Better unicode text handling for subprocesses

code/client/managedsoftwareupdate

@@ -95,7 +95,7 @@ def getIdleSeconds():
     proc = subprocess.Popen(cmd, shell=False, stdin=subprocess.PIPE,
                             stdout=subprocess.PIPE, stderr=subprocess.PIPE)
     (output, dummy_err) = proc.communicate()
-    ioreglines = str(output).splitlines()
+    ioreglines = output.decode("UTF-8").splitlines()
     idle_time = 0
     regex = re.compile(r'"?HIDIdleTime"?\s+=\s+(\d+)')
     for line in ioreglines:
@@ -117,7 +117,7 @@ def networkUp():
     proc = subprocess.Popen(cmd, shell=False, stdin=subprocess.PIPE,
                             stdout=subprocess.PIPE, stderr=subprocess.PIPE)
     (output, dummy_err) = proc.communicate()
-    lines = str(output).splitlines()
+    lines = output.decode('UTF-8').splitlines()
     for line in lines:
         if 'inet' in line:
             parts = line.split()

code/client/munkilib/adobeutils/core.py

@@ -97,9 +97,9 @@ class AdobeInstallProgressMonitor(object):
         proc = subprocess.Popen(['/bin/ls', '-t1', logpath],
                                 bufsize=-1, stdout=subprocess.PIPE,
                                 stderr=subprocess.PIPE)
-        (output, dummy_err) = proc.communicate()
+        output = proc.communicate()[0].decode('UTF-8')
         if output:
-            firstitem = str(output).splitlines()[0]
+            firstitem = output.splitlines()[0]
             if firstitem.endswith(".log"):
                 # store path of most recently modified log file
                 recent_adobe_log = os.path.join(logpath, firstitem)
@@ -144,9 +144,9 @@ class AdobeInstallProgressMonitor(object):
             proc = subprocess.Popen(cmd, bufsize=-1,
                                     stdout=subprocess.PIPE,
                                     stderr=subprocess.PIPE)
-            (output, dummy_err) = proc.communicate()
+            output = proc.communicate()[0].decode('UTF-8')
             if output:
-                lines = str(output).splitlines()
+                lines = output.splitlines()
                 completed_payloads = len(lines)
 
                 if (logfile not in self.payload_count

code/client/munkilib/appleupdates/au.py

@@ -229,7 +229,7 @@ class AppleUpdates(object):
         proc = subprocess.Popen(cmd, shell=False, bufsize=1,
                                 stdin=subprocess.PIPE,
                                 stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        output, dummy_err = proc.communicate()
+        output = proc.communicate()[0].decode('UTF-8')
 
         current_apple_packages_checksum = hashlib.sha256(output).hexdigest()
         old_apple_packages_checksum = prefs.pref(

code/client/munkilib/appleupdates/su_prefs.py

@@ -125,9 +125,9 @@ def set_custom_catalogurl(catalog_url):
             bufsize=-1, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
         (output, err) = proc.communicate()
         if output:
-            display.display_detail(output)
+            display.display_detail(output.decode('UTF-8'))
         if err:
-            display.display_error(err)
+            display.display_error(err.decode('UTF-8'))
 
 
 def reset_original_catalogurl():
@@ -163,9 +163,9 @@ def reset_original_catalogurl():
                                 stderr=subprocess.PIPE)
         (output, err) = proc.communicate()
         if output:
-            display.display_detail(output)
+            display.display_detail(output.decode('UTF-8'))
         if err:
-            display.display_error(err)
+            display.display_error(err.decode('UTF-8'))
 
     # remove ORIGINAL_CATALOG_URL_KEY
     CFPreferencesSetValue(

code/client/munkilib/dmgutils.py

@@ -92,7 +92,7 @@ def hdiutil_info():
         bufsize=-1, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
     (out, err) = proc.communicate()
     if err:
-        display.display_error(u'hdiutil info error: %s', err)
+        display.display_error(u'hdiutil info error: %s', err.decode("UTF-8"))
     (pliststr, out) = utils.getFirstPlist(out)
     if pliststr:
         try:
@@ -245,7 +245,8 @@ def unmountdmg(mountpoint):
                                 stderr=subprocess.PIPE)
         (dummy_output, err) = proc.communicate()
         if proc.returncode:
-            display.display_warning('Failed to unmount %s: %s', mountpoint, err)
+            display.display_warning(
+                'Failed to unmount %s: %s', mountpoint, err.decode("UTF-8"))
 
 
 if __name__ == '__main__':

code/client/munkilib/iconutils.py

@@ -149,13 +149,13 @@ def extractAppIconsFromFlatPkg(pkg_path):
         proc = subprocess.Popen(cmd, shell=False, bufsize=-1,
                                 stdin=subprocess.PIPE, stdout=subprocess.PIPE,
                                 stderr=subprocess.STDOUT)
-        output = proc.communicate()[0]
+        output = proc.communicate()[0].decode('UTF-8')
         if proc.returncode:
             display.display_error(u'Could not lsbom %s', bomfile)
         # record paths to all app Info.plist files
         pkg_dict[pkgname] = [
             os.path.normpath(line)
-            for line in output.decode('utf-8').splitlines()
+            for line in output.splitlines()
             if line.endswith(u'.app/Contents/Info.plist')]
         if not pkg_dict[pkgname]:
             # remove empty lists

code/client/munkilib/info.py

@@ -554,7 +554,7 @@ def get_hardware_info():
     proc = subprocess.Popen(cmd, shell=False, bufsize=-1,
                             stdin=subprocess.PIPE,
                             stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    (output, dummy_error) = proc.communicate()
+    output = proc.communicate()[0]
     try:
         plist = FoundationPlist.readPlistFromString(output)
         # system_profiler xml is an array
@@ -574,7 +574,7 @@ def get_ip_addresses(kind):
     proc = subprocess.Popen(cmd, shell=False, bufsize=-1,
                             stdin=subprocess.PIPE,
                             stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    (output, dummy_error) = proc.communicate()
+    output = proc.communicate()[0]
     try:
         plist = FoundationPlist.readPlistFromString(output)
         # system_profiler xml is an array of length 1

code/client/munkilib/installer/pkg.py

@@ -88,8 +88,8 @@ def pkg_needs_restart(pkgpath, options):
     proc = subprocess.Popen(cmd, shell=False, bufsize=-1,
                             stdin=subprocess.PIPE,
                             stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    (output, dummy_err) = proc.communicate()
-    restartaction = output.decode('UTF-8').rstrip('\n')
+    output = proc.communicate()[0].decode('UTF-8')
+    restartaction = output.rstrip('\n')
     return (restartaction == 'RequireRestart' or
             restartaction == 'RecommendRestart')
 

code/client/munkilib/installer/rmpkgs.py

@@ -285,7 +285,7 @@ def import_bom(bompath, curs):
     proc = subprocess.Popen(["/usr/sbin/pkgutil", "--pkg-info-plist", pkgid],
                             bufsize=-1, stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE)
-    (pliststr, dummy_err) = proc.communicate()
+    pliststr = proc.communicate()[0]
     if pliststr:
         plist = FoundationPlist.readPlistFromString(pliststr)
         if "install-location" in plist:
@@ -328,7 +328,7 @@ def import_from_pkgutil(pkgname, curs):
     proc = subprocess.Popen(["/usr/sbin/pkgutil", "--pkg-info-plist", pkgid],
                             bufsize=-1, stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE)
-    (pliststr, dummy_err) = proc.communicate()
+    pliststr = proc.communicate()[0]
     if pliststr:
         plist = FoundationPlist.readPlistFromString(pliststr)
         if "pkg-version" in plist:
@@ -616,10 +616,9 @@ def remove_receipts(pkgkeylist, noupdateapplepkgdb):
             proc = subprocess.Popen(cmd, bufsize=-1,
                                     stdout=subprocess.PIPE,
                                     stderr=subprocess.PIPE)
-            (output, dummy_err) = proc.communicate()
+            output = proc.communicate()[0].decode('UTF-8')
             if output:
-                display.display_detail(
-                    output.decode('UTF-8').rstrip('\n'))
+                display.display_detail(output.rstrip('\n'))
 
     display.display_percent_done(2, 4)
 

code/client/munkilib/keychain.py

@@ -157,7 +157,7 @@ def get_client_cert_common_name():
         proc = subprocess.Popen(cmd,
                                 bufsize=-1, stdout=subprocess.PIPE,
                                 stderr=subprocess.PIPE, stdin=subprocess.PIPE)
-        (out, _err) = proc.communicate()
+        out = proc.communicate()[0].decode("UTF-8")
         if out:
             for i in out.split('/'):
                 if i.startswith('CN='):
@@ -517,8 +517,8 @@ def security(verb_name, *args):
         stdout=subprocess.PIPE, stderr=subprocess.PIPE)
     (output, err) = proc.communicate()
     if proc.returncode:
-        raise SecurityError('%s: %s' % (proc.returncode, err))
-    return output or err
+        raise SecurityError('%s: %s' % (proc.returncode, err.decode("UTF-8")))
+    return (output or err).decode("UTF-8")
 
 
 def get_keychain_path():

code/client/munkilib/launchd/__init__.py

@@ -81,11 +81,11 @@ def job_info(job_label):
                             stdin=subprocess.PIPE,
                             stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE)
-    output = proc.communicate()[0]
+    output = proc.communicate()[0].decode('UTF-8')
     if proc.returncode or not output:
         return info
     else:
-        lines = output.decode("UTF-8").splitlines()
+        lines = output.splitlines()
         # search launchctl list output for our job label
         job_lines = [item for item in lines
                      if item.endswith('\t' + job_label)]
@@ -116,7 +116,7 @@ def stop_job(job_label):
                             stdin=subprocess.PIPE,
                             stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE)
-    err = proc.communicate()[1]
+    err = proc.communicate()[1].decode('UTF-8')
     if proc.returncode:
         raise LaunchdJobException(err)
 
@@ -128,7 +128,7 @@ def remove_job(job_label):
                             stdin=subprocess.PIPE,
                             stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE)
-    err = proc.communicate()[1]
+    err = proc.communicate()[1].decode('UTF-8')
     if proc.returncode:
         raise LaunchdJobException(err)
 
@@ -174,7 +174,7 @@ class Job(object):
                                 stdin=subprocess.PIPE,
                                 stdout=subprocess.PIPE,
                                 stderr=subprocess.PIPE)
-        err = proc.communicate()[1]
+        err = proc.communicate()[1].decode('UTF-8')
         if proc.returncode:
             raise LaunchdJobException(err)
 

code/client/munkilib/munkirepo/GitFileRepo.py

@@ -37,8 +37,11 @@ class MunkiGit(object):
                                 stdout=subprocess.PIPE,
                                 stderr=subprocess.PIPE)
         (output, error) = proc.communicate()
-        self.results = {"output": output,
-                        "error": error, "returncode": proc.returncode}
+        self.results = {
+            "output": output.decode('UTF-8'),
+            "error": error.decode('UTF-8'),
+            "returncode": proc.returncode
+        }
         return self.results
 
     def path_is_gitignored(self, a_path):

code/client/munkilib/munkirepo/MWA2APIRepo.py

@@ -14,10 +14,7 @@ try:
 except ImportError:
     from urllib.parse import quote
 
-from xml.parsers.expat import ExpatError
-
 from munkilib.munkirepo import Repo, RepoError
-
 from munkilib.wrappers import get_input, readPlistFromString, PlistReadError
 
 DEBUG = False
@@ -26,15 +23,6 @@ DEBUG = False
 CURL_CMD = '/usr/bin/curl'
 
 
-def get_input(prompt=None):
-    '''Python 2 and 3 wrapper for raw_input/input'''
-    try:
-        return raw_input(prompt)
-    except NameError:
-        # raw_input doesn't exist in Python 3
-        return input(prompt)
-
-
 class CurlError(Exception):
     '''Error for curl operations'''
     pass
@@ -116,7 +104,8 @@ class MWA2APIRepo(Repo):
                                 stdin=subprocess.PIPE,
                                 stdout=subprocess.PIPE, stderr=subprocess.PIPE)
         output, err = proc.communicate()
-
+        output = output.decode('UTF-8')
+        err = err.decode('UTF-8')
         if DEBUG:
             # save our curl_directives for debugging
             fileref = open(directivepath)

code/client/munkilib/osutils.py

@@ -119,8 +119,8 @@ def currentGUIusers():
     proc = subprocess.Popen('/usr/bin/who', shell=False,
                             stdin=subprocess.PIPE,
                             stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    (output, dummy_err) = proc.communicate()
-    lines = str(output).splitlines()
+    output = proc.communicate()[0].decode("UTF-8")
+    lines = output.splitlines()
     for line in lines:
         if 'console' in line:
             parts = line.split()
@@ -139,7 +139,7 @@ def pythonScriptRunning(scriptname):
     proc = subprocess.Popen(cmd, shell=False, bufsize=1,
                             stdin=subprocess.PIPE,
                             stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    (out, dummy_err) = proc.communicate()
+    out = proc.communicate()[0].decode("UTF-8")
     mypid = os.getpid()
     lines = str(out).splitlines()
     for line in lines:
@@ -177,7 +177,7 @@ def osascript(osastring):
                             stdout=subprocess.PIPE, stderr=subprocess.PIPE)
     (out, err) = proc.communicate()
     if proc.returncode != 0:
-        print('Error: ', err, file=sys.stderr)
+        print('Error: ', err.decode('UTF-8'), file=sys.stderr)
     if out:
         return out.decode('UTF-8').rstrip('\n')
     return u''

code/client/munkilib/pkgutils.py

@@ -44,8 +44,6 @@ from . import osutils
 from . import utils
 from . import FoundationPlist
 
-from .wrappers import is_a_string
-
 
 # we use lots of camelCase-style names. Deal with it.
 # pylint: disable=C0103
@@ -66,12 +64,14 @@ def getPkgRestartInfo(filename):
                             stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE)
     (out, err) = proc.communicate()
+    out = out.decode('UTF-8')
+    err = err.decode('UTF-8')
     if proc.returncode:
         display.display_error("installer -query failed: %s %s", out, err)
         return {}
 
     if out:
-        restartAction = str(out).rstrip('\n')
+        restartAction = out.rstrip('\n')
         if restartAction != 'None':
             installerinfo['RestartAction'] = restartAction
 
@@ -133,8 +133,7 @@ class MunkiLooseVersion(version.LooseVersion):
                 # integer is less than character/string
                 if isinstance(value, int):
                     return -1
-                else:
-                    return 1
+                return 1
             else:
                 if cmp_result:
                     return cmp_result
@@ -450,7 +449,7 @@ def getFlatPackageInfo(pkgpath):
             display.display_warning(
                 'No valid Distribution or PackageInfo found.')
     else:
-        display.display_warning(err)
+        display.display_warning(err.decode('UTF-8'))
 
     # change back to original working dir
     os.chdir(cwd)
@@ -476,7 +475,7 @@ def getBomList(pkgpath):
                                 shell=False, stdin=subprocess.PIPE,
                                 stdout=subprocess.PIPE,
                                 stderr=subprocess.PIPE)
-        (output, dummy_err) = proc.communicate()
+        output = proc.communicate()[0].decode('UTF-8')
         if proc.returncode == 0:
             return output.splitlines()
     return []
@@ -622,7 +621,7 @@ def getInstalledPackageVersion(pkgid):
                             bufsize=1,
                             stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE)
-    (out, dummy_err) = proc.communicate()
+    out = proc.communicate()[0]
 
     if out:
         try:
@@ -761,7 +760,7 @@ def getChoiceChangesXML(pkgitem):
         proc = subprocess.Popen(
             ['/usr/sbin/installer', '-showChoiceChangesXML', '-pkg', pkgitem],
             bufsize=1, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        (out, dummy_err) = proc.communicate()
+        out = proc.communicate()[0]
         if out:
             plist = FoundationPlist.readPlistFromString(out)
 
@@ -860,7 +859,7 @@ def getInstalledPackages():
     proc = subprocess.Popen(['/usr/sbin/pkgutil', '--regexp',
                              '--pkg-info-plist', '.*'], bufsize=8192,
                             stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    (out, dummy_err) = proc.communicate()
+    out = proc.communicate()[0]
     while out:
         (pliststr, out) = utils.getFirstPlist(out)
         if pliststr:

code/client/munkilib/processes.py

@@ -37,9 +37,9 @@ def get_running_processes():
                             shell=False, stdin=subprocess.PIPE,
                             stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE)
-    (output, dummy_err) = proc.communicate()
+    output = proc.communicate()[0].decode('UTF-8')
     if proc.returncode == 0:
-        proc_list = [item for item in output.decode("UTF-8").splitlines()
+        proc_list = [item for item in output.splitlines()
                      if item.startswith('/')]
         launchcfmapp = ('/System/Library/Frameworks/Carbon.framework'
                         '/Versions/A/Support/LaunchCFMApp')
@@ -49,10 +49,10 @@ def get_running_processes():
                                     shell=False, stdin=subprocess.PIPE,
                                     stdout=subprocess.PIPE,
                                     stderr=subprocess.PIPE)
-            (output, dummy_err) = proc.communicate()
+            output = proc.communicate()[0].decode('UTF-8')
             if proc.returncode == 0:
                 carbon_apps = [item[len(launchcfmapp)+1:]
-                               for item in output.decode("UTF-8").splitlines()
+                               for item in output.splitlines()
                                if item.startswith(launchcfmapp)]
                 if carbon_apps:
                     proc_list.extend(carbon_apps)
@@ -138,7 +138,7 @@ def find_processes(user=None, exe=None):
     argv = ['/bin/ps', '-x', '-w', '-w', '-a', '-o', 'pid=,user=,comm=']
     ps_proc = subprocess.Popen(
         argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    (stdout, dummy_stderr) = ps_proc.communicate()
+    stdout = ps_proc.communicate()[0].decode('UTF-8')
 
     pids = {}
 

code/client/munkilib/profiles.py

@@ -54,7 +54,7 @@ def config_profile_info(ignore_cache=False):
     # so let's redirect everything to stdout and just use that
     proc = subprocess.Popen(
         cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-    stdout = proc.communicate()[0]
+    stdout = proc.communicate()[0].decode('UTF-8')
     if proc.returncode != 0:
         display.display_error(
             'Could not obtain configuration profile info: %s' % stdout)
@@ -162,7 +162,8 @@ def read_signed_profile(profile_path):
     if proc.returncode:
         # security cms -D couldn't decode the file
         display.display_error(
-            'Error reading profile %s: %s' % (profile_path, stderr))
+            'Error reading profile %s: %s'
+            % (profile_path, stderr.decode('UTF-8')))
         return {}
     try:
         return FoundationPlist.readPlistFromString(stdout)
@@ -207,11 +208,11 @@ def install_profile(profile_path, profile_identifier):
     # so let's redirect everything to stdout and just use that
     proc = subprocess.Popen(
         cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-    stdout = proc.communicate()[0]
+    stdout = proc.communicate()[0].decode('UTF-8')
     if proc.returncode != 0:
         display.display_error(
             u'Profile %s installation failed: %s'
-            % (os.path.basename(profile_path), stdout.decode('UTF-8')))
+            % (os.path.basename(profile_path), stdout))
         return False
     if profile_identifier:
         record_profile_receipt(profile_path, profile_identifier)
@@ -232,7 +233,7 @@ def remove_profile(identifier):
     # so let's redirect everything to stdout and just use that
     proc = subprocess.Popen(
         cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-    stdout = proc.communicate()[0]
+    stdout = proc.communicate()[0].decode('UTF-8')
     if proc.returncode != 0:
         display.display_error(
             'Profile %s removal failed: %s' % (identifier, stdout))

code/client/munkilib/reports.py

@@ -126,7 +126,7 @@ def archive_report():
         proc = subprocess.Popen(['/bin/ls', '-t1', archivepath],
                                 bufsize=1, stdout=subprocess.PIPE,
                                 stderr=subprocess.PIPE)
-        (output, dummy_err) = proc.communicate()
+        output = proc.communicate()[0].decode('UTF-8')
         if output:
             archiveitems = [item
                             for item in str(output).splitlines()

code/client/munkilib/utils.py

@@ -178,7 +178,7 @@ def getPIDforProcessName(processname):
                 pass
             else:
                 if process.find(processname) != -1:
-                    return str(pid)
+                    return pid
 
     return 0
 

code/client/repoclean

@@ -377,7 +377,7 @@ class RepoCleaner(object):
             # we don't print stdout -- too much info
             #print output.rstrip('\n').decode('UTF-8')
 
-        errors = proc.stderr.read()
+        errors = proc.stderr.read().decode('UTF-8')
         if errors:
             print('\nThe following issues occurred while building catalogs:\n')
             print(errors)

code/pkgtemplate/Scripts_launchd/postinstall

@@ -24,7 +24,7 @@ def launchctld(identifier):
             cmd = ['/bin/launchctl', 'load', path]
             proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                                     stderr=subprocess.PIPE)
-            output, err = proc.communicate()
+            output = proc.communicate()[0].decode("UTF-8")
             return output
         except KeyError:
             pass