wrap os.listdir() to decode utf-8 unicode into unicode objects

change all code locations using os.listdir() to use munkicommon.listdir()



git-svn-id: http://munki.googlecode.com/svn/trunk@941 a4e17f2e-e282-11dd-95e1-755cbddbdd66

code/client/makecatalogs

@@ -104,7 +104,7 @@ def makecatalogs(repopath):
     if not os.path.exists(catalogpath):
         os.mkdir(catalogpath)
     else:
-        for item in os.listdir(catalogpath):
+        for item in munkicommon.listdir(catalogpath):
             itempath = os.path.join(catalogpath, item)
             if os.path.isfile(itempath):
                 os.remove(itempath)

code/client/makepkginfo

@@ -97,7 +97,7 @@ def getCatalogInfoFromDmg(dmgpath, options):
                 cataloginfo['package_path'] = options.pkgname
     elif not options.item:
         # search for first package at root
-        for fsitem in os.listdir(mountpoints[0]):
+        for fsitem in munkicommon.listdir(mountpoints[0]):
             itempath = os.path.join(mountpoints[0], fsitem)
             if itempath.endswith('.pkg') or itempath.endswith('.mpkg'):
                 cataloginfo = munkicommon.getPackageMetaData(itempath)
@@ -140,7 +140,7 @@ def getCatalogInfoFromDmg(dmgpath, options):
             # no item specified; look for an application at root of
             # mounted dmg
             item = ''
-            for itemname in os.listdir(mountpoints[0]):
+            for itemname in munkicommon.listdir(mountpoints[0]):
                 itempath = os.path.join(mountpoints[0], itemname)
                 if munkicommon.isApplication(itempath):
                     item = itemname

code/client/munkilib/adobeutils.py

@@ -107,7 +107,7 @@ def getPayloadInfo(dirpath):
     payloadinfo = {}
     # look for .proxy.xml file dir
     if os.path.isdir(dirpath):
-        for item in os.listdir(dirpath):
+        for item in munkicommon.listdir(dirpath):
             if item.endswith('.proxy.xml'):
                 xmlpath = os.path.join(dirpath, item)
                 dom = minidom.parse(xmlpath)
@@ -184,7 +184,7 @@ def getAdobeSetupInfo(installroot):
                             for node in element.childNodes:
                                 mediaSignature += node.nodeValue
                             
-            for item in os.listdir(path):
+            for item in munkicommon.listdir(path):
                 payloadpath = os.path.join(path, item)
                 payloadinfo = getPayloadInfo(payloadpath)
                 if payloadinfo:
@@ -204,7 +204,7 @@ def getAdobeSetupInfo(installroot):
         # look for an extensions folder; almost certainly this is an Updater
         for (path, unused_dirs, unused_files) in os.walk(installroot):
             if path.endswith("/extensions"):
-                for item in os.listdir(path):
+                for item in munkicommon.listdir(path):
                     #skip LanguagePacks
                     if item.find("LanguagePack") == -1:
                         itempath = os.path.join(path, item)
@@ -352,12 +352,12 @@ def getAdobeInstallProgressInfo(previous_completedpayloads,
 
 def countPayloads(dirpath):
     '''Attempts to count the payloads in the Adobe installation item'''
-    for item in os.listdir(dirpath):
+    for item in munkicommon.listdir(dirpath):
         itempath = os.path.join(dirpath, item)
         if os.path.isdir(itempath):
             if item == "payloads":
                 count = 0
-                for subitem in os.listdir(itempath):
+                for subitem in munkicommon.listdir(itempath):
                     subitempath = os.path.join(itempath, subitem)
                     if os.path.isdir(subitempath):
                         count = count + 1
@@ -673,7 +673,7 @@ def runAdobeCS5AAMEEInstall(dmgpath):
         realsetupdir = os.path.join(basepath,"Setup")
         tmpsetupdir = os.path.join(tmpdir, "Setup")
         os.mkdir(tmpsetupdir)
-        for item in os.listdir(realsetupdir):
+        for item in munkicommon.listdir(realsetupdir):
             os.symlink(os.path.join(realsetupdir, item), 
                                             os.path.join(tmpsetupdir, item))
                                             

code/client/munkilib/appleupdates.py

@@ -270,7 +270,7 @@ def getRestartInfo(installitemdir):
     weight['None'] = 0
     
     restartAction = "None"
-    for item in os.listdir(installitemdir):
+    for item in munkicommon.listdir(installitemdir):
         if item.endswith(".dist") or item.endswith(".pkg") or \
                 item.endswith(".mpkg"):
             installeritem = os.path.join(installitemdir, item)
@@ -346,7 +346,7 @@ def getSoftwareUpdateInfo():
             updatename = products[product_key]
             installitem = os.path.join(updatesdir, updatename)
             if os.path.exists(installitem) and os.path.isdir(installitem):
-                for subitem in os.listdir(installitem):
+                for subitem in munkicommon.listdir(installitem):
                     if subitem.endswith('.dist'):
                         distfile = os.path.join(installitem, subitem)
                         (title, vers, 

code/client/munkilib/installer.py

@@ -195,7 +195,7 @@ def installall(dirpath, choicesXMLpath=None, suppressBundleRelocation=False):
     """
     retcode = 0
     restartflag = False
-    installitems = os.listdir(dirpath)
+    installitems = munkicommon.listdir(dirpath)
     for item in installitems:
         if munkicommon.stopRequested():
             return (retcode, restartflag)
@@ -247,7 +247,7 @@ def copyAppFromDMG(dmgpath):
         appname = None
         mountpoint = mountpoints[0]
         # find an app at the root level, copy it to /Applications
-        for item in os.listdir(mountpoint):
+        for item in munkicommon.listdir(mountpoint):
             itempath = os.path.join(mountpoint, item)
             if munkicommon.isApplication(itempath):
                 appname = item

code/client/munkilib/munkicommon.py

@@ -819,7 +819,7 @@ def getExtendedVersion(bundlepath):
     infopath = os.path.join(bundlepath, 'Contents', 'Resources',
                                 'English.lproj')
     if os.path.exists(infopath):
-        for item in os.listdir(infopath):
+        for item in listdir(infopath):
             if os.path.join(infopath, item).endswith('.info'):
                 infofile = os.path.join(infopath, item)
                 fileobj = open(infofile, mode='r')
@@ -908,7 +908,7 @@ def getFlatPackageInfo(pkgpath):
         if not infoarray:
             # didn't get any packageid info or no PackageInfo file
             # look for subpackages at the top level
-            for item in os.listdir(currentdir):
+            for item in listdir(currentdir):
                 itempath = os.path.join(currentdir, item)
                 if itempath.endswith('.pkg') and os.path.isdir(itempath):
                     packageinfofile = os.path.join(itempath, 'PackageInfo')
@@ -961,7 +961,7 @@ def getOnePackageInfo(pkgpath):
         infopath = os.path.join(pkgpath, 'Contents', 'Resources',
                                     'English.lproj')
         if os.path.exists(infopath):
-            for item in os.listdir(infopath):
+            for item in listdir(infopath):
                 if os.path.join(infopath, item).endswith('.info'):
                     pkginfo['filename'] = os.path.basename(pkgpath)
                     pkginfo['packageid'] = os.path.basename(pkgpath)
@@ -1004,7 +1004,7 @@ def getBundlePackageInfo(pkgpath):
 
     bundlecontents = os.path.join(pkgpath, 'Contents')
     if os.path.exists(bundlecontents):
-        for item in os.listdir(bundlecontents):
+        for item in listdir(bundlecontents):
             if item.endswith('.dist'):
                 filename = os.path.join(bundlecontents, item)
                 dom = minidom.parse(filename)
@@ -1040,7 +1040,7 @@ def getBundlePackageInfo(pkgpath):
         for subdir in dirsToSearch:
             searchdir = os.path.join(pkgpath, subdir)
             if os.path.exists(searchdir):
-                for item in os.listdir(searchdir):
+                for item in listdir(searchdir):
                     itempath = os.path.join(searchdir, item)
                     if os.path.isdir(itempath):
                         if itempath.endswith('.pkg'):
@@ -1058,7 +1058,7 @@ def getBundlePackageInfo(pkgpath):
             # couldn't find any subpackages,
             # just return info from the .dist file
             # if it exists
-            for item in os.listdir(bundlecontents):
+            for item in listdir(bundlecontents):
                 if item.endswith('.dist'):
                     distfile = os.path.join(bundlecontents, item)
                     infoarray.extend(parsePkgRefs(distfile))
@@ -1117,7 +1117,7 @@ def getInstalledPackageVersion(pkgid):
     # Check /Library/Receipts
     receiptsdir = '/Library/Receipts'
     if os.path.exists(receiptsdir):
-        installitems = os.listdir(receiptsdir)
+        installitems = listdir(receiptsdir)
         highestversion = '0'
         for item in installitems:
             if item.endswith('.pkg'):
@@ -1170,7 +1170,7 @@ def findInstallerItem(path):
         # Apple Software Updates download as directories
         # with .dist files and .pkgs
         if os.path.exists(path) and os.path.isdir(path):
-            for item in os.listdir(path):
+            for item in listdir(path):
                 if item.endswith('.pkg'):
                     return path
 
@@ -1178,7 +1178,7 @@ def findInstallerItem(path):
             # look for a Packages dir
             path = os.path.join(path,'Packages')
             if os.path.exists(path) and os.path.isdir(path):
-                for item in os.listdir(path):
+                for item in listdir(path):
                     if item.endswith('.pkg'):
                         return path
     # found nothing!
@@ -1582,6 +1582,36 @@ def cleanUpTmpDir():
         tmpdir = None
 
 
+def listdir(path):
+    """OSX HFS+ string encoding safe listdir().
+
+    Args:
+        path: str or unicode, path to list contents of
+    Returns:
+        list of contents, items as str or unicode types
+    """
+    # on python2.4-2.6 on OSX, if path supplied to listdir() is unicode, the
+    # return list items are unicode also.  so the behavior of this method
+    # will change as various paths are supplied to it, possibly concatenated
+    # from XML output of plists, etc.
+    l = os.listdir(path)
+
+    for i in xrange(len(l)):
+        # 'i' is possibly a decomposed UTF-8 string in a str instance.
+        # fix it before it is exposed to the rest of Python.
+        #
+        # for background info:
+        # http://developer.apple.com/library/mac/#qa/qa2001/qa1235.html
+        # http://lists.zerezo.com/git/msg643117.html
+        # http://unicode.org/reports/tr15/    section 1.2
+
+        if type(l[i]) is not unicode:
+            u = unicode(l[i], 'utf-8')
+            l[i] = u
+
+    return l
+
+
 # module globals
 #debug = False
 verbose = 1

code/client/munkilib/removepackages.py

@@ -128,7 +128,7 @@ def shouldRebuildDB(pkgdbpath):
         receiptsdir_modtime = os.stat(receiptsdir).st_mtime
         if packagedb_modtime < receiptsdir_modtime:
             return True
-        receiptlist = os.listdir(receiptsdir)
+        receiptlist = munkicommon.listdir(receiptsdir)
         for item in receiptlist:
             if item.endswith(".pkg"):
                 pkgpath = os.path.join(receiptsdir, item)
@@ -140,7 +140,7 @@ def shouldRebuildDB(pkgdbpath):
         bomsdir_modtime = os.stat(bomsdir).st_mtime
         if packagedb_modtime < bomsdir_modtime:
             return True
-        bomlist = os.listdir(bomsdir)
+        bomlist = munkicommon.listdir(bomsdir)
         for item in bomlist:
             if item.endswith(".bom"):
                 bompath = os.path.join(bomsdir, item)
@@ -152,7 +152,7 @@ def shouldRebuildDB(pkgdbpath):
         receiptsdir_modtime = os.stat(sl_receiptsdir).st_mtime
         if packagedb_modtime < receiptsdir_modtime:
             return True
-        receiptlist = os.listdir(sl_receiptsdir)
+        receiptlist = munkicommon.listdir(sl_receiptsdir)
         for item in receiptlist:
             if item.endswith(".bom") or item.endswith(".plist"):
                 pkgpath = os.path.join(sl_receiptsdir, item)
@@ -206,7 +206,7 @@ def findBundleReceiptFromID(pkgid):
     if not pkgid:
         return ''
     receiptsdir = "/Library/Receipts"
-    for item in os.listdir(receiptsdir):
+    for item in munkicommon.listdir(receiptsdir):
         itempath = os.path.join(receiptsdir, item)
         if item.endswith('.pkg') and os.path.isdir(itempath):
             info = munkicommon.getOnePackageInfo(itempath)
@@ -542,12 +542,12 @@ def initDatabase(forcerebuild=False):
     receiptsdir = "/Library/Receipts"
     bomsdir = "/Library/Receipts/boms"
     if os.path.exists(receiptsdir):
-        receiptlist = os.listdir(receiptsdir)
+        receiptlist = munkicommon.listdir(receiptsdir)
         for item in receiptlist:
             if item.endswith(".pkg"):
                 pkgcount += 1
     if os.path.exists(bomsdir):
-        bomslist = os.listdir(bomsdir)
+        bomslist = munkicommon.listdir(bomsdir)
         for item in bomslist:
             if item.endswith(".bom"):
                 pkgcount += 1
@@ -576,7 +576,7 @@ def initDatabase(forcerebuild=False):
     local_display_percent_done(0, pkgcount)
     
     if os.path.exists(receiptsdir):
-        receiptlist = os.listdir(receiptsdir)
+        receiptlist = munkicommon.listdir(receiptsdir)
         for item in receiptlist:
             if munkicommon.stopRequested():
                 curs.close()
@@ -593,7 +593,7 @@ def initDatabase(forcerebuild=False):
                 local_display_percent_done(currentpkgindex, pkgcount)
                 
     if os.path.exists(bomsdir):
-        bomslist = os.listdir(bomsdir)
+        bomslist = munkicommon.listdir(bomsdir)
         for item in bomslist:
             if munkicommon.stopRequested():
                 curs.close()
@@ -930,7 +930,7 @@ def removeFilesystemItems(removalpaths, forcedeletebundles):
             munkicommon.display_detail("Removing: " + pathtoremove)
             if (os.path.isdir(pathtoremove) and \
                not os.path.islink(pathtoremove)):
-                diritems = os.listdir(pathtoremove)
+                diritems = munkicommon.listdir(pathtoremove)
                 if diritems == ['.DS_Store']:
                     # If there's only a .DS_Store file
                     # we'll consider it empty
@@ -939,7 +939,7 @@ def removeFilesystemItems(removalpaths, forcedeletebundles):
                         os.remove(ds_storepath)
                     except (OSError, IOError):
                         pass
-                    diritems = os.listdir(pathtoremove)
+                    diritems = munkicommon.listdir(pathtoremove)
                 if diritems == []:
                     # directory is empty
                     try:

code/client/munkilib/updatecheck.py

@@ -162,7 +162,7 @@ def getInstalledPackages():
     # Now check /Library/Receipts
     receiptsdir = '/Library/Receipts'
     if os.path.exists(receiptsdir):
-        installitems = os.listdir(receiptsdir)
+        installitems = munkicommon.listdir(receiptsdir)
         for item in installitems:
             if item.endswith('.pkg'):
                 pkginfo = munkicommon.getOnePackageInfo(
@@ -2597,7 +2597,7 @@ def check(client_id='', localmanifestpath=None):
                            for item in installinfo['removals']
                            if item.get('uninstaller_item')])
         cachedir = os.path.join(ManagedInstallDir, 'Cache')
-        for item in os.listdir(cachedir):
+        for item in munkicommon.listdir(cachedir):
             if item.endswith('.download'):
                 # we have a partial download here
                 # remove the '.download' from the end of the filename