From ffe6722d877b4cb04bd632868d3a75e925cbbb8d Mon Sep 17 00:00:00 2001 From: John Randolph Date: Tue, 30 Nov 2010 16:28:45 +0000 Subject: [PATCH] wrap os.listdir() to decode utf-8 unicode into unicode objects change all code locations using os.listdir() to use munkicommon.listdir() git-svn-id: http://munki.googlecode.com/svn/trunk@941 a4e17f2e-e282-11dd-95e1-755cbddbdd66 --- code/client/makecatalogs | 2 +- code/client/makepkginfo | 4 +-- code/client/munkilib/adobeutils.py | 12 +++---- code/client/munkilib/appleupdates.py | 4 +-- code/client/munkilib/installer.py | 4 +-- code/client/munkilib/munkicommon.py | 48 +++++++++++++++++++++----- code/client/munkilib/removepackages.py | 20 +++++------ code/client/munkilib/updatecheck.py | 4 +-- 8 files changed, 64 insertions(+), 34 deletions(-) diff --git a/code/client/makecatalogs b/code/client/makecatalogs index 1f470ecf..a59a0148 100755 --- a/code/client/makecatalogs +++ b/code/client/makecatalogs @@ -104,7 +104,7 @@ def makecatalogs(repopath): if not os.path.exists(catalogpath): os.mkdir(catalogpath) else: - for item in os.listdir(catalogpath): + for item in munkicommon.listdir(catalogpath): itempath = os.path.join(catalogpath, item) if os.path.isfile(itempath): os.remove(itempath) diff --git a/code/client/makepkginfo b/code/client/makepkginfo index b4fc1b18..5a9536f5 100755 --- a/code/client/makepkginfo +++ b/code/client/makepkginfo @@ -97,7 +97,7 @@ def getCatalogInfoFromDmg(dmgpath, options): cataloginfo['package_path'] = options.pkgname elif not options.item: # search for first package at root - for fsitem in os.listdir(mountpoints[0]): + for fsitem in munkicommon.listdir(mountpoints[0]): itempath = os.path.join(mountpoints[0], fsitem) if itempath.endswith('.pkg') or itempath.endswith('.mpkg'): cataloginfo = munkicommon.getPackageMetaData(itempath) @@ -140,7 +140,7 @@ def getCatalogInfoFromDmg(dmgpath, options): # no item specified; look for an application at root of # mounted dmg item = '' - for itemname in os.listdir(mountpoints[0]): + for itemname in munkicommon.listdir(mountpoints[0]): itempath = os.path.join(mountpoints[0], itemname) if munkicommon.isApplication(itempath): item = itemname diff --git a/code/client/munkilib/adobeutils.py b/code/client/munkilib/adobeutils.py index babc83ff..a0876424 100644 --- a/code/client/munkilib/adobeutils.py +++ b/code/client/munkilib/adobeutils.py @@ -107,7 +107,7 @@ def getPayloadInfo(dirpath): payloadinfo = {} # look for .proxy.xml file dir if os.path.isdir(dirpath): - for item in os.listdir(dirpath): + for item in munkicommon.listdir(dirpath): if item.endswith('.proxy.xml'): xmlpath = os.path.join(dirpath, item) dom = minidom.parse(xmlpath) @@ -184,7 +184,7 @@ def getAdobeSetupInfo(installroot): for node in element.childNodes: mediaSignature += node.nodeValue - for item in os.listdir(path): + for item in munkicommon.listdir(path): payloadpath = os.path.join(path, item) payloadinfo = getPayloadInfo(payloadpath) if payloadinfo: @@ -204,7 +204,7 @@ def getAdobeSetupInfo(installroot): # look for an extensions folder; almost certainly this is an Updater for (path, unused_dirs, unused_files) in os.walk(installroot): if path.endswith("/extensions"): - for item in os.listdir(path): + for item in munkicommon.listdir(path): #skip LanguagePacks if item.find("LanguagePack") == -1: itempath = os.path.join(path, item) @@ -352,12 +352,12 @@ def getAdobeInstallProgressInfo(previous_completedpayloads, def countPayloads(dirpath): '''Attempts to count the payloads in the Adobe installation item''' - for item in os.listdir(dirpath): + for item in munkicommon.listdir(dirpath): itempath = os.path.join(dirpath, item) if os.path.isdir(itempath): if item == "payloads": count = 0 - for subitem in os.listdir(itempath): + for subitem in munkicommon.listdir(itempath): subitempath = os.path.join(itempath, subitem) if os.path.isdir(subitempath): count = count + 1 @@ -673,7 +673,7 @@ def runAdobeCS5AAMEEInstall(dmgpath): realsetupdir = os.path.join(basepath,"Setup") tmpsetupdir = os.path.join(tmpdir, "Setup") os.mkdir(tmpsetupdir) - for item in os.listdir(realsetupdir): + for item in munkicommon.listdir(realsetupdir): os.symlink(os.path.join(realsetupdir, item), os.path.join(tmpsetupdir, item)) diff --git a/code/client/munkilib/appleupdates.py b/code/client/munkilib/appleupdates.py index 6611ffa8..9db5f2a3 100755 --- a/code/client/munkilib/appleupdates.py +++ b/code/client/munkilib/appleupdates.py @@ -270,7 +270,7 @@ def getRestartInfo(installitemdir): weight['None'] = 0 restartAction = "None" - for item in os.listdir(installitemdir): + for item in munkicommon.listdir(installitemdir): if item.endswith(".dist") or item.endswith(".pkg") or \ item.endswith(".mpkg"): installeritem = os.path.join(installitemdir, item) @@ -346,7 +346,7 @@ def getSoftwareUpdateInfo(): updatename = products[product_key] installitem = os.path.join(updatesdir, updatename) if os.path.exists(installitem) and os.path.isdir(installitem): - for subitem in os.listdir(installitem): + for subitem in munkicommon.listdir(installitem): if subitem.endswith('.dist'): distfile = os.path.join(installitem, subitem) (title, vers, diff --git a/code/client/munkilib/installer.py b/code/client/munkilib/installer.py index 64da0982..c9cccb75 100644 --- a/code/client/munkilib/installer.py +++ b/code/client/munkilib/installer.py @@ -195,7 +195,7 @@ def installall(dirpath, choicesXMLpath=None, suppressBundleRelocation=False): """ retcode = 0 restartflag = False - installitems = os.listdir(dirpath) + installitems = munkicommon.listdir(dirpath) for item in installitems: if munkicommon.stopRequested(): return (retcode, restartflag) @@ -247,7 +247,7 @@ def copyAppFromDMG(dmgpath): appname = None mountpoint = mountpoints[0] # find an app at the root level, copy it to /Applications - for item in os.listdir(mountpoint): + for item in munkicommon.listdir(mountpoint): itempath = os.path.join(mountpoint, item) if munkicommon.isApplication(itempath): appname = item diff --git a/code/client/munkilib/munkicommon.py b/code/client/munkilib/munkicommon.py index d0486d88..73b68d79 100644 --- a/code/client/munkilib/munkicommon.py +++ b/code/client/munkilib/munkicommon.py @@ -819,7 +819,7 @@ def getExtendedVersion(bundlepath): infopath = os.path.join(bundlepath, 'Contents', 'Resources', 'English.lproj') if os.path.exists(infopath): - for item in os.listdir(infopath): + for item in listdir(infopath): if os.path.join(infopath, item).endswith('.info'): infofile = os.path.join(infopath, item) fileobj = open(infofile, mode='r') @@ -908,7 +908,7 @@ def getFlatPackageInfo(pkgpath): if not infoarray: # didn't get any packageid info or no PackageInfo file # look for subpackages at the top level - for item in os.listdir(currentdir): + for item in listdir(currentdir): itempath = os.path.join(currentdir, item) if itempath.endswith('.pkg') and os.path.isdir(itempath): packageinfofile = os.path.join(itempath, 'PackageInfo') @@ -961,7 +961,7 @@ def getOnePackageInfo(pkgpath): infopath = os.path.join(pkgpath, 'Contents', 'Resources', 'English.lproj') if os.path.exists(infopath): - for item in os.listdir(infopath): + for item in listdir(infopath): if os.path.join(infopath, item).endswith('.info'): pkginfo['filename'] = os.path.basename(pkgpath) pkginfo['packageid'] = os.path.basename(pkgpath) @@ -1004,7 +1004,7 @@ def getBundlePackageInfo(pkgpath): bundlecontents = os.path.join(pkgpath, 'Contents') if os.path.exists(bundlecontents): - for item in os.listdir(bundlecontents): + for item in listdir(bundlecontents): if item.endswith('.dist'): filename = os.path.join(bundlecontents, item) dom = minidom.parse(filename) @@ -1040,7 +1040,7 @@ def getBundlePackageInfo(pkgpath): for subdir in dirsToSearch: searchdir = os.path.join(pkgpath, subdir) if os.path.exists(searchdir): - for item in os.listdir(searchdir): + for item in listdir(searchdir): itempath = os.path.join(searchdir, item) if os.path.isdir(itempath): if itempath.endswith('.pkg'): @@ -1058,7 +1058,7 @@ def getBundlePackageInfo(pkgpath): # couldn't find any subpackages, # just return info from the .dist file # if it exists - for item in os.listdir(bundlecontents): + for item in listdir(bundlecontents): if item.endswith('.dist'): distfile = os.path.join(bundlecontents, item) infoarray.extend(parsePkgRefs(distfile)) @@ -1117,7 +1117,7 @@ def getInstalledPackageVersion(pkgid): # Check /Library/Receipts receiptsdir = '/Library/Receipts' if os.path.exists(receiptsdir): - installitems = os.listdir(receiptsdir) + installitems = listdir(receiptsdir) highestversion = '0' for item in installitems: if item.endswith('.pkg'): @@ -1170,7 +1170,7 @@ def findInstallerItem(path): # Apple Software Updates download as directories # with .dist files and .pkgs if os.path.exists(path) and os.path.isdir(path): - for item in os.listdir(path): + for item in listdir(path): if item.endswith('.pkg'): return path @@ -1178,7 +1178,7 @@ def findInstallerItem(path): # look for a Packages dir path = os.path.join(path,'Packages') if os.path.exists(path) and os.path.isdir(path): - for item in os.listdir(path): + for item in listdir(path): if item.endswith('.pkg'): return path # found nothing! @@ -1582,6 +1582,36 @@ def cleanUpTmpDir(): tmpdir = None +def listdir(path): + """OSX HFS+ string encoding safe listdir(). + + Args: + path: str or unicode, path to list contents of + Returns: + list of contents, items as str or unicode types + """ + # on python2.4-2.6 on OSX, if path supplied to listdir() is unicode, the + # return list items are unicode also. so the behavior of this method + # will change as various paths are supplied to it, possibly concatenated + # from XML output of plists, etc. + l = os.listdir(path) + + for i in xrange(len(l)): + # 'i' is possibly a decomposed UTF-8 string in a str instance. + # fix it before it is exposed to the rest of Python. + # + # for background info: + # http://developer.apple.com/library/mac/#qa/qa2001/qa1235.html + # http://lists.zerezo.com/git/msg643117.html + # http://unicode.org/reports/tr15/ section 1.2 + + if type(l[i]) is not unicode: + u = unicode(l[i], 'utf-8') + l[i] = u + + return l + + # module globals #debug = False verbose = 1 diff --git a/code/client/munkilib/removepackages.py b/code/client/munkilib/removepackages.py index 39e2a59b..f77928bb 100755 --- a/code/client/munkilib/removepackages.py +++ b/code/client/munkilib/removepackages.py @@ -128,7 +128,7 @@ def shouldRebuildDB(pkgdbpath): receiptsdir_modtime = os.stat(receiptsdir).st_mtime if packagedb_modtime < receiptsdir_modtime: return True - receiptlist = os.listdir(receiptsdir) + receiptlist = munkicommon.listdir(receiptsdir) for item in receiptlist: if item.endswith(".pkg"): pkgpath = os.path.join(receiptsdir, item) @@ -140,7 +140,7 @@ def shouldRebuildDB(pkgdbpath): bomsdir_modtime = os.stat(bomsdir).st_mtime if packagedb_modtime < bomsdir_modtime: return True - bomlist = os.listdir(bomsdir) + bomlist = munkicommon.listdir(bomsdir) for item in bomlist: if item.endswith(".bom"): bompath = os.path.join(bomsdir, item) @@ -152,7 +152,7 @@ def shouldRebuildDB(pkgdbpath): receiptsdir_modtime = os.stat(sl_receiptsdir).st_mtime if packagedb_modtime < receiptsdir_modtime: return True - receiptlist = os.listdir(sl_receiptsdir) + receiptlist = munkicommon.listdir(sl_receiptsdir) for item in receiptlist: if item.endswith(".bom") or item.endswith(".plist"): pkgpath = os.path.join(sl_receiptsdir, item) @@ -206,7 +206,7 @@ def findBundleReceiptFromID(pkgid): if not pkgid: return '' receiptsdir = "/Library/Receipts" - for item in os.listdir(receiptsdir): + for item in munkicommon.listdir(receiptsdir): itempath = os.path.join(receiptsdir, item) if item.endswith('.pkg') and os.path.isdir(itempath): info = munkicommon.getOnePackageInfo(itempath) @@ -542,12 +542,12 @@ def initDatabase(forcerebuild=False): receiptsdir = "/Library/Receipts" bomsdir = "/Library/Receipts/boms" if os.path.exists(receiptsdir): - receiptlist = os.listdir(receiptsdir) + receiptlist = munkicommon.listdir(receiptsdir) for item in receiptlist: if item.endswith(".pkg"): pkgcount += 1 if os.path.exists(bomsdir): - bomslist = os.listdir(bomsdir) + bomslist = munkicommon.listdir(bomsdir) for item in bomslist: if item.endswith(".bom"): pkgcount += 1 @@ -576,7 +576,7 @@ def initDatabase(forcerebuild=False): local_display_percent_done(0, pkgcount) if os.path.exists(receiptsdir): - receiptlist = os.listdir(receiptsdir) + receiptlist = munkicommon.listdir(receiptsdir) for item in receiptlist: if munkicommon.stopRequested(): curs.close() @@ -593,7 +593,7 @@ def initDatabase(forcerebuild=False): local_display_percent_done(currentpkgindex, pkgcount) if os.path.exists(bomsdir): - bomslist = os.listdir(bomsdir) + bomslist = munkicommon.listdir(bomsdir) for item in bomslist: if munkicommon.stopRequested(): curs.close() @@ -930,7 +930,7 @@ def removeFilesystemItems(removalpaths, forcedeletebundles): munkicommon.display_detail("Removing: " + pathtoremove) if (os.path.isdir(pathtoremove) and \ not os.path.islink(pathtoremove)): - diritems = os.listdir(pathtoremove) + diritems = munkicommon.listdir(pathtoremove) if diritems == ['.DS_Store']: # If there's only a .DS_Store file # we'll consider it empty @@ -939,7 +939,7 @@ def removeFilesystemItems(removalpaths, forcedeletebundles): os.remove(ds_storepath) except (OSError, IOError): pass - diritems = os.listdir(pathtoremove) + diritems = munkicommon.listdir(pathtoremove) if diritems == []: # directory is empty try: diff --git a/code/client/munkilib/updatecheck.py b/code/client/munkilib/updatecheck.py index 48546919..058fac61 100644 --- a/code/client/munkilib/updatecheck.py +++ b/code/client/munkilib/updatecheck.py @@ -162,7 +162,7 @@ def getInstalledPackages(): # Now check /Library/Receipts receiptsdir = '/Library/Receipts' if os.path.exists(receiptsdir): - installitems = os.listdir(receiptsdir) + installitems = munkicommon.listdir(receiptsdir) for item in installitems: if item.endswith('.pkg'): pkginfo = munkicommon.getOnePackageInfo( @@ -2597,7 +2597,7 @@ def check(client_id='', localmanifestpath=None): for item in installinfo['removals'] if item.get('uninstaller_item')]) cachedir = os.path.join(ManagedInstallDir, 'Cache') - for item in os.listdir(cachedir): + for item in munkicommon.listdir(cachedir): if item.endswith('.download'): # we have a partial download here # remove the '.download' from the end of the filename