appleupdates.py now creates its own .dist files to manage installation of downloaded Apple Software Updates, replacing the earlier technique of just installing the individual packages in filename order.

In most invocations of subprocess.Popen, change the bufsize from 1 to -1, which changes from line-buffered to "big" buffers.

Other minor changes to align with the appleupdates,py changes and some pylint-recommended cleanup.

git-svn-id: http://munki.googlecode.com/svn/trunk@995 a4e17f2e-e282-11dd-95e1-755cbddbdd66

code/client/makepkginfo

@@ -51,7 +51,7 @@ def DMGhasSLA(dmgpath):
     hasSLA = False
     proc = subprocess.Popen(
                 ['/usr/bin/hdiutil', 'imageinfo', dmgpath, '-plist'],
-                bufsize=1, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+                bufsize=-1, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
     (pliststr, err) = proc.communicate()
     if err:
         print >> sys.stderr, ("hdiutil error %s with image %s." 
@@ -456,9 +456,9 @@ def main():
                 exit(-1)
             
             if options.description:
-              catinfo['description'] = options.description
+                catinfo['description'] = options.description
             if options.displayname:
-              catinfo['display_name'] = options.displayname
+                catinfo['display_name'] = options.displayname
 
             catinfo['installer_item_size'] = int(itemsize/1024)
             catinfo['installer_item_hash'] = itemhash
@@ -543,7 +543,7 @@ def main():
                 installs.append(iteminfodict)
             else:
                 print >> sys.stderr, (
-                                "Item %s doesn't exist. Skipping." % fitem)
+                    "Item %s doesn't exist. Skipping." % fitem)
                             
     if catinfo:
         catinfo['autoremove'] = False

code/client/managedsoftwareupdate

@@ -489,7 +489,7 @@ def main():
                 if not munkicommon.pythonScriptRunning(myname):
                     break
                 # or user clicks Stop
-                if munkistatus.getStopButtonState() == 1:
+                if munkicommon.stopRequested():
                     break
                 time.sleep(0.5)
 

code/client/munkiimport

@@ -43,7 +43,7 @@ def makeDMG(pkgpath):
     diskimagename = os.path.splitext(pkgname)[0] + '.dmg'
     diskimagepath = os.path.join(munkicommon.tmpdir, diskimagename)
     cmd = ['/usr/bin/hdiutil', 'create', '-srcfolder', pkgpath, diskimagepath]
-    proc = subprocess.Popen(cmd, shell=False, bufsize=1, 
+    proc = subprocess.Popen(cmd, shell=False, bufsize=-1, 
                             stdin=subprocess.PIPE, stdout=subprocess.PIPE,
                             stderr=subprocess.STDOUT)
     while True: 
@@ -394,7 +394,7 @@ def makePkgInfo(item_path):
         # didn't find it; assume the default install path
         makepkginfo_path = '/usr/local/munki/makepkginfo'
     proc = subprocess.Popen([makepkginfo_path, item_path],
-                            bufsize=1, stdout=subprocess.PIPE, 
+                            bufsize=-1, stdout=subprocess.PIPE, 
                             stderr=subprocess.PIPE)
     (pliststr, err) = proc.communicate()
     if proc.returncode:
@@ -417,7 +417,7 @@ def makeCatalogs():
     if not os.path.exists(repo_path):
         raise RepoCopyError('Could not connect to munki repo.')
     proc = subprocess.Popen([makecatalogs_path, repo_path],
-                            bufsize=1, stdout=subprocess.PIPE, 
+                            bufsize=-1, stdout=subprocess.PIPE, 
                             stderr=subprocess.PIPE)
     while True:
         output = proc.stdout.readline()

code/client/munkilib/appleupdates.py

@@ -25,6 +25,7 @@ import os
 import stat
 import subprocess
 from xml.dom import minidom
+from xml.parsers.expat import ExpatError
 
 from Foundation import NSDate
 
@@ -51,10 +52,13 @@ def getCurrentSoftwareUpdateServer():
 
 def selectSoftwareUpdateServer():
     '''Switch to our preferred Software Update Server if supplied'''
-    if munkicommon.pref('SoftwareUpdateServerURL'):
+    localCatalogURL = munkicommon.pref('SoftwareUpdateServerURL')
+    if localCatalogURL:
+        munkicommon.display_detail('Setting Apple Software Update '
+                                   'CatalogURL to %s' % localCatalogURL)
         cmd = ['/usr/bin/defaults', 'write',
                '/Library/Preferences/com.apple.SoftwareUpdate',
-               'CatalogURL', munkicommon.pref('SoftwareUpdateServerURL')]
+               'CatalogURL', localCatalogURL]
         unused_retcode = subprocess.call(cmd)
 
 
@@ -62,16 +66,20 @@ def restoreSoftwareUpdateServer(theurl):
     '''Switch back to original Software Update server (if there was one)'''
     if munkicommon.pref('SoftwareUpdateServerURL'):
         if theurl:
+            munkicommon.display_detail('Resetting Apple Software Update '
+                                       'CatalogURL to %s' % theurl)
             cmd = ['/usr/bin/defaults', 'write',
                    '/Library/Preferences/com.apple.SoftwareUpdate',
                    'CatalogURL', theurl]
         else:
+            munkicommon.display_detail('Resetting Apple Software Update '
+                                       'CatalogURL to the default')
             cmd = ['/usr/bin/defaults', 'delete',
                    '/Library/Preferences/com.apple.SoftwareUpdate', 
                    'CatalogURL']
         unused_retcode = subprocess.call(cmd)
-        
-        
+
+
 def setupSoftwareUpdateCheck():
     '''Set defaults for root user and current host.
     Needed for Leopard.'''
@@ -87,7 +95,7 @@ def setupSoftwareUpdateCheck():
            'com.apple.SoftwareUpdate', 'LaunchAppInBackground', 
            '-bool', 'YES']
     unused_retcode = subprocess.call(cmd)
-    
+
 
 CACHEDUPDATELIST = None
 def softwareUpdateList():
@@ -107,7 +115,7 @@ def softwareUpdateList():
                            stdout=subprocess.PIPE, stderr=subprocess.PIPE)
     (output, unused_err) = proc.communicate()                     
     if proc.returncode == 0:
-       updates = [str(item)[5:] for item in output.splitlines() 
+        updates = [str(item)[5:] for item in output.splitlines() 
                        if str(item).startswith('   * ')]
     munkicommon.display_detail(
         'softwareupdate returned %s updates' % len(updates))
@@ -117,14 +125,14 @@ def softwareUpdateList():
     
 def checkForSoftwareUpdates():
     '''Does our Apple Software Update check'''
+    msg = "Checking for available Apple Software Updates..."
     if munkicommon.munkistatusoutput:
-        munkistatus.message("Checking for available "
-                            "Apple Software Updates...")
+        munkistatus.message(msg)
         munkistatus.detail("")
         munkistatus.percent(-1)
+        munkicommon.log(msg)
     else:
-        munkicommon.display_status("Checking for available "
-                                   "Apple Software Updates...")
+        munkicommon.display_status(msg)
     # save the current SUS URL
     original_url = getCurrentSoftwareUpdateServer()
     # switch to a different SUS server if specified
@@ -147,30 +155,52 @@ def checkForSoftwareUpdates():
         os.chmod(softwareupdateapp, newmode)
         
         cmd = [ softwareupdatecheck ]
-    elif osvers == 10:
+    elif osvers > 9:
         # in Snow Leopard we can just use /usr/sbin/softwareupdate, since it
         # now downloads updates the same way as SoftwareUpdateCheck
-        cmd = ['/usr/sbin/softwareupdate', '-d', '-a']
+        cmd = ['/usr/sbin/softwareupdate', '-v', '-d', '-a']
     else:
         # unsupported os version
         return -1
-       
+        
+    # bump up verboseness so we get download percentage done feedback.
+    oldverbose = munkicommon.verbose
+    munkicommon.verbose = oldverbose + 1
+    
     # now check for updates
-    proc = subprocess.Popen(cmd, shell=False, bufsize=1,
-                            stdin=subprocess.PIPE, 
-                            stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-                            
+    proc = subprocess.Popen(cmd, shell=False, bufsize=-1,
+                            stdin=subprocess.PIPE,
+                            stdout=subprocess.PIPE,
+                            stderr=subprocess.STDOUT)
     while True: 
         output = proc.stdout.readline().decode('UTF-8')
         if munkicommon.munkistatusoutput:
-            if munkistatus.getStopButtonState() == 1:
+            if munkicommon.stopRequested():
                 os.kill(proc.pid, 15) #15 is SIGTERM
                 break
         if not output and (proc.poll() != None):
             break
         # send the output to STDOUT or MunkiStatus as applicable
-        # But first, filter out some noise...
-        if "Missing bundle identifier" not in output:
+        if output.startswith('Downloading '):
+            munkicommon.display_status(output.rstrip('\n'))
+        elif output.startswith('   Progress: '):
+            try:
+                percent = int(output[13:].rstrip('\n%'))
+            except ValueError:
+                percent = -1
+            munkicommon.display_percent_done(percent, 100)
+        elif output.startswith('Installed '):
+            # don't display this, it's just confusing
+            pass
+        elif output.startswith('x '):
+            # don't display this, it's just confusing
+            pass
+        elif 'Missing bundle identifier' in output:
+            # don't display this, it's noise
+            pass
+        elif output.rstrip() == '':
+            pass
+        else:
             munkicommon.display_status(output.rstrip('\n'))
     
     retcode = proc.poll()
@@ -194,35 +224,145 @@ def checkForSoftwareUpdates():
     if osvers == 9:
         # put mode back for Software Update.app
         os.chmod(softwareupdateapp, oldmode)
+        
+    # set verboseness back.
+    munkicommon.verbose = oldverbose
     
     # switch back to the original SUS server
     restoreSoftwareUpdateServer(original_url)
     return retcode
     
     
+#
+# Apple information on Distribution ('.dist') files:
+#
+# http://developer.apple.com/library/mac/#documentation/DeveloperTools/
+# Reference/DistributionDefinitionRef/200-Distribution_XML_Ref/
+# Distribution_XML_Ref.html
+#
+# Referred to elsewhere in this code as 'Distribution_XML_Ref.html'
+#
+
+def get_pkgrefs(xml_element):
+    '''Gets all the pkg-refs that are children of the xml_element
+       Returns a list of dictionaries.'''
+    pkgs = []
+    pkgrefs = xml_element.getElementsByTagName('pkg-ref')
+    if pkgrefs:
+        for ref in pkgrefs:
+            keys = ref.attributes.keys()
+            if 'id' in keys:
+                pkgid = ref.attributes['id'].value
+                pkg = {}
+                pkg['id'] = pkgid
+                if 'installKBytes' in keys:
+                    pkg['installKBytes'] = \
+                        ref.attributes['installKBytes'].value
+                # Distribution_XML_Ref.html
+                # says either 'installKBytes' or 'archiveKBytes' is valid
+                if 'archiveKBytes' in keys:
+                    pkg['installKBytes'] = \
+                        ref.attributes['archiveKBytes'].value
+                if 'version' in keys:
+                    pkg['version'] = \
+                        ref.attributes['version'].value
+                if 'auth' in keys:
+                    pkg['auth'] = \
+                        ref.attributes['auth'].value
+                if 'onConclusion' in keys:
+                    pkg['onConclusion'] = \
+                        ref.attributes['onConclusion'].value
+                if ref.firstChild:
+                    pkgfile = ref.firstChild.nodeValue
+                    pkgfile = os.path.basename(pkgfile).lstrip('#./')
+                    if pkgfile:
+                        pkg['package_file'] = pkgfile
+                pkgs.append(pkg)
+    return pkgs
+
+
 def parseDist(filename):
-    '''Attempts to extract:
-    SU_TITLE, SU_VERS, and SU_DESCRIPTION
-    from a .dist file in a Software Update download.'''
-    text = ""
+    '''Parses a dist file, looking for infomation of interest to
+       munki. Returns a dictionary.'''
+    su_name = ""
+    title = ""
+
     dom = minidom.parse(filename)
-    gui_scripts = dom.getElementsByTagName("installer-gui-script")
-    if gui_scripts:
-        localizations = gui_scripts[0].getElementsByTagName("localization")
-        if localizations:
-            string_elements = localizations[0].getElementsByTagName("strings")
-            if string_elements:
-                strings = string_elements[0]
-                for node in strings.childNodes:
-                    text += node.nodeValue
-                    
-                #if 'language' in strings.attributes.keys():
-                #    if strings.attributes['language'
-                #                             ].value.encode(
-                #                                   'UTF-8') == "English":
-                #        for node in strings.childNodes:
-                #            text += node.nodeValue
-                           
+
+    title_elements = dom.getElementsByTagName('title')
+    if title_elements and title_elements[0].firstChild:
+        title = title_elements[0].firstChild.nodeValue
+
+    outlines = {}
+    choices_outlines = dom.getElementsByTagName('choices-outline')
+    if choices_outlines:
+        for outline in choices_outlines:
+            if 'ui' in outline.attributes.keys():
+                # I wonder if we should convert to all lowercase...
+                ui_name = outline.attributes['ui'].value
+            else:
+                ui_name = u'Installer'
+            if not ui_name in outlines:
+                outlines[ui_name] = []
+                # this gets all lines, even children of lines
+                # so we get a flattened list, which is fine
+                # for our purposes for now.
+                # may need to rework if we need tree-style
+                # data in the future
+                lines = outline.getElementsByTagName('line')
+                for line in lines:
+                    if 'choice' in line.attributes.keys():
+                        outlines[ui_name].append(
+                            line.attributes['choice'].value)
+            else:
+                # more than one choices-outline with the same ui-name.
+                # we should throw an exception until we understand how to deal 
+                # with this.
+                # Maybe we can safely merge them, but we'll play it
+                # conversative for now
+                raise AppleUpdateParseError(
+                    'More than one choices-outline with ui=%s in %s'
+                    % (ui_name, filename))
+
+    choices = {}
+    choice_elements = dom.getElementsByTagName("choice")
+    if choice_elements:
+        for choice in choice_elements:
+            keys = choice.attributes.keys()
+            if 'id' in keys:
+                choice_id = choice.attributes['id'].value
+                if not choice_id in choices:
+                    choices[choice_id] = {}
+                pkgrefs = get_pkgrefs(choice)
+                if pkgrefs:
+                    choices[choice_id]['pkg-refs'] = pkgrefs
+            if 'suDisabledGroupID' in keys:
+                # this is the name as displayed from
+                # /usr/sbin/softwareupdate -l
+                su_name = choice.attributes[
+                    'suDisabledGroupID'].value
+
+    # now look in top-level of xml for more pkg-ref info
+    # this gets pkg-refs in child choice elements, too
+    root_pkgrefs = get_pkgrefs(dom)
+    # so remove the ones that we already found in choice elements
+    already_seen_pkgrefs = []
+    for key in choices.keys():
+        for pkgref in choices[key].get('pkg-refs', []):
+            already_seen_pkgrefs.append(pkgref)
+    root_pkgrefs = [item for item in root_pkgrefs
+                    if item not in already_seen_pkgrefs]
+
+    text = ""
+    localizations = dom.getElementsByTagName('localization')
+    if localizations:
+        string_elements = localizations[0].getElementsByTagName('strings')
+        if string_elements:
+            strings = string_elements[0]
+            if strings.firstChild:
+                text = strings.firstChild.nodeValue
+
+    # get title, version and description as displayed in Software Update
     title = vers = description = ""
     keep = False
     for line in text.split('\n'):
@@ -239,7 +379,7 @@ def parseDist(filename):
             line = line[16:]
             # lop off everything up through '
             line = line[line.find("'")+1:]
-        
+
         if keep:
             # replace escaped single quotes
             line = line.replace("\\'","'")
@@ -253,70 +393,255 @@ def parseDist(filename):
             else:
                 # append the line to the description
                 description += line + "\n"
-            
-    # now try to extract the size
+
+    # now try to determine the total installed size
     itemsize = 0
-    if gui_scripts:
-        pkgrefs = gui_scripts[0].getElementsByTagName("pkg-ref")
-        if pkgrefs:
-            for ref in pkgrefs:
-                keys = ref.attributes.keys()
-                if 'installKBytes' in keys:
-                    itemsize = int(
-                            ref.attributes[
-                            'installKBytes'].value.encode('UTF-8'))
-                    break
-            
+    for pkgref in root_pkgrefs:
+        if 'installKBytes' in pkgref:
+            itemsize += int(pkgref['installKBytes'])
+
     if itemsize == 0:
+        # just add up the size of the files in this directory
         for (path, unused_dirs, files) in os.walk(os.path.dirname(filename)):
             for name in files:
                 pathname = os.path.join(path, name)
                 # use os.lstat so we don't follow symlinks
                 itemsize += int(os.lstat(pathname).st_size)
         # convert to kbytes
-        itemsize = int(itemsize/1024)   
-                 
-    return title, vers, description, itemsize
+        itemsize = int(itemsize/1024)
+
+    dist = {}
+    dist['su_name'] = su_name
+    dist['title'] = title
+    dist['version'] = vers
+    dist['installed_size'] = itemsize
+    dist['description'] = description
+    dist['choices-outlines'] = outlines
+    dist['choices'] = choices
+    dist['pkg-refs'] = root_pkgrefs
+    return dist
 
 
-def getRestartInfo(installitemdir):
-    '''Looks at all the RestartActions for all the items in the
-     directory and returns the highest weighted of:
-       RequireRestart
-       RecommendRestart
-       RequireLogout
-       RecommendLogout
-       None'''
-    
-    weight = {}
-    weight['RequireRestart'] = 4
-    weight['RecommendRestart'] = 3
-    weight['RequireLogout'] = 2
-    weight['RecommendLogout'] = 1
-    weight['None'] = 0
-    
+def getRestartInfo(distfile):
+    '''Returns RestartInfo for distfile'''
     restartAction = "None"
-    for item in munkicommon.listdir(installitemdir):
-        if item.endswith(".dist") or item.endswith(".pkg") or \
-                item.endswith(".mpkg"):
-            installeritem = os.path.join(installitemdir, item)
-
-            proc = subprocess.Popen(["/usr/sbin/installer",
-                                    "-query", "RestartAction", 
-                                    "-pkg", installeritem], 
-                                    bufsize=1, 
-                                    stdout=subprocess.PIPE, 
-                                    stderr=subprocess.PIPE)
-            (out, unused_err) = proc.communicate()
-            if out:
-                thisAction = str(out).rstrip('\n')
-                if thisAction in weight.keys():
-                    if weight[thisAction] > weight[restartAction]:
-                        restartAction = thisAction
-            
+    proc = subprocess.Popen(["/usr/sbin/installer",
+                            "-query", "RestartAction", 
+                            "-pkg", distfile], 
+                            bufsize=1, 
+                            stdout=subprocess.PIPE, 
+                            stderr=subprocess.PIPE)
+    (out, unused_err) = proc.communicate()
+    if out:
+        restartAction = out.rstrip('\n')
     return restartAction
 
 
+def actionWeight(action):
+    '''Returns an integer representing the weight of an
+    onConclusion action'''
+    weight = {}
+    weight['RequireShutdown'] = 4
+    weight['RequireRestart'] = 3
+    weight['RecommendRestart'] = 2
+    weight['RequireLogout'] = 1
+    weight['None'] = 0
+    return weight.get(action, 'None')
+
+
+def deDupPkgRefList(pkgref_list):
+    '''some dists have the same package file listed
+       more than once with different attributes
+       we need to de-dupe the list'''
+
+    deduped_list = []
+    for pkg_ref in pkgref_list:
+        matchingitems = [item for item in deduped_list 
+            if item['package_file'] == pkg_ref['package_file']]
+        if matchingitems:
+            # we have a duplicate; we should keep the one that has
+            # the higher weighted 'onConclusion' action
+            if (actionWeight(pkg_ref.get('onConclusion', 'None')) >
+                actionWeight(matchingitems[0].get('onConclusion', 'None'))):
+                deduped_list.remove(matchingitems[0])
+                deduped_list.append(pkg_ref)
+            else:
+                # keep existing item in deduped_list
+                pass
+        else:
+            deduped_list.append(pkg_ref)
+    return deduped_list
+
+
+def getPkgsToInstall(dist, dist_dir=None):
+    '''Given a processed dist dictionary (from parseDist()),
+    Returns a list of pkg-ref dictionaries in the order of install'''
+
+    # Distribution_XML_Ref.html
+    #
+    # The name of the application that is to display the choices specified by  
+    # this element. Values: "Installer" (default), "SoftwareUpdate", or 
+    # "Invisible".
+    # "invisible" seems to be in use as well...
+    if 'SoftwareUpdate' in dist['choices-outlines']:
+        ui_names = ['SoftwareUpdate', 'invisible', 'Invisible']
+    else:
+        ui_names = ['Installer', 'invisible', 'Invisible']
+
+    pkgref_list = []
+    for ui_name in ui_names:
+        if ui_name in dist['choices-outlines']:
+            outline = dist['choices-outlines'][ui_name]
+            choices = dist['choices']
+            for line in outline:
+                if line in choices:
+                    for pkg_ref in choices[line].get('pkg-refs', []):
+                        if 'package_file' in pkg_ref:
+                            if dist_dir:
+                                # make sure pkg is present in dist_dir
+                                # before adding to the list
+                                package_path = os.path.join(dist_dir,
+                                    pkg_ref['package_file'])
+                                if os.path.exists(package_path):
+                                    pkgref_list.append(pkg_ref)
+                            else:
+                                # just add it
+                                pkgref_list.append(pkg_ref)
+
+    return deDupPkgRefList(pkgref_list)
+
+
+def makeFakeDist(title, pkg_refs_to_install):
+    '''Builds a dist script for the list of pkg_refs_to_install
+       Returns xml object'''
+    xmlout = minidom.parseString(
+'''<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<installer-gui-script minSpecVersion="1">
+    <options hostArchitectures="ppc,i386" customize="never"></options>
+    <title></title>
+    <platforms>
+        <client arch="ppc,i386"></client><server arch="ppc,i386"></server>
+    </platforms>
+    <choices-outline ui="SoftwareUpdate">
+        <line choice="su"></line>
+    </choices-outline>
+    <choices-outline>
+        <line choice="su"></line>
+    </choices-outline>
+    <choice id="su" title="">
+    </choice>
+</installer-gui-script>
+''')
+    xmlinst = xmlout.getElementsByTagName('installer-gui-script')[0]
+    xmlchoice = xmlinst.getElementsByTagName('choice')[0]
+    xmltitle = xmlinst.getElementsByTagName('title')[0]
+    for pkg_ref in pkg_refs_to_install:
+        node = xmlout.createElement('pkg-ref')
+        node.setAttribute("id", pkg_ref['id'])
+        if 'auth' in pkg_ref:
+            node.setAttribute('auth', pkg_ref['auth'])
+        if 'onConclusion' in pkg_ref:
+            node.setAttribute('onConclusion', pkg_ref['onConclusion'])
+        node.appendChild(
+            xmlout.createTextNode(pkg_ref.get('package_file','')))
+        # add to choice
+        xmlchoice.appendChild(node)
+
+        node = xmlout.createElement("pkg-ref")
+        node.setAttribute("id", pkg_ref['id'])
+        if 'installKBytes' in pkg_ref:
+            node.setAttribute('installKBytes', pkg_ref['installKBytes'])
+        if 'version' in pkg_ref:
+            node.setAttribute('version', pkg_ref['version'])
+        # add to root of installer-gui-script
+        xmlinst.appendChild(node)
+
+    xmlchoice.setAttribute("title", title)
+    xmltitle.appendChild(xmlout.createTextNode(title))    
+
+    return xmlout
+
+
+class AppleUpdateParseError(Exception):
+    '''We raise this exception when we encounter something
+    unexpected in the update processing'''
+    pass
+    
+    
+def processSoftwareUpdateDownload(appleupdatedir, 
+                            verifypkgsexist=True, writefile=True):
+    '''Given a directory containing an update downloaded by softwareupdate -d 
+    or SoftwareUpdateCheck, attempts to create a simplified .dist file that
+    /usr/sbin/installer can use to successfully install the downloaded
+    update. 
+    Returns dist info as dictionary, 
+    or raises AppleUpdateParseError exception.'''
+
+    availabledists = []
+    availablepkgs = []
+    generated_dist_file = os.path.join(appleupdatedir, 'MunkiGenerated.dist')
+    try:
+        os.unlink(generated_dist_file)
+    except OSError:
+        pass
+
+    # What files do we have to work with? Do we have an appropriate quantity?
+    diritems = os.listdir(appleupdatedir)
+    for diritem in diritems:
+        if diritem.endswith('.dist'):
+            availabledists.append(diritem)
+        elif diritem.endswith('.pkg') or diritem.endswith('.mpkg'):
+            availablepkgs.append(diritem)
+
+    if len(availabledists) != 1:
+        raise AppleUpdateParseError(
+            'Multiple .dist files in update directory %s' % appleupdatedir)
+    if verifypkgsexist and len(availablepkgs) < 1:
+        raise AppleUpdateParseError(
+            'No packages in update directory %s' % appleupdatedir)
+
+    appledistfile = os.path.join(appleupdatedir, availabledists[0])
+    try:
+        dist = parseDist(appledistfile)
+    except (ExpatError, IOError):
+        raise AppleUpdateParseError(
+            'Could not parse .dist file %s' % appleupdatedir)
+
+    if verifypkgsexist:
+        pkg_refs_to_install = getPkgsToInstall(dist, appleupdatedir)
+    else:
+        pkg_refs_to_install = getPkgsToInstall(dist)
+
+    if len(pkg_refs_to_install) == 0:
+        raise AppleUpdateParseError(
+            'Nothing was found to install in %s' % appleupdatedir)
+
+    if verifypkgsexist:
+        pkg_files_to_install = [item['package_file'] 
+                                for item in pkg_refs_to_install]
+        for pkg in availablepkgs:
+            if not pkg in pkg_files_to_install:
+                raise AppleUpdateParseError(
+                    'Package %s missing from list of packages to install '
+                    'in %s' % (pkg, appleupdatedir))
+
+    # combine info from the root pkg-refs and the ones to be installed
+    for choice_pkg_ref in pkg_refs_to_install:
+        root_match = [item for item in dist['pkg-refs']
+                      if choice_pkg_ref['id'] == item['id']]
+        for item in root_match:
+            for key in item.keys():
+                choice_pkg_ref[key] = item[key]
+
+    xmlout = makeFakeDist(dist['title'], pkg_refs_to_install)
+    if writefile:
+        f = open(generated_dist_file, 'w')
+        f.write(xmlout.toxml('utf-8'))
+        f.close()
+    
+    return dist
+
+
 def getSoftwareUpdateInfo():
     '''Parses the Software Update index.plist and the downloaded updates,
     extracting info in the format munki expects. Returns an array of
@@ -372,28 +697,25 @@ def getSoftwareUpdateInfo():
             updatename = products[product_key]
             installitem = os.path.join(updatesdir, updatename)
             if os.path.exists(installitem) and os.path.isdir(installitem):
-                for subitem in munkicommon.listdir(installitem):
-                    if subitem.endswith('.dist'):
-                        distfile = os.path.join(installitem, subitem)
-                        (title, vers, 
-                            description, 
-                            installedsize) = parseDist(distfile)
-                        iteminfo = {}
-                        iteminfo["installer_item"] = updatename
-                        iteminfo["name"] = title
-                        iteminfo["description"] = description
-                        if iteminfo["description"] == '':
-                            iteminfo["description"] = \
-                                            "Updated Apple software."
-                        iteminfo["version_to_install"] = vers
-                        iteminfo['display_name'] = title
-                        iteminfo['installed_size'] = installedsize
-                        restartAction = getRestartInfo(installitem)
-                        if restartAction != "None":
-                            iteminfo['RestartAction'] = restartAction
-                        
-                        infoarray.append(iteminfo)
-                        break
+                try:
+                    dist = processSoftwareUpdateDownload(installitem)
+                except AppleUpdateParseError, e:
+                    munkicommon.display_error('%s' % e)
+                else:
+                    iteminfo = {}
+                    iteminfo["installer_item"] = os.path.join(
+                        updatename, 'MunkiGenerated.dist')
+                    iteminfo["name"] = dist['su_name']
+                    iteminfo["description"] = (
+                        dist['description'] or "Updated Apple software.")
+                    iteminfo["version_to_install"] = dist['version']
+                    iteminfo['display_name'] = dist['title']
+                    iteminfo['installed_size'] = dist['installed_size']
+                    restartAction = getRestartInfo(
+                        os.path.join(installitem, 'MunkiGenerated.dist'))
+                    if restartAction != "None":
+                        iteminfo['RestartAction'] = restartAction
+                    infoarray.append(iteminfo)
 
     return infoarray
 
@@ -469,8 +791,8 @@ def appleSoftwareUpdatesAvailable(forcecheck=False, suppresscheck=False):
         if now.timeIntervalSinceDate_(nextSUcheck) >= 0:
             unused_retcode = checkForSoftwareUpdates()
         else:
-            munkicommon.log("Skipping Apple Software Update check because "
-                            "we last checked on %s..." % lastSUcheck)
+            munkicommon.log('Skipping Apple Software Update check because '
+                            'we last checked on %s...' % lastSUcheck)
 
     if writeAppleUpdatesFile():
         displayAppleUpdateInfo()
@@ -492,9 +814,7 @@ def clearAppleUpdateInfo():
 
 def installAppleUpdates():
     '''Uses /usr/sbin/installer to install updates previously
-    downloaded. Some items downloaded by SoftwareUpdate are not
-    installable by /usr/sbin/installer, so this approach may fail
-    to install all downloaded updates'''
+    downloaded.'''
 
     restartneeded = False
     appleupdatelist = getSoftwareUpdateInfo()
@@ -503,26 +823,20 @@ def installAppleUpdates():
     if appleupdatelist:
         munkicommon.report['AppleUpdateList'] = appleupdatelist
         munkicommon.savereport()
-        try:
-            # once we start, we should remove /Library/Updates/index.plist
-            # because it will point to items we've already installed
-            os.unlink('/Library/Updates/index.plist')
-            # remove the appleupdatesfile 
-            # so Managed Software Update.app doesn't display these
-            # updates again
-            os.unlink(appleUpdatesFile)
-        except (OSError, IOError):
-            pass
-        # now try to install the updates
         (restartneeded, unused_skipped_installs) = \
                 installer.installWithInfo("/Library/Updates",
                                           appleupdatelist)
         if restartneeded:
             munkicommon.report['RestartRequired'] = True
         munkicommon.savereport()
+        clearAppleUpdateInfo()
+                
     return restartneeded
 
 
+
+
+
 # define this here so we can access it in multiple functions
 appleUpdatesFile = os.path.join(munkicommon.pref('ManagedInstallDir'),
                                 'AppleUpdates.plist')

code/client/munkilib/installer.py

@@ -129,12 +129,12 @@ def install(pkgpath, choicesXMLpath=None, suppressBundleRelocation=False):
                                   '-target', '/']
     if choicesXMLpath:
         cmd.extend(['-applyChoiceChangesXML', choicesXMLpath])
-    proc = subprocess.Popen(cmd, shell=False, bufsize=1,
+    proc = subprocess.Popen(cmd, shell=False, bufsize=-1,
                             stdin=subprocess.PIPE,
                             stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
 
     while True:
-        installinfo =  proc.stdout.readline().decode('UTF-8')
+        installinfo = proc.stdout.readline().decode('UTF-8')
         if not installinfo and (proc.poll() != None):
             break
         if installinfo.startswith("installer:"):
@@ -480,8 +480,7 @@ def installWithInfo(dirpath, installlist, only_forced=False):
         if munkicommon.stopRequested():
             return restartflag
         if "installer_item" in item:
-            display_name = item.get('display_name') or item.get('name') or \
-                           item.get('manifestitem')
+            display_name = item.get('display_name') or item.get('name')
             version_to_install = item.get('version_to_install','')
             if munkicommon.munkistatusoutput:
                 munkistatus.message("Installing %s (%s of %s)..." %
@@ -583,20 +582,12 @@ def installWithInfo(dirpath, installlist, only_forced=False):
                         restartflag = True
                     munkicommon.unmountdmg(mountpoints[0])
                 else:
-                    itempath = munkicommon.findInstallerItem(itempath)
-                    if (itempath.endswith(".pkg") or \
-                            itempath.endswith(".mpkg")):
-                        (retcode, needtorestart) = install(itempath,
-                                                           choicesXMLfile,
-                                                    suppressBundleRelocation)
-                        if needtorestart:
-                            restartflag = True
-                    elif os.path.isdir(itempath):
-                        # directory of packages,
-                        # like what we get from Software Update
-                        (retcode, needtorestart) = installall(itempath,
-                                                              choicesXMLfile,
-                                                    suppressBundleRelocation)
+                    if (itempath.endswith(".pkg") or 
+                        itempath.endswith(".mpkg") or 
+                        itempath.endswith(".dist")):
+                        (retcode, needtorestart) = \
+                            install(itempath, choicesXMLfile,
+                                    suppressBundleRelocation)
                         if needtorestart:
                             restartflag = True
 
@@ -649,12 +640,20 @@ def installWithInfo(dirpath, installlist, only_forced=False):
                 if os.path.exists(itempath):
                     if os.path.isdir(itempath):
                         retcode = subprocess.call(
-                                                ["/bin/rm", "-rf", itempath])
+                            ["/bin/rm", "-rf", itempath])
+                    elif itempath.endswith('MunkiGenerated.dist'):
+                        # softwareupdate item handled by munki
+                        # remove enclosing directory
+                        retcode = subprocess.call(
+                            ["/bin/rm", "-rf", os.path.dirname(itempath)])
                     else:
+                        # flat pkg or dmg
                         retcode = subprocess.call(["/bin/rm", itempath])
-                shadowfile = os.path.join(itempath,".shadow")
-                if os.path.exists(shadowfile):
-                    retcode = subprocess.call(["/bin/rm", shadowfile])
+                        if itempath.endswith('.dmg'):
+                            shadowfile = os.path.join(itempath,".shadow")
+                            if os.path.exists(shadowfile):
+                                retcode = subprocess.call(
+                                    ["/bin/rm", shadowfile])
                     
     return (restartflag, skipped_installs)
 
@@ -673,6 +672,7 @@ def writefile(stringdata, path):
 
 
 def runUninstallScript(name, path):
+    '''Runs the uninstall script'''
     if munkicommon.munkistatusoutput:
         munkistatus.message("Running uninstall script "
                             "for %s..." % name)
@@ -738,8 +738,7 @@ def processRemovals(removallist, only_forced=False):
             continue
 
         index += 1
-        name = item.get('display_name') or item.get('name') or \
-               item.get('manifestitem')
+        name = item.get('display_name') or item.get('name')
         if munkicommon.munkistatusoutput:
             munkistatus.message("Removing %s (%s of %s)..." %
                                 (name, index, len(removallist)))

code/client/munkilib/munkicommon.py

@@ -74,7 +74,7 @@ class PreferencesError(Error):
 def get_version():
     """Returns version of munkitools, reading version.plist
     and svnversion"""
-    version = "UNKNOWN"
+    vers = "UNKNOWN"
     build = ""
     # find the munkilib directory, and the version files
     munkilibdir = os.path.dirname(os.path.abspath(__file__))
@@ -86,7 +86,7 @@ def get_version():
             pass
         else:
             try:
-                version = vers_plist['CFBundleShortVersionString']
+                vers = vers_plist['CFBundleShortVersionString']
             except KeyError:
                 pass
     svnversionfile = os.path.join(munkilibdir, "svnversion")
@@ -99,8 +99,8 @@ def get_version():
         except OSError:
             pass
     if build:
-        version = version + " Build " + build
-    return version
+        vers = vers + " Build " + build
+    return vers
 
 
 # output and logging functions
@@ -271,7 +271,7 @@ def display_warning(msg, *args):
     # collect the warning for later reporting
     if not 'Warnings' in report:
         report['Warnings'] = []
-    report['Warnings'].append(str(msg))
+    report['Warnings'].append('%s' % msg)
 
 
 def reset_errors():
@@ -294,7 +294,7 @@ def display_error(msg, *args):
     # collect the errors for later reporting
     if not 'Errors' in report:
         report['Errors'] = []
-    report['Errors'].append(str(msg))
+    report['Errors'].append('%s' % msg)
 
 
 def format_time(timestamp=None):
@@ -1159,28 +1159,13 @@ def nameAndVersion(aString):
     return (aString, '')
 
 
-def findInstallerItem(path):
-    """Find an installer item in the directory given by path"""
-    if path.endswith('.pkg') or path.endswith('.mpkg') or \
-       path.endswith('.dmg'):
-        return path
+def isInstallerItem(path):
+    """Verifies we have an installer item"""
+    if (path.endswith('.pkg') or path.endswith('.mpkg') or
+        path.endswith('.dmg') or path.endswith('.dist')):
+        return True
     else:
-        # Apple Software Updates download as directories
-        # with .dist files and .pkgs
-        if os.path.exists(path) and os.path.isdir(path):
-            for item in listdir(path):
-                if item.endswith('.pkg'):
-                    return path
-
-            # we didn't find a pkg at this level
-            # look for a Packages dir
-            path = os.path.join(path,'Packages')
-            if os.path.exists(path) and os.path.isdir(path):
-                for item in listdir(path):
-                    if item.endswith('.pkg'):
-                        return path
-    # found nothing!
-    return ''
+        return False
 
 
 def getPackageMetaData(pkgitem):
@@ -1202,8 +1187,7 @@ def getPackageMetaData(pkgitem):
               (some may not be installed on some machines)
     """
 
-    pkgitem = findInstallerItem(pkgitem)
-    if pkgitem == None:
+    if not isInstallerItem(pkgitem):
         return {}
 
     # first get the data /usr/sbin/installer will give us
@@ -1421,7 +1405,7 @@ def getSpotlightInstalledApplications():
     """
     argv = ['/usr/bin/mdfind', '-0', 'kMDItemKind = \'Application\'']
     p = subprocess.Popen(argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    (stdout, stderr) = p.communicate()
+    (stdout, unused_stderr) = p.communicate()
     rc = p.wait()
 
     applist = []
@@ -1445,7 +1429,7 @@ def getLSInstalledApplications():
     apps = LaunchServices._LSCopyAllApplicationURLs(None)
     applist = []
     for app in apps:
-        (status, fsobj, url) = LaunchServices.LSGetApplicationForURL(
+        (status, fsobj, unused_url) = LaunchServices.LSGetApplicationForURL(
             app, _unsigned(LaunchServices.kLSRolesAll), None, None)
         if status != 0:
             continue