Merge branch 'master' of https://code.google.com/p/munki

code/Managed Software Update/munki.py

@@ -125,6 +125,7 @@ def pref(pref_name):
     default_prefs = {
         'ManagedInstallDir': '/Library/Managed Installs',
         'InstallAppleSoftwareUpdates': False,
+        'AppleSoftwareUpdatesOnly': False,
         'ShowRemovalDetail': False,
         'InstallRequiresLogout': False
     }
@@ -269,7 +270,8 @@ def getAppleUpdates():
     plist = {}
     appleUpdatesFile = os.path.join(managedinstallbase, 'AppleUpdates.plist')
     if (os.path.exists(appleUpdatesFile) and
-            pref('InstallAppleSoftwareUpdates')):
+            (pref('InstallAppleSoftwareUpdates') or
+            pref('AppleSoftwareUpdatesOnly'))):
         try:
             plist = FoundationPlist.readPlist(appleUpdatesFile)
         except FoundationPlist.NSPropertyListSerializationException:

code/client/munkilib/installer.py

@@ -260,6 +260,8 @@ def install(pkgpath, choicesXMLpath=None, suppressBundleRelocation=False,
     # installer exited
     retcode = job.returncode()
     if retcode != 0:
+        # append stdout to our installer output
+        installeroutput.extend(job.stderr.read().splitlines())
         munkicommon.display_status_minor(
             "Install of %s failed with return code %s" % (packagename, retcode))
         munkicommon.display_error("-"*78)

code/client/munkilib/munkicommon.py

@@ -137,8 +137,8 @@ def set_file_nonblock(f, non_blocking=True):
 
 
 class Popen(subprocess.Popen):
-    '''Subclass of subprocess.Popen to add support for
-    timeouts for some operations.'''
+    """Subclass of subprocess.Popen to add support for timeouts."""
+
     def timed_readline(self, f, timeout):
         """Perform readline-like operation with timeout.
 
@@ -198,8 +198,9 @@ class Popen(subprocess.Popen):
         if self.stderr is not None:
             set_file_nonblock(self.stderr)
             fds.append(self.stderr)
-        if input is not None and sys.stdin is not None:
-            sys.stdin.write(input)
+
+        if std_in is not None and sys.stdin is not None:
+            sys.stdin.write(std_in)
 
         returncode = None
         inactive = 0
@@ -1982,10 +1983,15 @@ def getSPApplicationData():
     global SP_APPCACHE
     if not SP_APPCACHE:
         cmd = ['/usr/sbin/system_profiler', 'SPApplicationsDataType', '-xml']
-        proc = subprocess.Popen(cmd, shell=False, bufsize=-1,
-                                stdin=subprocess.PIPE,
-                                stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        (output, unused_error) = proc.communicate()
+        proc = Popen(cmd, shell=False, bufsize=-1,
+                     stdin=subprocess.PIPE, stdout=subprocess.PIPE,
+                     stderr=subprocess.PIPE)
+        try:
+            output, unused_error = proc.communicate(timeout=60)
+        except TimeoutError:
+            display_error(
+                'system_profiler hung; skipping SPApplicationsDataType query')
+            return SP_APPCACHE
         try:
             plist = FoundationPlist.readPlistFromString(output)
             # system_profiler xml is an array

code/client/munkilib/removepackages.py

@@ -218,7 +218,7 @@ def ImportPackage(packagepath, curs):
     pkgname = os.path.basename(packagepath)
 
     if not os.path.exists(packagepath):
-        munkicommon.display_error("%s not found." % packagepath)
+        munkicommon.display_error("%s not found.", packagepath)
         return
 
     if not os.path.isdir(packagepath):
@@ -226,7 +226,7 @@ def ImportPackage(packagepath, curs):
         # so we won't print a warning for that specific one.
         if pkgname != "BSD.pkg":
             munkicommon.display_warning(
-                "%s is not a valid receipt. Skipping." % packagepath)
+                "%s is not a valid receipt. Skipping.", packagepath)
         return
 
     if not os.path.exists(bompath):
@@ -235,13 +235,13 @@ def ImportPackage(packagepath, curs):
         bompath = os.path.join(packagepath, "Contents/Resources",
                                 bomname)
         if not os.path.exists(bompath):
-            munkicommon.display_warning("%s has no BOM file. Skipping." %
-                                        packagepath)
+            munkicommon.display_warning(
+                "%s has no BOM file. Skipping.", packagepath)
             return
 
     if not os.path.exists(infopath):
-        munkicommon.display_warning("%s has no Info.plist. Skipping." %
-                                    packagepath)
+        munkicommon.display_warning(
+            "%s has no Info.plist. Skipping.", packagepath)
         return
 
     timestamp = os.stat(packagepath).st_mtime
@@ -578,7 +578,7 @@ def initDatabase(forcerebuild=False):
 
             if item.endswith(".pkg"):
                 receiptpath = os.path.join(receiptsdir, item)
-                munkicommon.display_detail("Importing %s..." % receiptpath)
+                munkicommon.display_detail("Importing %s...", receiptpath)
                 ImportPackage(receiptpath, curs)
                 currentpkgindex += 1
                 munkicommon.display_percent_done(currentpkgindex, pkgcount)
@@ -595,7 +595,7 @@ def initDatabase(forcerebuild=False):
 
             if item.endswith(".bom"):
                 bompath = os.path.join(bomsdir, item)
-                munkicommon.display_detail("Importing %s..." % bompath)
+                munkicommon.display_detail("Importing %s...", bompath)
                 ImportBom(bompath, curs)
                 currentpkgindex += 1
                 munkicommon.display_percent_done(currentpkgindex, pkgcount)
@@ -608,7 +608,7 @@ def initDatabase(forcerebuild=False):
                 os.remove(packagedb)
                 return False
 
-            munkicommon.display_detail("Importing %s..." % pkg)
+            munkicommon.display_detail("Importing %s...", pkg)
             ImportFromPkgutil(pkg, curs)
             currentpkgindex += 1
             munkicommon.display_percent_done(currentpkgindex, pkgcount)
@@ -640,18 +640,18 @@ def getpkgkeys(pkgnames):
     for pkg in pkgnames:
         values_t = (pkg, )
         munkicommon.display_debug1(
-                    "select pkg_key from pkgs where pkgid = %s" % pkg)
+                    "select pkg_key from pkgs where pkgid = %s", pkg)
         pkg_keys = curs.execute('select pkg_key from pkgs where pkgid = ?',
                              values_t).fetchall()
         if not pkg_keys:
             # try pkgname
             munkicommon.display_debug1(
-                        "select pkg_key from pkgs where pkgname = %s" % pkg)
+                        "select pkg_key from pkgs where pkgname = %s", pkg)
             pkg_keys = curs.execute(
                 'select pkg_key from pkgs where pkgname = ?',
                                                         values_t).fetchall()
         if not pkg_keys:
-            munkicommon.display_error("%s not found in database." % pkg)
+            munkicommon.display_error("%s not found in database.", pkg)
             pkgerror = True
         else:
             for row in pkg_keys:
@@ -662,7 +662,7 @@ def getpkgkeys(pkgnames):
 
     curs.close()
     conn.close()
-    munkicommon.display_debug1("pkgkeys: %s" % pkgkeyslist)
+    munkicommon.display_debug1("pkgkeys: %s", pkgkeyslist)
     return pkgkeyslist
 
 
@@ -763,7 +763,7 @@ def removeReceipts(pkgkeylist, noupdateapplepkgdb):
                 receiptpath = findBundleReceiptFromID(pkgid)
 
             if receiptpath and os.path.exists(receiptpath):
-                munkicommon.display_detail("Removing %s..." % receiptpath)
+                munkicommon.display_detail("Removing %s...", receiptpath)
                 unused_retcode = subprocess.call(
                                             ["/bin/rm", "-rf", receiptpath])
 
@@ -937,8 +937,8 @@ def removeFilesystemItems(removalpaths, forcedeletebundles):
                     try:
                         os.rmdir(pathtoremove)
                     except (OSError, IOError), err:
-                        msg = "Couldn't remove directory %s - %s" % \
-                               (pathtoremove, err)
+                        msg = "Couldn't remove directory %s - %s" % (
+                            pathtoremove, err)
                         munkicommon.display_error(msg)
                         removalerrors = removalerrors + "\n" + msg
                 else:
@@ -948,7 +948,7 @@ def removeFilesystemItems(removalpaths, forcedeletebundles):
                     # around
                     if (forcedeletebundles and isBundle(pathtoremove)):
                         munkicommon.display_warning(
-                            "Removing non-empty bundle: %s" % pathtoremove)
+                            "Removing non-empty bundle: %s", pathtoremove)
                         retcode = subprocess.call(['/bin/rm', '-r',
                                                    pathtoremove])
                         if retcode:

code/client/munkilib/updatecheck.py

@@ -1071,6 +1071,16 @@ def getItemDetail(name, cataloglist, vers=''):
                         rejected_items.append(reason)
                         continue
 
+                if item.get('installable_condition'):
+                    pkginfo_predicate = item['installable_condition']
+                    if not predicateEvaluatesAsTrue(pkginfo_predicate):
+                        reason = (('Rejected item %s, version %s '
+                                    'with installable_condition: %s.')
+                                    % (item['name'], item['version'],
+                                       item['installable_condition']))
+                        rejected_items.append(reason)
+                        continue
+
                 # item name, version, minimum_os_version, and
                 # supported_architecture are all OK
                 munkicommon.display_debug1(

code/client/supervisor

@@ -18,6 +18,7 @@
 """Tool to supervise launch other binaries."""
 
 
+import errno
 import getopt
 import logging
 import logging.handlers
@@ -26,6 +27,7 @@ import random
 import signal
 import subprocess
 import sys
+import tempfile
 import time
 
 
@@ -33,6 +35,10 @@ class Error(Exception):
     """Base error."""
 
 
+class ExecuteError(Error):
+    """Error executing."""
+
+
 class OptionError(Error):
     """Option error."""
 
@@ -41,28 +47,58 @@ class TimeoutError(Error):
     """Timeout while execute() running."""
 
 
+DEFAULT_ERROR_EXEC_EXIT_CODES = [1]
+EXIT_STATUS_TIMEOUT = -99
+KILL_WAIT_SECS = 1
+
+
 class Supervisor(object):
-    def __init__(self):
-        """Init."""
+    def __init__(self, delayrandom_abort=False):
+        """Init.
+
+        Args:
+            delayrandom_abort: bool, default False. If True, sending
+                a SIGUSR1 to the process will stop any initial delayrandom
+                from continuing to countdown, and will immediately end the
+                delay. Note that setting this on multiple Supervisor instances
+                in one process might not work too well depending on the
+                timing of the execute() calls, see below.
+        """
         self.options = {
+            'error-exec': None,
+            'error-exec-exit-codes': None,
             'timeout': None,
             'delayrandom': None,
             'stdout': None,
             'stderr': None,
             'debug': None,
         }
+        self.exit_status = None
+        self.delayrandom_abort = delayrandom_abort
 
     def setOptions(self, **kwargs):
         for k in kwargs:
             self.options[k] = kwargs[k]
 
+    def signalHandler(self, signum, unused_frame):
+        if signum == signal.SIGUSR1:
+            self.continue_sleeping = False
+
     def execute(self, args):
         """Exec.
 
         Args:
             args: list, arguments to execute, args[0] is binary name
         """
-        logging.debug('execute(%s)' % args)
+        logging.debug('execute(%s)' % str(args))
+
+        if self.delayrandom_abort:
+            # A second Supervisor process will not take over the previous
+            # Supervisor process who is holding this signal now.
+            if signal.getsignal(signal.SIGUSR1) == signal.SIG_DFL:
+                signal.signal(signal.SIGUSR1, self.signalHandler)
+            self.continue_sleeping = True
+
         if 'delayrandom' in self.options and self.options['delayrandom']:
           max_secs = self.options['delayrandom']
           random_secs = random.randrange(0, max_secs)
@@ -71,12 +107,39 @@ class Supervisor(object):
               max_secs, random_secs)
           time.sleep(random_secs)
 
-        proc = subprocess.Popen(
-            args,
-            preexec_fn=lambda: os.setpgid(os.getpid(), os.getpid()),
-        )
+        if self.delayrandom_abort:
+          if not self.continue_sleeping:
+            logging.debug('Awoken from random delay by signal')
+          signal.signal(signal.SIGUSR1, signal.SIG_DFL)
 
-        self.returncode = None
+        if self.options['error-exec']:
+            self.stdout = tempfile.NamedTemporaryFile()
+            stdout_pipe = self.stdout
+            self.stderr = tempfile.NamedTemporaryFile()
+            stderr_pipe = self.stderr
+            # Parse error-exec-exit-codes, or set default if not provided.
+            exit_codes = self.options['error-exec-exit-codes']
+            if exit_codes:
+                self.error_exec_codes = [int(i) for i in exit_codes.split(',')]
+            else:
+                self.error_exec_codes = DEFAULT_ERROR_EXEC_EXIT_CODES
+        else:
+            stdout_pipe = None
+            stderr_pipe = None
+
+        try:
+            proc = subprocess.Popen(
+                args,
+                preexec_fn=lambda: os.setpgid(os.getpid(), os.getpid()),
+                stdout=stdout_pipe,
+                stderr=stderr_pipe,
+            )
+        except OSError, e:
+            self.exit_status = 127
+            raise ExecuteError(str(e))
+
+        self.exit_status = None
+        self.continue_sleeping = True
 
         start_time = time.time()
 
@@ -84,9 +147,9 @@ class Supervisor(object):
             while 1:
                 slept = 0
 
-                returncode = proc.poll()
-                if returncode is not None:
-                    self.returncode = returncode
+                exit_status = proc.poll()
+                if exit_status is not None:
+                    self.exit_status = exit_status
                     break
 
                 if 'timeout' in self.options and self.options['timeout']:
@@ -102,13 +165,69 @@ class Supervisor(object):
 
         except TimeoutError:
             logging.critical('Timeout error executing %s', ' '.join(args))
-            signal.signal(signal.SIGCHLD, signal.SIG_IGN)
-            os.kill(-1 * proc.pid, signal.SIGTERM)
-            signal.signal(signal.SIGCHLD, signal.SIG_DFL)
+            self.killPid(proc.pid)
+            self.exit_status = EXIT_STATUS_TIMEOUT
             raise
 
-    def GetReturnCode(self):
-        return self.returncode
+
+    def killPid(self, pid):
+        """Kill a pid, aggressively if necessary."""
+
+        exited = {}
+
+        class __ChildExit(Exception):
+            """Child exited."""
+
+        def __sigchld_handler(signum, frame):
+            if signum == signal.SIGCHLD:
+                os.waitpid(pid, os.WNOHANG)
+                exited[pid] = True
+
+        try:
+            signal.signal(signal.SIGCHLD, __sigchld_handler)
+            logging.warning('Sending SIGTERM to %d', pid)
+            os.kill(-1 * pid, signal.SIGTERM)  # *-1 = entire process group
+            time.sleep(KILL_WAIT_SECS)
+            if pid in exited:
+                return
+            logging.warning('Sending SIGKILL to %d', pid)
+            os.kill(-1 * pid, signal.SIGKILL)
+            time.sleep(KILL_WAIT_SECS)
+        except OSError, e:
+            if e.args[0] == errno.ESRCH:
+                logging.warning('pid %d died on its own')
+            else:
+                logging.critical('killPid: %s', str(e))
+
+        if pid in exited:
+            return
+
+        logging.debug('pid %d will not die', pid)
+
+    def getExitStatus(self):
+        return self.exit_status
+
+    def cleanup(self):
+        """Handle errors and call error-exec specified bin."""
+        if not self.options['error-exec']:
+            return
+
+        if self.exit_status in self.error_exec_codes:
+            did_timeout = int(bool(self.exit_status is EXIT_STATUS_TIMEOUT))
+            arg_str = self.options['error-exec']
+            arg_str = arg_str.replace('{EXIT}', str(self.exit_status))
+            arg_str = arg_str.replace('{TIMEOUT}', str(did_timeout))
+            arg_str = arg_str.replace('{STDOUT}', self.stdout.name)
+            arg_str = arg_str.replace('{STDERR}', self.stderr.name)
+            args = ('/bin/sh', '-c', arg_str)
+            error_supv = Supervisor()
+            error_supv.setOptions(timeout=5 * 3600)
+            error_supv.execute(args)
+
+        self.stdout.close()
+        self.stdout = None
+        self.stderr.close()
+        self.stderr = None
 
 
 def parseOpts(argv):
@@ -124,6 +243,7 @@ def parseOpts(argv):
             argv, '',
             [
                 'timeout=', 'delayrandom=', 'debug', 'help',
+                'error-exec=', 'error-exec-exit-codes=',
             ])
     except getopt.GetoptError, e:
         raise OptionError(str(e))
@@ -146,8 +266,32 @@ def Usage():
             after n seconds, terminate the executable
         --delayrandom n
             delay the execution of executable by random seconds up to n
+        --error-exec "path and options string"
+            exec path when executable returns non zero exit status.
+
+            in this mode the stdout and stderr from the supervised
+            executable are recorded to temp files.
+
+            the path and options string can include tokens which will be
+            replaced with values.  note the braces {} should be included.
+
+                {EXIT} = exit status
+                {TIMEOUT} = 1 or 0, timeout did or did not occur
+                {STDOUT} = path to stdout file
+                {STDERR} = path to stderr file
+
+            the error-exec bin may use the stdin, stderr files while it is
+            executing, but it should assume they will disappear when
+            the error-exec bin returns with any exit status.
+
+            the bin should not run more than 5 minutes or it will be
+            terminated.
+        --error-exec-exit-codes "1,100,203"
+            comma-delimited list of integer exit status codes.  If the
+            supervised script exits with one of these codes, the error-exec
+            executable will be run.  Default: "1" 
         --debug
-            enable debugging output
+            enable debugging output, all logs to stderr and not syslog.
         --help
             this text
         --
@@ -175,6 +319,17 @@ def processOpts(options, args):
     return True
 
 
+def setupSyslog():
+    """Setup syslog as a logger."""
+    logger = logging.getLogger()
+    syslog = logging.handlers.SysLogHandler('/var/run/syslog')
+    formatter = logging.Formatter(
+        '%(filename)s[%(process)d]: %(levelname)s %(message)s')
+    syslog.setFormatter(formatter)
+    syslog.setLevel(logging.DEBUG)
+    logger.addHandler(syslog)
+
+
 def main(argv):
     try:
         options, args = parseOpts(argv[1:])
@@ -184,15 +339,30 @@ def main(argv):
         logging.error(str(e))
         return 1
 
+    if options.get('debug', None) is None:
+        setupSyslog()
+
     try:
-        sp = Supervisor()
+        sp = Supervisor(delayrandom_abort=True)
         sp.setOptions(**options)
-        sp.execute(args)
-        return sp.GetReturnCode()
     except Error, e:
-        logging.debug('%s %s', e.__class__.__name__, str(e))
+        logging.exception('%s %s', e.__class__.__name__, str(e))
         return 1
 
+    ex = 0
+
+    try:
+        sp.execute(args)
+        ex = sp.getExitStatus()
+    except TimeoutError, e:
+        ex = 1
+    except Error, e:
+        logging.exception('%s %s', e.__class__.__name__, str(e))
+        ex = 1
+
+    sp.cleanup()
+    return ex
+
 
 if __name__ == '__main__':
     sys.exit(main(sys.argv))