mirror/munki
1
0
Fork 0
mirror of https://github.com/munki/munki.git synced 2026-04-22 12:38:23 -05:00
Code Issues Packages Projects Releases Wiki Activity

Supervisor improvements:

Browse Source 
- more robost killing; sigterm, then sigkill processes if they're still running.
- new --error-exec option, to execute an external script if primary process exits with one of --error-exec-exit-codes.  Intended to be used to notify admins of Munki crashes, by calling report_broken_client or similar script to email/post to server/etc.
This commit is contained in:
Justin McWilliams
2012-11-12 14:19:07 -05:00
parent b31e82905f
commit e6cd53188d
1 changed files with 191 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
code/client/supervisor
+191 -21
View File
@@ -18,6 +18,7 @@
"""Tool to supervise launch other binaries."""
import errno
import getopt
import logging
import logging.handlers
@@ -26,6 +27,7 @@ import random
import signal
import subprocess
import sys
import tempfile
import time
@@ -33,6 +35,10 @@ class Error(Exception):
"""Base error."""
class ExecuteError(Error):
"""Error executing."""
class OptionError(Error):
"""Option error."""
@@ -41,28 +47,58 @@ class TimeoutError(Error):
"""Timeout while execute() running."""
DEFAULT_ERROR_EXEC_EXIT_CODES = [1]
EXIT_STATUS_TIMEOUT = -99
KILL_WAIT_SECS = 1
class Supervisor(object):
def __init__(self):
"""Init."""
def __init__(self, delayrandom_abort=False):
"""Init.
Args:
delayrandom_abort: bool, default False. If True, sending
a SIGUSR1 to the process will stop any initial delayrandom
from continuing to countdown, and will immediately end the
delay. Note that setting this on multiple Supervisor instances
in one process might not work too well depending on the
timing of the execute() calls, see below.
"""
self.options = {
'error-exec': None,
'error-exec-exit-codes': None,
'timeout': None,
'delayrandom': None,
'stdout': None,
'stderr': None,
'debug': None,
}
self.exit_status = None
self.delayrandom_abort = delayrandom_abort
def setOptions(self, **kwargs):
for k in kwargs:
self.options[k] = kwargs[k]
def signalHandler(self, signum, unused_frame):
if signum == signal.SIGUSR1:
self.continue_sleeping = False
def execute(self, args):
"""Exec.
Args:
args: list, arguments to execute, args[0] is binary name
"""
logging.debug('execute(%s)' % args)
logging.debug('execute(%s)' % str(args))
if self.delayrandom_abort:
# A second Supervisor process will not take over the previous
# Supervisor process who is holding this signal now.
if signal.getsignal(signal.SIGUSR1) == signal.SIG_DFL:
signal.signal(signal.SIGUSR1, self.signalHandler)
self.continue_sleeping = True
if 'delayrandom' in self.options and self.options['delayrandom']:
max_secs = self.options['delayrandom']
random_secs = random.randrange(0, max_secs)
@@ -71,12 +107,39 @@ class Supervisor(object):
max_secs, random_secs)
time.sleep(random_secs)
proc = subprocess.Popen(
args,
preexec_fn=lambda: os.setpgid(os.getpid(), os.getpid()),
)
if self.delayrandom_abort:
if not self.continue_sleeping:
logging.debug('Awoken from random delay by signal')
signal.signal(signal.SIGUSR1, signal.SIG_DFL)
self.returncode = None
if self.options['error-exec']:
self.stdout = tempfile.NamedTemporaryFile()
stdout_pipe = self.stdout
self.stderr = tempfile.NamedTemporaryFile()
stderr_pipe = self.stderr
# Parse error-exec-exit-codes, or set default if not provided.
exit_codes = self.options['error-exec-exit-codes']
if exit_codes:
self.error_exec_codes = [int(i) for i in exit_codes.split(',')]
else:
self.error_exec_codes = DEFAULT_ERROR_EXEC_EXIT_CODES
else:
stdout_pipe = None
stderr_pipe = None
try:
proc = subprocess.Popen(
args,
preexec_fn=lambda: os.setpgid(os.getpid(), os.getpid()),
stdout=stdout_pipe,
stderr=stderr_pipe,
)
except OSError, e:
self.exit_status = 127
raise ExecuteError(str(e))
self.exit_status = None
self.continue_sleeping = True
start_time = time.time()
@@ -84,9 +147,9 @@ class Supervisor(object):
while 1:
slept = 0
returncode = proc.poll()
if returncode is not None:
self.returncode = returncode
exit_status = proc.poll()
if exit_status is not None:
self.exit_status = exit_status
break
if 'timeout' in self.options and self.options['timeout']:
@@ -102,13 +165,69 @@ class Supervisor(object):
except TimeoutError:
logging.critical('Timeout error executing %s', ' '.join(args))
signal.signal(signal.SIGCHLD, signal.SIG_IGN)
os.kill(-1 * proc.pid, signal.SIGTERM)
signal.signal(signal.SIGCHLD, signal.SIG_DFL)
self.killPid(proc.pid)
self.exit_status = EXIT_STATUS_TIMEOUT
raise
def GetReturnCode(self):
return self.returncode
def killPid(self, pid):
"""Kill a pid, aggressively if necessary."""
exited = {}
class __ChildExit(Exception):
"""Child exited."""
def __sigchld_handler(signum, frame):
if signum == signal.SIGCHLD:
os.waitpid(pid, os.WNOHANG)
exited[pid] = True
try:
signal.signal(signal.SIGCHLD, __sigchld_handler)
logging.warning('Sending SIGTERM to %d', pid)
os.kill(-1 * pid, signal.SIGTERM) # *-1 = entire process group
time.sleep(KILL_WAIT_SECS)
if pid in exited:
return
logging.warning('Sending SIGKILL to %d', pid)
os.kill(-1 * pid, signal.SIGKILL)
time.sleep(KILL_WAIT_SECS)
except OSError, e:
if e.args[0] == errno.ESRCH:
logging.warning('pid %d died on its own')
else:
logging.critical('killPid: %s', str(e))
if pid in exited:
return
logging.debug('pid %d will not die', pid)
def getExitStatus(self):
return self.exit_status
def cleanup(self):
"""Handle errors and call error-exec specified bin."""
if not self.options['error-exec']:
return
if self.exit_status in self.error_exec_codes:
did_timeout = int(bool(self.exit_status is EXIT_STATUS_TIMEOUT))
arg_str = self.options['error-exec']
arg_str = arg_str.replace('{EXIT}', str(self.exit_status))
arg_str = arg_str.replace('{TIMEOUT}', str(did_timeout))
arg_str = arg_str.replace('{STDOUT}', self.stdout.name)
arg_str = arg_str.replace('{STDERR}', self.stderr.name)
args = ('/bin/sh', '-c', arg_str)
error_supv = Supervisor()
error_supv.setOptions(timeout=5 * 3600)
error_supv.execute(args)
self.stdout.close()
self.stdout = None
self.stderr.close()
self.stderr = None
def parseOpts(argv):
@@ -124,6 +243,7 @@ def parseOpts(argv):
argv, '',
[
'timeout=', 'delayrandom=', 'debug', 'help',
'error-exec=', 'error-exec-exit-codes=',
])
except getopt.GetoptError, e:
raise OptionError(str(e))
@@ -146,8 +266,32 @@ def Usage():
after n seconds, terminate the executable
--delayrandom n
delay the execution of executable by random seconds up to n
--error-exec "path and options string"
exec path when executable returns non zero exit status.
in this mode the stdout and stderr from the supervised
executable are recorded to temp files.
the path and options string can include tokens which will be
replaced with values. note the braces {} should be included.
{EXIT} = exit status
{TIMEOUT} = 1 or 0, timeout did or did not occur
{STDOUT} = path to stdout file
{STDERR} = path to stderr file
the error-exec bin may use the stdin, stderr files while it is
executing, but it should assume they will disappear when
the error-exec bin returns with any exit status.
the bin should not run more than 5 minutes or it will be
terminated.
--error-exec-exit-codes "1,100,203"
comma-delimited list of integer exit status codes. If the
supervised script exits with one of these codes, the error-exec
executable will be run. Default: "1"
--debug
enable debugging output
enable debugging output, all logs to stderr and not syslog.
--help
this text
--
@@ -175,6 +319,17 @@ def processOpts(options, args):
return True
def setupSyslog():
"""Setup syslog as a logger."""
logger = logging.getLogger()
syslog = logging.handlers.SysLogHandler('/var/run/syslog')
formatter = logging.Formatter(
'%(filename)s[%(process)d]: %(levelname)s %(message)s')
syslog.setFormatter(formatter)
syslog.setLevel(logging.DEBUG)
logger.addHandler(syslog)
def main(argv):
try:
options, args = parseOpts(argv[1:])
@@ -184,15 +339,30 @@ def main(argv):
logging.error(str(e))
return 1
if options.get('debug', None) is None:
setupSyslog()
try:
sp = Supervisor()
sp = Supervisor(delayrandom_abort=True)
sp.setOptions(**options)
sp.execute(args)
return sp.GetReturnCode()
except Error, e:
logging.debug('%s %s', e.__class__.__name__, str(e))
logging.exception('%s %s', e.__class__.__name__, str(e))
return 1
ex = 0
try:
sp.execute(args)
ex = sp.getExitStatus()
except TimeoutError, e:
ex = 1
except Error, e:
logging.exception('%s %s', e.__class__.__name__, str(e))
ex = 1
sp.cleanup()
return ex
if __name__ == '__main__':
sys.exit(main(sys.argv))