#!/usr/bin/python # encoding: utf-8 # # Copyright 2009-2011 Greg Neagle. # # Licensed under the Apache License, Version 2.0 (the 'License'); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an 'AS IS' BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ munkiimport Created by Greg Neagle on 2010-09-29. Assists with importing installer items into the munki repo """ import sys import os import subprocess import time from optparse import OptionParser, BadOptionError, AmbiguousOptionError from munkilib import munkicommon from munkilib import FoundationPlist class PassThroughOptionParser(OptionParser): """ An unknown option pass-through implementation of OptionParser. When unknown arguments are encountered, bundle with largs and try again, until rargs is depleted. sys.exit(status) will still be called if a known argument is passed incorrectly (e.g. missing arguments or bad argument types, etc.) """ def _process_args(self, largs, rargs, values): while rargs: try: OptionParser._process_args(self, largs, rargs, values) except (BadOptionError, AmbiguousOptionError), e: largs.append(e.opt_str) def format_epilog(self, formatter): if not self.epilog: self.epilog = "" return self.epilog def makeDMG(pkgpath): """Wraps a non-flat package into a disk image. Returns path to newly-created disk image.""" pkgname = os.path.basename(pkgpath) print 'Making disk image containing %s...' % pkgname diskimagename = os.path.splitext(pkgname)[0] + '.dmg' diskimagepath = os.path.join(munkicommon.tmpdir, diskimagename) cmd = ['/usr/bin/hdiutil', 'create', '-srcfolder', pkgpath, diskimagepath] proc = subprocess.Popen(cmd, shell=False, bufsize=-1, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) while True: output = proc.stdout.readline() if not output and (proc.poll() != None): break print output.rstrip('\n') sys.stdout.flush() retcode = proc.poll() if retcode: print >> sys.stderr, 'Disk image creation failed.' return '' else: print 'Disk image created at: %s' % diskimagepath return diskimagepath def repoAvailable(): """Checks the repo path for proper directory structure. If the directories look wrong we probably don't have a valid repo path. Returns True if things look OK.""" repo_path = pref('repo_path') if not repo_path: print >> sys.stderr, 'No repo path specified.' return False if not os.path.exists(repo_path): mountRepoCLI() if not os.path.exists(repo_path): return False for subdir in ['catalogs', 'manifests', 'pkgs', 'pkgsinfo']: if not os.path.exists(os.path.join(repo_path, subdir)): print >> sys.stderr, "%s is missing %s" % (repo_path, subdir) return False # if we get this far, the repo path looks OK return True def mountRepoGUI(): """Attempts to connect to the repo fileshare Returns nothing whether we succeed or fail""" repo_path = pref('repo_path') repo_url = pref('repo_url') if not repo_path or not repo_url: return print 'Attempting to connect to munki repo...' cmd = ['/usr/bin/open', repo_url] unused_retcode = subprocess.call(cmd) for unused_i in range(60): # wait up to 60 seconds to connect to repo if os.path.exists(repo_path): break time.sleep(1) def mountRepoCLI(): """Attempts to connect to the repo fileshare""" global WE_MOUNTED_THE_REPO repo_path = pref('repo_path') repo_url = pref('repo_url') if os.path.exists(repo_path): return os.mkdir(repo_path) print 'Attempting to mount fileshare %s:' % repo_url if repo_url.startswith('afp:'): cmd = ['/sbin/mount_afp', '-i', repo_url, repo_path] elif repo_url.startswith('smb:'): cmd = ['/sbin/mount_smbfs', repo_url[4:], repo_path] elif repo_url.startswith('nfs://'): cmd = ['/sbin/mount_nfs', repo_url[6:], repo_path] else: print >> sys.stderr, 'Unsupported filesystem URL!' return retcode = subprocess.call(cmd) if retcode: os.rmdir(repo_path) else: WE_MOUNTED_THE_REPO = True def unmountRepoCLI(): """Attempts to unmount the repo fileshare""" repo_path = pref('repo_path') if not os.path.exists(repo_path): return cmd = ['/sbin/umount', repo_path] return subprocess.call(cmd) class RepoCopyError(Exception): """Error copying installer item to repo""" pass def copyItemToRepo(itempath, vers, subdirectory=''): """Copies an item to the appropriate place in the repo. If itempath is a path within the repo/pkgs directory, copies nothing. Renames the item if an item already exists with that name. Returns the relative path to the item.""" repo_path = pref('repo_path') if not os.path.exists(repo_path): raise RepoCopyError('Could not connect to munki repo.') destination_path = os.path.join(repo_path, 'pkgs', subdirectory) if not os.path.exists(destination_path): try: os.makedirs(destination_path) except OSError, errmsg: raise RepoCopyError('Could not create %s: %s' % (destination_path, errmsg)) item_name = os.path.basename(itempath) destination_path_name = os.path.join(destination_path, item_name) if itempath == destination_path_name: # we've been asked to 'import' a repo item. # just return the relative path return os.path.join(subdirectory, item_name) if os.path.exists(destination_path_name) and vers: if not vers in item_name: # try adding the version item_name = '%s-%s%s' % (os.path.splitext(item_name)[0], vers, os.path.splitext(item_name)[1]) destination_path_name = os.path.join(destination_path, item_name) index = 0 while os.path.exists(destination_path_name): print 'File %s already exists...' % destination_path_name index += 1 original_name = os.path.basename(itempath) item_name = '%s__%s%s' % (os.path.splitext(original_name)[0], index, os.path.splitext(original_name)[1]) destination_path_name = os.path.join(destination_path, item_name) print 'Copying %s to %s...' % (os.path.basename(itempath), destination_path_name) cmd = ['/bin/cp', itempath, destination_path_name] retcode = subprocess.call(cmd) if retcode: raise RepoCopyError('Unable to copy %s to %s' % (itempath, destination_path_name)) else: return os.path.join(subdirectory, item_name) def copyPkginfoToRepo(pkginfo, subdirectory=''): """Saves pkginfo to munki_repo_path/pkgsinfo/subdirectory""" # less error checking because we copy the installer_item # first and bail if it fails... repo_path = pref('repo_path') destination_path = os.path.join(repo_path, 'pkgsinfo', subdirectory) if not os.path.exists(destination_path): try: os.makedirs(destination_path) except OSError, errmsg: raise RepoCopyError('Could not create %s: %s' % (destination_path, errmsg)) pkginfo_ext = pref('pkginfo_extension') or '' if pkginfo_ext and not pkginfo_ext.startswith('.'): pkginfo_ext = '.' + pkginfo_ext pkginfo_name = '%s-%s%s' % (pkginfo['name'], pkginfo['version'], pkginfo_ext) pkginfo_path = os.path.join(destination_path, pkginfo_name) index = 0 while os.path.exists(pkginfo_path): index += 1 pkginfo_name = '%s-%s__%s%s' % (pkginfo['name'], pkginfo['version'], index, pkginfo_ext) pkginfo_path = os.path.join(destination_path, pkginfo_name) print 'Saving pkginfo to %s...' % pkginfo_path try: FoundationPlist.writePlist(pkginfo, pkginfo_path) except FoundationPlist.NSPropertyListWriteException, errmsg: raise RepoCopyError(errmsg) return pkginfo_path def openPkginfoInEditor(pkginfo_path): """Opens pkginfo list in the user's chosen editor.""" editor = pref('editor') if editor: if editor.endswith('.app'): cmd = ['/usr/bin/open', '-a', editor, pkginfo_path] else: cmd = [editor, pkginfo_path] unused_returncode = subprocess.call(cmd) def promptForSubdirectory(subdirectory): """Prompts the user for a subdirectory for the pkg and pkginfo""" while True: newdir = raw_input( 'Upload item to subdirectory path [%s]: ' % subdirectory) if newdir: repo_path = pref('repo_path') if not repoAvailable(): raise RepoCopyError('Could not connect to munki repo.') destination_path = os.path.join(repo_path, 'pkgs', newdir) if not os.path.exists(destination_path): answer = raw_input('Path %s doesn\'t exist. Create it? [y/n] ' % destination_path) if answer.lower().startswith('y'): break else: break else: return subdirectory return newdir class CatalogDBException(Exception): '''Exception to throw if we can't make a pkginfo DB''' pass def makeCatalogDB(): """Returns a dict we can use like a database""" all_items_path = os.path.join(pref('repo_path'), 'catalogs', 'all') if not os.path.exists(all_items_path): raise CatalogDBException try: catalogitems = FoundationPlist.readPlist(all_items_path) except FoundationPlist.NSPropertyListSerializationException: raise CatalogDBException pkgid_table = {} app_table = {} installer_item_table = {} hash_table = {} itemindex = -1 for item in catalogitems: itemindex = itemindex + 1 name = item.get('name', 'NO NAME') vers = item.get('version', 'NO VERSION') if name == 'NO NAME' or vers == 'NO VERSION': munkicommon.display_warning('Bad pkginfo: %s' % item) # add to hash table if 'installer_item_hash' in item: if not item['installer_item_hash'] in hash_table: hash_table[item['installer_item_hash']] = [] hash_table[item['installer_item_hash']].append(itemindex) # add to installer item table if 'installer_item_location' in item: installer_item_name = os.path.basename( item['installer_item_location']) if not installer_item_name in installer_item_table: installer_item_table[installer_item_name] = {} if not vers in installer_item_table[installer_item_name]: installer_item_table[installer_item_name][vers] = [] installer_item_table[installer_item_name][vers].append(itemindex) # add to table of receipts for receipt in item.get('receipts', []): if 'packageid' in receipt and 'version' in receipt: if not receipt['packageid'] in pkgid_table: pkgid_table[receipt['packageid']] = {} if not vers in pkgid_table[receipt['packageid']]: pkgid_table[receipt['packageid']][vers] = [] pkgid_table[receipt['packageid']][vers].append(itemindex) # add to table of installed applications for install in item.get('installs', []): if install.get('type') == 'application': if 'path' in install: if not install['path'] in app_table: app_table[install['path']] = {} if not vers in app_table[install['path']]: app_table[install['path']][vers] = [] app_table[install['path']][vers].append(itemindex) pkgdb = {} pkgdb['hashes'] = hash_table pkgdb['receipts'] = pkgid_table pkgdb['applications'] = app_table pkgdb['installer_items'] = installer_item_table pkgdb['items'] = catalogitems return pkgdb def findMatchingPkginfo(pkginfo): """Looks through repo catalogs looking for matching pkginfo Returns a pkginfo dictionary, or an empty dict""" def compare_version_keys(a, b): """Internal comparison function for use in sorting""" return cmp(munkicommon.MunkiLooseVersion(b), munkicommon.MunkiLooseVersion(a)) try: db = makeCatalogDB() except CatalogDBException: return {} if 'installer_item_hash' in pkginfo: matchingindexes = db['hashes'].get( pkginfo['installer_item_hash']) if matchingindexes: return db['items'][matchingindexes[0]] if 'receipts' in pkginfo: pkgids = [item['packageid'] for item in pkginfo['receipts'] if 'packageid' in item] if pkgids: possiblematches = db['receipts'].get(pkgids[0]) if possiblematches: versionlist = possiblematches.keys() versionlist.sort(compare_version_keys) # go through possible matches, newest version first for versionkey in versionlist: testpkgindexes = possiblematches[versionkey] for pkgindex in testpkgindexes: testpkginfo = db['items'][pkgindex] testpkgids = [item['packageid'] for item in testpkginfo.get('receipts',[]) if 'packageid' in item] if set(testpkgids) == set(pkgids): return testpkginfo if 'installs' in pkginfo: applist = [item for item in pkginfo['installs'] if item['type'] == 'application' and 'path' in item] if applist: app = applist[0]['path'] possiblematches = db['applications'].get(app) if possiblematches: versionlist = possiblematches.keys() versionlist.sort(compare_version_keys) indexes = db['applications'][app][versionlist[0]] return db['items'][indexes[0]] # no matches by receipts or installed applications, # let's try to match based on installer_item_name installer_item_name = os.path.basename(pkginfo['installer_item_location']) possiblematches = db['installer_items'].get(installer_item_name) if possiblematches: versionlist = possiblematches.keys() versionlist.sort(compare_version_keys) indexes = db['installer_items'][installer_item_name][versionlist[0]] return db['items'][indexes[0]] # if we get here, we found no matches return {} def makePkgInfo(options=None, test_mode=False): """Calls makepkginfo to generate the pkginfo for item_path.""" # first look for a makepkginfo in the same dir as us mydir = os.path.dirname(os.path.abspath(__file__)) makepkginfo_path = os.path.join(mydir, 'makepkginfo') if not os.path.exists(makepkginfo_path): # didn't find it; assume the default install path makepkginfo_path = '/usr/local/munki/makepkginfo' if test_mode: # prepend verification option if in test mode options = ['--verify-options-only'] + options # build makepkginfo command from discovered path and options cmd = [makepkginfo_path] + options proc = subprocess.Popen(cmd, bufsize=-1, stdout=subprocess.PIPE, stderr=subprocess.PIPE) (stdout, stderr) = proc.communicate() if test_mode: if proc.returncode == 2: # option syntax error or unknown option syntax_error = [error for error in stderr.splitlines() if 'error' in error] print >> sys.stderr, ('Option syntax error: %s' % syntax_error[-1].split(': ',2)[-1]) print >> sys.stderr, ('See \'%s --help\' for valid options that ' 'can be used with munkiimport.') % makepkginfo_path exit(-1) elif proc.returncode: # catch-all for any other error if stderr: print >> sys.stderr, stderr.rstrip('\n') return {} else: return stdout.rstrip('\n') if proc.returncode: print >> sys.stderr, stderr.rstrip('\n') return {} return FoundationPlist.readPlistFromString(stdout) def makeCatalogs(): """Calls makecatalogs to rebuild our catalogs""" # first look for a makecatalogs in the same dir as us mydir = os.path.dirname(os.path.abspath(__file__)) makecatalogs_path = os.path.join(mydir, 'makecatalogs') if not os.path.exists(makecatalogs_path): # didn't find it; assume the default install path makecatalogs_path = '/usr/local/munki/makecatalogs' repo_path = pref('repo_path') if not repoAvailable(): raise RepoCopyError('Could not connect to munki repo.') if not VERBOSE: print 'Rebuilding catalogs at %s...' % repo_path proc = subprocess.Popen([makecatalogs_path, repo_path], bufsize=-1, stdout=subprocess.PIPE, stderr=subprocess.PIPE) while True: output = proc.stdout.readline() if not output and (proc.poll() != None): break if VERBOSE: print output.rstrip('\n') errors = proc.stderr.read() if errors: print '\nThe following errors occurred while building catalogs:\n' print errors def cleanupAndExit(exitcode): """Unmounts the repo if we mounted it, then exits""" result = 0 if WE_MOUNTED_THE_REPO: if not NOINTERACTIVE: answer = raw_input('Unmount the repo fileshare? [y/n] ') if answer.lower().startswith('y'): result = unmountRepoCLI() else: result = unmountRepoCLI() exit(exitcode or result) def pref(prefname): """Returns a preference for prefname""" try: _prefs = FoundationPlist.readPlist(PREFSPATH) except FoundationPlist.NSPropertyListSerializationException: return None if prefname in _prefs: return _prefs[prefname] else: return None def configure(): """Configures munkiimport for use""" _prefs = {} for (key, prompt) in [ ('repo_path', 'Path to munki repo (example: /Volumes/repo)'), ('repo_url', 'Repo fileshare URL (example: afp://munki.example.com/repo)'), ('pkginfo_extension', 'pkginfo extension (Example: .plist)'), ('editor', 'pkginfo editor (examples: /usr/bin/vi or TextMate.app)')]: newvalue = raw_input('%15s [%s]: ' % (prompt, pref(key))) _prefs[key] = newvalue or pref(key) or '' try: FoundationPlist.writePlist(_prefs, PREFSPATH) except FoundationPlist.NSPropertyListWriteException: print >> sys.stderr, 'Could not save configuration to %s' % PREFSPATH PREFSNAME = 'com.googlecode.munki.munkiimport.plist' PREFSPATH = os.path.expanduser(os.path.join('~/Library/Preferences', PREFSNAME)) NOINTERACTIVE = False WE_MOUNTED_THE_REPO = False VERBOSE = False def main(): """Main routine""" global NOINTERACTIVE global VERBOSE usage = """usage: %prog [options] /path/to/installer_item Imports an installer item into a munki repo. Installer item can be a pkg, mpkg, dmg, or app. Bundle-style pkgs and apps are wrapped in a dmg file before upload. Example: munkiimport --subdirectory apps /path/to/installer_item """ epilog = """\nExtended Options: (makepkginfo options) In addition to the options described above, options used with 'makepkginfo' may also be specified to customize the resulting pkginfo file. Example: munkiimport --subdirectory apps -c production --minimum_os_vers 10.6.8 /path/to/installer_item\n""" p = PassThroughOptionParser(usage=usage, epilog=epilog) p.add_option('--configure', action='store_true', help="""Configure munkiimport with details about your munki repo, preferred editor, and the like. Any other options and arguments are ignored.""") p.add_option('--subdirectory', '-d', default='', help="""When importing an installer item, item will be uploaded to this subdirectory path in the repo pkgs directory, and the pkginfo file will be stored under this subdirectory under the pkgsinfo directory.""") p.add_option('--nointeractive', '-n', action='store_true', help="""No interactive prompts. May cause a failure if repo path is unavailable.""") p.add_option('--version', '-V', action='store_true', help='Print the version of the munki tools and exit.') p.add_option('--verbose', '-v', action='store_true', help='Print more output.') options, arguments = p.parse_args() if options.version: print munkicommon.get_version() exit(0) if options.configure: configure() exit(0) NOINTERACTIVE = options.nointeractive VERBOSE = options.verbose if len(arguments) == 0: p.print_usage() exit(0) # Verify that arguments, presumed to be for # 'makepkginfo' are valid and return installer_item installer_item = makePkgInfo(arguments, True) if not installer_item: cleanupAndExit(-1) # Remove the installer_item from arguments arguments.remove(installer_item) # Strip trailing '/' from installer_item installer_item = installer_item.rstrip('/') if not munkicommon.hasValidInstallerItemExt(installer_item) and \ not munkicommon.isApplication(installer_item): print >> sys.stderr, ( 'Unknown installer item type: "%s"' % installer_item) exit(-1) if not os.path.exists(installer_item): print >> sys.stderr, '%s does not exist!' % installer_item exit(-1) if not pref('repo_path'): print >> sys.stderr, ('Path to munki repo has not been defined. ' 'Run with --configure option to configure this ' 'tool.') exit(-1) if not repoAvailable(): print >> sys.stderr, ('Could not connect to munki repo. Check the ' 'configuration and try again.') exit(-1) if os.path.isdir(installer_item): if munkicommon.hasValidDiskImageExt(installer_item): # a directory named foo.dmg or foo.iso! print >> sys.stderr, '%s is an unknown type.' % installer_item cleanupAndExit(-1) else: # we need to convert to dmg dmg_path = makeDMG(installer_item) if dmg_path: installer_item = dmg_path else: print >> sys.stderr, ('Could not convert %s to a disk image.' % installer_item) cleanupAndExit(-1) # append the installer_item to arguments which # may have changed if bundle was wrapped into dmg arguments.append(installer_item) pkginfo = makePkgInfo(arguments, False) if not pkginfo: # makepkginfo returned an error print >> sys.stderr, 'Getting package info failed.' cleanupAndExit(-1) if not options.nointeractive: # try to find existing pkginfo items that match this one matchingpkginfo = findMatchingPkginfo(pkginfo) exactmatch = False if matchingpkginfo: if ('installer_item_hash' in matchingpkginfo and matchingpkginfo['installer_item_hash'] == pkginfo.get('installer_item_hash')): exactmatch = True print ('***This item is identical to an existing item in ' 'the repo***:') else: print 'This item is similar to an existing item in the repo:' fields = (('Item name', 'name'), ('Display name', 'display_name'), ('Description', 'description'), ('Version', 'version'), ('Installer item path', 'installer_item_location')) for (name, key) in fields: print '%21s: %s' % (name, matchingpkginfo.get( key,'').encode('UTF-8')) print if exactmatch: answer = raw_input('Import this item anyway? [y/n] ') if not answer.lower().startswith('y'): cleanupAndExit(0) answer = raw_input('Use existing item as a template? [y/n] ') if answer.lower().startswith('y'): pkginfo['name'] = matchingpkginfo['name'] pkginfo['display_name'] = pkginfo.get('display_name') or \ matchingpkginfo.get('display_name', matchingpkginfo['name']) pkginfo['description'] = pkginfo.get('description') or \ matchingpkginfo.get('description', '') if (options.subdirectory == '' and matchingpkginfo.get('installer_item_location')): options.subdirectory = os.path.dirname( matchingpkginfo['installer_item_location']) for key in ['blocking_applications', 'forced_install', 'forced_uninstall', 'unattended_install', 'unattended_uninstall', 'requires', 'update_for']: if key in matchingpkginfo: print 'Copying %s: %s' % (key, matchingpkginfo[key]) pkginfo[key] = matchingpkginfo[key] # now let user do some basic editing editfields = (('Item name', 'name'), ('Display name', 'display_name'), ('Description', 'description'), ('Version', 'version')) for (name, key) in editfields: newvalue = raw_input('%15s [%s]: ' % (name, pkginfo.get(key,'').encode('UTF-8'))) if newvalue: pkginfo[key] = newvalue newvalue = raw_input('%15s [%s]: ' % ('Catalogs', ', '.join(pkginfo['catalogs']))) if newvalue: pkginfo['catalogs'] = [item.strip() for item in newvalue.split(',')] if 'receipts' not in pkginfo and 'installs' not in pkginfo: print >> sys.stderr, ('WARNING: There are no receipts and no ' '\'installs\' items for this installer ' 'item. You will need to add at least one ' 'item to the \'installs\' list.') #TO-DO: provide a way to add 'installs' items right here print for (name, key) in editfields: print '%15s: %s' % (name, pkginfo.get(key,'').encode('UTF-8')) print '%15s: %s' % ( 'Catalogs', ', '.join(pkginfo['catalogs']).encode('UTF-8')) print answer = raw_input('Import this item? [y/n] ') if not answer.lower().startswith('y'): cleanupAndExit(0) if options.subdirectory == '': pkgs_path = os.path.join(pref('repo_path'), 'pkgs') if installer_item.startswith(pkgs_path): # the installer item is already in the repo. # use its relative path as the subdirectory installer_item_dirpath = os.path.dirname(installer_item) options.subdirectory = \ installer_item_dirpath[len(pkgs_path)+1:] options.subdirectory = promptForSubdirectory( options.subdirectory) # fix in case user accidentally starts subdirectory with a slash if options.subdirectory.startswith('/'): options.subdirectory = options.subdirectory[1:] try: uploaded_pkgpath = copyItemToRepo(installer_item, pkginfo.get('version'), options.subdirectory) except RepoCopyError, errmsg: print >> sys.stderr, errmsg cleanupAndExit(-1) # adjust the installer_item_location to match the actual location and name pkginfo['installer_item_location'] = uploaded_pkgpath # installer_item upload was successful, so upload pkginfo to repo try: pkginfo_path = copyPkginfoToRepo(pkginfo, options.subdirectory) except RepoCopyError, errmsg: print >> sys.stderr, errmsg cleanupAndExit(-1) if not options.nointeractive: # open the pkginfo file in the user's editor openPkginfoInEditor(pkginfo_path) answer = raw_input('Rebuild catalogs? [y/n] ') if answer.lower().startswith('y'): try: makeCatalogs() except RepoCopyError, errmsg: print >> sys.stderr, errmsg cleanupAndExit(-1) cleanupAndExit(0) if __name__ == '__main__': main()