#!/usr/bin/python # encoding: utf-8 # # Copyright 2009-2011 Greg Neagle. # # Licensed under the Apache License, Version 2.0 (the 'License'); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an 'AS IS' BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ munkiimport Created by Greg Neagle on 2010-09-29. Assists with importing installer items into the munki repo """ import sys import os import optparse import subprocess import time #from distutils import version from munkilib import munkicommon from munkilib import FoundationPlist def makeDMG(pkgpath): """Wraps a non-flat package into a disk image. Returns path to newly-created disk image.""" pkgname = os.path.basename(pkgpath) print 'Making disk image containing %s...' % pkgname diskimagename = os.path.splitext(pkgname)[0] + '.dmg' diskimagepath = os.path.join(munkicommon.tmpdir, diskimagename) cmd = ['/usr/bin/hdiutil', 'create', '-srcfolder', pkgpath, diskimagepath] proc = subprocess.Popen(cmd, shell=False, bufsize=-1, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) while True: output = proc.stdout.readline() if not output and (proc.poll() != None): break print output.rstrip('\n') sys.stdout.flush() retcode = proc.poll() if retcode: print >> sys.stderr, 'Disk image creation failed.' return '' else: print 'Disk image created at: %s' % diskimagepath return diskimagepath def repoAvailable(): """Checks the repo path for proper directory structure. If the directories look wrong we probably don't have a valid repo path. Returns True if things look OK.""" repo_path = pref('repo_path') if not repo_path: print >> sys.stderr, 'No repo path specified.' return False if not os.path.exists(repo_path): mountRepoCLI() if not os.path.exists(repo_path): return False for subdir in ['catalogs', 'manifests', 'pkgs', 'pkgsinfo']: if not os.path.exists(os.path.join(repo_path, subdir)): print >> sys.stderr, "%s is missing %s" % (repo_path, subdir) return False # if we get this far, the repo path looks OK return True def mountRepoGUI(): """Attempts to connect to the repo fileshare Returns nothing whether we succeed or fail""" repo_path = pref('repo_path') repo_url = pref('repo_url') if not repo_path or not repo_url: return print 'Attempting to connect to munki repo...' cmd = ['/usr/bin/open', repo_url] unused_retcode = subprocess.call(cmd) for unused_i in range(60): # wait up to 60 seconds to connect to repo if os.path.exists(repo_path): break time.sleep(1) def mountRepoCLI(): """Attempts to connect to the repo fileshare""" global WE_MOUNTED_THE_REPO repo_path = pref('repo_path') repo_url = pref('repo_url') if os.path.exists(repo_path): return os.mkdir(repo_path) print 'Attempting to mount fileshare %s:' % repo_url if repo_url.startswith('afp:'): cmd = ['/sbin/mount_afp', '-i', repo_url, repo_path] elif repo_url.startswith('smb:'): cmd = ['/sbin/mount_smbfs', repo_url[4:], repo_path] elif repo_url.startswith('nfs://'): cmd = ['/sbin/mount_nfs', repo_url[6:], repo_path] else: print >> sys.stderr, 'Unsupported filesystem URL!' return retcode = subprocess.call(cmd) if retcode: os.rmdir(repo_path) else: WE_MOUNTED_THE_REPO = True def unmountRepoCLI(): """Attempts to unmount the repo fileshare""" repo_path = pref('repo_path') if not os.path.exists(repo_path): return cmd = ['/sbin/umount', repo_path] return subprocess.call(cmd) class RepoCopyError(Exception): """Error copying installer item to repo""" pass def copyItemToRepo(itempath, vers, subdirectory=''): """Copies an item to the appropriate place in the repo. If itempath is a path within the repo/pkgs directory, copies nothing. Renames the item if an item already exists with that name. Returns the relative path to the item.""" repo_path = pref('repo_path') if not os.path.exists(repo_path): raise RepoCopyError('Could not connect to munki repo.') destination_path = os.path.join(repo_path, 'pkgs', subdirectory) if not os.path.exists(destination_path): try: os.makedirs(destination_path) except OSError, errmsg: raise RepoCopyError('Could not create %s: %s' % (destination_path, errmsg)) item_name = os.path.basename(itempath) destination_path_name = os.path.join(destination_path, item_name) if itempath == destination_path_name: # we've been asked to 'import' a repo item. # just return the relative path return os.path.join(subdirectory, item_name) if os.path.exists(destination_path_name) and vers: if not vers in item_name: # try adding the version item_name = '%s-%s%s' % (os.path.splitext(item_name)[0], vers, os.path.splitext(item_name)[1]) destination_path_name = os.path.join(destination_path, item_name) index = 0 while os.path.exists(destination_path_name): print 'File %s already exists...' % destination_path_name index += 1 original_name = os.path.basename(itempath) item_name = '%s__%s%s' % (os.path.splitext(original_name)[0], index, os.path.splitext(original_name)[1]) destination_path_name = os.path.join(destination_path, item_name) print 'Copying %s to %s...' % (os.path.basename(itempath), destination_path_name) cmd = ['/bin/cp', itempath, destination_path_name] retcode = subprocess.call(cmd) if retcode: raise RepoCopyError('Unable to copy %s to %s' % (itempath, destination_path_name)) else: return os.path.join(subdirectory, item_name) def copyPkginfoToRepo(pkginfo, subdirectory=''): """Saves pkginfo to munki_repo_path/pkgsinfo/subdirectory""" # less error checking because we copy the installer_item # first and bail if it fails... repo_path = pref('repo_path') destination_path = os.path.join(repo_path, 'pkgsinfo', subdirectory) if not os.path.exists(destination_path): try: os.makedirs(destination_path) except OSError, errmsg: raise RepoCopyError('Could not create %s: %s' % (destination_path, errmsg)) pkginfo_ext = pref('pkginfo_extension') or '' if pkginfo_ext and not pkginfo_ext.startswith('.'): pkginfo_ext = '.' + pkginfo_ext pkginfo_name = '%s-%s%s' % (pkginfo['name'], pkginfo['version'], pkginfo_ext) pkginfo_path = os.path.join(destination_path, pkginfo_name) index = 0 while os.path.exists(pkginfo_path): index += 1 pkginfo_name = '%s-%s__%s%s' % (pkginfo['name'], pkginfo['version'], index, pkginfo_ext) pkginfo_path = os.path.join(destination_path, pkginfo_name) print 'Saving pkginfo to %s...' % pkginfo_path try: FoundationPlist.writePlist(pkginfo, pkginfo_path) except FoundationPlist.NSPropertyListWriteException, errmsg: raise RepoCopyError(errmsg) return pkginfo_path def openPkginfoInEditor(pkginfo_path): """Opens pkginfo list in the user's chosen editor.""" editor = pref('editor') if editor: if editor.endswith('.app'): cmd = ['/usr/bin/open', '-a', editor, pkginfo_path] else: cmd = [editor, pkginfo_path] unused_returncode = subprocess.call(cmd) def promptForSubdirectory(subdirectory): """Prompts the user for a subdirectory for the pkg and pkginfo""" while True: newdir = raw_input( 'Upload item to subdirectory path [%s]: ' % subdirectory) if newdir: repo_path = pref('repo_path') if not repoAvailable(): raise RepoCopyError('Could not connect to munki repo.') destination_path = os.path.join(repo_path, 'pkgs', newdir) if not os.path.exists(destination_path): answer = raw_input('Path %s doesn\'t exist. Create it? [y/n] ' % destination_path) if answer.lower().startswith('y'): break else: break else: return subdirectory return newdir class CatalogDBException(Exception): '''Exception to throw if we can't make a pkginfo DB''' pass def makeCatalogDB(): """Returns a dict we can use like a database""" all_items_path = os.path.join(pref('repo_path'), 'catalogs', 'all') if not os.path.exists(all_items_path): raise CatalogDBException try: catalogitems = FoundationPlist.readPlist(all_items_path) except FoundationPlist.NSPropertyListSerializationException: raise CatalogDBException pkgid_table = {} app_table = {} installer_item_table = {} hash_table = {} itemindex = -1 for item in catalogitems: itemindex = itemindex + 1 name = item.get('name', 'NO NAME') vers = item.get('version', 'NO VERSION') if name == 'NO NAME' or vers == 'NO VERSION': munkicommon.display_warning('Bad pkginfo: %s' % item) # add to hash table if 'installer_item_hash' in item: if not item['installer_item_hash'] in hash_table: hash_table[item['installer_item_hash']] = [] hash_table[item['installer_item_hash']].append(itemindex) # add to installer item table if 'installer_item_location' in item: installer_item_name = os.path.basename( item['installer_item_location']) if not installer_item_name in installer_item_table: installer_item_table[installer_item_name] = {} if not vers in installer_item_table[installer_item_name]: installer_item_table[installer_item_name][vers] = [] installer_item_table[installer_item_name][vers].append(itemindex) # add to table of receipts for receipt in item.get('receipts', []): if 'packageid' in receipt and 'version' in receipt: if not receipt['packageid'] in pkgid_table: pkgid_table[receipt['packageid']] = {} if not vers in pkgid_table[receipt['packageid']]: pkgid_table[receipt['packageid']][vers] = [] pkgid_table[receipt['packageid']][vers].append(itemindex) # add to table of installed applications for install in item.get('installs', []): if install.get('type') == 'application': if 'path' in install: if not install['path'] in app_table: app_table[install['path']] = {} if not vers in app_table[install['path']]: app_table[install['path']][vers] = [] app_table[install['path']][vers].append(itemindex) pkgdb = {} pkgdb['hashes'] = hash_table pkgdb['receipts'] = pkgid_table pkgdb['applications'] = app_table pkgdb['installer_items'] = installer_item_table pkgdb['items'] = catalogitems return pkgdb def findMatchingPkginfo(pkginfo): """Looks through repo catalogs looking for matching pkginfo Returns a pkginfo dictionary, or an empty dict""" def compare_version_keys(a, b): """Internal comparison function for use in sorting""" return cmp(munkicommon.MunkiLooseVersion(b), munkicommon.MunkiLooseVersion(a)) try: db = makeCatalogDB() except CatalogDBException: return {} if 'installer_item_hash' in pkginfo: matchingindexes = db['hashes'].get( pkginfo['installer_item_hash']) if matchingindexes: return db['items'][matchingindexes[0]] if 'receipts' in pkginfo: pkgids = [item['packageid'] for item in pkginfo['receipts'] if 'packageid' in item] if pkgids: possiblematches = db['receipts'].get(pkgids[0]) if possiblematches: versionlist = possiblematches.keys() versionlist.sort(compare_version_keys) # go through possible matches, newest version first for versionkey in versionlist: testpkgindexes = possiblematches[versionkey] for pkgindex in testpkgindexes: testpkginfo = db['items'][pkgindex] testpkgids = [item['packageid'] for item in testpkginfo.get('receipts',[]) if 'packageid' in item] if set(testpkgids) == set(pkgids): return testpkginfo if 'installs' in pkginfo: applist = [item for item in pkginfo['installs'] if item['type'] == 'application' and 'path' in item] if applist: app = applist[0]['path'] possiblematches = db['applications'].get(app) if possiblematches: versionlist = possiblematches.keys() versionlist.sort(compare_version_keys) indexes = db['applications'][app][versionlist[0]] return db['items'][indexes[0]] # no matches by receipts or installed applications, # let's try to match based on installer_item_name installer_item_name = os.path.basename(pkginfo['installer_item_location']) possiblematches = db['installer_items'].get(installer_item_name) if possiblematches: versionlist = possiblematches.keys() versionlist.sort(compare_version_keys) indexes = db['installer_items'][installer_item_name][versionlist[0]] return db['items'][indexes[0]] # if we get here, we found no matches return {} def makePkgInfo(item_path): """Calls makepkginfo to generate the pkginfo for item_path.""" # first look for a makepkginfo in the same dir as us mydir = os.path.dirname(os.path.abspath(__file__)) makepkginfo_path = os.path.join(mydir, 'makepkginfo') if not os.path.exists(makepkginfo_path): # didn't find it; assume the default install path makepkginfo_path = '/usr/local/munki/makepkginfo' proc = subprocess.Popen([makepkginfo_path, item_path], bufsize=-1, stdout=subprocess.PIPE, stderr=subprocess.PIPE) (pliststr, err) = proc.communicate() if proc.returncode: print >> sys.stderr, err return {} return FoundationPlist.readPlistFromString(pliststr) def makeCatalogs(): """Calls makecatalogs to rebuild our catalogs""" # first look for a makepkginfo in the same dir as us mydir = os.path.dirname(os.path.abspath(__file__)) makecatalogs_path = os.path.join(mydir, 'makecatalogs') if not os.path.exists(makecatalogs_path): # didn't find it; assume the default install path makecatalogs_path = '/usr/local/munki/makepkginfo' repo_path = pref('repo_path') if not repoAvailable(): raise RepoCopyError('Could not connect to munki repo.') proc = subprocess.Popen([makecatalogs_path, repo_path], bufsize=-1, stdout=subprocess.PIPE, stderr=subprocess.PIPE) while True: output = proc.stdout.readline() if not output and (proc.poll() != None): break if VERBOSE: print output.rstrip('\n') errors = proc.stderr.read() if errors: print '\nThe following errors occurred while building catalogs:\n' print errors def cleanupAndExit(exitcode): result = 0 if WE_MOUNTED_THE_REPO: if not NOINTERACTIVE: answer = raw_input('Unmount the repo fileshare? [y/n] ') if answer.lower().startswith('y'): result = unmountRepoCLI() else: result = unmountRepoCLI() exit(exitcode or result) def pref(prefname): """Returns a preference for prefname""" try: _prefs = FoundationPlist.readPlist(PREFSPATH) except FoundationPlist.NSPropertyListSerializationException: return None if prefname in _prefs: return _prefs[prefname] else: return None def configure(): """Configures munkiimport for use""" _prefs = {} for (key, prompt) in [ ('repo_path', 'Path to munki repo (example: /Volumes/repo)'), ('repo_url', 'Repo fileshare URL (example: afp://munki.example.com/repo)'), ('pkginfo_extension', 'pkginfo extension (Example: .plist)'), ('editor', 'pkginfo editor (examples: /usr/bin/vi or TextMate.app)')]: newvalue = raw_input('%15s [%s]: ' % (prompt, pref(key))) _prefs[key] = newvalue or pref(key) or '' try: FoundationPlist.writePlist(_prefs, PREFSPATH) except FoundationPlist.NSPropertyListWriteException: print >> sys.stderr, 'Could not save configuration to %s' % PREFSPATH PREFSNAME = 'com.googlecode.munki.munkiimport.plist' PREFSPATH = os.path.expanduser(os.path.join('~/Library/Preferences', PREFSNAME)) NOINTERACTIVE = False WE_MOUNTED_THE_REPO = False VERBOSE = False def main(): """Main routine""" global NOINTERACTIVE usage = """usage: %prog [options] [/path/to/installer_item] Imports an installer item into a munki repo. Installer item can be a pkg, mpkg, dmg, or app. Bundle-style pkgs and apps are wrapped in a dmg file before upload.""" p = optparse.OptionParser(usage=usage) p.add_option('--configure', action='store_true', help="""Configure munkiimport with details about your munki repo, preferred editor, and the like. Any other options and arguments are ignored.""") p.add_option('--subdirectory', '-d', default='', help="""When importing an installer item, item will be uploaded to this subdirectory path in the repo pkgs directory, and the pkginfo file will be stored under this subdirectory under the pkgsinfo directory.""") p.add_option('--nointeractive', '-n', action='store_true', help="""No interactive prompts. May cause a failure if repo path is unavailable.""") p.add_option('--catalog', default=[], action='append', help="""Specify target catalogs. (default: prompt)""") p.add_option('--version', '-V', action='store_true', help='Print the version of the munki tools and exit.') p.add_option('--verbose', '-v', action='store_true', help='Print more output.') options, arguments = p.parse_args() if options.version: print munkicommon.get_version() exit(0) if options.configure: configure() exit(0) NOINTERACTIVE = options.nointeractive VERBOSE = options.verbose if len(arguments) == 0: p.print_usage() exit(0) if len(arguments) > 1: print >> sys.stderr, \ 'This tool supports importing only one item at a time.' print >> sys.stderr, 'Too many items given:' print >> sys.stderr, '\t', '\n\t'.join(arguments) exit(-1) installer_item = arguments[0] if not os.path.exists(installer_item): print >> sys.stderr, '%s does not exist!' % installer_item exit(-1) item_ext = os.path.splitext(installer_item)[1] if item_ext not in ['.pkg', '.mpkg', '.dmg', '.app']: print >> sys.stderr, '%s is an unknown type.' % installer_item exit(-1) if not pref('repo_path'): print >> sys.stderr, ('Path to munki repo has not been defined. ' 'Run with --configure option to configure this ' 'tool.') exit(-1) if not repoAvailable(): print >> sys.stderr, ('Could not connect to munki repo. Check the ' 'configuration and try again.') exit(-1) if os.path.isdir(installer_item): if item_ext == '.dmg': # a directory named foo.dmg! print >> sys.stderr, '%s is an unknown type.' % installer_item cleanupAndExit(-1) else: # we need to convert to dmg dmg_path = makeDMG(installer_item) if dmg_path: installer_item = dmg_path else: print >> sys.stderr, ('Could not convert %s to a disk image.' % installer_item) cleanupAndExit(-1) # generate pkginfo for the item pkginfo = makePkgInfo(installer_item) if not pkginfo: # makepkginfo returned an error cleanupAndExit(-1) if not options.nointeractive: # try to find existing pkginfo items that match this one matchingpkginfo = findMatchingPkginfo(pkginfo) exactmatch = False if matchingpkginfo: if ('installer_item_hash' in matchingpkginfo and matchingpkginfo['installer_item_hash'] == pkginfo.get('installer_item_hash')): exactmatch = True print ('***This item is identical to an existing item in ' 'the repo***:') else: print 'This item is similar to an existing item in the repo:' fields = (('Item name', 'name'), ('Display name', 'display_name'), ('Description', 'description'), ('Version', 'version'), ('Installer item path', 'installer_item_location')) for (name, key) in fields: print '%21s: %s' % (name, matchingpkginfo.get( key,'').encode('UTF-8')) print if exactmatch: answer = raw_input('Import this item anyway? [y/n] ') if not answer.lower().startswith('y'): cleanupAndExit(0) answer = raw_input('Use existing item as a template? [y/n] ') if answer.lower().startswith('y'): pkginfo['name'] = matchingpkginfo['name'] pkginfo['display_name'] = pkginfo.get('display_name') or \ matchingpkginfo.get('display_name', matchingpkginfo['name']) pkginfo['description'] = pkginfo.get('description') or \ matchingpkginfo.get('description', '') if (options.subdirectory == '' and matchingpkginfo.get('installer_item_location')): options.subdirectory = os.path.dirname( matchingpkginfo['installer_item_location']) for key in ['blocking_applications', 'forced_install', 'forced_uninstall', 'unattended_install', 'unattended_uninstall', 'requires', 'update_for']: if key in matchingpkginfo: print 'Copying %s: %s' % (key, matchingpkginfo[key]) pkginfo[key] = matchingpkginfo[key] # now let user do some basic editing editfields = (('Item name', 'name'), ('Display name', 'display_name'), ('Description', 'description'), ('Version', 'version')) for (name, key) in editfields: newvalue = raw_input('%15s [%s]: ' % (name, pkginfo.get(key,'').encode('UTF-8'))) if newvalue: pkginfo[key] = newvalue if options.catalog: pkginfo['catalogs'] = options.catalog newvalue = raw_input('%15s [%s]: ' % ('Catalogs', ', '.join(pkginfo['catalogs']))) if newvalue: pkginfo['catalogs'] = [item.strip() for item in newvalue.split(',')] if 'receipts' not in pkginfo and 'installs' not in pkginfo: print >> sys.stderr, ('WARNING: There are no receipts and no ' '\'installs\' items for this installer ' 'item. You will need to add at least one ' 'item to the \'installs\' list.') #TO-DO: provide a way to add 'installs' items right here print for (name, key) in editfields: print '%15s: %s' % (name, pkginfo.get(key,'').encode('UTF-8')) print '%15s: %s' % ( 'Catalogs', ', '.join(pkginfo['catalogs']).encode('UTF-8')) print answer = raw_input('Import this item? [y/n] ') if not answer.lower().startswith('y'): cleanupAndExit(0) if options.subdirectory == '': pkgs_path = os.path.join(pref('repo_path'), 'pkgs') if installer_item.startswith(pkgs_path): # the installer item is already in the repo. # use its relative path as the subdirectory installer_item_dirpath = os.path.dirname(installer_item) options.subdirectory = \ installer_item_dirpath[len(pkgs_path)+1:] options.subdirectory = promptForSubdirectory( options.subdirectory) # fix in case user accidentally starts subdirectory with a slash if options.subdirectory.startswith('/'): options.subdirectory = options.subdirectory[1:] try: uploaded_pkgpath = copyItemToRepo(installer_item, pkginfo.get('version'), options.subdirectory) except RepoCopyError, errmsg: print >> sys.stderr, errmsg cleanupAndExit(-1) # adjust the installer_item_location to match the actual location and name pkginfo['installer_item_location'] = uploaded_pkgpath # installer_item upload was successful, so upload pkginfo to repo try: pkginfo_path = copyPkginfoToRepo(pkginfo, options.subdirectory) except RepoCopyError, errmsg: print >> sys.stderr, errmsg cleanupAndExit(-1) if not options.nointeractive: # open the pkginfo file in the user's editor openPkginfoInEditor(pkginfo_path) answer = raw_input('Rebuild catalogs? [y/n] ') if answer.lower().startswith('y'): try: makeCatalogs() except RepoCopyError, errmsg: print >> sys.stderr, errmsg cleanupAndExit(-1) cleanupAndExit(0) if __name__ == '__main__': main()