mirror/munki
1
0
Fork 0
mirror of https://github.com/munki/munki.git synced 2026-02-25 10:29:07 -06:00
Code Issues Packages Projects Releases Wiki Activity

Update repoclean to work with repo plugins

Browse Source
This commit is contained in:
Greg Neagle
2017-03-15 16:36:27 -07:00
parent 7859066b57
commit 96d2bfcdf0
1 changed files with 113 additions and 195 deletions
Download Patch File Download Diff File Expand all files Collapse all files

308
code/client/repoclean
View File

@@ -23,74 +23,21 @@ A tool to remove older, unused software items from a Munki repo.
"""
import plistlib
import subprocess
import sys
import os
import optparse
from distutils.version import LooseVersion
from xml.parsers.expat import ExpatError
from munkilib.cliutils import get_version, pref, path2url
from munkilib.cliutils import print_utf8, print_err_utf8
from munkilib import munkirepo
try:
from munkilib import FoundationPlist as plistlib
LOCAL_PREFS_SUPPORT = True
except ImportError:
try:
import FoundationPlist as plistlib
LOCAL_PREFS_SUPPORT = True
except ImportError:
# maybe we're not on an OS X machine...
print >> sys.stderr, ("WARNING: FoundationPlist is not available, "
"using plistlib instead.")
import plistlib
LOCAL_PREFS_SUPPORT = False
try:
from munkilib.munkicommon import listdir, get_version
except ImportError:
# munkilib is not available
def listdir(path):
"""OS X HFS+ string encoding safe listdir().
Args:
path: path to list contents of
Returns:
list of contents, items as str or unicode types
"""
# if os.listdir() is supplied a unicode object for the path,
# it will return unicode filenames instead of their raw fs-dependent
# version, which is decomposed utf-8 on OS X.
#
# we use this to our advantage here and have Python do the decoding
# work for us, instead of decoding each item in the output list.
#
# references:
# https://docs.python.org/howto/unicode.html#unicode-filenames
# https://developer.apple.com/library/mac/#qa/qa2001/qa1235.html
# http://lists.zerezo.com/git/msg643117.html
# http://unicode.org/reports/tr15/ section 1.2
if type(path) is str:
path = unicode(path, 'utf-8')
elif type(path) is not unicode:
path = unicode(path)
return os.listdir(path)
def get_version():
'''Placeholder if munkilib is not available'''
return 'UNKNOWN'
def print_utf8(text):
'''Print Unicode text as UTF-8'''
print text.encode('UTF-8')
def print_err_utf8(text):
'''Print Unicode text to stderr as UTF-8'''
print >> sys.stderr, text.encode('UTF-8')
def nameAndVersion(aString):
def nameAndVersion(a_string):
"""Splits a string into the name and version number.
Name and version must be seperated with a hyphen ('-')
@@ -100,17 +47,17 @@ def nameAndVersion(aString):
'MicrosoftOffice2008-12.2.1' becomes ('MicrosoftOffice2008', '12.2.1')
"""
for delim in ('--', '-'):
if aString.count(delim) > 0:
chunks = aString.split(delim)
if a_string.count(delim) > 0:
chunks = a_string.split(delim)
vers = chunks.pop()
name = delim.join(chunks)
if vers[0] in '0123456789':
return (name, vers)
return (aString, '')
return (a_string, '')
def humanReadable(bytes):
def humanReadable(size_in_bytes):
"""Returns sizes in human-readable units."""
units = [(" bytes", 2**10),
(" KB", 2**20),
@@ -118,35 +65,43 @@ def humanReadable(bytes):
(" GB", 2**40),
(" TB", 2**50),]
for suffix, limit in units:
if bytes > limit:
if size_in_bytes > limit:
continue
else:
return str(round(bytes/float(limit/2**10), 1)) + suffix
return str(round(size_in_bytes/float(limit/2**10), 1)) + suffix
def make_catalogs(repopath):
def make_catalogs(repo, options):
"""Calls makecatalogs to rebuild our catalogs"""
# first look for a makecatalogs in the same dir as us
if hasattr(repo, 'authtoken'):
# Build an environment dict so we can put the authtoken
# into makecatalogs' environment
env = {'MUNKIREPO_AUTHTOKEN': repo.authtoken}
else:
env = None
mydir = os.path.dirname(os.path.abspath(__file__))
makecatalogs_path = os.path.join(mydir, 'makecatalogs')
if not os.path.exists(makecatalogs_path):
# didn't find it; assume the default install path
makecatalogs_path = '/usr/local/munki/makecatalogs'
print 'Rebuilding catalogs at %s...' % repopath
proc = subprocess.Popen([makecatalogs_path, repopath],
bufsize=-1, stdout=subprocess.PIPE,
print 'Rebuilding catalogs at %s...' % options.repo_url
cmd = [makecatalogs_path]
cmd.append('--repo-url')
cmd.append(options.repo_url)
cmd.append('--plugin')
cmd.append(options.plugin)
proc = subprocess.Popen(cmd, bufsize=-1, env=env, stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
while True:
output = proc.stdout.readline()
if not output and (proc.poll() != None):
break
if 0:
print output.rstrip('\n').decode('UTF-8')
print output.rstrip('\n').decode('UTF-8')
errors = proc.stderr.read()
if errors:
print '\nThe following errors occurred while building catalogs:\n'
print '\nThe following issues occurred while building catalogs:\n'
print errors
@@ -158,97 +113,44 @@ def count_pkgs_to_delete(items_to_delete, pkgs_to_keep):
return count
def get_file_sizes(repopath, items_to_delete, pkgs_to_keep):
def get_file_sizes(items_to_delete, pkgs_to_keep):
'''Returns human-readable sizes for the pkginfo items and
pkgs that are to be deleted'''
pkginfo_total_size = 0
pkg_total_size = 0
for item in items_to_delete:
if 'relative_path' in item:
try:
pkginfo_total_size += os.stat(
os.path.join(repopath, 'pkgsinfo',
item['relative_path'])).st_size
except (OSError, IOError):
pass
pkginfo_total_size += int(item.get('item_size', 0))
if 'pkg_path' in item and not item['pkg_path'] in pkgs_to_keep:
try:
pkg_total_size += os.stat(
os.path.join(repopath, 'pkgs',
item['pkg_path'])).st_size
except (OSError, IOError):
pass
pkg_total_size += int(item.get('pkg_size', 0))
return (humanReadable(pkginfo_total_size), humanReadable(pkg_total_size))
def delete_items(repopath, items_to_delete, pkgs_to_keep):
def delete_items(repo, items_to_delete, pkgs_to_keep):
'''Deletes items from the repo'''
for item in items_to_delete:
if 'relative_path' in item:
path_to_remove = os.path.join(
repopath, 'pkgsinfo', item['relative_path'])
print_utf8('Removing %s' % path_to_remove)
item_to_remove = os.path.join('pkgsinfo', item['relative_path'])
print_utf8('Removing %s' % item_to_remove)
try:
os.unlink(path_to_remove)
except (OSError, IOError), err:
repo.delete(item_to_remove)
except munkirepo.RepoError, err:
print_err_utf8(unicode(err))
if 'pkg_path' in item and not item['pkg_path'] in pkgs_to_keep:
path_to_remove = os.path.join(repopath, 'pkgs', item['pkg_path'])
print_utf8('Removing %s' % path_to_remove)
pkg_to_remove = os.path.join('pkgs', item['pkg_path'])
print_utf8('Removing %s' % pkg_to_remove)
try:
os.unlink(path_to_remove)
except (OSError, IOError), err:
repo.delete(pkg_to_remove)
except munkirepo.RepoError, err:
print_err_utf8(unicode(err))
def repo_plists(repo_subdir):
'''Generator function that returns plist filepaths'''
for dirpath, dirnames, filenames in os.walk(repo_subdir, followlinks=True):
for dirname in dirnames:
# don't recurse into directories that start
# with a period.
if dirname.startswith('.'):
dirnames.remove(dirname)
for filename in filenames:
if filename.startswith('.'):
# skip files that start with a period as well
continue
yield os.path.join(dirpath, filename)
def clean_repo(repopath, options):
def clean_repo(repo, options):
'''Clean it all up'''
def compare_versions(a, b):
"""sort highest version to top"""
return cmp(LooseVersion(b), LooseVersion(a))
# Make sure the manifests directory exists
manifestspath = os.path.join(repopath, 'manifests')
# make sure manifestpath is Unicode so that os.walk later gives us
# Unicode names back.
if type(manifestspath) is str:
manifestspath = unicode(manifestspath, 'utf-8')
elif type(manifestspath) is not unicode:
manifestspath = unicode(manifestspath)
if not os.path.exists(manifestspath):
print_err_utf8("manifests path %s doesn't exist!" % manifestspath)
exit(-1)
# Make sure the pkgsinfo directory exists
pkgsinfopath = os.path.join(repopath, 'pkgsinfo')
# make sure pkgsinfopath is Unicode so that os.walk later gives us
# Unicode names back.
if type(pkgsinfopath) is str:
pkgsinfopath = unicode(pkgsinfopath, 'utf-8')
elif type(pkgsinfopath) is not unicode:
pkgsinfopath = unicode(pkgsinfopath)
if not os.path.exists(pkgsinfopath):
print_err_utf8("pkgsinfo path %s doesn't exist!" % pkgsinfopath)
exit(-1)
errors = []
manifest_items = set()
manifest_items_with_versions = set()
@@ -256,14 +158,23 @@ def clean_repo(repopath, options):
print_utf8('Analyzing manifest files...')
# look through all manifests for "Foo-1.0" style items
# we need to note these so the specific referenced version is not deleted
for filepath in repo_plists(manifestspath):
try:
manifests_list = repo.itemlist('manifests')
except munkirepo.RepoError, err:
errors.append(
"Repo error getting list of manifests: %s" % unicode(err))
manifests_list = []
for manifest_name in manifests_list:
try:
manifest = plistlib.readPlist(filepath)
except IOError, inst:
errors.append("IO error for %s: %s" % (filepath, inst))
data = repo.get(os.path.join('manifests', manifest_name))
manifest = plistlib.readPlistFromString(data)
except munkirepo.RepoError, err:
errors.append(
"Repo error for %s: %s" % (manifest_name, unicode(err)))
continue
except BaseException, inst:
errors.append("Unexpected error for %s: %s" % (filepath, inst))
except (IOError, OSError, ExpatError), err:
errors.append(
"Unexpected error for %s: %s" % (manifest_name, unicode(err)))
continue
for key in ['managed_installs', 'managed_uninstalls',
'managed_updates', 'optional_installs']:
@@ -287,19 +198,30 @@ def clean_repo(repopath, options):
pkginfo_count = 0
print_utf8('Analyzing pkginfo files...')
for filepath in repo_plists(pkgsinfopath):
try:
pkgsinfo_list = repo.itemlist('pkgsinfo')
except munkirepo.RepoError, err:
errors.append(
"Repo error getting list of pkgsinfo: %s" % unicode(err))
pkgsinfo_list = []
for pkginfo_name in pkgsinfo_list:
pkginfo_identifier = os.path.join('pkgsinfo', pkginfo_name)
try:
pkginfo = plistlib.readPlist(filepath)
except IOError, inst:
errors.append("IO error for %s: %s" % (filepath, inst))
data = repo.get(pkginfo_identifier)
pkginfo = plistlib.readPlistFromString(data)
except munkirepo.RepoError, err:
errors.append(
"Repo error for %s: %s" % (pkginfo_name, unicode(err)))
continue
except BaseException, inst:
errors.append("Unexpected error for %s: %s" % (filepath, inst))
except (IOError, OSError, ExpatError), err:
errors.append(
"Unexpected error for %s: %s" % (pkginfo_name, unicode(err)))
continue
name = pkginfo['name']
version = pkginfo['version']
relpath = filepath[len(pkgsinfopath):].lstrip(os.path.sep)
pkgpath = pkginfo.get('installer_item_location', '')
pkgsize = pkginfo.get('installer_item_size', 0) * 1024
# track required items; if these are in "Foo-1.0" format, we need to
# note these so we don't delete the specific referenced version
@@ -356,8 +278,10 @@ def clean_repo(repopath, options):
pkginfodb[metakey][version].append({
'name': name,
'version': version,
'relative_path': relpath,
'resource_identifier': pkginfo_identifier,
'pkg_path': pkgpath,
'item_size': len(data),
'pkg_size': pkgsize
})
pkginfo_count += 1
@@ -399,22 +323,22 @@ def clean_repo(repopath, options):
line_info = (
"(multiple items share this version number) " + line_info)
else:
line_info = "(%s) %s" % (item['relative_path'], line_info)
line_info = "(%s) %s" % (item['resource_identifier'], line_info)
if print_this:
print " ", version, line_info
if len(item_list) > 1:
for item in item_list:
print " ", " " * len(version), "(%s)" % item['relative_path']
print " ", " " * len(version),
print "(%s)" % item['resource_identifier']
if print_this:
print
print_utf8("Total pkginfo items: %s" % pkginfo_count)
print_utf8("Item variants: %s" % len(pkginfodb.keys()))
print_utf8("pkginfo items to delete: %s" % len(items_to_delete))
print_utf8("pkgs to delete: %s" %
count_pkgs_to_delete(items_to_delete, pkgs_to_keep))
pkginfo_size, pkg_size = get_file_sizes(
repopath, items_to_delete, pkgs_to_keep)
print_utf8("pkgs to delete: %s"
% count_pkgs_to_delete(items_to_delete, pkgs_to_keep))
pkginfo_size, pkg_size = get_file_sizes(items_to_delete, pkgs_to_keep)
print_utf8("pkginfo space savings: %s" % pkginfo_size)
print_utf8("pkg space savings: %s" % pkg_size)
@@ -432,27 +356,10 @@ def clean_repo(repopath, options):
answer = raw_input(
'Are you sure? This action cannot be undone. [y/n] ')
if answer.lower().startswith('y'):
delete_items(repopath, items_to_delete, pkgs_to_keep)
make_catalogs(repopath)
delete_items(repo, items_to_delete, pkgs_to_keep)
make_catalogs(repo, options)
def pref(prefname):
"""Returns a preference for prefname"""
if not LOCAL_PREFS_SUPPORT:
return None
try:
_prefs = plistlib.readPlist(PREFSPATH)
except BaseException:
return None
if prefname in _prefs:
return _prefs[prefname]
else:
return None
PREFSNAME = 'com.googlecode.munki.munkiimport.plist'
PREFSPATH = os.path.expanduser(os.path.join('~/Library/Preferences',
PREFSNAME))
def main():
'''Main'''
usage = "usage: %prog [options] [/path/to/repo_root]"
@@ -467,13 +374,28 @@ def main():
parser.add_option('--delete-items-in-no-manifests', action='store_true',
help='Also delete items that are not referenced in any '
'manifests. Not yet implemented.')
parser.add_option('--repo_url', '--repo-url',
help='Optional repo URL. If specified, overrides any '
'repo_url specified via --configure.')
parser.add_option('--plugin', '--plugin', default=pref('plugin'),
help='Optional plugin to connect to repo. If specified, '
'overrides any plugin specified via --configure.')
options, arguments = parser.parse_args()
if options.version:
print get_version()
exit(0)
if not options.repo_url:
if arguments:
options.repo_url = path2url(arguments[0])
elif pref('repo_path'):
options.repo_url = path2url(pref('repo_path'))
if not options.plugin:
options.plugin = 'FileRepo'
try:
options.keep = int(options.keep)
except ValueError:
@@ -483,26 +405,22 @@ def main():
print_err_utf8('--keep value must be a positive integer!')
exit(-1)
# Make sure we have a path to work with
repopath = None
if len(arguments) == 0:
repopath = pref('repo_path')
if not repopath:
print_err_utf8("Need to specify a path to the repo root!")
exit(-1)
else:
print_utf8("Using repo path: %s" % repopath)
# Make sure we have a repo_url to work with
if not options.repo_url:
print_err_utf8("Need to specify a path to the repo root!")
exit(-1)
else:
repopath = arguments[0].rstrip("/")
print_utf8("Using repo url: %s" % options.repo_url)
# Make sure the repo path exists
if not os.path.exists(repopath):
print_err_utf8("Repo root path %s doesn't exist!" % repopath)
try:
repo = munkirepo.connect(options.repo_url, options.plugin)
except munkirepo.RepoError, err:
print >> sys.stderr, (u'Could not connect to munki repo: %s'
% unicode(err))
exit(-1)
# clean up the repo
clean_repo(repopath, options)
clean_repo(repo, options)
if __name__ == '__main__':
main()