mirror of
https://github.com/munki/munki.git
synced 2026-01-10 16:50:10 -06:00
427 lines
16 KiB
Python
Executable File
427 lines
16 KiB
Python
Executable File
#!/usr/bin/env python
|
|
# encoding: utf-8
|
|
#
|
|
# Copyright 2017 Greg Neagle.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# https://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
"""
|
|
repoclean
|
|
|
|
Created by Greg Neagle on 2016-06-22.
|
|
|
|
A tool to remove older, unused software items from a Munki repo.
|
|
|
|
"""
|
|
|
|
import plistlib
|
|
import subprocess
|
|
import sys
|
|
import os
|
|
import optparse
|
|
|
|
from distutils.version import LooseVersion
|
|
from xml.parsers.expat import ExpatError
|
|
|
|
from munkilib.cliutils import get_version, pref, path2url
|
|
from munkilib.cliutils import print_utf8, print_err_utf8
|
|
from munkilib import munkirepo
|
|
|
|
|
|
def nameAndVersion(a_string):
|
|
"""Splits a string into the name and version number.
|
|
|
|
Name and version must be seperated with a hyphen ('-')
|
|
or double hyphen ('--').
|
|
'TextWrangler-2.3b1' becomes ('TextWrangler', '2.3b1')
|
|
'AdobePhotoshopCS3--11.2.1' becomes ('AdobePhotoshopCS3', '11.2.1')
|
|
'MicrosoftOffice2008-12.2.1' becomes ('MicrosoftOffice2008', '12.2.1')
|
|
"""
|
|
for delim in ('--', '-'):
|
|
if a_string.count(delim) > 0:
|
|
chunks = a_string.split(delim)
|
|
vers = chunks.pop()
|
|
name = delim.join(chunks)
|
|
if vers[0] in '0123456789':
|
|
return (name, vers)
|
|
|
|
return (a_string, '')
|
|
|
|
|
|
def humanReadable(size_in_bytes):
|
|
"""Returns sizes in human-readable units."""
|
|
units = [(" bytes", 2**10),
|
|
(" KB", 2**20),
|
|
(" MB", 2**30),
|
|
(" GB", 2**40),
|
|
(" TB", 2**50),]
|
|
for suffix, limit in units:
|
|
if size_in_bytes > limit:
|
|
continue
|
|
else:
|
|
return str(round(size_in_bytes/float(limit/2**10), 1)) + suffix
|
|
|
|
|
|
def make_catalogs(repo, options):
|
|
"""Calls makecatalogs to rebuild our catalogs"""
|
|
# first look for a makecatalogs in the same dir as us
|
|
if hasattr(repo, 'authtoken'):
|
|
# Build an environment dict so we can put the authtoken
|
|
# into makecatalogs' environment
|
|
env = {'MUNKIREPO_AUTHTOKEN': repo.authtoken}
|
|
else:
|
|
env = None
|
|
mydir = os.path.dirname(os.path.abspath(__file__))
|
|
makecatalogs_path = os.path.join(mydir, 'makecatalogs')
|
|
if not os.path.exists(makecatalogs_path):
|
|
# didn't find it; assume the default install path
|
|
makecatalogs_path = '/usr/local/munki/makecatalogs'
|
|
print 'Rebuilding catalogs at %s...' % options.repo_url
|
|
cmd = [makecatalogs_path]
|
|
cmd.append('--repo-url')
|
|
cmd.append(options.repo_url)
|
|
cmd.append('--plugin')
|
|
cmd.append(options.plugin)
|
|
proc = subprocess.Popen(cmd, bufsize=-1, env=env, stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE)
|
|
while True:
|
|
output = proc.stdout.readline()
|
|
if not output and (proc.poll() != None):
|
|
break
|
|
print output.rstrip('\n').decode('UTF-8')
|
|
|
|
errors = proc.stderr.read()
|
|
if errors:
|
|
print '\nThe following issues occurred while building catalogs:\n'
|
|
print errors
|
|
|
|
|
|
def count_pkgs_to_delete(items_to_delete, pkgs_to_keep):
|
|
count = 0
|
|
for item in items_to_delete:
|
|
if 'pkg_path' in item and not item['pkg_path'] in pkgs_to_keep:
|
|
count += 1
|
|
return count
|
|
|
|
|
|
def get_file_sizes(items_to_delete, pkgs_to_keep):
|
|
'''Returns human-readable sizes for the pkginfo items and
|
|
pkgs that are to be deleted'''
|
|
pkginfo_total_size = 0
|
|
pkg_total_size = 0
|
|
for item in items_to_delete:
|
|
pkginfo_total_size += int(item.get('item_size', 0))
|
|
if 'pkg_path' in item and not item['pkg_path'] in pkgs_to_keep:
|
|
pkg_total_size += int(item.get('pkg_size', 0))
|
|
return (humanReadable(pkginfo_total_size), humanReadable(pkg_total_size))
|
|
|
|
|
|
def delete_items(repo, items_to_delete, pkgs_to_keep):
|
|
'''Deletes items from the repo'''
|
|
for item in items_to_delete:
|
|
if 'relative_path' in item:
|
|
item_to_remove = os.path.join('pkgsinfo', item['relative_path'])
|
|
print_utf8('Removing %s' % item_to_remove)
|
|
try:
|
|
repo.delete(item_to_remove)
|
|
except munkirepo.RepoError, err:
|
|
print_err_utf8(unicode(err))
|
|
if 'pkg_path' in item and not item['pkg_path'] in pkgs_to_keep:
|
|
pkg_to_remove = os.path.join('pkgs', item['pkg_path'])
|
|
print_utf8('Removing %s' % pkg_to_remove)
|
|
try:
|
|
repo.delete(pkg_to_remove)
|
|
except munkirepo.RepoError, err:
|
|
print_err_utf8(unicode(err))
|
|
|
|
|
|
def clean_repo(repo, options):
|
|
'''Clean it all up'''
|
|
|
|
def compare_versions(a, b):
|
|
"""sort highest version to top"""
|
|
return cmp(LooseVersion(b), LooseVersion(a))
|
|
|
|
errors = []
|
|
manifest_items = set()
|
|
manifest_items_with_versions = set()
|
|
|
|
print_utf8('Analyzing manifest files...')
|
|
# look through all manifests for "Foo-1.0" style items
|
|
# we need to note these so the specific referenced version is not deleted
|
|
try:
|
|
manifests_list = repo.itemlist('manifests')
|
|
except munkirepo.RepoError, err:
|
|
errors.append(
|
|
"Repo error getting list of manifests: %s" % unicode(err))
|
|
manifests_list = []
|
|
for manifest_name in manifests_list:
|
|
try:
|
|
data = repo.get(os.path.join('manifests', manifest_name))
|
|
manifest = plistlib.readPlistFromString(data)
|
|
except munkirepo.RepoError, err:
|
|
errors.append(
|
|
"Repo error for %s: %s" % (manifest_name, unicode(err)))
|
|
continue
|
|
except (IOError, OSError, ExpatError), err:
|
|
errors.append(
|
|
"Unexpected error for %s: %s" % (manifest_name, unicode(err)))
|
|
continue
|
|
for key in ['managed_installs', 'managed_uninstalls',
|
|
'managed_updates', 'optional_installs']:
|
|
for item in manifest.get(key, []):
|
|
itemname, itemvers = nameAndVersion(item)
|
|
manifest_items.add(itemname)
|
|
if itemvers:
|
|
manifest_items_with_versions.add((itemname, itemvers))
|
|
# next check conditional_items within the manifest
|
|
for conditional_item in manifest.get('conditional_items', []):
|
|
for key in ['managed_installs', 'managed_uninstalls',
|
|
'managed_updates', 'optional_installs']:
|
|
for item in conditional_item.get(key, []):
|
|
itemname, itemvers = nameAndVersion(item)
|
|
manifest_items.add(itemname)
|
|
if itemvers:
|
|
manifest_items_with_versions.add((itemname, itemvers))
|
|
|
|
pkginfodb = {}
|
|
required_items = set()
|
|
pkginfo_count = 0
|
|
|
|
print_utf8('Analyzing pkginfo files...')
|
|
try:
|
|
pkgsinfo_list = repo.itemlist('pkgsinfo')
|
|
except munkirepo.RepoError, err:
|
|
errors.append(
|
|
"Repo error getting list of pkgsinfo: %s" % unicode(err))
|
|
pkgsinfo_list = []
|
|
|
|
for pkginfo_name in pkgsinfo_list:
|
|
pkginfo_identifier = os.path.join('pkgsinfo', pkginfo_name)
|
|
try:
|
|
data = repo.get(pkginfo_identifier)
|
|
pkginfo = plistlib.readPlistFromString(data)
|
|
except munkirepo.RepoError, err:
|
|
errors.append(
|
|
"Repo error for %s: %s" % (pkginfo_name, unicode(err)))
|
|
continue
|
|
except (IOError, OSError, ExpatError), err:
|
|
errors.append(
|
|
"Unexpected error for %s: %s" % (pkginfo_name, unicode(err)))
|
|
continue
|
|
name = pkginfo['name']
|
|
version = pkginfo['version']
|
|
pkgpath = pkginfo.get('installer_item_location', '')
|
|
pkgsize = pkginfo.get('installer_item_size', 0) * 1024
|
|
|
|
# track required items; if these are in "Foo-1.0" format, we need to
|
|
# note these so we don't delete the specific referenced version
|
|
if 'requires' in pkginfo:
|
|
dependencies = pkginfo['requires']
|
|
# fix things if 'requires' was specified as a string
|
|
# instead of an array of strings
|
|
if isinstance(dependencies, basestring):
|
|
dependencies = [dependencies]
|
|
for dependency in dependencies:
|
|
required_name, required_vers = nameAndVersion(dependency)
|
|
if required_vers:
|
|
required_items.add((required_name, required_vers))
|
|
# if this item is in a manifest, then anything it requires
|
|
# should be treated as if it, too, is in a manifest.
|
|
if name in manifest_items:
|
|
manifest_items.add(required_name)
|
|
|
|
# now process update_for: if this is an update_for an item that is
|
|
# in manifest_items, it should be treated as if it, too is in a
|
|
# manifest
|
|
if 'update_for' in pkginfo:
|
|
update_items = pkginfo['update_for']
|
|
# fix things if 'update_for' was specified as a string
|
|
# instead of an array of strings
|
|
if isinstance(update_items, basestring):
|
|
update_items = [update_items]
|
|
for update_item in update_items:
|
|
update_item_name, update_item_vers = nameAndVersion(update_item)
|
|
if update_item_name in manifest_items:
|
|
# add our name
|
|
manifest_items.add(name)
|
|
|
|
metakey = ''
|
|
keys_to_hash = ['name', 'catalogs', 'minimum_munki_version',
|
|
'minimum_os_version', 'maximum_os_version',
|
|
'supported_architectures', 'installable_condition']
|
|
if pkginfo.get('uninstall_method') == 'removepackages':
|
|
keys_to_hash.append('receipts')
|
|
for key in keys_to_hash:
|
|
if pkginfo.get(key):
|
|
value = pkginfo[key]
|
|
if key == 'catalogs':
|
|
value = ', '.join(value)
|
|
if key == 'receipts':
|
|
value = ', '.join(
|
|
[item.get('packageid', '') for item in value])
|
|
metakey += u"%s: %s\n" % (key, value)
|
|
metakey = metakey.rstrip('\n')
|
|
if metakey not in pkginfodb:
|
|
pkginfodb[metakey] = {}
|
|
if version not in pkginfodb[metakey]:
|
|
pkginfodb[metakey][version] = []
|
|
pkginfodb[metakey][version].append({
|
|
'name': name,
|
|
'version': version,
|
|
'resource_identifier': pkginfo_identifier,
|
|
'pkg_path': pkgpath,
|
|
'item_size': len(data),
|
|
'pkg_size': pkgsize
|
|
})
|
|
pkginfo_count += 1
|
|
|
|
items_to_delete = []
|
|
pkgs_to_keep = set()
|
|
for key in sorted(pkginfodb.keys()):
|
|
print_this = (options.show_all or
|
|
len(pkginfodb[key].keys()) > options.keep)
|
|
item_name = pkginfodb[key][pkginfodb[key].keys()[0]][0]['name']
|
|
if print_this:
|
|
print key
|
|
if item_name not in manifest_items:
|
|
print "[not in any manifests]"
|
|
print "versions:"
|
|
index = 0
|
|
for version in sorted(pkginfodb[key].keys(), compare_versions):
|
|
line_info = ''
|
|
index += 1
|
|
item_list = pkginfodb[key][version]
|
|
if (item_list[0]['name'], version) in manifest_items_with_versions:
|
|
for item in item_list:
|
|
if item['pkg_path']:
|
|
pkgs_to_keep.add(item['pkg_path'])
|
|
line_info = "(REQUIRED by a manifest)"
|
|
elif (item_list[0]['name'], version) in required_items:
|
|
for item in item_list:
|
|
if item['pkg_path']:
|
|
pkgs_to_keep.add(item['pkg_path'])
|
|
line_info = "(REQUIRED by another pkginfo item)"
|
|
elif index <= options.keep:
|
|
for item in item_list:
|
|
if item['pkg_path']:
|
|
pkgs_to_keep.add(item['pkg_path'])
|
|
else:
|
|
for item in item_list:
|
|
items_to_delete.append(item)
|
|
line_info = "[to be DELETED]"
|
|
if len(item_list) > 1:
|
|
line_info = (
|
|
"(multiple items share this version number) " + line_info)
|
|
else:
|
|
line_info = "(%s) %s" % (item['resource_identifier'], line_info)
|
|
if print_this:
|
|
print " ", version, line_info
|
|
if len(item_list) > 1:
|
|
for item in item_list:
|
|
print " ", " " * len(version),
|
|
print "(%s)" % item['resource_identifier']
|
|
if print_this:
|
|
print
|
|
|
|
print_utf8("Total pkginfo items: %s" % pkginfo_count)
|
|
print_utf8("Item variants: %s" % len(pkginfodb.keys()))
|
|
print_utf8("pkginfo items to delete: %s" % len(items_to_delete))
|
|
print_utf8("pkgs to delete: %s"
|
|
% count_pkgs_to_delete(items_to_delete, pkgs_to_keep))
|
|
pkginfo_size, pkg_size = get_file_sizes(items_to_delete, pkgs_to_keep)
|
|
print_utf8("pkginfo space savings: %s" % pkginfo_size)
|
|
print_utf8("pkg space savings: %s" % pkg_size)
|
|
|
|
if errors:
|
|
print_err_utf8("\nErrors encountered when processing repo:\n")
|
|
for error in errors:
|
|
print_err_utf8(error)
|
|
|
|
if len(items_to_delete):
|
|
print
|
|
answer = raw_input(
|
|
'Delete pkginfo and pkg items marked as [to be DELETED]? '
|
|
'WARNING: This action cannot be undone. [y/n] ')
|
|
if answer.lower().startswith('y'):
|
|
answer = raw_input(
|
|
'Are you sure? This action cannot be undone. [y/n] ')
|
|
if answer.lower().startswith('y'):
|
|
delete_items(repo, items_to_delete, pkgs_to_keep)
|
|
make_catalogs(repo, options)
|
|
|
|
|
|
def main():
|
|
'''Main'''
|
|
usage = "usage: %prog [options] [/path/to/repo_root]"
|
|
parser = optparse.OptionParser(usage=usage)
|
|
parser.add_option('--version', '-V', action='store_true',
|
|
help='Print the version of the munki tools and exit.')
|
|
parser.add_option('--keep', '-k', default=2,
|
|
help='Keep this many versions of a specific variation. '
|
|
'Defaults to 2.')
|
|
parser.add_option('--show-all', action='store_true',
|
|
help='Show all items even if none will be deleted.')
|
|
parser.add_option('--delete-items-in-no-manifests', action='store_true',
|
|
help='Also delete items that are not referenced in any '
|
|
'manifests. Not yet implemented.')
|
|
parser.add_option('--repo_url', '--repo-url',
|
|
help='Optional repo URL. If specified, overrides any '
|
|
'repo_url specified via --configure.')
|
|
parser.add_option('--plugin', '--plugin', default=pref('plugin'),
|
|
help='Optional plugin to connect to repo. If specified, '
|
|
'overrides any plugin specified via --configure.')
|
|
|
|
options, arguments = parser.parse_args()
|
|
|
|
if options.version:
|
|
print get_version()
|
|
exit(0)
|
|
|
|
if not options.repo_url:
|
|
if arguments:
|
|
options.repo_url = path2url(arguments[0])
|
|
elif pref('repo_path'):
|
|
options.repo_url = path2url(pref('repo_path'))
|
|
|
|
if not options.plugin:
|
|
options.plugin = 'FileRepo'
|
|
|
|
try:
|
|
options.keep = int(options.keep)
|
|
except ValueError:
|
|
print_err_utf8('--keep value must be a positive integer!')
|
|
exit(-1)
|
|
if options.keep < 1:
|
|
print_err_utf8('--keep value must be a positive integer!')
|
|
exit(-1)
|
|
|
|
# Make sure we have a repo_url to work with
|
|
if not options.repo_url:
|
|
print_err_utf8("Need to specify a path to the repo root!")
|
|
exit(-1)
|
|
else:
|
|
print_utf8("Using repo url: %s" % options.repo_url)
|
|
|
|
try:
|
|
repo = munkirepo.connect(options.repo_url, options.plugin)
|
|
except munkirepo.RepoError, err:
|
|
print >> sys.stderr, (u'Could not connect to munki repo: %s'
|
|
% unicode(err))
|
|
exit(-1)
|
|
|
|
# clean up the repo
|
|
clean_repo(repo, options)
|
|
|
|
if __name__ == '__main__':
|
|
main()
|