Files
munki/code/client/munkilib/updatecheck/catalogs.py

647 lines
26 KiB
Python

# encoding: utf-8
#
# Copyright 2009-2020 Greg Neagle.
#
# Licensed under the Apache License, Version 2.0 (the 'License');
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an 'AS IS' BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
updatecheck.catalogs
Created by Greg Neagle on 2017-01-01.
Functions for working with Munki catalogs
"""
from __future__ import absolute_import, print_function
import os
from . import download
from .. import display
from .. import info
from .. import pkgutils
from .. import prefs
from .. import utils
from .. import FoundationPlist
from ..wrappers import is_a_string
def make_catalog_db(catalogitems):
"""Takes an array of catalog items and builds some indexes so we can
get our common data faster. Returns a dict we can use like a database"""
name_table = {}
pkgid_table = {}
itemindex = -1
for item in catalogitems:
itemindex = itemindex + 1
name = item.get('name', 'NO NAME')
vers = item.get('version', 'NO VERSION')
if name == 'NO NAME' or vers == 'NO VERSION':
display.display_warning('Bad pkginfo: %s', item)
# normalize the version number
vers = pkgutils.trim_version_string(vers)
# build indexes for items by name and version
if not name in name_table:
name_table[name] = {}
if not vers in name_table[name]:
name_table[name][vers] = []
name_table[name][vers].append(itemindex)
# build table of receipts
for receipt in item.get('receipts', []):
if 'packageid' in receipt and 'version' in receipt:
pkg_id = receipt['packageid']
version = receipt['version']
if not pkg_id in pkgid_table:
pkgid_table[pkg_id] = {}
if not version in pkgid_table[pkg_id]:
pkgid_table[pkg_id][version] = []
pkgid_table[pkg_id][version].append(itemindex)
# build table of update items with a list comprehension --
# filter all items from the catalogitems that have a non-empty
# 'update_for' list
updaters = [item for item in catalogitems if item.get('update_for')]
# now fix possible admin errors where 'update_for' is a string instead
# of a list of strings
for update in updaters:
if is_a_string(update['update_for']):
# convert to list of strings
update['update_for'] = [update['update_for']]
# build table of autoremove items with a list comprehension --
# filter all items from the catalogitems that have a non-empty
# 'autoremove' list
# autoremove items are automatically removed if they are not in the
# managed_install list (either directly or indirectly via included
# manifests)
autoremoveitems = [item.get('name') for item in catalogitems
if item.get('autoremove')]
# convert to set and back to list to get list of unique names
autoremoveitems = list(set(autoremoveitems))
pkgdb = {}
pkgdb['named'] = name_table
pkgdb['receipts'] = pkgid_table
pkgdb['updaters'] = updaters
pkgdb['autoremoveitems'] = autoremoveitems
pkgdb['items'] = catalogitems
return pkgdb
def add_package_ids(catalogitems, itemname_to_pkgid, pkgid_to_itemname):
"""Adds packageids from each catalogitem to two dictionaries.
One maps itemnames to receipt pkgids, the other maps receipt pkgids
to itemnames"""
for item in catalogitems:
name = item.get('name')
if not name:
continue
if item.get('receipts'):
if not name in itemname_to_pkgid:
itemname_to_pkgid[name] = {}
for receipt in item['receipts']:
if 'packageid' in receipt and 'version' in receipt:
pkgid = receipt['packageid']
vers = receipt['version']
if not pkgid in itemname_to_pkgid[name]:
itemname_to_pkgid[name][pkgid] = []
if not vers in itemname_to_pkgid[name][pkgid]:
itemname_to_pkgid[name][pkgid].append(vers)
if not pkgid in pkgid_to_itemname:
pkgid_to_itemname[pkgid] = {}
if not name in pkgid_to_itemname[pkgid]:
pkgid_to_itemname[pkgid][name] = []
if not vers in pkgid_to_itemname[pkgid][name]:
pkgid_to_itemname[pkgid][name].append(vers)
def split_name_and_version(some_string):
"""Splits a string into the name and version number.
Name and version must be separated with a hyphen ('-')
or double hyphen ('--').
'TextWrangler-2.3b1' becomes ('TextWrangler', '2.3b1')
'AdobePhotoshopCS3--11.2.1' becomes ('AdobePhotoshopCS3', '11.2.1')
'MicrosoftOffice2008-12.2.1' becomes ('MicrosoftOffice2008', '12.2.1')
"""
for delim in ('--', '-'):
if some_string.count(delim) > 0:
chunks = some_string.split(delim)
vers = chunks.pop()
name = delim.join(chunks)
if vers and vers[0] in '0123456789':
return (name, vers)
return (some_string, '')
def get_all_items_with_name(name, cataloglist):
"""Searches the catalogs in a list for all items matching a given name.
Returns:
list of pkginfo items; sorted with newest version first. No precedence
is given to catalog order.
"""
def item_version(item):
"""Returns a MunkiLooseVersion for pkginfo item"""
return pkgutils.MunkiLooseVersion(item['version'])
itemlist = []
# we'll throw away any included version info
name = split_name_and_version(name)[0]
display.display_debug1('Looking for all items matching: %s...', name)
for catalogname in cataloglist:
if not catalogname in list(_CATALOG.keys()):
# in case catalogname refers to a non-existent catalog...
continue
# is name in the catalog name table?
if name in _CATALOG[catalogname]['named']:
versionsmatchingname = _CATALOG[catalogname]['named'][name]
for vers in versionsmatchingname:
if vers == 'latest':
continue
indexlist = _CATALOG[catalogname]['named'][name][vers]
for index in indexlist:
thisitem = _CATALOG[catalogname]['items'][index]
if not thisitem in itemlist:
display.display_debug1(
'Adding item %s, version %s from catalog %s...',
name, thisitem['version'], catalogname)
itemlist.append(thisitem)
if itemlist:
# sort so latest version is first
itemlist.sort(key=item_version, reverse=True)
return itemlist
def get_auto_removal_items(installinfo, cataloglist):
"""Gets a list of items marked for automatic removal from the catalogs
in cataloglist. Filters those against items in the processed_installs
list, which should contain everything that is supposed to be installed.
Then filters against the removals list, which contains all the removals
that have already been processed.
"""
autoremovalnames = []
for catalogname in cataloglist or []:
if catalogname in list(_CATALOG.keys()):
autoremovalnames += _CATALOG[catalogname]['autoremoveitems']
processed_installs_names = [split_name_and_version(item)[0]
for item in installinfo['processed_installs']]
autoremovalnames = [item for item in autoremovalnames
if item not in processed_installs_names
and item not in installinfo['processed_uninstalls']]
return autoremovalnames
def look_for_updates(itemname, cataloglist):
"""Looks for updates for a given manifest item that is either
installed or scheduled to be installed or removed. This handles not only
specific application updates, but also updates that aren't simply
later versions of the manifest item.
For example, AdobeCameraRaw is an update for Adobe Photoshop, but
doesn't update the version of Adobe Photoshop.
Returns a list of manifestitem names that are updates for
manifestitem.
"""
display.display_debug1('Looking for updates for: %s', itemname)
# get a list of catalog items that are updates for other items
update_list = []
for catalogname in cataloglist:
if catalogname not in _CATALOG:
# in case the list refers to a non-existent catalog
continue
updaters = _CATALOG[catalogname]['updaters']
# list comprehension coming up...
update_items = [catalogitem['name']
for catalogitem in updaters
if itemname in catalogitem.get('update_for', [])]
if update_items:
update_list.extend(update_items)
# make sure the list has only unique items:
update_list = list(set(update_list))
if update_list:
# updates were found, so let's display them
num_updates = len(update_list)
# format the update list for better on-screen viewing
update_list_display = ", ".join(str(x) for x in update_list)
display.display_debug1(
'Found %s update(s): %s', num_updates, update_list_display)
return update_list
def look_for_updates_for_version(itemname, itemversion, cataloglist):
"""Looks for updates for a specific version of an item. Since these
can appear in manifests and pkginfo as item-version or item--version
we have to search twice."""
name_and_version = '%s-%s' % (itemname, itemversion)
alt_name_and_version = '%s--%s' % (itemname, itemversion)
update_list = look_for_updates(name_and_version, cataloglist)
update_list.extend(look_for_updates(alt_name_and_version, cataloglist))
# make sure the list has only unique items:
update_list = list(set(update_list))
return update_list
def best_version_match(vers_num, item_dict):
'''Attempts to find the best match in item_dict for vers_num'''
vers_tuple = vers_num.split('.')
precision = 1
while precision <= len(vers_tuple):
test_vers = '.'.join(vers_tuple[0:precision])
match_names = []
for item in item_dict.keys():
for item_version in item_dict[item]:
if (item_version.startswith(test_vers) and
item not in match_names):
match_names.append(item)
if len(match_names) == 1:
return match_names[0]
precision = precision + 1
return None
@utils.Memoize
def analyze_installed_pkgs():
"""Analyze catalog data and installed packages in an attempt to determine
what is installed."""
pkgdata = {}
itemname_to_pkgid = {}
pkgid_to_itemname = {}
for catalogname in _CATALOG:
catalogitems = _CATALOG[catalogname]['items']
add_package_ids(catalogitems, itemname_to_pkgid, pkgid_to_itemname)
# itemname_to_pkgid now contains all receipts (pkgids) we know about
# from items in all available catalogs
installedpkgs = pkgutils.getInstalledPackages()
installed = []
partiallyinstalled = []
installedpkgsmatchedtoname = {}
for name in itemname_to_pkgid:
# name is a Munki install item name
foundpkgcount = 0
for pkgid in itemname_to_pkgid[name]:
if pkgid in installedpkgs:
foundpkgcount += 1
if not name in installedpkgsmatchedtoname:
installedpkgsmatchedtoname[name] = []
# record this pkgid for Munki install item name
installedpkgsmatchedtoname[name].append(pkgid)
if foundpkgcount > 0:
if foundpkgcount == len(itemname_to_pkgid[name]):
# we found all receipts by pkgid on disk
installed.append(name)
else:
# we found only some receipts for the item
# on disk
partiallyinstalled.append(name)
# we pay special attention to the items that seem partially installed.
# we need to see if there are any packages that are unique to this item
# if there aren't, then this item probably isn't installed, and we're
# just finding receipts that are shared with other items.
for name in partiallyinstalled:
# get a list of pkgs for this item that are installed
pkgsforthisname = installedpkgsmatchedtoname[name]
# now build a list of all the pkgs referred to by all the other
# items that are either partially or entirely installed
allotherpkgs = []
for othername in installed:
allotherpkgs.extend(installedpkgsmatchedtoname[othername])
for othername in partiallyinstalled:
if othername != name:
allotherpkgs.extend(installedpkgsmatchedtoname[othername])
# use Python sets to find pkgs that are unique to this name
uniquepkgs = list(set(pkgsforthisname) - set(allotherpkgs))
if uniquepkgs:
installed.append(name)
# now filter partiallyinstalled to remove those items we moved to installed
partiallyinstalled = [item for item in partiallyinstalled
if item not in installed]
# build our reference table. For each item we think is installed,
# record the receipts on disk matched to the item
references = {}
for name in installed:
for pkgid in installedpkgsmatchedtoname[name]:
if not pkgid in references:
references[pkgid] = []
references[pkgid].append(name)
# look through all our installedpkgs, looking for ones that have not been
# attached to any Munki names yet
orphans = [pkgid for pkgid in installedpkgs if pkgid not in references]
# attempt to match orphans to Munki item names
matched_orphans = []
for pkgid in orphans:
if pkgid in pkgid_to_itemname:
installed_pkgid_version = installedpkgs[pkgid]
possible_match_items = pkgid_to_itemname[pkgid]
best_match = best_version_match(
installed_pkgid_version, possible_match_items)
if best_match:
matched_orphans.append(best_match)
# process matched_orphans
for name in matched_orphans:
if name not in installed:
installed.append(name)
if name in partiallyinstalled:
partiallyinstalled.remove(name)
for pkgid in installedpkgsmatchedtoname[name]:
if not pkgid in references:
references[pkgid] = []
if not name in references[pkgid]:
references[pkgid].append(name)
pkgdata['receipts_for_name'] = installedpkgsmatchedtoname
pkgdata['installed_names'] = installed
pkgdata['pkg_references'] = references
# left here for future debugging/testing use....
#pkgdata['itemname_to_pkgid'] = itemname_to_pkgid
#pkgdata['pkgid_to_itemname'] = pkgid_to_itemname
#pkgdata['partiallyinstalled_names'] = partiallyinstalled
#pkgdata['orphans'] = orphans
#pkgdata['matched_orphans'] = matched_orphans
#ManagedInstallDir = prefs.pref('ManagedInstallDir')
#pkgdatapath = os.path.join(ManagedInstallDir, 'PackageData.plist')
#try:
# FoundationPlist.writePlist(pkgdata, pkgdatapath)
#except FoundationPlist.NSPropertyListWriteException:
# pass
#catalogdbpath = os.path.join(ManagedInstallDir, 'CatalogDB.plist')
#try:
# FoundationPlist.writePlist(CATALOG, catalogdbpath)
#except FoundationPlist.NSPropertyListWriteException:
# pass
return pkgdata
def get_item_detail(name, cataloglist, vers='',
skip_min_os_check=False, suppress_warnings=False):
"""Searches the catalogs in list for an item matching the given name that
can be installed on the current hardware/OS (optionally skipping the
minimum OS check so we can return an item that requires a higher OS)
If no version is supplied, but the version is appended to the name
('TextWrangler--2.3.0.0.0') that version is used.
If no version is given at all, the latest version is assumed.
Returns a pkginfo item, or None.
"""
rejected_items = []
machine = info.getMachineFacts()
# condition check functions
def munki_version_ok(item):
'''Returns a boolean to indicate if the current Munki version is high
enough to install this item. If not, also adds the failure reason to
the rejected_items list.'''
if item.get('minimum_munki_version'):
min_munki_vers = item['minimum_munki_version']
display.display_debug1(
'Considering item %s, version %s '
'with minimum Munki version required %s',
item['name'], item['version'], min_munki_vers)
display.display_debug1(
'Our Munki version is %s', machine['munki_version'])
if (pkgutils.MunkiLooseVersion(machine['munki_version'])
< pkgutils.MunkiLooseVersion(min_munki_vers)):
reason = (
'Rejected item %s, version %s with minimum Munki version '
'required %s. Our Munki version is %s.'
% (item['name'], item['version'],
item['minimum_munki_version'], machine['munki_version']))
rejected_items.append(reason)
return False
return True
def os_version_ok(item, skip_min_os_check=False):
'''Returns a boolean to indicate if the item is ok to install under
the current OS. If not, also adds the failure reason to the
rejected_items list. If skip_min_os_check is True, skips the minimum os
version check.'''
# Is the current OS version >= minimum_os_version for the item?
if item.get('minimum_os_version') and not skip_min_os_check:
min_os_vers = item['minimum_os_version']
display.display_debug1(
'Considering item %s, version %s '
'with minimum os version required %s',
item['name'], item['version'], min_os_vers)
display.display_debug1(
'Our OS version is %s', machine['os_vers'])
if (pkgutils.MunkiLooseVersion(machine['os_vers']) <
pkgutils.MunkiLooseVersion(min_os_vers)):
# skip this one, go to the next
reason = (
'Rejected item %s, version %s with minimum os version '
'required %s. Our OS version is %s.'
% (item['name'], item['version'],
item['minimum_os_version'], machine['os_vers']))
rejected_items.append(reason)
return False
# current OS version <= maximum_os_version?
if item.get('maximum_os_version'):
max_os_vers = item['maximum_os_version']
display.display_debug1(
'Considering item %s, version %s '
'with maximum os version supported %s',
item['name'], item['version'], max_os_vers)
display.display_debug1(
'Our OS version is %s', machine['os_vers'])
if (pkgutils.MunkiLooseVersion(machine['os_vers']) >
pkgutils.MunkiLooseVersion(max_os_vers)):
# skip this one, go to the next
reason = (
'Rejected item %s, version %s with maximum os version '
'required %s. Our OS version is %s.'
% (item['name'], item['version'],
item['maximum_os_version'], machine['os_vers']))
rejected_items.append(reason)
return False
return True
def cpu_arch_ok(item):
'''Returns a boolean to indicate if the item is ok to install under
the current CPU architecture. If not, also adds the failure reason to
the rejected_items list.'''
if item.get('supported_architectures'):
display.display_debug1(
'Considering item %s, version %s '
'with supported architectures: %s',
item['name'], item['version'], item['supported_architectures'])
display.display_debug1(
'Our architecture is %s', machine['arch'])
if machine['arch'] in item['supported_architectures']:
return True
if ('x86_64' in item['supported_architectures'] and
machine['arch'] == 'i386' and
machine['x86_64_capable'] is True):
return True
# we didn't find a supported architecture that
# matches this machine
reason = (
'Rejected item %s, version %s with supported architectures: '
'%s. Our architecture is %s.'
% (item['name'], item['version'],
item['supported_architectures'], machine['arch']))
rejected_items.append(reason)
return False
return True
def installable_condition_ok(item):
'''Returns a boolean to indicate if an installable_condition predicate
in the current item passes. If not, also adds the failure reason to
the rejected_items list.'''
if item.get('installable_condition'):
if not info.predicate_evaluates_as_true(
item['installable_condition']):
rejected_items.append(
'Rejected item %s, version %s with installable_condition: '
'%s.' % (item['name'], item['version'],
item['installable_condition']))
return False
return True
if vers == 'apple_update_metadata':
vers = 'latest'
else:
(name, includedversion) = split_name_and_version(name)
if includedversion and vers == '':
vers = includedversion
if vers:
vers = pkgutils.trim_version_string(vers)
else:
vers = 'latest'
if skip_min_os_check:
display.display_debug1(
'Looking for detail for: %s, version %s, '
'ignoring minimum_os_version...', name, vers)
else:
display.display_debug1(
'Looking for detail for: %s, version %s...', name, vers)
for catalogname in cataloglist:
# is name in the catalog?
if catalogname in _CATALOG and name in _CATALOG[catalogname]['named']:
itemsmatchingname = _CATALOG[catalogname]['named'][name]
indexlist = []
if vers == 'latest':
# order all our items, highest version first
versionlist = list(itemsmatchingname.keys())
versionlist.sort(key=pkgutils.MunkiLooseVersion, reverse=True)
for versionkey in versionlist:
indexlist.extend(itemsmatchingname[versionkey])
elif vers in list(itemsmatchingname.keys()):
# get the specific requested version
indexlist = itemsmatchingname[vers]
if indexlist:
display.display_debug1(
'Considering %s items with name %s from catalog %s' %
(len(indexlist), name, catalogname))
for index in indexlist:
# iterate through list of items with matching name, highest
# version first, looking for first one that passes all the
# conditional tests (if any)
item = _CATALOG[catalogname]['items'][index]
if (munki_version_ok(item) and
os_version_ok(item,
skip_min_os_check=skip_min_os_check) and
cpu_arch_ok(item) and
installable_condition_ok(item)):
display.display_debug1(
'Found %s, version %s in catalog %s',
item['name'], item['version'], catalogname)
return item
# if we got this far, we didn't find it.
display.display_debug1('Not found')
for reason in rejected_items:
if suppress_warnings:
display.display_debug1(reason)
else:
display.display_warning(reason)
return None
# global to hold our catalog DBs
_CATALOG = {}
def get_catalogs(cataloglist):
"""Retrieves the catalogs from the server and populates our catalogs
dictionary.
"""
#global _CATALOG
for catalogname in cataloglist:
if not catalogname in _CATALOG:
catalogpath = download.download_catalog(catalogname)
if catalogpath:
try:
catalogdata = FoundationPlist.readPlist(catalogpath)
except FoundationPlist.NSPropertyListSerializationException:
display.display_error(
'Retrieved catalog %s is invalid.', catalogname)
try:
os.unlink(catalogpath)
except (OSError, IOError):
pass
else:
_CATALOG[catalogname] = make_catalog_db(catalogdata)
def clean_up():
"""Removes any catalog files that are no longer in use by this client"""
catalog_dir = os.path.join(prefs.pref('ManagedInstallDir'),
'catalogs')
for item in os.listdir(catalog_dir):
if item not in _CATALOG:
os.unlink(os.path.join(catalog_dir, item))
def catalogs():
'''Returns our internal _CATALOG dict'''
return _CATALOG
if __name__ == '__main__':
print('This is a library of support tools for the Munki Suite.')