munki/code/client/makecatalogs

#!/usr/bin/env python
# encoding: utf-8
#
# Copyright 2009-2016 Greg Neagle.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
makecatalogs

Created by Greg Neagle on 2009-03-30.

Recursively scans a directory, looking for installer item info files.
Builds a repo catalog from these files.

Assumes a pkgsinfo directory under repopath.
User calling this needs to be able to write to repo/catalogs.

"""

import sys
import os
import optparse
import hashlib

try:
    from munkilib import FoundationPlist as plistlib
    LOCAL_PREFS_SUPPORT = True
except ImportError:
    try:
        import FoundationPlist as plistlib
        LOCAL_PREFS_SUPPORT = True
    except ImportError:
        # maybe we're not on an OS X machine...
        print >> sys.stderr, ("WARNING: FoundationPlist is not available, "
                              "using plistlib instead.")
        import plistlib
        LOCAL_PREFS_SUPPORT = False

try:
    from munkilib.munkicommon import listdir, get_version
except ImportError:
    # munkilib is not available
    def listdir(path):
        """OS X HFS+ string encoding safe listdir().

        Args:
            path: path to list contents of
        Returns:
            list of contents, items as str or unicode types
        """
        # if os.listdir() is supplied a unicode object for the path,
        # it will return unicode filenames instead of their raw fs-dependent
        # version, which is decomposed utf-8 on OS X.
        #
        # we use this to our advantage here and have Python do the decoding
        # work for us, instead of decoding each item in the output list.
        #
        # references:
        # https://docs.python.org/howto/unicode.html#unicode-filenames
        # https://developer.apple.com/library/mac/#qa/qa2001/qa1235.html
        # http://lists.zerezo.com/git/msg643117.html
        # http://unicode.org/reports/tr15/    section 1.2
        if type(path) is str:
            path = unicode(path, 'utf-8')
        elif type(path) is not unicode:
            path = unicode(path)
        return os.listdir(path)

    def get_version():
        '''Placeholder if munkilib is not available'''
        return 'UNKNOWN'


def print_utf8(text):
    '''Print Unicode text as UTF-8'''
    print text.encode('UTF-8')

def print_err_utf8(text):
    '''Print Unicode text to stderr as UTF-8'''
    print >> sys.stderr, text.encode('UTF-8')


def makecatalogs(repopath, options):
    '''Assembles all pkginfo files into catalogs.
    Assumes a pkgsinfo directory under repopath.
    User calling this needs to be able to write to the repo/catalogs
    directory.'''

    # start with no errors!
    errors = []
    exit_code = 0

    # Make sure the icons directory exists
    iconspath = os.path.join(repopath, 'icons')
    # make sure iconspath is Unicode so that os.walk later gives us
    # Unicode names back.
    if type(iconspath) is str:
        iconspath = unicode(iconspath, 'utf-8')
    elif type(iconspath) is not unicode:
        iconspath = unicode(iconspath)

    if not os.path.exists(iconspath):
        print_err_utf8("icons path %s doesn't exist, skipping hashing!"
                       % iconspath)
        iconhashing = False
        # exit(-1) don't exit so we don't break when no icons dir
    else:
        icons = {}
        iconhashing = True

        # Walk through the icon files
        for dirpath, dirnames, filenames in os.walk(iconspath):
            for dirname in dirnames:
                # don't recurse into directories that start
                # with a period.
                if dirname.startswith('.'):
                    dirnames.remove(dirname)
            for filename in filenames:
                if filename.startswith('.'):
                    # skip files that start with a period as well
                    continue

                filepath = os.path.join(dirpath, filename)

                iconpath = filepath.rsplit(iconspath + '/', 1)[1]

                # Try to read the icon file
                try:
                    print_utf8("Hashing %s..." % (iconpath))
                    icons[iconpath] = (
                        hashlib.sha256(open(filepath, 'rb').read()).hexdigest())
                except IOError, inst:
                    errors.append("IO error for %s: %s" % (filepath, inst))
                    exit_code = -1
                    continue
                except BaseException, inst:
                    errors.append("Unexpected error for %s: %s"
                                  % (filepath, inst))
                    exit_code = -1
                    continue

    # Make sure the pkgsinfo directory exists
    pkgsinfopath = os.path.join(repopath, 'pkgsinfo')
    # make sure pkgsinfopath is Unicode so that os.walk later gives us
    # Unicode names back.
    if type(pkgsinfopath) is str:
        pkgsinfopath = unicode(pkgsinfopath, 'utf-8')
    elif type(pkgsinfopath) is not unicode:
        pkgsinfopath = unicode(pkgsinfopath)

    if not os.path.exists(pkgsinfopath):
        print_err_utf8("pkgsinfo path %s doesn't exist!" % pkgsinfopath)
        exit(-1)

    # start with empty catalogs dict
    catalogs = {}
    catalogs['all'] = []

    # Walk through the pkginfo files
    for dirpath, dirnames, filenames in os.walk(pkgsinfopath, followlinks=True):
        for dirname in dirnames:
            # don't recurse into directories that start
            # with a period.
            if dirname.startswith('.'):
                dirnames.remove(dirname)
        for filename in filenames:
            if filename.startswith('.'):
                # skip files that start with a period as well
                continue

            filepath = os.path.join(dirpath, filename)

            # Try to read the pkginfo file
            try:
                pkginfo = plistlib.readPlist(filepath)
            except IOError, inst:
                errors.append("IO error for %s: %s" % (filepath, inst))
                exit_code = -1
                continue
            except BaseException, inst:
                errors.append("Unexpected error for %s: %s" % (filepath, inst))
                exit_code = -1
                continue

            if not 'name' in pkginfo:
                errors.append(
                    "WARNING: file %s is missing name"
                    % filepath[len(pkgsinfopath)+1:])
                continue

            # don't copy admin notes to catalogs.
            if pkginfo.get('notes'):
                del pkginfo['notes']
            # strip out any keys that start with "_"
            # (example: pkginfo _metadata)
            for key in pkginfo.keys():
                if key.startswith('_'):
                    del pkginfo[key]

            if iconhashing:

                name = pkginfo.get('name')

                if pkginfo.get('icon_name'):
                    iconhash = (icons.get(pkginfo['icon_name']) or
                                icons.get(pkginfo['icon_name'] + '.png'))
                    if not iconhash:
                        errors.append("WARNING: icon_name specified"
                                      " in info file %s but it does not exist"
                                      % filepath[len(pkgsinfopath) + 1:])
                else:
                    iconhash = icons.get(name + '.png')

                if iconhash is not None:
                    pkginfo['icon_hash'] = iconhash
                    iconhash = None

            #simple sanity checking
            do_pkg_check = True
            installer_type = pkginfo.get('installer_type')
            if installer_type in ['nopkg', 'apple_update_metadata']:
                do_pkg_check = False
            if pkginfo.get('PackageCompleteURL'):
                do_pkg_check = False
            if pkginfo.get('PackageURL'):
                do_pkg_check = False


            if do_pkg_check:
                if not 'installer_item_location' in pkginfo:
                    errors.append(
                        "WARNING: file %s is missing installer_item_location"
                        % filepath[len(pkgsinfopath)+1:])
                    # Skip this pkginfo unless we're running with force flag
                    if not options.force:
                        exit_code = -1
                        continue

                # Try to form a path and fail if the
                # installer_item_location is not a valid type
                try:
                    installeritempath = os.path.join(
                        repopath, "pkgs", pkginfo['installer_item_location'])
                except TypeError:
                    errors.append("WARNING: invalid installer_item_location "
                                  "in info file %s"
                                  % filepath[len(pkgsinfopath)+1:])
                    exit_code = -1
                    continue

                # Check if the installer item actually exists
                if not os.path.exists(installeritempath):
                    errors.append("WARNING: Info file %s refers to "
                                  "missing installer item: %s" %
                                  (filepath[len(pkgsinfopath)+1:],
                                   pkginfo['installer_item_location']))
                    # Skip this pkginfo unless we're running with force flag
                    if not options.force:
                        exit_code = -1
                        continue

            #uninstaller sanity checking
            uninstaller_type = pkginfo.get('uninstall_method')
            if uninstaller_type in ['AdobeCCPUninstaller']:
                # uninstaller_item_location is required
                if not 'uninstaller_item_location' in pkginfo:
                   errors.append(
                       "WARNING: file %s is missing uninstaller_item_location"
                       % filepath[len(pkgsinfopath)+1:])
                   # Skip this pkginfo unless we're running with force flag
                   if not options.force:
                       exit_code = -1
                       continue

            # if an uninstaller_item_location is specified, sanity-check it
            if 'uninstaller_item_location' in pkginfo:
                try:
                    uninstalleritempath = os.path.join(
                        repopath, "pkgs", pkginfo['uninstaller_item_location'])
                except TypeError:
                    errors.append("WARNING: invalid uninstaller_item_location "
                                  "in info file %s"
                                  % filepath[len(pkgsinfopath)+1:])
                    exit_code = -1
                    continue

                # Check if the uninstaller item actually exists
                if not os.path.exists(uninstalleritempath):
                    errors.append("WARNING: Info file %s refers to "
                                  "missing uninstaller item: %s" %
                                  (filepath[len(pkgsinfopath)+1:],
                                   pkginfo['uninstaller_item_location']))
                    # Skip this pkginfo unless we're running with force flag
                    if not options.force:
                        exit_code = -1
                        continue

            catalogs['all'].append(pkginfo)
            for catalogname in pkginfo.get("catalogs", []):
                infofilename = filepath[len(pkgsinfopath)+1:]
                if not catalogname:
                    errors.append("WARNING: Info file %s has an empty "
                                  "catalog name!" % infofilename)
                    exit_code = -1
                    continue
                if not catalogname in catalogs:
                    catalogs[catalogname] = []
                catalogs[catalogname].append(pkginfo)
                print_utf8("Adding %s to %s..." % (infofilename, catalogname))

    if errors:
        # group all errors at the end for better visibility
        print
        for error in errors:
            print_err_utf8(error)

    # clear out old catalogs
    catalogpath = os.path.join(repopath, "catalogs")
    if not os.path.exists(catalogpath):
        os.mkdir(catalogpath)
    else:
        for item in listdir(catalogpath):
            itempath = os.path.join(catalogpath, item)
            if os.path.isfile(itempath):
                os.remove(itempath)

    # write the new catalogs
    print
    for key in catalogs.keys():
        catalogpath = os.path.join(repopath, "catalogs", key)
        if os.path.exists(catalogpath):
            print_err_utf8(
                "WARNING: catalog %s already exists at %s. "
                "Perhaps this is a non-case sensitive filesystem and you "
                "have catalogs with names differing only in case?"
                % (key, catalogpath))
            exit_code = -1
        elif len(catalogs[key]) != 0:
            plistlib.writePlist(catalogs[key], catalogpath)
            print "Created catalog %s..." % (catalogpath)
        else:
            print_err_utf8(
                "WARNING: Did not create catalog %s "
                "because it is empty "
                % (key))
            exit_code = -1

    # Exit with "exit_code" if we got this far.
    # This will be -1 if there were any errors
    # that prevented the catalogs to be written.
    exit(exit_code)


def pref(prefname):
    """Returns a preference for prefname"""
    if not LOCAL_PREFS_SUPPORT:
        return None
    try:
        _prefs = plistlib.readPlist(PREFSPATH)
    except BaseException:
        return None
    if prefname in _prefs:
        return _prefs[prefname]
    else:
        return None


PREFSNAME = 'com.googlecode.munki.munkiimport.plist'
PREFSPATH = os.path.expanduser(os.path.join('~/Library/Preferences',
                                            PREFSNAME))
def main():
    '''Main'''
    usage = "usage: %prog [options] [/path/to/repo_root]"
    parser = optparse.OptionParser(usage=usage)
    parser.add_option('--version', '-V', action='store_true',
                      help='Print the version of the munki tools and exit.')
    parser.add_option('--force', '-f', action='store_true', dest='force',
                      help='Disable sanity checks.')
    parser.set_defaults(force=False)
    options, arguments = parser.parse_args()

    if options.version:
        print get_version()
        exit(0)

    # Make sure we have a path to work with
    repopath = None
    if len(arguments) == 0:
        repopath = pref('repo_path')
        if not repopath:
            print_err_utf8("Need to specify a path to the repo root!")
            exit(-1)
        else:
            print_utf8("Using repo path: %s" % repopath)
    else:
        repopath = arguments[0].rstrip("/")

    # Make sure the repo path exists
    if not os.path.exists(repopath):
        print_err_utf8("Repo root path %s doesn't exist!" % repopath)
        exit(-1)

    # Make the catalogs
    makecatalogs(repopath, options)

if __name__ == '__main__':
    main()