munki/code/client/manifestutil

#!/usr/local/munki/munki-python
# encoding: utf-8
#
# Copyright 2011-2021 Greg Neagle.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
manifestutil

Created by Greg Neagle on 2011-03-04.
"""
from __future__ import absolute_import, print_function

# TODO: add support for delete-manifest

import fnmatch
import optparse
import os
import readline
import shlex
import sys

# our libs
from munkilib.cliutils import ConfigurationSaveError
from munkilib.cliutils import configure as _configure
from munkilib.cliutils import libedit
from munkilib.cliutils import get_version, pref, path2url
from munkilib.wrappers import (get_input,
                               readPlistFromString, writePlistToString,
                               PlistReadError, PlistWriteError)

from munkilib import munkirepo


def get_installer_item_names(repo, catalog_limit_list):
    '''Returns a list of unique installer item (pkg) names
    from the given list of catalogs'''
    item_list = []
    try:
        catalogs_list = repo.itemlist('catalogs')
    except munkirepo.RepoError as err:
        print((
            u'Could not retrieve catalogs: %s' % err), file=sys.stderr)
        return []
    for catalog_name in catalogs_list:
        if catalog_name in catalog_limit_list:
            try:
                data = repo.get(os.path.join('catalogs', catalog_name))
                catalog = readPlistFromString(data)
            except munkirepo.RepoError as err:
                print((
                    u'Could not retrieve catalog %s: %s'
                    % (catalog_name, err)), file=sys.stderr)
            except (IOError, OSError, PlistReadError):
                # skip items that aren't valid plists
                # or that we can't read
                pass
            else:
                item_list.extend([item['name']
                                  for item in catalog
                                  if not item.get('update_for')])
    item_list = list(set(item_list))
    item_list.sort()
    return item_list


def get_manifest_names(repo):
    '''Returns a list of available manifests'''
    try:
        manifest_names = repo.itemlist('manifests')
    except munkirepo.RepoError as err:
        print((
            u'Could not retrieve manifests: %s' % err), file=sys.stderr)
        manifest_names = []
    manifest_names.sort()
    return manifest_names


def get_catalogs(repo):
    '''Returns a list of available catalogs'''
    try:
        catalog_names = repo.itemlist('catalogs')
    except munkirepo.RepoError as err:
        print((
            u'Could not retrieve catalogs: %s' % err), file=sys.stderr)
        catalog_names = []
    catalog_names.sort()
    return catalog_names


def get_manifest_pkg_sections():
    '''Returns a list of manifest sections that can contain pkg names'''
    return ['managed_installs',
            'managed_uninstalls',
            'managed_updates',
            'optional_installs']


def printplistitem(label, value, indent=0):
    """Prints a plist item in an 'attractive' way"""
    indentspace = '    '
    if value is None:
        print(indentspace*indent, '%s: !NONE!' % label)
    elif isinstance(value, list) or type(value).__name__ == 'NSCFArray':
        if label:
            print(indentspace*indent, '%s:' % label)
        for item in value:
            printplistitem('', item, indent+1)
    elif isinstance(value, dict) or type(value).__name__ == 'NSCFDictionary':
        if label:
            print(indentspace*indent, '%s:' % label)
        keys = list(value.keys())
        keys.sort()
        for subkey in keys:
            printplistitem(subkey, value[subkey], indent+1)
    else:
        if label:
            print(indentspace*indent, '%s: %s' % (label, value))
        else:
            print(indentspace*indent, '%s' % value)


def printplist(plistdict):
    """Prints plist dictionary in a pretty(?) way"""
    keys = list(plistdict.keys())
    keys.sort()
    for key in keys:
        printplistitem(key, plistdict[key])


def get_manifest(repo, manifest_name):
    '''Gets the contents of a manifest'''
    manifest_ref = os.path.join('manifests', manifest_name)
    try:
        data = repo.get(manifest_ref)
        return readPlistFromString(data)
    except munkirepo.RepoError as err:
        print(u'Could not retrieve manifest %s: %s'
              % (manifest_name, err), file=sys.stderr)
        return None
    except (IOError, OSError, PlistReadError) as err:
        print(u'Could not read manifest %s: %s' % (manifest_name, err),
              file=sys.stderr)
        return None


def save_manifest(repo, manifest_dict, manifest_name, overwrite_existing=False):
    '''Saves a manifest to disk'''
    existing_manifest_names = get_manifest_names(repo)
    if not overwrite_existing:
        if manifest_name in existing_manifest_names:
            print('%s already exists!' % manifest_name, file=sys.stderr)
            return False
    manifest_ref = os.path.join('manifests', manifest_name)
    try:
        data = writePlistToString(manifest_dict)
        repo.put(manifest_ref, data)
        return True
    except (IOError, OSError, PlistWriteError, munkirepo.RepoError) as err:
        print(u'Saving %s failed: %s' % (manifest_name, err),
              file=sys.stderr)
        return False


def manifest_rename(repo, source_manifest_name, dest_manifest_name,
                    overwrite_existing=False):
    '''Renames an existing manifest. Since the repo can be an API-based repo,
    we really have to save a copy under the new name and then delete the old
    one.'''
    source_manifest_ref = os.path.join('manifests', source_manifest_name)
    dest_manifest_ref = os.path.join('manifests', dest_manifest_name)
    existing_manifest_names = get_manifest_names(repo)
    if not overwrite_existing:
        if dest_manifest_name in existing_manifest_names:
            print(u'%s already exists!' % dest_manifest_name, file=sys.stderr)
            return False
    try:
        source_data = repo.get(source_manifest_ref)
        repo.put(dest_manifest_ref, source_data)
        repo.delete(source_manifest_ref)
        return True
    except munkirepo.RepoError as err:
        print(u'Renaming %s to %s failed: %s'
              % (source_manifest_name, dest_manifest_name, err),
              file=sys.stderr)
        return False


def cleanup_and_exit(exitcode):
    """Give the user the chance to unmount the repo when we exit"""
    result = 0
    # TODO: handle mounted repo
    #if repo and repo.mounted and repo.WE_MOUNTED_THE_REPO:
    #    answer = raw_input('Unmount the repo fileshare? [y/n] ')
    #    if answer.lower().startswith('y'):
    #        result = repo.unmount()
    exit(exitcode or result)


def update_cached_manifest_list(repo):
    '''Updates our cached list of available manifests so our completer
    will return all the available manifests.'''
    CMD_ARG_DICT['manifests'] = get_manifest_names(repo)


##### subcommand functions #####

class MyOptParseError(Exception):
    '''Exception for our custom option parser'''
    pass


class MyOptParseExit(Exception):
    '''Raised when optparse calls self.exit() so we can handle it instead
    of exiting'''
    pass


class MyOptionParser(optparse.OptionParser):
    '''Custom option parser that overrides the error handler and
    the exit handler so printing help doesn't exit the interactive
    pseudo-shell'''
    def error(self, msg):
        """error(msg : string)

        """
        self.print_usage(sys.stderr)
        raise MyOptParseError('option error: %s' % msg)

    def exit(self, status=0, msg=None):
        if msg:
            sys.stderr.write(msg)
        raise MyOptParseExit


def version(repo, args):
    '''Prints version number'''
    # we ignore repo arg but subcommand dispatcher sends it to all
    # subcommands
    if len(args) != 0:
        print('Usage: version', file=sys.stderr)
        return 22 # Invalid argument
    print(get_version())
    return 0


def configure(repo, args):
    '''Allow user to configure tool options'''
    # we ignore the repo arg, but all the other subcommands require it
    if len(args):
        print('Usage: configure', file=sys.stderr)
        return 22 # Invalid argument
    prompt_list = [
        ('repo_url', 'Repo URL (example: afp://munki.example.com/repo)'),
        ('plugin', 'Munki repo plugin (defaults to FileRepo)')
    ]
    try:
        _configure(prompt_list)
        return 0
    except ConfigurationSaveError:
        return 1 # Operation not permitted


def list_catalogs(repo, args):
    '''Prints the names of the available catalogs'''
    parser = MyOptionParser()
    parser.set_usage('''list-catalogs
       Prints the names of the available catalogs''')
    try:
        _, arguments = parser.parse_args(args)
    except MyOptParseError as errmsg:
        print(str(errmsg), file=sys.stderr)
        return 22 # Invalid argument
    except MyOptParseExit:
        return 0

    if len(arguments) != 0:
        parser.print_usage(sys.stderr)
        return 22 # Invalid argument
    for item in get_catalogs(repo):
        print(item)
    return 0


def list_catalog_items(repo, args):
    '''Lists items in the given catalogs'''
    parser = MyOptionParser()
    parser.set_usage('''list-catalog-items CATALOG_NAME [CATALOG_NAME ...]
       Lists items in the given catalogs''')
    try:
        _, arguments = parser.parse_args(args)
    except MyOptParseError as errmsg:
        print(str(errmsg), file=sys.stderr)
        return 22 # Invalid argument
    except MyOptParseExit:
        return 0

    if len(arguments) == 0:
        parser.print_usage(sys.stderr)
        return 1 # Operation not permitted
    available_catalogs = get_catalogs(repo)
    for catalog in arguments:
        if catalog not in available_catalogs:
            print('%s: no such catalog!' % catalog, file=sys.stderr)
            return 2 # No such file or directory
    for pkg in get_installer_item_names(repo, arguments):
        print(pkg)
    return 0


def list_manifests(repo, args):
    '''Prints names of available manifests, filtering on arg[0]
    similar to Unix file globbing'''
    parser = MyOptionParser()
    parser.set_usage('''list-manifests [FILE_NAME_MATCH_STRING]
       Prints names of available manifests.''')
    try:
        _, arguments = parser.parse_args(args)
    except MyOptParseError as errmsg:
        print(str(errmsg), file=sys.stderr)
        return 22 # Invalid argument
    except MyOptParseExit:
        return 0

    if len(arguments) == 1:
        list_filter = arguments[0]
    elif len(arguments) == 0:
        list_filter = '*'
    else:
        parser.print_usage(sys.stderr)
        return 7 # Argument list too long
    for item in get_manifest_names(repo):
        if fnmatch.fnmatch(item, list_filter):
            print(item)
    return 0


def find(repo, args):
    '''Find text in manifests, optionally searching just a specific manifest
    section specified by keyname'''
    parser = MyOptionParser()
    parser.set_usage('''find FIND_TEXT [--section SECTION_NAME]
       Find text in manifests, optionally searching a specific manifest
       section''')
    parser.add_option('--section',
                      metavar='SECTION_NAME',
                      help=('(Optional) Section of the manifest to search for '
                            'FIND_TEXT'))
    try:
        options, arguments = parser.parse_args(args)
    except MyOptParseError as errmsg:
        print(str(errmsg), file=sys.stderr)
        return 22 # Invalid argument
    except MyOptParseExit:
        return 0

    if not options.section and len(arguments) == 2:
        options.section = arguments[1]
        del arguments[1]
    if len(arguments) != 1:
        parser.print_usage(sys.stderr)
        return 7 # Argument list too long
    findtext = arguments[0]
    keyname = options.section

    count = 0
    for name in get_manifest_names(repo):
        manifest_ref = os.path.join('manifests', name)
        try:
            data = repo.get(manifest_ref)
            manifest = readPlistFromString(data)
        except (IOError, OSError, PlistReadError, munkirepo.RepoError) as err:
            print(u'Error reading %s: %s' % (manifest_ref, err),
                  file=sys.stderr)
            continue
        if keyname:
            if keyname in manifest:
                value = manifest[keyname]
                if type(value) == list or type(value).__name__ == 'NSCFArray':
                    for item in value:
                        try:
                            if findtext.upper() in item.upper():
                                print('%s: %s' % (name, item))
                                count += 1
                        except AttributeError:
                            pass
                elif findtext.upper() in value.upper():
                    print('%s: %s' % (name, value))
                    count += 1
        else:
            for key in manifest.keys():
                value = manifest[key]
                if type(value) == list or type(value).__name__ == 'NSCFArray':
                    for item in value:
                        try:
                            if findtext.upper() in item.upper():
                                print('%s (%s): %s' % (name, key, item))
                                count += 1
                        except AttributeError:
                            pass
                elif findtext.upper() in value.upper():
                    print('%s (%s): %s' % (name, key, value))
                    count += 1

    print('%s matches found.' % count)
    return 0


def display_manifest(repo, args):
    '''Prints contents of a given manifest'''
    parser = MyOptionParser()
    parser.set_usage('''display-manifest MANIFESTNAME
       Prints the contents of the specified manifest''')
    try:
        _, arguments = parser.parse_args(args)
    except MyOptParseError as errmsg:
        print(str(errmsg), file=sys.stderr)
        return 22 # Invalid argument
    except MyOptParseExit:
        return 0

    if len(arguments) != 1:
        parser.print_usage(sys.stderr)
        return 7 # Argument list too long
    manifestname = arguments[0]
    manifest = get_manifest(repo, manifestname)
    if manifest:
        printplist(manifest)
        return 0
    else:
        return 2 # No such file or directory


def expand_included_manifests(repo, args):
    '''Prints a manifest, expanding any included manifests.'''

    def manifest_recurser(repo, manifest):
        '''Recursive expansion of included manifests'''
        # No infinite loop checking! Be wary!
        if 'included_manifests' in manifest:
            for (index, item) in enumerate(manifest['included_manifests']):
                included_manifest = get_manifest(repo, item)
                if included_manifest:
                    included_manifest = manifest_recurser(repo, included_manifest)
                manifest['included_manifests'][index] = {
                    item: included_manifest
                }
        return manifest

    parser = MyOptionParser()
    parser.set_usage('''expand-included-manifest MANIFESTNAME
        Prints included manifests in the specified manifest''')
    try:
        _, arguments = parser.parse_args(args)
    except MyOptParseError as errmsg:
        print(str(errmsg), file=sys.stderr)
        return 22 # Invalid argument
    except MyOptParseExit:
        return 0

    if len(arguments) != 1:
        parser.print_usage(sys.stderr)
        return 7 # Argument list too long
    manifestname = arguments[0]
    manifest = get_manifest(repo, manifestname)
    if manifest:
        printplist(manifest_recurser(repo, manifest))
    else:
        return 2 # No such file or directory


def new_manifest(repo, args):
    '''Creates a new, empty manifest'''
    parser = MyOptionParser()
    parser.set_usage('''new-manifest MANIFESTNAME
       Creates a new empty manifest''')
    try:
        _, arguments = parser.parse_args(args)
    except MyOptParseError as errmsg:
        print(str(errmsg), file=sys.stderr)
        return 22 # Invalid argument
    except MyOptParseExit:
        return 0

    if len(arguments) != 1:
        parser.print_usage(sys.stderr)
        return 7 # Argument list too long
    manifest_name = arguments[0]
    manifest = {'catalogs': [],
                'included_manifests': [],
                'managed_installs': [],
                'managed_uninstalls': []}
    if save_manifest(repo, manifest, manifest_name):
        update_cached_manifest_list(repo)
        return 0
    else:
        return 1 # Operation not permitted


def copy_manifest(repo, args):
    '''Copies one manifest to another'''
    parser = MyOptionParser()
    parser.set_usage(
        '''copy-manifest SOURCE_MANIFEST DESTINATION_MANIFEST
       Copies the contents of one manifest to another''')
    try:
        _, arguments = parser.parse_args(args)
    except MyOptParseError as errmsg:
        print(str(errmsg), file=sys.stderr)
        return 22 # Invalid argument
    except MyOptParseExit:
        return 0

    if len(arguments) != 2:
        parser.print_usage(sys.stderr)
        return 7 # Argument list too long
    source_manifest = arguments[0]
    dest_manifest = arguments[1]
    manifest = get_manifest(repo, source_manifest)
    if manifest and save_manifest(repo, manifest, dest_manifest):
        update_cached_manifest_list(repo)
        return 0
    else:
        return 1 # Operation not permitted


def rename_manifest(repo, args):
    '''Renames a manifest'''
    parser = MyOptionParser()
    parser.set_usage(
        '''rename-manifest SOURCE_MANIFEST DESTINATION_MANIFEST
       Renames the manifest''')
    try:
        _, arguments = parser.parse_args(args)
    except MyOptParseError as errmsg:
        print(str(errmsg), file=sys.stderr)
        return 22 # Invalid argument
    except MyOptParseExit:
        return 0

    if len(arguments) != 2:
        parser.print_usage(sys.stderr)
        return 7 # Argument list too long
    source_manifest = arguments[0]
    dest_manifest = arguments[1]
    if manifest_rename(repo, source_manifest, dest_manifest):
        print ('Renamed manifest %s to %s.'
               % (source_manifest, dest_manifest))
        update_cached_manifest_list(repo)
        return 0
    else:
        return 1 # Operation not permitted


def add_pkg(repo, args):
    '''Adds a package to a manifest.'''
    parser = MyOptionParser()
    parser.set_usage(
        '''add-pkg PKGNAME --manifest MANIFESTNAME [--section SECTIONNAME]
       Adds a package to a manifest. Package is added to managed_installs
       unless a different manifest section is specified with the
       --section option''')
    parser.add_option('--manifest',
                      metavar='MANIFESTNAME',
                      help='name of manifest on which to operate')
    parser.add_option('--section', default='managed_installs',
                      metavar='SECTIONNAME',
                      help=('manifest section to which to add the package. '
                            'Defaults to managed_installs.'))
    try:
        options, arguments = parser.parse_args(args)
    except MyOptParseError as errmsg:
        print(str(errmsg), file=sys.stderr)
        return 22 # Invalid argument
    except MyOptParseExit:
        return 0

    if not options.manifest:
        if len(arguments) == 2:
            options.manifest = arguments[1]
            del arguments[1]
        else:
            parser.print_usage(sys.stderr)
            return 7 # Argument list too long
    if len(arguments) != 1:
        parser.print_usage(sys.stderr)
        return 7 # Argument list too long
    pkgname = arguments[0]

    manifest = get_manifest(repo, options.manifest)
    if not manifest:
        return 2 # No such file or directory
    for section in get_manifest_pkg_sections():
        if pkgname in manifest.get(section, []):
            print((
                'Package %s is already in section %s of manifest %s.'
                % (pkgname, section, options.manifest)), file=sys.stderr)
            return 1 # Operation not permitted

    manifest_catalogs = manifest.get('catalogs', [])
    available_pkgnames = get_installer_item_names(repo, manifest_catalogs)
    if pkgname not in available_pkgnames:
        print((
            'WARNING: Package %s is not available in catalogs %s '
            'of manifest %s.'
            % (pkgname, manifest_catalogs, options.manifest)), file=sys.stderr)
    if not options.section in manifest:
        manifest[options.section] = [pkgname]
    else:
        manifest[options.section].append(pkgname)
    if save_manifest(repo, manifest, options.manifest, overwrite_existing=True):
        print ('Added %s to section %s of manifest %s.'
               % (pkgname, options.section, options.manifest))
        return 0
    else:
        return 1 # Operation not permitted


def move_install_to_uninstall(repo, args):
    '''Moves a package from managed_installs to managed_uninstalls in a manifest.'''
    parser = MyOptionParser()
    parser.set_usage(
        '''move-install-to-uninstall PKGNAME --manifest MANIFESTNAME
       Moves a package from managed_installs to managed_uninstalls in a manifest.''')
    parser.add_option('--manifest',
                      metavar='MANIFESTNAME',
                      help='''name of manifest on which to operate''')
    try:
        options, arguments = parser.parse_args(args)
    except MyOptParseError as errmsg:
        print(str(errmsg), file=sys.stderr)
        return 22 # Invalid argument
    except MyOptParseExit:
        return 0

    if not options.manifest:
        if len(arguments) == 2:
            options.manifest = arguments[1]
            del arguments[1]
        else:
            parser.print_usage(sys.stderr)
            return 7 # Argument list too long
    if len(arguments) != 1:
        parser.print_usage(sys.stderr)
        return 7 # Argument list too long
    pkgname = arguments[0]

    manifest = get_manifest(repo, options.manifest)
    if not manifest:
        return 2 # No such file or directory
    else:
        if pkgname in manifest['managed_installs']:
            manifest['managed_installs'].remove(pkgname)
            print ('Removed %s from section %s of manifest %s'
                   % (pkgname, 'managed_installs', options.manifest))
        else:
            print('WARNING: Package %s is not in section %s '
                  'of manifest %s. No changes made.'
                  % (pkgname, 'managed_installs', options.manifest),
                  file=sys.stderr)
            return 1 # Operation not permitted
        if pkgname in manifest['managed_uninstalls']:
            print ('%s is already in section managed_uninstalls of manifest %s.'
                   % (pkgname, options.manifest))
        else:
            manifest['managed_uninstalls'].append(pkgname)
            print ('Added %s to section managed_uninstalls of manifest %s.'
                   % (pkgname, options.manifest))

        if save_manifest(repo, manifest, options.manifest,
                         overwrite_existing=True):
            return 0
        else:
            return 1 # Operation not permitted


def remove_pkg(repo, args):
    '''Removes a package from a manifest.'''
    parser = MyOptionParser()
    parser.set_usage(
        '''remove-pkg PKGNAME --manifest MANIFESTNAME [--section SECTIONNAME]
       Removes a package from a manifest. Package is removed from
       managed_installs unless a different manifest section is specified with
       the --section option''')
    parser.add_option('--manifest',
                      metavar='MANIFESTNAME',
                      help='''name of manifest on which to operate''')
    parser.add_option('--section', default='managed_installs',
                      metavar='SECTIONNAME',
                      help=('manifest section from which to remove the '
                            'package. Defaults to managed_installs.'))
    try:
        options, arguments = parser.parse_args(args)
    except MyOptParseError as errmsg:
        print(str(errmsg), file=sys.stderr)
        return 22 # Invalid argument
    except MyOptParseExit:
        return 0

    if not options.manifest:
        if len(arguments) == 2:
            options.manifest = arguments[1]
            del arguments[1]
        else:
            parser.print_usage(sys.stderr)
            return 7 # Argument list too long
    if len(arguments) != 1:
        parser.print_usage(sys.stderr)
        return 7 # Argument list too long
    pkgname = arguments[0]

    manifest = get_manifest(repo, options.manifest)
    if not manifest:
        return 2 # No such file or directory
    if not options.section in manifest:
        print('Section %s is not in manifest %s.'
              % (options.section, options.manifest), file=sys.stderr)
        return 1 # Operation not permitted
    if pkgname not in manifest[options.section]:
        print('Package %s is not in section %s of manifest %s.'
              % (pkgname, options.section, options.manifest), file=sys.stderr)
        return 1 # Operation not permitted
    else:
        manifest[options.section].remove(pkgname)
        if save_manifest(repo, manifest, options.manifest,
                         overwrite_existing=True):
            print ('Removed %s from section %s of manifest %s.'
                   % (pkgname, options.section, options.manifest))
            return 0
        else:
            return 1 # Operation not permitted


def add_catalog(repo, args):
    '''Adds a catalog to a manifest.'''
    parser = MyOptionParser()
    parser.set_usage('''add-catalog CATALOGNAME --manifest MANIFESTNAME
       Adds a catalog to a manifest''')
    parser.add_option('--manifest',
                      metavar='MANIFESTNAME',
                      help='name of manifest on which to operate')
    try:
        options, arguments = parser.parse_args(args)
    except MyOptParseError as errmsg:
        print(str(errmsg), file=sys.stderr)
        return 22 # Invalid argument
    except MyOptParseExit:
        return 0

    if not options.manifest:
        if len(arguments) == 2:
            options.manifest = arguments[1]
            del arguments[1]
        else:
            parser.print_usage(sys.stderr)
            return 7 # Argument list too long
    if len(arguments) != 1:
        parser.print_usage(sys.stderr)
        return 7 # Argument list too long
    catalogname = arguments[0]

    available_catalogs = get_catalogs(repo)
    if catalogname not in available_catalogs:
        print('Unknown catalog name: %s.' % catalogname, file=sys.stderr)
        return 2 # no such file or directory

    manifest = get_manifest(repo, options.manifest)
    if not manifest:
        return 2 # no such file or directory
    if not 'catalogs' in manifest:
        manifest['catalogs'] = []
    if catalogname in manifest['catalogs']:
        print((
            'Catalog %s is already in manifest %s.'
            % (catalogname, options.manifest)), file=sys.stderr)
        return 1 # Operation not permitted
    else:
        # put it at the front of the catalog list as that is usually
        # what is wanted...
        manifest['catalogs'].insert(0, catalogname)
        if save_manifest(repo, manifest, options.manifest,
                         overwrite_existing=True):
            print ('Added %s to catalogs of manifest %s.'
                   % (catalogname, options.manifest))
            return 0
        else:
            return 1 # Operation not permitted


def remove_catalog(repo, args):
    '''Removes a catalog from a manifest.'''
    parser = MyOptionParser()
    parser.set_usage('''remove-catalog CATALOGNAME --manifest MANIFESTNAME
       Removes a catalog from a manifest''')
    parser.add_option('--manifest',
                      metavar='MANIFESTNAME',
                      help='name of manifest on which to operate')
    try:
        options, arguments = parser.parse_args(args)
    except MyOptParseError as errmsg:
        print(str(errmsg), file=sys.stderr)
        return 22 # Invalid argument
    except MyOptParseExit:
        return 0

    if not options.manifest:
        if len(arguments) == 2:
            options.manifest = arguments[1]
            del arguments[1]
        else:
            parser.print_usage(sys.stderr)
            return 7 # Argument list too long
    if len(arguments) != 1:
        parser.print_usage(sys.stderr)
        return 7 # Argument list too long
    catalogname = arguments[0]

    manifest = get_manifest(repo, options.manifest)
    if not manifest:
        return 2 # no such file or directory
    if catalogname not in manifest.get('catalogs', []):
        print((
            'Catalog %s is not in manifest %s.'
            % (catalogname, options.manifest)), file=sys.stderr)
        return 1 # Operation not permitted
    else:
        manifest['catalogs'].remove(catalogname)
        if save_manifest(repo, manifest, options.manifest,
                         overwrite_existing=True):
            print ('Removed %s from catalogs of manifest %s.'
                   % (catalogname, options.manifest))
            return 0
        else:
            return 1 # Operation not permitted


def add_included_manifest(repo, args):
    '''Adds an included manifest to a manifest.'''
    parser = MyOptionParser()
    parser.set_usage(
        '''add-included-manifest MANIFEST_TO_INCLUDE --manifest TARGET_MANIFEST
       Adds a manifest to the included_manifests of the TARGET_MANIFEST''')
    parser.add_option('--manifest',
                      metavar='TARGET_MANIFEST',
                      help='name of manifest on which to operate')
    try:
        options, arguments = parser.parse_args(args)
    except MyOptParseError as errmsg:
        print(str(errmsg), file=sys.stderr)
        return 22 # Invalid argument
    except MyOptParseExit:
        return 0

    if not options.manifest:
        if len(arguments) == 2:
            options.manifest = arguments[1]
            del arguments[1]
        else:
            parser.print_usage(sys.stderr)
            return 7 # Argument list too long
    if len(arguments) != 1:
        parser.print_usage(sys.stderr)
        return 7 # Argument list too long
    manifest_to_include = arguments[0]

    available_manifests = get_manifest_names(repo)
    if manifest_to_include not in available_manifests:
        print('Unknown manifest name: %s.' % manifest_to_include,
              file=sys.stderr)
        return 2 # no such file or directory
    if manifest_to_include == options.manifest:
        print('Can\'t include %s in itself!.' % manifest_to_include,
              file=sys.stderr)
        return 1 # Operation not permitted

    manifest = get_manifest(repo, options.manifest)
    if not manifest:
        return 2 # no such file or directory
    if not 'included_manifests' in manifest:
        manifest['included_manifests'] = []
    if manifest_to_include in manifest['included_manifests']:
        print('Manifest %s is already included in manifest %s.'
              % (manifest_to_include, options.manifest), file=sys.stderr)
        return 1 # Operation not permitted
    else:
        manifest['included_manifests'].append(manifest_to_include)
        if save_manifest(repo, manifest, options.manifest,
                         overwrite_existing=True):
            print ('Added %s to included_manifests of manifest %s.'
                   % (manifest_to_include, options.manifest))
            return 0
        else:
            return 1 # Operation not permitted


def remove_included_manifest(repo, args):
    '''Removes an included manifest from a manifest.'''
    parser = MyOptionParser()
    parser.set_usage(
        '''remove-included_manifest INCLUDED_MANIFEST --manifest TARGET_MANIFEST
       Removes a manifest from the included_manifests of TARGET_MANIFEST''')
    parser.add_option('--manifest',
                      metavar='TARGET_MANIFEST',
                      help='name of manifest on which to operate')
    try:
        options, arguments = parser.parse_args(args)
    except MyOptParseError as errmsg:
        print(str(errmsg), file=sys.stderr)
        return 22 # Invalid argument
    except MyOptParseExit:
        return 0

    if not options.manifest:
        if len(arguments) == 2:
            options.manifest = arguments[1]
            del arguments[1]
        else:
            parser.print_usage(sys.stderr)
            return 7 # Argument list too long
    if len(arguments) != 1:
        parser.print_usage(sys.stderr)
        return 7 # Argument list too long
    included_manifest = arguments[0]

    manifest = get_manifest(repo, options.manifest)
    if not manifest:
        return 2 # no such file or directory
    if included_manifest not in manifest.get('included_manifests', []):
        print('Manifest %s is not included in manifest %s.'
              % (included_manifest, options.manifest), file=sys.stderr)
        return 1 # Operation not permitted
    else:
        manifest['included_manifests'].remove(included_manifest)
        if save_manifest(repo, manifest, options.manifest,
                         overwrite_existing=True):
            print('Removed %s from included_manifests of manifest %s.'
                  % (included_manifest, options.manifest))
            return 0
        else:
            return 1 # Operation not permitted


def refresh_cache(repo, args):
    '''Refreshes the repo data if changes were made while manifestutil was
    running. Updates manifests, catalogs, and packages.'''
    parser = MyOptionParser()
    parser.set_usage('''refresh-cache
        Refreshes the repo data''')
    try:
        _, arguments = parser.parse_args(args)
    except MyOptParseError as errmsg:
        print(str(errmsg), file=sys.stderr)
        return 22 # Invalid argument
    except MyOptParseExit:
        return 0

    if len(arguments) != 0:
        parser.print_usage(sys.stderr)
        return 22 # Invalid argument
    CMD_ARG_DICT['manifests'] = get_manifest_names(repo)
    CMD_ARG_DICT['catalogs'] = get_catalogs(repo)
    CMD_ARG_DICT['pkgs'] = get_installer_item_names(
        repo, CMD_ARG_DICT['catalogs'])

##### end subcommand functions

def show_help():
    '''Prints available subcommands'''
    print("Available sub-commands:")
    subcommands = list(CMD_ARG_DICT['cmds'].keys())
    subcommands.sort()
    for item in subcommands:
        print('\t%s' % item)
    return 0


def tab_completer(text, state):
    '''Called by the readline lib to calculate possible completions'''
    array_to_match = None
    if readline.get_begidx() == 0:
        # since we are at the start of the line
        # we are matching commands
        array_to_match = 'cmds'
        match_list = list(CMD_ARG_DICT.get('cmds', {}).keys())
    else:
        # we are matching args
        cmd_line = readline.get_line_buffer()[0:readline.get_begidx()]
        cmd = shlex.split(cmd_line)[-1]
        array_to_match = CMD_ARG_DICT.get('cmds', {}).get(cmd)
        if array_to_match:
            match_list = CMD_ARG_DICT[array_to_match]
        else:
            array_to_match = CMD_ARG_DICT.get('options', {}).get(cmd)
            if array_to_match:
                match_list = CMD_ARG_DICT[array_to_match]
            else:
                array_to_match = 'options'
                match_list = list(CMD_ARG_DICT.get('options', {}).keys())

    matches = [item for item in match_list
               if item.upper().startswith(text.upper())]
    try:
        return matches[state]
    except IndexError:
        return None


def set_up_tab_completer():
    '''Starts our tab-completer when running interactively'''
    readline.set_completer(tab_completer)
    if libedit:
        # readline module was compiled against libedit
        readline.parse_and_bind("bind ^I rl_complete")
    else:
        readline.parse_and_bind("tab: complete")


def handle_subcommand(repo, args):
    '''Does all our subcommands'''
    # check if any arguments are passed.
    # if not, list subcommands.
    if len(args) < 1:
        show_help()
        return 2

    # strip leading hyphens and
    # replace embedded hyphens with underscores
    # so '--add-pkg' becomes 'add_pkg'
    # and 'new-manifest' becomes 'new_manifest'
    subcommand = args[0].lstrip('-').replace('-', '_')

    # special case the exit command
    if subcommand == 'exit':
        cleanup_and_exit(0)

    if (subcommand not in ['version', 'configure', 'help']
            and '-h' not in args and '--help' not in args):
        if not repo:
            repo = connect_to_repo()

    try:
        # find function to call by looking in the global name table
        # for a function with a name matching the subcommand
        subcommand_function = globals()[subcommand]
    except MyOptParseExit:
        return 0
    except (TypeError, KeyError):
        print('Unknown subcommand: %s' % args[0], file=sys.stderr)
        show_help()
        return 2
    else:
        return subcommand_function(repo, args[1:])


def connect_to_repo():
    '''Connects to the Munki repo'''
    repo_url = pref('repo_url')
    repo_path = pref('repo_path')
    repo_plugin = pref('plugin')
    if not repo_url and repo_path:
        repo_url = path2url(repo_path)
    if not repo_url:
        print((
            u'No repo URL defined. Run %s --configure to define one.'
            % os.path.basename(__file__)), file=sys.stderr)
        exit(-1)
    try:
        repo = munkirepo.connect(repo_url, repo_plugin)
    except munkirepo.RepoError as err:
        print(u'Repo error: %s' % err, file=sys.stderr)
        exit(-1)
    return repo


CMD_ARG_DICT = {}

def main():
    '''Our main routine'''
    repo = None

    cmds = {'add-pkg':                   'pkgs',
            'add-catalog':               'catalogs',
            'add-included-manifest':     'manifests',
            'remove-pkg':                'pkgs',
            'move-install-to-uninstall': 'pkgs',
            'remove-catalog':            'catalogs',
            'remove-included-manifest':  'manifests',
            'list-manifests':            'manifests',
            'list-catalogs':             'default',
            'list-catalog-items':        'catalogs',
            'display-manifest':          'manifests',
            'expand-included-manifests': 'manifests',
            'find':                      'default',
            'new-manifest':              'default',
            'copy-manifest':             'manifests',
            'rename-manifest':           'manifests',
            'refresh-cache':             'default',
            'exit':                      'default',
            'help':                      'default',
            'configure':                 'default',
            'version':                   'default'
           }
    CMD_ARG_DICT['cmds'] = cmds

    if len(sys.argv) > 1:
        # some commands or options were passed at the command line
        cmd = sys.argv[1].lstrip('-')
        retcode = handle_subcommand(repo, sys.argv[1:])
        cleanup_and_exit(retcode)
    else:
        # if we get here, no options or commands,
        # so let's enter interactive mode
        # must have an available repo for interactive mode
        repo = connect_to_repo()
        # build the rest of our dict to enable tab completion
        CMD_ARG_DICT['options'] = {'--manifest': 'manifests',
                                   '--section':  'sections'}

        CMD_ARG_DICT['default'] = []
        CMD_ARG_DICT['sections'] = get_manifest_pkg_sections()
        CMD_ARG_DICT['manifests'] = get_manifest_names(repo)
        CMD_ARG_DICT['catalogs'] = get_catalogs(repo)
        CMD_ARG_DICT['pkgs'] = get_installer_item_names(
            repo, CMD_ARG_DICT['catalogs'])

        set_up_tab_completer()
        print('Entering interactive mode... (type "help" for commands)')
        while 1:
            try:
                cmd = get_input('> ')
            except (KeyboardInterrupt, EOFError):
                # React to Control-C and Control-D
                print() # so we finish off the raw_input line
                cleanup_and_exit(0)
            args = shlex.split(cmd)
            handle_subcommand(repo, args)

if __name__ == '__main__':
    main()