From f1af30de4901d2bc023f748083182c89fae7d902 Mon Sep 17 00:00:00 2001 From: David Symons Date: Mon, 22 Jun 2015 12:40:19 +1000 Subject: [PATCH 1/9] Update getmanifest to create unique filename for local copy of manifest --- code/client/munkilib/updatecheck.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/code/client/munkilib/updatecheck.py b/code/client/munkilib/updatecheck.py index 91dfc2cb..94bd1da4 100755 --- a/code/client/munkilib/updatecheck.py +++ b/code/client/munkilib/updatecheck.py @@ -2572,7 +2572,10 @@ def getmanifest(partialurl, suppress_errors=False): else: # request for nested manifest manifestdisplayname = partialurl - manifestname = os.path.split(partialurl)[1] + + valid_chars = "-_.() %s%s" % (string.ascii_letters, string.digits) + manifestname = ''.join(c if c in valid_chars else "_" for c in partialurl) + manifesturl = manifestbaseurl + urllib2.quote(partialurl) if manifestname in MANIFESTS: From aecf0191a85a564e033b0b4b7abf9fff2fc2f87d Mon Sep 17 00:00:00 2001 From: David Symons Date: Tue, 23 Jun 2015 13:14:22 +1000 Subject: [PATCH 2/9] Updated getmanifest to copy the folder structure on the server when storing client-side manifests --- code/client/munkilib/updatecheck.py | 50 ++++++++++++++++++++++++++--- 1 file changed, 46 insertions(+), 4 deletions(-) diff --git a/code/client/munkilib/updatecheck.py b/code/client/munkilib/updatecheck.py index 94bd1da4..e9d69869 100755 --- a/code/client/munkilib/updatecheck.py +++ b/code/client/munkilib/updatecheck.py @@ -2553,6 +2553,42 @@ def getmanifest(partialurl, suppress_errors=False): string local path to the downloaded manifest. """ #global MANIFESTS + + def walkManifestCache(path): + """Walks MANIFESTS to the specified path, creating any intermediate + dictionaries if they don't exist. + + Returns: + The dictionary at the specified path. + """ + branch = MANIFESTS + + if path: + for part in path.split('/'): + branch = branch.setdefault(part, { }) + + return branch + + def addToManifestCache(path, value): + """Sets the item at the specified path to the given value. + """ + parts = os.path.split(path); + + folder = walkManifestCache(parts[0]) + folder[parts[1]] = value + + + def getFromManifestCache(path): + """Retrieves the item at the specified path. + + Returns: + The item at path in MANIFESTS + """ + parts = os.path.split(path); + + folder = walkManifestCache(parts[0]) + return folder.get(parts[1], None) + manifestbaseurl = (munkicommon.pref('ManifestURL') or munkicommon.pref('SoftwareRepoURL') + '/manifests/') if (not manifestbaseurl.endswith('?') and @@ -2572,10 +2608,7 @@ def getmanifest(partialurl, suppress_errors=False): else: # request for nested manifest manifestdisplayname = partialurl - - valid_chars = "-_.() %s%s" % (string.ascii_letters, string.digits) - manifestname = ''.join(c if c in valid_chars else "_" for c in partialurl) - + manifestname = partialurl manifesturl = manifestbaseurl + urllib2.quote(partialurl) if manifestname in MANIFESTS: @@ -2584,6 +2617,15 @@ def getmanifest(partialurl, suppress_errors=False): munkicommon.display_debug2('Manifest base URL is: %s', manifestbaseurl) munkicommon.display_detail('Getting manifest %s...', manifestdisplayname) manifestpath = os.path.join(manifest_dir, manifestname) + manifestdir = os.path.dirname(manifestpath) + + # Create the folder the manifest shall be stored in + try: + os.makedirs(manifestdir) + except OSError: + if not os.path.isdir(manifestdir): + raise + message = 'Retrieving list of software for this machine...' try: dummy_value = getResourceIfChangedAtomically( From 96b80d7ce1c3f1883a3c0a0e5131b2b2d04d54f3 Mon Sep 17 00:00:00 2001 From: David Symons Date: Tue, 23 Jun 2015 13:42:36 +1000 Subject: [PATCH 3/9] Forgot to use addToManifestCache and getFromManifestCache... --- code/client/munkilib/updatecheck.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/code/client/munkilib/updatecheck.py b/code/client/munkilib/updatecheck.py index e9d69869..6a952601 100755 --- a/code/client/munkilib/updatecheck.py +++ b/code/client/munkilib/updatecheck.py @@ -2611,8 +2611,8 @@ def getmanifest(partialurl, suppress_errors=False): manifestname = partialurl manifesturl = manifestbaseurl + urllib2.quote(partialurl) - if manifestname in MANIFESTS: - return MANIFESTS[manifestname] + if getFromManifestCache(manifestname) + return getFromManifestCache(manifestname) munkicommon.display_debug2('Manifest base URL is: %s', manifestbaseurl) munkicommon.display_detail('Getting manifest %s...', manifestdisplayname) @@ -2650,7 +2650,7 @@ def getmanifest(partialurl, suppress_errors=False): raise ManifestException(errormsg) else: # plist is valid - MANIFESTS[manifestname] = manifestpath + addToManifestCache(manifestname, manifestpath) return manifestpath From d7512980f572215c15b56930016bab5969c71803 Mon Sep 17 00:00:00 2001 From: David Symons Date: Tue, 23 Jun 2015 16:02:22 +1000 Subject: [PATCH 4/9] Flattened MANIFESTS, removing unnecessary sub-dictionaries. Also updated variable names to be less confusing --- code/client/munkilib/updatecheck.py | 54 +++++------------------------ 1 file changed, 9 insertions(+), 45 deletions(-) diff --git a/code/client/munkilib/updatecheck.py b/code/client/munkilib/updatecheck.py index 6a952601..8e747d9e 100755 --- a/code/client/munkilib/updatecheck.py +++ b/code/client/munkilib/updatecheck.py @@ -2553,48 +2553,12 @@ def getmanifest(partialurl, suppress_errors=False): string local path to the downloaded manifest. """ #global MANIFESTS - - def walkManifestCache(path): - """Walks MANIFESTS to the specified path, creating any intermediate - dictionaries if they don't exist. - - Returns: - The dictionary at the specified path. - """ - branch = MANIFESTS - - if path: - for part in path.split('/'): - branch = branch.setdefault(part, { }) - - return branch - - def addToManifestCache(path, value): - """Sets the item at the specified path to the given value. - """ - parts = os.path.split(path); - - folder = walkManifestCache(parts[0]) - folder[parts[1]] = value - - - def getFromManifestCache(path): - """Retrieves the item at the specified path. - - Returns: - The item at path in MANIFESTS - """ - parts = os.path.split(path); - - folder = walkManifestCache(parts[0]) - return folder.get(parts[1], None) - manifestbaseurl = (munkicommon.pref('ManifestURL') or munkicommon.pref('SoftwareRepoURL') + '/manifests/') if (not manifestbaseurl.endswith('?') and not manifestbaseurl.endswith('/')): manifestbaseurl = manifestbaseurl + '/' - manifest_dir = os.path.join(munkicommon.pref('ManagedInstallDir'), + manifestrootdir = os.path.join(munkicommon.pref('ManagedInstallDir'), 'manifests') if (partialurl.startswith('http://') or @@ -2611,19 +2575,19 @@ def getmanifest(partialurl, suppress_errors=False): manifestname = partialurl manifesturl = manifestbaseurl + urllib2.quote(partialurl) - if getFromManifestCache(manifestname) - return getFromManifestCache(manifestname) + if manifestname in MANIFESTS: + return MANIFESTS[manifestname] munkicommon.display_debug2('Manifest base URL is: %s', manifestbaseurl) munkicommon.display_detail('Getting manifest %s...', manifestdisplayname) - manifestpath = os.path.join(manifest_dir, manifestname) - manifestdir = os.path.dirname(manifestpath) - + manifestpath = os.path.join(manifestrootdir, manifestname) + # Create the folder the manifest shall be stored in + manifestdestdir = os.path.dirname(manifestpath) try: - os.makedirs(manifestdir) + os.makedirs(manifestdestdir) except OSError: - if not os.path.isdir(manifestdir): + if not os.path.isdir(manifestdestdir): raise message = 'Retrieving list of software for this machine...' @@ -2650,7 +2614,7 @@ def getmanifest(partialurl, suppress_errors=False): raise ManifestException(errormsg) else: # plist is valid - addToManifestCache(manifestname, manifestpath) + MANIFESTS[manifestname] = manifestpath return manifestpath From b1840af0acc35af0439fa981854a66f0e2c7a374 Mon Sep 17 00:00:00 2001 From: David Symons Date: Tue, 23 Jun 2015 16:43:24 +1000 Subject: [PATCH 5/9] Added cleanUpManifests to check over all manifests and remove those which are no longer used --- code/client/munkilib/updatecheck.py | 40 ++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 6 deletions(-) diff --git a/code/client/munkilib/updatecheck.py b/code/client/munkilib/updatecheck.py index 8e747d9e..764a5834 100755 --- a/code/client/munkilib/updatecheck.py +++ b/code/client/munkilib/updatecheck.py @@ -2558,7 +2558,7 @@ def getmanifest(partialurl, suppress_errors=False): if (not manifestbaseurl.endswith('?') and not manifestbaseurl.endswith('/')): manifestbaseurl = manifestbaseurl + '/' - manifestrootdir = os.path.join(munkicommon.pref('ManagedInstallDir'), + manifest_dir = os.path.join(munkicommon.pref('ManagedInstallDir'), 'manifests') if (partialurl.startswith('http://') or @@ -2580,14 +2580,14 @@ def getmanifest(partialurl, suppress_errors=False): munkicommon.display_debug2('Manifest base URL is: %s', manifestbaseurl) munkicommon.display_detail('Getting manifest %s...', manifestdisplayname) - manifestpath = os.path.join(manifestrootdir, manifestname) - + manifestpath = os.path.join(manifest_dir, manifestname) + # Create the folder the manifest shall be stored in - manifestdestdir = os.path.dirname(manifestpath) + destinationdir = os.path.dirname(manifestpath) try: - os.makedirs(manifestdestdir) + os.makedirs(destinationdir) except OSError: - if not os.path.isdir(manifestdestdir): + if not os.path.isdir(destinationdir): raise message = 'Retrieving list of software for this machine...' @@ -2617,6 +2617,31 @@ def getmanifest(partialurl, suppress_errors=False): MANIFESTS[manifestname] = manifestpath return manifestpath +def cleanUpManifests(subdir=""): + """Removes any manifest files that are no longer in use by this client""" + manifest_dir = os.path.join(munkicommon.pref('ManagedInstallDir'), + 'manifests') + + # For recursive calls (checking subfolders), append subdir to root manifest directory + if subdir: + manifest_dir = os.path.join(manifest_dir, subdir) + + for item in os.listdir(manifest_dir): + cachename = os.path.join(subdir, item) + if cachename not in MANIFESTS.keys(): + path = os.path.join(manifest_dir, item) + if os.path.isdir(path): + cleanUpManifests(cachename) + else: + os.unlink(path) + + # Remove folder is empty + try: + os.rmdir(manifest_dir) + except OSError: + pass + + def getPrimaryManifest(alternate_id): """Gets the client manifest from the server.""" @@ -3140,6 +3165,9 @@ def check(client_id='', localmanifestpath=None): # clean up catalogs directory cleanUpCatalogs() + # clean up manifests directory + cleanUpManifests() + # clean up cache dir # remove any item in the cache that isn't scheduled # to be used for an install or removal From d041d90b6f86f583968593bf6c9ee7aad956c3c2 Mon Sep 17 00:00:00 2001 From: David Symons Date: Tue, 23 Jun 2015 16:46:58 +1000 Subject: [PATCH 6/9] Fixing spelling mistake --- code/client/munkilib/updatecheck.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/client/munkilib/updatecheck.py b/code/client/munkilib/updatecheck.py index 764a5834..7580328d 100755 --- a/code/client/munkilib/updatecheck.py +++ b/code/client/munkilib/updatecheck.py @@ -2635,7 +2635,7 @@ def cleanUpManifests(subdir=""): else: os.unlink(path) - # Remove folder is empty + # Remove folder if empty try: os.rmdir(manifest_dir) except OSError: From 495873e7a12d46ab22c46947ae5219ae95efe45a Mon Sep 17 00:00:00 2001 From: David Symons Date: Tue, 23 Jun 2015 16:58:15 +1000 Subject: [PATCH 7/9] Added exception for SelfServeManifest to cleanUpManifests --- code/client/munkilib/updatecheck.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/code/client/munkilib/updatecheck.py b/code/client/munkilib/updatecheck.py index 7580328d..07f6e675 100755 --- a/code/client/munkilib/updatecheck.py +++ b/code/client/munkilib/updatecheck.py @@ -2622,12 +2622,20 @@ def cleanUpManifests(subdir=""): manifest_dir = os.path.join(munkicommon.pref('ManagedInstallDir'), 'manifests') + exceptions = [ + "SelfServeManifest" + ] + # For recursive calls (checking subfolders), append subdir to root manifest directory if subdir: manifest_dir = os.path.join(manifest_dir, subdir) for item in os.listdir(manifest_dir): cachename = os.path.join(subdir, item) + + if cachename in exceptions: + continue + if cachename not in MANIFESTS.keys(): path = os.path.join(manifest_dir, item) if os.path.isdir(path): From cb5117b18de89d0ac5d0b9d6264796d80d1cc8b7 Mon Sep 17 00:00:00 2001 From: David Symons Date: Thu, 25 Jun 2015 14:30:13 +1000 Subject: [PATCH 8/9] Reworked cleanUpManifests to use os.walk instead of recursion, cleaning it up. Added better exception handling to manifest folder creation in getmanifest --- code/client/munkilib/updatecheck.py | 46 +++++++++++++++-------------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/code/client/munkilib/updatecheck.py b/code/client/munkilib/updatecheck.py index 07f6e675..5eb3f888 100755 --- a/code/client/munkilib/updatecheck.py +++ b/code/client/munkilib/updatecheck.py @@ -2586,9 +2586,16 @@ def getmanifest(partialurl, suppress_errors=False): destinationdir = os.path.dirname(manifestpath) try: os.makedirs(destinationdir) - except OSError: + except OSError as e: + # OSError will be raised if destinationdir exists, ignore this case if not os.path.isdir(destinationdir): - raise + if not suppress_errors: + munkicommon.display_error( + 'Could not create folder to store manifest %s: %s', + manifestdisplayname, e + ) + + return None message = 'Retrieving list of software for this machine...' try: @@ -2617,7 +2624,7 @@ def getmanifest(partialurl, suppress_errors=False): MANIFESTS[manifestname] = manifestpath return manifestpath -def cleanUpManifests(subdir=""): +def cleanUpManifests(): """Removes any manifest files that are no longer in use by this client""" manifest_dir = os.path.join(munkicommon.pref('ManagedInstallDir'), 'manifests') @@ -2626,28 +2633,23 @@ def cleanUpManifests(subdir=""): "SelfServeManifest" ] - # For recursive calls (checking subfolders), append subdir to root manifest directory - if subdir: - manifest_dir = os.path.join(manifest_dir, subdir) + for (dirpath, dirnames, filenames) in os.walk(manifest_dir, topdown=False): + for name in filenames: + abs_path = os.path.join(dirpath, name) + rel_path = abs_path[len(manifest_dir):].lstrip("/") - for item in os.listdir(manifest_dir): - cachename = os.path.join(subdir, item) + if rel_path in exceptions: + continue - if cachename in exceptions: - continue - - if cachename not in MANIFESTS.keys(): - path = os.path.join(manifest_dir, item) - if os.path.isdir(path): - cleanUpManifests(cachename) - else: - os.unlink(path) + if rel_path not in MANIFESTS.keys(): + os.unlink(abs_path) - # Remove folder if empty - try: - os.rmdir(manifest_dir) - except OSError: - pass + # Try and remove the directory (rmdir will fail if directory is not empty) + try: + if dirpath != manifest_dir: + os.rmdir(dirpath) + except OSError: + pass From 3e4935ec9bf6c6e615cb16ce913e011e8223bb0b Mon Sep 17 00:00:00 2001 From: David Symons Date: Thu, 25 Jun 2015 14:38:04 +1000 Subject: [PATCH 9/9] Changed filename checking in cleanUpManifests to ignore all files named 'SelfServeManifest' --- code/client/munkilib/updatecheck.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/code/client/munkilib/updatecheck.py b/code/client/munkilib/updatecheck.py index 5eb3f888..cf4baa86 100755 --- a/code/client/munkilib/updatecheck.py +++ b/code/client/munkilib/updatecheck.py @@ -2635,12 +2635,13 @@ def cleanUpManifests(): for (dirpath, dirnames, filenames) in os.walk(manifest_dir, topdown=False): for name in filenames: + + if name in exceptions: + continue + abs_path = os.path.join(dirpath, name) rel_path = abs_path[len(manifest_dir):].lstrip("/") - if rel_path in exceptions: - continue - if rel_path not in MANIFESTS.keys(): os.unlink(abs_path)