mirror/munki
1
0
Fork 0
mirror of https://github.com/munki/munki.git synced 2026-01-06 06:29:56 -06:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'Munki3dev' into swiftapps

Browse Source
This commit is contained in:
Greg Neagle
2018-11-16 11:08:32 -08:00
parent 33dbdfb3a6 67bfb482f0
commit b3e33b34ac
5 changed files with 146 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
code/client/munkilib/updatecheck/core.py
View File

@@ -30,7 +30,6 @@ from . import catalogs
from . import download
from . import licensing
from . import manifestutils
from . import precaching
from .. import display
from .. import info
@@ -483,7 +482,7 @@ def check(client_id='', localmanifestpath=None):
# start our precaching agent
# note -- this must happen _after_ InstallInfo.plist gets written to disk.
precaching.run_agent()
download.run_precaching_agent()
if installcount or removalcount:
return 1

164
code/client/munkilib/updatecheck/download.py
View File

@@ -29,6 +29,7 @@ import urlparse
from .. import display
from .. import fetch
from .. import info
from .. import launchd
from .. import munkihash
from .. import osutils
from .. import prefs
@@ -38,12 +39,27 @@ from .. import FoundationPlist
ICON_HASHES_PLIST_NAME = '_icon_hashes.plist'
def get_url_basename(url):
"""For a URL, absolute or relative, return the basename string.
def enough_disk_space(item_pl,
installlist=None,
uninstalling=False,
warn=True,
precaching=False):
e.g. "http://foo/bar/path/foo.dmg" => "foo.dmg"
"/path/foo.dmg" => "foo.dmg"
"""
url_parse = urlparse.urlparse(url)
return os.path.basename(url_parse.path)
def get_download_cache_path(url):
"""For a URL, return the path that the download should cache to.
Returns a string."""
cachedir = os.path.join(prefs.pref('ManagedInstallDir'), 'Cache')
return os.path.join(cachedir, get_url_basename(url))
def enough_disk_space(item_pl, installlist=None,
uninstalling=False, warn=True, precaching=False):
"""Determine if there is enough disk space to download the installer
item."""
# fudgefactor is set to 100MB
@@ -68,12 +84,19 @@ def enough_disk_space(item_pl,
for item in installlist:
# subtract space needed for other items that are to be installed
if item.get('installer_item'):
availablediskspace = availablediskspace - \
int(item.get('installed_size', 0))
availablediskspace = (availablediskspace -
int(item.get('installed_size', 0)))
if availablediskspace > diskspaceneeded:
if diskspaceneeded > availablediskspace and not precaching:
# try to clear space by deleting some precached items
uncache(diskspaceneeded - availablediskspace)
availablediskspace = info.available_disk_space()
if availablediskspace >= diskspaceneeded:
return True
elif warn:
# we don't have enough space
if warn:
if uninstalling:
display.display_warning('There is insufficient disk space to '
'download the uninstaller for %s.',
@@ -108,9 +131,7 @@ def get_download_cache_path(url):
def download_installeritem(item_pl,
installinfo,
uninstalling=False,
precaching=False):
installinfo, uninstalling=False, precaching=False):
"""Downloads an (un)installer item.
Returns True if the item was downloaded, False if it was already cached.
Raises an error if there are issues..."""
@@ -152,14 +173,10 @@ def download_installeritem(item_pl,
if not os.path.exists(destinationpath):
# check to see if there is enough free space to download and install
if not enough_disk_space(item_pl,
if not enough_disk_space(item_pl,
installinfo['managed_installs'],
uninstalling=uninstalling,
precaching=precaching):
if not precaching:
# see if there are any precached items we can delete to free
# up space
pass
raise fetch.DownloadError(
'Insufficient disk space to download and install %s' % pkgname)
else:
@@ -191,7 +208,7 @@ def clean_up_icons_dir(icons_to_keep):
os.unlink(icon_path)
except (IOError, OSError):
pass
if len(osutils.listdir(dirpath)) == 0:
if osutils.listdir(dirpath):
# did we empty out this directory (or is it already empty)?
# if so, remove it
try:
@@ -266,14 +283,11 @@ def download_icons(item_list):
try:
fetch.munki_resource(
icon_url, icon_path, message=message)
fetch.writeCachedChecksum(icon_path)
except fetch.Error, err:
display.display_debug1(
'Could not retrieve icon %s from the server: %s',
'Error when retrieving icon %s from the server: %s',
icon_name, err)
else:
# if we downloaded it, store the hash for later use
if os.path.isfile(icon_path):
fetch.writeCachedChecksum(icon_path)
# delete any previously downloaded icons we no longer need
clean_up_icons_dir(icons_to_keep)
@@ -359,5 +373,109 @@ def download_catalog(catalogname):
return None
### precaching support ###
def _installinfo():
'''Get the install info from InstallInfo.plist'''
managed_install_dir = prefs.pref('ManagedInstallDir')
install_info_plist = os.path.join(managed_install_dir, 'InstallInfo.plist')
try:
return FoundationPlist.readPlist(install_info_plist)
except FoundationPlist.FoundationPlistException:
return {}
def _items_to_precache(install_info):
'''Returns a list of items from InstallInfo.plist's optional_installs
that have precache=True and installed=False'''
optional_install_items = install_info.get('optional_installs', [])
precache_items = [item for item in optional_install_items
if item.get('precache') and not item.get('installed')]
return precache_items
def cache():
'''Download any applicable precache items into our Cache folder'''
install_info = _installinfo()
for item in _items_to_precache(install_info):
try:
download_installeritem(item, install_info, precaching=True)
except fetch.Error, err:
display.display_warning(
'Failed to precache the installer for %s because %s',
item['name'], unicode(err))
def uncache(space_needed_in_kb):
'''Discard precached items to free up space for managed installs'''
install_info = _installinfo()
precached_items = [
[os.path.basename(item['installer_item_location'])]
for item in _items_to_precache(install_info)
if item.get('installer_item_location')]
if not precached_items:
return
cachedir = os.path.join(prefs.pref('ManagedInstallDir'), 'Cache')
precached_size = 0
for item in precached_items:
# item is [itemname]
item_path = os.path.join(cachedir, item[0])
itemsize = int(os.path.getsize(item_path)/1024)
precached_size += itemsize
item.append(itemsize)
# item is now [itemname, itemsize]
if precached_size < space_needed_in_kb:
# we can't clear enough space, so don't bother removing anything.
# otherwise we'll clear some space, but still can't download the large
# managed install, but then we'll have enough space to redownload the
# precachable items and so we will (and possibly do this over and
# over -- delete some, redownload, delete some, redownload...)
return
# sort reversed by size; smallest at end
precached_items.sort(key=lambda x: x[1], reverse=True)
deleted_kb = 0
while precached_items:
if deleted_kb >= space_needed_in_kb:
break
# remove and return last item in precached_items
# we delete the smallest item first, proceeeding until we've freed up
# enough space or deleted all the items
item = precached_items.pop()
item_path = os.path.join(cachedir, item[0])
item_size = item[1]
try:
os.remove(item_path)
deleted_kb += item_size
except OSError, err:
display.display_error(
"Could not remove precached item %s: %s" % (item_path, err))
def run_precaching_agent():
'''Kick off a run of our precaching agent, which allows the precaching to
run in the background after a normal Munki run'''
parent_dir = (
os.path.dirname(
os.path.dirname(
os.path.dirname(
os.path.abspath(__file__)))))
precache_agent_path = os.path.join(parent_dir, 'precache_agent')
if not os.path.exists(precache_agent_path):
# try absolute path in Munki's normal install dir
precache_agent_path = '/usr/local/munki/precache_agent'
if os.path.exists(precache_agent_path):
try:
job = launchd.Job([precache_agent_path], cleanup_at_exit=False)
job.start()
except launchd.LaunchdJobException as err:
display.display_error(
'Error with launchd job (%s): %s', precache_agent_path, err)
else:
display.display_error("Could not find precache_agent")
if __name__ == '__main__':
print 'This is a library of support tools for the Munki Suite.'

91
code/client/munkilib/updatecheck/precaching.py
View File

@@ -1,91 +0,0 @@
#!/usr/bin/python
# encoding: utf-8
#
# Copyright 2018 Greg Neagle.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
precaching
Created by Greg Neagle on 2018-05-06.
Module for precaching optional installs with installed=False and precache=True.
"""
import os
import sys
from . import download
from .. import FoundationPlist
from .. import display
from .. import fetch
from .. import launchd
from .. import prefs
def _installinfo():
'''Get the install info from InstallInfo.plist'''
managed_install_dir = prefs.pref('ManagedInstallDir')
install_info_plist = os.path.join(managed_install_dir, 'InstallInfo.plist')
try:
return FoundationPlist.readPlist(install_info_plist)
except FoundationPlist.FoundationPlistException:
return {}
def items_to_precache(install_info):
'''Returns a list of items from InstallInfo.plist's optional_installs
that have precache=True and installed=False'''
optional_install_items = install_info.get('optional_installs', [])
precache_items = [item for item in optional_install_items
if item.get('precache') and not item.get('installed')]
return precache_items
def cache():
'''Download any applicable precache items into our Cache folder'''
install_info = _installinfo()
for item in items_to_precache(install_info):
try:
download.download_installeritem(item, install_info, precaching=True)
except fetch.Error, err:
display.display_warning(
'Failed to precache the installer for %s because %s',
item['name'], unicode(err))
def run_agent():
'''Kick off a run of our precaching agent, which allows the precaching to
run in the background after a normal Munki run'''
parent_dir = (
os.path.dirname(
os.path.dirname(
os.path.dirname(
os.path.abspath(__file__)))))
precache_agent_path = os.path.join(parent_dir, 'precache_agent')
if not os.path.exists(precache_agent_path):
# try absolute path in Munki's normal install dir
precache_agent_path = '/usr/local/munki/precache_agent'
if os.path.exists(precache_agent_path):
try:
job = launchd.Job([precache_agent_path] , cleanup_at_exit=False)
job.start()
except launchd.LaunchdJobException as err:
display.display_error(
'Error with launchd job (%s): %s', precache_agent_path, err)
else:
display.display_error("Could not find precache_agent")
if __name__ == '__main__':
print 'This is a library of support tools for the Munki Suite.'

5
code/client/precache_agent
View File

@@ -23,8 +23,9 @@ A privileged agent that downloads optional installs items marked for precaching.
"""
import time
from munkilib.updatecheck import precaching
from munkilib.updatecheck import download
if __name__ == '__main__':
precaching.cache()
download.cache()
# sleep 10 seconds to prevent launchd from complaining
time.sleep(10)