Files
munki/code/client/munkilib/gurl.py
2020-01-01 08:53:37 -08:00

704 lines
29 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
# encoding: utf-8
#
# Copyright 2009-2020 Greg Neagle.
#
# Licensed under the Apache License, Version 2.0 (the 'License');
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an 'AS IS' BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
gurl.py
Created by Greg Neagle on 2013-11-21.
Modified in Feb 2016 to add support for NSURLSession.
Updated June 2019 for compatibility with Python 3 and PyObjC 5.1.2+
curl replacement using NSURLConnection and friends
Tested with PyObjC 2.5.1 (inlcuded with macOS)
and with PyObjC 5.2b1. Should also work with PyObjC 5.1.2.
May fail with other versions of PyObjC due to issues with completion handler
signatures.
"""
from __future__ import absolute_import, print_function
import os
import xattr
try:
# Python 2
from urlparse import urlparse
except ImportError:
# Python 3
from urllib.parse import urlparse
# builtin super doesn't work with Cocoa classes in recent PyObjC releases.
# pylint: disable=redefined-builtin,no-name-in-module
from objc import super
# pylint: enable=redefined-builtin,no-name-in-module
# PyLint cannot properly find names inside Cocoa libraries, so issues bogus
# No name 'Foo' in module 'Bar' warnings. Disable them.
# pylint: disable=E0611
from Foundation import (NSBundle, NSRunLoop, NSData, NSDate,
NSObject, NSURL, NSURLConnection,
NSMutableURLRequest,
NSURLRequestReloadIgnoringLocalCacheData,
NSURLResponseUnknownLength,
NSLog,
NSURLCredential, NSURLCredentialPersistenceNone,
NSPropertyListSerialization,
NSPropertyListMutableContainersAndLeaves,
NSPropertyListXMLFormat_v1_0)
try:
from Foundation import NSURLSession, NSURLSessionConfiguration
from CFNetwork import (kCFNetworkProxiesHTTPSEnable,
kCFNetworkProxiesHTTPEnable)
NSURLSESSION_AVAILABLE = True
except ImportError:
NSURLSESSION_AVAILABLE = False
# Disable PyLint complaining about 'invalid' names
# pylint: disable=C0103
if NSURLSESSION_AVAILABLE:
# NSURLSessionAuthChallengeDisposition enum constants
NSURLSessionAuthChallengeUseCredential = 0
NSURLSessionAuthChallengePerformDefaultHandling = 1
NSURLSessionAuthChallengeCancelAuthenticationChallenge = 2
NSURLSessionAuthChallengeRejectProtectionSpace = 3
# NSURLSessionResponseDisposition enum constants
NSURLSessionResponseCancel = 0
NSURLSessionResponseAllow = 1
NSURLSessionResponseBecomeDownload = 2
# TLS/SSLProtocol enum constants
kSSLProtocolUnknown = 0
kSSLProtocol3 = 2
kTLSProtocol1 = 4
kTLSProtocol11 = 7
kTLSProtocol12 = 8
kDTLSProtocol1 = 9
# define a helper function for block callbacks
import ctypes
import objc
CALLBACK_HELPER_AVAILABLE = True
try:
_objc_so = ctypes.cdll.LoadLibrary(
os.path.join(objc.__path__[0], '_objc.so'))
except OSError:
# could not load _objc.so
CALLBACK_HELPER_AVAILABLE = False
else:
PyObjCMethodSignature_WithMetaData = (
_objc_so.PyObjCMethodSignature_WithMetaData)
PyObjCMethodSignature_WithMetaData.restype = ctypes.py_object
def objc_method_signature(signature_str):
'''Return a PyObjCMethodSignature given a call signature in string
format'''
return PyObjCMethodSignature_WithMetaData(
ctypes.create_string_buffer(signature_str), None, False)
# pylint: enable=E0611
# disturbing hack warning!
# this works around an issue with App Transport Security on 10.11
bundle = NSBundle.mainBundle()
info = bundle.localizedInfoDictionary() or bundle.infoDictionary()
info['NSAppTransportSecurity'] = {'NSAllowsArbitraryLoads': True}
def NSLogWrapper(message):
'''A wrapper function for NSLog to prevent format string errors'''
NSLog('%@', message)
ssl_error_codes = {
-9800: u'SSL protocol error',
-9801: u'Cipher Suite negotiation failure',
-9802: u'Fatal alert',
-9803: u'I/O would block (not fatal)',
-9804: u'Attempt to restore an unknown session',
-9805: u'Connection closed gracefully',
-9806: u'Connection closed via error',
-9807: u'Invalid certificate chain',
-9808: u'Bad certificate format',
-9809: u'Underlying cryptographic error',
-9810: u'Internal error',
-9811: u'Module attach failure',
-9812: u'Valid cert chain, untrusted root',
-9813: u'Cert chain not verified by root',
-9814: u'Chain had an expired cert',
-9815: u'Chain had a cert not yet valid',
-9816: u'Server closed session with no notification',
-9817: u'Insufficient buffer provided',
-9818: u'Bad SSLCipherSuite',
-9819: u'Unexpected message received',
-9820: u'Bad MAC',
-9821: u'Decryption failed',
-9822: u'Record overflow',
-9823: u'Decompression failure',
-9824: u'Handshake failure',
-9825: u'Misc. bad certificate',
-9826: u'Bad unsupported cert format',
-9827: u'Certificate revoked',
-9828: u'Certificate expired',
-9829: u'Unknown certificate',
-9830: u'Illegal parameter',
-9831: u'Unknown Cert Authority',
-9832: u'Access denied',
-9833: u'Decoding error',
-9834: u'Decryption error',
-9835: u'Export restriction',
-9836: u'Bad protocol version',
-9837: u'Insufficient security',
-9838: u'Internal error',
-9839: u'User canceled',
-9840: u'No renegotiation allowed',
-9841: u'Peer cert is valid, or was ignored if verification disabled',
-9842: u'Server has requested a client cert',
-9843: u'Peer host name mismatch',
-9844: u'Peer dropped connection before responding',
-9845: u'Decryption failure',
-9846: u'Bad MAC',
-9847: u'Record overflow',
-9848: u'Configuration error',
-9849: u'Unexpected (skipped) record in DTLS'}
class Gurl(NSObject):
'''A class for getting content from a URL
using NSURLConnection/NSURLSession and friends'''
# since we inherit from NSObject, PyLint issues a few bogus warnings
# pylint: disable=W0232,E1002
# Don't want to define the attributes twice that are initialized in
# initWithOptions_(), so:
# pylint: disable=E1101,W0201
GURL_XATTR = 'com.googlecode.munki.downloadData'
def initWithOptions_(self, options):
'''Set up our Gurl object'''
self = super(Gurl, self).init()
if not self:
return None
self.follow_redirects = options.get('follow_redirects', False)
self.ignore_system_proxy = options.get('ignore_system_proxy', False)
self.destination_path = options.get('file')
self.can_resume = options.get('can_resume', False)
self.url = options.get('url')
self.additional_headers = options.get('additional_headers', {})
self.username = options.get('username')
self.password = options.get('password')
self.download_only_if_changed = options.get(
'download_only_if_changed', False)
self.cache_data = options.get('cache_data')
self.connection_timeout = options.get('connection_timeout', 60)
if NSURLSESSION_AVAILABLE:
self.minimum_tls_protocol = options.get(
'minimum_tls_protocol', kTLSProtocol1)
self.log = options.get('logging_function', NSLogWrapper)
self.resume = False
self.response = None
self.headers = None
self.status = None
self.error = None
self.SSLerror = None
self.done = False
self.redirection = []
self.destination = None
self.bytesReceived = 0
self.expectedLength = -1
self.percentComplete = 0
self.connection = None
self.session = None
self.task = None
return self
def start(self):
'''Start the connection'''
if not self.destination_path:
self.log('No output file specified.')
self.done = True
return
url = NSURL.URLWithString_(self.url)
request = (
NSMutableURLRequest.requestWithURL_cachePolicy_timeoutInterval_(
url, NSURLRequestReloadIgnoringLocalCacheData,
self.connection_timeout))
if self.additional_headers:
for header, value in self.additional_headers.items():
request.setValue_forHTTPHeaderField_(value, header)
# does the file already exist? See if we can resume a partial download
if os.path.isfile(self.destination_path):
stored_data = self.getStoredHeaders()
if (self.can_resume and 'expected-length' in stored_data and
('last-modified' in stored_data or 'etag' in stored_data)):
# we have a partial file and we're allowed to resume
self.resume = True
local_filesize = os.path.getsize(self.destination_path)
byte_range = 'bytes=%s-' % local_filesize
request.setValue_forHTTPHeaderField_(byte_range, 'Range')
if self.download_only_if_changed and not self.resume:
stored_data = self.cache_data or self.getStoredHeaders()
if 'last-modified' in stored_data:
request.setValue_forHTTPHeaderField_(
stored_data['last-modified'], 'if-modified-since')
if 'etag' in stored_data:
request.setValue_forHTTPHeaderField_(
stored_data['etag'], 'if-none-match')
if NSURLSESSION_AVAILABLE:
configuration = (
NSURLSessionConfiguration.defaultSessionConfiguration())
# optional: ignore system http/https proxies (10.9+ only)
if self.ignore_system_proxy is True:
configuration.setConnectionProxyDictionary_(
{kCFNetworkProxiesHTTPEnable: False,
kCFNetworkProxiesHTTPSEnable: False})
# set minimum supported TLS protocol (defaults to TLS1)
configuration.setTLSMinimumSupportedProtocol_(
self.minimum_tls_protocol)
self.session = (
NSURLSession.sessionWithConfiguration_delegate_delegateQueue_(
configuration, self, None))
self.task = self.session.dataTaskWithRequest_(request)
self.task.resume()
else:
self.connection = NSURLConnection.alloc().initWithRequest_delegate_(
request, self)
def cancel(self):
'''Cancel the connection'''
if self.connection:
if NSURLSESSION_AVAILABLE:
self.session.invalidateAndCancel()
else:
self.connection.cancel()
self.done = True
def isDone(self):
'''Check if the connection request is complete. As a side effect,
allow the delegates to work by letting the run loop run for a bit'''
if self.done:
return self.done
# let the delegates do their thing
NSRunLoop.currentRunLoop().runUntilDate_(
NSDate.dateWithTimeIntervalSinceNow_(.1))
return self.done
def getStoredHeaders(self):
'''Returns any stored headers for self.destination_path'''
# try to read stored headers
try:
stored_plist_bytestr = xattr.getxattr(
self.destination_path, self.GURL_XATTR)
except (KeyError, IOError):
return {}
data = NSData.dataWithBytes_length_(
stored_plist_bytestr, len(stored_plist_bytestr))
dataObject, _plistFormat, error = (
NSPropertyListSerialization.
propertyListFromData_mutabilityOption_format_errorDescription_(
data, NSPropertyListMutableContainersAndLeaves, None, None))
if error:
return {}
return dataObject
def storeHeaders_(self, headers):
'''Store dictionary data as an xattr for self.destination_path'''
plistData, error = (
NSPropertyListSerialization.
dataFromPropertyList_format_errorDescription_(
headers, NSPropertyListXMLFormat_v1_0, None))
if error:
byte_string = b''
else:
try:
byte_string = bytes(plistData)
except NameError:
byte_string = str(plistData)
try:
xattr.setxattr(self.destination_path, self.GURL_XATTR, byte_string)
except IOError as err:
self.log('Could not store metadata to %s: %s'
% (self.destination_path, err))
def normalizeHeaderDict_(self, a_dict):
'''Since HTTP header names are not case-sensitive, we normalize a
dictionary of HTTP headers by converting all the key names to
lower case'''
# yes, we don't use 'self'!
# pylint: disable=R0201
new_dict = {}
for key, value in a_dict.items():
new_dict[key.lower()] = value
return new_dict
def recordError_(self, error):
'''Record any error info from completed connection/session'''
self.error = error
# If this was an SSL error, try to extract the SSL error code.
if 'NSUnderlyingError' in error.userInfo():
ssl_code = error.userInfo()['NSUnderlyingError'].userInfo().get(
'_kCFNetworkCFStreamSSLErrorOriginalValue', None)
if ssl_code:
self.SSLerror = (ssl_code, ssl_error_codes.get(
ssl_code, 'Unknown SSL error'))
def removeExpectedSizeFromStoredHeaders(self):
'''If a successful transfer, clear the expected size so we
don\'t attempt to resume the download next time'''
if str(self.status).startswith('2'):
# remove the expected-size from the stored headers
headers = self.getStoredHeaders()
if 'expected-length' in headers:
del headers['expected-length']
self.storeHeaders_(headers)
def URLSession_task_didCompleteWithError_(self, _session, _task, error):
'''NSURLSessionTaskDelegate method.'''
if self.destination and self.destination_path:
self.destination.close()
self.removeExpectedSizeFromStoredHeaders()
if error:
self.recordError_(error)
self.done = True
def connection_didFailWithError_(self, _connection, error):
'''NSURLConnectionDelegate method
Sent when a connection fails to load its request successfully.'''
self.recordError_(error)
self.done = True
if self.destination and self.destination_path:
self.destination.close()
def connectionDidFinishLoading_(self, _connection):
'''NSURLConnectionDataDelegate method
Sent when a connection has finished loading successfully.'''
self.done = True
if self.destination and self.destination_path:
self.destination.close()
self.removeExpectedSizeFromStoredHeaders()
def handleResponse_withCompletionHandler_(
self, response, completionHandler):
'''Handle the response to the connection'''
self.response = response
self.bytesReceived = 0
self.percentComplete = -1
self.expectedLength = response.expectedContentLength()
download_data = {}
if response.className() == u'NSHTTPURLResponse':
# Headers and status code only available for HTTP/S transfers
self.status = response.statusCode()
self.headers = dict(response.allHeaderFields())
normalized_headers = self.normalizeHeaderDict_(self.headers)
if 'last-modified' in normalized_headers:
download_data['last-modified'] = normalized_headers[
'last-modified']
if 'etag' in normalized_headers:
download_data['etag'] = normalized_headers['etag']
download_data['expected-length'] = self.expectedLength
# self.destination is defined in initWithOptions_
# pylint: disable=E0203
if not self.destination and self.destination_path:
if self.status == 206 and self.resume:
# 206 is Partial Content response
stored_data = self.getStoredHeaders()
if (not stored_data or
stored_data.get('etag') != download_data.get('etag') or
stored_data.get('last-modified') != download_data.get(
'last-modified')):
# file on server is different than the one
# we have a partial for
self.log(
'Can\'t resume download; file on server has changed.')
if completionHandler:
# tell the session task to cancel
completionHandler(NSURLSessionResponseCancel)
else:
# cancel the connection
self.connection.cancel()
self.log('Removing %s' % self.destination_path)
os.unlink(self.destination_path)
# restart and attempt to download the entire file
self.log(
'Restarting download of %s' % self.destination_path)
os.unlink(self.destination_path)
self.start()
return
# try to resume
self.log('Resuming download for %s' % self.destination_path)
# add existing file size to bytesReceived so far
local_filesize = os.path.getsize(self.destination_path)
self.bytesReceived = local_filesize
self.expectedLength += local_filesize
# open file for append
self.destination = open(self.destination_path, 'ab')
elif str(self.status).startswith('2'):
# not resuming, just open the file for writing
self.destination = open(self.destination_path, 'wb')
# store some headers with the file for use if we need to resume
# the download and for future checking if the file on the server
# has changed
self.storeHeaders_(download_data)
if completionHandler:
# tell the session task to continue
completionHandler(NSURLSessionResponseAllow)
def URLSession_dataTask_didReceiveResponse_completionHandler_(
self, _session, _task, response, completionHandler):
'''NSURLSessionDataDelegate method'''
if CALLBACK_HELPER_AVAILABLE:
completionHandler.__block_signature__ = objc_method_signature(b'v@i')
self.handleResponse_withCompletionHandler_(response, completionHandler)
def connection_didReceiveResponse_(self, _connection, response):
'''NSURLConnectionDataDelegate delegate method
Sent when the connection has received sufficient data to construct the
URL response for its request.'''
self.handleResponse_withCompletionHandler_(response, None)
def handleRedirect_newRequest_withCompletionHandler_(
self, response, request, completionHandler):
'''Handle the redirect request'''
def allowRedirect():
'''Allow the redirect'''
if completionHandler:
completionHandler(request)
return None
return request
def denyRedirect():
'''Deny the redirect'''
if completionHandler:
completionHandler(None)
return None
newURL = request.URL().absoluteString()
if response is None:
# the request has changed the NSURLRequest in order to standardize
# its format, for example, changing a request for
# http://www.apple.com to http://www.apple.com/. This occurs because
# the standardized, or canonical, version of the request is used for
# cache management. Pass the request back as-is
# (it appears that at some point Apple also defined a redirect like
# http://developer.apple.com to https://developer.apple.com to be
# 'merely' a change in the canonical URL.)
# Further -- it appears that this delegate method isn't called at
# all in this scenario, unlike NSConnectionDelegate method
# connection:willSendRequest:redirectResponse:
# we'll leave this here anyway in case we're wrong about that
self.log('Allowing redirect to: %s' % newURL)
return allowRedirect()
# If we get here, it appears to be a real redirect attempt
# Annoyingly, we apparently can't get access to the headers from the
# site that told us to redirect. All we know is that we were told
# to redirect and where the new location is.
self.redirection.append([newURL, dict(response.allHeaderFields())])
newParsedURL = urlparse(newURL)
# This code was largely based on the work of Andreas Fuchs
# (https://github.com/munki/munki/pull/465)
if self.follow_redirects is True or self.follow_redirects == 'all':
# Allow the redirect
self.log('Allowing redirect to: %s' % newURL)
return allowRedirect()
elif (self.follow_redirects == 'https'
and newParsedURL.scheme == 'https'):
# Once again, allow the redirect
self.log('Allowing redirect to: %s' % newURL)
return allowRedirect()
# If we're down here either the preference was set to 'none',
# the url we're forwarding on to isn't https or follow_redirects
# was explicitly set to False
self.log('Denying redirect to: %s' % newURL)
return denyRedirect()
# we don't control the API, so
# pylint: disable=too-many-arguments
def URLSession_task_willPerformHTTPRedirection_newRequest_completionHandler_(
self, _session, _task, response, request, completionHandler):
'''NSURLSessionTaskDelegate method'''
self.log(
'URLSession_task_willPerformHTTPRedirection_newRequest_'
'completionHandler_')
if CALLBACK_HELPER_AVAILABLE:
completionHandler.__block_signature__ = objc_method_signature(b'v@@')
self.handleRedirect_newRequest_withCompletionHandler_(
response, request, completionHandler)
# pylint: enable=too-many-arguments
def connection_willSendRequest_redirectResponse_(
self, _connection, request, response):
'''NSURLConnectionDataDelegate method
Sent when the connection determines that it must change URLs in order
to continue loading a request.'''
self.log('connection_willSendRequest_redirectResponse_')
return self.handleRedirect_newRequest_withCompletionHandler_(
response, request, None)
def connection_canAuthenticateAgainstProtectionSpace_(
self, _connection, protectionSpace):
'''NSURLConnection delegate method
Sent to determine whether the delegate is able to respond to a
protection spaces form of authentication.
Deprecated in 10.10'''
# this is not called in 10.5.x.
self.log('connection_canAuthenticateAgainstProtectionSpace_')
if protectionSpace:
host = protectionSpace.host()
realm = protectionSpace.realm()
authenticationMethod = protectionSpace.authenticationMethod()
self.log('Protection space found. Host: %s Realm: %s AuthMethod: %s'
% (host, realm, authenticationMethod))
if self.username and self.password and authenticationMethod in [
'NSURLAuthenticationMethodDefault',
'NSURLAuthenticationMethodHTTPBasic',
'NSURLAuthenticationMethodHTTPDigest']:
# we know how to handle this
self.log('Can handle this authentication request')
return True
# we don't know how to handle this; let the OS try
self.log('Allowing OS to handle authentication request')
return False
def handleChallenge_withCompletionHandler_(
self, challenge, completionHandler):
'''Handle an authentication challenge'''
protectionSpace = challenge.protectionSpace()
host = protectionSpace.host()
realm = protectionSpace.realm()
authenticationMethod = protectionSpace.authenticationMethod()
self.log(
'Authentication challenge for Host: %s Realm: %s AuthMethod: %s'
% (host, realm, authenticationMethod))
if challenge.previousFailureCount() > 0:
# we have the wrong credentials. just fail
self.log('Previous authentication attempt failed.')
if completionHandler:
completionHandler(
NSURLSessionAuthChallengeCancelAuthenticationChallenge,
None)
else:
challenge.sender().cancelAuthenticationChallenge_(challenge)
if self.username and self.password and authenticationMethod in [
'NSURLAuthenticationMethodDefault',
'NSURLAuthenticationMethodHTTPBasic',
'NSURLAuthenticationMethodHTTPDigest']:
self.log('Will attempt to authenticate.')
self.log('Username: %s Password: %s'
% (self.username, ('*' * len(self.password or ''))))
credential = (
NSURLCredential.credentialWithUser_password_persistence_(
self.username, self.password,
NSURLCredentialPersistenceNone))
if completionHandler:
completionHandler(
NSURLSessionAuthChallengeUseCredential, credential)
else:
challenge.sender().useCredential_forAuthenticationChallenge_(
credential, challenge)
else:
# fall back to system-provided default behavior
self.log('Allowing OS to handle authentication request')
if completionHandler:
completionHandler(
NSURLSessionAuthChallengePerformDefaultHandling, None)
else:
if (challenge.sender().respondsToSelector_(
'performDefaultHandlingForAuthenticationChallenge:')):
self.log('Allowing OS to handle authentication request')
challenge.sender(
).performDefaultHandlingForAuthenticationChallenge_(
challenge)
else:
# Mac OS X 10.6 doesn't support
# performDefaultHandlingForAuthenticationChallenge:
self.log('Continuing without credential.')
challenge.sender(
).continueWithoutCredentialForAuthenticationChallenge_(
challenge)
def connection_willSendRequestForAuthenticationChallenge_(
self, _connection, challenge):
'''NSURLConnection delegate method
Tells the delegate that the connection will send a request for an
authentication challenge. New in 10.7.'''
self.log('connection_willSendRequestForAuthenticationChallenge_')
self.handleChallenge_withCompletionHandler_(challenge, None)
def URLSession_task_didReceiveChallenge_completionHandler_(
self, _session, _task, challenge, completionHandler):
'''NSURLSessionTaskDelegate method'''
if CALLBACK_HELPER_AVAILABLE:
completionHandler.__block_signature__ = objc_method_signature(b'v@i@')
self.log('URLSession_task_didReceiveChallenge_completionHandler_')
self.handleChallenge_withCompletionHandler_(
challenge, completionHandler)
def connection_didReceiveAuthenticationChallenge_(
self, _connection, challenge):
'''NSURLConnection delegate method
Sent when a connection must authenticate a challenge in order to
download its request. Deprecated in 10.10'''
self.log('connection_didReceiveAuthenticationChallenge_')
self.handleChallenge_withCompletionHandler_(challenge, None)
def handleReceivedData_(self, data):
'''Handle received data'''
if self.destination:
self.destination.write(data)
else:
try:
self.log(str(data))
except Exception:
pass
self.bytesReceived += len(data)
if self.expectedLength != NSURLResponseUnknownLength:
# pylint: disable=old-division
self.percentComplete = int(
float(self.bytesReceived)/float(self.expectedLength) * 100.0)
# pylint: enable=old-division
def URLSession_dataTask_didReceiveData_(self, _session, _task, data):
'''NSURLSessionDataDelegate method'''
self.handleReceivedData_(data)
def connection_didReceiveData_(self, _connection, data):
'''NSURLConnectionDataDelegate method
Sent as a connection loads data incrementally'''
self.handleReceivedData_(data)
if __name__ == '__main__':
print('This is a library of support tools for the Munki Suite.')