├── .gitignore ├── LICENSE.md ├── README.md └── fetch-installer-pkg.py /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | *.pyc 3 | *.pkg 4 | content/ 5 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | Licensed under the Apache License, Version 2.0 (the "License"); 2 | you may not use this source code except in compliance with the License. 3 | You may obtain a copy of the License at 4 | 5 | https://www.apache.org/licenses/LICENSE-2.0 6 | 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # fetch-installer-pkg 2 | 3 | This is a script based [on Greg Neagle's `installinstallmacos.py`](https://github.com/munki/macadmin-scripts/blob/main/installinstallmacos.py). 4 | 5 | This script will find the latest macOS Big Sur entry in Apple's software update catalogs and download the "InstallAssistant" pkg which installs the "Install macOS Big Sur" application on your system. 6 | 7 | When you run the script it will present you with all the Big Sur Installers it finds in the software update catalog. Choose one by entering the number and it will download the pkg file. 8 | 9 | ``` 10 | ./fetch-installer-pkg.py 11 | ``` 12 | 13 | Add the `--help` argument for further options. 14 | 15 | I have a [post on different strategies to deploy the macOS installer application](https://scriptingosx.com/2020/11/deploying-the-big-sur-installer-application/) on my blog. 16 | 17 | ### Credits 18 | 19 | Many thanks to Greg Neagle for the [original script](https://github.com/munki/macadmin-scripts/blob/main/installinstallmacos.py) and lots of advice and Mike Lynn for helping me figure out the software update catalog. 20 | -------------------------------------------------------------------------------- /fetch-installer-pkg.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # encoding: utf-8 3 | # 4 | # Copyright 2020 Armin Briegel. 5 | # 6 | # based on Greg Neagle's 'installinstallmacos.py' 7 | # https://github.com/munki/macadmin-scripts/blob/main/installinstallmacos.py 8 | # 9 | # with many thanks to Greg Neagle for the original script and lots of advice 10 | # and Mike Lynn for helping me figure out the software update catalog 11 | # Graham R Pugh for figurung out the 11.1 download 12 | # see his combined version of mine and Greg's script here: 13 | # https://github.com/grahampugh/erase-install/tree/pkg 14 | 15 | # 16 | # Licensed under the Apache License, Version 2.0 (the "License"); 17 | # you may not use this file except in compliance with the License. 18 | # You may obtain a copy of the License at 19 | # 20 | # http://www.apache.org/licenses/LICENSE-2.0 21 | # 22 | # Unless required by applicable law or agreed to in writing, software 23 | # distributed under the License is distributed on an "AS IS" BASIS, 24 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 25 | # See the License for the specific language governing permissions and 26 | # limitations under the License. 27 | # 28 | 29 | '''fetch-full-installer.py 30 | A tool to download the a pkg installer for the Install macOS app from Apple's 31 | softwareupdate servers''' 32 | 33 | # Python 3 compatibility shims 34 | from __future__ import ( 35 | absolute_import, division, print_function, unicode_literals) 36 | 37 | import argparse 38 | import gzip 39 | import os 40 | import plistlib 41 | import subprocess 42 | import sys 43 | try: 44 | # python 2 45 | from urllib.parse import urlsplit 46 | except ImportError: 47 | # python 3 48 | from urlparse import urlsplit 49 | from xml.dom import minidom 50 | from xml.parsers.expat import ExpatError 51 | import xattr 52 | 53 | 54 | DEFAULT_SUCATALOGS = { 55 | '17': 'https://swscan.apple.com/content/catalogs/others/' 56 | 'index-10.13-10.12-10.11-10.10-10.9' 57 | '-mountainlion-lion-snowleopard-leopard.merged-1.sucatalog', 58 | '18': 'https://swscan.apple.com/content/catalogs/others/' 59 | 'index-10.14-10.13-10.12-10.11-10.10-10.9' 60 | '-mountainlion-lion-snowleopard-leopard.merged-1.sucatalog', 61 | '19': 'https://swscan.apple.com/content/catalogs/others/' 62 | 'index-10.15-10.14-10.13-10.12-10.11-10.10-10.9' 63 | '-mountainlion-lion-snowleopard-leopard.merged-1.sucatalog', 64 | '20': 'https://swscan.apple.com/content/catalogs/others/' 65 | 'index-10.16-10.15-10.14-10.13-10.12-10.11-10.10-10.9' 66 | '-mountainlion-lion-snowleopard-leopard.merged-1.sucatalog', 67 | '21': 'https://swscan.apple.com/content/catalogs/others/' 68 | 'index-12-10.16-10.15-10.14-10.13-10.12-10.11-10.10-10.9' 69 | '-mountainlion-lion-snowleopard-leopard.merged-1.sucatalog', 70 | } 71 | 72 | SEED_CATALOGS_PLIST = ( 73 | '/System/Library/PrivateFrameworks/Seeding.framework/Versions/Current/' 74 | 'Resources/SeedCatalogs.plist' 75 | ) 76 | 77 | 78 | def get_input(prompt=None): 79 | '''Python 2 and 3 wrapper for raw_input/input''' 80 | try: 81 | return raw_input(prompt) 82 | except NameError: 83 | # raw_input doesn't exist in Python 3 84 | return input(prompt) 85 | 86 | 87 | def read_plist(filepath): 88 | '''Wrapper for the differences between Python 2 and Python 3's plistlib''' 89 | try: 90 | with open(filepath, "rb") as fileobj: 91 | return plistlib.load(fileobj) 92 | except AttributeError: 93 | # plistlib module doesn't have a load function (as in Python 2) 94 | return plistlib.readPlist(filepath) 95 | 96 | 97 | def read_plist_from_string(bytestring): 98 | '''Wrapper for the differences between Python 2 and Python 3's plistlib''' 99 | try: 100 | return plistlib.loads(bytestring) 101 | except AttributeError: 102 | # plistlib module doesn't have a load function (as in Python 2) 103 | return plistlib.readPlistFromString(bytestring) 104 | 105 | 106 | def get_seeding_program(sucatalog_url): 107 | '''Returns a seeding program name based on the sucatalog_url''' 108 | try: 109 | seed_catalogs = read_plist(SEED_CATALOGS_PLIST) 110 | for key, value in seed_catalogs.items(): 111 | if sucatalog_url == value: 112 | return key 113 | return '' 114 | except (OSError, IOError, ExpatError, AttributeError, KeyError) as err: 115 | print(err, file=sys.stderr) 116 | return '' 117 | 118 | 119 | def get_seed_catalog(seedname='DeveloperSeed'): 120 | '''Returns the developer seed sucatalog''' 121 | try: 122 | seed_catalogs = read_plist(SEED_CATALOGS_PLIST) 123 | return seed_catalogs.get(seedname) 124 | except (OSError, IOError, ExpatError, AttributeError, KeyError) as err: 125 | print(err, file=sys.stderr) 126 | return '' 127 | 128 | 129 | def get_seeding_programs(): 130 | '''Returns the list of seeding program names''' 131 | try: 132 | seed_catalogs = read_plist(SEED_CATALOGS_PLIST) 133 | return list(seed_catalogs.keys()) 134 | except (OSError, IOError, ExpatError, AttributeError, KeyError) as err: 135 | print(err, file=sys.stderr) 136 | return '' 137 | 138 | 139 | def get_default_catalog(): 140 | '''Returns the default softwareupdate catalog for the current OS''' 141 | darwin_major = os.uname()[2].split('.')[0] 142 | return DEFAULT_SUCATALOGS.get(darwin_major) 143 | 144 | 145 | class ReplicationError(Exception): 146 | '''A custom error when replication fails''' 147 | pass 148 | 149 | 150 | def replicate_url(full_url, 151 | root_dir='/tmp', 152 | show_progress=False, 153 | ignore_cache=False, 154 | attempt_resume=False): 155 | '''Downloads a URL and stores it in the same relative path on our 156 | filesystem. Returns a path to the replicated file.''' 157 | 158 | path = urlsplit(full_url)[2] 159 | relative_url = path.lstrip('/') 160 | relative_url = os.path.normpath(relative_url) 161 | local_file_path = os.path.join(root_dir, relative_url) 162 | if show_progress: 163 | options = '-fL' 164 | else: 165 | options = '-sfL' 166 | curl_cmd = ['/usr/bin/curl', options, 167 | '--create-dirs', 168 | '-o', local_file_path] 169 | if not full_url.endswith(".gz"): 170 | # stupid hack for stupid Apple behavior where it sometimes returns 171 | # compressed files even when not asked for 172 | curl_cmd.append('--compressed') 173 | if not ignore_cache and os.path.exists(local_file_path): 174 | curl_cmd.extend(['-z', local_file_path]) 175 | if attempt_resume: 176 | curl_cmd.extend(['-C', '-']) 177 | curl_cmd.append(full_url) 178 | print("Downloading %s..." % full_url) 179 | try: 180 | subprocess.check_call(curl_cmd) 181 | except subprocess.CalledProcessError as err: 182 | raise ReplicationError(err) 183 | return local_file_path 184 | 185 | 186 | def parse_server_metadata(filename): 187 | '''Parses a softwareupdate server metadata file, looking for information 188 | of interest. 189 | Returns a dictionary containing title, version, and description.''' 190 | title = '' 191 | vers = '' 192 | try: 193 | md_plist = read_plist(filename) 194 | except (OSError, IOError, ExpatError) as err: 195 | print('Error reading %s: %s' % (filename, err), file=sys.stderr) 196 | return {} 197 | vers = md_plist.get('CFBundleShortVersionString', '') 198 | localization = md_plist.get('localization', {}) 199 | preferred_localization = (localization.get('English') or 200 | localization.get('en')) 201 | if preferred_localization: 202 | title = preferred_localization.get('title', '') 203 | 204 | metadata = {} 205 | metadata['title'] = title 206 | metadata['version'] = vers 207 | return metadata 208 | 209 | 210 | def get_server_metadata(catalog, product_key, workdir, ignore_cache=False): 211 | '''Replicate ServerMetaData''' 212 | try: 213 | url = catalog['Products'][product_key]['ServerMetadataURL'] 214 | try: 215 | smd_path = replicate_url( 216 | url, root_dir=workdir, ignore_cache=ignore_cache) 217 | return smd_path 218 | except ReplicationError as err: 219 | print('Could not replicate %s: %s' % (url, err), file=sys.stderr) 220 | return None 221 | except KeyError: 222 | #print('Malformed catalog.', file=sys.stderr) 223 | return None 224 | 225 | 226 | def parse_dist(filename): 227 | '''Parses a softwareupdate dist file, returning a dict of info of 228 | interest''' 229 | dist_info = {} 230 | try: 231 | dom = minidom.parse(filename) 232 | except ExpatError: 233 | print('Invalid XML in %s' % filename, file=sys.stderr) 234 | return dist_info 235 | except IOError as err: 236 | print('Error reading %s: %s' % (filename, err), file=sys.stderr) 237 | return dist_info 238 | 239 | titles = dom.getElementsByTagName('title') 240 | if titles: 241 | dist_info['title_from_dist'] = titles[0].firstChild.wholeText 242 | 243 | auxinfos = dom.getElementsByTagName('auxinfo') 244 | if not auxinfos: 245 | return dist_info 246 | auxinfo = auxinfos[0] 247 | key = None 248 | value = None 249 | children = auxinfo.childNodes 250 | # handle the possibility that keys from auxinfo may be nested 251 | # within a 'dict' element 252 | dict_nodes = [n for n in auxinfo.childNodes 253 | if n.nodeType == n.ELEMENT_NODE and 254 | n.tagName == 'dict'] 255 | if dict_nodes: 256 | children = dict_nodes[0].childNodes 257 | for node in children: 258 | if node.nodeType == node.ELEMENT_NODE and node.tagName == 'key': 259 | key = node.firstChild.wholeText 260 | if node.nodeType == node.ELEMENT_NODE and node.tagName == 'string': 261 | value = node.firstChild.wholeText 262 | if key and value: 263 | dist_info[key] = value 264 | key = None 265 | value = None 266 | return dist_info 267 | 268 | 269 | def download_and_parse_sucatalog(sucatalog, workdir, ignore_cache=False): 270 | '''Downloads and returns a parsed softwareupdate catalog''' 271 | try: 272 | localcatalogpath = replicate_url( 273 | sucatalog, root_dir=workdir, ignore_cache=ignore_cache) 274 | except ReplicationError as err: 275 | print('Could not replicate %s: %s' % (sucatalog, err), file=sys.stderr) 276 | exit(-1) 277 | if os.path.splitext(localcatalogpath)[1] == '.gz': 278 | with gzip.open(localcatalogpath) as the_file: 279 | content = the_file.read() 280 | try: 281 | catalog = read_plist_from_string(content) 282 | return catalog 283 | except ExpatError as err: 284 | print('Error reading %s: %s' % (localcatalogpath, err), 285 | file=sys.stderr) 286 | exit(-1) 287 | else: 288 | try: 289 | catalog = read_plist(localcatalogpath) 290 | return catalog 291 | except (OSError, IOError, ExpatError) as err: 292 | print('Error reading %s: %s' % (localcatalogpath, err), 293 | file=sys.stderr) 294 | exit(-1) 295 | 296 | 297 | def find_mac_os_installers(catalog, installassistant_pkg_only=False): 298 | '''Return a list of product identifiers for what appear to be macOS 299 | installers''' 300 | mac_os_installer_products = [] 301 | if 'Products' in catalog: 302 | for product_key in catalog['Products'].keys(): 303 | product = catalog['Products'][product_key] 304 | try: 305 | if product['ExtendedMetaInfo']['InstallAssistantPackageIdentifiers']: 306 | if product['ExtendedMetaInfo']['InstallAssistantPackageIdentifiers' 307 | ]['SharedSupport']: 308 | mac_os_installer_products.append(product_key) 309 | except KeyError: 310 | continue 311 | return mac_os_installer_products 312 | 313 | def os_installer_product_info(catalog, workdir, ignore_cache=False): 314 | '''Returns a dict of info about products that look like macOS installers''' 315 | product_info = {} 316 | installer_products = find_mac_os_installers(catalog) 317 | for product_key in installer_products: 318 | product_info[product_key] = {} 319 | filename = get_server_metadata(catalog, product_key, workdir) 320 | if filename: 321 | product_info[product_key] = parse_server_metadata(filename) 322 | else: 323 | print('No server metadata for %s' % product_key) 324 | product_info[product_key]['title'] = None 325 | product_info[product_key]['version'] = None 326 | 327 | product = catalog['Products'][product_key] 328 | product_info[product_key]['PostDate'] = product['PostDate'] 329 | distributions = product['Distributions'] 330 | dist_url = distributions.get('English') or distributions.get('en') 331 | try: 332 | dist_path = replicate_url( 333 | dist_url, root_dir=workdir, show_progress=False, ignore_cache=ignore_cache) 334 | except ReplicationError as err: 335 | print('Could not replicate %s: %s' % (dist_url, err), 336 | file=sys.stderr) 337 | else: 338 | dist_info = parse_dist(dist_path) 339 | product_info[product_key]['DistributionPath'] = dist_path 340 | product_info[product_key].update(dist_info) 341 | if not product_info[product_key]['title']: 342 | product_info[product_key]['title'] = dist_info.get('title_from_dist') 343 | if not product_info[product_key]['version']: 344 | product_info[product_key]['version'] = dist_info.get('VERSION') 345 | 346 | return product_info 347 | 348 | 349 | def replicate_product(catalog, product_id, workdir, ignore_cache=False): 350 | '''Downloads all the packages for a product''' 351 | product = catalog['Products'][product_id] 352 | for package in product.get('Packages', []): 353 | # TO-DO: Check 'Size' attribute and make sure 354 | # we have enough space on the target 355 | # filesystem before attempting to download 356 | if 'URL' in package: 357 | try: 358 | replicate_url( 359 | package['URL'], root_dir=workdir, 360 | show_progress=True, ignore_cache=ignore_cache, 361 | attempt_resume=(not ignore_cache)) 362 | except ReplicationError as err: 363 | print('Could not replicate %s: %s' % (package['URL'], err), 364 | file=sys.stderr) 365 | exit(-1) 366 | if 'MetadataURL' in package: 367 | try: 368 | replicate_url(package['MetadataURL'], root_dir=workdir, 369 | ignore_cache=ignore_cache) 370 | except ReplicationError as err: 371 | print('Could not replicate %s: %s' 372 | % (package['MetadataURL'], err), file=sys.stderr) 373 | exit(-1) 374 | 375 | 376 | def main(): 377 | '''Do the main thing here''' 378 | parser = argparse.ArgumentParser() 379 | parser.add_argument('--seedprogram', default='', 380 | help='Which Seed Program catalog to use. Valid values ' 381 | 'are %s.' % ', '.join(get_seeding_programs())) 382 | parser.add_argument('--catalogurl', default='', 383 | help='Software Update catalog URL. This option ' 384 | 'overrides any seedprogram option.') 385 | parser.add_argument('--workdir', metavar='path_to_working_dir', 386 | default='.', 387 | help='Path to working directory on a volume with over ' 388 | '10G of available space. Defaults to current working ' 389 | 'directory.') 390 | parser.add_argument('--ignore-cache', action='store_true', 391 | help='Ignore any previously cached files.') 392 | parser.add_argument('--latest', action='store_true', 393 | help='Download the latest version with no user interaction.') 394 | parser.add_argument('--version', default='', 395 | help='Download the latest version with no user interaction.') 396 | args = parser.parse_args() 397 | 398 | current_dir = os.getcwd() 399 | 400 | if args.catalogurl: 401 | su_catalog_url = args.catalogurl 402 | elif args.seedprogram: 403 | su_catalog_url = get_seed_catalog(args.seedprogram) 404 | if not su_catalog_url: 405 | print('Could not find a catalog url for seed program %s' 406 | % args.seedprogram, file=sys.stderr) 407 | print('Valid seeding programs are: %s' 408 | % ', '.join(get_seeding_programs()), file=sys.stderr) 409 | exit(-1) 410 | else: 411 | su_catalog_url = get_default_catalog() 412 | if not su_catalog_url: 413 | print('Could not find a default catalog url for this OS version.', 414 | file=sys.stderr) 415 | exit(-1) 416 | 417 | # download sucatalog and look for products that are for macOS installers 418 | catalog = download_and_parse_sucatalog( 419 | su_catalog_url, args.workdir, ignore_cache=args.ignore_cache) 420 | 421 | # print(catalog) 422 | product_info = os_installer_product_info( 423 | catalog, args.workdir, ignore_cache=args.ignore_cache) 424 | 425 | if not product_info: 426 | print('No macOS installer products found in the sucatalog.', 427 | file=sys.stderr) 428 | exit(-1) 429 | 430 | if len(product_info) > 1: 431 | # display a menu of choices (some seed catalogs have multiple installers) 432 | print('%2s %14s %10s %8s %11s %s' 433 | % ('#', 'ProductID', 'Version', 'Build', 'Post Date', 'Title')) 434 | # sort the list by release date 435 | sorted_product_info = sorted(product_info, key=lambda k: product_info[k]['PostDate'], reverse=True) 436 | 437 | if args.latest: 438 | product_id = sorted_product_info[0] 439 | elif args.version: 440 | found_version = False 441 | for index, product_id in enumerate(sorted_product_info): 442 | if (product_info[product_id]['version'] == args.version): 443 | found_version = True 444 | break 445 | if found_version != True: 446 | print("Couldn't find version, Exiting.") 447 | exit(1) 448 | else: 449 | for index, product_id in enumerate(sorted_product_info): 450 | print('%2s %14s %10s %8s %11s %s' % ( 451 | index + 1, 452 | product_id, 453 | product_info[product_id].get('version', 'UNKNOWN'), 454 | product_info[product_id].get('BUILD', 'UNKNOWN'), 455 | product_info[product_id]['PostDate'].strftime('%Y-%m-%d'), 456 | product_info[product_id]['title'] 457 | )) 458 | answer = get_input( 459 | '\nChoose a product to download (1-%s): ' % len(product_info)) 460 | try: 461 | index = int(answer) - 1 462 | if index < 0: 463 | raise ValueError 464 | product_id = sorted_product_info[index] 465 | except (ValueError, IndexError): 466 | print('Exiting.') 467 | exit(0) 468 | else: # only one product found 469 | product_id = list(product_info.keys())[0] 470 | print("Found a single installer:") 471 | 472 | 473 | product = catalog['Products'][product_id] 474 | 475 | print('%14s %10s %8s %11s %s' % ( 476 | product_id, 477 | product_info[product_id].get('version', 'UNKNOWN'), 478 | product_info[product_id].get('BUILD', 'UNKNOWN'), 479 | product_info[product_id]['PostDate'].strftime('%Y-%m-%d'), 480 | product_info[product_id]['title'] 481 | )) 482 | 483 | # determine the InstallAssistant pkg url 484 | for package in product['Packages']: 485 | package_url = package['URL'] 486 | if package_url.endswith('InstallAssistant.pkg'): 487 | break 488 | 489 | # print("Package URL is %s" % package_url) 490 | download_pkg = replicate_url(package_url, args.workdir, True, ignore_cache=args.ignore_cache) 491 | 492 | pkg_name = ('InstallAssistant-%s-%s.pkg' % (product_info[product_id]['version'], 493 | product_info[product_id]['BUILD'])) 494 | 495 | # hard link the downloaded file to cwd 496 | local_pkg = os.path.join(args.workdir, pkg_name) 497 | os.link(download_pkg, local_pkg) 498 | 499 | # unlink download 500 | #os.unlink(download_pkg) 501 | 502 | # reveal in Finder 503 | open_cmd = ['open', '-R', local_pkg] 504 | subprocess.check_call(open_cmd) 505 | 506 | 507 | if __name__ == '__main__': 508 | main() 509 | --------------------------------------------------------------------------------