Unofficial Fruit Company Open Source Git Mirror

├── footer.html ├── fetch-all.sh ├── Makefile ├── fetch.sh ├── gitify └── xref.sh ├── header.html ├── README ├── mirror.rb └── gitify.rb /footer.html: -------------------------------------------------------------------------------- 1 |

This page was generated on %DATE%

2 | 3 | 4 | -------------------------------------------------------------------------------- /fetch-all.sh: -------------------------------------------------------------------------------- 1 | curl http://www.opensource.apple.com|grep 'href="/release/mac-os-x'|sed -e 's/.*mac-os-x-$[0-9][^/]*$\/.*/\1/'|sort|xargs -n 20 -P 5 ./fetch.sh 2 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | index.html: header.html projects.html footer.html 2 | cat header.html projects.html footer.html | sed -e 's/%DATE%/'"`date`"/g > $@ 3 | 4 | projects.html: gitify.rb 5 | ./gitify.rb 6 | 7 | -------------------------------------------------------------------------------- /fetch.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | for release; do 4 | ./mirror.rb --release "$release" | grep -v '^#' | xargs ./mirror.rb --release "$release" 5 | [ ${PIPESTATUS[0]} -eq 0 ] && [ ${PIPESTATUS[0]} -eq 0 ] 6 | done -------------------------------------------------------------------------------- /gitify/xref.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | for project in * ; do 4 | ( 5 | echo -n $project 6 | cd $project 7 | versions=$(git tag|grep -v os-x-|tr "\n" " ") 8 | echo -n " versions:" 9 | for version in $versions ; do 10 | inosversions=$(git tag --points-at $version|grep os-x-|tr "\n" " ") 11 | echo -n " $version" 12 | for osversion in $inosversions ; do 13 | echo -n " ($osversion)" 14 | xattr -w nu.dll.aosm.${osversion} `date +%s` ../../projects/${project}-${version} 15 | done 16 | done 17 | echo "" 18 | ) 19 | done 20 | -------------------------------------------------------------------------------- /header.html: -------------------------------------------------------------------------------- 1 | Unofficial Fruit Company Open Source Git Mirror 2 | 3 | 4 |

Unofficial Fruit Company Open Source Git Mirror

5 | 6 |

The Unofficial Fruit Company Open Source Git Mirror, or AOSM for short, is a collection of git repositories created from the Fruit Company Open Source web page, with tags for each project version and OS release. This makes it easy to, for example, compare xnu in 10.7.1 to xnu in 10.7.4 using regular git diffing tools, or see which OS releases contained zlib 37.2 (upstream zlib version 1.2.5) (project version numbers generally don't correspond to upstream project version numbers). 7 |

8 | -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | -*- Mode: auto-fill -*- 2 | 3 | These scripts helps you download and extract packages from 4 | opensource.apple.com or even maintain an extracted mirror with the 5 | latest open source packages from a Mac OS X release (10.8.1 by 6 | default). 7 | 8 | Note that the .tar.gz will not be saved; only its extracted contents. 9 | 10 | Prerequisite: 11 | 12 | sudo gem install plist minitar 13 | 14 | Usage: 15 | 16 | Run "./mirror.rb" to list all available packages. 17 | Run "./mirror.rb package1 package2 ..." to download package1, package2 18 | and so on. 19 | The script will skip packages that have already been downloaded. 20 | 21 | To mirror all packages in a certain OS X release from 22 | opensource.apple.com, you can run the following command: 23 | ./fetch.sh 10.8.1 24 | 25 | As of 10.8.1, the CF and WebKit2 packages are missing, and a full 26 | mirror uses about 2.1 GB of disk space. 27 | -------------------------------------------------------------------------------- /mirror.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # -*- coding: utf-8 -*- 3 | 4 | require 'open-uri' 5 | require 'rubygems' 6 | require 'plist' 7 | require 'zlib' 8 | require 'archive/tar/minitar' 9 | include Archive::Tar 10 | 11 | release = 'mac-os-x-1081' 12 | targetdir = 'projects' 13 | tmpdir = '.projects.tmp' 14 | releaseurl = "http://opensource.apple.com/release/#{release}/" 15 | 16 | if ARGV[0] == "--release" 17 | ARGV.shift 18 | release = "mac-os-x-" + ARGV.shift.gsub(/\./, "") 19 | if release =~ /^mac-os-x-109/ || release =~ /^mac-os-x-1010/ 20 | release = release.sub(/^mac-/, "") 21 | end 22 | end 23 | 24 | releaseplist = "http://opensource.apple.com/plist/#{release}.plist" 25 | print "# Downloading package list from #{releaseplist}…" 26 | STDOUT.flush 27 | data = Plist.parse_xml(open(releaseplist).read) 28 | puts " OK" 29 | 30 | if ARGV.length == 0 31 | puts "# Listing all projects from #{releaseurl}" 32 | data["projects"].each do | key, value | 33 | puts " #{key}" 34 | end 35 | exit(0) 36 | end 37 | 38 | def tag_project(dir, release) 39 | `xattr -w nu.dll.aosm.#{release} \`date +%s\` "#{dir}"` 40 | end 41 | 42 | if not File.exists?(targetdir) 43 | Dir.mkdir(targetdir) 44 | end 45 | 46 | if not File.exists?(tmpdir) 47 | Dir.mkdir(tmpdir) 48 | end 49 | 50 | ARGV.each do | projectname | 51 | project = data["projects"][projectname] 52 | projectdir = "#{projectname}-#{project['version']}" 53 | tgzurl = "http://opensource.apple.com/tarballs/#{projectname}/#{projectdir}.tar.gz" 54 | fulltarget = File.join(targetdir, projectdir) 55 | if File.exists?(fulltarget) 56 | puts "#{projectdir} already downloaded, skipping." 57 | tag_project(fulltarget, release) 58 | next 59 | end 60 | print "Downloading and untarring #{tgzurl}…" 61 | STDOUT.flush 62 | begin 63 | tmptarget = File.join(tmpdir, projectdir) 64 | if not File.exists?(tmptarget) 65 | Dir.mkdir(tmptarget) 66 | File.chmod(0755, tmptarget) 67 | end 68 | tgz = Zlib::GzipReader.new(open(tgzurl)) 69 | Minitar.unpack(tgz, tmpdir) 70 | File.rename(tmptarget, fulltarget) 71 | tag_project(fulltarget, release) 72 | puts " OK." 73 | rescue OpenURI::HTTPError 74 | puts " failed." 75 | end 76 | end 77 | puts "Kthxbye." 78 | -------------------------------------------------------------------------------- /gitify.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # 3 | # This script iterates over all projects downloaded by mirror.rb and fetch.sh, 4 | # creates a git repository for each project and attempts to create a git commit 5 | # for each project version in the correct order. It also creates git tags for 6 | # the OS releases containing a particular version, making it easy to track 7 | # changes between OS releases and e.g. get diffs between arbitrary versions 8 | # 9 | 10 | # problematic projects 11 | ignore = [ 'libauto' ] 12 | 13 | require 'strscan' 14 | 15 | # https://github.com/jordi/version_sorter/blob/master/version_sorter.rb 16 | module VersionSorter 17 | extend self 18 | 19 | def sort(list) 20 | ss = StringScanner.new '' 21 | pad_to = 0 22 | list.each { |li| pad_to = li.size if li.size > pad_to } 23 | 24 | list.sort_by do |li| 25 | ss.string = li 26 | parts = '' 27 | 28 | begin 29 | if match = ss.scan_until(/\d+|[a-z]+/i) 30 | parts << match.rjust(pad_to) 31 | else 32 | break 33 | end 34 | end until ss.eos? 35 | 36 | parts 37 | end 38 | end 39 | 40 | def rsort(list) 41 | sort(list).reverse! 42 | end 43 | end 44 | 45 | if not File.exists?("projects") 46 | puts "No projects found." 47 | exit 1 48 | end 49 | 50 | projects = {} 51 | 52 | Dir.entries("projects").each do | projectdir | 53 | next if projectdir == "." or projectdir == ".." 54 | projectname, projectversion = projectdir.split("-") 55 | next if ignore.include?(projectname) 56 | if ARGV.count > 0 57 | inargs = false 58 | ARGV.each do | arg | 59 | if projectname == arg 60 | inargs = true 61 | end 62 | end 63 | next unless inargs; 64 | end 65 | if not projects[projectname] 66 | projects[projectname] = [ projectversion ] 67 | else 68 | projects[projectname].push(projectversion) 69 | end 70 | end 71 | 72 | basedir = Dir.pwd 73 | 74 | if not ENV.include?('GITHUB_AUTH') 75 | puts "Warning: GITHUB_AUTH not defined, will not create GitHub projects" 76 | end 77 | 78 | index = open("projects.html", "w") 79 | 80 | basehref = "http://github.com/aosm/" 81 | index.puts "

#{project}: " 86 | 87 | Dir.chdir(basedir) 88 | versions = VersionSorter.sort(projects[project]) 89 | gitdir = "gitify/#{project}" 90 | if not File.exists?(gitdir) 91 | Dir.mkdir(gitdir) 92 | end 93 | if not File.exists?("#{gitdir}/.git") 94 | system "git init #{gitdir}" 95 | end 96 | 97 | Dir.chdir(gitdir) 98 | json_file = "../../metadata/#{project}.github.json" 99 | if not File.size?(json_file) and ENV.include?('GITHUB_AUTH') 100 | puts "Creating GitHub repo" 101 | system "curl -f -i -u #{ENV['GITHUB_AUTH']} -H X-GitHub-OTP:#{ENV['GITHUB_OTP']} https://api.github.com/orgs/aosm/repos -d '{\"name\":\"#{project}\"}' > #{json_file}" 102 | end 103 | if ENV.include?('GITHUB_AUTH') and `git remote`.chomp.eql?("") 104 | puts "Adding remote" 105 | system "git remote add origin git@github.com:aosm/#{project}" 106 | end 107 | 108 | puts "Generating HTML" 109 | index.puts "versions:
- #{version} " 113 | srcdir = "#{basedir}/projects/#{project}-#{version}" 114 | xattrs = `xattr -l #{srcdir} | grep '^nu\.dll.aosm\.' | cut -f 1 -d :`.split(/[\r\n]+/) 115 | index.puts " in OS releases: " 116 | xattrs.each do | xattr | 117 | tag = xattr.sub("nu.dll.aosm.", "") 118 | if not version_os_printed.include?(tag) 119 | index.puts "#{tag} " 120 | version_os_printed[tag] = true 121 | end 122 | end 123 | end 124 | index.puts "
" 125 | 126 | if not `git tag -l #{versions[-1]}`.chomp.eql?("") 127 | puts "The latest version (#{versions[-1]}) already imported to git" 128 | else 129 | versions.each do | version | 130 | puts "VERSION: #{version}" 131 | if not `git tag -l #{version}`.chomp.eql?("") 132 | puts "Version #{version} already imported to git" 133 | next 134 | end 135 | puts "Importing version #{version}" 136 | srcdir="#{basedir}/projects/#{project}-#{version}" 137 | xattrs=`xattr -l #{srcdir} | grep '^nu\.dll.aosm\.' | cut -f 1 -d :`.split(/[\r\n]+/) 138 | if not system "rsync -avz --delete --exclude=.git #{srcdir}/ ." 139 | puts "Error: failed to rsync #{project} version #{version}" 140 | exit 1 141 | end 142 | if not system "git add ." 143 | puts "Error: failed to add #{project} version #{version} to git" 144 | exit 1 145 | end 146 | gitresult = `git commit -am \"version #{version}\"` 147 | puts gitresult 148 | if $?.to_i != 0 149 | if gitresult =~ /nothing to commit/ 150 | puts "Warning: nothing to commit for #{project} version #{version}" 151 | else 152 | puts "Error: failed to commit #{project} version #{version} to git" 153 | exit 1 154 | end 155 | end 156 | if not system "git tag #{version}" 157 | puts "Error: failed to tag #{project} version #{version}" 158 | exit 1 159 | end 160 | xattrs.each do | xattr | 161 | tag = xattr.sub("nu.dll.aosm.", "") 162 | if not system "git tag #{tag}" 163 | puts "Error: failed to set release tag #{tag} for #{project} version #{version}" 164 | exit 1 165 | end 166 | end 167 | end 168 | end 169 | tree_push_file = "../../metadata/#{project}-#{versions[-1]}.tree.pushed" 170 | tags_push_file = "../../metadata/#{project}-#{versions[-1]}.tags.pushed" 171 | if not File.exists?(tree_push_file) 172 | if system "git push --all -u origin" 173 | system "touch #{tree_push_file}" 174 | end 175 | end 176 | if not File.exists?(tags_push_file) 177 | if system "git push --tags" 178 | system "touch #{tags_push_file}" 179 | end 180 | end 181 | end 182 | index.puts "

" 183 | index.close 184 | --------------------------------------------------------------------------------