├── Makefile
├── README
├── fetch-all.sh
├── fetch.sh
├── footer.html
├── gitify.rb
├── gitify
    └── xref.sh
├── header.html
└── mirror.rb


/Makefile:
--------------------------------------------------------------------------------
1 | index.html: header.html projects.html footer.html
2 | 	cat header.html projects.html footer.html | sed -e 's/%DATE%/'"`date`"/g > $@
3 | 
4 | projects.html: gitify.rb
5 | 	./gitify.rb
6 | 
7 | 


--------------------------------------------------------------------------------
/README:
--------------------------------------------------------------------------------
 1 | -*- Mode: auto-fill -*-
 2 | 
 3 | These scripts helps you download and extract packages from
 4 | opensource.apple.com or even maintain an extracted mirror with the
 5 | latest open source packages from a Mac OS X release (10.8.1 by
 6 | default).
 7 | 
 8 | Note that the .tar.gz will not be saved; only its extracted contents.
 9 | 
10 | Prerequisite:
11 | 
12 |  sudo gem install plist minitar
13 | 
14 | Usage:
15 | 
16 | Run "./mirror.rb" to list all available packages.
17 | Run "./mirror.rb package1 package2 ..." to download package1, package2
18 | and so on.
19 | The script will skip packages that have already been downloaded.
20 | 
21 | To mirror all packages in a certain OS X release from
22 | opensource.apple.com, you can run the following command:
23 |   ./fetch.sh 10.8.1
24 | 
25 | As of 10.8.1, the CF and WebKit2 packages are missing, and a full
26 | mirror uses about 2.1 GB of disk space.
27 | 


--------------------------------------------------------------------------------
/fetch-all.sh:
--------------------------------------------------------------------------------
1 | curl http://www.opensource.apple.com|grep 'href="/release/mac-os-x'|sed -e 's/.*mac-os-x-\([0-9][^/]*\)\/.*/\1/'|sort|xargs -n 20 -P 5 ./fetch.sh 
2 | 


--------------------------------------------------------------------------------
/fetch.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash -e
2 | 
3 | for release; do
4 |     ./mirror.rb --release "$release" | grep -v '^#' | xargs ./mirror.rb --release "$release"
5 |     [ ${PIPESTATUS[0]} -eq 0 ] && [ ${PIPESTATUS[0]} -eq 0 ]
6 | done


--------------------------------------------------------------------------------
/footer.html:
--------------------------------------------------------------------------------
1 | <p>This page was generated on %DATE%</p>
2 | </body>
3 | </html>
4 | 


--------------------------------------------------------------------------------
/gitify.rb:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env ruby
  2 | #
  3 | # This script iterates over all projects downloaded by mirror.rb and fetch.sh,
  4 | # creates a git repository for each project and attempts to create a git commit
  5 | # for each project version in the correct order. It also creates git tags for
  6 | # the OS releases containing a particular version, making it easy to track
  7 | # changes between OS releases and e.g. get diffs between arbitrary versions
  8 | #
  9 | 
 10 | # problematic projects
 11 | ignore = [ 'libauto' ]
 12 | 
 13 | require 'strscan'
 14 | 
 15 | # https://github.com/jordi/version_sorter/blob/master/version_sorter.rb
 16 | module VersionSorter
 17 |   extend self
 18 | 
 19 |   def sort(list)
 20 |     ss     = StringScanner.new ''
 21 |     pad_to = 0
 22 |     list.each { |li| pad_to = li.size if li.size > pad_to }
 23 | 
 24 |     list.sort_by do |li|
 25 |       ss.string = li
 26 |       parts     = ''
 27 | 
 28 |       begin
 29 |         if match = ss.scan_until(/\d+|[a-z]+/i)
 30 |           parts << match.rjust(pad_to)
 31 |         else
 32 |           break
 33 |         end
 34 |       end until ss.eos?
 35 | 
 36 |       parts
 37 |     end
 38 |   end
 39 | 
 40 |   def rsort(list)
 41 |     sort(list).reverse!
 42 |   end
 43 | end
 44 | 
 45 | if not File.exists?("projects")
 46 |   puts "No projects found."
 47 |   exit 1
 48 | end
 49 | 
 50 | projects = {}
 51 | 
 52 | Dir.entries("projects").each do | projectdir |
 53 |   next if projectdir == "." or projectdir == ".."
 54 |   projectname, projectversion = projectdir.split("-")
 55 |   next if ignore.include?(projectname)
 56 |   if ARGV.count > 0
 57 |     inargs = false
 58 |     ARGV.each do | arg |
 59 |       if projectname == arg
 60 |         inargs = true
 61 |       end
 62 |     end
 63 |     next unless inargs;
 64 |   end
 65 |   if not projects[projectname]
 66 |     projects[projectname] = [ projectversion ]
 67 |   else
 68 |     projects[projectname].push(projectversion)
 69 |   end
 70 | end
 71 | 
 72 | basedir = Dir.pwd
 73 | 
 74 | if not ENV.include?('GITHUB_AUTH')
 75 |   puts "Warning: GITHUB_AUTH not defined, will not create GitHub projects"
 76 | end
 77 | 
 78 | index = open("projects.html", "w")
 79 | 
 80 | basehref = "http://github.com/aosm/"
 81 | index.puts "<ul>"
 82 | 
 83 | projects.keys.sort.each do | project | 
 84 |   puts "PROJECT: #{project}"
 85 |   index.puts "<li><a id=\"#{project}\" href=\"#{basehref}#{project}\">#{project}</a>: "
 86 | 
 87 |   Dir.chdir(basedir)
 88 |   versions = VersionSorter.sort(projects[project])
 89 |   gitdir = "gitify/#{project}"
 90 |   if not File.exists?(gitdir)
 91 |     Dir.mkdir(gitdir)
 92 |   end
 93 |   if not File.exists?("#{gitdir}/.git")
 94 |     system "git init #{gitdir}"
 95 |   end
 96 | 
 97 |   Dir.chdir(gitdir)
 98 |   json_file = "../../metadata/#{project}.github.json"
 99 |   if not File.size?(json_file) and ENV.include?('GITHUB_AUTH')
100 |     puts "Creating GitHub repo"
101 |     system "curl -f -i -u #{ENV['GITHUB_AUTH']} -H X-GitHub-OTP:#{ENV['GITHUB_OTP']} https://api.github.com/orgs/aosm/repos -d '{\"name\":\"#{project}\"}' > #{json_file}"
102 |   end
103 |   if ENV.include?('GITHUB_AUTH') and `git remote`.chomp.eql?("")
104 |     puts "Adding remote"
105 |     system "git remote add origin git@github.com:aosm/#{project}"
106 |   end
107 | 
108 |   puts "Generating HTML"
109 |   index.puts "versions: <ul>"
110 |   versions.each do | version |
111 |     version_os_printed = {}
112 |     index.puts "<li><a id=\"#{project}-#{version}\" href=\"#{basehref}#{project}/tree/#{version}\">#{version}</a> "
113 |     srcdir = "#{basedir}/projects/#{project}-#{version}"
114 |     xattrs = `xattr -l #{srcdir} | grep '^nu\.dll.aosm\.' | cut -f 1 -d :`.split(/[\r\n]+/)
115 |     index.puts " in OS releases: "
116 |     xattrs.each do | xattr |
117 |       tag = xattr.sub("nu.dll.aosm.", "")
118 |       if not version_os_printed.include?(tag)
119 |         index.puts "<a href=\"#{basehref}#{project}/tree/#{tag}\">#{tag}</a> "
120 |         version_os_printed[tag] = true
121 |       end
122 |     end
123 |   end
124 |   index.puts "</ul>"
125 | 
126 |   if not `git tag -l #{versions[-1]}`.chomp.eql?("")
127 |     puts "The latest version (#{versions[-1]}) already imported to git"
128 |   else
129 |     versions.each do | version | 
130 |       puts "VERSION: #{version}"
131 |       if not `git tag -l #{version}`.chomp.eql?("")
132 |         puts "Version #{version} already imported to git"
133 |         next
134 |       end
135 |       puts "Importing version #{version}"
136 |       srcdir="#{basedir}/projects/#{project}-#{version}"
137 |       xattrs=`xattr -l #{srcdir} | grep '^nu\.dll.aosm\.' | cut -f 1 -d :`.split(/[\r\n]+/)
138 |       if not system "rsync -avz --delete --exclude=.git #{srcdir}/ ."
139 |         puts "Error: failed to rsync #{project} version #{version}"
140 |         exit 1
141 |       end
142 |       if not system "git add ."
143 |         puts "Error: failed to add #{project} version #{version} to git"
144 |         exit 1
145 |       end
146 |       gitresult = `git commit -am \"version #{version}\"`
147 |       puts gitresult
148 |       if $?.to_i != 0
149 |         if gitresult =~ /nothing to commit/
150 |           puts "Warning: nothing to commit for #{project} version #{version}"
151 |         else
152 |           puts "Error: failed to commit #{project} version #{version} to git"
153 |           exit 1
154 |         end
155 |       end    
156 |       if not system "git tag #{version}"
157 |         puts "Error: failed to tag #{project} version #{version}"
158 |         exit 1
159 |       end
160 |       xattrs.each do | xattr |
161 |         tag = xattr.sub("nu.dll.aosm.", "")
162 |         if not system "git tag #{tag}" 
163 |           puts "Error: failed to set release tag #{tag} for #{project} version #{version}"
164 |           exit 1
165 |         end
166 |       end
167 |     end
168 |   end
169 |   tree_push_file = "../../metadata/#{project}-#{versions[-1]}.tree.pushed"
170 |   tags_push_file = "../../metadata/#{project}-#{versions[-1]}.tags.pushed"
171 |   if not File.exists?(tree_push_file)
172 |     if system "git push --all -u origin"
173 |       system "touch #{tree_push_file}"
174 |     end
175 |   end
176 |   if not File.exists?(tags_push_file)
177 |     if system "git push --tags"
178 |       system "touch #{tags_push_file}"
179 |     end
180 |   end
181 | end
182 | index.puts "</ul>"
183 | index.close
184 | 


--------------------------------------------------------------------------------
/gitify/xref.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | for project in * ; do
 4 |     (
 5 | 	echo -n $project
 6 | 	cd $project
 7 | 	versions=$(git tag|grep -v os-x-|tr "\n" " ")
 8 | 	echo -n " versions:"
 9 | 	for version in $versions ; do
10 | 	    inosversions=$(git tag --points-at $version|grep os-x-|tr "\n" " ")
11 | 	    echo -n " $version"
12 | 	    for osversion in $inosversions ; do
13 | 		echo -n " ($osversion)"
14 | 		xattr -w nu.dll.aosm.${osversion} `date +%s` ../../projects/${project}-${version}
15 | 	    done
16 | 	done
17 | 	echo ""
18 |     )
19 | done
20 | 


--------------------------------------------------------------------------------
/header.html:
--------------------------------------------------------------------------------
1 | <html><head><title>Unofficial Fruit Company Open Source Git Mirror</title></head>
2 | <body>
3 | 
4 | <h2>Unofficial Fruit Company Open Source Git Mirror</h2>
5 | 
6 | <p>The <a href="https://github.com/aosm">Unofficial Fruit Company Open Source Git Mirror</a>, or AOSM for short, is a collection of git repositories created from the <a href="http://www.opensource.apple.com">Fruit Company Open Source web page</a>, with tags for each project version and OS release. This makes it easy to, for example, compare <a href="https://github.com/aosm/xnu/tree/mac-os-x-1071">xnu in 10.7.1</a> to <a href="https://github.com/aosm/xnu/tree/mac-os-x-1074">xnu in 10.7.4</a> using regular git diffing tools, or see which OS releases contained <a href="#zlib-37.2">zlib 37.2 (upstream zlib version 1.2.5)</a> (project version numbers generally don't correspond to upstream project version numbers).
7 | </p>
8 | 


--------------------------------------------------------------------------------
/mirror.rb:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env ruby
 2 | # -*- coding: utf-8 -*-
 3 | 
 4 | require 'open-uri'
 5 | require 'rubygems'
 6 | require 'plist'
 7 | require 'zlib'
 8 | require 'archive/tar/minitar'
 9 | include Archive::Tar
10 | 
11 | release = 'mac-os-x-1081'
12 | targetdir = 'projects'
13 | tmpdir = '.projects.tmp'
14 | releaseurl = "http://opensource.apple.com/release/#{release}/"
15 | 
16 | if ARGV[0] == "--release"
17 |   ARGV.shift
18 |   release = "mac-os-x-" + ARGV.shift.gsub(/\./, "")
19 |   if release =~ /^mac-os-x-109/  || release =~ /^mac-os-x-1010/
20 |      release = release.sub(/^mac-/, "")
21 |   end
22 | end
23 | 
24 | releaseplist = "http://opensource.apple.com/plist/#{release}.plist"
25 | print "# Downloading package list from #{releaseplist}…"
26 | STDOUT.flush
27 | data = Plist.parse_xml(open(releaseplist).read)
28 | puts " OK"
29 | 
30 | if ARGV.length == 0
31 |   puts "# Listing all projects from #{releaseurl}"
32 |   data["projects"].each do | key, value |
33 |     puts "   #{key}"
34 |   end
35 |   exit(0)
36 | end
37 | 
38 | def tag_project(dir, release)
39 |     `xattr -w nu.dll.aosm.#{release} \`date +%s\` "#{dir}"`
40 | end
41 | 
42 | if not File.exists?(targetdir)
43 |   Dir.mkdir(targetdir)
44 | end
45 | 
46 | if not File.exists?(tmpdir)
47 |   Dir.mkdir(tmpdir)
48 | end
49 | 
50 | ARGV.each do | projectname |
51 |   project = data["projects"][projectname]
52 |   projectdir = "#{projectname}-#{project['version']}"
53 |   tgzurl = "http://opensource.apple.com/tarballs/#{projectname}/#{projectdir}.tar.gz"
54 |   fulltarget = File.join(targetdir, projectdir)
55 |   if File.exists?(fulltarget)
56 |     puts "#{projectdir} already downloaded, skipping."
57 |     tag_project(fulltarget, release)
58 |     next
59 |   end
60 |   print "Downloading and untarring #{tgzurl}…"
61 |   STDOUT.flush
62 |   begin
63 |     tmptarget = File.join(tmpdir, projectdir)
64 |     if not File.exists?(tmptarget)
65 |       Dir.mkdir(tmptarget)
66 |       File.chmod(0755, tmptarget)
67 |     end
68 |     tgz = Zlib::GzipReader.new(open(tgzurl))
69 |     Minitar.unpack(tgz, tmpdir)
70 |     File.rename(tmptarget, fulltarget)
71 |     tag_project(fulltarget, release)
72 |     puts " OK."
73 |   rescue OpenURI::HTTPError
74 |     puts " failed."
75 |   end
76 | end
77 | puts "Kthxbye."
78 | 


--------------------------------------------------------------------------------