├── arch_mirror.py
├── listcache.go
├── mkimage-arch-pacman.conf
├── mkimage-arch.sh
└── sync_all.py
/arch_mirror.py:
--------------------------------------------------------------------------------
1 | import datetime
2 | import os
3 | import subprocess
4 |
5 | MIRROR_SOURCE_ROOT = 'rsync://mirror.rit.edu/archlinux'
6 | MIRROR_DEST_ROOT = '/mirror/archlinux'
7 | PKG_ROOT = '/mirror/archlinux/all'
8 |
9 | cur_date = datetime.date.today()
10 |
11 | def get_latest_mirror():
12 | path = MIRROR_DEST_ROOT
13 |
14 | for i in xrange(3):
15 | dirs = os.listdir(path)
16 | try:
17 | dirs.remove('latest')
18 | except ValueError:
19 | pass
20 | try:
21 | dirs.remove('all')
22 | except ValueError:
23 | pass
24 | if len(dirs) == 0:
25 | return
26 | latest_dir = sorted(dirs)[-1]
27 | path = os.path.join(path, latest_dir)
28 |
29 | return path
30 |
31 | base_args = [
32 | 'rsync',
33 | '-rtlHh',
34 | '--progress',
35 | '--delete-after',
36 | '--delay-updates',
37 | '--copy-links',
38 | '--safe-links',
39 | '--max-delete=1000',
40 | '--delete-excluded',
41 | '--exclude=.*',
42 | '--hard-links',
43 | '--ignore-errors',
44 | ]
45 |
46 | repo_args = []
47 |
48 | print '***************************************************'
49 |
50 | for repo in ['core', 'extra', 'community', 'multilib']:
51 | args = list(base_args)
52 |
53 | source = MIRROR_SOURCE_ROOT + '/%s/os/x86_64/' % repo
54 | print 'source:', source
55 |
56 | dest = os.path.join(MIRROR_DEST_ROOT, cur_date.strftime('%Y/%m/%d'),
57 | '%s/os/x86_64/' % repo)
58 | print 'dest:', dest
59 |
60 | link_dest = get_latest_mirror()
61 | if link_dest:
62 | link_dest = os.path.join(link_dest, '%s/os/x86_64/' % repo)
63 |
64 | if link_dest != dest:
65 | args.append('--link-dest=%s' % link_dest)
66 | print 'link-dest:', link_dest
67 |
68 | args += [
69 | source,
70 | dest,
71 | ]
72 |
73 | repo_args.append(args)
74 |
75 | print '***************************************************'
76 |
77 | confirm = raw_input('Continue (y/N): ').lower()
78 | if confirm != 'y':
79 | exit(1)
80 |
81 | for args in repo_args:
82 | if not os.path.exists(args[-1]):
83 | os.makedirs(args[-1])
84 |
85 | subprocess.check_call(args)
86 |
87 | os.remove(os.path.join(MIRROR_DEST_ROOT, 'latest'))
88 | subprocess.check_call(['ln', '-s',
89 | os.path.join(MIRROR_DEST_ROOT, cur_date.strftime('%Y/%m/%d')),
90 | os.path.join(MIRROR_DEST_ROOT, 'latest'),
91 | ])
92 |
93 | src_dir = get_latest_mirror()
94 | for repo in ['core', 'extra', 'community', 'multilib']:
95 | path = os.path.join(src_dir, repo, 'os/x86_64')
96 |
97 | for pkg_name in os.listdir(path):
98 | if pkg_name.endswith('sig'):
99 | continue
100 |
101 | pkg_short = pkg_name.replace('-any.pkg.tar.xz', '')
102 | pkg_short = pkg_short.replace('-x86_64.pkg.tar.xz', '')
103 |
104 | pkg_path = os.path.join(path, pkg_name)
105 | sig_path = pkg_path + '.sig'
106 |
107 | pkg_out_path = os.path.join(PKG_ROOT, pkg_short)
108 | sig_out_path = pkg_out_path + '.sig'
109 |
110 | subprocess.check_call(['ln', '-sfn', pkg_path, pkg_out_path])
111 | subprocess.check_call(['ln', '-sfn', sig_path, sig_out_path])
112 |
--------------------------------------------------------------------------------
/listcache.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | "io/ioutil"
6 | "os"
7 | "path/filepath"
8 | "sort"
9 | "strings"
10 | )
11 |
12 | const body = `
13 | Index of %s
14 |
15 | Index of %s
../
16 | %s
17 |
18 | `
19 |
20 | type Item struct {
21 | Name string
22 | IsDir bool
23 | Formatted string
24 | }
25 |
26 | type Items struct {
27 | items []Item
28 | }
29 |
30 | func (s *Items) Len() (n int) {
31 | n = len(s.items)
32 | return
33 | }
34 |
35 | func (s *Items) Less(i int, j int) bool {
36 | if s.items[i].IsDir && !s.items[j].IsDir {
37 | return true
38 | }
39 | if s.items[i].Name[:1] == "." && s.items[i].Name[:1] != "." {
40 | return true
41 | }
42 | return s.items[i].Name < s.items[j].Name
43 | }
44 |
45 | func (s *Items) Swap(i int, j int) {
46 | s.items[i], s.items[j] = s.items[j], s.items[i]
47 | }
48 |
49 | func (s *Items) Add(item Item) {
50 | s.items = append(s.items, item)
51 | }
52 |
53 | func (s *Items) Sort() {
54 | sort.Sort(s)
55 | }
56 |
57 | func (s *Items) Join(sep string) (data string) {
58 | for i, item := range s.items {
59 | if i != 0 {
60 | data += sep
61 | }
62 | data += item.Formatted
63 | }
64 | return
65 | }
66 |
67 | func main() {
68 | path := filepath.Clean(os.Args[1])
69 | pathFrm := strings.Replace(path, "/mirror", "", 1) + "/"
70 | if pathFrm[:1] != "/" {
71 | pathFrm = "/" + pathFrm
72 | }
73 |
74 | items := &Items{}
75 |
76 | itemsAll, err := ioutil.ReadDir(path)
77 | if err != nil {
78 | panic(err)
79 | }
80 |
81 | for _, item := range itemsAll {
82 | name := item.Name()
83 | if name == "index.html" {
84 | continue
85 | }
86 |
87 | modTime := item.ModTime().Format("02-Jan-2006 15:04")
88 |
89 | if item.Mode()&os.ModeSymlink != 0 {
90 | linkPath, err := os.Readlink(filepath.Join(path, item.Name()))
91 | if err != nil {
92 | panic(err)
93 | }
94 |
95 | itm, err := os.Lstat(linkPath)
96 | if err != nil {
97 | if os.IsNotExist(err) {
98 | continue
99 | }
100 | panic(err)
101 | }
102 | item = itm
103 | }
104 |
105 | size := ""
106 | if item.IsDir() {
107 | name += "/"
108 | size = "-"
109 | } else {
110 | size = fmt.Sprintf("%d", item.Size())
111 | }
112 |
113 | formattedName := name
114 | if len(formattedName) > 50 {
115 | formattedName = formattedName[:47] + "..>"
116 | }
117 |
118 | items.Add(Item{
119 | Name: name,
120 | IsDir: item.IsDir(),
121 | Formatted: fmt.Sprintf(
122 | ``, name) + fmt.Sprintf(
123 | "%-54s %s % 19s", formattedName+"", modTime, size),
124 | })
125 | }
126 |
127 | items.Sort()
128 |
129 | fmt.Printf(body, pathFrm, pathFrm, items.Join("\n"))
130 | }
131 |
--------------------------------------------------------------------------------
/mkimage-arch-pacman.conf:
--------------------------------------------------------------------------------
1 | #
2 | # /etc/pacman.conf
3 | #
4 | # See the pacman.conf(5) manpage for option and repository directives
5 |
6 | #
7 | # GENERAL OPTIONS
8 | #
9 | [options]
10 | # The following paths are commented out with their default values listed.
11 | # If you wish to use different paths, uncomment and update the paths.
12 | #RootDir = /
13 | #DBPath = /var/lib/pacman/
14 | #CacheDir = /var/cache/pacman/pkg/
15 | #LogFile = /var/log/pacman.log
16 | #GPGDir = /etc/pacman.d/gnupg/
17 | HoldPkg = pacman glibc
18 | #XferCommand = /usr/bin/curl -C - -f %u > %o
19 | #XferCommand = /usr/bin/wget --passive-ftp -c -O %o %u
20 | #CleanMethod = KeepInstalled
21 | #UseDelta = 0.7
22 | Architecture = auto
23 |
24 | # Pacman won't upgrade packages listed in IgnorePkg and members of IgnoreGroup
25 | #IgnorePkg =
26 | #IgnoreGroup =
27 |
28 | #NoUpgrade =
29 | #NoExtract =
30 |
31 | # Misc options
32 | #UseSyslog
33 | #Color
34 | #TotalDownload
35 | #CheckSpace
36 | #VerbosePkgLists
37 |
38 | # By default, pacman accepts packages signed by keys that its local keyring
39 | # trusts (see pacman-key and its man page), as well as unsigned packages.
40 | SigLevel = Required DatabaseOptional
41 | LocalFileSigLevel = Optional
42 | #RemoteFileSigLevel = Required
43 |
44 | # NOTE: You must run `pacman-key --init` before first using pacman; the local
45 | # keyring can then be populated with the keys of all official Arch Linux
46 | # packagers with `pacman-key --populate archlinux`.
47 |
48 | #
49 | # REPOSITORIES
50 | # - can be defined here or included from another file
51 | # - pacman will search repositories in the order defined here
52 | # - local/custom mirrors can be added here or in separate files
53 | # - repositories listed first will take precedence when packages
54 | # have identical names, regardless of version number
55 | # - URLs will have $repo replaced by the name of the current repo
56 | # - URLs will have $arch replaced by the name of the architecture
57 | #
58 | # Repository entries are of the format:
59 | # [repo-name]
60 | # Server = ServerName
61 | # Include = IncludePath
62 | #
63 | # The header [repo-name] is crucial - it must be present and
64 | # uncommented to enable the repo.
65 | #
66 |
67 | # The testing repositories are disabled by default. To enable, uncomment the
68 | # repo name header and Include lines. You can add preferred servers immediately
69 | # after the header, and they will be used before the default mirrors.
70 |
71 | #[testing]
72 | #Include = /etc/pacman.d/mirrorlist
73 |
74 | [core]
75 | Include = /etc/pacman.d/mirrorlist
76 |
77 | [extra]
78 | Include = /etc/pacman.d/mirrorlist
79 |
80 | #[community-testing]
81 | #Include = /etc/pacman.d/mirrorlist
82 |
83 | [community]
84 | Include = /etc/pacman.d/mirrorlist
85 |
86 | # If you want to run 32 bit applications on your x86_64 system,
87 | # enable the multilib repositories as required here.
88 |
89 | #[multilib-testing]
90 | #Include = /etc/pacman.d/mirrorlist
91 |
92 | #[multilib]
93 | #Include = /etc/pacman.d/mirrorlist
94 |
95 | # An example of a custom package repository. See the pacman manpage for
96 | # tips on creating your own repositories.
97 | #[custom]
98 | #SigLevel = Optional TrustAll
99 | #Server = file:///home/custompkgs
100 |
--------------------------------------------------------------------------------
/mkimage-arch.sh:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env bash
2 | # Generate a minimal filesystem for archlinux and load it into the local
3 | # docker as "archlinux"
4 | # requires root
5 | set -e
6 |
7 | hash pacstrap &> /dev/null || {
8 | echo "Could not find pacstrap. Run pacman -S arch-install-scripts"
9 | exit 1
10 | }
11 |
12 | hash expect &> /dev/null || {
13 | echo "Could not find expect. Run pacman -S expect"
14 | exit 1
15 | }
16 |
17 | REPO_DATE=$1
18 | ROOTFS=$(mktemp -d ${TMPDIR:-/var/tmp}/rootfs-archlinux-XXXXXXXXXX)
19 | chmod 755 $ROOTFS
20 |
21 | echo $REPO_DATE | grep -q . || {
22 | echo "Must supply repo date"
23 | exit 1
24 | }
25 |
26 | curl -o /dev/null --silent --head --write-out '%{http_code}' 'http://mirror.pritunl.com/archlinux/'$REPO_DATE'/' | grep -q 200 || {
27 | echo 'Could not verify mirror. http://mirror.pritunl.com/archlinux/'$REPO_DATE'/'
28 | exit 1
29 | }
30 |
31 | # packages to ignore for space savings
32 | PKGIGNORE=(
33 | cryptsetup
34 | device-mapper
35 | dhcpcd
36 | iproute2
37 | jfsutils
38 | linux
39 | lvm2
40 | man-db
41 | man-pages
42 | mdadm
43 | nano
44 | netctl
45 | openresolv
46 | pciutils
47 | pcmciautils
48 | reiserfsprogs
49 | s-nail
50 | systemd-sysvcompat
51 | usbutils
52 | vi
53 | xfsprogs
54 | )
55 | IFS=','
56 | PKGIGNORE="${PKGIGNORE[*]}"
57 | unset IFS
58 |
59 | expect < $ROOTFS/etc/locale.gen
77 | arch-chroot $ROOTFS locale-gen
78 | arch-chroot $ROOTFS /bin/sh -c 'echo "Server = http://mirror.pritunl.com/archlinux/'$REPO_DATE'/\$repo/os/\$arch" > /etc/pacman.d/mirrorlist'
79 |
80 | # udev doesn't work in containers, rebuild /dev
81 | DEV=$ROOTFS/dev
82 | rm -rf $DEV
83 | mkdir -p $DEV
84 | mknod -m 666 $DEV/null c 1 3
85 | mknod -m 666 $DEV/zero c 1 5
86 | mknod -m 666 $DEV/random c 1 8
87 | mknod -m 666 $DEV/urandom c 1 9
88 | mkdir -m 755 $DEV/pts
89 | mkdir -m 1777 $DEV/shm
90 | mknod -m 666 $DEV/tty c 5 0
91 | mknod -m 600 $DEV/console c 5 1
92 | mknod -m 666 $DEV/tty0 c 4 0
93 | mknod -m 666 $DEV/full c 1 7
94 | mknod -m 600 $DEV/initctl p
95 | mknod -m 666 $DEV/ptmx c 5 2
96 | ln -sf /proc/self/fd $DEV/fd
97 |
98 | tar --numeric-owner --xattrs --acls -C $ROOTFS -c . | docker import - pritunl/archlinux
99 | docker run --rm -t pritunl/archlinux echo Success.
100 | rm -rf $ROOTFS
101 |
--------------------------------------------------------------------------------
/sync_all.py:
--------------------------------------------------------------------------------
1 | import os
2 | import subprocess
3 |
4 | MIRROR_DEST_ROOT = '/mirror/archlinux'
5 | PKG_ROOT = '/mirror/archlinux/all'
6 |
7 | def get_latest_mirror():
8 | path = MIRROR_DEST_ROOT
9 |
10 | for i in xrange(3):
11 | dirs = os.listdir(path)
12 | try:
13 | dirs.remove('latest')
14 | except ValueError:
15 | pass
16 | try:
17 | dirs.remove('all')
18 | except ValueError:
19 | pass
20 | if len(dirs) == 0:
21 | return
22 | latest_dir = sorted(dirs)[-1]
23 | path = os.path.join(path, latest_dir)
24 |
25 | return path
26 |
27 | src_dir = get_latest_mirror()
28 | for repo in ['core', 'extra', 'community', 'multilib']:
29 | path = os.path.join(src_dir, repo, 'os/x86_64')
30 | for pkg_name in os.listdir(path):
31 | if pkg_name.endswith('sig'):
32 | continue
33 |
34 | pkg_short = pkg_name.replace('-any.pkg.tar.xz', '')
35 | pkg_short = pkg_short.replace('-x86_64.pkg.tar.xz', '')
36 |
37 | pkg_path = os.path.join(path, pkg_name)
38 | sig_path = pkg_path + '.sig'
39 |
40 | pkg_out_path = os.path.join(PKG_ROOT, pkg_short)
41 | sig_out_path = pkg_out_path + '.sig'
42 |
43 | subprocess.check_call(['ln', '-sfn', pkg_path, pkg_out_path])
44 | subprocess.check_call(['ln', '-sfn', sig_path, sig_out_path])
45 |
--------------------------------------------------------------------------------