├── .gitignore
├── .gobuild.yml
├── Makefile
├── README.md
├── Vagrantfile
├── dlrootfs.go
├── dlrootfs
    └── dlrootfs.go
├── git.go
├── integration
    └── integration_test.go
├── pulling_job.go
├── queue.go
├── sample_configs
    ├── container.json
    └── lxc-config
├── setup.sh
├── utils.go
└── vendor.sh


/.gitignore:
--------------------------------------------------------------------------------
1 | rootfs
2 | .vagrant
3 | vendor
4 | release
5 | dlrootfs/dlrootfs
6 | 


--------------------------------------------------------------------------------
/.gobuild.yml:
--------------------------------------------------------------------------------
 1 | author: robinmonjo@gmail.com
 2 | description: >
 3 |   Command line tool to pull images from the docker hub, without docker installed
 4 | 
 5 | filesets:
 6 |   includes:
 7 |     - README.md
 8 |   excludes:
 9 |     - \.git
10 |   depth: 3
11 | 
12 | settings:
13 |   build: |
14 |     sh vendor.sh && export GOPATH=$GOPATH:`pwd`/vendor && cd dlrootfs && go install -v
15 | 


--------------------------------------------------------------------------------
/Makefile:
--------------------------------------------------------------------------------
 1 | HARDWARE=$(shell uname -m)
 2 | 
 3 | build:
 4 | 	cd dlrootfs && go build
 5 | 
 6 | release:
 7 | 	mkdir -p release
 8 | 	cd dlrootfs && GOOS=linux go build -o ../release/dlrootfs
 9 | 	cd release && tar -zcf dlrootfs_$(HARDWARE).tgz dlrootfs
10 | 
11 | 	rm release/dlrootfs
12 | 
13 | test:
14 | 	cd dlrootfs && go install
15 | 	cd integration && go test
16 | 
17 | clean:
18 | 	rm -rf release
19 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | > **dlrootfs is now [cargo](https://github.com/robinmonjo/cargo)**
 2 | 
 3 | > **cargo include dlrootfs features and brings the possibility to push on the docker hub**
 4 | >
 5 | 
 6 | 
 7 | ## dlrootfs (deprecated see [cargo](https://github.com/robinmonjo/cargo))
 8 | 
 9 | Download root file systems from the [docker hub](https://registry.hub.docker.com/) without docker
10 | 
11 | ````bash
12 | Usage: dlrootfs <image_name>:[<image_tag>] [-d <rootfs_destination>] [-u <username>:<password>] [-g]
13 | 
14 | Examples:
15 |   dlrootfs ubuntu  #if no tag, use latest
16 |   dlrootfs ubuntu:precise -d ubuntu_rootfs
17 |   dlrootfs dockefile/elasticsearch:latest
18 |   dlrootfs my_repo/my_image:latest -u username:password
19 |   dlrootfs version
20 | Default:
21 |   -d="./rootfs": destination of the resulting rootfs directory
22 |   -g=false: use git layering
23 |   -u="": docker hub credentials: <username>:<password>
24 | ````
25 | #### `-g` flag
26 | 
27 | As explained [in the doc](https://docs.docker.com/terms/layer/), docker images are a set of layers. Using the `-g` flag,
28 | `dlrootfs` will download the file system in a git repository where each layer is downloaded in a separate branch:
29 | 
30 | ![Alt text](https://dl.dropboxusercontent.com/u/6543817/dlrootfs-readme/dlrootfs-g.png)
31 | 
32 | The screenshot above is the resulting rootfs of `dlrootfs ubuntu -g`. We can clearly see the image is composed of 5 layers.
33 | `layer(n)_*` results from `git checkout -b layer(n-1)_*` with data from `layer(n)`.
34 | 
35 | It allows to use git to see diffs between layers, checkout a new branch, work on the rootfs with a container engine, review
36 | and commit changes, etc. It also opens the path for `docker push` without docker (coming soon).
37 | 
38 | ### Installation
39 | 
40 | ````bash
41 | curl -sL https://github.com/robinmonjo/dlrootfs/releases/download/v1.4.0/dlrootfs_x86_64.tgz | tar -C /usr/local/bin -zxf -
42 | ````
43 | 
44 | Provided binary is linux only but `dlrootfs` may be used on OSX and (probably) windows too.
45 | The difference is, when ran on a linux box, `dlrootfs` will perform `lchown` during layer extraction,
46 | it won't otherwise.
47 | 
48 | Some images require you to be root during extraction (the official busybox image for example) why others won't
49 | (the official debian one).
50 | 
51 | ### Why dlrootfs ?
52 | 
53 | Docker has become really popular and lots of people and organisations are building docker images they store
54 | and share on the [docker hub](https://registry.hub.docker.com/). However these images are only available for
55 | docker's user. `dlrootfs` allows to download root file systems from the docker hub so they can be used
56 | with other container engines ([LXC](https://linuxcontainers.org/), [nsinit (`libcontainer`)](https://github.com/docker/libcontainer), [systemd-nspawn](http://0pointer.de/public/systemd-man/systemd-nspawn.html) ...)
57 | 
58 | 
59 | ##### Using docker images with nsinit
60 | 
61 | 1. Browse the [docker hub](https://registry.hub.docker.com/) and find the image you want (say [ubuntu](https://registry.hub.docker.com/u/library/ubuntu/))
62 | 2. Download ubuntu rootfs: `dlrootfs ubuntu`
63 | 3. `cd` to `rootfs` and create a `container.json` file (needed by `libcontainer`, you can use the sample config of this repository `sample_configs/container.json`).
64 | 4. Launch bash in the official Docker ubuntu image: `nsinit exec /bin/bash`
65 | 
66 | ##### Using docker images with LXC
67 | 
68 | 1. Browse the [docker hub](https://registry.hub.docker.com/) and find the image you want (say [ubuntu](https://registry.hub.docker.com/u/library/ubuntu/))
69 | 2. Download ubuntu rootfs: `dlrootfs ubuntu`
70 | 3. Create a `config` file (for examples the one you can find in `sample_configs/lxc-config`)
71 | 4. Do not forget to change the `config` to match your settings (especially rootfs location)
72 | 5. Launch bash in the "official Docker ubuntu image LXC container": `lxc-start -n ubuntu -f <config file> /bin/bash`
73 | 
74 | ### Warnings
75 | 
76 | * Untaring on the `vagrant` shared folder will fail
77 | * `cgroup-lite` is required for `nsinit`
78 | 
79 | ### License
80 | 
81 | MIT
82 | 


--------------------------------------------------------------------------------
/Vagrantfile:
--------------------------------------------------------------------------------
 1 | # -*- mode: ruby -*-
 2 | # vi: set ft=ruby :
 3 | 
 4 | # Vagrantfile API/syntax version. Don't touch unless you know what you're doing!
 5 | VAGRANTFILE_API_VERSION = "2"
 6 | 
 7 | Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
 8 |   config.vm.box = "trusty64"
 9 |   config.vm.box_url = "https://cloud-images.ubuntu.com/vagrant/trusty/current/trusty-server-cloudimg-amd64-vagrant-disk1.box"
10 | 
11 |   #supposed clean workspace: eg ~/code/go
12 |   config.vm.synced_folder "~/code/go", "/home/vagrant/code/go/"
13 | 
14 |   config.vm.provision "shell", path: "setup.sh"
15 | 
16 |   config.vm.network :public_network
17 | 
18 | end
19 | 


--------------------------------------------------------------------------------
/dlrootfs.go:
--------------------------------------------------------------------------------
  1 | package dlrootfs
  2 | 
  3 | import (
  4 | 	"encoding/json"
  5 | 	"fmt"
  6 | 	"io/ioutil"
  7 | 	"os"
  8 | 	"path"
  9 | 	"runtime"
 10 | 	"strconv"
 11 | 
 12 | 	"github.com/docker/docker/pkg/archive"
 13 | 	"github.com/docker/docker/registry"
 14 | )
 15 | 
 16 | const MAX_DL_CONCURRENCY int = 7
 17 | 
 18 | type HubSession struct {
 19 | 	registry.Session
 20 | 	RepoData *registry.RepositoryData
 21 | }
 22 | 
 23 | func NewHubSession(imageName, userName, password string) (*HubSession, error) {
 24 | 	hostname, _, err := registry.ResolveRepositoryName(imageName)
 25 | 	if err != nil {
 26 | 		return nil, fmt.Errorf("failed to find repository for image %v: %v", imageName, err)
 27 | 	}
 28 | 	endpoint, err := registry.NewEndpoint(hostname, []string{})
 29 | 	if err != nil {
 30 | 		return nil, err
 31 | 	}
 32 | 
 33 | 	authConfig := &registry.AuthConfig{}
 34 | 	if userName != "" && password != "" {
 35 | 		authConfig.Username = userName
 36 | 		authConfig.Password = password
 37 | 	}
 38 | 
 39 | 	var metaHeaders map[string][]string
 40 | 
 41 | 	session, err := registry.NewSession(authConfig, registry.HTTPRequestFactory(metaHeaders), endpoint, true)
 42 | 	if err != nil {
 43 | 		return nil, fmt.Errorf("failed to create docker hub session %v", err)
 44 | 	}
 45 | 
 46 | 	repoData, err := session.GetRepositoryData(imageName)
 47 | 	if err != nil {
 48 | 		return nil, fmt.Errorf("failed to get repository data %v", err)
 49 | 	}
 50 | 
 51 | 	return &HubSession{*session, repoData}, nil
 52 | }
 53 | 
 54 | func (s *HubSession) DownloadFlattenedImage(imageName, imageTag, rootfsDest string, gitLayering, printProgress bool) error {
 55 | 
 56 | 	tagsList, err := s.GetRemoteTags(s.RepoData.Endpoints, imageName, s.RepoData.Tokens)
 57 | 	if err != nil {
 58 | 		return fmt.Errorf("failed to retrieve tag list %v", err)
 59 | 	}
 60 | 
 61 | 	imageId := tagsList[imageTag]
 62 | 	if printProgress {
 63 | 		fmt.Printf("Image ID: %v\n", imageId)
 64 | 	}
 65 | 
 66 | 	//Download image history
 67 | 	var imageHistory []string
 68 | 	for _, ep := range s.RepoData.Endpoints {
 69 | 		imageHistory, err = s.GetRemoteHistory(imageId, ep, s.RepoData.Tokens)
 70 | 		if err == nil {
 71 | 			break
 72 | 		}
 73 | 	}
 74 | 	if err != nil {
 75 | 		return fmt.Errorf("failed to get back image history %v", err)
 76 | 	}
 77 | 
 78 | 	err = os.MkdirAll(rootfsDest, 0700)
 79 | 	if err != nil {
 80 | 		return fmt.Errorf("failed to create directory %v: %v", rootfsDest, err)
 81 | 	}
 82 | 
 83 | 	var gitRepo *GitRepo
 84 | 	if gitLayering {
 85 | 		if gitRepo, err = NewGitRepo(rootfsDest); err != nil {
 86 | 			return fmt.Errorf("failed to create git repository %v", err)
 87 | 		}
 88 | 	}
 89 | 
 90 | 	queue := NewQueue(MAX_DL_CONCURRENCY)
 91 | 
 92 | 	if printProgress {
 93 | 		fmt.Printf("Pulling %d layers:\n", len(imageHistory))
 94 | 	}
 95 | 
 96 | 	for i := len(imageHistory) - 1; i >= 0; i-- {
 97 | 		layerId := imageHistory[i]
 98 | 		job := NewPullingJob(s, layerId)
 99 | 		queue.Enqueue(job)
100 | 	}
101 | 	<-queue.DoneChan
102 | 
103 | 	if printProgress {
104 | 		fmt.Printf("Downloading layers:\n")
105 | 	}
106 | 
107 | 	//no lchown if not on linux
108 | 	tarOptions := &archive.TarOptions{NoLchown: false}
109 | 	if runtime.GOOS != "linux" {
110 | 		tarOptions.NoLchown = true
111 | 	}
112 | 
113 | 	cpt := 0
114 | 
115 | 	for i := len(imageHistory) - 1; i >= 0; i-- {
116 | 
117 | 		//for each layers
118 | 		layerId := imageHistory[i]
119 | 
120 | 		if printProgress {
121 | 			fmt.Printf("\t%v ... ", truncateID(layerId))
122 | 		}
123 | 
124 | 		if gitLayering {
125 | 			//create a git branch
126 | 			if _, err = gitRepo.CheckoutB("layer" + strconv.Itoa(cpt) + "_" + layerId); err != nil {
127 | 				return fmt.Errorf("failed to checkout %v", err)
128 | 			}
129 | 		}
130 | 
131 | 		//download and untar the layer
132 | 		job := queue.CompletedJobWithID(layerId).(*PullingJob)
133 | 		err = archive.Untar(job.LayerData, rootfsDest, tarOptions)
134 | 		job.LayerData.Close()
135 | 		if err != nil {
136 | 			return err
137 | 		}
138 | 
139 | 		//write image info
140 | 		var layerInfo map[string]interface{}
141 | 		err = json.Unmarshal(job.LayerInfo, &layerInfo)
142 | 		if err != nil {
143 | 			return err
144 | 		}
145 | 
146 | 		prettyInfo, _ := json.MarshalIndent(layerInfo, "", "  ")
147 | 		ioutil.WriteFile(path.Join(rootfsDest, "json"), prettyInfo, 0644)
148 | 		if gitLayering {
149 | 			ioutil.WriteFile(path.Join(rootfsDest, "layersize"), []byte(strconv.Itoa(job.LayerSize)), 0644)
150 | 		}
151 | 
152 | 		if gitLayering {
153 | 			_, err = gitRepo.Add(".")
154 | 			if err != nil {
155 | 				return fmt.Errorf("failed to add changes %v", err)
156 | 			}
157 | 			_, err = gitRepo.Commit("adding layer " + strconv.Itoa(cpt))
158 | 			if err != nil {
159 | 				return fmt.Errorf("failed to commit changes %v", err)
160 | 			}
161 | 		}
162 | 
163 | 		cpt++
164 | 
165 | 		if printProgress {
166 | 			fmt.Printf("done\n")
167 | 		}
168 | 	}
169 | 	return nil
170 | }
171 | 


--------------------------------------------------------------------------------
/dlrootfs/dlrootfs.go:
--------------------------------------------------------------------------------
 1 | package main
 2 | 
 3 | import (
 4 | 	"flag"
 5 | 	"fmt"
 6 | 	"log"
 7 | 	"os"
 8 | 
 9 | 	"github.com/rmonjo/dlrootfs"
10 | )
11 | 
12 | const VERSION string = "1.4.0"
13 | 
14 | var (
15 | 	flagset = flag.NewFlagSet("dlrootfs", flag.ExitOnError)
16 | 
17 | 	rootfsDest  *string = flagset.String("d", "./rootfs", "destination of the resulting rootfs directory")
18 | 	credentials *string = flagset.String("u", "", "docker hub credentials: <username>:<password>")
19 | 	gitLayering *bool   = flagset.Bool("g", false, "use git layering")
20 | )
21 | 
22 | func init() {
23 | 	flagset.Usage = func() {
24 | 		fmt.Fprintf(os.Stderr, "Usage: dlrootfs <image_name>:[<image_tag>] [-d <rootfs_destination>] [-u <username>:<password>] [-g]\n\n")
25 | 		fmt.Fprintf(os.Stderr, "Examples:\n")
26 | 		fmt.Fprintf(os.Stderr, "  dlrootfs ubuntu  #if no tag, use latest\n")
27 | 		fmt.Fprintf(os.Stderr, "  dlrootfs ubuntu:precise -d ubuntu_rootfs\n")
28 | 		fmt.Fprintf(os.Stderr, "  dlrootfs dockefile/elasticsearch:latest\n")
29 | 		fmt.Fprintf(os.Stderr, "  dlrootfs my_repo/my_image:latest -u username:password\n")
30 | 		fmt.Fprintf(os.Stderr, "  dlrootfs version\n")
31 | 		fmt.Fprintf(os.Stderr, "Default:\n")
32 | 		flagset.PrintDefaults()
33 | 	}
34 | }
35 | 
36 | func main() {
37 | 
38 | 	if len(os.Args) <= 1 {
39 | 		flagset.Usage()
40 | 		return
41 | 	}
42 | 
43 | 	imageNameTag := os.Args[1]
44 | 
45 | 	switch imageNameTag {
46 | 	case "version":
47 | 		fmt.Println(VERSION)
48 | 	case "":
49 | 		flagset.Usage()
50 | 	default:
51 | 		pullImage(imageNameTag, os.Args[2:])
52 | 	}
53 | }
54 | 
55 | func pullImage(imageNameTag string, args []string) {
56 | 	flagset.Parse(args)
57 | 
58 | 	imageName, imageTag := dlrootfs.ParseImageNameTag(imageNameTag)
59 | 	userName, password := dlrootfs.ParseCredentials(*credentials)
60 | 
61 | 	fmt.Printf("Opening a session for %v ...\n", imageName)
62 | 	session, err := dlrootfs.NewHubSession(imageName, userName, password)
63 | 	if err != nil {
64 | 		log.Fatal(err)
65 | 	}
66 | 
67 | 	err = session.DownloadFlattenedImage(imageName, imageTag, *rootfsDest, *gitLayering, true)
68 | 	if err != nil {
69 | 		log.Fatal(err)
70 | 	}
71 | 
72 | 	fmt.Printf("\nRootfs of %v:%v in %v\n", imageName, imageTag, *rootfsDest)
73 | 	if *credentials != "" {
74 | 		fmt.Printf("WARNING: don't forget to remove your docker hub credentials from your history !!\n")
75 | 	}
76 | }
77 | 


--------------------------------------------------------------------------------
/git.go:
--------------------------------------------------------------------------------
 1 | package dlrootfs
 2 | 
 3 | import (
 4 | 	"os"
 5 | 	"os/exec"
 6 | )
 7 | 
 8 | type GitRepo struct {
 9 | 	Path string
10 | }
11 | 
12 | func NewGitRepo(path string) (*GitRepo, error) {
13 | 	r := &GitRepo{Path: path}
14 | 
15 | 	if _, err := os.Stat(path + "/.git"); err == nil {
16 | 		return r, nil
17 | 	}
18 | 
19 | 	_, err := r.exec("init", path)
20 | 	if err != nil {
21 | 		return nil, err
22 | 	}
23 | 	_, err = r.execInWorkTree("config", "user.email", "mail@mail.com")
24 | 	if err != nil {
25 | 		return nil, err
26 | 	}
27 | 
28 | 	_, err = r.execInWorkTree("config", "user.name", "dlrootfs")
29 | 	if err != nil {
30 | 		return nil, err
31 | 	}
32 | 
33 | 	return r, nil
34 | }
35 | 
36 | func (r *GitRepo) Checkout(branch string) ([]byte, error) {
37 | 	return r.execInWorkTree("checkout", branch)
38 | }
39 | 
40 | func (r *GitRepo) CheckoutB(branch string) ([]byte, error) {
41 | 	return r.execInWorkTree("checkout", "-b", branch)
42 | }
43 | 
44 | func (r *GitRepo) Add(file string) ([]byte, error) {
45 | 	return r.execInWorkTree("add", file)
46 | }
47 | 
48 | func (r *GitRepo) Commit(message string) ([]byte, error) {
49 | 	return r.execInWorkTree("commit", "-m", message)
50 | }
51 | 
52 | func (r *GitRepo) Branch() ([]byte, error) {
53 | 	return r.execInWorkTree("branch")
54 | }
55 | 
56 | func (r *GitRepo) execInWorkTree(args ...string) ([]byte, error) {
57 | 	args = append([]string{"--git-dir=" + r.Path + "/.git", "--work-tree=" + r.Path}, args...)
58 | 	return r.exec(args...)
59 | }
60 | 
61 | func (r *GitRepo) exec(args ...string) ([]byte, error) {
62 | 	gitPath, err := exec.LookPath("git")
63 | 	if err != nil {
64 | 		return nil, err
65 | 	}
66 | 	cmd := exec.Command(gitPath, args...)
67 | 	return cmd.CombinedOutput()
68 | }
69 | 


--------------------------------------------------------------------------------
/integration/integration_test.go:
--------------------------------------------------------------------------------
  1 | package main
  2 | 
  3 | import (
  4 | 	"fmt"
  5 | 	"os"
  6 | 	"os/exec"
  7 | 	"strings"
  8 | 	"testing"
  9 | 
 10 | 	"github.com/rmonjo/dlrootfs"
 11 | )
 12 | 
 13 | const CREDS_ENV string = "DLROOTFS_CREDS"
 14 | 
 15 | var (
 16 | 	dlrootfsBinary string   = "dlrootfs"
 17 | 	testImages     []string = []string{"busybox", "progrium/busybox"}
 18 | 	unknownImage   string   = "unknownZRTFGHUIJKLMOPRST"
 19 | 	privateImage   string   = "robinmonjo/debian"
 20 | 	gitImage       string   = "ubuntu:14.04"
 21 | 
 22 | 	minimalLinuxRootDirs []string = []string{"bin", "dev", "etc", "home", "lib", "mnt", "opt", "proc", "root", "run", "sbin", "sys", "tmp", "usr", "var"}
 23 | )
 24 | 
 25 | func assertErrNil(err error, t *testing.T) {
 26 | 	if err != nil {
 27 | 		t.Fatal(err)
 28 | 	}
 29 | }
 30 | 
 31 | func assertErrNotNil(err error, t *testing.T) {
 32 | 	if err == nil {
 33 | 		t.Fatal(err)
 34 | 	}
 35 | }
 36 | 
 37 | func Test_downloadImage(t *testing.T) {
 38 | 	fmt.Printf("Testing unknown %v image ... ", unknownImage)
 39 | 	downloadImage(unknownImage, "tmp_unknown", "", false, assertErrNotNil, t)
 40 | 	fmt.Printf("Ok\n")
 41 | 
 42 | 	for _, imageName := range testImages {
 43 | 		fmt.Printf("Testing %v image ... ", imageName)
 44 | 		downloadImage(imageName, "tmp_test", "", true, assertErrNil, t)
 45 | 		fmt.Printf("Ok\n")
 46 | 	}
 47 | }
 48 | 
 49 | func Test_downloadPrivateImage(t *testing.T) {
 50 | 	creds := os.Getenv(CREDS_ENV)
 51 | 	if creds == "" {
 52 | 		fmt.Printf("Skipping private image test (%v not set)\n", CREDS_ENV)
 53 | 		return
 54 | 	}
 55 | 	fmt.Printf("Testing private %v image ... ", privateImage)
 56 | 	downloadImage(privateImage, "tmp_priv_test", creds, true, assertErrNil, t)
 57 | 	fmt.Printf("Ok\n")
 58 | }
 59 | 
 60 | func downloadImage(imageName, rootfsDest, credentials string, checkFs bool, assert func(error, *testing.T), t *testing.T) {
 61 | 	defer os.RemoveAll(rootfsDest)
 62 | 
 63 | 	cmd := exec.Command(dlrootfsBinary, imageName, "-d", rootfsDest, "-u", credentials)
 64 | 	err := cmd.Start()
 65 | 	assertErrNil(err, t)
 66 | 
 67 | 	err = cmd.Wait()
 68 | 	assert(err, t)
 69 | 
 70 | 	if !checkFs {
 71 | 		return
 72 | 	}
 73 | 
 74 | 	fmt.Printf("Checking FS ... ")
 75 | 	for _, dir := range minimalLinuxRootDirs {
 76 | 		checkDirExists(rootfsDest+"/"+dir, t)
 77 | 	}
 78 | 
 79 | }
 80 | 
 81 | func checkDirExists(dir string, t *testing.T) {
 82 | 	src, err := os.Stat(dir)
 83 | 	assertErrNil(err, t)
 84 | 
 85 | 	if !src.IsDir() {
 86 | 		t.Fatal(dir, "not a directory")
 87 | 	}
 88 | }
 89 | 
 90 | func Test_downloadWithGitLayers(t *testing.T) {
 91 | 	fmt.Printf("Testing git layering ... ")
 92 | 	rootfsDest := "./ubuntu"
 93 | 	defer os.RemoveAll(rootfsDest)
 94 | 	cmd := exec.Command(dlrootfsBinary, gitImage, "-d", rootfsDest, "-g")
 95 | 	err := cmd.Start()
 96 | 	assertErrNil(err, t)
 97 | 
 98 | 	err = cmd.Wait()
 99 | 	assertErrNil(err, t)
100 | 
101 | 	gitRepo, _ := dlrootfs.NewGitRepo(rootfsDest)
102 | 	out, _ := gitRepo.Branch()
103 | 
104 | 	expectedBranches := []string{
105 | 		"layer0_511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158",
106 | 		"layer1_c7b7c64195686444123ef370322b5270b098c77dc2d62208e8a9ce28a11a63f9",
107 | 		"layer2_70c8faa62a44b9f6a70ec3a018ec14ec95717ebed2016430e57fec1abc90a879",
108 | 		"layer3_d735006ad9c1b1563e021d7a4fecfd75ed36d4384e2a1c42e78d8261b83d6271",
109 | 		"* layer4_04c5d3b7b0656168630d3ba35d8889bd0e9caafcaeb3004d2bfbc47e7c5d35d2",
110 | 	}
111 | 
112 | 	branches := strings.Split(string(out), "\n")
113 | 
114 | 	for i, branch := range branches {
115 | 		trimmedBranch := strings.Trim(branch, " \n")
116 | 		if trimmedBranch == "" {
117 | 			continue
118 | 		}
119 | 		expectedBranch := expectedBranches[i]
120 | 		if trimmedBranch != expectedBranch {
121 | 			t.Fatal("Expected branch", expectedBranch, " got ", trimmedBranch)
122 | 		}
123 | 	}
124 | 	fmt.Printf("OK\n")
125 | }
126 | 


--------------------------------------------------------------------------------
/pulling_job.go:
--------------------------------------------------------------------------------
 1 | package dlrootfs
 2 | 
 3 | import (
 4 | 	"fmt"
 5 | 	"io"
 6 | )
 7 | 
 8 | type PullingJob struct {
 9 | 	Session *HubSession
10 | 
11 | 	LayerId string
12 | 
13 | 	LayerData io.ReadCloser
14 | 	LayerInfo []byte
15 | 	LayerSize int
16 | 
17 | 	Err error
18 | }
19 | 
20 | func NewPullingJob(session *HubSession, layerId string) *PullingJob {
21 | 	return &PullingJob{Session: session, LayerId: layerId}
22 | }
23 | 
24 | func (job *PullingJob) Start() {
25 | 	fmt.Printf("\tPulling fs layer %v\n", truncateID(job.LayerId))
26 | 	endpoints := job.Session.RepoData.Endpoints
27 | 	tokens := job.Session.RepoData.Tokens
28 | 
29 | 	for _, ep := range endpoints {
30 | 		job.LayerInfo, job.LayerSize, job.Err = job.Session.GetRemoteImageJSON(job.LayerId, ep, tokens)
31 | 		if job.Err != nil {
32 | 			continue
33 | 		}
34 | 		job.LayerData, job.Err = job.Session.GetRemoteImageLayer(job.LayerId, ep, tokens, int64(job.LayerSize))
35 | 	}
36 | 
37 | 	fmt.Printf("\tDone %v\n", truncateID(job.LayerId))
38 | }
39 | 
40 | func (job *PullingJob) Error() error {
41 | 	return job.Err
42 | }
43 | 
44 | func (job *PullingJob) ID() string {
45 | 	return job.LayerId
46 | }
47 | 


--------------------------------------------------------------------------------
/queue.go:
--------------------------------------------------------------------------------
 1 | package dlrootfs
 2 | 
 3 | import (
 4 | 	"log"
 5 | 	"sync"
 6 | )
 7 | 
 8 | type Job interface {
 9 | 	Start()
10 | 	Error() error
11 | 	ID() string
12 | }
13 | 
14 | type Queue struct {
15 | 	Concurrency   int
16 | 	NbRunningJob  int
17 | 	WaitingJobs   []Job
18 | 	Lock          *sync.Mutex
19 | 	DoneChan      chan bool
20 | 	PerJobChan    chan string
21 | 	CompletedJobs map[string]Job
22 | }
23 | 
24 | func NewQueue(concurrency int) *Queue {
25 | 	doneChan := make(chan bool)
26 | 	perJobChan := make(chan string, 10000)
27 | 	return &Queue{Concurrency: concurrency, Lock: &sync.Mutex{}, DoneChan: doneChan, PerJobChan: perJobChan, CompletedJobs: make(map[string]Job)}
28 | }
29 | 
30 | func (queue *Queue) Enqueue(job Job) {
31 | 	queue.Lock.Lock()
32 | 	defer queue.Lock.Unlock()
33 | 
34 | 	if !queue.canLaunchJob() {
35 | 		//concurrency limit reached, make the job wait
36 | 		queue.WaitingJobs = append(queue.WaitingJobs, job)
37 | 		return
38 | 	}
39 | 	queue.startJob(job)
40 | }
41 | 
42 | func (queue *Queue) startJob(job Job) {
43 | 	queue.NbRunningJob++
44 | 	go func() {
45 | 		//start the job
46 | 		job.Start()
47 | 		queue.dequeue(job)
48 | 	}()
49 | }
50 | 
51 | func (queue *Queue) dequeue(job Job) {
52 | 	queue.Lock.Lock()
53 | 	defer queue.Lock.Unlock()
54 | 
55 | 	if job.Error() != nil {
56 | 		log.Fatal(job.Error())
57 | 	}
58 | 	queue.CompletedJobs[job.ID()] = job
59 | 	queue.PerJobChan <- job.ID()
60 | 
61 | 	queue.NbRunningJob--
62 | 	if queue.canLaunchJob() && len(queue.WaitingJobs) > 0 {
63 | 		queue.startJob(queue.WaitingJobs[0])
64 | 		queue.WaitingJobs = append(queue.WaitingJobs[:0], queue.WaitingJobs[1:]...)
65 | 	}
66 | 	if len(queue.WaitingJobs) == 0 && queue.NbRunningJob == 0 {
67 | 		queue.DoneChan <- true
68 | 	}
69 | }
70 | 
71 | func (queue *Queue) canLaunchJob() bool {
72 | 	return queue.NbRunningJob < queue.Concurrency
73 | }
74 | 
75 | func (queue *Queue) CompletedJobWithID(jobId string) Job {
76 | 	return queue.CompletedJobs[jobId]
77 | }
78 | 


--------------------------------------------------------------------------------
/sample_configs/container.json:
--------------------------------------------------------------------------------
  1 | {
  2 |     "capabilities": [
  3 |         "CHOWN",
  4 |         "DAC_OVERRIDE",
  5 |         "FOWNER",
  6 |         "MKNOD",
  7 |         "NET_RAW",
  8 |         "SETGID",
  9 |         "SETUID",
 10 |         "SETFCAP",
 11 |         "SETPCAP",
 12 |         "NET_BIND_SERVICE",
 13 |         "SYS_CHROOT",
 14 |         "KILL"
 15 |     ],
 16 |     "cgroups": {
 17 |         "allowed_devices": [
 18 |             {
 19 |                 "cgroup_permissions": "m",
 20 |                 "major_number": -1,
 21 |                 "minor_number": -1,
 22 |                 "type": 99
 23 |             },
 24 |             {
 25 |                 "cgroup_permissions": "m",
 26 |                 "major_number": -1,
 27 |                 "minor_number": -1,
 28 |                 "type": 98
 29 |             },
 30 |             {
 31 |                 "cgroup_permissions": "rwm",
 32 |                 "major_number": 5,
 33 |                 "minor_number": 1,
 34 |                 "path": "/dev/console",
 35 |                 "type": 99
 36 |             },
 37 |             {
 38 |                 "cgroup_permissions": "rwm",
 39 |                 "major_number": 4,
 40 |                 "path": "/dev/tty0",
 41 |                 "type": 99
 42 |             },
 43 |             {
 44 |                 "cgroup_permissions": "rwm",
 45 |                 "major_number": 4,
 46 |                 "minor_number": 1,
 47 |                 "path": "/dev/tty1",
 48 |                 "type": 99
 49 |             },
 50 |             {
 51 |                 "cgroup_permissions": "rwm",
 52 |                 "major_number": 136,
 53 |                 "minor_number": -1,
 54 |                 "type": 99
 55 |             },
 56 |             {
 57 |                 "cgroup_permissions": "rwm",
 58 |                 "major_number": 5,
 59 |                 "minor_number": 2,
 60 |                 "type": 99
 61 |             },
 62 |             {
 63 |                 "cgroup_permissions": "rwm",
 64 |                 "major_number": 10,
 65 |                 "minor_number": 200,
 66 |                 "type": 99
 67 |             },
 68 |             {
 69 |                 "cgroup_permissions": "rwm",
 70 |                 "file_mode": 438,
 71 |                 "major_number": 1,
 72 |                 "minor_number": 3,
 73 |                 "path": "/dev/null",
 74 |                 "type": 99
 75 |             },
 76 |             {
 77 |                 "cgroup_permissions": "rwm",
 78 |                 "file_mode": 438,
 79 |                 "major_number": 1,
 80 |                 "minor_number": 5,
 81 |                 "path": "/dev/zero",
 82 |                 "type": 99
 83 |             },
 84 |             {
 85 |                 "cgroup_permissions": "rwm",
 86 |                 "file_mode": 438,
 87 |                 "major_number": 1,
 88 |                 "minor_number": 7,
 89 |                 "path": "/dev/full",
 90 |                 "type": 99
 91 |             },
 92 |             {
 93 |                 "cgroup_permissions": "rwm",
 94 |                 "file_mode": 438,
 95 |                 "major_number": 5,
 96 |                 "path": "/dev/tty",
 97 |                 "type": 99
 98 |             },
 99 |             {
100 |                 "cgroup_permissions": "rwm",
101 |                 "file_mode": 438,
102 |                 "major_number": 1,
103 |                 "minor_number": 9,
104 |                 "path": "/dev/urandom",
105 |                 "type": 99
106 |             },
107 |             {
108 |                 "cgroup_permissions": "rwm",
109 |                 "file_mode": 438,
110 |                 "major_number": 1,
111 |                 "minor_number": 8,
112 |                 "path": "/dev/random",
113 |                 "type": 99
114 |             }
115 |         ]
116 |     },
117 |     "restrict_sys": true,
118 |     "mount_config": {
119 |         "device_nodes": [
120 |             {
121 |                 "cgroup_permissions": "rwm",
122 |                 "file_mode": 438,
123 |                 "major_number": 1,
124 |                 "minor_number": 3,
125 |                 "path": "/dev/null",
126 |                 "type": 99
127 |             },
128 |             {
129 |                 "cgroup_permissions": "rwm",
130 |                 "file_mode": 438,
131 |                 "major_number": 1,
132 |                 "minor_number": 5,
133 |                 "path": "/dev/zero",
134 |                 "type": 99
135 |             },
136 |             {
137 |                 "cgroup_permissions": "rwm",
138 |                 "file_mode": 438,
139 |                 "major_number": 1,
140 |                 "minor_number": 7,
141 |                 "path": "/dev/full",
142 |                 "type": 99
143 |             },
144 |             {
145 |                 "cgroup_permissions": "rwm",
146 |                 "file_mode": 438,
147 |                 "major_number": 5,
148 |                 "path": "/dev/tty",
149 |                 "type": 99
150 |             },
151 |             {
152 |                 "cgroup_permissions": "rwm",
153 |                 "file_mode": 438,
154 |                 "major_number": 1,
155 |                 "minor_number": 9,
156 |                 "path": "/dev/urandom",
157 |                 "type": 99
158 |             },
159 |             {
160 |                 "cgroup_permissions": "rwm",
161 |                 "file_mode": 438,
162 |                 "major_number": 1,
163 |                 "minor_number": 8,
164 |                 "path": "/dev/random",
165 |                 "type": 99
166 |             }
167 |         ],
168 |         "mounts": [
169 |             {
170 |                 "type": "tmpfs",
171 |                 "destination": "/tmp"
172 |             }
173 |         ]
174 |     },
175 |     "environment": [
176 |         "HOME=/",
177 |         "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
178 |         "HOSTNAME=dlrootfs-demo",
179 |         "TERM=xterm"
180 |     ],
181 |     "hostname": "dlrootfs-demo",
182 |     "namespaces": {
183 |         "NEWIPC": true,
184 |         "NEWNET": true,
185 |         "NEWNS": true,
186 |         "NEWPID": true,
187 |         "NEWUTS": true
188 |     },
189 |     "networks": [
190 |         {
191 |             "address": "127.0.0.1/0",
192 |             "gateway": "localhost",
193 |             "mtu": 1500,
194 |             "type": "loopback"
195 |         }
196 |     ],
197 |     "tty": true,
198 |     "user": "root"
199 | }
200 | 


--------------------------------------------------------------------------------
/sample_configs/lxc-config:
--------------------------------------------------------------------------------
 1 | 
 2 | # Container specific configuration
 3 | #CHANGE ME
 4 | lxc.rootfs = /root/ubuntu
 5 | lxc.utsname = dlrootfs
 6 | lxc.arch = amd64
 7 | 
 8 | # Network configuration
 9 | lxc.network.type = veth
10 | 
11 | # Default pivot location
12 | lxc.pivotdir = lxc_putold
13 | 
14 | # Default mount entries
15 | lxc.mount.entry = proc proc proc nodev,noexec,nosuid 0 0
16 | lxc.mount.entry = sysfs sys sysfs defaults 0 0
17 | lxc.mount.entry = /sys/fs/fuse/connections sys/fs/fuse/connections none bind,optional 0 0
18 | lxc.mount.entry = /sys/kernel/debug sys/kernel/debug none bind,optional 0 0
19 | lxc.mount.entry = /sys/kernel/security sys/kernel/security none bind,optional 0 0
20 | lxc.mount.entry = /sys/fs/pstore sys/fs/pstore none bind,optional 0 0
21 | 
22 | # Default console settings
23 | lxc.devttydir = lxc
24 | lxc.tty = 4
25 | lxc.pts = 1024
26 | 
27 | # Default capabilities
28 | lxc.cap.drop = sys_module mac_admin mac_override sys_time
29 | 
30 | # When using LXC with apparmor, the container will be confined by default.
31 | # If you wish for it to instead run unconfined, copy the following line
32 | # (uncommented) to the container's configuration file.
33 | #lxc.aa_profile = unconfined
34 | 
35 | # To support container nesting on an Ubuntu host while retaining most of
36 | # apparmor's added security, use the following two lines instead.
37 | #lxc.aa_profile = lxc-container-default-with-nesting
38 | #lxc.hook.mount = /usr/share/lxc/hooks/mountcgroups
39 | 
40 | # Uncomment the following line to autodetect squid-deb-proxy configuration on the
41 | # host and forward it to the guest at start time.
42 | #lxc.hook.pre-start = /usr/share/lxc/hooks/squid-deb-proxy-client
43 | 
44 | # If you wish to allow mounting block filesystems, then use the following
45 | # line instead, and make sure to grant access to the block device and/or loop
46 | # devices below in lxc.cgroup.devices.allow.
47 | #lxc.aa_profile = lxc-container-default-with-mounting
48 | 
49 | # Default cgroup limits
50 | lxc.cgroup.devices.deny = a
51 | ## Allow any mknod (but not using the node)
52 | lxc.cgroup.devices.allow = c *:* m
53 | lxc.cgroup.devices.allow = b *:* m
54 | ## /dev/null and zero
55 | lxc.cgroup.devices.allow = c 1:3 rwm
56 | lxc.cgroup.devices.allow = c 1:5 rwm
57 | ## consoles
58 | lxc.cgroup.devices.allow = c 5:0 rwm
59 | lxc.cgroup.devices.allow = c 5:1 rwm
60 | ## /dev/{,u}random
61 | lxc.cgroup.devices.allow = c 1:8 rwm
62 | lxc.cgroup.devices.allow = c 1:9 rwm
63 | ## /dev/pts/*
64 | lxc.cgroup.devices.allow = c 5:2 rwm
65 | lxc.cgroup.devices.allow = c 136:* rwm
66 | ## rtc
67 | lxc.cgroup.devices.allow = c 254:0 rm
68 | ## fuse
69 | lxc.cgroup.devices.allow = c 10:229 rwm
70 | ## tun
71 | lxc.cgroup.devices.allow = c 10:200 rwm
72 | ## full
73 | lxc.cgroup.devices.allow = c 1:7 rwm
74 | ## hpet
75 | lxc.cgroup.devices.allow = c 10:228 rwm
76 | ## kvm
77 | lxc.cgroup.devices.allow = c 10:232 rwm
78 | ## To use loop devices, copy the following line to the container's
79 | ## configuration file (uncommented).
80 | #lxc.cgroup.devices.allow = b 7:* rwm
81 | 
82 | # Blacklist some syscalls which are not safe in privileged
83 | # containers
84 | lxc.seccomp = /usr/share/lxc/config/common.seccomp
85 | 


--------------------------------------------------------------------------------
/setup.sh:
--------------------------------------------------------------------------------
 1 | ##!/bin/ash
 2 | 
 3 | set -e
 4 | 
 5 | sudo apt-get update -qq
 6 | 
 7 | echo "Installing base stack"
 8 | 
 9 | packagelist=(
10 | 	cgroup-lite             #this is important !!
11 |   curl
12 |   build-essential
13 |   bison
14 |   openssl
15 |   libreadline6
16 |   libreadline-dev
17 |   git-core
18 |   zlib1g
19 |   zlib1g-dev
20 |   libssl-dev
21 |   libyaml-dev
22 |   libxml2-dev
23 |   libxslt-dev
24 |   autoconf
25 |   ssl-cert
26 |   libcurl4-openssl-dev
27 |   lxc
28 | 	zsh
29 | 	git
30 |   python-software-properties
31 |   golang
32 | )
33 | 
34 | sudo DEBIAN_FRONTEND=noninteractive apt-get install -y ${packagelist[@]}
35 | 
36 | curl -L http://install.ohmyz.sh | sh
37 | 
38 | echo "GOPATH=~/code/go" >> ~/.bashrc
39 | 


--------------------------------------------------------------------------------
/utils.go:
--------------------------------------------------------------------------------
 1 | package dlrootfs
 2 | 
 3 | import (
 4 | 	"strings"
 5 | )
 6 | 
 7 | func truncateID(id string) string {
 8 | 	shortLen := 12
 9 | 	if len(id) < shortLen {
10 | 		shortLen = len(id)
11 | 	}
12 | 	return id[:shortLen]
13 | }
14 | 
15 | func ParseCredentials(credentials string) (string, string) {
16 | 	credentialsSplit := strings.SplitN(credentials, ":", 2)
17 | 	if len(credentialsSplit) != 2 {
18 | 		return "", ""
19 | 	}
20 | 	return credentialsSplit[0], credentialsSplit[1]
21 | }
22 | 
23 | func ParseImageNameTag(imageNameTag string) (imageName string, imageTag string) {
24 | 	if strings.Contains(imageNameTag, ":") {
25 | 		imageName = strings.SplitN(imageNameTag, ":", 2)[0]
26 | 		imageTag = strings.SplitN(imageNameTag, ":", 2)[1]
27 | 	} else {
28 | 		imageName = imageNameTag
29 | 		imageTag = "latest"
30 | 	}
31 | 
32 | 	if !strings.Contains(imageName, "/") {
33 | 		imageName = "library/" + imageName
34 | 	}
35 | 	return
36 | }
37 | 


--------------------------------------------------------------------------------
/vendor.sh:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env bash
 2 | set -e
 3 | 
 4 | cd "$(dirname "$BASH_SOURCE")"
 5 | 
 6 | # Downloads dependencies into vendor/ directory
 7 | mkdir -p vendor
 8 | cd vendor
 9 | 
10 | clone() {
11 | 	vcs=$1
12 | 	pkg=$2
13 | 	rev=$3
14 | 
15 | 	pkg_url=https://$pkg
16 | 	target_dir=src/$pkg
17 | 
18 | 	echo "$pkg @ $rev: "
19 | 
20 | 	if [ -d $target_dir ]; then
21 | 		echo "rm old, $pkg"
22 | 		rm -fr $target_dir
23 | 	fi
24 | 
25 | 	echo "clone, $pkg"
26 | 	case $vcs in
27 | 		git)
28 | 			git clone --quiet --no-checkout $pkg_url $target_dir
29 | 			( cd $target_dir && git reset --quiet --hard $rev )
30 | 			;;
31 | 		hg)
32 | 			hg clone --quiet --updaterev $rev $pkg_url $target_dir
33 | 			;;
34 | 	esac
35 | 
36 | 	echo "rm VCS, $vcs"
37 | 	( cd $target_dir && rm -rf .{git,hg} )
38 | 
39 | 	echo "done"
40 | }
41 | 
42 | clone git github.com/docker/docker v1.4.0
43 | clone git github.com/gorilla/mux 136d54f81f
44 | clone git github.com/gorilla/context 14f550f51a
45 | clone git github.com/Sirupsen/logrus v0.6.0
46 | 
47 | echo "don't forget to add vendor folder to your GOPATH (export GOPATH=\$GOPATH:\`pwd\`/vendor)"
48 | 


--------------------------------------------------------------------------------