├── .gitignore ├── actions ├── test │ ├── Dockerfile │ └── run-tests └── release │ ├── Dockerfile │ └── run-release ├── pkg ├── provider │ ├── doc.go │ ├── utils.go │ ├── google.go │ ├── github.go │ └── docker.go ├── image │ ├── doc.go │ ├── provider_test.go │ ├── utils.go │ ├── manifest.go │ ├── url_test.go │ ├── provider.go │ ├── remote_test.go │ ├── url.go │ ├── remote.go │ ├── untar.go │ └── store.go └── lock │ ├── lock_test.go │ └── lock.go ├── .goreleaser.yml ├── .github └── main.workflow ├── go.mod ├── makefile ├── LICENSE.md ├── go.sum ├── README.md └── roots.go /.gitignore: -------------------------------------------------------------------------------- 1 | dist -------------------------------------------------------------------------------- /actions/test/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang 2 | COPY run-tests /bin/run-tests 3 | -------------------------------------------------------------------------------- /actions/release/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang 2 | COPY run-release /bin/run-release 3 | -------------------------------------------------------------------------------- /pkg/provider/doc.go: -------------------------------------------------------------------------------- 1 | // Package provider implements specific providers for different Docker 2 | // container registries. 3 | package provider 4 | -------------------------------------------------------------------------------- /pkg/image/doc.go: -------------------------------------------------------------------------------- 1 | // Package image provides access to container registries supporting the Docker 2 | // Registry V2 API with the ability to download and extract layers provided 3 | // by those registries. 4 | package image 5 | -------------------------------------------------------------------------------- /.goreleaser.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | before: 4 | hooks: 5 | - go mod download 6 | 7 | builds: 8 | - env: 9 | - CGO_ENABLED=1 10 | goos: 11 | - linux 12 | goarch: 13 | - amd64 14 | 15 | archives: 16 | - format: binary 17 | 18 | checksum: 19 | name_template: 'checksums.txt' 20 | 21 | snapshot: 22 | name_template: "{{ .Tag }}-next" 23 | 24 | changelog: 25 | sort: asc 26 | filters: 27 | exclude: 28 | - '^Document' 29 | - typo 30 | -------------------------------------------------------------------------------- /.github/main.workflow: -------------------------------------------------------------------------------- 1 | workflow "Test & Release" { 2 | on = "push" 3 | resolves = "Release" 4 | } 5 | 6 | action "Test" { 7 | uses = "seantis/roots/actions/test@master" 8 | runs = "run-tests" 9 | } 10 | 11 | action "Tagged" { 12 | needs = "Test" 13 | uses = "actions/bin/filter@master" 14 | args = "tag v*" 15 | } 16 | 17 | action "Release" { 18 | needs = "Tagged" 19 | uses = "seantis/roots/actions/release@master" 20 | runs = "run-release" 21 | secrets = ["GITHUB_TOKEN"] 22 | } 23 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/seantis/roots 2 | 3 | go 1.22 4 | 5 | require ( 6 | github.com/alexflint/go-filemutex v1.3.0 7 | github.com/dankinder/httpmock v1.0.4 8 | github.com/jawher/mow.cli v1.2.0 9 | github.com/stretchr/testify v1.9.0 10 | golang.org/x/oauth2 v0.21.0 11 | ) 12 | 13 | require ( 14 | cloud.google.com/go/compute/metadata v0.3.0 // indirect 15 | github.com/davecgh/go-spew v1.1.1 // indirect 16 | github.com/pmezard/go-difflib v1.0.0 // indirect 17 | github.com/stretchr/objx v0.5.2 // indirect 18 | golang.org/x/sys v0.16.0 // indirect 19 | gopkg.in/yaml.v3 v3.0.1 // indirect 20 | ) 21 | -------------------------------------------------------------------------------- /pkg/lock/lock_test.go: -------------------------------------------------------------------------------- 1 | package lock 2 | 3 | import ( 4 | "os" 5 | "path" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | // TestLockSimple tests the interprocess locking using a single process 12 | func TestLockSimple(t *testing.T) { 13 | dir, _ := os.MkdirTemp("", "locks") 14 | 15 | foo := &InterProcessLock{Path: path.Join(dir, "foo")} 16 | bar := &InterProcessLock{Path: path.Join(dir, "bar")} 17 | 18 | assert.NoError(t, foo.Lock(), "error locking foo") 19 | assert.NoError(t, bar.Lock(), "error locking bar") 20 | 21 | assert.NoError(t, foo.Unlock(), "error unlocking foo") 22 | assert.NoError(t, bar.Unlock(), "error unlocking bar") 23 | } 24 | -------------------------------------------------------------------------------- /pkg/provider/utils.go: -------------------------------------------------------------------------------- 1 | package provider 2 | 3 | import "net/http" 4 | 5 | type boundHeadersTransport struct { 6 | base http.RoundTripper 7 | headers map[string]string 8 | } 9 | 10 | func (t *boundHeadersTransport) RoundTrip(req *http.Request) (*http.Response, error) { 11 | for k, v := range t.headers { 12 | req.Header.Add(k, v) 13 | } 14 | 15 | return t.base.RoundTrip(req) 16 | } 17 | 18 | // clientWithHeader returns an http.Client which sets the given headers on 19 | // each request sent to the server 20 | func clientWithHeaders(headers map[string]string) *http.Client { 21 | return &http.Client{ 22 | Transport: &boundHeadersTransport{ 23 | headers: headers, 24 | base: http.DefaultTransport, 25 | }, 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /makefile: -------------------------------------------------------------------------------- 1 | .PHONY: test 2 | test: 3 | @ go test ./... 4 | 5 | .PHONY: test-all 6 | test-all: 7 | @ docker build -t roots-test-action actions/test 8 | @ docker run -v $(PWD):/github/workspace --rm -it roots-test-action run-tests 9 | 10 | .PHONY: test-release 11 | test-release: 12 | @ docker build -t roots-release-action actions/release 13 | @ docker run -e "GITHUB_REF=refs/tags/v0.0.0" -v $(PWD):/github/workspace --rm -it roots-release-action run-release --skip=publish --snapshot --clean 14 | 15 | .PHONY: release 16 | release: 17 | @ docker build -t roots-release-action actions/release 18 | @ docker run -e GITHUB_TOKEN=$(GITHUB_TOKEN) -e GITHUB_REF=refs/tags/$(VERSION) -v $(PWD):/github/workspace --rm -it roots-release-action run-release --clean 19 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | Copyright 2018 Seantis GmbH 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 8 | -------------------------------------------------------------------------------- /actions/release/run-release: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euo pipefail 3 | 4 | GITHUB_REF="${GITHUB_REF:-}" 5 | GITHUB_WORKSPACE="${GITHUB_WORKSPACE:-/github/workspace}" 6 | 7 | if ! echo "$GITHUB_REF" | grep -E "^refs/tags/v[0-9]+\.[0-9]+\.[0-9]+"; then 8 | echo "'$GITHUB_REF' does not refer to a valid tag" 9 | exit 1 10 | fi 11 | 12 | if ! test -d "$GITHUB_WORKSPACE"; then 13 | echo "$GITHUB_WORKSPACE is missing" 14 | exit 1 15 | fi 16 | 17 | VERSION=$(echo "$GITHUB_REF" | grep -Eo '[0-9]+\.[0-9]+\.[0-9]+') 18 | 19 | # get latest goreleaser release 20 | echo "🌲 Downloading goreleaser" 21 | 22 | GORELEASER_DEB=$(curl -s -L https://api.github.com/repos/goreleaser/goreleaser/releases/latest \ 23 | | grep browser_download_url \ 24 | | grep amd64.deb \ 25 | | cut -d '"' -f 4) 26 | 27 | curl -s -L -o goreleaser.deb "$GORELEASER_DEB" 28 | 29 | echo "🌲 Installing goreleaser" 30 | dpkg -i goreleaser.deb 31 | 32 | echo "🌲 Releasing $VERSION" 33 | cd "$GITHUB_WORKSPACE" || exit 1 34 | git config --global --add safe.directory "$GITHUB_WORKSPACE" 35 | goreleaser "$@" 36 | -------------------------------------------------------------------------------- /pkg/image/provider_test.go: -------------------------------------------------------------------------------- 1 | package image 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | type nullProvider struct{} 12 | 13 | func (p *nullProvider) GetClient(url URL, auth string) (*http.Client, error) { 14 | return nil, fmt.Errorf("not implemented") 15 | } 16 | 17 | type falseProvider struct { 18 | *nullProvider 19 | } 20 | 21 | func (p *nullProvider) Supports(url URL) bool { 22 | return false 23 | } 24 | 25 | type trueProvider struct { 26 | *nullProvider 27 | } 28 | 29 | func (p *trueProvider) Supports(url URL) bool { 30 | return true 31 | } 32 | 33 | // TestRegistryLookup tests the registry lookup priority 34 | func TestRegistryLookup(t *testing.T) { 35 | defer ClearProviderRegistry() 36 | 37 | foo := &falseProvider{} 38 | bar := &trueProvider{} 39 | baz := &falseProvider{} 40 | 41 | RegisterProvider("foo", foo) 42 | RegisterProvider("bar", bar) 43 | RegisterProvider("baz", baz) 44 | 45 | provider, _ := LookupProvider(URL{}) 46 | 47 | assert.Equal(t, provider, bar, "provider registry lookup failure") 48 | } 49 | -------------------------------------------------------------------------------- /actions/test/run-tests: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euo pipefail 3 | 4 | GITHUB_WORKSPACE="${GITHUB_WORKSPACE:-/github/workspace}" 5 | 6 | if ! test -d "$GITHUB_WORKSPACE"; then 7 | echo "$GITHUB_WORKSPACE is missing" 8 | exit 1 9 | fi 10 | 11 | cd "$GITHUB_WORKSPACE" || exit 1 12 | git config --global --add safe.directory "$GITHUB_WORKSPACE" 13 | 14 | echo "🌲 Downloading dependencies" 15 | go mod download 16 | 17 | # install go get outside our repository to ignore the mod file 18 | pushd / 19 | go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest 20 | popd 21 | 22 | echo "🌲 Running unit tests" 23 | go test ./... 24 | echo "" 25 | 26 | echo "🌲 Running golangci-lint" 27 | golangci-lint run 28 | 29 | echo "🌲 Testing version" 30 | go run roots.go version 31 | echo "" 32 | 33 | echo "🌲 Testing Docker Hub pull with race detection" 34 | go run -race roots.go pull debian /tmp/debian 35 | echo "" 36 | 37 | echo "🌲 Testing GCR pull" 38 | go run roots.go pull gcr.io/google-containers/etcd:3.3.10 /tmp/etcd 39 | echo "" 40 | 41 | echo "🌲 Testing GHCR pull" 42 | go run roots.go pull ghcr.io/github/issue_metrics:latest /tmp/issue_metrics 43 | echo "" 44 | 45 | echo "🌲 Testing Cache Purge" 46 | go run roots.go purge 47 | -------------------------------------------------------------------------------- /pkg/image/utils.go: -------------------------------------------------------------------------------- 1 | package image 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "strings" 7 | ) 8 | 9 | func bisect(text string, delimiter string) (string, string) { 10 | split := strings.SplitN(text, delimiter, 2) 11 | return split[0], split[1] 12 | } 13 | 14 | // mustNewRequest calls http.NewRequest, but panics if there's an error (as those 15 | // are most certainly errors that we catch during testing) 16 | func mustNewRequest(method string, url string) *http.Request { 17 | res, err := http.NewRequest(method, url, nil) 18 | if err != nil { 19 | panic(err) 20 | } 21 | 22 | return res 23 | } 24 | 25 | func requireSupportedMimeTypes(client *http.Client, url URL) error { 26 | ref := url.Endpoint("manifests", url.Reference()) 27 | 28 | req := mustNewRequest("HEAD", ref) 29 | req.Header.Add("Accept", fmt.Sprintf("%s, */*", ManifestMimeType)) 30 | 31 | res, err := client.Do(req) 32 | if err != nil { 33 | return fmt.Errorf("error requesting %s: %v", ref, err) 34 | } 35 | if res.StatusCode != 200 { 36 | return fmt.Errorf("HEAD %s returned %s", ref, res.Status) 37 | } 38 | 39 | mime := res.Header.Get("Content-Type") 40 | if mime != ManifestMimeType && mime != ManifestListMimeType { 41 | return fmt.Errorf("no schema version 2 support by %s", url) 42 | } 43 | 44 | return nil 45 | } 46 | -------------------------------------------------------------------------------- /pkg/image/manifest.go: -------------------------------------------------------------------------------- 1 | package image 2 | 3 | import "fmt" 4 | 5 | var ( 6 | // ManifestListMimeType is the mime type used to get the manifest list 7 | ManifestListMimeType = "application/vnd.docker.distribution.manifest.list.v2+json" 8 | 9 | // ManifestMimeType is the mime type used to get the manifest 10 | ManifestMimeType = "application/vnd.docker.distribution.manifest.v2+json" 11 | ) 12 | 13 | // ManifestList represents the Docker Manifest List: 14 | // * https://github.com/docker/distribution/blob/master/docs/spec/manifest-v2-2.md 15 | // * application/vnd.docker.distribution.manifest.list.v2+json 16 | type ManifestList struct { 17 | Manifests []PlatformManifest `json:"manifests"` 18 | } 19 | 20 | // PlatformManifest represents an entry in a Manifest List 21 | type PlatformManifest struct { 22 | *ManifestLayer 23 | Platform Platform `json:"platform"` 24 | } 25 | 26 | // Platform represents the platform description in a PlatformManifest 27 | type Platform struct { 28 | Architecture string `json:"architecture"` 29 | OS string `json:"os"` 30 | } 31 | 32 | func (p *Platform) String() string { 33 | return fmt.Sprintf("%s/%s", p.OS, p.Architecture) 34 | } 35 | 36 | // Manifest represents a Docker Image Manifest 37 | // * https://github.com/docker/distribution/blob/master/docs/spec/manifest-v2-2.md 38 | // * application/vnd.docker.distribution.manifest.v2+json 39 | type Manifest struct { 40 | Digest string 41 | SchemaVersion int `json:"schemaVersion"` 42 | MediaType string `json:"mediaType"` 43 | Layers []ManifestLayer `json:"layers"` 44 | } 45 | 46 | // ManifestLayer represents a Docker Image Layer 47 | type ManifestLayer struct { 48 | MediaType string `json:"mediaType"` 49 | Size int `json:"size"` 50 | Digest string `json:"digest"` 51 | } 52 | -------------------------------------------------------------------------------- /pkg/image/url_test.go: -------------------------------------------------------------------------------- 1 | package image 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | var cases = []struct { 10 | url string 11 | expected URL 12 | format string 13 | }{ 14 | { 15 | "ubuntu", URL{ 16 | Name: "ubuntu", 17 | Tag: "latest", 18 | Repository: "library", 19 | Host: "registry-1.docker.io", 20 | }, 21 | "registry-1.docker.io/library/ubuntu:latest", 22 | }, 23 | { 24 | "ubuntu:18.04", URL{ 25 | Name: "ubuntu", 26 | Tag: "18.04", 27 | Repository: "library", 28 | Host: "registry-1.docker.io", 29 | }, 30 | "registry-1.docker.io/library/ubuntu:18.04", 31 | }, 32 | { 33 | "gcr.io/google-containers/ubuntu", URL{ 34 | Name: "ubuntu", 35 | Tag: "latest", 36 | Repository: "google-containers", 37 | Host: "gcr.io", 38 | }, 39 | "gcr.io/google-containers/ubuntu:latest", 40 | }, 41 | { 42 | "foo/bar", URL{ 43 | Name: "bar", 44 | Tag: "latest", 45 | Repository: "foo", 46 | Host: "registry-1.docker.io", 47 | }, 48 | "registry-1.docker.io/foo/bar:latest", 49 | }, 50 | { 51 | "foo/bar@sha256:0xdeadbeef", URL{ 52 | Name: "bar", 53 | Tag: "latest", 54 | Repository: "foo", 55 | Host: "registry-1.docker.io", 56 | Digest: "sha256:0xdeadbeef", 57 | }, 58 | "registry-1.docker.io/foo/bar:latest@sha256:0xdeadbeef", 59 | }, 60 | { 61 | "", URL{}, "", 62 | }, 63 | { 64 | "@", URL{}, "", 65 | }, 66 | { 67 | "/////@@", URL{}, "", 68 | }, 69 | { 70 | " ", URL{}, "", 71 | }, 72 | } 73 | 74 | // TestParse tests the image URL parsing 75 | func TestParse(t *testing.T) { 76 | for _, c := range cases { 77 | t.Run(c.url, func(t *testing.T) { 78 | result, _ := Parse(c.url) 79 | 80 | assert.Equal(t, c.expected, *result, "unexpected url") 81 | 82 | format := result.String() 83 | assert.Equal(t, format, c.format, "unexpected format") 84 | }) 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /pkg/image/provider.go: -------------------------------------------------------------------------------- 1 | package image 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | ) 7 | 8 | var ( 9 | registry = make(map[string]Provider) 10 | priority = []string{} 11 | ) 12 | 13 | // Provider provides an authenticated client for a given URL. 14 | type Provider interface { 15 | 16 | // GetClient returns an net/http Client that is authenticated 17 | // to interact with the repository for all URLs the provider supports. 18 | // 19 | // It is called once for each new URL. It is up to the provider to reuse 20 | // clients when called multiple times as this depends on the registry. 21 | // 22 | // The 'auth' parameter is an optional string used for authentication. Its 23 | // meaning is determined by the provider itself. It may be a path, a token 24 | // a username and password etc. - The CLI passes the auth value as is. 25 | GetClient(url URL, auth string) (*http.Client, error) 26 | 27 | // Supports returns true if the provider supports the given URL - multiple 28 | // providers may support the same URL - in this case, the first provider 29 | // in order of registration is chosen 30 | Supports(url URL) bool 31 | } 32 | 33 | // LookupProvider takes an image.URL and returns the associated provider 34 | func LookupProvider(url URL) (Provider, error) { 35 | for _, name := range priority { 36 | provider := registry[name] 37 | 38 | if provider.Supports(url) { 39 | return provider, nil 40 | } 41 | } 42 | 43 | return nil, fmt.Errorf("no provider for %s", url) 44 | } 45 | 46 | // RegisterProvider registers a provider with the given name. Providers are 47 | // meant to be registered once during initialization and doing so concurrently 48 | // is not safe. If a provider with the same name exists, it is overwritten. 49 | func RegisterProvider(name string, provider Provider) { 50 | registry[name] = provider 51 | priority = append(priority, name) 52 | } 53 | 54 | // ClearProviderRegistry clears the provider registry (mainly useful for tests) 55 | func ClearProviderRegistry() { 56 | registry = make(map[string]Provider) 57 | priority = []string{} 58 | } 59 | -------------------------------------------------------------------------------- /pkg/provider/google.go: -------------------------------------------------------------------------------- 1 | package provider 2 | 3 | import ( 4 | "context" 5 | "net/http" 6 | "os" 7 | "regexp" 8 | "sync" 9 | 10 | "github.com/seantis/roots/pkg/image" 11 | "golang.org/x/oauth2/google" 12 | ) 13 | 14 | // GCRProvider authenticates clients against the Google Cloud Registry 15 | type GCRProvider struct { 16 | clients map[string]*http.Client 17 | mu sync.Mutex 18 | } 19 | 20 | func init() { 21 | image.RegisterProvider("gcr", &GCRProvider{ 22 | clients: make(map[string]*http.Client), 23 | }) 24 | } 25 | 26 | var gcrhosts = regexp.MustCompile(`([a-z]+?\.)?gcr\.io`) 27 | var gcrscope = "https://www.googleapis.com/auth/devstorage.read_only" 28 | 29 | // Supports returns true if the URLs host is one of the google cloud registry hosts 30 | func (p *GCRProvider) Supports(url image.URL) bool { 31 | return gcrhosts.MatchString(url.Host) 32 | } 33 | 34 | // GetClient returns a client authenticated with the Google Cloud Registry - 35 | // the auth string is supposed to be the path to a service account json file 36 | // the required scope is limit to https://www.googleapis.com/auth/devstorage.read_only 37 | func (p *GCRProvider) GetClient(url image.URL, auth string) (*http.Client, error) { 38 | 39 | p.mu.Lock() 40 | defer p.mu.Unlock() 41 | 42 | // The client for GCR is only bound to the auth string 43 | if p.clients[auth] == nil { 44 | client, err := p.newClient(auth) 45 | 46 | if err != nil { 47 | return nil, err 48 | } 49 | 50 | p.clients[auth] = client 51 | } 52 | 53 | return p.clients[auth], nil 54 | } 55 | 56 | // newClient spawns a new http client for GCR given the path to an account json 57 | // file, or an empty string (for anonymous access) 58 | func (p *GCRProvider) newClient(auth string) (*http.Client, error) { 59 | 60 | // we try to get the Google's default client and fall back on the 61 | // unauthenticated client if that doesn't work 62 | if len(auth) != 0 { 63 | os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", auth) 64 | } 65 | 66 | client, err := google.DefaultClient(context.Background(), gcrscope) 67 | 68 | // we got logged in! 69 | if err == nil { 70 | return client, nil 71 | } 72 | 73 | // we are not authenticated 74 | return &http.Client{}, nil 75 | } 76 | -------------------------------------------------------------------------------- /pkg/lock/lock.go: -------------------------------------------------------------------------------- 1 | // Package lock provides interprocess locking using a combination of flock 2 | // and process-local mutex-es 3 | package lock 4 | 5 | import ( 6 | "fmt" 7 | "sync" 8 | 9 | "github.com/alexflint/go-filemutex" 10 | ) 11 | 12 | var ( 13 | locksmu = &sync.Mutex{} 14 | locks = make(map[string]*sync.Mutex) 15 | ) 16 | 17 | // InterProcessLock provides a mutex that works across the current process and 18 | // across all other processes. It works by first acquiring a local lock and 19 | // then a file lock. 20 | // 21 | // The reason that a local process lock is used first, is due to the limits 22 | // of interprocess locking in Linux -> we have to avoid reusing the same lock 23 | // file multiple times in the same process or closing one of the locks will 24 | // unlock all the others. See: http://0pointer.de/blog/projects/locking.html 25 | type InterProcessLock struct { 26 | Path string 27 | filelock *filemutex.FileMutex 28 | } 29 | 30 | func (l *InterProcessLock) localMutex() *sync.Mutex { 31 | locksmu.Lock() 32 | defer locksmu.Unlock() 33 | 34 | if locks[l.Path] == nil { 35 | locks[l.Path] = &sync.Mutex{} 36 | } 37 | 38 | return locks[l.Path] 39 | } 40 | 41 | // Lock the lock, blocking until the lock has been acquired 42 | func (l *InterProcessLock) Lock() error { 43 | local := l.localMutex() 44 | local.Lock() 45 | 46 | if l.filelock != nil { 47 | return fmt.Errorf("expected filelock to be nil") 48 | } 49 | 50 | var err error 51 | 52 | if l.filelock, err = filemutex.New(l.Path); err != nil { 53 | return fmt.Errorf("could not acquire lock: %v", err) 54 | } 55 | 56 | if err = l.filelock.Lock(); err != nil { 57 | return fmt.Errorf("could not acquire file lock: %v", err) 58 | } 59 | 60 | return nil 61 | } 62 | 63 | // Unlock the lock 64 | func (l *InterProcessLock) Unlock() error { 65 | if err := l.filelock.Unlock(); err != nil { 66 | return fmt.Errorf("could not unlock file lock: %v", err) 67 | } 68 | 69 | l.localMutex().Unlock() 70 | return nil 71 | } 72 | 73 | // MustLock engages the lock and panics if that fails (it will still block 74 | // if the lock is already locked, since that is not an error) 75 | func (l *InterProcessLock) MustLock() { 76 | if err := l.Lock(); err != nil { 77 | panic(err) 78 | } 79 | } 80 | 81 | // MustUnlock removes the lock and panics if that fails 82 | func (l *InterProcessLock) MustUnlock() { 83 | if err := l.Unlock(); err != nil { 84 | panic(err) 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /pkg/provider/github.go: -------------------------------------------------------------------------------- 1 | package provider 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "net/http" 7 | "regexp" 8 | "sync" 9 | 10 | "github.com/seantis/roots/pkg/image" 11 | ) 12 | 13 | // GHProvider does not authenticate at the moment 14 | type GHProvider struct { 15 | clients map[string]*http.Client 16 | mu sync.Mutex 17 | } 18 | 19 | func init() { 20 | image.RegisterProvider("gh", &GHProvider{ 21 | clients: make(map[string]*http.Client), 22 | }) 23 | } 24 | 25 | var ghhosts = regexp.MustCompile(`ghcr\.io`) 26 | 27 | // Supports returns true if the URLs host is one of the GitHub Container 28 | // Registry hosts 29 | func (p *GHProvider) Supports(url image.URL) bool { 30 | return ghhosts.MatchString(url.Host) 31 | } 32 | 33 | // GetClient returns a client for the GitHub Container Registry. Currently 34 | // there's no support for private repositories and 'auth' is ignored. 35 | func (p *GHProvider) GetClient(url image.URL, auth string) (*http.Client, error) { 36 | 37 | p.mu.Lock() 38 | defer p.mu.Unlock() 39 | 40 | // The client for Docker is bound to the repository 41 | if p.clients[url.Repository] == nil { 42 | client, err := p.newClient(url.Repository, url.Name, auth) 43 | 44 | if err != nil { 45 | return nil, err 46 | } 47 | 48 | p.clients[url.Repository] = client 49 | } 50 | 51 | return p.clients[url.Repository], nil 52 | } 53 | 54 | // newClient spawns a new unauthenticated http client for GitHub Container 55 | // Repository 56 | func (p *GHProvider) newClient(repository string, name string, auth string) (*http.Client, error) { 57 | // even public api connections need an authorization token 58 | t := "https://ghcr.io/token?scope=repository:%s/%s:pull" 59 | u := fmt.Sprintf(t, repository, name) 60 | 61 | res, err := http.Get(u) 62 | if err != nil { 63 | return nil, fmt.Errorf("error getting access-token via %s: %v", u, err) 64 | } 65 | 66 | if res.StatusCode != 200 { 67 | return nil, fmt.Errorf("GET %s failed with %s", u, res.Status) 68 | } 69 | 70 | // we'll get it from the json response 71 | tr := &dockerTokenResponse{} 72 | err = json.NewDecoder(res.Body).Decode(&tr) 73 | 74 | if err != nil { 75 | return nil, fmt.Errorf("error parsing response: %e", err) 76 | } 77 | 78 | if len(tr.Token) == 0 { 79 | return nil, fmt.Errorf("%s did not return a token", u) 80 | } 81 | 82 | // we then use it to create a client with a proper bearer token set 83 | return clientWithHeaders(map[string]string{ 84 | "Authorization": fmt.Sprintf("Bearer %s", tr.Token), 85 | }), err 86 | } 87 | -------------------------------------------------------------------------------- /pkg/image/remote_test.go: -------------------------------------------------------------------------------- 1 | package image 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/http" 7 | "testing" 8 | 9 | "github.com/dankinder/httpmock" 10 | "github.com/stretchr/testify/assert" 11 | "github.com/stretchr/testify/mock" 12 | ) 13 | 14 | type mockProvider struct { 15 | Server *httpmock.Server 16 | } 17 | 18 | func (p *mockProvider) GetClient(url URL, auth string) (*http.Client, error) { 19 | return http.DefaultClient, nil 20 | } 21 | 22 | func (p *mockProvider) Supports(url URL) bool { 23 | return true 24 | } 25 | 26 | func mockServer() *httpmock.Server { 27 | downstream := &httpmock.MockHandler{} 28 | 29 | header := make(http.Header) 30 | header.Add("Docker-Content-Digest", "foobar") 31 | header.Add("Content-Type", ManifestListMimeType) 32 | 33 | downstream.On("Handle", "HEAD", "/v2/library/ubuntu/manifests/latest", mock.Anything).Return(httpmock.Response{ 34 | Header: header, 35 | }) 36 | 37 | downstream.On("Handle", "GET", "/v2/library/ubuntu/manifests/latest", mock.Anything).Return(httpmock.Response{ 38 | Header: header, 39 | Body: []byte(` 40 | { 41 | "schemaVersion": 2, 42 | "mediaType": "application/vnd.docker.distribution.manifest.list.v2+json", 43 | "manifests": [ 44 | { 45 | "mediaType": "application/vnd.docker.distribution.manifest.v2+json", 46 | "size": 123, 47 | "digest": "foobar", 48 | "platform": { 49 | "architecture": "amd64", 50 | "os": "linux" 51 | } 52 | } 53 | ] 54 | } 55 | `), 56 | }) 57 | 58 | return httpmock.NewServer(downstream) 59 | } 60 | 61 | // TestRemoteDigest tests the lookup of the digest on a mock provider 62 | func TestRemoteDigest(t *testing.T) { 63 | defer ClearProviderRegistry() 64 | 65 | server := mockServer() 66 | defer server.Close() 67 | 68 | RegisterProvider("mock", &mockProvider{ 69 | Server: server, 70 | }) 71 | 72 | url := URL{ 73 | Host: server.URL(), 74 | Name: "ubuntu", 75 | Repository: "library", 76 | Tag: "latest", 77 | } 78 | 79 | remote, _ := NewRemote(context.Background(), url, "") 80 | 81 | digest, err := remote.Digest() 82 | assert.NoError(t, err, "error during mock lookup") 83 | assert.Equal(t, "foobar", digest, "could not lookup mock digest") 84 | 85 | remote.WithPlatform(&Platform{ 86 | Architecture: "arm", 87 | OS: "linux", 88 | }) 89 | digest, err = remote.Digest() 90 | assert.EqualError(t, err, fmt.Sprintf("no manifest found for %s linux/arm", url), "unexpected error") 91 | assert.Equal(t, "", digest, "could not lookup mock digest") 92 | } 93 | -------------------------------------------------------------------------------- /pkg/provider/docker.go: -------------------------------------------------------------------------------- 1 | package provider 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "net/http" 7 | "regexp" 8 | "sync" 9 | 10 | "github.com/seantis/roots/pkg/image" 11 | ) 12 | 13 | // DockerProvider authenticates clients against the Docker Hub 14 | type DockerProvider struct { 15 | clients map[string]*http.Client 16 | mu sync.Mutex 17 | } 18 | 19 | type dockerTokenResponse struct { 20 | Token string `json:"token"` 21 | } 22 | 23 | var dockerhosts = regexp.MustCompile(`([a-z0-9-]+\.)?docker\.io`) 24 | 25 | func init() { 26 | image.RegisterProvider("docker", &DockerProvider{ 27 | clients: make(map[string]*http.Client), 28 | }) 29 | } 30 | 31 | // Supports returns true if the URLs host is one of the google cloud registry hosts 32 | func (p *DockerProvider) Supports(url image.URL) bool { 33 | return dockerhosts.MatchString(url.Host) 34 | } 35 | 36 | // GetClient returns a client authenticated with the Docker Hub. Currently 37 | // there's no support for private repositories and 'auth' is ignored. Note also 38 | // that the token given by Docker Hub expires after 5 minutes - renewal logic 39 | // has not been implemented yet. 40 | func (p *DockerProvider) GetClient(url image.URL, auth string) (*http.Client, error) { 41 | 42 | p.mu.Lock() 43 | defer p.mu.Unlock() 44 | 45 | // The client for Docker is bound to the repository 46 | if p.clients[url.Repository] == nil { 47 | client, err := p.newClient(url.Repository, url.Name, auth) 48 | 49 | if err != nil { 50 | return nil, err 51 | } 52 | 53 | p.clients[url.Repository] = client 54 | } 55 | 56 | return p.clients[url.Repository], nil 57 | } 58 | 59 | // newClient returns a new client authenitcated with the Docker Hub 60 | func (p *DockerProvider) newClient(repository string, name string, auth string) (*http.Client, error) { 61 | // even public api connections need an authorization token 62 | t := "https://auth.docker.io/token?service=registry.docker.io&scope=repository:%s/%s:pull" 63 | u := fmt.Sprintf(t, repository, name) 64 | 65 | res, err := http.Get(u) 66 | if err != nil { 67 | return nil, fmt.Errorf("error getting access-token via %s: %v", u, err) 68 | } 69 | 70 | if res.StatusCode != 200 { 71 | return nil, fmt.Errorf("GET %s failed with %s", u, res.Status) 72 | } 73 | 74 | // we'll get it from the json response 75 | tr := &dockerTokenResponse{} 76 | err = json.NewDecoder(res.Body).Decode(&tr) 77 | 78 | if err != nil { 79 | return nil, fmt.Errorf("error parsing response: %e", err) 80 | } 81 | 82 | if len(tr.Token) == 0 { 83 | return nil, fmt.Errorf("%s did not return a token", u) 84 | } 85 | 86 | // we then use it to create a client with a proper bearer token set 87 | return clientWithHeaders(map[string]string{ 88 | "Authorization": fmt.Sprintf("Bearer %s", tr.Token), 89 | }), err 90 | } 91 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc= 2 | cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= 3 | github.com/alexflint/go-filemutex v1.3.0 h1:LgE+nTUWnQCyRKbpoceKZsPQbs84LivvgwUymZXdOcM= 4 | github.com/alexflint/go-filemutex v1.3.0/go.mod h1:U0+VA/i30mGBlLCrFPGtTe9y6wGQfNAWPBTekHQ+c8A= 5 | github.com/dankinder/httpmock v1.0.4 h1:jGiak5b4VKB1qjSXF2O/DcoYNfGVID+NwuE/dBm5H7Y= 6 | github.com/dankinder/httpmock v1.0.4/go.mod h1:ixH0HJU1412LcL7yn20EuEK/E8kO5VVH3y8Hj+QU1sg= 7 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 8 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 9 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 10 | github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= 11 | github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 12 | github.com/jawher/mow.cli v1.2.0 h1:e6ViPPy+82A/NFF/cfbq3Lr6q4JHKT9tyHwTCcUQgQw= 13 | github.com/jawher/mow.cli v1.2.0/go.mod h1:y+pcA3jBAdo/GIZx/0rFjw/K2bVEODP9rfZOfaiq8Ko= 14 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 15 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 16 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 17 | github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= 18 | github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= 19 | github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= 20 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 21 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 22 | github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= 23 | github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 24 | golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= 25 | golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= 26 | golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= 27 | golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 28 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 29 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 30 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 31 | gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 32 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 33 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 34 | -------------------------------------------------------------------------------- /pkg/image/url.go: -------------------------------------------------------------------------------- 1 | package image 2 | 3 | import ( 4 | "fmt" 5 | "regexp" 6 | "strings" 7 | ) 8 | 9 | var localurl = regexp.MustCompile(`(?i)^http://(127\.[\d.]+|[0:]+1|localhost)`) 10 | 11 | // URL contains the result of a parsed container url like the following: 12 | // * ubuntu:latest 13 | // * gcr.io/google-containers/alpine 14 | // * busybox:123@foobar 15 | // See also https://stackoverflow.com/q/37861791 16 | type URL struct { 17 | Name string 18 | Host string 19 | Repository string 20 | Tag string 21 | Digest string 22 | } 23 | 24 | // String returns the normalized form of the URL (i.e the longer form with 25 | // a guaranteed host, repository and tag name) - if the URL is empty, "" 26 | // is returned 27 | func (url URL) String() string { 28 | if len(url.Name) == 0 { 29 | return "" 30 | } 31 | 32 | if len(url.Digest) == 0 { 33 | return fmt.Sprintf("%s/%s/%s:%s", 34 | url.Host, 35 | url.Repository, 36 | url.Name, 37 | url.Tag) 38 | } 39 | 40 | return fmt.Sprintf("%s/%s/%s:%s@%s", 41 | url.Host, 42 | url.Repository, 43 | url.Name, 44 | url.Tag, 45 | url.Digest) 46 | } 47 | 48 | // Endpoint returns an API endpoint of the v2 registry API 49 | func (url URL) Endpoint(segments ...string) string { 50 | // by default, no protocol is given and we force https 51 | host := fmt.Sprintf("https://%s", url.Host) 52 | 53 | // the host may include the http protocol if it points to a local address 54 | if localurl.MatchString(url.Host) { 55 | host = url.Host 56 | } 57 | 58 | return fmt.Sprintf("%s/v2/%s/%s/%s", 59 | host, 60 | url.Repository, 61 | url.Name, 62 | strings.Join(segments, "/")) 63 | } 64 | 65 | // Reference returns either the digest or, if the digest is absent, the tag 66 | func (url URL) Reference() string { 67 | if len(url.Digest) > 0 { 68 | return url.Digest 69 | } 70 | 71 | return url.Tag 72 | } 73 | 74 | // Parse parses the given URL and returns an error if it doesn't look correct 75 | func Parse(url string) (*URL, error) { 76 | url = strings.Trim(url, " \n\t") 77 | 78 | if len(url) == 0 { 79 | return &URL{}, fmt.Errorf("passed an empty url") 80 | } 81 | 82 | p := &URL{} 83 | 84 | // if there's an @, we got our digest 85 | if strings.Contains(url, "@") { 86 | url, p.Digest = bisect(url, "@") 87 | } 88 | 89 | // before the slash is the host and repository, after it the name and tag 90 | parts := strings.Split(url, "/") 91 | 92 | // if there is a slash and we got a dot or a colon we found a host name 93 | if strings.Contains(url, "/") && strings.ContainsAny(parts[0], ".:") { 94 | p.Host, parts = parts[0], parts[1:] 95 | } 96 | 97 | // if there's a colon in the last part, we got a tag 98 | if strings.Contains(parts[len(parts)-1], ":") { 99 | parts[len(parts)-1], p.Tag = bisect(parts[len(parts)-1], ":") 100 | } 101 | 102 | // the rest should be the name and possibly the repository 103 | switch len(parts) { 104 | case 1: 105 | p.Name = parts[0] 106 | case 2: 107 | p.Repository, p.Name = parts[0], parts[1] 108 | default: 109 | return &URL{}, fmt.Errorf("too many slashes in %s", url) 110 | } 111 | 112 | if len(p.Name) == 0 { 113 | return &URL{}, fmt.Errorf("could not find a name for %s", url) 114 | } 115 | 116 | // finally, we add some defaults that are set in practice 117 | if len(p.Host) == 0 { 118 | p.Host = "registry-1.docker.io" 119 | } 120 | 121 | if len(p.Tag) == 0 { 122 | p.Tag = "latest" 123 | } 124 | 125 | if len(p.Repository) == 0 { 126 | p.Repository = "library" 127 | } 128 | 129 | return p, nil 130 | } 131 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Roots 2 | 3 | Pulls containers from registries and extracts them into a folder. The resulting 4 | root tree can be used to inspect all the files of a container and it can be run 5 | directly using [systemd-nspawn](https://www.freedesktop.org/software/systemd/man/systemd-nspawn.html). 6 | 7 | There are other tools that can accomplish the same thing, but they all do 8 | more than roots does. Roots fetches image layers, extracts them and calls it a day. 9 | 10 | [![Go Report Card](https://goreportcard.com/badge/github.com/seantis/roots)](https://goreportcard.com/report/github.com/seantis/roots) 11 | 12 | ## Container Pull 13 | 14 | Pull a container, extract it and run it using systemd-nspawn: 15 | 16 | ```bash 17 | roots pull debian:bookworm ./debian 18 | sudo systemd-nspawn -D ./debian /bin/bash 19 | ``` 20 | 21 | Existing directories can be overwritten using `--force`: 22 | 23 | ```bash 24 | roots pull debian:bookworm ./debian --force 25 | ``` 26 | 27 | ## Container Digest 28 | 29 | Roots supports checking the digest of images, which is useful to check if 30 | an image has had an update: 31 | 32 | ```bash 33 | roots digest debian:bookworm 34 | ``` 35 | 36 | ## Cache 37 | 38 | Roots keeps downloaded layers in a cache. This cache can be purged periodically: 39 | 40 | ```bash 41 | roots purge 42 | ``` 43 | 44 | The default cache directory is `/var/cache/roots` for root users or 45 | `~/.cache/seantis/roots` for any other user. You can override this with the 46 | cache option: 47 | 48 | ```bash 49 | roots pull debian ./debian --cache /tmp/cache 50 | roots purge --cache /tmp/cache 51 | ``` 52 | 53 | Or you can disable the cache entirely as follows: 54 | 55 | ```bash 56 | roots pull debian ./debian --cache=no 57 | ``` 58 | 59 | You can also set this value through the `ROOTS_CACHE` environment variable. 60 | 61 | ## Private Registries 62 | 63 | Private registries are supported, though currently only the Google Container 64 | Registry has been implemented (pull requests welcome!): 65 | 66 | ```bash 67 | roots pull gcr.io/google-containers/etcd:3.3.10 ./etcd --auth account.json 68 | ``` 69 | 70 | ## Multi-Arch 71 | 72 | It is possible to select a specific architecture/os for the image if it supports 73 | multi-arch manifests (a.k.a fat manifests): 74 | 75 | ```bash 76 | roots pull gcr.io/google-containers/etcd:3.3.10 --arch arm --os linux 77 | ``` 78 | 79 | If the image does not support multiple platforms, using --arch/--os will result 80 | in an error. If the image does support multiple platforms and --arch/--os is 81 | omitted, the default manifest defined by the registry is used. 82 | 83 | ## Requirements / Limitations 84 | 85 | Roots has only been tested on Linux/MacOS. 86 | 87 | ## Installation 88 | 89 | To install the roots command-line run the following command: 90 | 91 | ```bash 92 | go install github.com/seantis/roots@latest 93 | ``` 94 | 95 | ## Multiple Processes 96 | 97 | It is possible to run multiple roots processes at the same time, however its 98 | use is quite limited as cache and destination are locked during pull/purge. 99 | 100 | That only leaves the digest operation, which doesn't write anything, as well as 101 | the option to use no cache or separate caches with differing destinations. 102 | 103 | Feel free to open an issue if you have a use case for this. 104 | 105 | ## Tests 106 | 107 | Unit tests can be run as follows: 108 | 109 | ```bash 110 | make test 111 | ``` 112 | 113 | Additional tests are run using GitHub actions. To try those locally, run the 114 | following command (requires docker): 115 | 116 | ```bash 117 | make test-all 118 | ``` 119 | 120 | ## Releases 121 | 122 | There's a release process defined with GitHub Actions, but it is currently 123 | defunct as public repositories do not get properly triggered when tagging 124 | a commit. 125 | 126 | Therefore, this is the current manual release process: 127 | 128 | ```bash 129 | git tag vX.Y.Z 130 | git push --tags 131 | 132 | GITHUB_TOKEN="foobar" VERSION=vX.Y.Z make release 133 | ``` 134 | 135 | GITHUB_TOKEN is a [personal access token (classic)](https://github.com/settings/tokens) 136 | with `repo` scope. 137 | 138 | In the future this step should happen automatically if the tests pass, with 139 | the only requirement being a tagged commit. 140 | 141 | Note also that currently only linux/amd64 is offered as a prebuilt binary. Due 142 | to our use of the os/user. Currently we need to use CGO, which makes cross 143 | compilation a bit tricky. Other platforms are currently required to install this 144 | tool using the default `go install github.com/seantis/roots@latest` approach. 145 | 146 | ## Test-Releases 147 | 148 | You can create a test release using: 149 | 150 | ```bash 151 | make test-release 152 | ``` 153 | 154 | Test releases have the version `0.0.0`. 155 | -------------------------------------------------------------------------------- /pkg/image/remote.go: -------------------------------------------------------------------------------- 1 | package image 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "io" 8 | "net/http" 9 | ) 10 | 11 | // Remote represents an image on a remote repository 12 | type Remote struct { 13 | client *http.Client 14 | url URL 15 | platform *Platform 16 | ctx context.Context 17 | } 18 | 19 | func (r *Remote) String() string { 20 | if r.platform != nil { 21 | return fmt.Sprintf("%s %s", r.url, r.platform) 22 | } 23 | 24 | return r.url.String() 25 | } 26 | 27 | // NewRemote returns a new remote instance. An error is returned if the 28 | // remote instance cannot be accessed due to lack of permissions. 29 | func NewRemote(ctx context.Context, url URL, auth string) (*Remote, error) { 30 | provider, err := LookupProvider(url) 31 | if err != nil { 32 | return nil, err 33 | } 34 | 35 | client, err := provider.GetClient(url, auth) 36 | if err != nil { 37 | return nil, err 38 | } 39 | 40 | err = requireSupportedMimeTypes(client, url) 41 | if err != nil { 42 | return nil, err 43 | } 44 | 45 | return &Remote{ 46 | url: url, 47 | client: client, 48 | ctx: ctx, 49 | }, nil 50 | } 51 | 52 | // Platforms returns all the platforms the image supports. Nil is is 53 | // returned if the image does not have multi-platform support (i.e. there is 54 | // no manifest list). 55 | // 56 | // If the image has platforms, you should bind the required platform to the 57 | // Remote using WithPlatform, before using other methods, as you will otherwise 58 | // get whatever the registry deems to be the default platform of the manifest, 59 | // which might not be what you want. 60 | func (r *Remote) Platforms() ([]*Platform, error) { 61 | 62 | // try to get the manifest list (not all images have this) 63 | l, err := r.ManifestList() 64 | if err != nil || l == nil { 65 | return nil, err 66 | } 67 | 68 | // each manifest has exactly one platform 69 | platforms := make([]*Platform, len(l.Manifests)) 70 | for i, m := range l.Manifests { 71 | platforms[i] = &m.Platform 72 | } 73 | 74 | return platforms, nil 75 | } 76 | 77 | // WithPlatform binds the given platform to the remote and uses it to 78 | // scope the Digest and Manifest methods 79 | func (r *Remote) WithPlatform(p *Platform) { 80 | r.platform = p 81 | } 82 | 83 | // ManifestList queries the remote for the manifest list and parses the result. 84 | // If the manifest list does not exist, the method returns nil, nil instead of 85 | // an error, as manifest lists are not available for most images today. 86 | func (r *Remote) ManifestList() (*ManifestList, error) { 87 | 88 | // not having a manifest list is no error 89 | res, err := r.request("GET", ManifestListMimeType, "manifests", r.url.Reference()) 90 | if err != nil { 91 | return nil, nil 92 | } 93 | 94 | // not being able to parse an existing list is however 95 | lst := &ManifestList{} 96 | if err := r.unmarshal(res, lst); err != nil { 97 | return nil, fmt.Errorf("error parsing manifest list: %v", err) 98 | } 99 | 100 | return lst, nil 101 | } 102 | 103 | // Manifest gets the manifest of the image. The current platform is 104 | // respected if one was set through WithPlatform. 105 | func (r *Remote) Manifest() (*Manifest, error) { 106 | 107 | // the digest is bound to the platform 108 | digest, err := r.Digest() 109 | if err != nil { 110 | return nil, err 111 | } 112 | 113 | // it should almost certainly be fetchable at this point 114 | res, err := r.request("GET", ManifestMimeType, "manifests", digest) 115 | if err != nil { 116 | return nil, fmt.Errorf("error requesting manifest@%s: %v", digest, err) 117 | } 118 | 119 | // if the server responds with a manifest list, our digest is not correct 120 | if res.Header.Get("Content-Type") != ManifestMimeType { 121 | return nil, fmt.Errorf("content type for %s cannot be %s", digest, res.Header.Get("Content-Type")) 122 | } 123 | 124 | // we must also be able to parse it 125 | m := &Manifest{Digest: digest} 126 | if err := r.unmarshal(res, &m); err != nil { 127 | return nil, fmt.Errorf("error parsing manifest: %v", err) 128 | } 129 | 130 | return m, nil 131 | } 132 | 133 | // Digest gets the latest digest of the image. The current platform is 134 | // respected if one was set through WithPlatform. 135 | func (r *Remote) Digest() (string, error) { 136 | // due to https://github.com/docker/distribution/issues/2395 we always 137 | // have to request the manifest list, even if it doesn't exist, as images 138 | // with manifest lists on docker hub will not return the expected digest 139 | lst, err := r.ManifestList() 140 | if err != nil { 141 | return "", err 142 | } 143 | 144 | // if there's a list, but no platform, take the first item 145 | // 146 | // we could be cleverer here by picking the platform or we could let 147 | // the user know that he should pick one 148 | if r.platform == nil && lst != nil && len(lst.Manifests) != 0 { 149 | return lst.Manifests[0].Digest, nil 150 | } 151 | 152 | // if there's no list and no platform, fall back to whatever the server 153 | // gives us through the docker-content-digest header 154 | if r.platform == nil && (lst == nil || len(lst.Manifests) == 0) { 155 | res, err := r.request("HEAD", ManifestMimeType, "manifests", r.url.Reference()) 156 | 157 | if err != nil { 158 | return "", fmt.Errorf("failed to fetch manifest: %v", err) 159 | } 160 | 161 | return res.Header.Get("Docker-Content-Digest"), nil 162 | } 163 | 164 | // if there is a platform, we require a list 165 | if lst == nil { 166 | return "", fmt.Errorf("no multi-platform support: %s", r.url) 167 | } 168 | 169 | for _, m := range lst.Manifests { 170 | if m.Platform == *r.platform { 171 | return m.Digest, nil 172 | } 173 | } 174 | 175 | // there was no match 176 | return "", fmt.Errorf("no manifest found for %s", r) 177 | } 178 | 179 | // Layers returns the layers of the image. The current plaform is 180 | func (r *Remote) Layers() ([]ManifestLayer, error) { 181 | 182 | m, err := r.Manifest() 183 | if err != nil { 184 | return nil, err 185 | } 186 | 187 | return m.Layers, nil 188 | } 189 | 190 | // DownloadLayer downloads a layer to a Writer 191 | func (r *Remote) DownloadLayer(digest string, w io.Writer) error { 192 | 193 | res, err := r.request("GET", "*", "blobs", digest) 194 | if err != nil { 195 | return fmt.Errorf("failed to download %s: %v", digest, err) 196 | } 197 | 198 | // copy the downloads using the default buffer 199 | defer res.Body.Close() 200 | 201 | _, err = io.Copy(w, res.Body) 202 | if err != nil { 203 | return fmt.Errorf("error downloading %s: %v", digest, err) 204 | } 205 | 206 | return nil 207 | } 208 | 209 | func (r *Remote) request(method string, accept string, segments ...string) (*http.Response, error) { 210 | req, err := http.NewRequest(method, r.url.Endpoint(segments...), nil) 211 | if err != nil { 212 | return nil, fmt.Errorf("error requesting %s: %v", req.URL, err) 213 | } 214 | 215 | req = req.WithContext(r.ctx) 216 | 217 | req.Header.Add("Accept", accept) 218 | res, err := r.client.Do(req) 219 | 220 | if err != nil { 221 | return nil, fmt.Errorf("error requesting %s: %v", req.URL, err) 222 | } 223 | 224 | if res.StatusCode != 200 { 225 | return nil, fmt.Errorf("%s %s failed with %s", method, req.URL, res.Status) 226 | } 227 | 228 | return res, nil 229 | } 230 | 231 | func (r *Remote) unmarshal(res *http.Response, v interface{}) error { 232 | body, err := io.ReadAll(res.Body) 233 | defer res.Body.Close() 234 | 235 | if err != nil { 236 | return fmt.Errorf("error reading response body: %v", err) 237 | } 238 | 239 | err = json.Unmarshal(body, &v) 240 | if err != nil { 241 | return fmt.Errorf("error unmarshaling response into %v: %v", v, err) 242 | } 243 | 244 | return nil 245 | } 246 | -------------------------------------------------------------------------------- /pkg/image/untar.go: -------------------------------------------------------------------------------- 1 | package image 2 | 3 | import ( 4 | "archive/tar" 5 | "compress/gzip" 6 | "context" 7 | "errors" 8 | "fmt" 9 | "io" 10 | "os" 11 | "path" 12 | "path/filepath" 13 | "regexp" 14 | "sort" 15 | "strings" 16 | ) 17 | 18 | // detect relative paths that try to escape the destination directory 19 | var unsafepath = regexp.MustCompile(`/?\.\./`) 20 | 21 | // walkHandler takes a tar.Header and handles it, returning an optional error 22 | type walkHandler func(*tar.Header, *tar.Reader) error 23 | 24 | // untarLayer takes an OCI layer and extracts it into a directory, observing 25 | // any whiteouts that might be specified in the layer. 26 | // See: https://github.com/opencontainers/image-spec/blob/master/layer.md 27 | func untarLayer(ctx context.Context, archive, dst string, dirmodes map[string]os.FileMode) error { 28 | r, err := os.Open(archive) 29 | if err == nil { 30 | defer r.Close() 31 | } else { 32 | return err 33 | } 34 | 35 | gzr, err := gzip.NewReader(r) 36 | if err == nil { 37 | defer gzr.Close() 38 | } else { 39 | return err 40 | } 41 | 42 | reset := func() { 43 | if _, err := r.Seek(0, 0); err != nil { 44 | panic(fmt.Errorf("failed to seek %s: %v", archive, err)) 45 | } 46 | 47 | if err := gzr.Reset(r); err != nil { 48 | panic(fmt.Errorf("failed to reset %s: %v", archive, err)) 49 | } 50 | } 51 | 52 | // pre-process the archive 53 | err = walkTar(ctx, gzr, func(h *tar.Header, r *tar.Reader) error { 54 | 55 | // apply whiteout files 56 | if isWhiteoutPath(h.Name) { 57 | if err := applyWhiteout(dst, h.Name); err != nil { 58 | return err 59 | } 60 | } 61 | 62 | // detect unsafe filenames and stop everything if found 63 | if unsafepath.MatchString(h.Name) { 64 | return fmt.Errorf("refusing to extract unsafe path: %s", h.Name) 65 | } 66 | 67 | // create directory structure 68 | if h.Typeflag == tar.TypeDir { 69 | file := filepath.Join(dst, h.Name) 70 | 71 | if err := os.MkdirAll(file, 0755); err != nil { 72 | return fmt.Errorf("error creating directory %s: %v", file, err) 73 | } 74 | 75 | // store actual file mode of directories to set them later 76 | dirmodes[file] = os.FileMode(h.Mode) 77 | } 78 | 79 | return nil 80 | }) 81 | 82 | if err != nil { 83 | return err 84 | } 85 | 86 | reset() 87 | 88 | // create all regular files 89 | err = walkTar(ctx, gzr, func(h *tar.Header, r *tar.Reader) error { 90 | 91 | // skip anything but regular files 92 | if h.Typeflag != tar.TypeReg { 93 | return nil 94 | } 95 | 96 | // skip whiteout files 97 | if isWhiteoutPath(h.Name) { 98 | return nil 99 | } 100 | 101 | // remove the file if it exists 102 | file := filepath.Join(dst, h.Name) 103 | 104 | if info, err := os.Stat(file); err == nil && !info.IsDir() { 105 | if err := os.Remove(file); err != nil { 106 | return fmt.Errorf("error replacing %s: %v", file, err) 107 | } 108 | } 109 | 110 | // write the file, (re-)setting the mode at the end, which is the 111 | // only way to make absolutely sure that is set correctly 112 | mode := h.FileInfo().Mode() 113 | 114 | f, err := os.OpenFile(file, os.O_CREATE|os.O_RDWR, mode) 115 | if err != nil { 116 | return fmt.Errorf("error creating %s: %v", file, err) 117 | } 118 | 119 | if _, err := io.Copy(f, r); err != nil { 120 | return fmt.Errorf("error copying %s: %v", file, err) 121 | } 122 | 123 | if err := os.Chmod(file, mode); err != nil { 124 | return fmt.Errorf("error setting mode for %s: %v", file, err) 125 | } 126 | 127 | return f.Close() 128 | }) 129 | 130 | if err != nil { 131 | return err 132 | } 133 | 134 | reset() 135 | 136 | // create links 137 | return walkTar(ctx, gzr, func(h *tar.Header, r *tar.Reader) error { 138 | 139 | // skip anything that isn't a link 140 | if h.Typeflag != tar.TypeLink && h.Typeflag != tar.TypeSymlink { 141 | return nil 142 | } 143 | 144 | new := filepath.Join(dst, h.Name) 145 | 146 | var old string 147 | if h.Linkname[0] == '.' || !strings.Contains(h.Linkname, "/") { 148 | old = filepath.Join(filepath.Dir(new), h.Linkname) 149 | } else { 150 | old = filepath.Join(dst, h.Linkname) 151 | } 152 | 153 | // remove the link if it exists 154 | if info, err := os.Lstat(new); err == nil && !info.IsDir() { 155 | if err := os.Remove(new); err != nil { 156 | return fmt.Errorf("error replacing %s: %v", new, err) 157 | } 158 | } 159 | 160 | // create hard links 161 | if h.Typeflag == tar.TypeLink { 162 | if err := os.Link(old, new); err != nil { 163 | return fmt.Errorf("error creating hard link %s->%s: %v", new, old, err) 164 | } 165 | return nil 166 | } 167 | 168 | // create symbolic links 169 | if err := os.Symlink(h.Linkname, new); err != nil { 170 | return fmt.Errorf("error creating symbolic link %s->%s: %v", new, old, err) 171 | } 172 | 173 | return nil 174 | }) 175 | } 176 | 177 | // walkTar takes a gzip.Reader and calls a handler function 178 | func walkTar(ctx context.Context, gzr *gzip.Reader, handler walkHandler) error { 179 | tr := tar.NewReader(gzr) 180 | 181 | for { 182 | header, err := tr.Next() 183 | 184 | if err != nil { 185 | if err != io.EOF { 186 | return fmt.Errorf("failed to walk tar: %v", err) 187 | } 188 | return nil 189 | } 190 | 191 | select { 192 | case <-ctx.Done(): 193 | return errors.New("interrupted") 194 | default: 195 | err = handler(header, tr) 196 | 197 | if err != nil { 198 | return err 199 | } 200 | } 201 | } 202 | } 203 | 204 | // setDirectoryPermissions takes a list of directories with file permissions 205 | // and applies the permissions to those files 206 | func setDirectoryPermissions(dirmodes map[string]os.FileMode) error { 207 | 208 | // process directories with longer paths first, to set the permissions 209 | // of children before setting the permissions of parents 210 | order := make([]string, 0, len(dirmodes)) 211 | for path := range dirmodes { 212 | 213 | // it's possible that certain paths do not exist anymore, if a 214 | // whiteout was applied in the process 215 | if info, err := os.Stat(path); os.IsNotExist(err) { 216 | continue 217 | } else if err != nil { 218 | return fmt.Errorf("error accessing %s: %v", path, err) 219 | } else if !info.IsDir() { 220 | return fmt.Errorf("not a directory: %s", path) 221 | } 222 | 223 | order = append(order, path) 224 | } 225 | 226 | sort.Slice(order, func(j, k int) bool { 227 | return len(order[j]) > len(order[k]) 228 | }) 229 | 230 | for _, path := range order { 231 | if err := os.Chmod(path, dirmodes[path]); err != nil { 232 | return fmt.Errorf("error setting %04o on %s: %v", dirmodes[path], path, err) 233 | } 234 | } 235 | 236 | return nil 237 | } 238 | 239 | // applyWhiteout takes a destination and a relative whiteout path and applies it 240 | func applyWhiteout(dst, whiteout string) error { 241 | if strings.HasSuffix(whiteout, ".wh..wh..opq") { 242 | return applyOpaqueWhiteout(dst, whiteout) 243 | } 244 | 245 | return applySimpleWhiteout(dst, whiteout) 246 | } 247 | 248 | func applyOpaqueWhiteout(dst, whiteout string) error { 249 | base := path.Join(dst, filepath.Dir(whiteout)) 250 | 251 | f, err := os.Open(base) 252 | if err != nil { 253 | if os.IsNotExist(err) { 254 | return nil 255 | } 256 | 257 | return err 258 | } 259 | defer f.Close() 260 | 261 | const buffer = 10 262 | 263 | for { 264 | lst, err := f.Readdir(buffer) 265 | 266 | if err == io.EOF { 267 | return nil 268 | } 269 | 270 | for _, info := range lst { 271 | file := path.Join(base, info.Name()) 272 | 273 | if info.IsDir() { 274 | err = os.RemoveAll(file) 275 | 276 | if err != nil { 277 | return err 278 | } 279 | } 280 | 281 | if err := os.Remove(file); err != nil && !os.IsNotExist(err) { 282 | return err 283 | } 284 | } 285 | } 286 | } 287 | 288 | func applySimpleWhiteout(dst, whiteout string) error { 289 | file := path.Join(dst, filepath.Dir(whiteout), filepath.Base(whiteout)[4:]) 290 | info, err := os.Stat(file) 291 | 292 | if err != nil { 293 | if os.IsNotExist(err) { 294 | return nil 295 | } 296 | 297 | return err 298 | } 299 | 300 | if info.IsDir() { 301 | return os.RemoveAll(file) 302 | } 303 | 304 | return os.Remove(file) 305 | } 306 | 307 | func isWhiteoutPath(p string) bool { 308 | return strings.HasPrefix(filepath.Base(p), ".wh.") 309 | } 310 | -------------------------------------------------------------------------------- /pkg/image/store.go: -------------------------------------------------------------------------------- 1 | package image 2 | 3 | import ( 4 | "bufio" 5 | "context" 6 | "crypto/md5" 7 | "fmt" 8 | "os" 9 | "path" 10 | "path/filepath" 11 | "strings" 12 | 13 | "github.com/seantis/roots/pkg/lock" 14 | ) 15 | 16 | // Store negotiates between the local destination and the remote image, 17 | // optionally caching layers and offering a way to purge the cache. 18 | type Store struct { 19 | Path string 20 | } 21 | 22 | // StoreResult contains the result of a DownloadLayer call 23 | type StoreResult struct { 24 | Path string 25 | Digest string 26 | Error error 27 | } 28 | 29 | // NewStore returns a new store 30 | func NewStore(folder string) (*Store, error) { 31 | 32 | // ignore path creation errors - if it's serious, we'll know about it later 33 | _ = os.Mkdir(path.Join(folder, "layers"), 0755) 34 | _ = os.Mkdir(path.Join(folder, "links"), 0755) 35 | 36 | return &Store{ 37 | Path: folder, 38 | }, nil 39 | } 40 | 41 | // Purge removes all the unused data from the cache 42 | func (s *Store) Purge() error { 43 | 44 | // lock the whole cache 45 | defer s.lockCache().MustUnlock() 46 | 47 | // load the destination folders and the layers connected to them 48 | links, err := s.readLinks() 49 | if err != nil { 50 | return err 51 | } 52 | 53 | // keep a list of known layers 54 | layers := make(map[string]bool) 55 | 56 | for dst, digests := range links { 57 | _, err := os.Stat(dst) 58 | 59 | if err != nil { 60 | if !os.IsNotExist(err) { 61 | return fmt.Errorf("error reading %s: %v", dst, err) 62 | } 63 | 64 | // the destination does not exist anymore, remove the link 65 | if err := os.Remove(s.LinkPath(dst)); err != nil { 66 | return fmt.Errorf("error removing %s: %v", dst, err) 67 | } 68 | 69 | continue 70 | } 71 | 72 | // the destination still exists, add its digest to the known layers 73 | for _, digest := range digests { 74 | layers[digest] = true 75 | } 76 | } 77 | 78 | // go through all the cached layers and remove the unknown ones 79 | selector := fmt.Sprintf("%s/layers/*.layer", s.Path) 80 | cached, err := filepath.Glob(selector) 81 | if err != nil { 82 | return fmt.Errorf("error reading %s: %v", selector, err) 83 | } 84 | 85 | for _, file := range cached { 86 | digest := strings.TrimSuffix(filepath.Base(file), ".layer") 87 | 88 | if !layers[digest] { 89 | if err := os.Remove(file); err != nil { 90 | return fmt.Errorf("error removing %s: %v", file, err) 91 | } 92 | } 93 | } 94 | 95 | return nil 96 | } 97 | 98 | // LinkPath returns the path to the link file in the cache 99 | func (s *Store) LinkPath(dst string) string { 100 | return path.Join(s.Path, "links", fmt.Sprintf("%x.link", md5.Sum([]byte(dst)))) 101 | } 102 | 103 | // LayerPath returns the path to the layer file in the cache 104 | func (s *Store) LayerPath(digest string) string { 105 | return path.Join(s.Path, "layers", fmt.Sprintf("%s.layer", digest)) 106 | } 107 | 108 | // Extract takes a remote, downloads the layers and stores them at dst 109 | func (s *Store) Extract(ctx context.Context, r *Remote, dst string) error { 110 | 111 | // fetch the layers 112 | layers, err := r.Layers() 113 | if err != nil { 114 | return fmt.Errorf("error querying layers for %s: %v", r, err) 115 | } 116 | 117 | if len(layers) == 0 { 118 | return fmt.Errorf("no layers found for %s", r) 119 | } 120 | 121 | // lock the whole destination as well as the cache 122 | defer s.lockCache().MustUnlock() 123 | defer s.lockDestination(dst).MustUnlock() 124 | 125 | // ensure the destination is empty 126 | entries, err := os.ReadDir(dst) 127 | if err != nil { 128 | return fmt.Errorf("error extracting to %s: %v", dst, err) 129 | } 130 | 131 | if len(entries) > 1 { 132 | return fmt.Errorf("directory %s is not empty", dst) 133 | } 134 | 135 | // download the layers concurrently 136 | results := make([]chan *StoreResult, len(layers)) 137 | for i, l := range layers { 138 | results[i], err = s.downloadLayer(ctx, r, l.Digest) 139 | 140 | if err != nil { 141 | return fmt.Errorf("error writing %s: %v", l.Digest, err) 142 | } 143 | } 144 | 145 | // process the layers in order 146 | digests := make([]string, len(results)) 147 | dirmodes := make(map[string]os.FileMode) 148 | 149 | for i := range results { 150 | result := <-results[i] 151 | 152 | if result.Error != nil { 153 | return fmt.Errorf("error downloading %s: %v", result.Digest, result.Error) 154 | } 155 | 156 | err := untarLayer(ctx, result.Path, dst, dirmodes) 157 | 158 | if err != nil { 159 | return fmt.Errorf("error extracting %s: %v", result.Path, err) 160 | } 161 | 162 | digests[i] = result.Digest 163 | } 164 | 165 | // set the correct permissions for all directories 166 | if err := setDirectoryPermissions(dirmodes); err != nil { 167 | return fmt.Errorf("error setting directory permissions: %v", err) 168 | } 169 | 170 | // record the destination in the cache 171 | return s.saveLink(dst, digests) 172 | } 173 | 174 | // downloadLayer downloads the given layer into the cache and sends a path 175 | // through the given channel, once the download is complete. 176 | // If the layer was downloaded already, the path will be sent to the channel 177 | // right away. 178 | func (s *Store) downloadLayer(ctx context.Context, r *Remote, digest string) (chan *StoreResult, error) { 179 | 180 | // we need a buffer of 1 so we can send to the channel even if the other 181 | // side has not yet started listening 182 | out := make(chan *StoreResult, 1) 183 | dst := s.LayerPath(digest) 184 | 185 | // if the layer already exists, send it right away 186 | _, err := os.Stat(dst) 187 | if err == nil { 188 | out <- &StoreResult{ 189 | Path: dst, 190 | Error: nil, 191 | Digest: digest, 192 | } 193 | return out, nil 194 | } 195 | 196 | // otherwise create the file 197 | w, err := os.Create(dst) 198 | if err != nil { 199 | return nil, err 200 | } 201 | 202 | // then download it in the background 203 | go func() { 204 | defer w.Close() 205 | err := r.DownloadLayer(digest, w) 206 | 207 | out <- &StoreResult{ 208 | Path: dst, 209 | Error: err, 210 | Digest: digest, 211 | } 212 | }() 213 | 214 | return out, nil 215 | } 216 | 217 | // saveLink takes a destination and a list of layer digests and records it in 218 | // the cache. The resulting files are used to only Purge what is necessary. 219 | // 220 | // note that this function does not do any locking -> it assumes the cache 221 | // has been locked already 222 | func (s *Store) saveLink(dst string, digests []string) error { 223 | 224 | file := s.LinkPath(dst) 225 | f, err := os.Create(file) 226 | if err != nil { 227 | return fmt.Errorf("error creating %s: %v", file, err) 228 | } 229 | 230 | // the first line is the header 231 | if _, err := f.WriteString(fmt.Sprintf("%s\n", dst)); err != nil { 232 | return fmt.Errorf("error writing %s: %v", file, err) 233 | } 234 | 235 | // the other lines are the digests 236 | for _, digest := range digests { 237 | if _, err := f.WriteString(fmt.Sprintf("%s\n", digest)); err != nil { 238 | return fmt.Errorf("error writing %s: %v", file, err) 239 | } 240 | } 241 | 242 | return nil 243 | } 244 | 245 | // readLinks walks through the stored links and returns a map of the 246 | // destinations and the digests they're associated with 247 | func (s *Store) readLinks() (map[string][]string, error) { 248 | selector := fmt.Sprintf("%s/links/*.link", s.Path) 249 | 250 | files, err := filepath.Glob(selector) 251 | if err != nil { 252 | return nil, fmt.Errorf("error reading %s: %v", selector, err) 253 | } 254 | 255 | links := make(map[string][]string) 256 | 257 | for _, file := range files { 258 | f, err := os.Open(file) 259 | if err != nil { 260 | return nil, fmt.Errorf("error reading %s: %v", file, err) 261 | } 262 | 263 | var dst string 264 | 265 | scanner := bufio.NewScanner(f) 266 | for scanner.Scan() { 267 | 268 | // the first line contains the destination 269 | if dst == "" { 270 | dst = scanner.Text() 271 | continue 272 | } 273 | 274 | // subsequent lines contain layers 275 | links[dst] = append(links[dst], scanner.Text()) 276 | } 277 | 278 | // manually close instead of deferring, otherwise files are kept open 279 | // until the function returns 280 | f.Close() 281 | 282 | if err := scanner.Err(); err != nil { 283 | return nil, fmt.Errorf("error reading %s: %v", file, err) 284 | } 285 | } 286 | 287 | return links, nil 288 | } 289 | 290 | func (s *Store) lockCache() *lock.InterProcessLock { 291 | l := &lock.InterProcessLock{Path: path.Join(s.Path, ".lock")} 292 | l.MustLock() 293 | 294 | return l 295 | } 296 | 297 | func (s *Store) lockDestination(dst string) *lock.InterProcessLock { 298 | l := &lock.InterProcessLock{Path: fmt.Sprintf("%s.lock", dst)} 299 | l.MustLock() 300 | 301 | return l 302 | } 303 | -------------------------------------------------------------------------------- /roots.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "log" 7 | "os" 8 | "os/signal" 9 | "os/user" 10 | "path" 11 | "runtime" 12 | "strings" 13 | 14 | cli "github.com/jawher/mow.cli" 15 | "github.com/seantis/roots/pkg/image" 16 | _ "github.com/seantis/roots/pkg/provider" // to register providers 17 | ) 18 | 19 | var ( 20 | version = "dev" 21 | commit = "none" 22 | date = "unknown" 23 | ) 24 | 25 | func main() { 26 | app := cli.App("roots", "Download and extract containers") 27 | ctx := newInterruptableContext() 28 | 29 | // disable datetime output 30 | log.SetFlags(0) 31 | 32 | app.Command("version", "Show version", func(cmd *cli.Cmd) { 33 | cmd.Action = func() { 34 | fmt.Printf("roots %s, commit %s, built at %s\n", version, commit, date) 35 | } 36 | }) 37 | 38 | app.Command("digest", "Show the latest digest", func(cmd *cli.Cmd) { 39 | cmd.Spec = "CONTAINER [--auth] [--arch] [--os]" 40 | 41 | var ( 42 | url = newURLArg(cmd) 43 | auth = newAuthOpt(cmd) 44 | arch = newArchOpt(cmd) 45 | ops = newOSOpt(cmd) 46 | ) 47 | 48 | cmd.Action = func() { 49 | digest, err := newRemote(ctx, url, auth, arch, ops).Digest() 50 | 51 | if err != nil { 52 | log.Fatal(err) 53 | } 54 | 55 | fmt.Println(digest) 56 | } 57 | }) 58 | 59 | app.Command("purge", "Purge unused files from the cache", func(cmd *cli.Cmd) { 60 | cmd.Spec = "[--cache]" 61 | 62 | var ( 63 | cache = newCacheOpt(cmd) 64 | ) 65 | 66 | cmd.Action = func() { 67 | // setup the cache 68 | if *cache == "" { 69 | *cache = os.Getenv("ROOTS_CACHE") 70 | } 71 | 72 | if *cache == "" { 73 | *cache = defaultCache() 74 | } 75 | 76 | entries, err := os.ReadDir(*cache) 77 | if err != nil { 78 | log.Fatalf("error accessing %s: %v", *cache, err) 79 | } 80 | 81 | if len(entries) == 0 { 82 | log.Fatalf("not a cache directory: %s", *cache) 83 | } 84 | 85 | valid := false 86 | for _, info := range entries { 87 | if info.Name() == "layers" { 88 | valid = true 89 | break 90 | } 91 | } 92 | 93 | if !valid { 94 | log.Fatalf("not a cache directory: %s", *cache) 95 | } 96 | 97 | store, err := image.NewStore(*cache) 98 | if err != nil { 99 | log.Fatalf("could not create store at %s: %v", *cache, err) 100 | } 101 | 102 | if err := store.Purge(); err != nil { 103 | log.Fatalf("error during purge of %s: %v", *cache, err) 104 | } 105 | } 106 | }) 107 | 108 | app.Command("pull", "Download and extract", func(cmd *cli.Cmd) { 109 | cmd.Spec = "CONTAINER DEST [--auth] [--arch] [--os] [--cache] [--force]" 110 | 111 | var ( 112 | url = newURLArg(cmd) 113 | dest = newDestArg(cmd) 114 | auth = newAuthOpt(cmd) 115 | arch = newArchOpt(cmd) 116 | ops = newOSOpt(cmd) 117 | cache = newCacheOpt(cmd) 118 | force = newForceOpt(cmd) 119 | ) 120 | 121 | cmd.Action = func() { 122 | 123 | // setup the cache 124 | if *cache == "" { 125 | *cache = os.Getenv("ROOTS_CACHE") 126 | } 127 | 128 | if strings.ToLower(*cache) == "no" { 129 | temp, err := os.MkdirTemp("", "store") 130 | if err != nil { 131 | log.Fatal(err) 132 | } 133 | defer os.RemoveAll(temp) 134 | 135 | *cache = temp 136 | } 137 | 138 | if *cache == "" { 139 | *cache = defaultCache() 140 | } 141 | 142 | if err := os.MkdirAll(*cache, 0755); err != nil { 143 | log.Fatalf("could not create cache at %s: %v", *cache, err) 144 | } 145 | 146 | store, err := image.NewStore(*cache) 147 | if err != nil { 148 | log.Fatalf("could not create store at %s: %v", *cache, err) 149 | } 150 | 151 | // create the destination 152 | if *force { 153 | 154 | // let's not be responsible for wiping out an actual root fs 155 | if strings.Count(*dest, "/") <= 2 { 156 | log.Fatalf("not enough path separators to force-remove: %s", *dest) 157 | } 158 | 159 | if err := os.RemoveAll(*dest); err != nil { 160 | log.Fatalf("could note force-remove %s: %v", *dest, err) 161 | } 162 | 163 | } 164 | 165 | if err := os.MkdirAll(*dest, 0755); err != nil { 166 | log.Fatalf("could not create destination at %s: %v", *dest, err) 167 | } 168 | 169 | // pull & extract the image 170 | remote := newRemote(ctx, url, auth, arch, ops) 171 | 172 | if err := store.Extract(ctx, remote, *dest); err != nil { 173 | log.Fatalf("error during pull: %v", err) 174 | } 175 | } 176 | }) 177 | 178 | err := app.Run(os.Args) 179 | if err != nil { 180 | log.Fatalf("error running command: %v", err) 181 | } 182 | } 183 | 184 | func defaultCache() string { 185 | usr, err := user.Current() 186 | 187 | if err != nil { 188 | log.Fatalf("error looking up current user: %v", err) 189 | } 190 | 191 | if usr.Uid == "0" || usr.HomeDir == "" { 192 | return "/var/cache/roots" 193 | } 194 | 195 | return path.Join(usr.HomeDir, ".cache", "seantis", "roots") 196 | } 197 | 198 | func newInterruptableContext() context.Context { 199 | ctx, cancel := context.WithCancel(context.Background()) 200 | 201 | c := make(chan os.Signal, 1) 202 | signal.Notify(c, os.Interrupt) 203 | go func() { 204 | <-c 205 | signal.Stop(c) 206 | cancel() 207 | }() 208 | 209 | return ctx 210 | } 211 | 212 | func newRemote(ctx context.Context, urlstring, auth, arch, ops *string) *image.Remote { 213 | 214 | if *auth == "" { 215 | *auth = os.Getenv("ROOTS_AUTH") 216 | } 217 | 218 | if *arch == "" { 219 | *arch = os.Getenv("ROOTS_ARCH") 220 | } 221 | 222 | if *ops == "" { 223 | *ops = os.Getenv("ROOTS_OS") 224 | } 225 | 226 | url, err := image.Parse(*urlstring) 227 | if err != nil { 228 | log.Fatalf("failed to parse image url %s: %v", *urlstring, err) 229 | } 230 | 231 | remote, err := image.NewRemote(ctx, *url, *auth) 232 | if err != nil { 233 | log.Fatalf("failed to connect to %s: %v", *urlstring, err) 234 | } 235 | 236 | if len(*arch) > 0 || len(*ops) > 0 { 237 | if len(*arch) == 0 { 238 | *arch = runtime.GOARCH 239 | } 240 | 241 | if len(*ops) == 0 { 242 | *ops = "linux" 243 | } 244 | 245 | remote.WithPlatform(&image.Platform{ 246 | Architecture: *arch, 247 | OS: *ops, 248 | }) 249 | } 250 | 251 | return remote 252 | } 253 | 254 | func newURLArg(cmd *cli.Cmd) *string { 255 | return cmd.StringArg("CONTAINER", "", 256 | `The url of the container, example values: 257 | 258 | - ubuntu:latest 259 | - gcr.io/google-containers/etcd:3.3.10 260 | `) 261 | } 262 | 263 | func newDestArg(cmd *cli.Cmd) *string { 264 | return cmd.StringArg("DEST", "", "The destination folder") 265 | } 266 | 267 | func newAuthOpt(cmd *cli.Cmd) *string { 268 | return cmd.StringOpt("auth", "", 269 | `Authentication for the following providers: 270 | 271 | * Google Container Registry: 272 | Path to service worker json file, with the following scope: 273 | 274 | 275 | This value can also be set through the env var ROOTS_AUTH, 276 | though the flag takes precedence. 277 | `) 278 | } 279 | 280 | func newArchOpt(cmd *cli.Cmd) *string { 281 | return cmd.StringOpt("arch", "", 282 | `Force the given architecture, example values: 283 | 284 | * amd64 285 | * arm 286 | 287 | See https://github.com/golang/go/blob/master/src/go/build/syslist.go 288 | 289 | Requires multi-arch support by the container. 290 | 291 | This value can also be set through the env var ROOTS_ARCH, 292 | though the flag takes precedence. 293 | `) 294 | } 295 | 296 | func newOSOpt(cmd *cli.Cmd) *string { 297 | return cmd.StringOpt("os", "", 298 | `Force the given OS, example values: 299 | 300 | * linux 301 | * windows 302 | 303 | See https://github.com/golang/go/blob/master/src/go/build/syslist.go 304 | 305 | Requires multi-arch support by the container. 306 | 307 | This value can also be set through the env var ROOTS_OS, 308 | though the flag takes precedence. 309 | `) 310 | } 311 | 312 | func newCacheOpt(cmd *cli.Cmd) *string { 313 | return cmd.StringOpt("cache", "", 314 | `Sets the cache folder that should be used. Defaults: 315 | 316 | * For non-root users: 317 | ~/.cache/seantis/roots 318 | 319 | * For root users: 320 | /var/cache/roots 321 | 322 | If the special value 'no' is given, a temporary folder will 323 | be used during the lifetime of the process. 324 | 325 | This value can also be set through the env var ROOTS_CACHE, 326 | though the flag takes precedence. 327 | `) 328 | } 329 | 330 | func newForceOpt(cmd *cli.Cmd) *bool { 331 | return cmd.BoolOpt("force", false, `Remove the destination before pulling 332 | 333 | Note that this only works if there are at least two path 334 | separatores in the destination. So you can force remove 335 | /var/roots/ubuntu, but not / or /var/lib. 336 | `) 337 | } 338 | --------------------------------------------------------------------------------