├── .github ├── dependabot.yml └── workflows │ ├── build.yml │ └── release.yml ├── .gitignore ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── config.yaml ├── docker_auth.go ├── docker_auth_darwin.go ├── duration.go ├── ecr_private_manager.go ├── ecr_public_manager.go ├── go.mod ├── go.sum ├── main.go ├── mirror.go └── mirror_test.go /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: gomod 4 | directory: "/" 5 | schedule: 6 | interval: daily 7 | time: "10:00" 8 | open-pull-requests-limit: 10 9 | ignore: 10 | - dependency-name: github.com/aws/aws-sdk-go-v2/config 11 | versions: 12 | - 1.1.2 13 | - 1.1.3 14 | - 1.1.4 15 | - 1.1.5 16 | - dependency-name: github.com/aws/aws-sdk-go-v2/service/ecr 17 | versions: 18 | - 1.1.2 19 | - 1.2.0 20 | - 1.2.1 21 | - dependency-name: github.com/fsouza/go-dockerclient 22 | versions: 23 | - 1.7.0 24 | - 1.7.1 25 | - dependency-name: github.com/sirupsen/logrus 26 | versions: 27 | - 1.7.0 28 | - 1.7.1 29 | - 1.8.0 30 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: build 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | build: 7 | runs-on: ubuntu-20.04 8 | steps: 9 | - name: Setup go 10 | uses: actions/setup-go@v1 11 | with: 12 | go-version: 1.15 13 | - name: Checkout repository 14 | uses: actions/checkout@v2 15 | - name: Build 16 | run: make -j dist 17 | - name: Test 18 | run: go test -timeout=600s -v 19 | - name: List output 20 | run: ls -la build 21 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: release 2 | 3 | on: 4 | release: 5 | types: [published] 6 | 7 | jobs: 8 | release: 9 | runs-on: ubuntu-20.04 10 | steps: 11 | - name: Setup go 12 | uses: actions/setup-go@v1 13 | with: 14 | go-version: 1.15 15 | - name: Checkout repository 16 | uses: actions/checkout@v2 17 | - name: Build 18 | run: make -j dist 19 | - name: Test 20 | run: go test -timeout=600s -v 21 | - name: Upload release assets 22 | uses: svenstaro/upload-release-action@v2 23 | with: 24 | repo_token: ${{ secrets.GITHUB_TOKEN }} 25 | file: build/docker-mirror-linux-amd64 26 | tag: ${{ github.ref }} 27 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | vendor/ 2 | build/ 3 | coverage.txt 4 | codecov.bash 5 | cover/ 6 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.15-alpine 2 | # Adding ca-certificates for external communication and git for dependency installation 3 | RUN apk add --no-cache ca-certificates git 4 | WORKDIR /go/src/github.com/seatgeek/docker-mirror/ 5 | COPY . /go/src/github.com/seatgeek/docker-mirror/ 6 | RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o build/docker-mirror . 7 | 8 | FROM alpine:latest 9 | RUN apk add --no-cache ca-certificates 10 | COPY --from=0 /go/src/github.com/seatgeek/docker-mirror/build/docker-mirror /usr/local/bin/ 11 | CMD ["docker-mirror"] 12 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2017, SeatGeek 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | * Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | * Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | * Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # build config 2 | BUILD_DIR ?= $(abspath build) 3 | GET_GOARCH = $(word 2,$(subst -, ,$1)) 4 | GET_GOOS = $(word 1,$(subst -, ,$1)) 5 | GOBUILD ?= $(shell go env GOOS)-$(shell go env GOARCH) 6 | GOFILES_NOCACHE = $(shell find . -type f -name '*.go' -not -path "./cache/*") 7 | VETARGS? =-all 8 | 9 | $(BUILD_DIR): 10 | mkdir -p $@ 11 | 12 | .PHONY: build 13 | build: 14 | go install 15 | 16 | .PHONY: fmt 17 | fmt: 18 | @echo "=> Running go fmt" ; 19 | @if [ -n "`go fmt ${GOFILES_NOCACHE}`" ]; then \ 20 | echo "[ERR] go fmt updated formatting. Please commit formatted code first."; \ 21 | exit 1; \ 22 | fi 23 | 24 | .PHONY: vet 25 | vet: fmt 26 | @echo "=> Running go vet $(VETARGS) ${GOFILES_NOCACHE}" 27 | @go vet $(VETARGS) ${GOFILES_NOCACHE} ; if [ $$? -eq 1 ]; then \ 28 | echo ""; \ 29 | echo "[LINT] Vet found suspicious constructs. Please check the reported constructs"; \ 30 | echo "and fix them if necessary before submitting the code for review."; \ 31 | fi 32 | 33 | BINARIES = $(addprefix $(BUILD_DIR)/docker-mirror-, $(GOBUILD)) 34 | $(BINARIES): $(BUILD_DIR)/docker-mirror-%: $(BUILD_DIR) 35 | @echo "=> building $@ ..." 36 | GOOS=$(call GET_GOOS,$*) GOARCH=$(call GET_GOARCH,$*) CGO_ENABLED=0 go build -o $@ 37 | 38 | .PHONY: dist 39 | dist: fmt vet 40 | @echo "=> building ..." 41 | $(MAKE) -j $(BINARIES) 42 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # docker-mirror 2 | 3 | [![build](https://github.com/seatgeek/docker-mirror/actions/workflows/build.yml/badge.svg)](https://github.com/seatgeek/docker-mirror/actions/workflows/build.yml) 4 | 5 | This project will copy public DockerHub, Quay or GCR repositories to a private registry. 6 | 7 | 8 | 9 | - [docker-mirror](#docker-mirror) 10 | - [Install / Building](#install--building) 11 | - [Using](#using) 12 | - [Adding new mirror repository](#adding-new-mirror-repository) 13 | - [Updating / resync an existing repository](#updating--resync-an-existing-repository) 14 | - [Update all repositories](#update-all-repositories) 15 | - [Example config.yaml](#example-configyaml) 16 | - [Environment Variables](#environment-variables) 17 | 18 | 19 | 20 | ## Install / Building 21 | 22 | - make sure you got Go 1.15 or newer 23 | - OSX: `brew install go` 24 | - make sure you have `CGO` enabled 25 | - `export CGO_ENABLED=1` 26 | - clone this repository to `$HOME/src/github.com/seatgeek/docker-mirror` 27 | - change your working directory to `$HOME/go/src/github.com/seatgeek/docker-mirror` 28 | - run `go install` to build and install the `docker-mirror` binary into your `$HOME/go/bin/` directory 29 | - alternative: `go build` to build the binary and put it in the current working directory 30 | 31 | ## Using 32 | 33 | Make sure that your local Docker agent is logged into to `ECR`. 34 | - To login to ECR private registries: \ 35 | `aws ecr get-login-password --region us-east-1 | docker login -u AWS --password-stdin ACCOUNT_ID.dkr.REGION.amazonaws.com` 36 | - To login to ECR public registries: \ 37 | `aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/YOUR_ECR_PUBLIC_SUFFIX` \ 38 | Note that the region must be `us-east-1` for ECR public registry authentication. 39 | 40 | _See [AWS ECR documentation](https://docs.aws.amazon.com/ecr/index.html) for more details_ 41 | 42 | `docker-mirror` will automatically create the ECR repository on demand, so you do not need to login and do any UI operations in the AWS Console. 43 | 44 | `docker-mirror` will look for your AWS credentials in all the default locations (`env`, `~/.aws/` and so forth like normal AWS tools do) 45 | 46 | ### Configuration File 47 | 48 | There are several configuration options you can use in your `config.yaml` below. Please see the `config.yaml` file in the repository for a full example. 49 | 50 | - `ignore_tag:` This option sets tags that can be ignored on pulls. (i.e. `ignore_tag: - "*-alpine"`) 51 | 52 | - `match_tag:` This option sets the tags that you want to match on for pulls. (i.e. `match_tag: - "3*"`) 53 | 54 | - `max_tag_age:` This option sets the max tag age you wish to pull from. (i.e. `max_tag_age: 4w`) 55 | 56 | - `name:` This option sets the name of your repository. (i.e. `name: elasticsearch`) 57 | 58 | - `host:` This options sets where do you want to mirror repositories from. Accepted values include `hub.docker.com`, `quay.io` and `gcr.io`. If not set, images will be pulled from Docker Hub. 59 | 60 | - `private_registry:` This option allows you to set a private Docker registry prefix for docker pulls. It will prefix any of your `name:` options with the `private_registry` name and a slash to allow you to customize where your images are being pulled through. This is particularly useful if you use a proxy to dockerhub. i.e. (`private_registry: "private-registry-name"`) 61 | 62 | ### Adding new mirror repository 63 | 64 | - add the new repository to the `config.yaml` file 65 | - TIP: omit the `max_tag_age` for the initial sync to mirror all historic tags (`match_tag` is fine to use in all cases) 66 | - run `PREFIX=${reopsitory_name} docker-mirror` to trigger a sync for the specific new repository (you probably don't want to sync all the existing repositories) 67 | - add the `max_tag_age` filter to the newly added repository so future syns won't cosider all historic tags 68 | 69 | ### Updating / resync an existing repository 70 | 71 | - run `PREFIX=${reopsitory_name} docker-mirror` to trigger a sync for the specific repository 72 | - TIP: Consider if the tags you want to sync fits within the `max_tag_age` and other filters 73 | 74 | ### Update all repositories 75 | 76 | - run `docker-mirror` and wait (for a while) 77 | 78 | ## Example config.yaml 79 | 80 | ```yml 81 | --- 82 | cleanup: true # (optional) Clean the mirrored images (default: false) 83 | target: 84 | # where to copy images to 85 | # Below is an example of the ECR private registry. 86 | # To mirror repositories to a ECR public registry, replace this value with public.ecr.aws/YOUR_ECR_PUBLIC_ALIAS 87 | registry: ACCOUNT_ID.dkr.REGION.amazonaws.com 88 | 89 | # (optional) prefix all repositories with this name 90 | # ACCOUNT_ID.dkr.REGION.amazonaws.com/hub/jippi/hashi-ui 91 | prefix: "hub/" 92 | 93 | # what repositories to copy 94 | repositories: 95 | # will automatically know it's a "library" repository in dockerhub 96 | - name: elasticsearch 97 | match_tag: # tags to match, can be specific or glob pattern 98 | - "5.6.8" # specific tag match 99 | - "6.*" # glob patterns will match 100 | ignore_tag: # tags to never match on (even if its matched by `tag`) 101 | - "*-alpine" # support both glob or specific strings 102 | 103 | - name: yotpo/resec 104 | host: hub.docker.com # mirror the repository from Docker Hub 105 | max_tag_age: 8w # only import tags that are 8w or less old 106 | 107 | - name: jippi/hashi-ui 108 | max_tags: 10 # only copy the 10 latest tags 109 | match_tag: 110 | - "v*" 111 | 112 | - name: kubebuilder/kube-rbac-proxy 113 | host: gcr.io # mirror the repository from Google Container Registry 114 | 115 | - name: jippi/go-metadataproxy # import all tags 116 | ``` 117 | 118 | ## Environment Variables 119 | 120 | Environment Variable | Default | Description 121 | ----------------------| ---------------| ------------------------------------------------- 122 | CONFIG_FILE | config.yaml | config file to use 123 | DOCKERHUB_USER | unset | optional user to authenticate to docker hub with 124 | DOCKERHUB_PASSWORD | unset | optional password to authenticate to docker hub with 125 | LOG_LEVEL | unset | optional control the log level output 126 | PREFIX | unset | optional only mirror images that match the defined prefix 127 | -------------------------------------------------------------------------------- /config.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | target: 3 | registry: ACCOUNT_ID.dkr.REGION.amazonaws.com 4 | prefix: "hub/" 5 | 6 | repositories: 7 | - private_registry: "private-registry-name" 8 | name: elasticsearch 9 | max_tag_age: 4w 10 | ignore_tag: 11 | - "*-alpine" 12 | 13 | - name: redis 14 | max_tag_age: 4w 15 | match_tag: 16 | - "3*" 17 | - "4*" 18 | - "latest" 19 | ignore_tag: 20 | - "*32bit*" 21 | - "*alpine*" 22 | - "*nanoserver*" 23 | - "*windowsservercore*" 24 | 25 | - name: yotpo/resec 26 | max_tag_age: 4w 27 | 28 | - name: jippi/hashi-ui 29 | max_tag_age: 4w 30 | match_tag: 31 | - "v*" 32 | 33 | - name: kubebuilder/kube-rbac-proxy 34 | host: gcr.io 35 | match_tag: 36 | - "v*" 37 | -------------------------------------------------------------------------------- /docker_auth.go: -------------------------------------------------------------------------------- 1 | // +build !darwin 2 | 3 | package main 4 | 5 | import ( 6 | "fmt" 7 | 8 | "github.com/fsouza/go-dockerclient" 9 | log "github.com/sirupsen/logrus" 10 | ) 11 | 12 | func getDockerCredentials(registry string) (*docker.AuthConfiguration, error) { 13 | authOptions, err := docker.NewAuthConfigurationsFromDockerCfg() 14 | if err != nil { 15 | log.Fatal(err) 16 | } 17 | 18 | creds, ok := authOptions.Configs[registry] 19 | if !ok { 20 | return nil, fmt.Errorf("No auth found for %s", registry) 21 | } 22 | 23 | return &creds, nil 24 | } 25 | -------------------------------------------------------------------------------- /docker_auth_darwin.go: -------------------------------------------------------------------------------- 1 | // +build darwin 2 | 3 | package main 4 | 5 | import ( 6 | "github.com/docker/docker-credential-helpers/osxkeychain" 7 | "github.com/fsouza/go-dockerclient" 8 | ) 9 | 10 | var ( 11 | keychain = osxkeychain.Osxkeychain{} 12 | ) 13 | 14 | func getDockerCredentials(registry string) (*docker.AuthConfiguration, error) { 15 | user, pass, err := keychain.Get(registry) 16 | if err != nil { 17 | return nil, err 18 | } 19 | 20 | return &docker.AuthConfiguration{ 21 | Username: user, 22 | Password: pass, 23 | }, nil 24 | } 25 | -------------------------------------------------------------------------------- /duration.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "regexp" 6 | "strconv" 7 | "time" 8 | ) 9 | 10 | // Duration with support for periodes larger than "hours" 11 | type Duration time.Duration 12 | 13 | var durationRE = regexp.MustCompile("^([0-9]+)(y|w|d|h|m|s|ms)$") 14 | 15 | // ParseDuration parses a string into a time.Duration, assuming that a year 16 | // always has 365d, a week always has 7d, and a day always has 24h. 17 | func ParseDuration(durationStr string) (Duration, error) { 18 | matches := durationRE.FindStringSubmatch(durationStr) 19 | if len(matches) != 3 { 20 | return 0, fmt.Errorf("not a valid duration string: %q", durationStr) 21 | } 22 | var ( 23 | n, _ = strconv.Atoi(matches[1]) 24 | dur = time.Duration(n) * time.Millisecond 25 | ) 26 | switch unit := matches[2]; unit { 27 | case "y": 28 | dur *= 1000 * 60 * 60 * 24 * 365 29 | case "w": 30 | dur *= 1000 * 60 * 60 * 24 * 7 31 | case "d": 32 | dur *= 1000 * 60 * 60 * 24 33 | case "h": 34 | dur *= 1000 * 60 * 60 35 | case "m": 36 | dur *= 1000 * 60 37 | case "s": 38 | dur *= 1000 39 | case "ms": 40 | // Value already correct 41 | default: 42 | return 0, fmt.Errorf("invalid time unit in duration string: %q", unit) 43 | } 44 | return Duration(dur), nil 45 | } 46 | 47 | func (d Duration) String() string { 48 | var ( 49 | ms = int64(time.Duration(d) / time.Millisecond) 50 | unit = "ms" 51 | ) 52 | if ms == 0 { 53 | return "0s" 54 | } 55 | factors := map[string]int64{ 56 | "y": 1000 * 60 * 60 * 24 * 365, 57 | "w": 1000 * 60 * 60 * 24 * 7, 58 | "d": 1000 * 60 * 60 * 24, 59 | "h": 1000 * 60 * 60, 60 | "m": 1000 * 60, 61 | "s": 1000, 62 | "ms": 1, 63 | } 64 | 65 | switch int64(0) { 66 | case ms % factors["y"]: 67 | unit = "y" 68 | case ms % factors["w"]: 69 | unit = "w" 70 | case ms % factors["d"]: 71 | unit = "d" 72 | case ms % factors["h"]: 73 | unit = "h" 74 | case ms % factors["m"]: 75 | unit = "m" 76 | case ms % factors["s"]: 77 | unit = "s" 78 | } 79 | return fmt.Sprintf("%v%v", ms/factors[unit], unit) 80 | } 81 | 82 | // MarshalYAML implements the yaml.Marshaler interface. 83 | func (d Duration) MarshalYAML() (interface{}, error) { 84 | return d.String(), nil 85 | } 86 | 87 | // UnmarshalYAML implements the yaml.Unmarshaler interface. 88 | func (d *Duration) UnmarshalYAML(unmarshal func(interface{}) error) error { 89 | var s string 90 | if err := unmarshal(&s); err != nil { 91 | return err 92 | } 93 | dur, err := ParseDuration(s) 94 | if err != nil { 95 | return err 96 | } 97 | *d = dur 98 | return nil 99 | } 100 | -------------------------------------------------------------------------------- /ecr_private_manager.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/aws/aws-sdk-go-v2/service/ecr" 7 | "github.com/cenkalti/backoff" 8 | log "github.com/sirupsen/logrus" 9 | ) 10 | 11 | type ecrPrivateManager struct { 12 | client *ecr.Client // AWS ECR client 13 | repositories map[string]bool // list of repositories in ECR 14 | } 15 | 16 | func (e *ecrPrivateManager) exists(name string) bool { 17 | if _, ok := e.repositories[name]; ok { 18 | return true 19 | } 20 | 21 | return false 22 | } 23 | 24 | func (e *ecrPrivateManager) ensure(name string) error { 25 | if e.exists(name) { 26 | return nil 27 | } 28 | 29 | return e.create(name) 30 | } 31 | 32 | func (e *ecrPrivateManager) create(name string) error { 33 | _, err := e.client.CreateRepository(context.TODO(), &ecr.CreateRepositoryInput{ 34 | RepositoryName: &name, 35 | }) 36 | 37 | if err != nil { 38 | return err 39 | } 40 | 41 | e.repositories[name] = true 42 | return nil 43 | } 44 | 45 | func (e *ecrPrivateManager) buildCache(nextToken *string) error { 46 | if nextToken == nil { 47 | log.Info("Loading list of ECR repositories") 48 | } 49 | 50 | resp, err := e.client.DescribeRepositories(context.TODO(), &ecr.DescribeRepositoriesInput{ 51 | NextToken: nextToken, 52 | }) 53 | if err != nil { 54 | return err 55 | } 56 | 57 | if e.repositories == nil { 58 | e.repositories = make(map[string]bool) 59 | } 60 | 61 | for _, repo := range resp.Repositories { 62 | e.repositories[*repo.RepositoryName] = true 63 | } 64 | 65 | // keep paging as long as there is a token for the next page 66 | if resp.NextToken != nil { 67 | e.buildCache(resp.NextToken) 68 | } 69 | 70 | // no next token means we hit the last page 71 | if nextToken == nil { 72 | log.Info("Done loading ECR repositories") 73 | } 74 | 75 | return nil 76 | } 77 | 78 | func (e *ecrPrivateManager) buildCacheBackoff() backoff.Operation { 79 | return func() error { 80 | return e.buildCache(nil) 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /ecr_public_manager.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/aws/aws-sdk-go-v2/service/ecrpublic" 7 | "github.com/cenkalti/backoff" 8 | log "github.com/sirupsen/logrus" 9 | ) 10 | 11 | type ecrPublicManager struct { 12 | client *ecrpublic.Client // AWS public ECR client 13 | repositories map[string]bool // list of repositories in public ECR 14 | } 15 | 16 | func (e *ecrPublicManager) exists(name string) bool { 17 | if _, ok := e.repositories[name]; ok { 18 | return true 19 | } 20 | 21 | return false 22 | } 23 | 24 | func (e *ecrPublicManager) ensure(name string) error { 25 | if e.exists(name) { 26 | return nil 27 | } 28 | 29 | return e.create(name) 30 | } 31 | 32 | func (e *ecrPublicManager) create(name string) error { 33 | _, err := e.client.CreateRepository(context.TODO(), &ecrpublic.CreateRepositoryInput{ 34 | RepositoryName: &name, 35 | }) 36 | if err != nil { 37 | return err 38 | } 39 | 40 | e.repositories[name] = true 41 | return nil 42 | } 43 | 44 | func (e *ecrPublicManager) buildCache(nextToken *string) error { 45 | if nextToken == nil { 46 | log.Info("Loading the list of ECR public repositories") 47 | } 48 | 49 | resp, err := e.client.DescribeRepositories(context.TODO(), &ecrpublic.DescribeRepositoriesInput{ 50 | NextToken: nextToken, 51 | }) 52 | if err != nil { 53 | return err 54 | } 55 | 56 | if e.repositories == nil { 57 | e.repositories = make(map[string]bool) 58 | } 59 | 60 | for _, repo := range resp.Repositories { 61 | e.repositories[*repo.RepositoryName] = true 62 | } 63 | 64 | // keep paging as long as there is a token for the next page 65 | if resp.NextToken != nil { 66 | err := e.buildCache(resp.NextToken) 67 | if err != nil { 68 | return err 69 | } 70 | } 71 | 72 | // no next token means we hit the last page 73 | if nextToken == nil { 74 | log.Info("Done loading ECR public repositories") 75 | } 76 | 77 | return nil 78 | } 79 | 80 | func (e *ecrPublicManager) buildCacheBackoff() backoff.Operation { 81 | return func() error { 82 | return e.buildCache(nil) 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/seatgeek/docker-mirror 2 | 3 | go 1.17 4 | 5 | require ( 6 | github.com/aws/aws-sdk-go-v2/config v1.1.1 7 | github.com/aws/aws-sdk-go-v2/service/ecr v1.1.1 8 | github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.13.3 9 | github.com/cenkalti/backoff v2.2.1+incompatible 10 | github.com/docker/docker-credential-helpers v0.6.4 11 | github.com/fsouza/go-dockerclient v1.6.6 12 | github.com/google/go-github v17.0.0+incompatible 13 | github.com/ryanuber/go-glob v0.0.0-20160226084822-572520ed46db 14 | github.com/sirupsen/logrus v1.6.0 15 | gopkg.in/yaml.v2 v2.4.0 16 | ) 17 | 18 | require ( 19 | github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 // indirect 20 | github.com/Microsoft/go-winio v0.4.15-0.20200113171025-3fe6c5262873 // indirect 21 | github.com/Microsoft/hcsshim v0.8.9 // indirect 22 | github.com/aws/aws-sdk-go-v2 v1.16.2 // indirect 23 | github.com/aws/aws-sdk-go-v2/credentials v1.1.1 // indirect 24 | github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.0.2 // indirect 25 | github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.9 // indirect 26 | github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.3 // indirect 27 | github.com/aws/aws-sdk-go-v2/internal/ini v1.1.0 // indirect 28 | github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.0.2 // indirect 29 | github.com/aws/aws-sdk-go-v2/service/sso v1.1.1 // indirect 30 | github.com/aws/aws-sdk-go-v2/service/sts v1.1.1 // indirect 31 | github.com/aws/smithy-go v1.11.2 // indirect 32 | github.com/containerd/containerd v1.3.4 // indirect 33 | github.com/containerd/continuity v0.0.0-20200413184840-d3ef23f19fbb // indirect 34 | github.com/docker/distribution v2.7.1+incompatible // indirect 35 | github.com/docker/docker v17.12.0-ce-rc1.0.20200505174321-1655290016ac+incompatible // indirect 36 | github.com/docker/go-connections v0.4.0 // indirect 37 | github.com/docker/go-units v0.4.0 // indirect 38 | github.com/gogo/protobuf v1.3.1 // indirect 39 | github.com/golang/protobuf v1.3.3 // indirect 40 | github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135 // indirect 41 | github.com/jmespath/go-jmespath v0.4.0 // indirect 42 | github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect 43 | github.com/moby/sys/mount v0.1.0 // indirect 44 | github.com/moby/sys/mountinfo v0.1.0 // indirect 45 | github.com/moby/term v0.0.0-20200429084858-129dac9f73f6 // indirect 46 | github.com/morikuni/aec v1.0.0 // indirect 47 | github.com/opencontainers/go-digest v1.0.0-rc1 // indirect 48 | github.com/opencontainers/image-spec v1.0.1 // indirect 49 | github.com/opencontainers/runc v0.1.1 // indirect 50 | github.com/pkg/errors v0.9.1 // indirect 51 | github.com/stretchr/testify v1.6.1 // indirect 52 | golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7 // indirect 53 | golang.org/x/sync v0.0.0-20190423024810-112230192c58 // indirect 54 | golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4 // indirect 55 | google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 // indirect 56 | google.golang.org/grpc v1.29.1 // indirect 57 | ) 58 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | bazil.org/fuse v0.0.0-20160811212531-371fbbdaa898/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8= 2 | cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 3 | github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8= 4 | github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= 5 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 6 | github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw= 7 | github.com/Microsoft/go-winio v0.4.15-0.20200113171025-3fe6c5262873 h1:93nQ7k53GjoMQ07HVP8g6Zj1fQZDDj7Xy2VkNNtvX8o= 8 | github.com/Microsoft/go-winio v0.4.15-0.20200113171025-3fe6c5262873/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw= 9 | github.com/Microsoft/hcsshim v0.8.9 h1:VrfodqvztU8YSOvygU+DN1BGaSGxmrNfqOv5oOuX2Bk= 10 | github.com/Microsoft/hcsshim v0.8.9/go.mod h1:5692vkUqntj1idxauYlpoINNKeqCiG6Sg38RRsjT5y8= 11 | github.com/aws/aws-sdk-go-v2 v1.2.0/go.mod h1:zEQs02YRBw1DjK0PoJv3ygDYOFTre1ejlJWl8FwAuQo= 12 | github.com/aws/aws-sdk-go-v2 v1.7.0/go.mod h1:tb9wi5s61kTDA5qCkcDbt3KRVV74GGslQkl/DRdX/P4= 13 | github.com/aws/aws-sdk-go-v2 v1.16.2 h1:fqlCk6Iy3bnCumtrLz9r3mJ/2gUT0pJ0wLFVIdWh+JA= 14 | github.com/aws/aws-sdk-go-v2 v1.16.2/go.mod h1:ytwTPBG6fXTZLxxeeCCWj2/EMYp/xDUgX+OET6TLNNU= 15 | github.com/aws/aws-sdk-go-v2/config v1.1.1 h1:ZAoq32boMzcaTW9bcUacBswAmHTbvlvDJICgHFZuECo= 16 | github.com/aws/aws-sdk-go-v2/config v1.1.1/go.mod h1:0XsVy9lBI/BCXm+2Tuvt39YmdHwS5unDQmxZOYe8F5Y= 17 | github.com/aws/aws-sdk-go-v2/credentials v1.1.1 h1:NbvWIM1Mx6sNPTxowHgS2ewXCRp+NGTzUYb/96FZJbY= 18 | github.com/aws/aws-sdk-go-v2/credentials v1.1.1/go.mod h1:mM2iIjwl7LULWtS6JCACyInboHirisUUdkBPoTHMOUo= 19 | github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.0.2 h1:EtEU7WRaWliitZh2nmuxEXrN0Cb8EgPUFGIoTMeqbzI= 20 | github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.0.2/go.mod h1:3hGg3PpiEjHnrkrlasTfxFqUsZ2GCk/fMUn4CbKgSkM= 21 | github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.9 h1:onz/VaaxZ7Z4V+WIN9Txly9XLTmoOh1oJ8XcAC3pako= 22 | github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.9/go.mod h1:AnVH5pvai0pAF4lXRq0bmhbes1u9R8wTE+g+183bZNM= 23 | github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.3 h1:9stUQR/u2KXU6HkFJYlqnZEjBnbgrVbG6I5HN09xZh0= 24 | github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.3/go.mod h1:ssOhaLpRlh88H3UmEcsBoVKq309quMvm3Ds8e9d4eJM= 25 | github.com/aws/aws-sdk-go-v2/internal/ini v1.1.0 h1:DJq/vXXF+LAFaa/kQX9C6arlf4xX4uaaqGWIyAKOCpM= 26 | github.com/aws/aws-sdk-go-v2/internal/ini v1.1.0/go.mod h1:qGQ/9IfkZonRNSNLE99/yBJ7EPA/h8jlWEqtJCcaj+Q= 27 | github.com/aws/aws-sdk-go-v2/service/ecr v1.1.1 h1:idXCsD7Rl3LtE/MFFw81a1C1tVRSP3AOnv96U0TsRUo= 28 | github.com/aws/aws-sdk-go-v2/service/ecr v1.1.1/go.mod h1:NGFCwbEd03lj5kwG8vO5qS5m4CfvHE4ir3pA5ozrlUM= 29 | github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.13.3 h1:2XpcXse156FZfnvnrzqTb8uwJuWUcT1ryiU7dZOzBYc= 30 | github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.13.3/go.mod h1:JojDs/ei43SWG9m059FtaOBJK607XPF5RuRJZ8NTWTk= 31 | github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.0.2 h1:4AH9fFjUlVktQMznF+YN33aWNXaR4VgDXyP28qokJC0= 32 | github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.0.2/go.mod h1:45MfaXZ0cNbeuT0KQ1XJylq8A6+OpVV2E5kvY/Kq+u8= 33 | github.com/aws/aws-sdk-go-v2/service/sso v1.1.1 h1:37QubsarExl5ZuCBlnRP+7l1tNwZPBSTqpTBrPH98RU= 34 | github.com/aws/aws-sdk-go-v2/service/sso v1.1.1/go.mod h1:SuZJxklHxLAXgLTc1iFXbEWkXs7QRTQpCLGaKIprQW0= 35 | github.com/aws/aws-sdk-go-v2/service/sts v1.1.1 h1:TJoIfnIFubCX0ACVeJ0w46HEH5MwjwYN4iFhuYIhfIY= 36 | github.com/aws/aws-sdk-go-v2/service/sts v1.1.1/go.mod h1:Wi0EBZwiz/K44YliU0EKxqTCJGUfYTWXrrBwkq736bM= 37 | github.com/aws/smithy-go v1.1.0/go.mod h1:EzMw8dbp/YJL4A5/sbhGddag+NPT7q084agLbB9LgIw= 38 | github.com/aws/smithy-go v1.5.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= 39 | github.com/aws/smithy-go v1.11.2 h1:eG/N+CcUMAvsdffgMvjMKwfyDzIkjM6pfxMJ8Mzc6mE= 40 | github.com/aws/smithy-go v1.11.2/go.mod h1:3xHYmszWVx2c0kIwQeEVf9uSm4fYZt67FBJnwub1bgM= 41 | github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= 42 | github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= 43 | github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= 44 | github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= 45 | github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= 46 | github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko= 47 | github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= 48 | github.com/containerd/containerd v1.3.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= 49 | github.com/containerd/containerd v1.3.4 h1:3o0smo5SKY7H6AJCmJhsnCjR2/V2T8VmiHt7seN2/kI= 50 | github.com/containerd/containerd v1.3.4/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= 51 | github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= 52 | github.com/containerd/continuity v0.0.0-20200413184840-d3ef23f19fbb h1:nXPkFq8X1a9ycY3GYQpFNxHh3j2JgY7zDZfq2EXMIzk= 53 | github.com/containerd/continuity v0.0.0-20200413184840-d3ef23f19fbb/go.mod h1:Dq467ZllaHgAtVp4p1xUQWBrFXR9s/wyoTpG8zOJGkY= 54 | github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= 55 | github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= 56 | github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= 57 | github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc= 58 | github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= 59 | github.com/creack/pty v1.1.9 h1:uDmaGzcdjhF4i/plgjmEsriH11Y0o7RKapEf/LDaM3w= 60 | github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= 61 | github.com/danieljoos/wincred v1.1.0/go.mod h1:XYlo+eRTsVA9aHGp7NGjFkPla4m+DCL7hqDjlFjiygg= 62 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 63 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 64 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 65 | github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= 66 | github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= 67 | github.com/docker/docker v17.12.0-ce-rc1.0.20200505174321-1655290016ac+incompatible h1:ZxJX4ZSNg1LORBsStUojbrLfkrE3Ut122XhzyZnN110= 68 | github.com/docker/docker v17.12.0-ce-rc1.0.20200505174321-1655290016ac+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= 69 | github.com/docker/docker-credential-helpers v0.6.4 h1:axCks+yV+2MR3/kZhAmy07yC56WZ2Pwu/fKWtKuZB0o= 70 | github.com/docker/docker-credential-helpers v0.6.4/go.mod h1:ofX3UI0Gz1TteYBjtgs07O36Pyasyp66D2uKT7H8W1c= 71 | github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= 72 | github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= 73 | github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= 74 | github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= 75 | github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= 76 | github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= 77 | github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= 78 | github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= 79 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= 80 | github.com/fsouza/go-dockerclient v1.6.6 h1:9e3xkBrVkPb81gzYq23i7iDUEd6sx2ooeJA/gnYU6R4= 81 | github.com/fsouza/go-dockerclient v1.6.6/go.mod h1:3/oRIWoe7uT6bwtAayj/EmJmepBjeL4pYvt7ZxC7Rnk= 82 | github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= 83 | github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= 84 | github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= 85 | github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= 86 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= 87 | github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 88 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 89 | github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 90 | github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 91 | github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I= 92 | github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= 93 | github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= 94 | github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 95 | github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 96 | github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 97 | github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 98 | github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 99 | github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o= 100 | github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= 101 | github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY= 102 | github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= 103 | github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135 h1:zLTLjkaOFEFIOxY5BWLFLwh+cL8vOBW4XJ2aqLE/Tf0= 104 | github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= 105 | github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= 106 | github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= 107 | github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= 108 | github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= 109 | github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= 110 | github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= 111 | github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= 112 | github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= 113 | github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= 114 | github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= 115 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 116 | github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 117 | github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= 118 | github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 119 | github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= 120 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 121 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 122 | github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= 123 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 124 | github.com/moby/sys/mount v0.1.0 h1:Ytx78EatgFKtrqZ0BvJ0UtJE472ZvawVmil6pIfuCCU= 125 | github.com/moby/sys/mount v0.1.0/go.mod h1:FVQFLDRWwyBjDTBNQXDlWnSFREqOo3OKX9aqhmeoo74= 126 | github.com/moby/sys/mountinfo v0.1.0 h1:r8vMRbMAFEAfiNptYVokP+nfxPJzvRuia5e2vzXtENo= 127 | github.com/moby/sys/mountinfo v0.1.0/go.mod h1:w2t2Avltqx8vE7gX5l+QiBKxODu2TX0+Syr3h52Tw4o= 128 | github.com/moby/term v0.0.0-20200429084858-129dac9f73f6 h1:3Y9aosU6S5Bo8GYH0s+t1ej4m30GuUKvQ3c9ZLqdL28= 129 | github.com/moby/term v0.0.0-20200429084858-129dac9f73f6/go.mod h1:or9wGItza1sRcM4Wd3dIv8DsFHYQuFsMHEdxUIlUxms= 130 | github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= 131 | github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= 132 | github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 133 | github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 134 | github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= 135 | github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= 136 | github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ= 137 | github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= 138 | github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI= 139 | github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= 140 | github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= 141 | github.com/opencontainers/runc v0.1.1 h1:GlxAyO6x8rfZYN9Tt0Kti5a/cP41iuiO2yYT0IJGY8Y= 142 | github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= 143 | github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= 144 | github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 145 | github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 146 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 147 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 148 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 149 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 150 | github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 151 | github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= 152 | github.com/ryanuber/go-glob v0.0.0-20160226084822-572520ed46db h1:ge9atzKq16843f793fDVxKUhmTb4H5muzjJQ6PgsnHg= 153 | github.com/ryanuber/go-glob v0.0.0-20160226084822-572520ed46db/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= 154 | github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= 155 | github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= 156 | github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= 157 | github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= 158 | github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= 159 | github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= 160 | github.com/spf13/pflag v1.0.1-0.20171106142849-4c012f6dcd95/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= 161 | github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= 162 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 163 | github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 164 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 165 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 166 | github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= 167 | github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= 168 | github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 169 | github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= 170 | go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= 171 | golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 172 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 173 | golang.org/x/crypto v0.0.0-20200429183012-4b2356b1ed79 h1:IaQbIIB2X/Mp/DKctl6ROxz1KyMlKp4uyvL6+kQ7C88= 174 | golang.org/x/crypto v0.0.0-20200429183012-4b2356b1ed79/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 175 | golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 176 | golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 177 | golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= 178 | golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 179 | golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 180 | golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 181 | golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 182 | golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 183 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 184 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 185 | golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 186 | golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 187 | golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7 h1:AeiKBIuRw3UomYXSbLy0Mc2dDLfdtbT/IVn4keq83P0= 188 | golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= 189 | golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= 190 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 191 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 192 | golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 193 | golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= 194 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 195 | golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 196 | golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 197 | golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 198 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 199 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 200 | golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 201 | golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 202 | golang.org/x/sys v0.0.0-20190514135907-3a4b5fb9f71f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 203 | golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 204 | golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 205 | golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 206 | golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 207 | golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4 h1:myAQVi0cGEoqQVR5POX+8RR2mrocKqNN1hmeMqhX27k= 208 | golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 209 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 210 | golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= 211 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 212 | golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 213 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 214 | golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 215 | golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 216 | golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= 217 | golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 218 | golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= 219 | golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= 220 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= 221 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 222 | google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= 223 | google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 224 | google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= 225 | google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 226 | google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 227 | google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE= 228 | google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= 229 | google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= 230 | google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= 231 | google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= 232 | google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= 233 | google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= 234 | google.golang.org/grpc v1.29.1 h1:EC2SB8S04d2r73uptxphDSUG+kTKVgjRPF+N3xpxRB4= 235 | google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= 236 | gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= 237 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 238 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= 239 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 240 | gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= 241 | gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= 242 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= 243 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 244 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 245 | gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 246 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= 247 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 248 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= 249 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 250 | gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= 251 | gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= 252 | gotest.tools/v3 v3.0.2 h1:kG1BFyqVHuQoVQiR1bWGnfz/fmHvvuiSPIV7rvl360E= 253 | gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= 254 | honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 255 | honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 256 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io/ioutil" 7 | "os" 8 | "runtime" 9 | "strconv" 10 | "strings" 11 | "sync" 12 | "time" 13 | 14 | awsconfig "github.com/aws/aws-sdk-go-v2/config" 15 | "github.com/aws/aws-sdk-go-v2/service/ecr" 16 | "github.com/aws/aws-sdk-go-v2/service/ecrpublic" 17 | "github.com/cenkalti/backoff" 18 | docker "github.com/fsouza/go-dockerclient" 19 | log "github.com/sirupsen/logrus" 20 | "gopkg.in/yaml.v2" 21 | ) 22 | 23 | const ( 24 | ecrPublicRegistryPrefix = "public.ecr.aws" 25 | ecrPublicRegion = "us-east-1" 26 | ) 27 | 28 | var ( 29 | config Config 30 | isPrivateECR bool 31 | ) 32 | 33 | // ecrManager is an interface which defines the methods ECR private or public managers should implement. 34 | type ecrManager interface { 35 | exists(name string) bool 36 | ensure(name string) error 37 | create(name string) error 38 | buildCache(nextToken *string) error 39 | buildCacheBackoff() backoff.Operation 40 | } 41 | 42 | // Config is the result of the parsed yaml file 43 | type Config struct { 44 | Cleanup bool `yaml:"cleanup"` 45 | Workers int `yaml:"workers"` 46 | Repositories []Repository `yaml:"repositories,flow"` 47 | Target TargetConfig `yaml:"target"` 48 | } 49 | 50 | // TargetConfig contains info on where to mirror repositories to 51 | type TargetConfig struct { 52 | Registry string `yaml:"registry"` 53 | Prefix string `yaml:"prefix"` 54 | } 55 | 56 | // Repository is a single docker hub repository to mirror 57 | type Repository struct { 58 | PrivateRegistry string `yaml:"private_registry"` 59 | Name string `yaml:"name"` 60 | MatchTags []string `yaml:"match_tag"` 61 | DropTags []string `yaml:"ignore_tag"` 62 | MaxTags int `yaml:"max_tags"` 63 | MaxTagAge *Duration `yaml:"max_tag_age"` 64 | RemoteTagSource string `yaml:"remote_tags_source"` 65 | RemoteTagConfig map[string]string `yaml:"remote_tags_config"` 66 | TargetPrefix *string `yaml:"target_prefix"` 67 | Host string `yaml:"host"` 68 | } 69 | 70 | func createDockerClient() (*docker.Client, error) { 71 | client, err := docker.NewClientFromEnv() 72 | return client, err 73 | } 74 | 75 | func main() { 76 | // log level 77 | if rawLevel := os.Getenv("LOG_LEVEL"); rawLevel != "" { 78 | logLevel, err := log.ParseLevel(rawLevel) 79 | if err != nil { 80 | log.Fatal(err) 81 | } 82 | log.SetLevel(logLevel) 83 | } 84 | 85 | // mirror file to read 86 | configFile := "config.yaml" 87 | if f := os.Getenv("CONFIG_FILE"); f != "" { 88 | configFile = f 89 | } 90 | 91 | content, err := ioutil.ReadFile(configFile) 92 | if err != nil { 93 | log.Fatal(fmt.Sprintf("Could not read config file: %s", err)) 94 | } 95 | 96 | if err := yaml.Unmarshal(content, &config); err != nil { 97 | log.Fatal(fmt.Sprintf("Could not parse config file: %s", err)) 98 | } 99 | 100 | if config.Target.Registry == "" { 101 | log.Fatal("Missing `target -> registry` yaml config") 102 | } 103 | 104 | isPrivateECR = !strings.HasPrefix(config.Target.Registry, ecrPublicRegistryPrefix) 105 | 106 | if config.Workers == 0 { 107 | config.Workers = runtime.NumCPU() 108 | } 109 | 110 | // number of workers 111 | if w := os.Getenv("NUM_WORKERS"); w != "" { 112 | p, err := strconv.Atoi(w) 113 | if err != nil { 114 | log.Fatal(fmt.Sprintf("Could not parse NUM_WORKERS env: %s", err)) 115 | } 116 | 117 | config.Workers = p 118 | } 119 | 120 | // init Docker client 121 | log.Info("Creating Docker client") 122 | var client DockerClient 123 | client, err = createDockerClient() 124 | if err != nil { 125 | log.Fatalf("Could not create Docker client: %s", err.Error()) 126 | } 127 | 128 | info, err := client.Info() 129 | if err != nil { 130 | log.Fatalf("Could not get Docker info: %s", err.Error()) 131 | } 132 | log.Infof("Connected to Docker daemon: %s @ %s", info.Name, info.ServerVersion) 133 | 134 | // init AWS client 135 | log.Info("Creating AWS client") 136 | cfg, err := awsconfig.LoadDefaultConfig(context.TODO()) 137 | if err != nil { 138 | log.Fatalf("Unable to load AWS SDK config, " + err.Error()) 139 | } 140 | 141 | // pre-load ECR repositories 142 | var ecrManager ecrManager 143 | 144 | if !isPrivateECR { 145 | // Override the AWS region with the ecrPublicRegion for ECR authentication. 146 | cfg.Region = ecrPublicRegion 147 | ecrManager = &ecrPublicManager{client: ecrpublic.NewFromConfig(cfg)} 148 | } else { 149 | ecrManager = &ecrPrivateManager{client: ecr.NewFromConfig(cfg)} 150 | } 151 | 152 | backoffSettings := backoff.NewExponentialBackOff() 153 | backoffSettings.InitialInterval = 1 * time.Second 154 | backoffSettings.MaxElapsedTime = 10 * time.Second 155 | 156 | notifyError := func(err error, d time.Duration) { 157 | log.Errorf("%v (%s)", err, d.String()) 158 | } 159 | 160 | if err = backoff.RetryNotify(ecrManager.buildCacheBackoff(), backoffSettings, notifyError); err != nil { 161 | log.Fatalf("Could not build ECR cache: %s", err) 162 | } 163 | 164 | workerCh := make(chan Repository, 5) 165 | var wg sync.WaitGroup 166 | 167 | // start background workers 168 | for i := 0; i < config.Workers; i++ { 169 | go worker(&wg, workerCh, &client, ecrManager) 170 | } 171 | 172 | prefix := os.Getenv("PREFIX") 173 | 174 | // add jobs for the workers 175 | for _, repo := range config.Repositories { 176 | if prefix != "" && !strings.HasPrefix(repo.Name, prefix) { 177 | continue 178 | } 179 | 180 | wg.Add(1) 181 | workerCh <- repo 182 | } 183 | 184 | // wait for all workers to complete 185 | wg.Wait() 186 | log.Info("Done") 187 | } 188 | 189 | func worker(wg *sync.WaitGroup, workerCh chan Repository, dc *DockerClient, ecrm ecrManager) { 190 | log.Debug("Starting worker") 191 | 192 | for { 193 | select { 194 | case repo := <-workerCh: 195 | // Check if the given host is from our support list. 196 | if repo.Host != "" && repo.Host != dockerHub && repo.Host != quay && repo.Host != gcr && repo.Host != k8s { 197 | log.Errorf("Could not pull images from host: %s. We support %s, %s, %s, and %s", repo.Host, dockerHub, quay, gcr, k8s) 198 | wg.Done() 199 | continue 200 | } 201 | 202 | // If Host is not specified, will mirror repos from Docker Hub. 203 | if repo.Host == "" { 204 | repo.Host = dockerHub 205 | } 206 | 207 | m := mirror{ 208 | dockerClient: dc, 209 | ecrManager: ecrm, 210 | } 211 | if err := m.setup(repo); err != nil { 212 | log.Errorf("Failed to setup mirror for repository %s: %s", repo.Name, err) 213 | wg.Done() 214 | continue 215 | } 216 | 217 | m.work() 218 | wg.Done() 219 | } 220 | } 221 | } 222 | -------------------------------------------------------------------------------- /mirror.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/json" 7 | "fmt" 8 | "net/http" 9 | "os" 10 | "sort" 11 | "strconv" 12 | "strings" 13 | "time" 14 | 15 | docker "github.com/fsouza/go-dockerclient" 16 | "github.com/google/go-github/github" 17 | "github.com/ryanuber/go-glob" 18 | log "github.com/sirupsen/logrus" 19 | ) 20 | 21 | const ( 22 | dockerHub = "hub.docker.com" 23 | quay = "quay.io" 24 | gcr = "gcr.io" 25 | k8s = "k8s.gcr.io" 26 | ) 27 | 28 | var ( 29 | PTransport = &http.Transport{Proxy: http.ProxyFromEnvironment} 30 | httpClient = &http.Client{Timeout: 10 * time.Second, Transport: PTransport} 31 | ) 32 | 33 | // DockerTagsResponse is Docker Registry v2 compatible struct 34 | type DockerTagsResponse struct { 35 | Count int `json:"count"` 36 | Next *string `json:"next"` 37 | Previous *string `json:"previous"` 38 | Results []RepositoryTag `json:"results"` 39 | } 40 | 41 | // QuayTagsResponse is Quay API v1 compatible struct 42 | type QuayTagsResponse struct { 43 | HasAdditional bool `json:"has_additional"` 44 | Page int `json:"page"` 45 | Tags []RepositoryTag `json:"tags"` 46 | } 47 | 48 | // GCRTagsResponse is GCR API v2 compatible struct 49 | type GCRTagsResponse struct { 50 | Name string `json:"name"` 51 | Tags []string `json:"tags"` 52 | } 53 | 54 | // RepositoryTag is Docker, Quay, GCR API compatible struct, holding the individual 55 | // tags for the requested repository 56 | type RepositoryTag struct { 57 | Name string `json:"name"` 58 | LastUpdated time.Time `json:"last_updated"` 59 | LastModified time.Time `json:"last_modified"` 60 | } 61 | 62 | // logWriter is a io.Writer compatible wrapper, piping the output 63 | // to a specific logrus entry 64 | type logWriter struct { 65 | logger *log.Entry 66 | } 67 | 68 | func (l logWriter) Write(p []byte) (n int, err error) { 69 | l.logger.Debug(strings.Trim(string(p), "\n")) 70 | return len(p), nil 71 | } 72 | 73 | type DockerClient interface { 74 | Info() (*docker.DockerInfo, error) 75 | TagImage(string, docker.TagImageOptions) error 76 | PullImage(docker.PullImageOptions, docker.AuthConfiguration) error 77 | PushImage(docker.PushImageOptions, docker.AuthConfiguration) error 78 | RemoveImage(string) error 79 | } 80 | 81 | type mirror struct { 82 | dockerClient *DockerClient // docker client used to pull, tag and push images 83 | ecrManager ecrManager // ECR manager, used to ensure the ECR repository exist 84 | log *log.Entry // logrus logger with the relevant custom fields 85 | repo Repository // repository the mirror 86 | remoteTags []RepositoryTag // list of remote repository tags (post filtering) 87 | } 88 | 89 | const defaultSleepDuration time.Duration = 60 * time.Second 90 | 91 | func (m *mirror) setup(repo Repository) (err error) { 92 | m.log = log.WithField("full_repo", repo.Name) 93 | m.repo = repo 94 | // specific tag to mirror 95 | if strings.Contains(repo.Name, ":") { 96 | chunk := strings.SplitN(repo.Name, ":", 2) 97 | m.repo.Name = chunk[0] 98 | m.repo.MatchTags = []string{chunk[1]} 99 | } 100 | 101 | // fetch remote tags 102 | m.remoteTags, err = m.getRemoteTags() 103 | if err != nil { 104 | return err 105 | } 106 | 107 | m.filterTags() 108 | 109 | m.log = m.log.WithField("repo", m.repo.Name) 110 | m.log = m.log.WithField("num_tags", len(m.remoteTags)) 111 | return nil 112 | } 113 | 114 | // filter tags by 115 | // - by matching tag name (with glob support) 116 | // - by exluding tag name (with glob support) 117 | // - by tag age 118 | // - by max number of tags to process 119 | func (m *mirror) filterTags() { 120 | now := time.Now() 121 | res := make([]RepositoryTag, 0) 122 | 123 | for _, remoteTag := range m.remoteTags { 124 | // match tags, with glob 125 | if len(m.repo.MatchTags) > 0 { 126 | keep := false 127 | for _, tag := range m.repo.MatchTags { 128 | if !glob.Glob(tag, remoteTag.Name) { 129 | m.log.Debugf("Dropping tag '%s', it doesn't match glob pattern '%s'", remoteTag.Name, tag) 130 | continue 131 | } 132 | 133 | keep = true 134 | } 135 | 136 | if !keep { 137 | continue 138 | } 139 | } 140 | 141 | // filter all tags what should be ignored, with glob 142 | if len(m.repo.DropTags) > 0 { 143 | keep := true 144 | for _, tag := range m.repo.DropTags { 145 | if glob.Glob(tag, remoteTag.Name) { 146 | m.log.Debugf("Dropping tag '%s', its ignored by glob '%s'", remoteTag.Name, tag) 147 | keep = false 148 | break 149 | } 150 | } 151 | 152 | if !keep { 153 | continue 154 | } 155 | } 156 | 157 | // filter on tag age 158 | if m.repo.MaxTagAge != nil { 159 | dur := time.Duration(*m.repo.MaxTagAge) 160 | if now.Sub(remoteTag.LastUpdated) > dur { 161 | m.log.Debugf("Dropping tag '%s', its older than %s", remoteTag.Name, m.repo.MaxTagAge.String()) 162 | continue 163 | } 164 | } 165 | 166 | res = append(res, remoteTag) 167 | } 168 | 169 | // limit list of tags to $n newest (sorted by age by default) 170 | if m.repo.MaxTags > 0 && len(res) > m.repo.MaxTags { 171 | m.log.Debugf("Dropping %d tags, only need %d newest", len(res)-m.repo.MaxTags, m.repo.MaxTags) 172 | res = res[:m.repo.MaxTags] 173 | } 174 | 175 | m.remoteTags = res 176 | } 177 | 178 | // return the name of repostiory, as it should be on the target 179 | // this include any target repository prefix + the repository name in DockerHub 180 | func (m *mirror) targetRepositoryName() string { 181 | if m.repo.TargetPrefix != nil { 182 | return fmt.Sprintf("%s%s", *m.repo.TargetPrefix, m.repo.Name) 183 | } 184 | 185 | return fmt.Sprintf("%s%s", config.Target.Prefix, m.repo.Name) 186 | } 187 | 188 | // pull the image from remote repository to local docker agent 189 | func (m *mirror) pullImage(tag string) error { 190 | m.log.Info("Starting docker pull") 191 | defer m.timeTrack(time.Now(), "Completed docker pull") 192 | 193 | pullOptions := docker.PullImageOptions{ 194 | Tag: tag, 195 | InactivityTimeout: 1 * time.Minute, 196 | OutputStream: &logWriter{logger: m.log.WithField("docker_action", "pull")}, 197 | } 198 | authConfig := docker.AuthConfiguration{} 199 | 200 | switch m.repo.Host { 201 | case dockerHub: 202 | pullOptions.Repository = m.repo.Name 203 | 204 | if os.Getenv("DOCKERHUB_USER") != "" && os.Getenv("DOCKERHUB_PASSWORD") != "" { 205 | m.log.Info("Using docker hub credentials from environment") 206 | authConfig.Username = os.Getenv("DOCKERHUB_USER") 207 | authConfig.Password = os.Getenv("DOCKERHUB_PASSWORD") 208 | } 209 | 210 | if m.repo.PrivateRegistry != "" { 211 | pullOptions.Repository = m.repo.PrivateRegistry + "/" + m.repo.Name 212 | return (*m.dockerClient).PullImage(pullOptions, authConfig) 213 | } 214 | case quay: 215 | pullOptions.Repository = quay + "/" + m.repo.Name 216 | case gcr: 217 | pullOptions.Repository = gcr + "/" + m.repo.Name 218 | case k8s: 219 | pullOptions.Repository = k8s + "/" + m.repo.Name 220 | } 221 | 222 | return (*m.dockerClient).PullImage(pullOptions, authConfig) 223 | } 224 | 225 | // (re)tag the (local) docker image with the target repository name 226 | func (m *mirror) tagImage(tag string) error { 227 | m.log.Info("Starting docker tag") 228 | defer m.timeTrack(time.Now(), "Completed docker tag") 229 | 230 | tagOptions := docker.TagImageOptions{ 231 | Repo: fmt.Sprintf("%s/%s", config.Target.Registry, m.targetRepositoryName()), 232 | Tag: tag, 233 | Force: true, 234 | } 235 | 236 | switch m.repo.Host { 237 | case dockerHub: 238 | return (*m.dockerClient).TagImage(fmt.Sprintf("%s:%s", m.repo.Name, tag), tagOptions) 239 | case quay: 240 | return (*m.dockerClient).TagImage(fmt.Sprintf("%s/%s:%s", quay, m.repo.Name, tag), tagOptions) 241 | case gcr: 242 | return (*m.dockerClient).TagImage(fmt.Sprintf("%s/%s:%s", gcr, m.repo.Name, tag), tagOptions) 243 | case k8s: 244 | return (*m.dockerClient).TagImage(fmt.Sprintf("%s/%s:%s", k8s, m.repo.Name, tag), tagOptions) 245 | } 246 | 247 | return nil 248 | } 249 | 250 | // push the local (re)tagged image to the target docker registry 251 | func (m *mirror) pushImage(tag string) error { 252 | m.log.Info("Starting docker push") 253 | defer m.timeTrack(time.Now(), "Completed docker push") 254 | 255 | pushOptions := docker.PushImageOptions{ 256 | Name: fmt.Sprintf("%s/%s", config.Target.Registry, m.targetRepositoryName()), 257 | Registry: config.Target.Registry, 258 | Tag: tag, 259 | OutputStream: &logWriter{logger: m.log.WithField("docker_action", "push")}, 260 | InactivityTimeout: 1 * time.Minute, 261 | } 262 | 263 | var ( 264 | creds *docker.AuthConfiguration 265 | err error 266 | ) 267 | 268 | if !isPrivateECR { 269 | creds, err = getDockerCredentials(ecrPublicRegistryPrefix) 270 | } else { 271 | creds, err = getDockerCredentials(config.Target.Registry) 272 | } 273 | if err != nil { 274 | return err 275 | } 276 | 277 | return (*m.dockerClient).PushImage(pushOptions, *creds) 278 | } 279 | 280 | func (m *mirror) deleteImage(tag string) error { 281 | var repository string 282 | switch m.repo.Host { 283 | case dockerHub: 284 | repository = fmt.Sprintf("%s:%s", m.repo.Name, tag) 285 | case quay: 286 | repository = fmt.Sprintf("%s/%s:%s", quay, m.repo.Name, tag) 287 | case gcr: 288 | repository = fmt.Sprintf("%s/%s:%s", gcr, m.repo.Name, tag) 289 | case k8s: 290 | repository = fmt.Sprintf("%s/%s:%s", k8s, m.repo.Name, tag) 291 | } 292 | m.log.Info("Cleaning images: " + repository) 293 | err := (*m.dockerClient).RemoveImage(repository) 294 | if err != nil { 295 | return err 296 | } 297 | 298 | target := fmt.Sprintf("%s/%s:%s", config.Target.Registry, m.targetRepositoryName(), tag) 299 | m.log.Info("Cleaning images: " + target) 300 | err = (*m.dockerClient).RemoveImage(target) 301 | if err != nil { 302 | return err 303 | } 304 | 305 | return nil 306 | } 307 | 308 | func (m *mirror) work() { 309 | m.log.Debugf("Starting work") 310 | 311 | if err := m.ecrManager.ensure(m.targetRepositoryName()); err != nil { 312 | log.Errorf("Failed to create ECR repo %s: %s", m.targetRepositoryName(), err) 313 | return 314 | } 315 | 316 | for _, tag := range m.remoteTags { 317 | m.log = m.log.WithField("tag", tag.Name) 318 | m.log.Info("Start mirror tag") 319 | 320 | if err := m.pullImage(tag.Name); err != nil { 321 | m.log.Errorf("Failed to pull docker image: %s", err) 322 | continue 323 | } 324 | 325 | if err := m.tagImage(tag.Name); err != nil { 326 | m.log.Errorf("Failed to (re)tag docker image: %s", err) 327 | continue 328 | } 329 | 330 | if err := m.pushImage(tag.Name); err != nil { 331 | m.log.Errorf("Failed to push (re)tagged image: %s", err) 332 | continue 333 | } 334 | 335 | if config.Cleanup == true { 336 | if err := m.deleteImage(tag.Name); err != nil { 337 | m.log.Errorf("Failed to clean image: %s", err) 338 | continue 339 | } 340 | } 341 | 342 | m.log.Info("Successfully pushed (re)tagged image") 343 | } 344 | 345 | m.log.WithField("tag", "") 346 | m.log.Info("Repository mirror completed") 347 | } 348 | 349 | // get the remote tags from the remote compatible registry. 350 | // read out the image tag and when it was updated, and sort by the updated time if applicable 351 | func (m *mirror) getRemoteTags() ([]RepositoryTag, error) { 352 | if m.repo.RemoteTagSource == "github" { 353 | client := github.NewClient(nil) 354 | limit, err := strconv.Atoi(m.repo.RemoteTagConfig["num_releases"]) 355 | if err != nil { 356 | return nil, fmt.Errorf("Invalid/missing int value for remote_tag_config -> num_releases") 357 | } 358 | 359 | remoteTags, _, err := client.Repositories.ListTags(context.Background(), m.repo.RemoteTagConfig["owner"], m.repo.RemoteTagConfig["repo"], &github.ListOptions{PerPage: limit}) 360 | if err != nil { 361 | return nil, err 362 | } 363 | 364 | var allTags []RepositoryTag 365 | for _, tag := range remoteTags { 366 | allTags = append(allTags, RepositoryTag{ 367 | Name: strings.TrimPrefix(*tag.Name, "v"), 368 | }) 369 | } 370 | 371 | return allTags, nil 372 | } 373 | 374 | // Get tags information from Docker Hub, Quay, GCR or k8s.gcr.io. 375 | var url string 376 | fullRepoName := m.repo.Name 377 | token := "" 378 | 379 | switch m.repo.Host { 380 | case dockerHub: 381 | if !strings.Contains(fullRepoName, "/") { 382 | fullRepoName = "library/" + m.repo.Name 383 | } 384 | 385 | if os.Getenv("DOCKERHUB_USER") != "" && os.Getenv("DOCKERHUB_PASSWORD") != "" { 386 | m.log.Info("Getting tags using docker hub credentials from environment") 387 | 388 | message, err := json.Marshal(map[string]string{ 389 | "username": os.Getenv("DOCKERHUB_USER"), 390 | "password": os.Getenv("DOCKERHUB_PASSWORD"), 391 | }) 392 | 393 | if err != nil { 394 | return nil, err 395 | } 396 | 397 | resp, err := http.Post("https://hub.docker.com/v2/users/login/", "application/json", bytes.NewBuffer(message)) 398 | if err != nil { 399 | return nil, err 400 | } 401 | 402 | var result map[string]interface{} 403 | 404 | json.NewDecoder(resp.Body).Decode(&result) 405 | token = result["token"].(string) 406 | } 407 | 408 | url = fmt.Sprintf("https://registry.hub.docker.com/v2/repositories/%s/tags/?page_size=2048", fullRepoName) 409 | case quay: 410 | url = fmt.Sprintf("https://quay.io/api/v1/repository/%s/tag", fullRepoName) 411 | case gcr: 412 | url = fmt.Sprintf("https://gcr.io/v2/%s/tags/list", fullRepoName) 413 | case k8s: 414 | url = fmt.Sprintf("https://k8s.gcr.io/v2/%s/tags/list", fullRepoName) 415 | } 416 | 417 | var allTags []RepositoryTag 418 | 419 | search: 420 | for { 421 | var ( 422 | err error 423 | res *http.Response 424 | req *http.Request 425 | retries int = 5 426 | ) 427 | 428 | for retries > 0 { 429 | req, err = http.NewRequest("GET", url, nil) 430 | if err != nil { 431 | return nil, err 432 | } 433 | 434 | if token != "" { 435 | req.Header.Set("Authorization", fmt.Sprintf("JWT %s", token)) 436 | } 437 | 438 | res, err = httpClient.Do(req) 439 | 440 | if err != nil { 441 | m.log.Warningf(err.Error()) 442 | m.log.Warningf("Failed to get %s, retrying", url) 443 | retries-- 444 | } else if res.StatusCode == 429 { 445 | sleepTime := getSleepTime(res.Header.Get("X-RateLimit-Reset"), time.Now()) 446 | m.log.Infof("Rate limited on %s, sleeping for %s", url, sleepTime) 447 | time.Sleep(sleepTime) 448 | retries-- 449 | } else if res.StatusCode < 200 || res.StatusCode >= 300 { 450 | m.log.Warningf("Get %s failed with %d, retrying", url, res.StatusCode) 451 | retries-- 452 | } else { 453 | break 454 | } 455 | 456 | } 457 | 458 | if err != nil { 459 | return nil, err 460 | } 461 | defer res.Body.Close() 462 | 463 | dc := json.NewDecoder(res.Body) 464 | 465 | switch m.repo.Host { 466 | case dockerHub: 467 | var tags DockerTagsResponse 468 | if err = dc.Decode(&tags); err != nil { 469 | return nil, err 470 | } 471 | 472 | allTags = append(allTags, tags.Results...) 473 | if tags.Next == nil { 474 | break search 475 | } 476 | 477 | url = *tags.Next 478 | case quay: 479 | var tags QuayTagsResponse 480 | if err := dc.Decode(&tags); err != nil { 481 | return nil, err 482 | } 483 | allTags = append(allTags, tags.Tags...) 484 | break search 485 | case gcr: 486 | var tags GCRTagsResponse 487 | if err := dc.Decode(&tags); err != nil { 488 | return nil, err 489 | } 490 | for _, tag := range tags.Tags { 491 | allTags = append(allTags, RepositoryTag{ 492 | Name: tag, 493 | }) 494 | } 495 | break search 496 | case k8s: 497 | var tags GCRTagsResponse 498 | if err := dc.Decode(&tags); err != nil { 499 | return nil, err 500 | } 501 | for _, tag := range tags.Tags { 502 | allTags = append(allTags, RepositoryTag{ 503 | Name: tag, 504 | }) 505 | } 506 | break search 507 | } 508 | } 509 | 510 | // sort the tags by updated/modified time if applicable, newest first 511 | switch m.repo.Host { 512 | case dockerHub: 513 | sort.Slice(allTags, func(i, j int) bool { 514 | return allTags[i].LastUpdated.After(allTags[j].LastUpdated) 515 | }) 516 | case quay: 517 | sort.Slice(allTags, func(i, j int) bool { 518 | return allTags[i].LastModified.After(allTags[j].LastModified) 519 | }) 520 | } 521 | 522 | return allTags, nil 523 | } 524 | 525 | // will help output how long time a function took to do its work 526 | func (m *mirror) timeTrack(start time.Time, name string) { 527 | elapsed := time.Since(start) 528 | m.log.Infof("%s in %s", name, elapsed) 529 | } 530 | 531 | func getSleepTime(rateLimitReset string, now time.Time) time.Duration { 532 | rateLimitResetInt, err := strconv.ParseInt(rateLimitReset, 10, 64) 533 | 534 | if err != nil { 535 | return defaultSleepDuration 536 | } 537 | 538 | sleepTime := time.Unix(rateLimitResetInt, 0) 539 | calculatedSleepTime := sleepTime.Sub(now) 540 | 541 | if calculatedSleepTime < (0 * time.Second) { 542 | return 0 * time.Second 543 | } 544 | 545 | return calculatedSleepTime 546 | } 547 | -------------------------------------------------------------------------------- /mirror_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "strconv" 5 | "testing" 6 | "time" 7 | 8 | docker "github.com/fsouza/go-dockerclient" 9 | ) 10 | 11 | func TestGetSleepTime(t *testing.T) { 12 | fakeNow := time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC) 13 | 14 | // Zero 15 | result := getSleepTime(getTimeAsString(time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC)), fakeNow) 16 | expected := 0 * time.Second 17 | if result != expected { 18 | t.Errorf("Expected %s got %s", expected, result) 19 | } 20 | 21 | // Default 22 | result = getSleepTime(getTimeAsString(time.Date(2021, 1, 1, 0, 0, 10, 0, time.UTC)), fakeNow) 23 | expected = 10 * time.Second 24 | if result != expected { 25 | t.Errorf("Expected %s got %s", expected, result) 26 | } 27 | 28 | // Random junk 29 | result = getSleepTime("random-string-of-rubbish", fakeNow) 30 | expected = 60 * time.Second 31 | if result != expected { 32 | t.Errorf("Expected %s got %s", expected, result) 33 | } 34 | 35 | // Negative 36 | result = getSleepTime(getTimeAsString(time.Date(2020, 12, 30, 0, 0, 10, 0, time.UTC)), fakeNow) 37 | expected = 0 * time.Second 38 | if result != expected { 39 | t.Errorf("Expected %s got %s", expected, result) 40 | } 41 | 42 | } 43 | 44 | type ResponseContainer struct { 45 | TagImageName string 46 | TagImageOptions docker.TagImageOptions 47 | PullImageOptions docker.PullImageOptions 48 | PullImageAuthConfiguration docker.AuthConfiguration 49 | PushImageOptions docker.PushImageOptions 50 | PushImageAuthConfiguration docker.AuthConfiguration 51 | RemoveImageName string 52 | } 53 | 54 | type TestDockerClient struct { 55 | ResponseContainer *ResponseContainer 56 | } 57 | 58 | func (t *TestDockerClient) Info() (*docker.DockerInfo, error) { 59 | return &docker.DockerInfo{}, nil 60 | } 61 | 62 | func (t *TestDockerClient) TagImage(name string, opts docker.TagImageOptions) error { 63 | t.ResponseContainer.TagImageName = name 64 | t.ResponseContainer.TagImageOptions = opts 65 | return nil 66 | } 67 | 68 | func (t *TestDockerClient) PullImage(opts docker.PullImageOptions, authConfig docker.AuthConfiguration) error { 69 | t.ResponseContainer.PullImageOptions = opts 70 | t.ResponseContainer.PullImageAuthConfiguration = authConfig 71 | return nil 72 | } 73 | 74 | func (t *TestDockerClient) PushImage(opts docker.PushImageOptions, auth docker.AuthConfiguration) error { 75 | t.ResponseContainer.PushImageOptions = opts 76 | t.ResponseContainer.PushImageAuthConfiguration = auth 77 | return nil 78 | } 79 | 80 | func (t *TestDockerClient) RemoveImage(name string) error { 81 | t.ResponseContainer.RemoveImageName = name 82 | return nil 83 | } 84 | 85 | func CreateTestDockerClient(responseContainer *ResponseContainer) *TestDockerClient { 86 | return &TestDockerClient{ResponseContainer: responseContainer} 87 | } 88 | 89 | func TestPullImage(t *testing.T) { 90 | 91 | t.Run("tests to ensure that PrivateRegistry creates the proper repo name", func(t *testing.T) { 92 | responseContainer := &ResponseContainer{} 93 | var client DockerClient 94 | client = CreateTestDockerClient(responseContainer) 95 | repo := Repository{ 96 | PrivateRegistry: "private-registry-name", 97 | Name: "elasticsearch", 98 | Host: "hub.docker.com", 99 | } 100 | 101 | m := mirror{ 102 | dockerClient: &client, 103 | } 104 | 105 | m.setup(repo) 106 | m.pullImage("latest") 107 | 108 | got := responseContainer.PullImageOptions.Repository 109 | want := "private-registry-name/elasticsearch" 110 | 111 | if got != want { 112 | t.Errorf("Expected %q, got %q", want, got) 113 | } 114 | }) 115 | 116 | t.Run("tests to ensure that without PrivateRegistry, a repo name is correct", func(t *testing.T) { 117 | responseContainer := &ResponseContainer{} 118 | var client DockerClient 119 | client = CreateTestDockerClient(responseContainer) 120 | repo := Repository{ 121 | Name: "elasticsearch", 122 | Host: "hub.docker.com", 123 | } 124 | 125 | m := mirror{ 126 | dockerClient: &client, 127 | } 128 | 129 | m.setup(repo) 130 | m.pullImage("latest") 131 | 132 | got := responseContainer.PullImageOptions.Repository 133 | want := "elasticsearch" 134 | 135 | if got != want { 136 | t.Errorf("Expected %q, got %q", want, got) 137 | } 138 | }) 139 | } 140 | 141 | func getTimeAsString(date time.Time) string { 142 | return strconv.FormatInt(date.Unix(), 10) 143 | } 144 | --------------------------------------------------------------------------------