├── .DockerIgnore ├── .github ├── .markdown-lint.yml ├── ISSUE_TEMPLATE │ ├── bug-report.md │ ├── config.yml │ └── feature_request.md ├── PULL_REQUEST_TEMPLATE.md └── workflows │ ├── check.yml │ ├── codecov.yml │ ├── config │ ├── test-auth.yml │ └── test-images.yml │ ├── release.yml │ └── synctest.yml ├── .gitignore ├── .spelling ├── Dockerfile ├── FAQs.md ├── LICENSE ├── Makefile ├── README-zh_CN.md ├── README.md ├── cmd └── image-syncer.go ├── examples ├── auth.json ├── auth.yaml ├── config.json ├── config.yaml ├── images.json └── images.yaml ├── go.mod ├── go.sum ├── main.go └── pkg ├── client ├── client.go ├── config.go └── logger.go ├── concurrent ├── counter.go ├── imageList.go └── list.go ├── sync ├── constants.go ├── destination.go ├── manifest.go └── source.go ├── task ├── blob.go ├── manifest.go ├── rule.go ├── task.go └── url.go └── utils ├── auth └── google.go ├── ctx.go ├── slice.go ├── string.go ├── string_test.go ├── types ├── auth.go └── imageList.go ├── url.go └── url_test.go /.DockerIgnore: -------------------------------------------------------------------------------- 1 | *.md 2 | *.yaml 3 | *.json 4 | .git/ 5 | .github/ 6 | images-syncer -------------------------------------------------------------------------------- /.github/.markdown-lint.yml: -------------------------------------------------------------------------------- 1 | MD024: 2 | allow_different_nesting: true 3 | siblings_only: true 4 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug-report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug Report 3 | about: Report a bug encountered while using image-syncer 4 | labels: kind/bug 5 | --- 6 | 7 | 11 | 12 | Bug Report 13 | --- 14 | 15 | Type: *bug report* 16 | 17 | ### What happened 18 | 19 | ### What you expected to happen 20 | 21 | ### How to reproduce it (as minimally and precisely as possible) 22 | 23 | ### Anything else we need to know? 24 | 25 | ### Environment 26 | - image-syncer version: 27 | - OS (e.g. `cat /etc/os-release`): 28 | - Registry version (e.g. `habor`): 29 | - Others: -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | blank_issues_enabled: false -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | labels: kind/feature 5 | --- 6 | 7 | 11 | 12 | Issue Description 13 | --- 14 | 15 | Type: *feature request* 16 | 17 | ### Describe what feature you want 18 | 19 | ### Additional context 20 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 5 | 6 | Pull Request Description 7 | --- 8 | 9 | ### Describe what this PR does / why we need it 10 | 11 | ### Does this pull request fix one issue? 12 | 13 | 14 | ### Describe how you did it 15 | 16 | ### Describe how to verify it 17 | 18 | ### Special notes for reviews -------------------------------------------------------------------------------- /.github/workflows/check.yml: -------------------------------------------------------------------------------- 1 | name: check 2 | 3 | on: 4 | push: { } 5 | pull_request: { } 6 | 7 | concurrency: 8 | group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.event.after }} 9 | cancel-in-progress: true 10 | 11 | jobs: 12 | go-mod: 13 | runs-on: ubuntu-latest 14 | steps: 15 | - uses: actions/checkout@v2 16 | - name: Set up Go 17 | uses: actions/setup-go@v2.1.3 18 | with: 19 | go-version: 1.20.4 20 | - uses: actions/cache@v2 21 | with: 22 | path: ~/go/pkg/mod 23 | key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} 24 | restore-keys: | 25 | ${{ runner.os }}-go- 26 | - name: Check module vendoring 27 | run: | 28 | go mod tidy 29 | go mod vendor 30 | git diff --exit-code || (echo "please run 'go mod tidy && go mod vendor', and submit your changes"; exit 1) 31 | 32 | golangci-lint: 33 | runs-on: ubuntu-latest 34 | steps: 35 | - uses: actions/checkout@v2 36 | - name: Set up Go 37 | uses: actions/setup-go@v2.1.3 38 | with: 39 | go-version: 1.20.4 40 | - name: Run golangci-lint 41 | uses: golangci/golangci-lint-action@v3.6.0 42 | with: 43 | version: v1.53 44 | args: --timeout 300s --skip-dirs-use-default -v -E goconst -E gofmt -E ineffassign -E goimports -E revive -E misspell -E vet -E deadcode 45 | 46 | go-test: 47 | runs-on: ubuntu-latest 48 | steps: 49 | - uses: actions/checkout@v2 50 | - name: Set up Go 51 | uses: actions/setup-go@v2.1.3 52 | with: 53 | go-version: 1.20.4 54 | - uses: actions/cache@v2 55 | with: 56 | path: ~/go/pkg/mod 57 | key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} 58 | restore-keys: | 59 | ${{ runner.os }}-go- 60 | - name: Test 61 | run: | 62 | go test -v ./... 63 | go test -race -v ./... 64 | 65 | shellcheck: 66 | runs-on: ubuntu-latest 67 | steps: 68 | - uses: actions/checkout@v2 69 | - name: Run ShellCheck 70 | uses: ludeeus/action-shellcheck@1.1.0 71 | env: 72 | SHELLCHECK_OPTS: -e SC2236,SC2162,SC2268 73 | with: 74 | ignore: tests 75 | 76 | super-linter: 77 | runs-on: ubuntu-latest 78 | steps: 79 | - uses: actions/checkout@v2 80 | - name: Lint Code Base 81 | uses: github/super-linter@v4.5.1 82 | env: 83 | VALIDATE_ALL_CODEBASE: true 84 | VALIDATE_MARKDOWN: true 85 | LINTER_RULES_PATH: .github 86 | MARKDOWN_CONFIG_FILE: .markdown-lint.yml 87 | VALIDATE_MD: true 88 | DEFAULT_BRANCH: main 89 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 90 | FILTER_REGEX_EXCLUDE: .github/.* 91 | -------------------------------------------------------------------------------- /.github/workflows/codecov.yml: -------------------------------------------------------------------------------- 1 | name: codecov 2 | 3 | on: 4 | push: 5 | branches: 6 | - 'master' 7 | pull_request: 8 | branches: 9 | - 'master' 10 | 11 | concurrency: 12 | group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.event.after }} 13 | cancel-in-progress: true 14 | 15 | jobs: 16 | codecov: 17 | runs-on: ubuntu-latest 18 | steps: 19 | - uses: actions/checkout@v2 20 | - name: Set up Go 21 | uses: actions/setup-go@v2.1.3 22 | with: 23 | go-version: 1.20.4 24 | - uses: actions/cache@v2 25 | with: 26 | path: ~/go/pkg/mod 27 | key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} 28 | restore-keys: | 29 | ${{ runner.os }}-go- 30 | - name: Test and update codecov 31 | run: | 32 | go test -race -coverprofile=coverage.txt -covermode=atomic ./... 33 | - uses: codecov/codecov-action@v1.5.0 34 | with: 35 | file: ./coverage.txt 36 | flags: unittests 37 | fail_ci_if_error: true -------------------------------------------------------------------------------- /.github/workflows/config/test-auth.yml: -------------------------------------------------------------------------------- 1 | localhost:5000: 2 | insecure: true -------------------------------------------------------------------------------- /.github/workflows/config/test-images.yml: -------------------------------------------------------------------------------- 1 | # single docker image 2 | quay.io/operator-framework/olm:v0.18.3: "localhost:5000/image-syncer-test/olm" 3 | 4 | # docker manifest list 5 | alpine:3.18.2: "localhost:5000/image-syncer-test/alpine" 6 | 7 | # OCI index 8 | hybridnetdev/hybridnet:v0.8.2: "localhost:5000/image-syncer-test/hybridnet" 9 | 10 | # OCI image 11 | hybridnetdev/hybridnet@sha256:14b267eb38aa85fd12d0e168fffa2d8a6187ac53a14a0212b0d4fce8d729598c: 12 | - "localhost:5000/image-syncer-test/hybridnet" 13 | - "localhost:5000/image-syncer-test/hybridnet:v0.8.2-test" -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: release 2 | 3 | on: 4 | release: 5 | types: [created] 6 | 7 | jobs: 8 | releases-linux-binaries: 9 | name: Release Go Binaries 10 | runs-on: ubuntu-latest 11 | strategy: 12 | matrix: 13 | goos: [linux, darwin] 14 | goarch: [amd64, arm64] 15 | steps: 16 | - uses: actions/checkout@v2 17 | - uses: wangyoucao577/go-release-action@v1.51 18 | with: 19 | github_token: ${{ secrets.GITHUB_TOKEN }} 20 | goos: ${{ matrix.goos }} 21 | goarch: ${{ matrix.goarch }} 22 | goversion: 1.20.4 23 | extra_files: LICENSE README.md 24 | -------------------------------------------------------------------------------- /.github/workflows/synctest.yml: -------------------------------------------------------------------------------- 1 | name: synctest 2 | 3 | on: 4 | push: { } 5 | pull_request: { } 6 | 7 | concurrency: 8 | group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.event.after }} 9 | cancel-in-progress: true 10 | 11 | jobs: 12 | test_sync_job: 13 | runs-on: ubuntu-latest 14 | steps: 15 | - name: Checkout 16 | uses: actions/checkout@v3 17 | - name: Set up Go 18 | uses: actions/setup-go@v2 19 | with: 20 | go-version: 1.20.4 21 | - name: Build 22 | run: make 23 | - name: Run docker registry 24 | run: docker run -d -p 5000:5000 --restart=always --name registry registry:2 25 | - name: Run image-syncer 26 | run: ./image-syncer --proc=6 --auth=./.github/workflows/config/test-auth.yml --images=./.github/workflows/config/test-images.yml 27 | - name: Use docker to check result 28 | run: | 29 | docker pull localhost:5000/image-syncer-test/alpine:3.18.2 && 30 | docker pull localhost:5000/image-syncer-test/olm:v0.18.3 && 31 | docker pull localhost:5000/image-syncer-test/hybridnet:v0.8.2 && 32 | docker pull localhost:5000/image-syncer-test/hybridnet@sha256:14b267eb38aa85fd12d0e168fffa2d8a6187ac53a14a0212b0d4fce8d729598c && 33 | docker pull localhost:5000/image-syncer-test/hybridnet:v0.8.2-test -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | image-syncer 2 | records 3 | .idea 4 | .vscode 5 | *.log 6 | *.conf 7 | *.json 8 | *.yaml -------------------------------------------------------------------------------- /.spelling: -------------------------------------------------------------------------------- 1 | image-syncer 2 | Stdout 3 | DIGEST_INVALID 4 | e.g. 5 | Alibaba 6 | Quay.io 7 | goroutine 8 | v1.1.0 9 | v1.2.0 -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.22.5 as builder 2 | WORKDIR /go/src/github.com/AliyunContainerService/image-syncer 3 | COPY ./ ./ 4 | ENV GOPROXY=https://proxy.golang.com.cn,direct 5 | RUN CGO_ENABLED=0 GOOS=linux make 6 | 7 | FROM alpine:latest 8 | WORKDIR /bin/ 9 | COPY --from=builder /go/src/github.com/AliyunContainerService/image-syncer/image-syncer ./ 10 | RUN chmod +x ./image-syncer 11 | RUN apk add -U --no-cache ca-certificates && rm -rf /var/cache/apk/* && mkdir -p /etc/ssl/certs \ 12 | && update-ca-certificates --fresh 13 | ENTRYPOINT ["image-syncer"] 14 | CMD ["--config", "/etc/image-syncer/image-syncer.json"] 15 | -------------------------------------------------------------------------------- /FAQs.md: -------------------------------------------------------------------------------- 1 | # FAQs 2 | 3 | ## How can I get the results of synchronization?（如何获得同步的执行结果？） 4 | 5 | After synchronization, `image-syncer` will count the failed and success synchronization tasks(images), outputs like "Finished, \ sync tasks failed, \ tasks generate failed" will be print to both Stdout and logs file. 6 | 7 | 在同步结束之后，`image-syncer` 会统计成功和失败的同步任务数目（每个同步任务代表一个镜像），并在标准输出和日志中打印 "Finished, \ sync tasks failed, \ tasks generate failed" 的字样，从而可以获得同步的结果 8 | 9 | ## How can I deal with the failed sync tasks?（同步失败的任务如何处理？） 10 | 11 | Failed tasks caused by *occasional* network problems will be restored by retries generally, if number of retries is enough. If number of retries is not large enough and synchronization is ended, you can just run `image-syncer` twice in the same way twice, synchronized images will be skipped as synchronization is incremental (with the same records file). 12 | 13 | 偶发网络问题导致的同步任务失败，在重试次数足够的情况下（默认为2次），通常会被重试机制恢复。如果重试次数不够多，并且程序结束时还有同步失败的任务，先排查是否是偶发的问题，如果是，可以直接重新再次以相同的方式运行程序，已同步的镜像会被跳过（依赖之前生成records文件）。 14 | 15 | ## Find "Put manifest ... unknown blob error" in logs during synchronization, while there is no error with PUT actions of specific blobs?（传输过程中遇到了"Put manifest ... unknown blob error"问题，但是put对应blob的时候没有错误） 16 | 17 | This problem may happens if the records file that already exists was not generated by yourself, this records file may includes the image blobs you have not synchronized. Use another records file path to generate a new one if you are not sure which is the correct records file, this may cause a much more longer synchronization. 18 | 19 | 在用错了records文件的情况下可能会发生，这个错误的records文件记录了之前没有同步过的镜像blog。如果不确定正确的records文件在哪，可以用个新的records文件路径生成一个新的records文件，但是这样同步会耗费更长的时间。 20 | 21 | image-syncer remove the dependence of records file after v1.1.0 22 | 23 | image-syncer 在 >= v1.1.0 版本中移除了对于records文件的依赖 24 | 25 | ## Occasional network problems（常见的偶发网络问题） 26 | 27 | "read: connection reset by peer" happens if the source of registry has network limits; "TLS handshake timeout" happens if network delay gets high; "DIGEST_INVALID" happens if image blobs is damaged in network traffic; 28 | 29 | "read: connection reset by peer"这种大概是源仓库所在registry的网络限流，连接被断掉了；"TLS handshake timeout"在网络延迟比较高的时候会出现；"DIGEST_INVALID"表示在网络传输过程中由于传输错误，镜像blob损坏； 30 | 31 | ## “ACR get tags failed” 32 | 33 | As getting tags of a ACR private repository needs to authenticate in another way defer from username/password, if you want to synchronize sources of ACR private repository, you need to specify all the tags that you want to synchronize. 34 | 35 | 由于`image-syncer`现在不支持password/username之外的认证方式，而如果要获取ACR私有仓库所有的tags，需要进行额外的认证；如果你需要以ACR私有仓库作为源同步镜像，需要手动指定所有需要同步的tag。 36 | 37 | ## “500 Internal Server Error” 38 | 39 | When you are confused with such error logs of image-syncer, check if some of manifests in your docker images are schema V1 40 | (which was deprecated by docker) first. As this tool doesn't support auto-transformation of manifest for schema V1 to schema 41 | V2, you cannot synchronize a docker image with schema V1 manifest to a registry which doesn't support it (e.g., Harbor, etc.). 42 | 43 | 当出现这种报错时，首先检查是否是镜像是旧版本，并且manifest类型为 Schema 1（已被docker弃用）。由于本工具不支持manifest类型的自动转换，并且目前新版本harbor不再支持类型为 Schema 1 的manifest，所以无法通过本工具将manifest类型为 Schema 1 的docker镜像同步到不支持该类型manifest的镜像存储中（比如harbor） -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: cmd clean 2 | 3 | cmd: $(wildcard ./pkg/client/*.go ./pkg/sync/*.go ./pkg/tools/*.go ./cmd/*.go ./*.go) 4 | go build -o image-syncer ./main.go 5 | 6 | clean: 7 | rm image-syncer -------------------------------------------------------------------------------- /README-zh_CN.md: -------------------------------------------------------------------------------- 1 | # image-syncer 2 | 3 | ![workflow check](https://github.com/AliyunContainerService/image-syncer/actions/workflows/check.yml/badge.svg) 4 | ![workflow build](https://github.com/AliyunContainerService/image-syncer/actions/workflows/synctest.yml/badge.svg) 5 | [![Version](https://img.shields.io/github/v/release/AliyunContainerService/image-syncer)](https://github.com/AliyunContainerService/image-syncer/releases) 6 | [![Go Report Card](https://goreportcard.com/badge/github.com/AliyunContainerService/image-syncer)](https://goreportcard.com/report/github.com/AliyunContainerService/image-syncer) 7 | [![Github All Releases](https://img.shields.io/github/downloads/AliyunContainerService/image-syncer/total.svg)](https://api.github.com/repos/AliyunContainerService/image-syncer/releases) 8 | [![codecov](https://codecov.io/gh/AliyunContainerService/image-syncer/graph/badge.svg)](https://codecov.io/gh/AliyunContainerService/image-syncer) 9 | [![License](https://img.shields.io/github/license/AliyunContainerService/image-syncer)](https://www.apache.org/licenses/LICENSE-2.0.html) 10 | 11 | `image-syncer` 是一个容器镜像同步工具，可用来进行多对多的镜像仓库同步，支持目前绝大多数主流的 docker 镜像仓库服务 12 | 13 | [English](./README.md) | 简体中文 14 | 15 | ## Features 16 | 17 | - 支持多对多镜像仓库同步 18 | - 支持基于 Docker Registry V2 搭建的镜像仓库服务 (如 Docker Hub、 Quay、 阿里云镜像服务 ACR、 Harbor 等) 19 | - 同步过程只经过内存和网络，不依赖磁盘存储，同步速度快 20 | - 自动增量同步, 自动忽略已同步且不需要修改的镜像 21 | - 支持镜像层级别的并发同步，可以通过配置文件调整并发数（可以理解为同一时间在同步的镜像层数量上限） 22 | - 自动重试失败的同步任务，可以解决大部分镜像同步中的偶发问题（限流、网络抖动），支持重试次数配置 23 | - 简单轻量，不依赖 docker 以及其他程序 24 | 25 | ## 使用 26 | 27 | ### GitHub Action 28 | 29 | 可以使用 [image-sync-action](https://github.com/marketplace/actions/image-sync-action) 这个 github action 来实现灵活触发的镜像同步作业（比如定时同步），不需要支付任何资源的同时，也可以解决国内外网访问的问题 30 | 31 | ### 下载和安装 32 | 33 | 在 [releases](https://github.com/AliyunContainerService/image-syncer/releases) 页面可下载源码以及二进制文件 34 | 35 | ### 手动编译 36 | 37 | ```bash 38 | go get github.com/AliyunContainerService/image-syncer 39 | cd $GOPATH/github.com/AliyunContainerService/image-syncer 40 | 41 | # This will create a binary file named image-syncer 42 | make 43 | ``` 44 | 45 | ### 命令用例 46 | 47 | ```bash 48 | # 获得帮助信息 49 | ./image-syncer -h 50 | 51 | ./image-syncer --proc=6 --auth=./auth.json --images=./images.json --retries=3 52 | ``` 53 | 54 | ### 配置文件 55 | 56 | 为了提高配置的灵活性，image-syncer 支持通过 `--auth` 参数以文件的形式传入认证信息，同时通过 `--images` 参数以文件的形式传入镜像同步规则。两种配置文件都同时支持 YAML 和 JSON 两种格式，其中认证信息是可选的，镜像同步规则是必须的。通过两者分离的方式，可以做到认证信息的灵活复用 57 | 58 | > 1.2.0 版本之前主要使用的、通过 `--config` 参数以一个配置文件同时传入认证信息和镜像同步规则的格式也是兼容的，可以参考 [config.yaml](examples/config.yaml) 和 [config.json](examples/config.json) 59 | 60 | #### 认证信息 61 | 62 | 认证信息中可以同时描述多个 registry（或者 registry/namespace）对象，一个对象可以包含账号和密码，其中，密码可能是一个 TOKEN 63 | 64 | > 注意，通常镜像源仓库需要具有 pull 以及访问 tags 权限，镜像目标仓库需要拥有 push 以及创建仓库权限；如果对应仓库没有提供认证信息，则默认匿名访问 65 | 66 | 认证信息文件通过 `--auth` 参数传入，具体文件样例可以参考 [auth.yaml](examples/auth.yaml) 和 [auth.json](examples/auth.json)，这里以 [auth.yaml](examples/auth.yaml) 为例： 67 | 68 | ```yaml 69 | quay.io: #支持 "registry" 和 "registry/namespace"（v1.0.3之后的版本） 的形式，image-syncer 会自动为镜像同步规则中的每个源/目标 url 查找认证信息，并且使用对应认证信息进行进行访问，如果匹配到了多个，用“最长匹配”的那个作为最终结果 70 | username: xxx 71 | password: xxxxxxxxx 72 | insecure: true # 可选，（v1.0.1 之后支持）registry是否是http服务，如果是，insecure 字段需要为 true，默认是 false 73 | registry.cn-beijing.aliyuncs.com: 74 | username: xxx # 可选，（v1.3.1 之后支持）value 使用 "${env}" 或者 "$env" 形式可以引用环境变量 75 | password: xxxxxxxxx # 可选，（v1.3.1 之后支持）value 使用 "${env}" 或者 "$env" 类型的字符串可以引用环境变量 76 | docker.io: 77 | username: "${env}" 78 | password: "$env" 79 | quay.io/coreos: 80 | username: abc 81 | password: xxxxxxxxx 82 | insecure: true 83 | ``` 84 | 85 | #### 镜像同步规则 86 | 87 | 每条镜像同步规则为一个 “源镜像 url: 目标镜像 url” 的键值对。无论是源镜像 url 还是目标镜像 url，字符串格式都和 docker pull 命令所使用的镜像 url 大致相同（registry/repository:tag、registry/repository@digest），但在 tag 和 digest 配置上和 docker pull 所使用的 url 存在区别，这里对整体逻辑进行描述： 88 | 89 | 1. 源镜像 url、目标镜像 url 都不能为空 90 | 2. 源镜像 url 不包含 tag 和 digest 时，代表同步源镜像 repository 中的所有镜像 tag 91 | 3. 源镜像 url 可以包含一个或多个 tag，多个 tag 之间用英文逗号分隔，代表同步源镜像 repository 中的多个指定镜像 tag 92 | 4. 源镜像 url 可以但最多只能包含一个 digest，此时如果目标镜像 url 包含 digest，digest 必须一致 93 | 5. 源镜像 url 的 "tag" 可以是一个正则表达式，需要额外在首尾加上 `/` 字符作为标识，源镜像 repository 中所有匹配正则表达式的镜像 tag 会被同步，不支持多个正则表达式 94 | 6. 目标镜像 url 可以不包含 tag 和 digest，表示所有需同步的镜像保持其镜像 tag 或者 digest 不变 95 | 7. 目标镜像 url 可以包含多个 tag 或者 digest，数量必须与源镜像 url 中的 tag 数量相同，此时，同步后的镜像 tag 会被修改成目标镜像 url 中指定的镜像 tag（按照从左到右顺序对应） 96 | 8. 支持同时指定多个目标镜像 url，此时 "目标镜像 url" 为数组的形式，数组的每个元素（字符串）都需要满足前面的规则 97 | 98 | 镜像同步规则文件通过 `--images` 参数传入，具体文件样例可以参考 [images.yaml](examples/images.yaml) 和 [images.json](examples/images.json)，这里以 [images.yaml](examples/images.yaml) 为例。 示例如下： 99 | 100 | ```yaml 101 | quay.io/coreos/kube-rbac-proxy: quay.io/ruohe/kube-rbac-proxy 102 | quay.io/coreos/kube-rbac-proxy:v1.0: quay.io/ruohe/kube-rbac-proxy 103 | quay.io/coreos/kube-rbac-proxy:v1.0,v2.0: quay.io/ruohe/kube-rbac-proxy 104 | quay.io/coreos/kube-rbac-proxy@sha256:14b267eb38aa85fd12d0e168fffa2d8a6187ac53a14a0212b0d4fce8d729598c: quay.io/ruohe/kube-rbac-proxy 105 | quay.io/coreos/kube-rbac-proxy:v1.1: 106 | - quay.io/ruohe/kube-rbac-proxy1 107 | - quay.io/ruohe/kube-rbac-proxy2 108 | quay.io/coreos/kube-rbac-proxy:/a+/: quay.io/ruohe/kube-rbac-proxy 109 | ``` 110 | 111 | ### 更多参数 112 | 113 | `image-syncer` 的使用比较简单，但同时也支持多个命令行参数的指定： 114 | 115 | ``` 116 | -h --help 使用说明，会打印出一些启动参数的当前默认值 117 | 118 | --config 设置用户提供的配置文件路径，使用之前需要创建此文件，默认为当前工作目录下的config.json文件。这个参数与 --auth和--images 的 119 | 作用相同，分解成两个参数可以更好地区分认证信息与镜像仓库同步规则。建议使用 --auth 和 --images. 120 | 121 | --auth 设置用户提供的认证文件所在路径，使用之前需要创建此认证文件，默认为当前工作目录下的auth.json文件 122 | 123 | --images 设置用户提供的镜像同步规则文件所在路径，使用之前需要创建此文件，默认为当前工作目录下的images.json文件 124 | 125 | --log 打印出来的log文件路径，默认打印到标准错误输出，如果将日志打印到文件将不会有命令行输出，此时需要通过cat对应的日志文件查看 126 | 127 | --proc 并发数，进行镜像同步的并发goroutine数量，默认为5 128 | 129 | --retries 失败同步任务的重试次数，默认为2，重试会在所有任务都被执行一遍之后开始，并且也会重新尝试对应次数生成失败任务的生成。一些偶尔出现的网络错误比如io timeout、TLS handshake timeout，都可以通过设置重试次数来减少失败的任务数量 130 | 131 | --os 用来过滤源 tag 的 os 列表，为空则没有任何过滤要求，只对非 docker v2 schema1 media 类型的镜像格式有效 132 | 133 | --arch 用来过滤源 tag 的 architecture 列表，为空则没有任何过滤要求 134 | 135 | --force 同步已经存在的、被忽略的镜像，这个操作会更新已存在镜像的时间戳 136 | ``` 137 | 138 | ### FAQs 139 | 140 | 同步中常见的问题汇总在[FAQs 文档](./FAQs.md)中 141 | 142 | ## Star History 143 | 144 | [![Star History Chart](https://api.star-history.com/svg?repos=AliyunContainerService/image-syncer&type=Date)](https://star-history.com/#AliyunContainerService/image-syncer) 145 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # image-syncer 2 | 3 | ![workflow check](https://github.com/AliyunContainerService/image-syncer/actions/workflows/check.yml/badge.svg) 4 | ![workflow build](https://github.com/AliyunContainerService/image-syncer/actions/workflows/synctest.yml/badge.svg) 5 | [![Version](https://img.shields.io/github/v/release/AliyunContainerService/image-syncer)](https://github.com/AliyunContainerService/image-syncer/releases) 6 | [![Go Report Card](https://goreportcard.com/badge/github.com/AliyunContainerService/image-syncer)](https://goreportcard.com/report/github.com/AliyunContainerService/image-syncer) 7 | [![Github All Releases](https://img.shields.io/github/downloads/AliyunContainerService/image-syncer/total.svg)](https://api.github.com/repos/AliyunContainerService/image-syncer/releases) 8 | [![codecov](https://codecov.io/gh/AliyunContainerService/image-syncer/graph/badge.svg)](https://codecov.io/gh/AliyunContainerService/image-syncer) 9 | [![License](https://img.shields.io/github/license/AliyunContainerService/image-syncer)](https://www.apache.org/licenses/LICENSE-2.0.html) 10 | 11 | `image-syncer` is a docker registry tools. With `image-syncer` you can synchronize docker images from some source registries to target registries, which include most popular public docker registry services. 12 | 13 | English | [简体中文](./README-zh_CN.md) 14 | 15 | ## Features 16 | 17 | - Support for many-to-many registry synchronization 18 | - Supports docker registry services based on Docker Registry V2 (e.g., Alibaba Cloud Container Registry Service, Docker Hub, Quay.io, Harbor, etc.) 19 | - Network & Memory Only, doesn't rely on any large disk storage, fast synchronization 20 | - Incremental Synchronization, ignore unchanged images automatically 21 | - BloB-Level Concurrent Synchronization, adjustable goroutine numbers 22 | - Automatic Retries of Failed Sync Tasks, to resolve the network problems (rate limit, etc.) while synchronizing 23 | - Doesn't rely on Docker daemon or other programs 24 | 25 | ## Usage 26 | 27 | ### GitHub Action 28 | 29 | You can use [image-sync-action](https://github.com/marketplace/actions/image-sync-action) to try image-syncer online without paying for any machine resources. 30 | 31 | ### Install image-syncer 32 | 33 | You can download the latest binary release [here](https://github.com/AliyunContainerService/image-syncer/releases) 34 | 35 | ### Compile Manually 36 | 37 | ```bash 38 | go get github.com/AliyunContainerService/image-syncer 39 | cd $GOPATH/github.com/AliyunContainerService/image-syncer 40 | 41 | # This will create a binary file named image-syncer 42 | make 43 | ``` 44 | 45 | ### Example 46 | 47 | ```bash 48 | # Get usage information 49 | ./image-syncer -h 50 | 51 | ./image-syncer --proc=6 --auth=./auth.json --images=./images.json --retries=3 52 | ``` 53 | 54 | ### Configure Files 55 | 56 | Image-syncer supports `--auth` and `--images` flag for passing authentication file and image sync configuration file, both of which supports YAML and JSON format. Seperate authentication information is more flexible to reuse it in different sync missions. 57 | 58 | > The older version (< v1.2.0) of configuration file is still supported via `--config` flag, you can find the example in [config.yaml](examples/config.yaml) and [config.json](examples/config.json). 59 | 60 | #### Authentication file 61 | 62 | Authentication file holds all the authentication information for each registry. For each registry (or namespace), there is a object which contains username and password. For each images sync rule in image sync configuration file, image-syncer will try to find a match in all the authentication information and use the best(longest) fit one. Access will be anonymous if no authentication information is found. 63 | 64 | You can find the example in [auth.yaml](examples/auth.yaml) and [auth.json](examples/auth.json), here we use [auth.yaml](examples/auth.yaml) for explaination: 65 | 66 | ```yaml 67 | quay.io: # This "registry" or "registry/namespace" string should be the same as registry or registry/namespace used below in image sync rules. And if an url match multiple objects, the "registry/namespace" string will actually be used. 68 | username: xxx 69 | password: xxxxxxxxx 70 | insecure: true # Optional, "insecure" field needs to be true if this registry is a http service, default value is false. 71 | registry.cn-beijing.aliyuncs.com: 72 | username: xxx # Optional, if the value string is a format of "${env}" or "$env", use the "env" environment variables as username. 73 | password: xxxxxxxxx # Optional, if the value string is a format of "${env}" or "$env", use the "env" environment variables as password. 74 | docker.io: 75 | username: "${env}" 76 | password: "$env" 77 | quay.io/coreos: 78 | username: abc 79 | password: xxxxxxxxx 80 | insecure: true 81 | ``` 82 | 83 | #### Image sync configuration file 84 | 85 | Image sync configuration file defines all the image sync rules. Each rule is a key/value pair, of which the key refers to "the source images url" and the value refers to "the destination images url". The source/destination images url is mostly the same with the url we use 86 | in `docker pull/push` commands, but still something different in the "tags and digest" part: 87 | 88 | 1. Neither of the source images url and the destination images url should be empty. 89 | 2. If the source images url contains no tags or digest, all the tags of source repository will be synced. 90 | 3. The source images url can have more than one tags, which should be seperated by comma, only the specified tags will be synced. 91 | 4. The source images url can have at most one digest, and the destination images url should only have no digest or the same digest at the same time. 92 | 5. The "tags" part of source images url can be a regular expression which needs to have an additional prefix and suffix string `/`. All the tags of source repository that matches the regular expression will be synced. Multiple regular expressions is not supported. 93 | 6. If the destination images url has no digest or tags, it means the source images will keep the same tags or digest after being synced. 94 | 7. The destination images url can have more than one tags, the number of which must be the same with the tags in the source images url, then all the source images' tags will be changed to a new one (correspond from left to right). 95 | 8. The "destination images url" can also be an array, each of which follows the rules above. 96 | 97 | You can find the example in [images.yaml](examples/images.yaml) and [images.json](examples/images.json), here we use [images.yaml](examples/images.yaml) for explaination: 98 | 99 | ```yaml 100 | quay.io/coreos/kube-rbac-proxy: quay.io/ruohe/kube-rbac-proxy 101 | quay.io/coreos/kube-rbac-proxy:v1.0: quay.io/ruohe/kube-rbac-proxy 102 | quay.io/coreos/kube-rbac-proxy:v1.0,v2.0: quay.io/ruohe/kube-rbac-proxy 103 | quay.io/coreos/kube-rbac-proxy@sha256:14b267eb38aa85fd12d0e168fffa2d8a6187ac53a14a0212b0d4fce8d729598c: quay.io/ruohe/kube-rbac-proxy 104 | quay.io/coreos/kube-rbac-proxy:v1.1: 105 | - quay.io/ruohe/kube-rbac-proxy1 106 | - quay.io/ruohe/kube-rbac-proxy2 107 | quay.io/coreos/kube-rbac-proxy:/a+/: quay.io/ruohe/kube-rbac-proxy 108 | ``` 109 | 110 | ### Parameters 111 | 112 | ``` 113 | -h --help Usage information 114 | 115 | --config Set the path of config file, this file need to be created before starting synchronization, default 116 | config file is at "current/working/directory/config.json". (This flag can be replaced with flag --auth 117 | and --images which for better orgnization.) 118 | 119 | --auth Set the path of authentication file, this file need to be created before starting synchronization, default 120 | config file is at "current/working/directory/auth.json". This flag need to be pair used with --images. 121 | 122 | --images Set the path of image rules file, this file need to be created before starting synchronization, default 123 | config file is at "current/working/directory/images.json". This flag need to be pair used with --auth. 124 | 125 | --log Set the path of log file, logs will be printed to Stderr by default 126 | 127 | --proc Number of goroutines, default value is 5 128 | 129 | --retries Times to retry failed tasks, default value is 2, the retries of failed tasks will start after all the tasks 130 | are executed once, this can resolve most occasional network problems during synchronization 131 | 132 | --os OS list to filter source tags, not works for docker v2 schema1 media, takes no effect if empty 133 | 134 | --arch Architecture list to filter source tags, takes no effect if empty 135 | 136 | --force Force update manifest whether the destination manifest exists 137 | ``` 138 | 139 | ### FAQs 140 | 141 | Frequently asked questions are listed in [FAQs](./FAQs.md) 142 | 143 | ## Star History 144 | 145 | [![Star History Chart](https://api.star-history.com/svg?repos=AliyunContainerService/image-syncer&type=Date)](https://star-history.com/#AliyunContainerService/image-syncer) 146 | -------------------------------------------------------------------------------- /cmd/image-syncer.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/AliyunContainerService/image-syncer/pkg/client" 8 | "github.com/AliyunContainerService/image-syncer/pkg/utils" 9 | 10 | "github.com/spf13/cobra" 11 | ) 12 | 13 | var ( 14 | logPath, configFile, authFile, imagesFile, successImagesFile, outputImagesFormat string 15 | 16 | procNum, retries int 17 | 18 | osFilterList, archFilterList []string 19 | 20 | forceUpdate bool 21 | ) 22 | 23 | // RootCmd describes "image-syncer" command 24 | var RootCmd = &cobra.Command{ 25 | Use: "image-syncer", 26 | Aliases: []string{"image-syncer"}, 27 | Short: "A docker registry image synchronization tool", 28 | Long: `A Fast and Flexible docker registry image synchronization tool implement by Go. 29 | 30 | Complete documentation is available at https://github.com/AliyunContainerService/image-syncer`, 31 | RunE: func(cmd *cobra.Command, args []string) error { 32 | cmd.SilenceErrors = true 33 | 34 | // work starts here 35 | client, err := client.NewSyncClient(configFile, authFile, imagesFile, logPath, successImagesFile, outputImagesFormat, 36 | procNum, retries, utils.RemoveEmptyItems(osFilterList), utils.RemoveEmptyItems(archFilterList), forceUpdate) 37 | if err != nil { 38 | return fmt.Errorf("init sync client error: %v", err) 39 | } 40 | 41 | cmd.SilenceUsage = true 42 | return client.Run() 43 | }, 44 | } 45 | 46 | func init() { 47 | RootCmd.PersistentFlags().StringVar(&configFile, "config", "", "config file path. This flag is deprecated and will be removed in the future. Please use --auth and --images instead.") 48 | RootCmd.PersistentFlags().StringVar(&authFile, "auth", "", "auth file path. This flag need to be pair used with --images.") 49 | RootCmd.PersistentFlags().StringVar(&imagesFile, "images", "", "images file path. This flag need to be pair used with --auth") 50 | RootCmd.PersistentFlags().StringVar(&logPath, "log", "", "log file path (default in os.Stderr)") 51 | RootCmd.PersistentFlags().IntVarP(&procNum, "proc", "p", 5, "numbers of working goroutines") 52 | RootCmd.PersistentFlags().IntVarP(&retries, "retries", "r", 2, "times to retry failed task") 53 | RootCmd.PersistentFlags().StringArrayVar(&osFilterList, "os", []string{}, "os list to filter source tags, not works for docker v2 schema1 and OCI media") 54 | RootCmd.PersistentFlags().StringArrayVar(&archFilterList, "arch", []string{}, "architecture list to filter source tags, not works for OCI media") 55 | RootCmd.PersistentFlags().BoolVar(&forceUpdate, "force", false, "force update manifest whether the destination manifest exists") 56 | RootCmd.PersistentFlags().StringVar(&successImagesFile, "output-success-images", "", "output success images in a new file") 57 | RootCmd.PersistentFlags().StringVar(&outputImagesFormat, "output-images-format", "yaml", "success images output format, json or yaml") 58 | } 59 | 60 | // Execute executes the RootCmd 61 | func Execute() { 62 | if err := RootCmd.Execute(); err != nil { 63 | fmt.Println(err) 64 | os.Exit(-1) 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /examples/auth.json: -------------------------------------------------------------------------------- 1 | { 2 | "quay.io": { 3 | "username": "xxx", 4 | "password": "xxxxxxxxx", 5 | "insecure": true 6 | }, 7 | "registry.cn-beijing.aliyuncs.com": { 8 | "username": "xxx", 9 | "password": "xxxxxxxxx" 10 | }, 11 | "docker.io": { 12 | "username": "xxx", 13 | "password": "xxxxxxxxxx" 14 | }, 15 | "quay.io/coreos": { 16 | "username": "abc", 17 | "password": "xxxxxxxxx", 18 | "insecure": true 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /examples/auth.yaml: -------------------------------------------------------------------------------- 1 | quay.io: 2 | username: xxx 3 | password: xxxxxxxxx 4 | insecure: true 5 | registry.cn-beijing.aliyuncs.com: 6 | username: xxx 7 | password: xxxxxxxxx 8 | docker.io: 9 | username: xxx 10 | password: xxxxxxxxxx 11 | quay.io/coreos: 12 | username: abc 13 | password: xxxxxxxxx 14 | insecure: true 15 | -------------------------------------------------------------------------------- /examples/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "auth": { 3 | "quay.io": { 4 | "username": "xxx", 5 | "password": "xxxxxxxxx", 6 | "insecure": true 7 | }, 8 | "registry.cn-beijing.aliyuncs.com": { 9 | "username": "xxx", 10 | "password": "xxxxxxxxx" 11 | }, 12 | "docker.io": { 13 | "username": "xxx", 14 | "password": "xxxxxxxxxx" 15 | }, 16 | "quay.io/coreos": { 17 | "username": "abc", 18 | "password": "xxxxxxxxx", 19 | "insecure": true 20 | } 21 | }, 22 | "images": { 23 | "quay.io/coreos/kube-rbac-proxy": "quay.io/ruohe/kube-rbac-proxy", 24 | 25 | "quay.io/coreos/kube-rbac-proxy:v1.0": "quay.io/ruohe/kube-rbac-proxy", 26 | 27 | "quay.io/coreos/kube-rbac-proxy:v1.0,v2.0": "quay.io/ruohe/kube-rbac-proxy" 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /examples/config.yaml: -------------------------------------------------------------------------------- 1 | auth: 2 | quay.io: 3 | username: xxx 4 | password: xxxxxxxxx 5 | insecure: true 6 | registry.cn-beijing.aliyuncs.com: 7 | username: xxx 8 | password: xxxxxxxxx 9 | docker.io: 10 | username: xxx 11 | password: xxxxxxxxxx 12 | quay.io/coreos: 13 | username: abc 14 | password: xxxxxxxxx 15 | insecure: true 16 | images: 17 | quay.io/coreos/kube-rbac-proxy: quay.io/ruohe/kube-rbac-proxy 18 | quay.io/coreos/kube-rbac-proxy:v1.0: quay.io/ruohe/kube-rbac-proxy 19 | quay.io/coreos/kube-rbac-proxy:v1.0,v2.0: quay.io/ruohe/kube-rbac-proxy -------------------------------------------------------------------------------- /examples/images.json: -------------------------------------------------------------------------------- 1 | { 2 | "quay.io/coreos/kube-rbac-proxy": "quay.io/ruohe/kube-rbac-proxy", 3 | "quay.io/coreos/kube-rbac-proxy:v1.0": "quay.io/ruohe/kube-rbac-proxy", 4 | "quay.io/coreos/kube-rbac-proxy:v1.0,v2.0": "quay.io/ruohe/kube-rbac-proxy", 5 | "quay.io/coreos/kube-rbac-proxy:v1.1": ["quay.io/ruohe/kube-rbac-proxy1", "quay.io/ruohe/kube-rbac-proxy2"], 6 | "quay.io/coreos/kube-rbac-proxy:/a+/": "quay.io/ruohe/kube-rbac-proxy" 7 | } 8 | -------------------------------------------------------------------------------- /examples/images.yaml: -------------------------------------------------------------------------------- 1 | quay.io/coreos/kube-rbac-proxy: quay.io/ruohe/kube-rbac-proxy 2 | quay.io/coreos/kube-rbac-proxy:v1.0: quay.io/ruohe/kube-rbac-proxy 3 | quay.io/coreos/kube-rbac-proxy:v1.0,v2.0: quay.io/ruohe/kube-rbac-proxy 4 | quay.io/coreos/kube-rbac-proxy@sha256:14b267eb38aa85fd12d0e168fffa2d8a6187ac53a14a0212b0d4fce8d729598c: quay.io/ruohe/kube-rbac-proxy 5 | quay.io/coreos/kube-rbac-proxy:v1.1: 6 | - quay.io/ruohe/kube-rbac-proxy1 7 | - quay.io/ruohe/kube-rbac-proxy2 8 | quay.io/coreos/kube-rbac-proxy:/a+/: quay.io/ruohe/kube-rbac-proxy -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/AliyunContainerService/image-syncer 2 | 3 | go 1.20 4 | 5 | require ( 6 | github.com/containers/image/v5 v5.29.0 7 | github.com/docker/go-units v0.5.0 8 | github.com/fatih/color v1.16.0 9 | github.com/opencontainers/go-digest v1.0.0 10 | github.com/opencontainers/image-spec v1.1.0-rc5 11 | github.com/panjf2000/ants/v2 v2.10.0 12 | github.com/sirupsen/logrus v1.9.3 13 | github.com/spf13/cobra v1.8.0 14 | github.com/stretchr/testify v1.8.4 15 | github.com/tidwall/gjson v1.17.0 16 | golang.org/x/oauth2 v0.15.0 17 | gopkg.in/yaml.v2 v2.4.0 18 | ) 19 | 20 | require ( 21 | cloud.google.com/go/compute v1.23.3 // indirect 22 | cloud.google.com/go/compute/metadata v0.2.3 // indirect 23 | github.com/BurntSushi/toml v1.3.2 // indirect 24 | github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01 // indirect 25 | github.com/containers/ocicrypt v1.1.9 // indirect 26 | github.com/containers/storage v1.51.0 // indirect 27 | github.com/davecgh/go-spew v1.1.1 // indirect 28 | github.com/distribution/reference v0.5.0 // indirect 29 | github.com/docker/distribution v2.8.3+incompatible // indirect 30 | github.com/docker/docker v24.0.7+incompatible // indirect 31 | github.com/docker/docker-credential-helpers v0.8.0 // indirect 32 | github.com/docker/go-connections v0.4.0 // indirect 33 | github.com/golang/protobuf v1.5.3 // indirect 34 | github.com/gorilla/mux v1.8.1 // indirect 35 | github.com/hashicorp/errwrap v1.1.0 // indirect 36 | github.com/hashicorp/go-multierror v1.1.1 // indirect 37 | github.com/inconshreveable/mousetrap v1.1.0 // indirect 38 | github.com/json-iterator/go v1.1.12 // indirect 39 | github.com/klauspost/compress v1.17.4 // indirect 40 | github.com/klauspost/pgzip v1.2.6 // indirect 41 | github.com/mattn/go-colorable v0.1.13 // indirect 42 | github.com/mattn/go-isatty v0.0.20 // indirect 43 | github.com/moby/sys/mountinfo v0.7.1 // indirect 44 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 45 | github.com/modern-go/reflect2 v1.0.2 // indirect 46 | github.com/opencontainers/runc v1.1.10 // indirect 47 | github.com/opencontainers/runtime-spec v1.1.0 // indirect 48 | github.com/pkg/errors v0.9.1 // indirect 49 | github.com/pmezard/go-difflib v1.0.0 // indirect 50 | github.com/spf13/pflag v1.0.5 // indirect 51 | github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect 52 | github.com/tidwall/match v1.1.1 // indirect 53 | github.com/tidwall/pretty v1.2.1 // indirect 54 | github.com/ulikunitz/xz v0.5.11 // indirect 55 | github.com/vbatts/tar-split v0.11.5 // indirect 56 | golang.org/x/exp v0.0.0-20231206192017-f3f8817b8deb // indirect 57 | golang.org/x/sync v0.7.0 // indirect 58 | golang.org/x/sys v0.15.0 // indirect 59 | google.golang.org/appengine v1.6.8 // indirect 60 | google.golang.org/protobuf v1.31.0 // indirect 61 | gopkg.in/yaml.v3 v3.0.1 // indirect 62 | ) 63 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | cloud.google.com/go/compute v1.23.3 h1:6sVlXXBmbd7jNX0Ipq0trII3e4n1/MsADLK6a+aiVlk= 2 | cloud.google.com/go/compute v1.23.3/go.mod h1:VCgBUoMnIVIR0CscqQiPJLAG25E3ZRZMzcFZeQ+h8CI= 3 | cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= 4 | cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= 5 | github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= 6 | github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= 7 | github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= 8 | github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= 9 | github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= 10 | github.com/containers/image/v5 v5.29.0 h1:9+nhS/ZM7c4Kuzu5tJ0NMpxrgoryOJ2HAYTgG8Ny7j4= 11 | github.com/containers/image/v5 v5.29.0/go.mod h1:kQ7qcDsps424ZAz24thD+x7+dJw1vgur3A9tTDsj97E= 12 | github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01 h1:Qzk5C6cYglewc+UyGf6lc8Mj2UaPTHy/iF2De0/77CA= 13 | github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01/go.mod h1:9rfv8iPl1ZP7aqh9YA68wnZv2NUDbXdcdPHVz0pFbPY= 14 | github.com/containers/ocicrypt v1.1.9 h1:2Csfba4jse85Raxk5HIyEk8OwZNjRvfkhEGijOjIdEM= 15 | github.com/containers/ocicrypt v1.1.9/go.mod h1:dTKx1918d8TDkxXvarscpNVY+lyPakPNFN4jwA9GBys= 16 | github.com/containers/storage v1.51.0 h1:AowbcpiWXzAjHosKz7MKvPEqpyX+ryZA/ZurytRrFNA= 17 | github.com/containers/storage v1.51.0/go.mod h1:ybl8a3j1PPtpyaEi/5A6TOFs+5TrEyObeKJzVtkUlfc= 18 | github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= 19 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 20 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 21 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 22 | github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0= 23 | github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= 24 | github.com/docker/cli v24.0.7+incompatible h1:wa/nIwYFW7BVTGa7SWPVyyXU9lgORqUb1xfI36MSkFg= 25 | github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk= 26 | github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= 27 | github.com/docker/docker v24.0.7+incompatible h1:Wo6l37AuwP3JaMnZa226lzVXGA3F9Ig1seQen0cKYlM= 28 | github.com/docker/docker v24.0.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= 29 | github.com/docker/docker-credential-helpers v0.8.0 h1:YQFtbBQb4VrpoPxhFuzEBPQ9E16qz5SpHLS+uswaCp8= 30 | github.com/docker/docker-credential-helpers v0.8.0/go.mod h1:UGFXcuoQ5TxPiB54nHOZ32AWRqQdECoh/Mg0AlEYb40= 31 | github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= 32 | github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= 33 | github.com/docker/go-metrics v0.0.1 h1:AgB/0SvBxihN0X8OR4SjsblXkbMvalQ8cjmtKQ2rQV8= 34 | github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= 35 | github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= 36 | github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= 37 | github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= 38 | github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= 39 | github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= 40 | github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= 41 | github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= 42 | github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 43 | github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= 44 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 45 | github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= 46 | github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= 47 | github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= 48 | github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= 49 | github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= 50 | github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= 51 | github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= 52 | github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= 53 | github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= 54 | github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= 55 | github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= 56 | github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= 57 | github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= 58 | github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= 59 | github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= 60 | github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= 61 | github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= 62 | github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= 63 | github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= 64 | github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= 65 | github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= 66 | github.com/moby/sys/mountinfo v0.7.1 h1:/tTvQaSJRr2FshkhXiIpux6fQ2Zvc4j7tAhMTStAG2g= 67 | github.com/moby/sys/mountinfo v0.7.1/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI= 68 | github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= 69 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 70 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= 71 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 72 | github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= 73 | github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= 74 | github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= 75 | github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= 76 | github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= 77 | github.com/opencontainers/image-spec v1.1.0-rc5 h1:Ygwkfw9bpDvs+c9E34SdgGOj41dX/cbdlwvlWt0pnFI= 78 | github.com/opencontainers/image-spec v1.1.0-rc5/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8= 79 | github.com/opencontainers/runc v1.1.10 h1:EaL5WeO9lv9wmS6SASjszOeQdSctvpbu0DdBQBizE40= 80 | github.com/opencontainers/runc v1.1.10/go.mod h1:+/R6+KmDlh+hOO8NkjmgkG9Qzvypzk0yXxAPYYR65+M= 81 | github.com/opencontainers/runtime-spec v1.1.0 h1:HHUyrt9mwHUjtasSbXSMvs4cyFxh+Bll4AjJ9odEGpg= 82 | github.com/opencontainers/runtime-spec v1.1.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= 83 | github.com/panjf2000/ants/v2 v2.10.0 h1:zhRg1pQUtkyRiOFo2Sbqwjp0GfBNo9cUY2/Grpx1p+8= 84 | github.com/panjf2000/ants/v2 v2.10.0/go.mod h1:7ZxyxsqE4vvW0M7LSD8aI3cKwgFhBHbxnlN8mDqHa1I= 85 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 86 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 87 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 88 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 89 | github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= 90 | github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= 91 | github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= 92 | github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI= 93 | github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= 94 | github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= 95 | github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= 96 | github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= 97 | github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= 98 | github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= 99 | github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= 100 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 101 | github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= 102 | github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= 103 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 104 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 105 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 106 | github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= 107 | github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= 108 | github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= 109 | github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= 110 | github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 h1:kdXcSzyDtseVEc4yCz2qF8ZrQvIDBJLl4S1c3GCXmoI= 111 | github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= 112 | github.com/tidwall/gjson v1.17.0 h1:/Jocvlh98kcTfpN2+JzGQWQcqrPQwDrVEMApx/M5ZwM= 113 | github.com/tidwall/gjson v1.17.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= 114 | github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= 115 | github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= 116 | github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= 117 | github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= 118 | github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= 119 | github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8= 120 | github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= 121 | github.com/vbatts/tar-split v0.11.5 h1:3bHCTIheBm1qFTcgh9oPu+nNBtX+XJIupG/vacinCts= 122 | github.com/vbatts/tar-split v0.11.5/go.mod h1:yZbwRsSeGjusneWgA781EKej9HF8vme8okylkAeNKLk= 123 | github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= 124 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 125 | golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= 126 | golang.org/x/exp v0.0.0-20231206192017-f3f8817b8deb h1:c0vyKkb6yr3KR7jEfJaOSv4lG7xPkbN6r52aJz1d8a8= 127 | golang.org/x/exp v0.0.0-20231206192017-f3f8817b8deb/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= 128 | golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= 129 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 130 | golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= 131 | golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= 132 | golang.org/x/oauth2 v0.15.0 h1:s8pnnxNVzjWyrvYdFUQq5llS1PX2zhPXmccZv99h7uQ= 133 | golang.org/x/oauth2 v0.15.0/go.mod h1:q48ptWNTY5XWf+JNten23lcvHpLJ0ZSxF5ttTHKVCAM= 134 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 135 | golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 136 | golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= 137 | golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= 138 | golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= 139 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 140 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 141 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 142 | golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 143 | golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 144 | golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 145 | golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 146 | golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 147 | golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= 148 | golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 149 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 150 | golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= 151 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 152 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 153 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= 154 | golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= 155 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 156 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 157 | golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= 158 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 159 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 160 | google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= 161 | google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= 162 | google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= 163 | google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= 164 | google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= 165 | google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= 166 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 167 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 168 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= 169 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 170 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 171 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 172 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 173 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/AliyunContainerService/image-syncer/cmd" 5 | ) 6 | 7 | func main() { 8 | cmd.Execute() 9 | } 10 | -------------------------------------------------------------------------------- /pkg/client/client.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "os" 7 | "time" 8 | 9 | "github.com/fatih/color" 10 | "github.com/panjf2000/ants/v2" 11 | "github.com/sirupsen/logrus" 12 | "gopkg.in/yaml.v2" 13 | 14 | "github.com/AliyunContainerService/image-syncer/pkg/concurrent" 15 | "github.com/AliyunContainerService/image-syncer/pkg/task" 16 | "github.com/AliyunContainerService/image-syncer/pkg/utils/types" 17 | ) 18 | 19 | // Client describes a synchronization client 20 | type Client struct { 21 | taskList *concurrent.List 22 | failedTaskList *concurrent.List 23 | 24 | taskCounter *concurrent.Counter 25 | failedTaskCounter *concurrent.Counter 26 | 27 | // TODO: print failed images? but this might be difficult because image sync rule might be illegal 28 | successImagesList *concurrent.ImageList 29 | successImagesFile, outputImagesFormat string 30 | 31 | config *Config 32 | 33 | routineNum int 34 | retries int 35 | logger *logrus.Logger 36 | 37 | forceUpdate bool 38 | } 39 | 40 | // NewSyncClient creates a synchronization client 41 | func NewSyncClient(configFile, authFile, imagesFile, logFile, successImagesFile, outputImagesFormat string, 42 | routineNum, retries int, osFilterList, archFilterList []string, forceUpdate bool) (*Client, error) { 43 | 44 | logger := NewFileLogger(logFile) 45 | 46 | config, err := NewSyncConfig(configFile, authFile, imagesFile, osFilterList, archFilterList, logger) 47 | if err != nil { 48 | return nil, fmt.Errorf("generate config error: %v", err) 49 | } 50 | 51 | return &Client{ 52 | taskList: concurrent.NewList(), 53 | failedTaskList: concurrent.NewList(), 54 | 55 | taskCounter: concurrent.NewCounter(0, 0), 56 | failedTaskCounter: concurrent.NewCounter(0, 0), 57 | 58 | successImagesList: concurrent.NewImageList(), 59 | successImagesFile: successImagesFile, 60 | outputImagesFormat: outputImagesFormat, 61 | 62 | config: config, 63 | routineNum: routineNum, 64 | retries: retries, 65 | logger: logger, 66 | 67 | forceUpdate: forceUpdate, 68 | }, nil 69 | } 70 | 71 | // Run is main function of a synchronization client 72 | func (c *Client) Run() error { 73 | start := time.Now() 74 | 75 | imageList, err := types.NewImageList(c.config.ImageList) 76 | if err != nil { 77 | return fmt.Errorf("failed to get image list: %v", err) 78 | } 79 | 80 | for source, destList := range imageList { 81 | for _, dest := range destList { 82 | // TODO: support multiple destinations for one task 83 | ruleTask, err := task.NewRuleTask(source, dest, 84 | c.config.osFilterList, c.config.archFilterList, 85 | func(repository string) types.Auth { 86 | auth, exist := c.config.GetAuth(repository) 87 | if !exist { 88 | c.logger.Infof("Auth information not found for %v, access will be anonymous.", repository) 89 | } 90 | return auth 91 | }, c.forceUpdate) 92 | if err != nil { 93 | return fmt.Errorf("failed to generate rule task for %s -> %s: %v", source, dest, err) 94 | } 95 | 96 | c.taskList.PushBack(ruleTask) 97 | c.taskCounter.IncreaseTotal() 98 | } 99 | } 100 | 101 | routinePool, _ := ants.NewPoolWithFunc(c.routineNum, func(i interface{}) { 102 | tTask, ok := i.(task.Task) 103 | if !ok { 104 | c.logger.Errorf("invalid task %v", i) 105 | return 106 | } 107 | 108 | nextTasks, message, err := tTask.Run() 109 | count, total := c.taskCounter.Increase() 110 | finishedNumString := color.New(color.FgGreen).Sprintf("%d", count) 111 | totalNumString := color.New(color.FgGreen).Sprintf("%d", total) 112 | 113 | if err != nil { 114 | c.failedTaskList.PushBack(tTask) 115 | c.failedTaskCounter.IncreaseTotal() 116 | c.logger.Errorf("Failed to executed %v: %v. Now %v/%v tasks have been processed.", tTask.String(), err, 117 | finishedNumString, totalNumString) 118 | } else { 119 | if tTask.Type() == task.ManifestType { 120 | // TODO: the ignored images will not be recorded in success images list 121 | c.successImagesList.Add(tTask.GetSource().String(), tTask.GetDestination().String()) 122 | } 123 | 124 | if len(message) != 0 { 125 | c.logger.Infof("Finish %v: %v. Now %v/%v tasks have been processed.", tTask.String(), message, 126 | finishedNumString, totalNumString) 127 | } else { 128 | c.logger.Infof("Finish %v. Now %v/%v tasks have been processed.", tTask.String(), 129 | finishedNumString, totalNumString) 130 | } 131 | } 132 | 133 | for _, t := range nextTasks { 134 | c.taskList.PushFront(t) 135 | c.taskCounter.IncreaseTotal() 136 | } 137 | }) 138 | defer routinePool.Release() 139 | 140 | if err = c.handleTasks(routinePool); err != nil { 141 | c.logger.Errorf("Failed to handle tasks: %v", err) 142 | } 143 | 144 | for times := 0; times < c.retries; times++ { 145 | c.taskCounter, c.failedTaskCounter = c.failedTaskCounter, concurrent.NewCounter(0, 0) 146 | 147 | if c.failedTaskList.Len() != 0 { 148 | c.taskList.PushBackList(c.failedTaskList) 149 | c.failedTaskList.Reset() 150 | } 151 | 152 | if c.taskList.Len() != 0 { 153 | // retry to handle task 154 | c.logger.Infof("Start to retry tasks, please wait ...") 155 | if err = c.handleTasks(routinePool); err != nil { 156 | c.logger.Errorf("Failed to handle tasks: %v", err) 157 | } 158 | } 159 | } 160 | 161 | endMsg := fmt.Sprintf("Synchronization finished, %v tasks failed, cost %v.", 162 | c.failedTaskList.Len(), time.Since(start).String()) 163 | c.logger.Infof(color.New(color.FgGreen).Sprintf(endMsg)) 164 | 165 | if len(c.successImagesFile) != 0 { 166 | file, err := os.OpenFile(c.successImagesFile, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0666) 167 | if err != nil { 168 | return fmt.Errorf("open file %v error: %v", c.successImagesFile, err) 169 | } 170 | 171 | if c.outputImagesFormat == "json" { 172 | encoder := json.NewEncoder(file) 173 | if err := encoder.Encode(c.successImagesList.Content()); err != nil { 174 | return fmt.Errorf("marshal success images error: %v", err) 175 | } 176 | } else { 177 | encoder := yaml.NewEncoder(file) 178 | if err := encoder.Encode(c.successImagesList.Content()); err != nil { 179 | return fmt.Errorf("marshal success images error: %v", err) 180 | } 181 | } 182 | } 183 | 184 | _, failedSyncTaskCountTotal := c.failedTaskCounter.Value() 185 | if failedSyncTaskCountTotal != 0 { 186 | return fmt.Errorf("failed tasks exist") 187 | } 188 | return nil 189 | } 190 | 191 | func (c *Client) handleTasks(routinePool *ants.PoolWithFunc) error { 192 | for { 193 | item := c.taskList.PopFront() 194 | // no more tasks need to handle 195 | if item == nil { 196 | if routinePool.Running() == 0 { 197 | break 198 | } 199 | time.Sleep(1 * time.Second) 200 | continue 201 | } 202 | 203 | if err := routinePool.Invoke(item); err != nil { 204 | return fmt.Errorf("failed to invoke routine: %v", err) 205 | } 206 | } 207 | return nil 208 | } 209 | -------------------------------------------------------------------------------- /pkg/client/config.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "os" 7 | "strings" 8 | 9 | "github.com/AliyunContainerService/image-syncer/pkg/utils/types" 10 | 11 | "github.com/sirupsen/logrus" 12 | 13 | "github.com/AliyunContainerService/image-syncer/pkg/utils" 14 | 15 | "gopkg.in/yaml.v2" 16 | ) 17 | 18 | // Config information of sync client 19 | type Config struct { 20 | // the authentication information of each registry 21 | AuthList map[string]types.Auth `json:"auth" yaml:"auth"` 22 | 23 | // a : map 24 | ImageList map[string]interface{} `json:"images" yaml:"images"` 25 | 26 | // only images with selected os can be sync 27 | osFilterList []string 28 | // only images with selected architecture can be sync 29 | archFilterList []string 30 | } 31 | 32 | // NewSyncConfig creates a Config struct 33 | func NewSyncConfig(configFile, authFilePath, imageFilePath string, 34 | osFilterList, archFilterList []string, logger *logrus.Logger) (*Config, error) { 35 | if len(configFile) == 0 && len(imageFilePath) == 0 { 36 | return nil, fmt.Errorf("neither config.json nor images.json is provided") 37 | } 38 | 39 | if len(configFile) == 0 && len(authFilePath) == 0 { 40 | logger.Warnf("[Warning] No authentication information found because neither " + 41 | "config.json nor auth.json provided, image-syncer may not work fine.") 42 | } 43 | 44 | var config Config 45 | 46 | if len(configFile) != 0 { 47 | if err := openAndDecode(configFile, &config); err != nil { 48 | return nil, fmt.Errorf("decode config file %v failed, error %v", configFile, err) 49 | } 50 | } else { 51 | if len(authFilePath) != 0 { 52 | if err := openAndDecode(authFilePath, &config.AuthList); err != nil { 53 | return nil, fmt.Errorf("decode auth file %v error: %v", authFilePath, err) 54 | } 55 | } 56 | config.AuthList = expandEnv(config.AuthList) 57 | 58 | if err := openAndDecode(imageFilePath, &config.ImageList); err != nil { 59 | return nil, fmt.Errorf("decode image file %v error: %v", imageFilePath, err) 60 | } 61 | } 62 | 63 | config.osFilterList = osFilterList 64 | config.archFilterList = archFilterList 65 | 66 | return &config, nil 67 | } 68 | 69 | // Open json file and decode into target interface 70 | func openAndDecode(filePath string, target interface{}) error { 71 | if !strings.HasSuffix(filePath, ".yaml") && 72 | !strings.HasSuffix(filePath, ".yml") && 73 | !strings.HasSuffix(filePath, ".json") { 74 | return fmt.Errorf("only one of yaml/yml/json format is supported") 75 | } 76 | 77 | if _, err := os.Stat(filePath); os.IsNotExist(err) { 78 | return fmt.Errorf("file %v not exist: %v", filePath, err) 79 | } 80 | 81 | file, err := os.OpenFile(filePath, os.O_RDONLY, 0666) 82 | if err != nil { 83 | return fmt.Errorf("open file %v error: %v", filePath, err) 84 | } 85 | 86 | if strings.HasSuffix(filePath, ".yaml") || strings.HasSuffix(filePath, ".yml") { 87 | decoder := yaml.NewDecoder(file) 88 | if err := decoder.Decode(target); err != nil { 89 | return fmt.Errorf("unmarshal config error: %v", err) 90 | } 91 | } else { 92 | decoder := json.NewDecoder(file) 93 | if err := decoder.Decode(target); err != nil { 94 | return fmt.Errorf("unmarshal config error: %v", err) 95 | } 96 | } 97 | 98 | return nil 99 | } 100 | 101 | // GetAuth gets the authentication information in Config 102 | func (c *Config) GetAuth(repository string) (types.Auth, bool) { 103 | auth := types.Auth{} 104 | prefixLen := 0 105 | exist := false 106 | 107 | for key, value := range c.AuthList { 108 | if matched := utils.RepoMathPrefix(repository, key); matched { 109 | if len(key) > prefixLen { 110 | auth = value 111 | exist = true 112 | } 113 | } 114 | } 115 | 116 | return auth, exist 117 | } 118 | 119 | func expandEnv(authMap map[string]types.Auth) map[string]types.Auth { 120 | result := make(map[string]types.Auth) 121 | 122 | for registry, auth := range authMap { 123 | pwd := os.ExpandEnv(auth.Password) 124 | name := os.ExpandEnv(auth.Username) 125 | newAuth := types.Auth{ 126 | Username: name, 127 | Password: pwd, 128 | Insecure: auth.Insecure, 129 | } 130 | result[registry] = newAuth 131 | } 132 | 133 | return result 134 | } 135 | -------------------------------------------------------------------------------- /pkg/client/logger.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "os" 5 | 6 | "github.com/sirupsen/logrus" 7 | ) 8 | 9 | const ( 10 | logTimestampFormat = "2006-01-02 15:04:05" 11 | ) 12 | 13 | // NewFileLogger creates a log file and init logger 14 | func NewFileLogger(path string) *logrus.Logger { 15 | logger := logrus.New() 16 | 17 | // disable color 18 | if len(path) != 0 { 19 | _ = os.Setenv("NO_COLOR", "true") 20 | } 21 | 22 | logger.Formatter = &logrus.TextFormatter{ 23 | FullTimestamp: true, 24 | TimestampFormat: logTimestampFormat, 25 | } 26 | 27 | if file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0666); err == nil { 28 | logger.Out = file 29 | } else { 30 | logger.Info("Failed to log to file, using default stderr") 31 | } 32 | 33 | return logger 34 | } 35 | -------------------------------------------------------------------------------- /pkg/concurrent/counter.go: -------------------------------------------------------------------------------- 1 | package concurrent 2 | 3 | import "sync" 4 | 5 | type Counter struct { 6 | sync.Mutex 7 | count int 8 | total int 9 | } 10 | 11 | func NewCounter(count, total int) *Counter { 12 | return &Counter{ 13 | count: count, 14 | total: total, 15 | } 16 | } 17 | 18 | func (c *Counter) Decrease() (int, int) { 19 | c.Lock() 20 | defer c.Unlock() 21 | 22 | if c.count > 0 { 23 | c.count-- 24 | } 25 | return c.count, c.total 26 | } 27 | 28 | func (c *Counter) Increase() (int, int) { 29 | c.Lock() 30 | defer c.Unlock() 31 | 32 | if c.count < c.total { 33 | c.count++ 34 | } 35 | return c.count, c.total 36 | } 37 | 38 | func (c *Counter) IncreaseTotal() (int, int) { 39 | c.Lock() 40 | defer c.Unlock() 41 | 42 | c.total++ 43 | return c.count, c.total 44 | } 45 | 46 | // Value return count and total 47 | func (c *Counter) Value() (int, int) { 48 | c.Lock() 49 | defer c.Unlock() 50 | 51 | return c.count, c.total 52 | } 53 | -------------------------------------------------------------------------------- /pkg/concurrent/imageList.go: -------------------------------------------------------------------------------- 1 | package concurrent 2 | 3 | import ( 4 | "sync" 5 | 6 | "github.com/AliyunContainerService/image-syncer/pkg/utils/types" 7 | ) 8 | 9 | type ImageList struct { 10 | sync.Mutex 11 | content types.ImageList 12 | } 13 | 14 | func NewImageList() *ImageList { 15 | return &ImageList{ 16 | content: types.ImageList{}, 17 | } 18 | } 19 | 20 | func (i *ImageList) Add(src, dst string) { 21 | i.Lock() 22 | defer i.Unlock() 23 | 24 | i.content.Add(src, dst) 25 | } 26 | 27 | func (i *ImageList) Query(src, dst string) bool { 28 | i.Lock() 29 | defer i.Unlock() 30 | 31 | return i.content.Query(src, dst) 32 | } 33 | 34 | func (i *ImageList) Delete(key string) { 35 | i.Lock() 36 | defer i.Unlock() 37 | 38 | delete(i.content, key) 39 | } 40 | 41 | func (i *ImageList) Rest() { 42 | i.Lock() 43 | defer i.Unlock() 44 | 45 | i.content = types.ImageList{} 46 | } 47 | 48 | func (i *ImageList) Content() types.ImageList { 49 | i.Lock() 50 | defer i.Unlock() 51 | 52 | return i.content 53 | } 54 | -------------------------------------------------------------------------------- /pkg/concurrent/list.go: -------------------------------------------------------------------------------- 1 | package concurrent 2 | 3 | import ( 4 | "container/list" 5 | "sync" 6 | ) 7 | 8 | type List struct { 9 | sync.Mutex 10 | items *list.List 11 | } 12 | 13 | func NewList() *List { 14 | return &List{ 15 | items: list.New(), 16 | } 17 | } 18 | 19 | func (l *List) PopFront() any { 20 | l.Lock() 21 | defer l.Unlock() 22 | 23 | item := l.items.Front() 24 | if item != nil { 25 | l.items.Remove(item) 26 | return item.Value 27 | } 28 | 29 | return nil 30 | } 31 | 32 | func (l *List) PushBack(value any) { 33 | l.Lock() 34 | defer l.Unlock() 35 | 36 | l.items.PushBack(value) 37 | } 38 | 39 | func (l *List) PushFront(value any) { 40 | l.Lock() 41 | defer l.Unlock() 42 | 43 | l.items.PushFront(value) 44 | } 45 | 46 | func (l *List) PushBackList(other *List) { 47 | l.Lock() 48 | defer l.Unlock() 49 | 50 | l.items.PushBackList(other.GetItems()) 51 | } 52 | 53 | func (l *List) GetItems() *list.List { 54 | l.Lock() 55 | defer l.Unlock() 56 | 57 | return l.items 58 | } 59 | 60 | func (l *List) Reset() { 61 | l.Lock() 62 | defer l.Unlock() 63 | 64 | l.items.Init() 65 | } 66 | 67 | func (l *List) Len() int { 68 | l.Lock() 69 | defer l.Unlock() 70 | 71 | return l.items.Len() 72 | } 73 | -------------------------------------------------------------------------------- /pkg/sync/constants.go: -------------------------------------------------------------------------------- 1 | package sync 2 | 3 | import "github.com/containers/image/v5/pkg/blobinfocache/none" 4 | 5 | var ( 6 | // NoCache used to disable a blobinfocache 7 | NoCache = none.NoCache 8 | ) 9 | -------------------------------------------------------------------------------- /pkg/sync/destination.go: -------------------------------------------------------------------------------- 1 | package sync 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "io" 8 | "reflect" 9 | "strings" 10 | 11 | "github.com/AliyunContainerService/image-syncer/pkg/utils/auth" 12 | 13 | "github.com/containers/image/v5/manifest" 14 | specsv1 "github.com/opencontainers/image-spec/specs-go/v1" 15 | 16 | "github.com/AliyunContainerService/image-syncer/pkg/utils" 17 | "github.com/containers/image/v5/docker" 18 | "github.com/containers/image/v5/types" 19 | "github.com/opencontainers/go-digest" 20 | ) 21 | 22 | // ImageDestination is a reference of a remote image we will push to 23 | type ImageDestination struct { 24 | ref types.ImageReference 25 | destination types.ImageDestination 26 | ctx context.Context 27 | sysctx *types.SystemContext 28 | 29 | // destination image description 30 | registry string 31 | repository string 32 | tagOrDigest string 33 | } 34 | 35 | // NewImageDestination generates an ImageDestination by repository, the repository string must include tag or digest. 36 | // If username or password is empty, access to repository will be anonymous. 37 | func NewImageDestination(registry, repository, tagOrDigest, username, password string, insecure bool) (*ImageDestination, error) { 38 | if strings.Contains(repository, ":") { 39 | return nil, fmt.Errorf("repository string should not include ':'") 40 | } 41 | 42 | // if tagOrDigest is empty, will attach to the "latest" tag 43 | destRef, err := docker.ParseReference("//" + registry + "/" + repository + utils.AttachConnectorToTagOrDigest(tagOrDigest)) 44 | if err != nil { 45 | return nil, err 46 | } 47 | 48 | var sysctx *types.SystemContext 49 | if insecure { 50 | // destination registry is http service 51 | sysctx = &types.SystemContext{ 52 | DockerInsecureSkipTLSVerify: types.OptionalBoolTrue, 53 | } 54 | } else { 55 | sysctx = &types.SystemContext{} 56 | } 57 | 58 | ctx := context.WithValue(context.Background(), utils.CTXKey("ImageDestination"), repository) 59 | if username != "" && password != "" { 60 | //fmt.Printf("Credential processing for %s/%s ...\n", registry, repository) 61 | if auth.IsGCRPermanentServiceAccountToken(registry, username) { 62 | fmt.Printf("Getting oauth2 token for %s...\n", username) 63 | token, expiry, err := auth.GCPTokenFromCreds(password) 64 | if err != nil { 65 | return nil, err 66 | } 67 | 68 | fmt.Printf("oauth2 token expiry: %s\n", expiry) 69 | password = token 70 | username = "oauth2accesstoken" 71 | } 72 | sysctx.DockerAuthConfig = &types.DockerAuthConfig{ 73 | Username: username, 74 | Password: password, 75 | } 76 | } 77 | 78 | destination, err := destRef.NewImageDestination(ctx, sysctx) 79 | if err != nil { 80 | return nil, err 81 | } 82 | 83 | return &ImageDestination{ 84 | ref: destRef, 85 | destination: destination, 86 | ctx: ctx, 87 | sysctx: sysctx, 88 | registry: registry, 89 | repository: repository, 90 | tagOrDigest: tagOrDigest, 91 | }, nil 92 | } 93 | 94 | // PushManifest push a manifest file to destination image. 95 | // If instanceDigest is not nil, it contains a digest of the specific manifest instance to write the manifest for 96 | // (when the primary manifest is a manifest list); this should always be nil if the primary manifest is not a manifest list. 97 | func (i *ImageDestination) PushManifest(manifestByte []byte, instanceDigest *digest.Digest) error { 98 | return i.destination.PutManifest(i.ctx, manifestByte, instanceDigest) 99 | } 100 | 101 | // CheckManifestChanged checks if manifest of specified tag or digest has changed. 102 | func (i *ImageDestination) CheckManifestChanged(destManifestBytes []byte, instanceDigest *digest.Digest) bool { 103 | existManifestBytes := i.GetManifest(instanceDigest) 104 | return !manifestEqual(existManifestBytes, destManifestBytes) 105 | } 106 | 107 | func (i *ImageDestination) GetManifest(instanceDigest *digest.Digest) []byte { 108 | var err error 109 | var srcRef types.ImageReference 110 | 111 | if instanceDigest != nil { 112 | manifestURL := i.registry + "/" + i.repository + utils.AttachConnectorToTagOrDigest(instanceDigest.String()) 113 | 114 | // create source to check manifest 115 | srcRef, err = docker.ParseReference("//" + manifestURL) 116 | if err != nil { 117 | return nil 118 | } 119 | } else { 120 | srcRef = i.ref 121 | } 122 | 123 | source, err := srcRef.NewImageSource(i.ctx, i.sysctx) 124 | if err != nil { 125 | // if the source cannot be created, manifest not exist 126 | return nil 127 | } 128 | 129 | tManifestByte, mineType, err := source.GetManifest(i.ctx, instanceDigest) 130 | if err != nil { 131 | // if error happens, it's considered that the manifest not exist 132 | return nil 133 | } 134 | 135 | // only for manifest list 136 | switch mineType { 137 | case manifest.DockerV2ListMediaType: 138 | manifestSchemaListObj, err := manifest.Schema2ListFromManifest(tManifestByte) 139 | if err != nil { 140 | return nil 141 | } 142 | 143 | for _, manifestDescriptorElem := range manifestSchemaListObj.Manifests { 144 | mfstBytes := i.GetManifest(&manifestDescriptorElem.Digest) 145 | if mfstBytes == nil { 146 | // cannot find sub manifest, manifest list not exist 147 | return nil 148 | } 149 | } 150 | 151 | case specsv1.MediaTypeImageIndex: 152 | ociIndexesObj, err := manifest.OCI1IndexFromManifest(tManifestByte) 153 | if err != nil { 154 | return nil 155 | } 156 | 157 | for _, manifestDescriptorElem := range ociIndexesObj.Manifests { 158 | mfstBytes := i.GetManifest(&manifestDescriptorElem.Digest) 159 | if mfstBytes == nil { 160 | // cannot find sub manifest, manifest list not exist 161 | return nil 162 | } 163 | } 164 | } 165 | 166 | return tManifestByte 167 | } 168 | 169 | // PutABlob push a blob to destination image 170 | func (i *ImageDestination) PutABlob(blob io.ReadCloser, blobInfo types.BlobInfo) error { 171 | _, err := i.destination.PutBlob(i.ctx, blob, types.BlobInfo{ 172 | Digest: blobInfo.Digest, 173 | Size: blobInfo.Size, 174 | }, NoCache, true) 175 | 176 | // io.ReadCloser need to be close 177 | defer blob.Close() 178 | 179 | return err 180 | } 181 | 182 | // CheckBlobExist checks if a blob exist for destination and reuse exist blobs 183 | func (i *ImageDestination) CheckBlobExist(blobInfo types.BlobInfo) (bool, error) { 184 | exist, _, err := i.destination.TryReusingBlob(i.ctx, types.BlobInfo{ 185 | Digest: blobInfo.Digest, 186 | Size: blobInfo.Size, 187 | }, NoCache, false) 188 | 189 | return exist, err 190 | } 191 | 192 | // Close a ImageDestination 193 | func (i *ImageDestination) Close() error { 194 | return i.destination.Close() 195 | } 196 | 197 | // GetRegistry returns the registry of a ImageDestination 198 | func (i *ImageDestination) GetRegistry() string { 199 | return i.registry 200 | } 201 | 202 | // GetRepository returns the repository of a ImageDestination 203 | func (i *ImageDestination) GetRepository() string { 204 | return i.repository 205 | } 206 | 207 | // GetTagOrDigest return the tag or digest of a ImageDestination 208 | func (i *ImageDestination) GetTagOrDigest() string { 209 | return i.tagOrDigest 210 | } 211 | 212 | func (i *ImageDestination) String() string { 213 | return i.registry + "/" + i.repository + utils.AttachConnectorToTagOrDigest(i.tagOrDigest) 214 | } 215 | 216 | func manifestEqual(m1, m2 []byte) bool { 217 | var a map[string]interface{} 218 | var b map[string]interface{} 219 | 220 | if err := json.Unmarshal(m1, &a); err != nil { 221 | //Received an unexpected manifest retrieval result, return false to trigger a fallback to the push task. 222 | return false 223 | } 224 | if err := json.Unmarshal(m2, &b); err != nil { 225 | return false 226 | } 227 | 228 | return reflect.DeepEqual(a, b) 229 | } 230 | -------------------------------------------------------------------------------- /pkg/sync/manifest.go: -------------------------------------------------------------------------------- 1 | package sync 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "strings" 7 | 8 | "github.com/opencontainers/go-digest" 9 | 10 | "github.com/containers/image/v5/manifest" 11 | specsv1 "github.com/opencontainers/image-spec/specs-go/v1" 12 | "github.com/tidwall/gjson" 13 | ) 14 | 15 | type ManifestInfo struct { 16 | Obj manifest.Manifest 17 | Digest *digest.Digest 18 | 19 | // (manifest.Manifest).Serialize() does not in general reproduce the original blob if this object was loaded from one, 20 | // even if no modifications were made! Non-list type image seems cannot use the result to push to dest 21 | // repo while it is a part of list-type image. 22 | Bytes []byte 23 | } 24 | 25 | // GenerateManifestObj returns a new manifest object along with its byte serialization, and a sub manifest object array, 26 | // and a digest array of sub manifests for list-type manifests. 27 | // For list type manifest, the origin manifest info might be modified because of platform filters, and a nil manifest 28 | // object will be returned if no sub manifest need to transport. 29 | // For non-list type manifests, which doesn't match the filters, a nil manifest object will be returned. 30 | func GenerateManifestObj(manifestBytes []byte, manifestType string, osFilterList, archFilterList []string, 31 | i *ImageSource, parent *manifest.Schema2List) (interface{}, []byte, []*ManifestInfo, error) { 32 | 33 | switch manifestType { 34 | case manifest.DockerV2Schema2MediaType: 35 | manifestObj, err := manifest.Schema2FromManifest(manifestBytes) 36 | if err != nil { 37 | return nil, nil, nil, err 38 | } 39 | 40 | // platform info stored in config blob 41 | if parent == nil && manifestObj.ConfigInfo().Digest != "" { 42 | blob, _, err := i.GetABlob(manifestObj.ConfigInfo()) 43 | if err != nil { 44 | return nil, nil, nil, err 45 | } 46 | defer blob.Close() 47 | bytes, err := io.ReadAll(blob) 48 | if err != nil { 49 | return nil, nil, nil, err 50 | } 51 | results := gjson.GetManyBytes(bytes, "architecture", "os") 52 | 53 | if !platformValidate(osFilterList, archFilterList, 54 | &manifest.Schema2PlatformSpec{Architecture: results[0].String(), OS: results[1].String()}) { 55 | return nil, nil, nil, nil 56 | } 57 | } 58 | 59 | return manifestObj, manifestBytes, nil, nil 60 | case manifest.DockerV2Schema1MediaType, manifest.DockerV2Schema1SignedMediaType: 61 | manifestObj, err := manifest.Schema1FromManifest(manifestBytes) 62 | if err != nil { 63 | return nil, nil, nil, err 64 | } 65 | 66 | // v1 only support architecture and this field is for information purposes and not currently used by the engine. 67 | if parent == nil && !platformValidate(osFilterList, archFilterList, 68 | &manifest.Schema2PlatformSpec{Architecture: manifestObj.Architecture}) { 69 | return nil, nil, nil, nil 70 | } 71 | 72 | return manifestObj, manifestBytes, nil, nil 73 | case specsv1.MediaTypeImageManifest: 74 | //TODO: platform filter? 75 | manifestObj, err := manifest.OCI1FromManifest(manifestBytes) 76 | if err != nil { 77 | return nil, nil, nil, err 78 | } 79 | return manifestObj, manifestBytes, nil, nil 80 | case manifest.DockerV2ListMediaType: 81 | var subManifestInfoSlice []*ManifestInfo 82 | 83 | manifestSchemaListObj, err := manifest.Schema2ListFromManifest(manifestBytes) 84 | if err != nil { 85 | return nil, nil, nil, err 86 | } 87 | 88 | var filteredDescriptors []manifest.Schema2ManifestDescriptor 89 | 90 | for index, manifestDescriptorElem := range manifestSchemaListObj.Manifests { 91 | // select os and arch 92 | if !platformValidate(osFilterList, archFilterList, &manifestDescriptorElem.Platform) { 93 | continue 94 | } 95 | 96 | filteredDescriptors = append(filteredDescriptors, manifestDescriptorElem) 97 | mfstBytes, mfstType, err := i.source.GetManifest(i.ctx, &manifestDescriptorElem.Digest) 98 | if err != nil { 99 | return nil, nil, nil, err 100 | } 101 | 102 | //TODO: will the sub manifest be list-type? 103 | subManifest, _, _, err := GenerateManifestObj(mfstBytes, mfstType, 104 | archFilterList, osFilterList, i, manifestSchemaListObj) 105 | if err != nil { 106 | return nil, nil, nil, err 107 | } 108 | 109 | if subManifest != nil { 110 | subManifestInfoSlice = append(subManifestInfoSlice, &ManifestInfo{ 111 | Obj: subManifest.(manifest.Manifest), 112 | 113 | // cannot use &manifestDescriptorElem.Digest here, because manifestDescriptorElem is a fixed copy object 114 | Digest: &manifestSchemaListObj.Manifests[index].Digest, 115 | Bytes: mfstBytes, 116 | }) 117 | } 118 | } 119 | 120 | // no sub manifests need to transport 121 | if len(filteredDescriptors) == 0 { 122 | return nil, nil, nil, nil 123 | } 124 | 125 | // return a new Schema2List 126 | if len(filteredDescriptors) != len(manifestSchemaListObj.Manifests) { 127 | manifestSchemaListObj.Manifests = filteredDescriptors 128 | } 129 | 130 | newManifestBytes, _ := manifestSchemaListObj.Serialize() 131 | 132 | return manifestSchemaListObj, newManifestBytes, subManifestInfoSlice, nil 133 | case specsv1.MediaTypeImageIndex: 134 | var subManifestInfoSlice []*ManifestInfo 135 | 136 | ociIndexesObj, err := manifest.OCI1IndexFromManifest(manifestBytes) 137 | if err != nil { 138 | return nil, nil, nil, err 139 | } 140 | 141 | var filteredDescriptors []specsv1.Descriptor 142 | 143 | for index, descriptor := range ociIndexesObj.Manifests { 144 | // select os and arch 145 | if !platformValidate(osFilterList, archFilterList, &manifest.Schema2PlatformSpec{ 146 | Architecture: descriptor.Platform.Architecture, 147 | OS: descriptor.Platform.OS, 148 | }) { 149 | continue 150 | } 151 | 152 | filteredDescriptors = append(filteredDescriptors, descriptor) 153 | 154 | mfstBytes, mfstType, innerErr := i.source.GetManifest(i.ctx, &descriptor.Digest) 155 | if innerErr != nil { 156 | return nil, nil, nil, innerErr 157 | } 158 | 159 | //TODO: will the sub manifest be list-type? 160 | subManifest, _, _, innerErr := GenerateManifestObj(mfstBytes, mfstType, 161 | archFilterList, osFilterList, i, nil) 162 | if innerErr != nil { 163 | return nil, nil, nil, err 164 | } 165 | 166 | if subManifest != nil { 167 | subManifestInfoSlice = append(subManifestInfoSlice, &ManifestInfo{ 168 | Obj: subManifest.(manifest.Manifest), 169 | 170 | // cannot use &descriptor.Digest here, because descriptor is a fixed copy object 171 | Digest: &ociIndexesObj.Manifests[index].Digest, 172 | Bytes: mfstBytes, 173 | }) 174 | } 175 | } 176 | 177 | // no sub manifests need to transport 178 | if len(filteredDescriptors) == 0 { 179 | return nil, nil, nil, nil 180 | } 181 | 182 | // return a new Schema2List 183 | if len(filteredDescriptors) != len(ociIndexesObj.Manifests) { 184 | ociIndexesObj.Manifests = filteredDescriptors 185 | } 186 | 187 | newManifestBytes, _ := ociIndexesObj.Serialize() 188 | 189 | return ociIndexesObj, newManifestBytes, subManifestInfoSlice, nil 190 | default: 191 | return nil, nil, nil, fmt.Errorf("unsupported manifest type: %v", manifestType) 192 | } 193 | } 194 | 195 | // compare first:second to pat, second is optional 196 | func colonMatch(pat string, first string, second string) bool { 197 | if strings.Index(pat, first) != 0 { 198 | return false 199 | } 200 | 201 | return len(first) == len(pat) || (pat[len(first)] == ':' && pat[len(first)+1:] == second) 202 | } 203 | 204 | // Match platform selector according to the source image and its platform. 205 | // If platform.OS is not specified, the manifest will never be filtered, the same with platform.Architecture. 206 | func platformValidate(osFilterList, archFilterList []string, platform *manifest.Schema2PlatformSpec) bool { 207 | osMatched := true 208 | archMatched := true 209 | 210 | if len(osFilterList) != 0 && platform.OS != "" { 211 | osMatched = false 212 | for _, o := range osFilterList { 213 | // match os:osversion 214 | if colonMatch(o, platform.OS, platform.OSVersion) { 215 | osMatched = true 216 | } 217 | } 218 | } 219 | 220 | if len(archFilterList) != 0 && platform.Architecture != "" { 221 | archMatched = false 222 | for _, a := range archFilterList { 223 | // match architecture:variant 224 | if colonMatch(a, platform.Architecture, platform.Variant) { 225 | archMatched = true 226 | } 227 | } 228 | } 229 | 230 | return osMatched && archMatched 231 | } 232 | -------------------------------------------------------------------------------- /pkg/sync/source.go: -------------------------------------------------------------------------------- 1 | package sync 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | "strings" 8 | 9 | "github.com/AliyunContainerService/image-syncer/pkg/utils" 10 | "github.com/containers/image/v5/docker" 11 | "github.com/containers/image/v5/manifest" 12 | "github.com/containers/image/v5/types" 13 | ) 14 | 15 | // ImageSource is a reference to a remote image need to be pulled. 16 | type ImageSource struct { 17 | ref types.ImageReference 18 | source types.ImageSource 19 | ctx context.Context 20 | sysctx *types.SystemContext 21 | 22 | // source image description 23 | registry string 24 | repository string 25 | tagOrDigest string 26 | } 27 | 28 | // NewImageSource generates a PullTask by repository, the repository string must include tag or digest, or it can only be used 29 | // to list tags. 30 | // If username or password is empty, access to repository will be anonymous. 31 | // A repository string is the rest part of the images url except tag digest and registry 32 | func NewImageSource(registry, repository, tagOrDigest, username, password string, insecure bool) (*ImageSource, error) { 33 | if strings.Contains(repository, ":") { 34 | return nil, fmt.Errorf("repository string should not include ':'") 35 | } 36 | 37 | srcRef, err := docker.ParseReference("//" + registry + "/" + repository + utils.AttachConnectorToTagOrDigest(tagOrDigest)) 38 | if err != nil { 39 | return nil, err 40 | } 41 | 42 | var sysctx *types.SystemContext 43 | if insecure { 44 | // destination registry is http service 45 | sysctx = &types.SystemContext{ 46 | DockerInsecureSkipTLSVerify: types.OptionalBoolTrue, 47 | } 48 | } else { 49 | sysctx = &types.SystemContext{} 50 | } 51 | 52 | ctx := context.WithValue(context.Background(), utils.CTXKey("ImageSource"), repository) 53 | if username != "" && password != "" { 54 | sysctx.DockerAuthConfig = &types.DockerAuthConfig{ 55 | Username: username, 56 | Password: password, 57 | } 58 | } 59 | 60 | var source types.ImageSource 61 | if tagOrDigest != "" { 62 | // if tagOrDigest is empty, will attach to the "latest" tag, and will get an error if "latest" is not exist 63 | source, err = srcRef.NewImageSource(ctx, sysctx) 64 | if err != nil { 65 | return nil, err 66 | } 67 | } 68 | 69 | return &ImageSource{ 70 | ref: srcRef, 71 | source: source, 72 | ctx: ctx, 73 | sysctx: sysctx, 74 | registry: registry, 75 | repository: repository, 76 | tagOrDigest: tagOrDigest, 77 | }, nil 78 | } 79 | 80 | // GetManifest get manifest file from source image 81 | func (i *ImageSource) GetManifest() ([]byte, string, error) { 82 | if i.source == nil { 83 | return nil, "", fmt.Errorf("cannot get manifest file without specified a tag or digest") 84 | } 85 | return i.source.GetManifest(i.ctx, nil) 86 | } 87 | 88 | // GetBlobInfos get blob infos from non-list type manifests. 89 | func (i *ImageSource) GetBlobInfos(manifestObjSlice ...manifest.Manifest) ([]types.BlobInfo, error) { 90 | if i.source == nil { 91 | return nil, fmt.Errorf("cannot get blobs without specified a tag or digest") 92 | } 93 | 94 | var srcBlobs []types.BlobInfo 95 | for _, manifestObj := range manifestObjSlice { 96 | blobInfos := manifestObj.LayerInfos() 97 | for _, l := range blobInfos { 98 | srcBlobs = append(srcBlobs, l.BlobInfo) 99 | } 100 | // append config blob info 101 | configBlob := manifestObj.ConfigInfo() 102 | if configBlob.Digest != "" { 103 | srcBlobs = append(srcBlobs, configBlob) 104 | } 105 | } 106 | 107 | return srcBlobs, nil 108 | } 109 | 110 | // GetABlob gets a blob from remote image 111 | func (i *ImageSource) GetABlob(blobInfo types.BlobInfo) (io.ReadCloser, int64, error) { 112 | return i.source.GetBlob(i.ctx, types.BlobInfo{Digest: blobInfo.Digest, URLs: blobInfo.URLs, Size: -1}, NoCache) 113 | } 114 | 115 | // Close an ImageSource 116 | func (i *ImageSource) Close() error { 117 | return i.source.Close() 118 | } 119 | 120 | // GetRegistry returns the registry of a ImageSource 121 | func (i *ImageSource) GetRegistry() string { 122 | return i.registry 123 | } 124 | 125 | // GetRepository returns the repository of a ImageSource 126 | func (i *ImageSource) GetRepository() string { 127 | return i.repository 128 | } 129 | 130 | // GetTagOrDigest returns the tag or digest a ImageSource 131 | func (i *ImageSource) GetTagOrDigest() string { 132 | return i.tagOrDigest 133 | } 134 | 135 | func (i *ImageSource) String() string { 136 | return i.registry + "/" + i.repository + utils.AttachConnectorToTagOrDigest(i.tagOrDigest) 137 | } 138 | 139 | // GetSourceRepoTags gets all the tags of a repository which ImageSource belongs to 140 | func (i *ImageSource) GetSourceRepoTags() ([]string, error) { 141 | // this function still works out even the tagOrDigest is empty 142 | return docker.GetRepositoryTags(i.ctx, i.sysctx, i.ref) 143 | } 144 | -------------------------------------------------------------------------------- /pkg/task/blob.go: -------------------------------------------------------------------------------- 1 | package task 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/docker/go-units" 7 | 8 | "github.com/AliyunContainerService/image-syncer/pkg/sync" 9 | "github.com/containers/image/v5/types" 10 | ) 11 | 12 | // BlobTask sync a blob which belongs to the primary ManifestTask. 13 | type BlobTask struct { 14 | primary Task 15 | 16 | info types.BlobInfo 17 | } 18 | 19 | func NewBlobTask(manifestTask Task, info types.BlobInfo) *BlobTask { 20 | return &BlobTask{ 21 | primary: manifestTask, 22 | info: info, 23 | } 24 | } 25 | 26 | func (b *BlobTask) Run() ([]Task, string, error) { 27 | var resultMsg string 28 | 29 | //// random failure test 30 | //rand.Seed(time.Now().UnixNano()) 31 | //if rand.Intn(100)%2 == 1 { 32 | // return nil, resultMsg, fmt.Errorf("random failure") 33 | //} 34 | 35 | dst := b.primary.GetDestination() 36 | src := b.primary.GetSource() 37 | 38 | blobExist, err := dst.CheckBlobExist(b.info) 39 | if err != nil { 40 | return nil, resultMsg, fmt.Errorf("failed to check blob %s(%v) exist: %v", b.info.Digest, b.info.Size, err) 41 | } 42 | 43 | // ignore exist blob 44 | if !blobExist { 45 | // pull a blob from source 46 | blob, size, err := src.GetABlob(b.info) 47 | if err != nil { 48 | return nil, resultMsg, fmt.Errorf("failed to get blob %s(%v): %v", b.info.Digest, size, err) 49 | } 50 | 51 | b.info.Size = size 52 | // push a blob to destination 53 | if err = dst.PutABlob(blob, b.info); err != nil { 54 | return nil, resultMsg, fmt.Errorf("failed to put blob %s(%v): %v", b.info.Digest, b.info.Size, err) 55 | } 56 | } else { 57 | resultMsg = "ignore exist blob" 58 | } 59 | 60 | if b.primary.ReleaseOnce() { 61 | resultMsg = "start to sync manifest" 62 | return []Task{b.primary}, resultMsg, nil 63 | } 64 | return nil, resultMsg, nil 65 | } 66 | 67 | func (b *BlobTask) GetPrimary() Task { 68 | return b.primary 69 | } 70 | 71 | func (b *BlobTask) Runnable() bool { 72 | // always runnable 73 | return true 74 | } 75 | 76 | func (b *BlobTask) ReleaseOnce() bool { 77 | // do nothing 78 | return true 79 | } 80 | 81 | func (b *BlobTask) GetSource() *sync.ImageSource { 82 | return b.primary.GetSource() 83 | } 84 | 85 | func (b *BlobTask) GetDestination() *sync.ImageDestination { 86 | return b.primary.GetDestination() 87 | } 88 | 89 | func (b *BlobTask) String() string { 90 | return fmt.Sprintf("synchronizing blob %s(%v) from %s to %s", 91 | b.info.Digest, units.HumanSize(float64(b.info.Size)), b.GetSource().String(), b.GetDestination().String()) 92 | } 93 | 94 | func (b *BlobTask) Type() Type { 95 | return BlobType 96 | } 97 | -------------------------------------------------------------------------------- /pkg/task/manifest.go: -------------------------------------------------------------------------------- 1 | package task 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/AliyunContainerService/image-syncer/pkg/utils" 7 | 8 | "github.com/AliyunContainerService/image-syncer/pkg/concurrent" 9 | "github.com/AliyunContainerService/image-syncer/pkg/sync" 10 | "github.com/opencontainers/go-digest" 11 | ) 12 | 13 | // ManifestTask sync a manifest from source to destination. 14 | type ManifestTask struct { 15 | source *sync.ImageSource 16 | destination *sync.ImageDestination 17 | 18 | // for manifest, this refers to a manifest list 19 | primary Task 20 | 21 | counter *concurrent.Counter 22 | 23 | bytes []byte 24 | digest *digest.Digest 25 | } 26 | 27 | func NewManifestTask(manifestListTask Task, source *sync.ImageSource, destination *sync.ImageDestination, 28 | counter *concurrent.Counter, bytes []byte, digest *digest.Digest) *ManifestTask { 29 | return &ManifestTask{ 30 | primary: manifestListTask, 31 | source: source, 32 | destination: destination, 33 | counter: counter, 34 | bytes: bytes, 35 | digest: digest, 36 | } 37 | } 38 | 39 | func (m *ManifestTask) Run() ([]Task, string, error) { 40 | var resultMsg string 41 | 42 | //// random failure test 43 | //rand.Seed(time.Now().UnixNano()) 44 | //if rand.Intn(100)%2 == 1 { 45 | // return nil, resultMsg, fmt.Errorf("random failure") 46 | //} 47 | 48 | if err := m.destination.PushManifest(m.bytes, m.digest); err != nil { 49 | return nil, resultMsg, fmt.Errorf("failed to put manifest: %v", err) 50 | } 51 | 52 | if m.primary == nil { 53 | return nil, resultMsg, nil 54 | } 55 | 56 | if m.primary.ReleaseOnce() { 57 | resultMsg = "start to sync manifest list" 58 | return []Task{m.primary}, resultMsg, nil 59 | } 60 | return nil, resultMsg, nil 61 | } 62 | 63 | func (m *ManifestTask) GetPrimary() Task { 64 | return m.primary 65 | } 66 | 67 | func (m *ManifestTask) Runnable() bool { 68 | count, _ := m.counter.Value() 69 | return count == 0 70 | } 71 | 72 | func (m *ManifestTask) ReleaseOnce() bool { 73 | count, _ := m.counter.Decrease() 74 | return count == 0 75 | } 76 | 77 | func (m *ManifestTask) GetSource() *sync.ImageSource { 78 | return m.source 79 | } 80 | 81 | func (m *ManifestTask) GetDestination() *sync.ImageDestination { 82 | return m.destination 83 | } 84 | 85 | func (m *ManifestTask) String() string { 86 | var srcTagOrDigest, dstTagOrDigest string 87 | if m.primary == nil { 88 | srcTagOrDigest = m.GetSource().GetTagOrDigest() 89 | dstTagOrDigest = m.GetDestination().GetTagOrDigest() 90 | } else { 91 | srcTagOrDigest = m.digest.String() 92 | dstTagOrDigest = m.digest.String() 93 | } 94 | 95 | return fmt.Sprintf("synchronizing manifest from %s/%s%s to %s/%s%s", 96 | m.GetSource().GetRegistry(), m.GetSource().GetRepository(), utils.AttachConnectorToTagOrDigest(srcTagOrDigest), 97 | m.GetDestination().GetRegistry(), m.GetDestination().GetRepository(), utils.AttachConnectorToTagOrDigest(dstTagOrDigest)) 98 | } 99 | 100 | func (m *ManifestTask) Type() Type { 101 | return ManifestType 102 | } 103 | -------------------------------------------------------------------------------- /pkg/task/rule.go: -------------------------------------------------------------------------------- 1 | package task 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/AliyunContainerService/image-syncer/pkg/utils/types" 7 | 8 | "github.com/AliyunContainerService/image-syncer/pkg/sync" 9 | "github.com/AliyunContainerService/image-syncer/pkg/utils" 10 | ) 11 | 12 | // RuleTask analyze an image config rule ("xxx:xxx") and generates URLTask(s). 13 | type RuleTask struct { 14 | source string 15 | destination string 16 | 17 | osFilterList, archFilterList []string 18 | 19 | getAuthFunc func(repository string) types.Auth 20 | 21 | forceUpdate bool 22 | } 23 | 24 | func NewRuleTask(source, destination string, 25 | osFilterList, archFilterList []string, 26 | getAuthFunc func(repository string) types.Auth, forceUpdate bool) (*RuleTask, error) { 27 | if source == "" { 28 | return nil, fmt.Errorf("source url should not be empty") 29 | } 30 | 31 | if destination == "" { 32 | return nil, fmt.Errorf("destination url should not be empty") 33 | } 34 | 35 | return &RuleTask{ 36 | source: source, 37 | destination: destination, 38 | getAuthFunc: getAuthFunc, 39 | osFilterList: osFilterList, 40 | archFilterList: archFilterList, 41 | forceUpdate: forceUpdate, 42 | }, nil 43 | } 44 | 45 | func (r *RuleTask) Run() ([]Task, string, error) { 46 | //// random failure test 47 | //rand.Seed(time.Now().UnixNano()) 48 | //if rand.Intn(100)%2 == 1 { 49 | // return nil, "", fmt.Errorf("random failure") 50 | //} 51 | 52 | // if source tag is not specific, get all tags of this source repo 53 | sourceURLs, err := utils.GenerateRepoURLs(r.source, r.listAllTags) 54 | if err != nil { 55 | return nil, "", fmt.Errorf("source url %s format error: %v", r.source, err) 56 | } 57 | 58 | // if destination tags or digest is not specific, reuse tags or digest of sourceURLs 59 | destinationURLs, err := utils.GenerateRepoURLs(r.destination, func(registry, repository string) ([]string, error) { 60 | var result []string 61 | for _, item := range sourceURLs { 62 | result = append(result, item.GetTagOrDigest()) 63 | } 64 | return result, nil 65 | }) 66 | if err != nil { 67 | return nil, "", fmt.Errorf("source url %s format error: %v", r.source, err) 68 | } 69 | 70 | // TODO: remove duplicated sourceURL and destinationURL pair? 71 | if err = checkSourceAndDestinationURLs(sourceURLs, destinationURLs); err != nil { 72 | return nil, "", fmt.Errorf("failed to check source and destination urls for %s:%s: %v", 73 | r.source, r.destination, err) 74 | } 75 | 76 | var results []Task 77 | for index, s := range sourceURLs { 78 | results = append(results, 79 | NewURLTask(s, destinationURLs[index], 80 | r.getAuthFunc(s.GetURLWithoutTagOrDigest()), 81 | r.getAuthFunc(destinationURLs[index].GetURLWithoutTagOrDigest()), 82 | r.osFilterList, r.archFilterList, r.forceUpdate, 83 | ), 84 | ) 85 | } 86 | 87 | return results, "", nil 88 | } 89 | 90 | func (r *RuleTask) GetPrimary() Task { 91 | return nil 92 | } 93 | 94 | func (r *RuleTask) Runnable() bool { 95 | // always runnable 96 | return true 97 | } 98 | 99 | func (r *RuleTask) ReleaseOnce() bool { 100 | // do nothing 101 | return true 102 | } 103 | 104 | func (r *RuleTask) GetSource() *sync.ImageSource { 105 | return nil 106 | } 107 | 108 | func (r *RuleTask) GetDestination() *sync.ImageDestination { 109 | return nil 110 | } 111 | 112 | func (r *RuleTask) String() string { 113 | return fmt.Sprintf("analyzing image rule for %s -> %s", r.source, r.destination) 114 | } 115 | 116 | func (r *RuleTask) Type() Type { 117 | return RuleType 118 | } 119 | 120 | func (r *RuleTask) listAllTags(sourceRegistry, sourceRepository string) ([]string, error) { 121 | auth := r.getAuthFunc(sourceRegistry + "/" + sourceRepository) 122 | 123 | imageSource, err := sync.NewImageSource(sourceRegistry, sourceRepository, "", 124 | auth.Username, auth.Password, auth.Insecure) 125 | if err != nil { 126 | return nil, fmt.Errorf("generate %s image source error: %v", sourceRegistry+"/"+sourceRepository, err) 127 | } 128 | 129 | return imageSource.GetSourceRepoTags() 130 | } 131 | 132 | func checkSourceAndDestinationURLs(sourceURLs, destinationURLs []*utils.RepoURL) error { 133 | if len(sourceURLs) != len(destinationURLs) { 134 | return fmt.Errorf("the number of tags of source and destination is not matched") 135 | } 136 | 137 | // digest must be the same 138 | if len(sourceURLs) == 1 && sourceURLs[0].HasDigest() && destinationURLs[0].HasDigest() { 139 | if sourceURLs[0].GetTagOrDigest() != destinationURLs[0].GetTagOrDigest() { 140 | return fmt.Errorf("the digest of source and destination must match") 141 | } 142 | } 143 | 144 | return nil 145 | } 146 | -------------------------------------------------------------------------------- /pkg/task/task.go: -------------------------------------------------------------------------------- 1 | package task 2 | 3 | import ( 4 | "github.com/AliyunContainerService/image-syncer/pkg/sync" 5 | ) 6 | 7 | type Type string 8 | 9 | const ( 10 | URLType = Type("URL") 11 | ManifestType = Type("Manifest") 12 | RuleType = Type("Rule") 13 | BlobType = Type("Blob") 14 | ) 15 | 16 | type Task interface { 17 | // Run returns primary task and result message if success while primary task is not nil and can run immediately. 18 | Run() ([]Task, string, error) 19 | 20 | // GetPrimary returns primary task, manifest for a blob, or manifest list for a manifest 21 | GetPrimary() Task 22 | 23 | // Runnable returns if the task can be executed immediately 24 | Runnable() bool 25 | 26 | // ReleaseOnce try to release once and return if the task is runnable after being released. 27 | ReleaseOnce() bool 28 | 29 | // GetSource return a source refers to the source images. 30 | GetSource() *sync.ImageSource 31 | 32 | // GetDestination return a source refers to the destination images 33 | GetDestination() *sync.ImageDestination 34 | 35 | String() string 36 | 37 | Type() Type 38 | } 39 | -------------------------------------------------------------------------------- /pkg/task/url.go: -------------------------------------------------------------------------------- 1 | package task 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "github.com/AliyunContainerService/image-syncer/pkg/utils/types" 8 | 9 | "github.com/AliyunContainerService/image-syncer/pkg/concurrent" 10 | "github.com/containers/image/v5/manifest" 11 | 12 | "github.com/AliyunContainerService/image-syncer/pkg/utils" 13 | 14 | "github.com/AliyunContainerService/image-syncer/pkg/sync" 15 | ) 16 | 17 | // URLTask converts an image RepoURL pair (specific tag) to BlobTask(s) and ManifestTask(s). 18 | type URLTask struct { 19 | source *utils.RepoURL 20 | destination *utils.RepoURL 21 | 22 | sourceAuth types.Auth 23 | destinationAuth types.Auth 24 | 25 | osFilterList, archFilterList []string 26 | 27 | forceUpdate bool 28 | } 29 | 30 | func NewURLTask(source, destination *utils.RepoURL, 31 | sourceAuth, destinationAuth types.Auth, 32 | osFilterList, archFilterList []string, 33 | forceUpdate bool) Task { 34 | return &URLTask{ 35 | source: source, 36 | destination: destination, 37 | sourceAuth: sourceAuth, 38 | destinationAuth: destinationAuth, 39 | osFilterList: osFilterList, 40 | archFilterList: archFilterList, 41 | forceUpdate: forceUpdate, 42 | } 43 | } 44 | 45 | func (u *URLTask) Run() ([]Task, string, error) { 46 | imageSource, err := sync.NewImageSource(u.source.GetRegistry(), u.source.GetRepo(), u.source.GetTagOrDigest(), 47 | u.sourceAuth.Username, u.sourceAuth.Password, u.sourceAuth.Insecure) 48 | if err != nil { 49 | return nil, "", fmt.Errorf("generate %s image source error: %v", u.source.String(), err) 50 | } 51 | 52 | imageDestination, err := sync.NewImageDestination(u.destination.GetRegistry(), u.destination.GetRepo(), 53 | u.destination.GetTagOrDigest(), u.destinationAuth.Username, u.destinationAuth.Password, u.destinationAuth.Insecure) 54 | if err != nil { 55 | return nil, "", fmt.Errorf("generate %s image destination error: %v", u.destination.String(), err) 56 | } 57 | 58 | tasks, msg, err := u.generateSyncTasks(imageSource, imageDestination, u.osFilterList, u.archFilterList) 59 | if err != nil { 60 | return nil, "", fmt.Errorf("failed to generate manifest/blob tasks: %v", err) 61 | } 62 | 63 | return tasks, msg, nil 64 | } 65 | 66 | func (u *URLTask) GetPrimary() Task { 67 | return nil 68 | } 69 | 70 | func (u *URLTask) Runnable() bool { 71 | // always runnable 72 | return true 73 | } 74 | 75 | func (u *URLTask) ReleaseOnce() bool { 76 | // do nothing 77 | return true 78 | } 79 | 80 | func (u *URLTask) GetSource() *sync.ImageSource { 81 | return nil 82 | } 83 | 84 | func (u *URLTask) GetDestination() *sync.ImageDestination { 85 | return nil 86 | } 87 | 88 | func (u *URLTask) String() string { 89 | return fmt.Sprintf("generating sync tasks from %s to %s", u.source, u.destination) 90 | } 91 | 92 | func (u *URLTask) Type() Type { 93 | return URLType 94 | } 95 | 96 | // generateSyncTasks generates blob/manifest tasks. 97 | func (u *URLTask) generateSyncTasks(source *sync.ImageSource, destination *sync.ImageDestination, 98 | osFilterList, archFilterList []string) ([]Task, string, error) { 99 | var results []Task 100 | var resultMsg string 101 | 102 | // get manifest from source 103 | manifestBytes, manifestType, err := source.GetManifest() 104 | if err != nil { 105 | return nil, resultMsg, fmt.Errorf("failed to get manifest: %v", err) 106 | } 107 | 108 | destManifestObj, destManifestBytes, subManifestInfoSlice, err := sync.GenerateManifestObj(manifestBytes, 109 | manifestType, osFilterList, archFilterList, source, nil) 110 | if err != nil { 111 | return nil, resultMsg, fmt.Errorf(" failed to get manifest info: %v", err) 112 | } 113 | 114 | if destManifestObj == nil { 115 | resultMsg = "skip synchronization because no manifest fits platform filters" 116 | return nil, resultMsg, nil 117 | } 118 | 119 | if changed := destination.CheckManifestChanged(destManifestBytes, nil); !u.forceUpdate && !changed { 120 | // do nothing if image is unchanged 121 | resultMsg = "skip synchronization because destination image exists" 122 | return nil, resultMsg, nil 123 | } 124 | 125 | destManifestTask := NewManifestTask(nil, 126 | source, destination, nil, destManifestBytes, nil) 127 | 128 | if len(subManifestInfoSlice) == 0 { 129 | // non-list type image 130 | blobInfos, err := source.GetBlobInfos(destManifestObj.(manifest.Manifest)) 131 | if err != nil { 132 | return nil, resultMsg, fmt.Errorf("failed to get blob infos: %v", err) 133 | } 134 | 135 | destManifestTask.counter = concurrent.NewCounter(len(blobInfos), len(blobInfos)) 136 | 137 | for _, info := range blobInfos { 138 | // only append blob tasks 139 | results = append(results, NewBlobTask(destManifestTask, info)) 140 | } 141 | } else { 142 | // list type image 143 | var noExistSubManifestCounter int 144 | var ignoredManifestDigests []string 145 | 146 | for _, mfstInfo := range subManifestInfoSlice { 147 | if changed := destination.CheckManifestChanged(mfstInfo.Bytes, mfstInfo.Digest); !u.forceUpdate && !changed { 148 | // do nothing if manifest is unchanged 149 | ignoredManifestDigests = append(ignoredManifestDigests, mfstInfo.Digest.String()) 150 | continue 151 | } 152 | 153 | noExistSubManifestCounter++ 154 | 155 | blobInfos, err := source.GetBlobInfos(mfstInfo.Obj) 156 | if err != nil { 157 | return nil, resultMsg, fmt.Errorf("failed to get blob infos for manifest %s: %v", mfstInfo.Digest, err) 158 | } 159 | 160 | subManifestTask := NewManifestTask(destManifestTask, source, destination, 161 | concurrent.NewCounter(len(blobInfos), len(blobInfos)), mfstInfo.Bytes, mfstInfo.Digest) 162 | 163 | for _, info := range blobInfos { 164 | // only append blob tasks 165 | results = append(results, NewBlobTask(subManifestTask, info)) 166 | } 167 | } 168 | destManifestTask.counter = concurrent.NewCounter(noExistSubManifestCounter, noExistSubManifestCounter) 169 | 170 | if noExistSubManifestCounter == 0 { 171 | // all the sub manifests are exist in destination 172 | results = append(results, destManifestTask) 173 | } 174 | 175 | if len(ignoredManifestDigests) != 0 { 176 | resultMsg = fmt.Sprintf("%v sub manifests in the list are ignored: %v", len(ignoredManifestDigests), 177 | strings.Join(ignoredManifestDigests, ", ")) 178 | } 179 | } 180 | 181 | return results, resultMsg, nil 182 | } 183 | -------------------------------------------------------------------------------- /pkg/utils/auth/google.go: -------------------------------------------------------------------------------- 1 | package auth 2 | 3 | import ( 4 | "context" 5 | "encoding/base64" 6 | "strings" 7 | "time" 8 | 9 | "golang.org/x/oauth2/google" 10 | ) 11 | 12 | const Oauth2User = "_oauth2_" 13 | 14 | // IsGCRPermanentServiceAccountToken returns true if user is a Google permanent service account token 15 | func IsGCRPermanentServiceAccountToken(registry string, username string) bool { 16 | return strings.Contains(registry, ".gcr.io") && strings.Compare(username, Oauth2User) == 0 17 | } 18 | 19 | // GCPTokenFromCreds creates oauth2 token from permanent service account token 20 | func GCPTokenFromCreds(creds string) (string, time.Time, error) { 21 | b, err := base64.StdEncoding.DecodeString(creds) 22 | if err != nil { 23 | return "", time.Time{}, err 24 | } 25 | conf, err := google.JWTConfigFromJSON( 26 | b, "https://www.googleapis.com/auth/devstorage.read_write") 27 | if err != nil { 28 | return "", time.Time{}, err 29 | } 30 | 31 | token, err := conf.TokenSource(context.Background()).Token() 32 | if err != nil { 33 | return "", time.Time{}, err 34 | } 35 | 36 | return token.AccessToken, token.Expiry, nil 37 | } 38 | -------------------------------------------------------------------------------- /pkg/utils/ctx.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | type CTXKey string 4 | -------------------------------------------------------------------------------- /pkg/utils/slice.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | func RemoveEmptyItems(slice []string) []string { 4 | var result []string 5 | for _, item := range slice { 6 | if item != "" { 7 | result = append(result, item) 8 | } 9 | } 10 | return result 11 | } 12 | 13 | func RemoveDuplicateItems(slice []string) []string { 14 | result := make([]string, 0, len(slice)) 15 | temp := map[string]struct{}{} 16 | for _, item := range slice { 17 | if _, ok := temp[item]; !ok { 18 | temp[item] = struct{}{} 19 | result = append(result, item) 20 | } 21 | } 22 | return result 23 | } 24 | -------------------------------------------------------------------------------- /pkg/utils/string.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "strings" 5 | ) 6 | 7 | func RepoMathPrefix(repo, prefix string) bool { 8 | if len(prefix) == 0 { 9 | return false 10 | } 11 | 12 | s := strings.TrimPrefix(repo, prefix) 13 | if s == repo { 14 | return false 15 | } 16 | 17 | return string(s[0]) == "/" || string(prefix[len(prefix)-1]) == "/" 18 | } 19 | -------------------------------------------------------------------------------- /pkg/utils/string_test.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestString(t *testing.T) { 10 | cases := [][]string{ 11 | { 12 | "gcr.io/knative-releases/github.com/knative/build/cmd/creds-init:v1", "gcr.io/knative-releases/github.com/knative/build/cmd", 13 | }, { 14 | "registry.hub.docker.io/library/nginx", "registry.hub.docker.io/library/", 15 | }, { 16 | "registry.hub.docker.io/library/nginx", "registry.hub.docker.io/libr", 17 | }, { 18 | "registry.hub.docker.io/library/nginx", "", 19 | }, 20 | } 21 | 22 | var results []bool 23 | for _, c := range cases { 24 | result := RepoMathPrefix(c[0], c[1]) 25 | results = append(results, result) 26 | } 27 | 28 | assert.Equal(t, true, results[0]) 29 | assert.Equal(t, true, results[1]) 30 | assert.Equal(t, false, results[2]) 31 | assert.Equal(t, false, results[3]) 32 | } 33 | -------------------------------------------------------------------------------- /pkg/utils/types/auth.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | // Auth describes the authentication information of a registry or a repository 4 | type Auth struct { 5 | Username string `json:"username" yaml:"username"` 6 | Password string `json:"password" yaml:"password"` 7 | Insecure bool `json:"insecure" yaml:"insecure"` 8 | } 9 | -------------------------------------------------------------------------------- /pkg/utils/types/imageList.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/AliyunContainerService/image-syncer/pkg/utils" 8 | ) 9 | 10 | type ImageList map[string][]string 11 | 12 | func NewImageList(origin map[string]interface{}) (ImageList, error) { 13 | result := map[string][]string{} 14 | 15 | for source, dest := range origin { 16 | convertErr := fmt.Errorf("invalid destination %v for source \"%v\", "+ 17 | "destination should only be string or []string", dest, source) 18 | 19 | emptyDestErr := fmt.Errorf("empty destination is not supported for source: %v", source) 20 | 21 | if destList, ok := dest.([]interface{}); ok { 22 | // check if is destination is a []string 23 | for _, d := range destList { 24 | destStr, ok := d.(string) 25 | if !ok { 26 | return nil, convertErr 27 | } 28 | 29 | if len(destStr) == 0 { 30 | return nil, emptyDestErr 31 | } 32 | result[source] = append(result[source], os.ExpandEnv(destStr)) 33 | } 34 | 35 | // empty slice is the same with an empty string 36 | if len(destList) == 0 { 37 | return nil, emptyDestErr 38 | } 39 | 40 | result[source] = utils.RemoveDuplicateItems(result[source]) 41 | } else if destStr, ok := dest.(string); ok { 42 | // check if is destination is a string 43 | if len(destStr) == 0 { 44 | return nil, emptyDestErr 45 | } 46 | result[source] = append(result[source], os.ExpandEnv(destStr)) 47 | } else { 48 | return nil, convertErr 49 | } 50 | } 51 | 52 | return result, nil 53 | } 54 | 55 | func (i ImageList) Query(src, dst string) bool { 56 | destList, exist := i[src] 57 | if exist { 58 | // check if is destination is a []string 59 | for _, d := range destList { 60 | if d == dst { 61 | return true 62 | } 63 | } 64 | } 65 | 66 | return false 67 | } 68 | 69 | func (i ImageList) Add(src, dst string) { 70 | if exist := i.Query(src, dst); !exist { 71 | i[src] = append(i[src], dst) 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /pkg/utils/url.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "fmt" 5 | "regexp" 6 | "strings" 7 | 8 | "github.com/opencontainers/go-digest" 9 | 10 | "github.com/containers/image/v5/docker/reference" 11 | ) 12 | 13 | const ( 14 | DockerHubURL = "docker.io" 15 | ) 16 | 17 | type RepoURL struct { 18 | // origin url 19 | ref reference.Reference 20 | 21 | // "namespace" is part of repo 22 | registry string 23 | repo string 24 | tagOrDigest string 25 | } 26 | 27 | // GenerateRepoURLs creates a RepoURL slice. 28 | // If url has no tags or digest, tags or digest should be provided by externalTagsOrDigest func, 29 | // and empty slice will be returned if no tags or digest is provided. 30 | func GenerateRepoURLs(url string, externalTagsOrDigest func(registry, repository string, 31 | ) (tagsOrDigest []string, err error)) ([]*RepoURL, error) { 32 | var result []*RepoURL 33 | ref, err := reference.ParseNormalizedNamed(url) 34 | 35 | var tagsOrDigest []string 36 | var urlWithoutTagOrDigest string 37 | 38 | if canonicalRef, ok := ref.(reference.Canonical); ok { 39 | // url has digest 40 | tagsOrDigest = append(tagsOrDigest, canonicalRef.Digest().String()) 41 | urlWithoutTagOrDigest = canonicalRef.Name() 42 | } else if taggedRef, ok := ref.(reference.NamedTagged); ok { 43 | // url has one normal tag 44 | tagsOrDigest = append(tagsOrDigest, taggedRef.Tag()) 45 | urlWithoutTagOrDigest = taggedRef.Name() 46 | } else if err == nil { 47 | // url has no specified digest or tag 48 | registry, repo := getRegistryAndRepositoryFromURLWithoutTagOrDigest(url) 49 | allTags, err := externalTagsOrDigest(registry, repo) 50 | if err != nil { 51 | return nil, fmt.Errorf("failed to get external tags: %v", err) 52 | } 53 | 54 | urlWithoutTagOrDigest = url 55 | tagsOrDigest = append(tagsOrDigest, allTags...) 56 | } else { 57 | // url might have special tags 58 | if strings.Contains(url, ":/") { 59 | // regex exist, /*/, etc. 60 | slice := strings.SplitN(url, ":/", 2) 61 | if len(slice) != 2 || !strings.HasSuffix(slice[1], "/") { 62 | return nil, fmt.Errorf("invalid tag regex url format %v, regex must start and end with \"/\"", url) 63 | } 64 | 65 | _, err = reference.ParseNormalizedNamed(slice[0]) 66 | if err != nil { 67 | return nil, fmt.Errorf("failed to parse repository url %v: %v", slice[0], err) 68 | } 69 | 70 | urlWithoutTagOrDigest = slice[0] 71 | regexStr := strings.TrimSuffix(slice[1], "/") 72 | regex, err := regexp.Compile(regexStr) 73 | if err != nil { 74 | return nil, fmt.Errorf("invalid tag regex: \"%v\": %v", regexStr, err) 75 | } 76 | 77 | registry, repo := getRegistryAndRepositoryFromURLWithoutTagOrDigest(urlWithoutTagOrDigest) 78 | allTags, err := externalTagsOrDigest(registry, repo) 79 | if err != nil { 80 | return nil, fmt.Errorf("failed to get external tags: %v", err) 81 | } 82 | 83 | for _, t := range allTags { 84 | if regex.MatchString(t) { 85 | tagsOrDigest = append(tagsOrDigest, t) 86 | } 87 | } 88 | } else { 89 | // multiple tags exist 90 | slice := strings.SplitN(url, ",", -1) 91 | if len(slice) < 1 { 92 | return nil, fmt.Errorf("invalid repository url: %v", url) 93 | } 94 | 95 | ref, err = reference.ParseNormalizedNamed(slice[0]) 96 | if err != nil { 97 | return nil, fmt.Errorf("failed to parse first tag with url %v: %v", slice[0], err) 98 | } 99 | 100 | urlWithoutTagOrDigest = ref.(reference.NamedTagged).Name() 101 | tagsOrDigest = append(tagsOrDigest, ref.(reference.NamedTagged).Tag()) 102 | tagsOrDigest = append(tagsOrDigest, slice[1:]...) 103 | } 104 | } 105 | 106 | registry, repo := getRegistryAndRepositoryFromURLWithoutTagOrDigest(urlWithoutTagOrDigest) 107 | 108 | // if no tags or digest provided, an empty slice will be returned 109 | for _, item := range tagsOrDigest { 110 | newURL := registry + "/" + repo + AttachConnectorToTagOrDigest(item) 111 | ref, err = reference.ParseNormalizedNamed(newURL) 112 | if err != nil { 113 | return nil, fmt.Errorf("failed to parese canonical url: %v", newURL) 114 | } 115 | 116 | result = append(result, &RepoURL{ 117 | ref: ref, 118 | registry: registry, 119 | repo: repo, 120 | tagOrDigest: item, 121 | }) 122 | } 123 | 124 | return result, nil 125 | } 126 | 127 | // GetURL returns the whole url 128 | func (r *RepoURL) String() string { 129 | return r.ref.String() 130 | } 131 | 132 | // GetRegistry returns the registry in a url 133 | func (r *RepoURL) GetRegistry() string { 134 | return r.registry 135 | } 136 | 137 | // GetRepo returns the repository in a url 138 | func (r *RepoURL) GetRepo() string { 139 | return r.repo 140 | } 141 | 142 | // GetTagOrDigest returns the tag in a url 143 | func (r *RepoURL) GetTagOrDigest() string { 144 | return r.tagOrDigest 145 | } 146 | 147 | // GetRepoWithTagOrDigest returns repository:tag in a url 148 | func (r *RepoURL) GetRepoWithTagOrDigest() string { 149 | if r.tagOrDigest == "" { 150 | return r.repo 151 | } 152 | 153 | return r.repo + AttachConnectorToTagOrDigest(r.tagOrDigest) 154 | } 155 | 156 | func (r *RepoURL) HasDigest() bool { 157 | _, result := r.ref.(reference.Canonical) 158 | return result 159 | } 160 | 161 | func (r *RepoURL) GetURLWithoutTagOrDigest() string { 162 | return r.registry + "/" + r.repo 163 | } 164 | 165 | func AttachConnectorToTagOrDigest(tagOrDigest string) string { 166 | if len(tagOrDigest) == 0 { 167 | return "" 168 | } 169 | 170 | tmpDigest := digest.Digest(tagOrDigest) 171 | if err := tmpDigest.Validate(); err != nil { 172 | return ":" + tagOrDigest 173 | } 174 | return "@" + tagOrDigest 175 | } 176 | 177 | func getRegistryAndRepositoryFromURLWithoutTagOrDigest(urlWithoutTagOrDigest string) (registry string, repo string) { 178 | slice := strings.SplitN(urlWithoutTagOrDigest, "/", 2) 179 | if len(slice) == 1 { 180 | registry = DockerHubURL 181 | repo = slice[0] 182 | } else { 183 | registry = slice[0] 184 | repo = slice[1] 185 | } 186 | 187 | return 188 | } 189 | -------------------------------------------------------------------------------- /pkg/utils/url_test.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestURL(t *testing.T) { 11 | urls := []string{ 12 | "gcr.io/knative-releases/github.com/knative/build/cmd/creds-init:v1", 13 | "registry.hub.docker.io/library/nginx", 14 | "nginx:v1", 15 | "127.0.0.1:300/library/nginx:v1", 16 | "127.0.0.1:300/library/nginx", 17 | "127.0.0.1:300/library/nginx:v1,v2", 18 | "registry.cn-beijing.aliyuncs.com/hhyasdf/hybridnet@sha256:df2ef9e979fc063645dcbed51374233c6bcf4ab49308c0478702565e96b9bc9e", 19 | "nginx", 20 | "test-regex/test:/b+/", 21 | } 22 | 23 | var repoURLs []*RepoURL 24 | for _, url := range urls { 25 | tmpUrls, err := GenerateRepoURLs(url, func(registry, repository string) (tags []string, err error) { 26 | if registry == "test-regex" { 27 | return []string{"aaa", "bbb"}, nil 28 | } 29 | return []string{"latest"}, nil 30 | }) 31 | if err != nil { 32 | fmt.Println("err: ", err) 33 | return 34 | } 35 | repoURLs = append(repoURLs, tmpUrls...) 36 | } 37 | 38 | assert.Equal(t, "gcr.io", repoURLs[0].GetRegistry()) 39 | assert.Equal(t, "gcr.io/knative-releases/github.com/knative/build/cmd/creds-init:v1", repoURLs[0].String()) 40 | assert.Equal(t, "knative-releases/github.com/knative/build/cmd/creds-init", repoURLs[0].GetRepo()) 41 | assert.Equal(t, "v1", repoURLs[0].GetTagOrDigest()) 42 | assert.Equal(t, "latest", repoURLs[1].GetTagOrDigest()) 43 | assert.Equal(t, "registry.hub.docker.io", repoURLs[1].GetRegistry()) 44 | assert.Equal(t, "library/nginx", repoURLs[1].GetRepo()) 45 | assert.Equal(t, DockerHubURL, repoURLs[2].GetRegistry()) 46 | assert.Equal(t, "v1", repoURLs[2].GetTagOrDigest()) 47 | assert.Equal(t, "library/nginx", repoURLs[2].GetRepo()) 48 | assert.Equal(t, "127.0.0.1:300", repoURLs[3].GetRegistry()) 49 | assert.Equal(t, "v1", repoURLs[3].GetTagOrDigest()) 50 | assert.Equal(t, "library/nginx:v1", repoURLs[3].GetRepoWithTagOrDigest()) 51 | assert.Equal(t, "127.0.0.1:300/library/nginx", repoURLs[4].GetURLWithoutTagOrDigest()) 52 | assert.Equal(t, "127.0.0.1:300", repoURLs[4].GetRegistry()) 53 | assert.Equal(t, "library/nginx:latest", repoURLs[4].GetRepoWithTagOrDigest()) 54 | assert.Equal(t, "v2", repoURLs[6].GetTagOrDigest()) 55 | assert.Equal(t, "sha256:df2ef9e979fc063645dcbed51374233c6bcf4ab49308c0478702565e96b9bc9e", repoURLs[7].GetTagOrDigest()) 56 | assert.Equal(t, "hhyasdf/hybridnet@sha256:df2ef9e979fc063645dcbed51374233c6bcf4ab49308c0478702565e96b9bc9e", 57 | repoURLs[7].GetRepoWithTagOrDigest()) 58 | assert.Equal(t, "hhyasdf/hybridnet", 59 | repoURLs[7].GetRepo()) 60 | assert.Equal(t, DockerHubURL, repoURLs[8].GetRegistry()) 61 | assert.Equal(t, "bbb", repoURLs[9].GetTagOrDigest()) 62 | } 63 | --------------------------------------------------------------------------------