├── .DS_Store ├── .gitignore ├── LICENSE ├── Makefile ├── README.md ├── cmd └── image-transfer │ └── main.go ├── configs └── config.go ├── docs └── arch.png ├── go.mod ├── go.sum ├── hack └── containerized │ ├── Dockerfile │ └── run.go └── pkg ├── apis ├── ccrapis │ └── ccrapi.go └── tcrapis │ └── tcrapi.go ├── flag └── flag.go ├── image-transfer ├── cmd.go ├── options │ ├── config.go │ └── options.go └── run.go ├── log ├── doc.go ├── encoder.go ├── flag.go ├── log.go ├── log_restful.go └── type.go ├── transfer ├── job.go ├── manifest.go ├── source.go └── target.go └── utils ├── ratelimiter.go └── utils.go /.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tkestack/image-transfer/c247c9360ca709d763298717a3cb5d9aacf54ae1/.DS_Store -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | _output/* 2 | *.yaml 3 | ccr_to_tcr_rules 4 | .vscode/* 5 | image-transfer 6 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | VERSION ?= $(shell git describe --match 'v[0-9]*' --dirty='.m' --tags --always) 2 | REVISION ?= $(shell git rev-parse HEAD)$(shell if ! git diff --no-ext-diff --quiet --exit-code; then echo .m; fi) 3 | 4 | .PHONY: binary clean 5 | 6 | binary: 7 | @echo "build image-transfer binary" 8 | @go build -o ./_output/image-transfer ./cmd/image-transfer/main.go 9 | 10 | transfer_image: binary 11 | @echo "build image-transfer image" 12 | @go build -o ./_output/run hack/containerized/run.go 13 | @docker build -t ccr.ccs.tencentyun.com/tcrimages/image-transfer:${VERSION} -f hack/containerized/Dockerfile . 14 | 15 | clean: 16 | @rm ./_output/* 17 | 18 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # 镜像迁移工具：image-transfer 2 | 3 | `image-transfer` 是一个docker镜像迁移工具，用于对不同镜像仓库的镜像进行批量迁移。 4 | 5 | ## 特性 6 | 7 | - 支持多对多镜像仓库迁移 8 | - 支持腾讯云 TCR 个人版(CCR)一键全量迁移至企业版 9 | - 支持基于 Docker Registry V2搭建的docker镜像仓库服务 (如 腾讯云TCR个人版(CCR)/TCR企业版、Docker Hub、 Quay、 阿里云镜像服务ACR、 Harbor等) 10 | - 支持自定义 qps 限速，避免迁移时对仓库造成过大压力 11 | - 同步不落盘，提升同步速度 12 | - 利用 pipeline 模型，提高任务执行效率 13 | - 增量同步, 通过对同步过的镜像 blob 信息落盘，不重复同步已同步的镜像 14 | - 并发同步，可以通过配置文件调整并发数 15 | - 自动重试失败的同步任务，可以解决大部分镜像同步中的网络抖动问题 16 | - 不依赖docker以及其他程序 17 | 18 | ## 模式 19 | 20 | tcr镜像迁移工具具有两种运行模式： 21 | 22 | - 通用模式：支持多种镜像仓库迁移 23 | - 腾讯云 CCR 一键全量迁移模式：腾讯云 TCR 个人版(CCR) -> TCR企业版 24 | 25 | ## 架构 26 | 27 | ![image](./docs/arch.png) 28 | 29 | ## 使用 30 | 31 | ### 下载和安装 32 | 33 | 在[releases](https://github.com/tkestack/image-transfer/releases)页面可下载源码以及二进制文件 34 | 35 | ### 手动编译 36 | 37 | ```bash 38 | git clone https://github.com/tkestack/image-transfer.git 39 | cd ./image-transfer 40 | 41 | # 编译 42 | make 43 | ``` 44 | 45 | ## 使用方法 46 | 47 | ### 使用帮助 48 | 49 | ```shell 50 | ./image-transfer -h 51 | ``` 52 | 53 | ### 使用示例1：通用模式 54 | 55 | 设置镜像鉴权配置文件为 registry-secret.yaml，镜像迁移仓库规则文件为 transfer-rule.yaml，默认迁移目标为腾讯云 TCR 个人版：ccr.ccs.tencentyun.com，默认 namespace 为 default，并发数为 5， 56 | 57 | ```shell 58 | ./image-transfer --securityFile=./registry-secret.yaml --ruleFile=./transfer-rule.yaml \ 59 | --ns=default --registry=ccr.ccs.tencentyun.com --routines=5 --retry=3 60 | ``` 61 | 62 | ### 使用示例2：腾讯云 CCR 一键全量迁移模式 63 | 64 | 打开ccr迁移模式 ccrToTcr=true, 设置镜像仓库鉴权配置文件为 registry-secret.yaml，腾讯云 API 调用密钥配置文件为 tencentcloud-secret.yaml，默认个人版及企业版实例所在地域均为 ap-guangzhou（广州） 65 | TCR 企业版实例示例名称为 image-transfer, 并发数为 5，失败重试次数为 3 66 | 67 | ```shell 68 | ./image-transfer --ccrToTcr=true --securityFile=./registry-secret.yaml --secretFile=./tencentcloud-secret.yaml \ 69 | --tcrName=image-transfer --tcrRegion=ap-guangzhou --ccrRegion=ap-guangzhou --routines=5 --retry=3 70 | ``` 71 | 72 | ### 配置文件参考 73 | 74 | #### 腾讯云 API 密钥配置文件 tencentcloud-secret.yaml 75 | 76 | 如需迁移至腾讯云容器镜像服务内，需配置腾讯云 API 调用密钥，可前往 控制台-访问管理-访问密钥-API密钥管理 获取密钥对。如果在同账号下将 TCR 个人版数据迁移至企业版实例内，可共用同一套密钥。 77 | 78 | ```yaml 79 | ccr: 80 | secretId: xxx 81 | secretKey: xxx 82 | tcr: 83 | secretId: xxx 84 | secretKey: xxx 85 | ``` 86 | 87 | #### 镜像仓库鉴权配置文件 registry-secret.yaml 88 | 89 | 配置源目标及迁移目标镜像仓库的访问凭证，即 Docker Login 所使用的用户名，密码。 90 | 91 | ```yaml 92 | ccr.ccs.tencentyun.com: 93 | username: xxx 94 | password: xxx 95 | image-transfer.tencentcloudcr.com: 96 | username: xxx 97 | password: xxx 98 | registry.hub.docker.com: 99 | username: xxx 100 | password: xxx 101 | registry.cn-guangzhou.aliyuncs.com: 102 | username: xxx 103 | password: xxx 104 | acr.cn-guangzhou.cr.aliyuncs.com: 105 | username: xxx 106 | password: xxx 107 | ``` 108 | 109 | #### 镜像迁移仓库配置文件 transfer-rule.yaml 110 | 111 | 配置镜像迁移规则，即镜像仓库，镜像版本在源目标及迁移目标内的映射关系。 112 | 113 | 注：目前只支持 repo、tag 级别，且可同时配置多条镜像迁移规则。 114 | 115 | ```yaml 116 | ## tag 117 | demo-ns/nginx:latest : image-transfer.tencentcloudcr.com/demo-ns/nginx:latest 118 | ## repo 119 | registry.hub.docker.com/{ns1}/{repo1}: image-transfer.tencentcloudcr.com/{ns1}/{repo1} 120 | registry.hub.docker.com/{ns2}/{repo2}: image-transfer.tencentcloudcr.com/{ns2}/{repo2} 121 | ``` 122 | -------------------------------------------------------------------------------- /cmd/image-transfer/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making TKEStack 3 | * available. 4 | * 5 | * Copyright (C) 2012-2020 Tencent. All Rights Reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package main 20 | 21 | import ( 22 | "fmt" 23 | tcr_image_transfer "tkestack.io/image-transfer/pkg/image-transfer" 24 | "math/rand" 25 | "os" 26 | "runtime" 27 | "time" 28 | ) 29 | 30 | func main() { 31 | 32 | rand.Seed(time.Now().UTC().UnixNano()) 33 | if len(os.Getenv("GOMAXPROCS")) == 0 { 34 | runtime.GOMAXPROCS(runtime.NumCPU()) 35 | } 36 | 37 | cmd := tcr_image_transfer.NewImageTransferCommand("image-transfer") 38 | if err := cmd.Execute(); err != nil { 39 | fmt.Fprintf(os.Stderr, "error: %v\n", err) 40 | os.Exit(1) 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /configs/config.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making TKEStack 3 | * available. 4 | * 5 | * Copyright (C) 2012-2020 Tencent. All Rights Reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package configs 20 | 21 | import ( 22 | "errors" 23 | "fmt" 24 | "gopkg.in/ini.v1" 25 | "gopkg.in/yaml.v2" 26 | "os" 27 | "strings" 28 | "sync" 29 | "tkestack.io/image-transfer/pkg/image-transfer/options" 30 | "tkestack.io/image-transfer/pkg/log" 31 | ) 32 | 33 | 34 | var ( 35 | once sync.Once 36 | instance *Configs 37 | //QPS rateLimit qps 38 | QPS int 39 | //sessionInstance *SessionConfigs 40 | ) 41 | 42 | const ( 43 | maxRatelimit int = 30000 44 | maxRoutineNums int = 50 45 | ) 46 | 47 | // Configs struct save of all config 48 | type Configs struct { 49 | FlagConf *options.ClientOptions 50 | Conf *ini.File 51 | Security map[string]Security 52 | ImageList map[string]string 53 | Secret map[string]Secret 54 | //ConfMap map[string]interface{} 55 | //ConfMapString map[string]string 56 | } 57 | 58 | // Security describes the authentication information of a registry 59 | type Security struct { 60 | Username string `json:"username" yaml:"username"` 61 | Password string `json:"password" yaml:"password"` 62 | Insecure bool `json:"insecure" yaml:"insecure"` 63 | } 64 | 65 | // Secret describes secret info for tencent cloud 66 | type Secret struct { 67 | SecretID string `json:"secretId" yaml:"secretId"` 68 | SecretKey string `json:"secretKey" yaml:"secretKey"` 69 | } 70 | 71 | 72 | // InitConfigs InitLogger initializes logger the way we want for tke. 73 | func InitConfigs(opts *options.ClientOptions) (*Configs, error) { 74 | //log.Println(opts.Config.ConfigFile) 75 | once.Do(func() { 76 | instance = &Configs{} 77 | 78 | instance.FlagConf = opts 79 | }) 80 | 81 | if instance.FlagConf.Config.CCRToTCR == true { 82 | if len(instance.FlagConf.Config.SecretFile) == 0 || len(instance.FlagConf.Config.SecurityFile) == 0 { 83 | return nil, errors.New("no SecretFile or security file is provided, Exit") 84 | } else if len(instance.FlagConf.Config.TCRName) == 0 { 85 | return nil, errors.New("no tcr name is provided, Exit") 86 | } else { 87 | secret, err := instance.GetSecret() 88 | if err != nil { 89 | return nil, err 90 | } 91 | instance.Secret = secret 92 | securityList, err := instance.GetSecurity() 93 | if err != nil { 94 | return nil, err 95 | } 96 | instance.Security = securityList 97 | } 98 | } else { 99 | if len(instance.FlagConf.Config.RuleFile) == 0 || len(instance.FlagConf.Config.SecurityFile) == 0 { 100 | return nil, errors.New("no rule file or security file is provided, Exit") 101 | } 102 | instance.ImageList = instance.GetImageList() 103 | 104 | securityList, err := instance.GetSecurity() 105 | if err != nil { 106 | return nil, err 107 | } 108 | instance.Security = securityList 109 | 110 | 111 | } 112 | 113 | if instance.FlagConf.Config.RoutineNums > maxRoutineNums { 114 | instance.FlagConf.Config.RoutineNums = maxRoutineNums 115 | } 116 | 117 | if instance.FlagConf.Config.QPS > maxRatelimit { 118 | instance.FlagConf.Config.QPS = maxRatelimit 119 | } 120 | 121 | QPS = instance.FlagConf.Config.QPS 122 | 123 | 124 | return instance, nil 125 | } 126 | 127 | // GetConfigs get config of Configs instance 128 | func GetConfigs() *Configs { 129 | /*if instance == nil{ 130 | log.Fatalf("Fail to get instance: %v", instance) 131 | }*/ 132 | return instance 133 | } 134 | 135 | // GetImageList get images list of configs instance 136 | func (c *Configs) GetImageList() map[string]string { 137 | var imageList map[string]string 138 | 139 | 140 | if err := openAndDecode(c.FlagConf.Config.RuleFile, &imageList); err != nil { 141 | log.Errorf("decode config file %v error: %v", c.FlagConf.Config.RuleFile, err) 142 | return nil 143 | } 144 | 145 | 146 | return imageList 147 | } 148 | 149 | // GetSecurity gets the Security information in Config 150 | func (c *Configs) GetSecurity() (map[string]Security, error) { 151 | var securityList map[string]Security 152 | 153 | if err := openAndDecode(c.FlagConf.Config.SecurityFile, &securityList); err != nil { 154 | log.Errorf("decode config file %v error: %v", c.FlagConf.Config.SecurityFile, err) 155 | return securityList, err 156 | } 157 | 158 | 159 | return securityList, nil 160 | } 161 | 162 | 163 | // GetSecuritySpecific gets the specific authentication information in Config 164 | func (c *Configs) GetSecuritySpecific(registry string, namespace string) (Security, bool) { 165 | 166 | // key of each AuthList item can be "registry/namespace" or "registry" only 167 | registryAndNamespace := registry + "/" + namespace 168 | 169 | if moreSpecificAuth, exist := c.Security[registryAndNamespace]; exist { 170 | return moreSpecificAuth, exist 171 | } 172 | auth, exist := c.Security[registry] 173 | return auth, exist 174 | } 175 | 176 | // GetSecret get secret from secret file 177 | func (c *Configs) GetSecret() (map[string]Secret, error) { 178 | var secret map[string]Secret 179 | 180 | if err := openAndDecode(c.FlagConf.Config.SecretFile, &secret); err != nil { 181 | log.Errorf("decode secret file %v error: %v", c.FlagConf.Config.SecretFile, err) 182 | return secret, err 183 | } 184 | 185 | return secret, nil 186 | 187 | } 188 | 189 | 190 | // Open yaml file and decode into target interface 191 | func openAndDecode(filePath string, target interface{}) error { 192 | if !strings.HasSuffix(filePath, ".yaml") { 193 | return fmt.Errorf("only support yaml format file") 194 | } 195 | 196 | if _, err := os.Stat(filePath); os.IsNotExist(err) { 197 | return fmt.Errorf("file %v not exist: %v", filePath, err) 198 | } 199 | 200 | file, err := os.OpenFile(filePath, os.O_RDONLY, 0666) 201 | if err != nil { 202 | return fmt.Errorf("open file %v error: %v", filePath, err) 203 | } 204 | 205 | 206 | decoder := yaml.NewDecoder(file) 207 | if err := decoder.Decode(target); err != nil { 208 | return fmt.Errorf("unmarshal config error: %v", err) 209 | } 210 | 211 | 212 | return nil 213 | } 214 | -------------------------------------------------------------------------------- /docs/arch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tkestack/image-transfer/c247c9360ca709d763298717a3cb5d9aacf54ae1/docs/arch.png -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module tkestack.io/image-transfer 2 | 3 | go 1.14 4 | 5 | require ( 6 | github.com/containers/image/v5 v5.11.1 7 | github.com/containers/libtrust v0.0.0-20200511145503-9c3a6c22cd9a // indirect 8 | github.com/docker/docker v1.13.1 // indirect 9 | github.com/emicklei/go-restful v2.15.0+incompatible 10 | github.com/gorilla/mux v1.8.0 // indirect 11 | github.com/opencontainers/go-digest v1.0.0 12 | github.com/pkg/errors v0.9.1 13 | github.com/skipor/goenv v0.0.0-20170219222015-cf3a15e6b664 14 | github.com/spf13/cobra v1.1.3 15 | github.com/spf13/pflag v1.0.5 16 | github.com/tencentcloud/tencentcloud-sdk-go v1.0.62 17 | go.uber.org/ratelimit v0.1.0 18 | go.uber.org/zap v1.10.0 19 | gopkg.in/ini.v1 v1.51.0 20 | gopkg.in/natefinch/lumberjack.v2 v2.0.0 21 | gopkg.in/yaml.v2 v2.4.0 22 | ) 23 | -------------------------------------------------------------------------------- /hack/containerized/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM centos:8 2 | 3 | ADD ./_output/run / 4 | ADD ./_output/image-transfer / 5 | 6 | CMD ["/run"] -------------------------------------------------------------------------------- /hack/containerized/run.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "os" 7 | "os/exec" 8 | "strings" 9 | "text/template" 10 | 11 | "github.com/spf13/cobra" 12 | "tkestack.io/image-transfer/pkg/apis/ccrapis" 13 | "tkestack.io/image-transfer/pkg/log" 14 | ) 15 | 16 | const ( 17 | secretFilePath = "/tmp/secret.yaml" 18 | securityFilePath = "/tmp/security.yaml" 19 | binaryFilePath = "/image-transfer" 20 | 21 | securityFileTemplate = `{{.TCRDomian}}: 22 | username: {{.TCRUsername}} 23 | password: "{{.TCRPassword}}" 24 | {{.CCRDomain}}: 25 | username: {{.CCRUsername}} 26 | password: "{{.CCRPassword}}"` 27 | secretFileTemplate = `ccr: 28 | secretId: {{.CCRSecretId}} 29 | secretKey: {{.CCRSecretKey}} 30 | tcr: 31 | secretId: {{.TCRSecretId}} 32 | secretKey: {{.TCRSecretKey}}` 33 | ) 34 | 35 | var ( 36 | // for same region and account 37 | secretId string 38 | secretKey string 39 | regionName string 40 | // for different region and account 41 | ccrSecretId string 42 | ccrSecretKey string 43 | tcrSecretId string 44 | tcrSecretKey string 45 | ccrRegionName string 46 | tcrRegionName string 47 | 48 | // common flag 49 | tcrName string 50 | ccrAuth string 51 | tcrAuth string 52 | tagNum int 53 | routines int 54 | 55 | rootCmd = &cobra.Command{ 56 | Use: "Example: run --tcrName=test-transfer --secretId=xxxx --secretKey=xxxx --regionName=ap-guangzhou --ccrAuth=user:pass --tcrAuth=user:pass --tagNum=50", 57 | Short: "run", 58 | Long: "Image migration tool for CCR and TCR", 59 | Run: run(), 60 | } 61 | ) 62 | 63 | type renderArgs struct { 64 | CCRSecretId string 65 | CCRSecretKey string 66 | TCRSecretId string 67 | TCRSecretKey string 68 | TCRDomian string 69 | CCRDomain string 70 | CCRUsername string 71 | CCRPassword string 72 | TCRUsername string 73 | TCRPassword string 74 | } 75 | 76 | func init() { 77 | rootCmd.Flags().StringVar(&secretId, "secretId", "", 78 | "yunapi secret Id for ccr and tcr") 79 | rootCmd.Flags().StringVar(&secretKey, "secretKey", "", 80 | "yunapi secret key for ccr and tcr") 81 | rootCmd.Flags().StringVar(®ionName, "regionName", "ap-guangzhou", 82 | "yunapi region name for ccr and tcr") 83 | 84 | rootCmd.Flags().StringVar(&ccrSecretId, "ccrSecretId", "", 85 | "yunapi secret Id for ccr") 86 | rootCmd.Flags().StringVar(&ccrSecretKey, "ccrSecretKey", "", 87 | "yunapi secret key for ccr") 88 | rootCmd.Flags().StringVar(&tcrSecretId, "tcrSecretId", "", 89 | "yunapi secret Id for tcr") 90 | rootCmd.Flags().StringVar(&tcrSecretKey, "tcrSecretKey", "", 91 | "yunapi secret key for tcr") 92 | 93 | rootCmd.Flags().StringVar(&ccrRegionName, "ccrRegionName", regionName, 94 | "yunapi region name for ccr") 95 | rootCmd.Flags().StringVar(&tcrRegionName, "tcrRegionName", regionName, 96 | "yunapi region name for tcr") 97 | 98 | rootCmd.Flags().StringVar(&tcrName, "tcrName", "", 99 | "tcr instance name") 100 | rootCmd.Flags().StringVar(&ccrAuth, "ccrAuth", "", 101 | "ccr auth secret, format is username:password") 102 | rootCmd.Flags().StringVar(&tcrAuth, "tcrAuth", "", 103 | "tcr auth secret, format is username:password") 104 | rootCmd.Flags().IntVar(&tagNum, "tagNum", 100, 105 | "number of recent tags in migration") 106 | rootCmd.Flags().IntVar(&routines, "routines", 5, 107 | "number of concurrent task in migration") 108 | 109 | } 110 | 111 | func generateRenderArgs() (*renderArgs, error) { 112 | ccrIdx := strings.Index(ccrAuth, ":") 113 | if ccrIdx == -1 { 114 | return nil, errors.New("Invalid ccrAuthSecret") 115 | } 116 | tcrIdx := strings.Index(tcrAuth, ":") 117 | if tcrIdx == -1 { 118 | return nil, errors.New("Invalid tcrAuthSecret") 119 | } 120 | ccrDomainPrefix, ok := ccrapis.RegionPrefix[ccrRegionName] 121 | if !ok { 122 | return nil, errors.New("Invalid ccrRegionName") 123 | } 124 | rargs := renderArgs{ 125 | CCRSecretId: ccrSecretId, 126 | CCRSecretKey: ccrSecretKey, 127 | TCRSecretId: tcrSecretId, 128 | TCRSecretKey: tcrSecretKey, 129 | TCRDomian: fmt.Sprintf("%s.tencentcloudcr.com", tcrName), 130 | CCRDomain: fmt.Sprintf("%s.ccs.tencentyun.com", ccrDomainPrefix), 131 | CCRUsername: ccrAuth[:ccrIdx], 132 | CCRPassword: ccrAuth[ccrIdx+1:], 133 | TCRUsername: tcrAuth[:tcrIdx], 134 | TCRPassword: tcrAuth[tcrIdx+1:], 135 | } 136 | 137 | return &rargs, nil 138 | } 139 | 140 | func render(args *renderArgs) error { 141 | 142 | renderInternal := func(templateStr, path string) error { 143 | 144 | tmpl, err := template.New(path).Parse(templateStr) 145 | if err != nil { 146 | log.Errorf("template.Parse error: %v", err) 147 | return err 148 | } 149 | outputFile, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0666) 150 | if err != nil { 151 | log.Errorf("os.OpenFile error: %v", err) 152 | return err 153 | } 154 | err = tmpl.Execute(outputFile, args) 155 | if err != nil { 156 | log.Errorf("tmpl.Execute error: %v", err) 157 | return err 158 | } 159 | return nil 160 | } 161 | 162 | // render secret.yaml 163 | if err := renderInternal(secretFileTemplate, secretFilePath); err != nil { 164 | return err 165 | } 166 | // render security.yaml 167 | if err := renderInternal(securityFileTemplate, securityFilePath); err != nil { 168 | return err 169 | } 170 | return nil 171 | } 172 | 173 | func validateArgs() { 174 | if ccrSecretId == "" || ccrSecretKey == "" || tcrSecretId == "" || tcrSecretKey == "" { 175 | log.Error("Require ccrSecretId or secretId, ccrSecretKey or secretKey") 176 | os.Exit(1) 177 | } 178 | if ccrRegionName == "" || tcrRegionName == "" { 179 | log.Error("Require ccrRegionName,tcrRegionName or regionName") 180 | os.Exit(1) 181 | } 182 | if tcrName == "" { 183 | log.Error("Require tcrName") 184 | os.Exit(1) 185 | } 186 | 187 | if ccrAuth == "" { 188 | log.Error("Require ccrAuth") 189 | os.Exit(1) 190 | } 191 | if tcrAuth == "" { 192 | log.Error("Require tcrAuth") 193 | os.Exit(1) 194 | } 195 | 196 | } 197 | 198 | func run() func(cmd *cobra.Command, args []string) { 199 | return func(cmd *cobra.Command, args []string) { 200 | // adapt same region or not 201 | if ccrSecretId == "" { 202 | ccrSecretId = secretId 203 | } 204 | if ccrSecretKey == "" { 205 | ccrSecretKey = secretKey 206 | } 207 | if tcrSecretId == "" { 208 | tcrSecretId = secretId 209 | } 210 | if tcrSecretKey == "" { 211 | tcrSecretKey = secretKey 212 | } 213 | if ccrRegionName == "" { 214 | ccrRegionName = regionName 215 | } 216 | if tcrRegionName == "" { 217 | tcrRegionName = regionName 218 | } 219 | 220 | validateArgs() 221 | 222 | rargs, err := generateRenderArgs() 223 | if err != nil { 224 | log.Errorf("generateRenderArgs error: %v", err) 225 | os.Exit(1) 226 | 227 | } 228 | if err := render(rargs); err != nil { 229 | log.Error("Render template error") 230 | os.Exit(1) 231 | } 232 | commandArgs := fmt.Sprintf("--ccrToTcr=true --retry=3 --routines=%d --ccrTagNums=%d --tcrName=%s --ccrRegion=%s --tcrRegion=%s --securityFile=%s --secretFile=%s", 233 | routines, tagNum, tcrName, ccrRegionName, tcrRegionName, securityFilePath, secretFilePath) 234 | 235 | transferCMD := exec.Command(binaryFilePath, strings.Split(commandArgs, " ")...) 236 | transferCMD.Stderr = os.Stderr 237 | transferCMD.Stdout = os.Stdout 238 | if err := transferCMD.Run(); err != nil { 239 | log.Errorf("transferCMD.Run() error: %v", err) 240 | os.Exit(1) 241 | } 242 | } 243 | } 244 | 245 | func main() { 246 | rootCmd.Execute() 247 | } 248 | -------------------------------------------------------------------------------- /pkg/apis/ccrapis/ccrapi.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making TKEStack 3 | * available. 4 | * 5 | * Copyright (C) 2012-2020 Tencent. All Rights Reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package ccrapis 20 | 21 | import ( 22 | "net/http" 23 | "strings" 24 | "sync" 25 | 26 | "github.com/pkg/errors" 27 | "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common" 28 | "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/profile" 29 | tcr "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/tcr/v20190924" 30 | "tkestack.io/image-transfer/configs" 31 | "tkestack.io/image-transfer/pkg/log" 32 | "tkestack.io/image-transfer/pkg/utils" 33 | ) 34 | 35 | // CCRAPIClient wrap http client 36 | type CCRAPIClient struct { 37 | httpClient *http.Client 38 | url string 39 | } 40 | 41 | // RegionPrefix is map tencent cloud region to ccr domain prefix 42 | var RegionPrefix = map[string]string{ 43 | "ap-guangzhou": "ccr", 44 | "ap-shanghai": "ccr", 45 | "ap-nanjing": "ccr", 46 | "ap-beijing": "ccr", 47 | "ap-shenzhen": "ccr", 48 | "ap-chongqing": "ccr", 49 | "ap-chengdu": "ccr", 50 | "ap-tianjin": "ccr", 51 | "ap-hongkong": "hkccr", 52 | "ap-shenzhen-fsi": "szjrccr", 53 | "ap-shanghai-fsi": "shjrccr", 54 | "ap-beijing-fsi": "bjjrccr", 55 | "ap-singapore": "sgccr", 56 | "ap-seoul": "krccr", 57 | "ap-tokyo": "jpccr", 58 | "ap-mumbai": "inccr", 59 | "ap-bangkok": "thccr", 60 | "na-toronto": "caccr", 61 | "na-siliconvalley": "uswccr", 62 | "na-ashburn": "useccr", 63 | "eu-frankfurt": "deccr", 64 | "eu-moscow": "ruccr", 65 | } 66 | 67 | // NewCCRAPIClient is new return *CCRAPIClient 68 | func NewCCRAPIClient() *CCRAPIClient { 69 | httpclient := http.Client{} 70 | ai := CCRAPIClient{httpClient: &httpclient} 71 | 72 | return &ai 73 | } 74 | 75 | // GetAllNamespaceByName get all ns from ccr name 76 | func (ai *CCRAPIClient) GetAllNamespaceByName(secret map[string]configs.Secret, region string) ([]string, error) { 77 | 78 | var nsList []string 79 | 80 | secretID, secretKey, err := GetCcrSecret(secret) 81 | 82 | if err != nil { 83 | log.Errorf("GetCcrSecret error: ", err) 84 | return nsList, err 85 | } 86 | 87 | offset := int64(0) 88 | count := 0 89 | limit := int64(100) 90 | 91 | for { 92 | resp, err := ai.DescribeNamespacePersonal(secretID, secretKey, region, offset, limit) 93 | if err != nil { 94 | log.Errorf("GetAllNamespaceByName error, ", err) 95 | return nsList, err 96 | } 97 | namespaceCount := *resp.Response.Data.NamespaceCount 98 | count += len(resp.Response.Data.NamespaceInfo) 99 | 100 | for _, ns := range resp.Response.Data.NamespaceInfo { 101 | nsList = append(nsList, *ns.Namespace) 102 | } 103 | log.Debugf("ccr namespace offset %d limit %d resp is %s", offset, limit, resp.ToJsonString()) 104 | 105 | if int64(count) >= namespaceCount { 106 | break 107 | } else { 108 | offset += limit 109 | } 110 | 111 | } 112 | 113 | return nsList, nil 114 | 115 | } 116 | 117 | //GetAllCcrRepo get all ccr repo send to repoChan 118 | func (ai *CCRAPIClient) GetAllCcrRepo(secret map[string]configs.Secret, ccrRegion string, 119 | failedNsList []string, tcrRegion string, tcrName string, repoChan chan string, toltalCount int64) error { 120 | secretID, secretKey, err := GetCcrSecret(secret) 121 | if err != nil { 122 | log.Errorf("GetCcrSecret error: %s", err) 123 | return err 124 | } 125 | 126 | offset := int64(0) 127 | limit := int64(100) 128 | maxGorutineNum := 5 129 | page := toltalCount / limit 130 | if toltalCount%limit > 0 { 131 | page++ 132 | } 133 | wg := sync.WaitGroup{} 134 | defer close(repoChan) 135 | leakCh := make(chan struct{}, maxGorutineNum) 136 | wg.Add(1) 137 | go func() { 138 | defer wg.Done() 139 | for i := 0; i < maxGorutineNum; i++ { 140 | leakCh <- struct{}{} 141 | } 142 | }() 143 | 144 | for j := int64(0); j < page; j++ { 145 | wg.Add(1) 146 | <-leakCh 147 | go func(n int64) { 148 | defer wg.Done() 149 | offset = n * limit 150 | resp, err := ai.DescribeRepositoryOwnerPersonal(secretID, secretKey, ccrRegion, offset, limit) 151 | if err != nil { 152 | log.Errorf("get ccr repo offset %d limit %d error, %s", offset, limit, err) 153 | return 154 | } 155 | for _, repo := range resp.Response.Data.RepoInfo { 156 | ns := strings.Split(*repo.RepoName, "/")[0] 157 | if len(failedNsList) == 0 || !utils.IsContain(failedNsList, ns) { 158 | repoChan <- *repo.RepoName 159 | } 160 | } 161 | leakCh <- struct{}{} 162 | }(j) 163 | } 164 | wg.Wait() 165 | return nil 166 | } 167 | 168 | // GetRepoTags get ccr repo tags 169 | func (ai *CCRAPIClient) GetRepoTags(secretID, secretKey, ccrRegion, repoName string, ccrTagNum int64) ([]string, error) { 170 | 171 | if ccrTagNum < 0 { 172 | return nil, errors.Errorf("Invalid value ccrTagNum %v", ccrTagNum) 173 | } 174 | 175 | offset := int64(0) 176 | count := int64(0) 177 | limit := int64(100) 178 | 179 | var result []string 180 | 181 | for { 182 | resp, err := ai.DescribeImagePersonal(secretID, secretKey, ccrRegion, repoName, offset, limit) 183 | if err != nil { 184 | return nil, err 185 | } 186 | var tagCount int64 187 | 188 | if resp.Response != nil && resp.Response.Data != nil { 189 | tagCount = *resp.Response.Data.TagCount 190 | } else { 191 | return nil, errors.New("DescribeImagePersonal resp is nil") 192 | } 193 | if tagCount == 0 { 194 | return nil, nil 195 | } 196 | if ccrTagNum >= tagCount { 197 | ccrTagNum = tagCount 198 | } 199 | 200 | count += int64(len(resp.Response.Data.TagInfo)) 201 | for _, tagInfo := range resp.Response.Data.TagInfo { 202 | result = append(result, *tagInfo.TagName) 203 | } 204 | 205 | if count >= ccrTagNum { 206 | break 207 | } 208 | 209 | if count >= tagCount { 210 | break 211 | } else { 212 | offset += limit 213 | } 214 | 215 | } 216 | // in case index out of range 217 | if ccrTagNum > int64(len(result)) { 218 | ccrTagNum = int64(len(result)) 219 | } 220 | 221 | return result[:ccrTagNum], nil 222 | } 223 | 224 | func (ai *CCRAPIClient) DescribeImagePersonal(secretID, secretKey, 225 | region, repoName string, offset, limit int64) (*tcr.DescribeImagePersonalResponse, error) { 226 | 227 | credential := common.NewCredential( 228 | secretID, 229 | secretKey, 230 | ) 231 | cpf := profile.NewClientProfile() 232 | cpf.HttpProfile.Endpoint = "tcr.tencentcloudapi.com" 233 | client, _ := tcr.NewClient(credential, region, cpf) 234 | 235 | request := tcr.NewDescribeImagePersonalRequest() 236 | 237 | request.Limit = common.Int64Ptr(limit) 238 | request.Offset = common.Int64Ptr(offset) 239 | request.RepoName = common.StringPtr(repoName) 240 | response, err := client.DescribeImagePersonal(request) 241 | 242 | if err != nil { 243 | log.Errorf("An error has returned: %s", err) 244 | return nil, err 245 | } 246 | 247 | return response, nil 248 | 249 | } 250 | 251 | // DescribeNamespacePersonal is ccr api DescribeNamespacePersonal 252 | func (ai *CCRAPIClient) DescribeNamespacePersonal(secretID, secretKey, 253 | region string, offset, limit int64) (*tcr.DescribeNamespacePersonalResponse, error) { 254 | 255 | credential := common.NewCredential( 256 | secretID, 257 | secretKey, 258 | ) 259 | cpf := profile.NewClientProfile() 260 | cpf.HttpProfile.Endpoint = "tcr.tencentcloudapi.com" 261 | client, _ := tcr.NewClient(credential, region, cpf) 262 | 263 | request := tcr.NewDescribeNamespacePersonalRequest() 264 | 265 | request.Namespace = common.StringPtr("") 266 | request.Limit = common.Int64Ptr(limit) 267 | request.Offset = common.Int64Ptr(offset) 268 | 269 | response, err := client.DescribeNamespacePersonal(request) 270 | 271 | if err != nil { 272 | log.Errorf("An error has returned: %s", err) 273 | return nil, err 274 | } 275 | 276 | return response, nil 277 | 278 | } 279 | 280 | // DescribeRepositoryOwnerPersonal is ccr api DescribeRepositoryOwnerPersonal 281 | func (ai *CCRAPIClient) DescribeRepositoryOwnerPersonal(secretID, secretKey, 282 | region string, offset, limit int64) (*tcr.DescribeRepositoryOwnerPersonalResponse, error) { 283 | 284 | credential := common.NewCredential( 285 | secretID, 286 | secretKey, 287 | ) 288 | cpf := profile.NewClientProfile() 289 | cpf.HttpProfile.Endpoint = "tcr.tencentcloudapi.com" 290 | client, _ := tcr.NewClient(credential, region, cpf) 291 | 292 | request := tcr.NewDescribeRepositoryOwnerPersonalRequest() 293 | 294 | request.Limit = common.Int64Ptr(limit) 295 | request.Offset = common.Int64Ptr(offset) 296 | 297 | response, err := client.DescribeRepositoryOwnerPersonal(request) 298 | 299 | if err != nil { 300 | log.Errorf("An error has returned: %s", err) 301 | return nil, err 302 | } 303 | 304 | return response, nil 305 | 306 | } 307 | 308 | // GetCcrSecret get ccr secret from configs 309 | func GetCcrSecret(secret map[string]configs.Secret) (string, string, error) { 310 | var secretID string 311 | var secretKey string 312 | 313 | if ccr, ok := secret["ccr"]; ok { 314 | //ccr secret存在 315 | secretID = ccr.SecretID 316 | secretKey = ccr.SecretKey 317 | } else if tcr, ok := secret["tcr"]; ok { 318 | //用tcr secret代替ccr 319 | secretID = tcr.SecretID 320 | secretKey = tcr.SecretKey 321 | } else { 322 | return "", "", errors.New("no matched secret provided in secret file") 323 | } 324 | 325 | return secretID, secretKey, nil 326 | } 327 | -------------------------------------------------------------------------------- /pkg/apis/tcrapis/tcrapi.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making TKEStack 3 | * available. 4 | * 5 | * Copyright (C) 2012-2020 Tencent. All Rights Reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package tcrapis 20 | 21 | import ( 22 | "errors" 23 | "net/http" 24 | 25 | "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common" 26 | "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/profile" 27 | tcr "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/tcr/v20190924" 28 | "tkestack.io/image-transfer/configs" 29 | "tkestack.io/image-transfer/pkg/log" 30 | ) 31 | 32 | // TCRAPIClient wrap http client 33 | type TCRAPIClient struct { 34 | httpClient *http.Client 35 | url string 36 | } 37 | 38 | // NewTCRAPIClient is new return *CCRAPIClient 39 | func NewTCRAPIClient() *TCRAPIClient { 40 | httpclient := http.Client{} 41 | ai := TCRAPIClient{httpClient: &httpclient} 42 | 43 | return &ai 44 | } 45 | 46 | // GetAllNamespaceByName get all ns from tcr name 47 | func (ai *TCRAPIClient) GetAllNamespaceByName(secret map[string]configs.Secret, 48 | region string, tcrName string) ([]string, string, error) { 49 | 50 | var nsList []string 51 | var tcrID string 52 | secretID, secretKey, err := GetTcrSecret(secret) 53 | 54 | if err != nil { 55 | log.Errorf("GetTcrSecret error: ", err) 56 | return nsList, tcrID, err 57 | } 58 | 59 | //get tcrId by tcr name 60 | filterValues := []string{tcrName} 61 | resp, err := ai.DescribeInstances(secretID, secretKey, region, 0, 100, "RegistryName", filterValues) 62 | if err != nil { 63 | log.Errorf("DescribeInstances error, ", err) 64 | return nsList, tcrID, err 65 | } 66 | 67 | tcrID = *resp.Response.Registries[0].RegistryId 68 | 69 | // tcr offset means page number, currently :( 70 | offset := int64(1) 71 | count := 0 72 | limit := int64(100) 73 | 74 | for { 75 | resp, err := ai.DescribeNamespaces(secretID, secretKey, region, offset, limit, tcrID) 76 | if err != nil { 77 | log.Errorf("DescribeNamespaces error, ", err) 78 | return nsList, tcrID, err 79 | } 80 | log.Debugf("tcr namespace offset %d limit %d resp is %s", offset, limit, resp.ToJsonString()) 81 | namespaceCount := *resp.Response.TotalCount 82 | count += len(resp.Response.NamespaceList) 83 | for _, ns := range resp.Response.NamespaceList { 84 | nsList = append(nsList, *ns.Name) 85 | } 86 | 87 | if int64(count) >= namespaceCount { 88 | break 89 | } else { 90 | offset += 1 91 | } 92 | 93 | } 94 | 95 | return nsList, tcrID, nil 96 | 97 | } 98 | 99 | // DescribeInstances is tcr api DescribeInstances 100 | func (ai *TCRAPIClient) DescribeInstances(secretID, secretKey, region string, offset, 101 | limit int64, filterName string, filterValues []string) (*tcr.DescribeInstancesResponse, error) { 102 | 103 | credential := common.NewCredential( 104 | secretID, 105 | secretKey, 106 | ) 107 | cpf := profile.NewClientProfile() 108 | cpf.HttpProfile.Endpoint = "tcr.tencentcloudapi.com" 109 | client, _ := tcr.NewClient(credential, region, cpf) 110 | 111 | request := tcr.NewDescribeInstancesRequest() 112 | 113 | request.Filters = []*tcr.Filter{ 114 | { 115 | Values: common.StringPtrs(filterValues), 116 | Name: common.StringPtr(filterName), 117 | }, 118 | } 119 | 120 | response, err := client.DescribeInstances(request) 121 | 122 | if err != nil { 123 | log.Errorf("An error has returned: %s", err) 124 | return nil, err 125 | } 126 | 127 | return response, nil 128 | 129 | } 130 | 131 | // DescribeNamespaces is tcr api DescribeNamespaces 132 | func (ai *TCRAPIClient) DescribeNamespaces(secretID, secretKey, region string, offset, 133 | limit int64, registryID string) (*tcr.DescribeNamespacesResponse, error) { 134 | 135 | credential := common.NewCredential( 136 | secretID, 137 | secretKey, 138 | ) 139 | cpf := profile.NewClientProfile() 140 | cpf.HttpProfile.Endpoint = "tcr.tencentcloudapi.com" 141 | client, _ := tcr.NewClient(credential, region, cpf) 142 | 143 | request := tcr.NewDescribeNamespacesRequest() 144 | 145 | request.RegistryId = common.StringPtr(registryID) 146 | request.Limit = common.Int64Ptr(limit) 147 | request.Offset = common.Int64Ptr(offset) 148 | 149 | response, err := client.DescribeNamespaces(request) 150 | 151 | if err != nil { 152 | log.Errorf("An error has returned: %s", err) 153 | return nil, err 154 | } 155 | 156 | return response, nil 157 | 158 | } 159 | 160 | // CreateNamespace is tcr api CreateNamespace 161 | func (ai *TCRAPIClient) CreateNamespace(secretID, secretKey, region string, 162 | registryID string, nsName string) (*tcr.CreateNamespaceResponse, error) { 163 | 164 | credential := common.NewCredential( 165 | secretID, 166 | secretKey, 167 | ) 168 | cpf := profile.NewClientProfile() 169 | cpf.HttpProfile.Endpoint = "tcr.tencentcloudapi.com" 170 | client, _ := tcr.NewClient(credential, region, cpf) 171 | 172 | request := tcr.NewCreateNamespaceRequest() 173 | 174 | request.RegistryId = common.StringPtr(registryID) 175 | request.NamespaceName = common.StringPtr(nsName) 176 | request.IsPublic = common.BoolPtr(false) 177 | 178 | response, err := client.CreateNamespace(request) 179 | 180 | if err != nil { 181 | log.Errorf("An error has returned: %s", err) 182 | return nil, err 183 | } 184 | 185 | return response, nil 186 | 187 | } 188 | 189 | // GetTcrSecret get tcr secret from config 190 | func GetTcrSecret(secret map[string]configs.Secret) (string, string, error) { 191 | var secretID string 192 | var secretKey string 193 | 194 | if tcr, ok := secret["tcr"]; ok { 195 | //tcr secret存在 196 | secretID = tcr.SecretID 197 | secretKey = tcr.SecretKey 198 | } else if ccr, ok := secret["ccr"]; ok { 199 | //用ccr secret代替tcr 200 | secretID = ccr.SecretID 201 | secretKey = ccr.SecretKey 202 | } else { 203 | return "", "", errors.New("no matched secret provided in secret file") 204 | } 205 | 206 | return secretID, secretKey, nil 207 | } 208 | 209 | // DescribeImages get the images list of tcr repo 210 | func (ai *TCRAPIClient) DescribeImages(secretID, secretKey, region, registryID, nsName, repositoryName string, offset, limit int64) (*tcr.DescribeImagesResponse, error) { 211 | credential := common.NewCredential( 212 | secretID, 213 | secretKey, 214 | ) 215 | cpf := profile.NewClientProfile() 216 | cpf.HttpProfile.Endpoint = "tcr.tencentcloudapi.com" 217 | client, _ := tcr.NewClient(credential, region, cpf) 218 | 219 | request := tcr.NewDescribeImagesRequest() 220 | request.RegistryId = common.StringPtr(registryID) 221 | request.NamespaceName = common.StringPtr(nsName) 222 | request.RepositoryName = common.StringPtr(repositoryName) 223 | request.Limit = common.Int64Ptr(limit) 224 | request.Offset = common.Int64Ptr(offset) 225 | 226 | response, err := client.DescribeImages(request) 227 | 228 | if err != nil { 229 | log.Errorf("An error has returned: %s", err) 230 | return nil, err 231 | } 232 | return response, nil 233 | } 234 | 235 | // GetRepoTags get tcr repo tag list 236 | func (ai *TCRAPIClient) GetRepoTags(secretID, secretKey, region, tcrName, nsName, repositoryName, instanceName string) ([]string, error) { 237 | var tags []string 238 | 239 | tcrID := instanceName 240 | if tcrID == "" { 241 | //get tcrId by tcr name 242 | filterValues := []string{tcrName} 243 | resp, err := ai.DescribeInstances(secretID, secretKey, region, 0, 100, "RegistryName", filterValues) 244 | if err != nil { 245 | log.Errorf("DescribeInstances error, %s", err) 246 | return tags, err 247 | } 248 | 249 | tcrID = *resp.Response.Registries[0].RegistryId 250 | } 251 | 252 | // tcr offset means page number, currently :( 253 | offset := int64(1) 254 | count := 0 255 | limit := int64(100) 256 | 257 | for { 258 | resp, err := ai.DescribeImages(secretID, secretKey, region, tcrID, nsName, repositoryName, offset, limit) 259 | if err != nil { 260 | log.Errorf("DescribeImages error, %s", err) 261 | return tags, err 262 | } 263 | tagsCount := *resp.Response.TotalCount 264 | count += len(resp.Response.ImageInfoList) 265 | for _, tagInfo := range resp.Response.ImageInfoList { 266 | tags = append(tags, *tagInfo.ImageVersion) 267 | } 268 | 269 | log.Debugf("tcr get tag repo %s/%s offset %d, limit %d, total count %d, current count %d, resp %v", nsName, repositoryName, offset, limit, tagsCount, count, resp.ToJsonString()) 270 | if int64(count) >= tagsCount { 271 | break 272 | } else { 273 | offset++ 274 | } 275 | } 276 | return tags, nil 277 | } 278 | -------------------------------------------------------------------------------- /pkg/flag/flag.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making TKEStack 3 | * available. 4 | * 5 | * Copyright (C) 2012-2020 Tencent. All Rights Reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package flag 20 | 21 | import ( 22 | "strings" 23 | 24 | "fmt" 25 | 26 | "tkestack.io/image-transfer/pkg/log" 27 | "github.com/spf13/pflag" 28 | ) 29 | 30 | // WordSepNormalizeFunc changes all flags that contain "_" separators 31 | func WordSepNormalizeFunc(f *pflag.FlagSet, name string) pflag.NormalizedName { 32 | if strings.Contains(name, "_") { 33 | return pflag.NormalizedName(strings.Replace(name, "_", "-", -1)) 34 | } 35 | return pflag.NormalizedName(name) 36 | } 37 | 38 | // WarnWordSepNormalizeFunc changes and warns for flags that contain "_" separators 39 | func WarnWordSepNormalizeFunc(f *pflag.FlagSet, name string) pflag.NormalizedName { 40 | if strings.Contains(name, "_") { 41 | normalizedName := strings.Replace(name, "_", "-", -1) 42 | log.Warn(fmt.Sprintf("%s is DEPRECATED and will be removed" + 43 | " in a future version. Use %s instead.", name, normalizedName)) 44 | return pflag.NormalizedName(normalizedName) 45 | } 46 | return pflag.NormalizedName(name) 47 | } 48 | 49 | // InitFlags normalizes, parses, then logs the command line flags 50 | func InitFlags() { 51 | pflag.CommandLine.SetNormalizeFunc(WordSepNormalizeFunc) 52 | } 53 | 54 | // PrintFlags logs the flags in the flagset 55 | func PrintFlags(flags *pflag.FlagSet) { 56 | flags.VisitAll(func(flag *pflag.Flag) { 57 | log.Debug("Flag value has been parsed", log.Any(flag.Name, flag.Value)) 58 | }) 59 | } 60 | -------------------------------------------------------------------------------- /pkg/image-transfer/cmd.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making TKEStack 3 | * available. 4 | * 5 | * Copyright (C) 2012-2020 Tencent. All Rights Reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package imagetransfer 20 | 21 | 22 | import ( 23 | "fmt" 24 | "tkestack.io/image-transfer/pkg/log" 25 | "tkestack.io/image-transfer/pkg/image-transfer/options" 26 | "github.com/spf13/cobra" 27 | flagUtil "tkestack.io/image-transfer/pkg/flag" 28 | "os" 29 | ) 30 | 31 | 32 | // RunFunc defines the alias of the command entry function 33 | type RunFunc func(cmd *cobra.Command, args []string) 34 | 35 | // NewImageTransferCommand creates a *cobra.Command object with default parameters 36 | func NewImageTransferCommand(basename string) *cobra.Command { 37 | flagUtil.InitFlags() 38 | 39 | opts := options.NewClientOptions() 40 | cmd := &cobra.Command{ 41 | Use: basename, 42 | Long: "image-transfer", 43 | Run: run(opts), 44 | } 45 | 46 | opts.AddFlags(cmd.Flags()) 47 | log.AddFlags(cmd.Flags()) 48 | return cmd 49 | } 50 | 51 | func run(opts *options.ClientOptions) RunFunc { 52 | return func(cmd *cobra.Command, args []string) { 53 | log.InitLogger() 54 | defer log.FlushLogger() 55 | 56 | flagUtil.PrintFlags(cmd.Flags()) 57 | 58 | 59 | client, err := NewTransferClient(opts) 60 | if err != nil { 61 | log.Errorf("init Transfer Client error: %v", err) 62 | os.Exit(1) 63 | } 64 | 65 | if err := client.Run(); err != nil { 66 | fmt.Fprintf(os.Stderr, "%v\n", err) 67 | os.Exit(1) 68 | } 69 | 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /pkg/image-transfer/options/config.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making TKEStack 3 | * available. 4 | * 5 | * Copyright (C) 2012-2020 Tencent. All Rights Reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package options 20 | 21 | import ( 22 | "github.com/spf13/pflag" 23 | ) 24 | 25 | // ConfigOptions 基础配置信息 26 | type ConfigOptions struct { 27 | SecurityFile string 28 | RuleFile string 29 | RoutineNums int 30 | RetryNums int 31 | QPS int 32 | DefaultRegistry string 33 | DefaultNamespace string 34 | CCRToTCR bool 35 | CCRRegion string 36 | CCRTagNums int 37 | TCRRegion string 38 | TCRName string 39 | SecretFile string 40 | // if target tag is exist override it 41 | TagExistOverridden bool 42 | } 43 | 44 | // NewConfigOptions creates a NewConfigOptions object with default 45 | // parameters. 46 | func NewConfigOptions() *ConfigOptions { 47 | return &ConfigOptions{} 48 | } 49 | 50 | // Validate is used to parse and validate the parameters entered by the user at 51 | // the command line when the program starts. 52 | func (o *ConfigOptions) Validate() []error { 53 | var allErrors []error 54 | 55 | return allErrors 56 | } 57 | 58 | // AddFlags adds flags related to authenticate for a specific APIServer to the 59 | // specified FlagSet 60 | func (o *ConfigOptions) AddFlags(fs *pflag.FlagSet) { 61 | fs.StringVar(&o.SecurityFile, "securityFile", o.SecurityFile, 62 | "Get registry auth config from config file path") 63 | fs.StringVar(&o.RuleFile, "ruleFile", o.RuleFile, 64 | "Get images rules config from config file path") 65 | fs.StringVar(&o.DefaultRegistry, "registry", o.DefaultRegistry, 66 | "default destinate registry url when destinate registry is not "+ 67 | "given in the config file, can also be set with DEFAULT_REGISTRY environment value") 68 | fs.StringVar(&o.DefaultNamespace, "ns", o.DefaultNamespace, 69 | "default destinate namespace when destinate namespace is not"+ 70 | " given in the config file, can also be set with DEFAULT_NAMESPACE environment value") 71 | fs.IntVar(&o.RoutineNums, "routines", 5, 72 | "number of goroutines, default value is 5, max routines is 50") 73 | fs.IntVar(&o.RetryNums, "retry", 2, 74 | "number of retries, default value is 2") 75 | fs.IntVar(&o.CCRTagNums, "ccrTagNums", 100, 76 | "number of ccr recent tags for every repo, default value is 100, set 0 to sync all tag") 77 | fs.IntVar(&o.QPS, "qps", 100, 78 | "QPS of request, default value is 100, max is 30000") 79 | fs.BoolVar(&o.CCRToTCR, "ccrToTcr", false, 80 | "mode: transfer ccr images to tcr, default value is false") 81 | fs.StringVar(&o.CCRRegion, "ccrRegion", "ap-guangzhou", 82 | "ccr region, default value is ap-guangzhou. this flag is used when flag ccrToTcr=true") 83 | fs.StringVar(&o.TCRRegion, "tcrRegion", "ap-guangzhou", 84 | "tcr region, default value is ap-guangzhou. this flag is used when flag ccrToTcr=true") 85 | fs.StringVar(&o.TCRName, "tcrName", o.TCRName, 86 | "tcr name. this flag is used when flag ccrToTcr=true") 87 | fs.StringVar(&o.SecretFile, "secretFile", o.SecretFile, 88 | "Tencent Cloud secretId 、secretKey for access ccr and tcr. this flag is used when flag ccrToTcr=true") 89 | fs.BoolVar(&o.TagExistOverridden, "tag-exist-overridden", true, "if target tag is exist, override it") 90 | } 91 | -------------------------------------------------------------------------------- /pkg/image-transfer/options/options.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making TKEStack 3 | * available. 4 | * 5 | * Copyright (C) 2012-2020 Tencent. All Rights Reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package options 20 | 21 | import ( 22 | "github.com/spf13/pflag" 23 | ) 24 | 25 | // ClientOptions contains the configuration required for api server to run. 26 | type ClientOptions struct { 27 | Config *ConfigOptions 28 | } 29 | 30 | // NewClientOptions creates a new ClientOptions object with default 31 | // parameters. 32 | func NewClientOptions() *ClientOptions { 33 | return &ClientOptions{ 34 | Config: NewConfigOptions(), 35 | } 36 | } 37 | 38 | // AddFlags adds flags for a specific api server to the specified FlagSet object. 39 | func (o *ClientOptions) AddFlags(fs *pflag.FlagSet) { 40 | o.Config.AddFlags(fs) 41 | } 42 | 43 | // Validate checks APIServerOptions and return a slice of found errors. 44 | func (o *ClientOptions) Validate() []error { 45 | var errors []error 46 | 47 | 48 | 49 | if errs := o.Config.Validate(); len(errs) > 0 { 50 | errors = append(errors, errs...) 51 | } 52 | 53 | 54 | return errors 55 | } 56 | -------------------------------------------------------------------------------- /pkg/image-transfer/run.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making TKEStack 3 | * available. 4 | * 5 | * Copyright (C) 2012-2020 Tencent. All Rights Reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package imagetransfer 20 | 21 | import ( 22 | "container/list" 23 | "fmt" 24 | "strings" 25 | "sync" 26 | "time" 27 | 28 | "tkestack.io/image-transfer/configs" 29 | "tkestack.io/image-transfer/pkg/apis/ccrapis" 30 | "tkestack.io/image-transfer/pkg/apis/tcrapis" 31 | "tkestack.io/image-transfer/pkg/image-transfer/options" 32 | "tkestack.io/image-transfer/pkg/log" 33 | "tkestack.io/image-transfer/pkg/transfer" 34 | "tkestack.io/image-transfer/pkg/utils" 35 | ) 36 | 37 | //Client is a transfer client 38 | type Client struct { 39 | // a Transfer.Job list 40 | jobList *list.List 41 | 42 | // a URLPair list store origin url pair from rule file 43 | urlPairList *list.List 44 | // store standard image url 45 | normalURLPairList *list.List 46 | 47 | // failed list 48 | failedJobList *list.List 49 | failedJobGenerateList *list.List 50 | failedGenNormalURLPairList *list.List 51 | 52 | config *configs.Configs 53 | 54 | //finished generate ccrToTcr urlPair 55 | urlPairFinished bool 56 | // mutex 57 | jobListMutex sync.Mutex 58 | urlPairListMutex sync.Mutex 59 | normalURLPairListMutex sync.Mutex 60 | failedJobListMutex sync.Mutex 61 | failedJobGenerateListMutex sync.Mutex 62 | failedGenNormalURLPairListMutex sync.Mutex 63 | urlPairFinishedMutex sync.Mutex 64 | } 65 | 66 | // URLPair is a pair of source and target url 67 | type URLPair struct { 68 | source string 69 | target string 70 | } 71 | 72 | // Run is main function of a transfer client 73 | func (c *Client) Run() error { 74 | 75 | if c.config.FlagConf.Config.CCRToTCR { 76 | return c.CCRToTCRTransfer() 77 | } 78 | 79 | return c.NormalTransfer(c.config.ImageList, nil, nil, nil) 80 | 81 | } 82 | 83 | //CCRToTCRTransfer transfer ccr to tcr 84 | func (c *Client) CCRToTCRTransfer() error { 85 | 86 | ccrClient := ccrapis.NewCCRAPIClient() 87 | ccrNs, err := ccrClient.GetAllNamespaceByName(c.config.Secret, c.config.FlagConf.Config.CCRRegion) 88 | log.Debugf("ccr namespaces is %s", ccrNs) 89 | if err != nil { 90 | log.Errorf("Get ccr ns returned error: %s", err) 91 | return err 92 | } 93 | 94 | tcrClient := tcrapis.NewTCRAPIClient() 95 | tcrNs, tcrID, err := tcrClient.GetAllNamespaceByName(c.config.Secret, 96 | c.config.FlagConf.Config.TCRRegion, c.config.FlagConf.Config.TCRName) 97 | log.Debugf("tcr namespaces is %s", tcrNs) 98 | if err != nil { 99 | log.Errorf("Get tcr ns returned error: %s", err) 100 | return err 101 | } 102 | 103 | //create ccr ns in tcr 104 | failedNsList, err := c.CreateTcrNs(tcrClient, ccrNs, tcrNs, c.config.Secret, c.config.FlagConf.Config.TCRRegion, tcrID) 105 | if err != nil { 106 | log.Errorf("CreateTcrNs error: %s", err) 107 | return err 108 | } 109 | 110 | //retry failedNsList 111 | if len(failedNsList) != 0 { 112 | log.Infof("some ccr namespace create failed in tcr, retry Create Tcr Ns.") 113 | for times := 0; times < c.config.FlagConf.Config.RetryNums && len(failedNsList) != 0; times++ { 114 | tmpFailedNsList, err := c.RetryCreateTcrNs(tcrClient, failedNsList, 115 | c.config.Secret, c.config.FlagConf.Config.TCRRegion) 116 | if err != nil { 117 | continue 118 | } else { 119 | failedNsList = tmpFailedNsList 120 | } 121 | } 122 | } 123 | 124 | if len(failedNsList) != 0 { 125 | log.Warnf("some ccr namespace create failed in tcr: %s", failedNsList) 126 | } 127 | 128 | //generate transfer rules 129 | repoChan, err := c.GenerateCcrToTcrRules(failedNsList, ccrClient, c.config.Secret, c.config.FlagConf.Config.CCRRegion, 130 | c.config.FlagConf.Config.TCRRegion, c.config.FlagConf.Config.TCRName) 131 | if err != nil { 132 | return err 133 | } 134 | 135 | return c.NormalTransfer(nil, ccrClient, tcrClient, repoChan) 136 | 137 | } 138 | 139 | //GenerateCcrToTcrRules generate rules of ccr transfer to tcr 140 | func (c *Client) GenerateCcrToTcrRules(failedNsList []string, ccrClient *ccrapis.CCRAPIClient, 141 | secret map[string]configs.Secret, ccrRegion string, tcrRegion string, tcrName string) (chan string, error) { 142 | 143 | secretID, secretKey, err := ccrapis.GetCcrSecret(secret) 144 | if err != nil { 145 | log.Errorf("GetCcrSecret error: %s", err) 146 | return nil, err 147 | } 148 | resp, err := ccrClient.DescribeRepositoryOwnerPersonal(secretID, secretKey, ccrRegion, 0, 1) 149 | if err != nil { 150 | log.Errorf("get ccr repo count error, %s", err) 151 | return nil, fmt.Errorf("get ccr repo count error %s", err) 152 | } 153 | totalRepo := *resp.Response.Data.TotalCount 154 | log.Debugf("total repo is %d", totalRepo) 155 | repoChan := make(chan string, totalRepo) 156 | err = ccrClient.GetAllCcrRepo(secret, ccrRegion, failedNsList, tcrRegion, tcrName, repoChan, totalRepo) 157 | if err != nil { 158 | log.Errorf("get ccr repo to tcr rules failed: %s", err) 159 | return nil, err 160 | } 161 | 162 | return repoChan, nil 163 | 164 | } 165 | 166 | //RetryCreateTcrNs retry to create tcr namespaces 167 | func (c *Client) RetryCreateTcrNs(tcrClient *tcrapis.TCRAPIClient, retryList []string, 168 | secret map[string]configs.Secret, region string) ([]string, error) { 169 | var failedList []string 170 | 171 | secretID, secretKey, err := tcrapis.GetTcrSecret(secret) 172 | if err != nil { 173 | log.Errorf("GetTcrSecret error: %s", err) 174 | return failedList, err 175 | } 176 | 177 | tcrNs, tcrID, err := tcrClient.GetAllNamespaceByName(c.config.Secret, 178 | c.config.FlagConf.Config.TCRRegion, c.config.FlagConf.Config.TCRName) 179 | log.Debugf("tcr namespaces is %s", tcrNs) 180 | if err != nil { 181 | log.Errorf("retry create tcr ns, get tcr ns error: ", err) 182 | return nil, err 183 | } 184 | 185 | for _, ns := range retryList { 186 | if !utils.IsContain(tcrNs, ns) { 187 | _, err := tcrClient.CreateNamespace(secretID, secretKey, region, tcrID, ns) 188 | if err != nil { 189 | log.Errorf("tcr CreateNamespace %s error: %s", ns, err) 190 | failedList = append(failedList, ns) 191 | } 192 | } 193 | } 194 | 195 | return failedList, nil 196 | 197 | } 198 | 199 | //CreateTcrNs create tcr namespaces 200 | func (c *Client) CreateTcrNs(tcrClient *tcrapis.TCRAPIClient, ccrNs, tcrNs []string, 201 | secret map[string]configs.Secret, region string, tcrID string) ([]string, error) { 202 | 203 | var failedList []string 204 | 205 | secretID, secretKey, err := tcrapis.GetTcrSecret(secret) 206 | 207 | if err != nil { 208 | log.Errorf("GetTcrSecret error: %s", err) 209 | return failedList, err 210 | } 211 | 212 | for _, ns := range ccrNs { 213 | if !utils.IsContain(tcrNs, ns) { 214 | log.Infof("create namespace %s", ns) 215 | _, err := tcrClient.CreateNamespace(secretID, secretKey, region, tcrID, ns) 216 | if err != nil { 217 | log.Errorf("tcr CreateNamespace %s error: %s", ns, err) 218 | failedList = append(failedList, ns) 219 | } 220 | } 221 | } 222 | 223 | return failedList, nil 224 | 225 | } 226 | 227 | //NormalTransfer is the normal mode of transfer 228 | func (c *Client) NormalTransfer(imageList map[string]string, ccrClient *ccrapis.CCRAPIClient, tcrClient *tcrapis.TCRAPIClient, repoChan chan string) error { 229 | jobListChan := make(chan *transfer.Job, c.config.FlagConf.Config.RoutineNums) 230 | fmt.Println("Start to handle transfer jobs, please wait ...") 231 | wg := sync.WaitGroup{} 232 | 233 | // generate goroutines to handle transfer jobs 234 | wg.Add(1) 235 | go func() { 236 | defer wg.Done() 237 | c.jobsHandler(jobListChan) 238 | }() 239 | 240 | // ccrToTcr progress is (ccr api)repo --> repochan --> NormalPairList --> jobListChan 241 | if c.config.FlagConf.Config.CCRToTCR { 242 | wg.Add(1) 243 | go func() { 244 | defer wg.Done() 245 | c.HandleCcrToTCrTags(repoChan) 246 | c.SetURLPairFinished() 247 | }() 248 | } else { 249 | // Normal progress is urlPairList --> NormalPairList --> jobListChan 250 | for source, target := range imageList { 251 | c.urlPairList.PushBack(&URLPair{ 252 | source: source, 253 | target: target, 254 | }) 255 | } 256 | 257 | // carry urlPair to NormalURLPair 258 | wg.Add(1) 259 | go func() { 260 | defer wg.Done() 261 | c.HandleURLPair() 262 | c.SetURLPairFinished() 263 | }() 264 | } 265 | 266 | // get item from NormalURLPair to jobListChan 267 | c.rulesHandler(jobListChan) 268 | 269 | wg.Wait() 270 | 271 | log.Infof("Start to retry failed jobs...") 272 | 273 | for times := 0; times < c.config.FlagConf.Config.RetryNums; times++ { 274 | log.Debugf("failedjobList len %d, failedGenNormalURLPairList len %d, failedJobGenerateList len %d", c.failedJobList.Len(), c.failedGenNormalURLPairList.Len(), c.failedJobGenerateList.Len()) 275 | c.Retry() 276 | } 277 | 278 | if c.failedJobList.Len() != 0 { 279 | log.Infof("################# %v failed transfer jobs: #################", c.failedJobList.Len()) 280 | for e := c.failedJobList.Front(); e != nil; e = e.Next() { 281 | log.Infof(e.Value.(*transfer.Job).Source.GetRegistry() + "/" + 282 | e.Value.(*transfer.Job).Source.GetRepository() + ":" + e.Value.(*transfer.Job).Source.GetTag()) 283 | 284 | } 285 | } 286 | 287 | if c.failedGenNormalURLPairList.Len() != 0 { 288 | log.Infof("################# %v failed generate Normal urlPair: #################", c.failedGenNormalURLPairList.Len()) 289 | for e := c.failedGenNormalURLPairList.Front(); e != nil; e = e.Next() { 290 | log.Infof(e.Value.(*URLPair).source + ": " + e.Value.(*URLPair).target) 291 | 292 | } 293 | } 294 | 295 | if c.failedJobGenerateList.Len() != 0 { 296 | log.Infof("################# %v failed generate jobs: #################", c.failedJobGenerateList.Len()) 297 | for e := c.failedJobGenerateList.Front(); e != nil; e = e.Next() { 298 | log.Infof(e.Value.(*URLPair).source + ": " + e.Value.(*URLPair).target) 299 | 300 | } 301 | } 302 | 303 | log.Infof("################# Finished, %v transfer jobs failed, %v normal urlPair generate failed, %v jobs generate failed #################", 304 | c.failedJobList.Len(), c.failedGenNormalURLPairList.Len(), c.failedJobGenerateList.Len()) 305 | 306 | return nil 307 | 308 | } 309 | 310 | //Retry is retry the failed job 311 | func (c *Client) Retry() { 312 | retryJobListChan := make(chan *transfer.Job, c.config.FlagConf.Config.RoutineNums) 313 | 314 | wg1 := sync.WaitGroup{} 315 | wg1.Add(1) 316 | go func() { 317 | defer func() { 318 | wg1.Done() 319 | }() 320 | c.jobsHandler(retryJobListChan) 321 | }() 322 | 323 | if c.failedJobList.Len() != 0 { 324 | failedJobListChan := make(chan *transfer.Job, c.failedJobList.Len()) 325 | for { 326 | failedJob := c.failedJobList.Front() 327 | if failedJob == nil { 328 | break 329 | } 330 | log.Infof("put failed job to failedJobListChan %s/%s:%s %s/%s:%s", failedJob.Value.(*transfer.Job).Source.GetRegistry(), failedJob.Value.(*transfer.Job).Source.GetRepository(), failedJob.Value.(*transfer.Job).Source.GetTag(), failedJob.Value.(*transfer.Job).Target.GetRegistry(), failedJob.Value.(*transfer.Job).Target.GetRepository(), failedJob.Value.(*transfer.Job).Target.GetTag()) 331 | failedJobListChan <- failedJob.Value.(*transfer.Job) 332 | c.failedJobList.Remove(failedJob) 333 | } 334 | close(failedJobListChan) 335 | wg1.Add(1) 336 | go func() { 337 | defer wg1.Done() 338 | c.jobsHandler(failedJobListChan) 339 | }() 340 | } 341 | 342 | if c.failedGenNormalURLPairList.Len() != 0 || c.failedJobGenerateList.Len() != 0 { 343 | if c.failedGenNormalURLPairList.Len() != 0 { 344 | c.urlPairList.PushBackList(c.failedGenNormalURLPairList) 345 | c.failedGenNormalURLPairList.Init() 346 | if !c.config.FlagConf.Config.CCRToTCR { 347 | c.HandleURLPair() 348 | } else { 349 | c.CcrtoTcrGenTagRetry() 350 | } 351 | } 352 | 353 | if c.failedJobGenerateList.Len() != 0 { 354 | c.normalURLPairList.PushBackList(c.failedJobGenerateList) 355 | c.failedJobGenerateList.Init() 356 | } 357 | c.rulesHandler(retryJobListChan) 358 | } else { 359 | close(retryJobListChan) 360 | } 361 | 362 | wg1.Wait() 363 | } 364 | 365 | // NewTransferClient creates a transfer client 366 | func NewTransferClient(opts *options.ClientOptions) (*Client, error) { 367 | 368 | clientConfig, err := configs.InitConfigs(opts) 369 | 370 | if err != nil { 371 | return nil, err 372 | } 373 | 374 | return &Client{ 375 | jobList: list.New(), 376 | urlPairList: list.New(), 377 | failedJobList: list.New(), 378 | failedJobGenerateList: list.New(), 379 | normalURLPairList: list.New(), 380 | failedGenNormalURLPairList: list.New(), 381 | config: clientConfig, 382 | jobListMutex: sync.Mutex{}, 383 | urlPairListMutex: sync.Mutex{}, 384 | failedJobListMutex: sync.Mutex{}, 385 | failedJobGenerateListMutex: sync.Mutex{}, 386 | urlPairFinishedMutex: sync.Mutex{}, 387 | failedGenNormalURLPairListMutex: sync.Mutex{}, 388 | }, nil 389 | } 390 | 391 | func (c *Client) rulesHandler(jobListChan chan *transfer.Job) { 392 | defer func() { 393 | close(jobListChan) 394 | }() 395 | 396 | routineNum := c.config.FlagConf.Config.RoutineNums 397 | wg := sync.WaitGroup{} 398 | 399 | for i := 0; i < routineNum; i++ { 400 | wg.Add(1) 401 | go func() { 402 | defer wg.Done() 403 | defer log.Debugf("exit rule handler main loop") 404 | for { 405 | urlPair, empty := c.GetNormalURLPair() 406 | // no more job to generate 407 | if empty && c.IsURLPairFinished() { 408 | log.Debugf("url pair is empty") 409 | break 410 | } 411 | if empty { 412 | log.Debugf("not finieshed but url pair is empty") 413 | time.Sleep(100 * time.Millisecond) 414 | continue 415 | } 416 | log.Infof("generate job source %s, target %s", urlPair.source, urlPair.target) 417 | err := c.GenerateTransferJob(jobListChan, urlPair.source, urlPair.target) 418 | if err != nil { 419 | log.Errorf("Generate transfer job %s to %s error: %v", urlPair.source, urlPair.target, err) 420 | // put to failedJobGenerateList 421 | c.PutAFailedURLPair(urlPair) 422 | } 423 | } 424 | }() 425 | } 426 | wg.Wait() 427 | } 428 | 429 | func (c *Client) jobsHandler(jobListChan chan *transfer.Job) { 430 | 431 | routineNum := c.config.FlagConf.Config.RoutineNums 432 | wg := sync.WaitGroup{} 433 | for i := 0; i < routineNum; i++ { 434 | wg.Add(1) 435 | go func() { 436 | defer wg.Done() 437 | for { 438 | job, ok := <-jobListChan 439 | if !ok { 440 | break 441 | } 442 | if err := job.Run(); err != nil { 443 | log.Errorf("handle job failed %s/%s:%s, %s", job.Source.GetRegistry(), job.Source.GetRepository(), job.Source.GetTag(), err) 444 | c.PutAFailedJob(job) 445 | } 446 | } 447 | }() 448 | } 449 | 450 | wg.Wait() 451 | 452 | } 453 | 454 | // GetURLPair gets a URLPair from urlPairList 455 | func (c *Client) GetURLPair() (*URLPair, bool) { 456 | c.urlPairListMutex.Lock() 457 | defer func() { 458 | c.urlPairListMutex.Unlock() 459 | }() 460 | 461 | urlPair := c.urlPairList.Front() 462 | if urlPair == nil { 463 | return nil, true 464 | } 465 | c.urlPairList.Remove(urlPair) 466 | 467 | return urlPair.Value.(*URLPair), false 468 | } 469 | 470 | // PutURLPair puts a URLPair to urlPairList 471 | func (c *Client) PutURLPair(urlPair *URLPair) { 472 | c.urlPairListMutex.Lock() 473 | defer c.urlPairListMutex.Unlock() 474 | c.urlPairList.PushBack(urlPair) 475 | } 476 | 477 | // GetNormalURLPair gets a URLPair from normalurlPairList 478 | func (c *Client) GetNormalURLPair() (*URLPair, bool) { 479 | c.normalURLPairListMutex.Lock() 480 | defer func() { 481 | c.normalURLPairListMutex.Unlock() 482 | }() 483 | 484 | urlPair := c.normalURLPairList.Front() 485 | if urlPair == nil { 486 | return nil, true 487 | } 488 | c.normalURLPairList.Remove(urlPair) 489 | 490 | return urlPair.Value.(*URLPair), false 491 | } 492 | 493 | // PutNormalURLPair puts a URLPair to normalurlPairList 494 | func (c *Client) PutNormalURLPair(urlPair *URLPair) { 495 | c.normalURLPairListMutex.Lock() 496 | defer c.normalURLPairListMutex.Unlock() 497 | c.normalURLPairList.PushBack(urlPair) 498 | } 499 | 500 | // GetJob return a transfer.Job struct if the job list is not empty 501 | func (c *Client) GetJob() (*transfer.Job, bool) { 502 | c.jobListMutex.Lock() 503 | defer func() { 504 | c.jobListMutex.Unlock() 505 | }() 506 | 507 | job := c.jobList.Front() 508 | if job == nil { 509 | return nil, true 510 | } 511 | c.jobList.Remove(job) 512 | 513 | return job.Value.(*transfer.Job), false 514 | } 515 | 516 | // PutJob puts a transfer.Job struct to job list 517 | func (c *Client) PutJob(job *transfer.Job) { 518 | c.jobListMutex.Lock() 519 | defer func() { 520 | c.jobListMutex.Unlock() 521 | }() 522 | 523 | if c.jobList != nil { 524 | c.jobList.PushBack(job) 525 | } 526 | } 527 | 528 | // GenerateTransferJob creates transfer jobs from normalURLPair 529 | func (c *Client) GenerateTransferJob(jobListChan chan *transfer.Job, source string, target string) error { 530 | if source == "" { 531 | return fmt.Errorf("source url should not be empty") 532 | } 533 | 534 | sourceURL, err := utils.NewRepoURL(source) 535 | if err != nil { 536 | return fmt.Errorf("url %s format error: %v", source, err) 537 | } 538 | 539 | if target == "" { 540 | return fmt.Errorf("target url should not be empty") 541 | } 542 | 543 | targetURL, err := utils.NewRepoURL(target) 544 | if err != nil { 545 | return fmt.Errorf("url %s format error: %v", target, err) 546 | } 547 | 548 | // if tag is not specific 549 | if sourceURL.GetTag() == "" { 550 | return fmt.Errorf("source tag empty, source: %s", sourceURL.GetURL()) 551 | } 552 | 553 | if targetURL.GetTag() == "" { 554 | return fmt.Errorf("target tag empty, target: %s", targetURL.GetURL()) 555 | } 556 | 557 | var imageSource *transfer.ImageSource 558 | var imageTarget *transfer.ImageTarget 559 | 560 | sourceSecurity, exist := c.config.GetSecuritySpecific(sourceURL.GetRegistry(), sourceURL.GetNamespace()) 561 | if exist { 562 | log.Infof("Find auth information for %v, username: %v", sourceURL.GetURL(), sourceSecurity.Username) 563 | } else { 564 | log.Infof("Cannot find auth information for %v, pull actions will be anonymous", sourceURL.GetURL()) 565 | } 566 | 567 | targetSecurity, exist := c.config.GetSecuritySpecific(targetURL.GetRegistry(), targetURL.GetNamespace()) 568 | if exist { 569 | log.Infof("Find auth information for %v, username: %v", targetURL.GetURL(), targetSecurity.Username) 570 | 571 | } else { 572 | log.Infof("Cannot find auth information for %v, push actions will be anonymous", targetURL.GetURL()) 573 | } 574 | 575 | imageSource, err = transfer.NewImageSource(sourceURL.GetRegistry(), sourceURL.GetRepoWithNamespace(), sourceURL.GetTag(), sourceSecurity.Username, sourceSecurity.Password, sourceSecurity.Insecure) 576 | if err != nil { 577 | return fmt.Errorf("generate %s image source error: %v", sourceURL.GetURL(), err) 578 | } 579 | 580 | imageTarget, err = transfer.NewImageTarget(targetURL.GetRegistry(), targetURL.GetRepoWithNamespace(), targetURL.GetTag(), targetSecurity.Username, targetSecurity.Password, targetSecurity.Insecure) 581 | if err != nil { 582 | return fmt.Errorf("generate %s image target error: %v", sourceURL.GetURL(), err) 583 | } 584 | 585 | jobListChan <- transfer.NewJob(imageSource, imageTarget) 586 | 587 | log.Infof("Generate a job for %s to %s", sourceURL.GetURL(), targetURL.GetURL()) 588 | return nil 589 | } 590 | 591 | // GetFailedJob gets a failed job from failedJobList 592 | func (c *Client) GetFailedJob() (*transfer.Job, bool) { 593 | c.failedJobListMutex.Lock() 594 | defer func() { 595 | c.failedJobListMutex.Unlock() 596 | }() 597 | 598 | failedJob := c.failedJobList.Front() 599 | if failedJob == nil { 600 | return nil, true 601 | } 602 | c.failedJobList.Remove(failedJob) 603 | 604 | return failedJob.Value.(*transfer.Job), false 605 | } 606 | 607 | // PutAFailedJob puts a failed job to failedJobList 608 | func (c *Client) PutAFailedJob(failedJob *transfer.Job) { 609 | 610 | c.failedJobListMutex.Lock() 611 | defer func() { 612 | c.failedJobListMutex.Unlock() 613 | }() 614 | 615 | if c.failedJobList != nil { 616 | c.failedJobList.PushBack(failedJob) 617 | } 618 | } 619 | 620 | // GetAFailedURLPair get a URLPair from failedJobGenerateList 621 | func (c *Client) GetAFailedURLPair() (*URLPair, bool) { 622 | c.failedJobGenerateListMutex.Lock() 623 | defer func() { 624 | c.failedJobGenerateListMutex.Unlock() 625 | }() 626 | 627 | failedURLPair := c.failedJobGenerateList.Front() 628 | if failedURLPair == nil { 629 | return nil, true 630 | } 631 | c.failedJobGenerateList.Remove(failedURLPair) 632 | 633 | return failedURLPair.Value.(*URLPair), false 634 | } 635 | 636 | // PutAFailedURLPair puts a URLPair to failedJobGenerateList 637 | func (c *Client) PutAFailedURLPair(failedURLPair *URLPair) { 638 | c.failedJobGenerateListMutex.Lock() 639 | defer func() { 640 | c.failedJobGenerateListMutex.Unlock() 641 | }() 642 | 643 | if c.failedJobGenerateList != nil { 644 | c.failedJobGenerateList.PushBack(failedURLPair) 645 | } 646 | 647 | } 648 | 649 | // GetAFailedGenNormalURLPair get a URLPair from failedGenNormalURLPairList 650 | func (c *Client) GetAFailedGenNormalURLPair() (*URLPair, bool) { 651 | c.failedGenNormalURLPairListMutex.Lock() 652 | defer func() { 653 | c.failedGenNormalURLPairListMutex.Unlock() 654 | }() 655 | 656 | failedURLPair := c.failedGenNormalURLPairList.Front() 657 | if failedURLPair == nil { 658 | return nil, true 659 | } 660 | c.failedGenNormalURLPairList.Remove(failedURLPair) 661 | 662 | return failedURLPair.Value.(*URLPair), false 663 | } 664 | 665 | // PutAFailedGenNormalURLPair puts a URLPair to failedGenNormalURLPairList 666 | func (c *Client) PutAFailedGenNormalURLPair(failedURLPair *URLPair) { 667 | c.failedGenNormalURLPairListMutex.Lock() 668 | defer func() { 669 | c.failedGenNormalURLPairListMutex.Unlock() 670 | }() 671 | 672 | if c.failedGenNormalURLPairList != nil { 673 | c.failedGenNormalURLPairList.PushBack(failedURLPair) 674 | } 675 | 676 | } 677 | 678 | // GenJobFilterTag is hornor by TagExistOverridden policy, skip generate job if tag in target and tag digest is same 679 | func (c *Client) GenJobFilterTag(sourceTags, targetTags []string, sourceURL, targetURL *utils.RepoURL, sourceSecurity, targetSecurity configs.Security, wg *sync.WaitGroup) { 680 | tagChan := make(chan string, len(sourceTags)) 681 | wg.Add(1) 682 | go func() { 683 | defer wg.Done() 684 | for _, tag := range sourceTags { 685 | tagChan <- tag 686 | } 687 | close(tagChan) 688 | }() 689 | 690 | for i := 0; i < 10; i++ { 691 | wg.Add(1) 692 | go func() { 693 | defer wg.Done() 694 | for tag := range tagChan { 695 | urlPair := &URLPair{ 696 | source: sourceURL.GetURL() + ":" + tag, 697 | target: targetURL.GetURL() + ":" + tag, 698 | } 699 | 700 | log.Debugf("handle tag %s", urlPair.source) 701 | //source tag exist in target 702 | if utils.IsContain(targetTags, tag) { 703 | if !c.config.FlagConf.Config.TagExistOverridden { 704 | log.Warnf("Skip push image, target image %s/%s:%s already exist, flag \"--tag-exist-overridden\" is set so skip", targetURL.GetRegistry(), targetURL.GetRepoWithNamespace(), tag) 705 | continue 706 | } 707 | imageSource, err := transfer.NewImageSource(sourceURL.GetRegistry(), sourceURL.GetRepoWithNamespace(), 708 | tag, sourceSecurity.Username, sourceSecurity.Password, sourceSecurity.Insecure) 709 | if err != nil { 710 | log.Errorf("generate %s image source error: %v", sourceURL.GetURL(), err) 711 | c.PutAFailedGenNormalURLPair(urlPair) 712 | continue 713 | } 714 | 715 | imageTarget, err := transfer.NewImageTarget(targetURL.GetRegistry(), targetURL.GetRepoWithNamespace(), tag, targetSecurity.Username, targetSecurity.Password, targetSecurity.Insecure) 716 | if err != nil { 717 | log.Errorf("generate %s image target error: %v", targetURL.GetURL(), err) 718 | c.PutAFailedGenNormalURLPair(urlPair) 719 | continue 720 | } 721 | sourceDigest, err := imageSource.GetImageDigest() 722 | if err != nil { 723 | log.Errorf("Failed to get source image digest from %s/%s:%s error: %v", imageSource.GetRegistry(), imageSource.GetRepository(), tag, err) 724 | c.PutAFailedGenNormalURLPair(urlPair) 725 | continue 726 | } 727 | targetDigest, err := imageTarget.GetImageDigest() 728 | if err != nil { 729 | log.Errorf("Failed to get target image digest from %s/%s:%s error: %v", imageTarget.GetRegistry(), imageTarget.GetRepository(), tag, err) 730 | c.PutAFailedGenNormalURLPair(urlPair) 731 | continue 732 | } 733 | 734 | if sourceDigest == targetDigest { 735 | log.Infof("Skip push image, target image %s/%s:%s already exist and has same digest %s", imageTarget.GetRegistry(), imageTarget.GetRepository(), imageTarget.GetTag(), sourceDigest) 736 | continue 737 | } 738 | 739 | if targetDigest != "" { 740 | log.Warnf("Target image %s/%s:%s already exist, target digest %s to be override as source digest %s", imageTarget.GetRegistry(), imageTarget.GetRepository(), imageTarget.GetTag(), targetDigest, sourceDigest) 741 | } 742 | } 743 | log.Infof("put normal url pair %v", urlPair) 744 | c.PutNormalURLPair(urlPair) 745 | } 746 | }() 747 | } 748 | } 749 | 750 | // HandleCcrToTCrTags get tags from ccr api and generate urlPair by filter tag 751 | func (c *Client) HandleCcrToTCrTags(repoChan chan string) error { 752 | wg := sync.WaitGroup{} 753 | for i := 0; i < 5; i++ { 754 | wg.Add(1) 755 | go func() { 756 | defer wg.Done() 757 | for ccrRepo := range repoChan { 758 | log.Infof("ccr repo is %s", ccrRepo) 759 | source := fmt.Sprintf("%s%s%s", ccrapis.RegionPrefix[c.config.FlagConf.Config.CCRRegion], ".ccs.tencentyun.com/", ccrRepo) 760 | target := c.config.FlagConf.Config.TCRName + ".tencentcloudcr.com/" + ccrRepo 761 | urlPair := &URLPair{ 762 | source: source, 763 | target: target, 764 | } 765 | 766 | err := c.GenCcrtoTcrTagURLPair(source, target, &wg) 767 | if err != nil { 768 | c.PutAFailedGenNormalURLPair(urlPair) 769 | log.Errorf("Handle repoChan tags failed error, %s", err) 770 | } 771 | } 772 | }() 773 | } 774 | wg.Wait() 775 | return nil 776 | } 777 | 778 | // SetURLPairFinished set finished flag 779 | func (c *Client) SetURLPairFinished() { 780 | c.urlPairFinishedMutex.Lock() 781 | defer c.urlPairFinishedMutex.Unlock() 782 | c.urlPairFinished = true 783 | } 784 | 785 | // IsURLPairFinished locked and return c.urlPairFinished 786 | func (c *Client) IsURLPairFinished() bool { 787 | c.urlPairFinishedMutex.Lock() 788 | defer c.urlPairFinishedMutex.Unlock() 789 | return c.urlPairFinished 790 | } 791 | 792 | // GenTagURLPair is generate normal image url that containt tag 793 | func (c *Client) GenTagURLPair(source string, target string, wg *sync.WaitGroup) error { 794 | if source == "" { 795 | return fmt.Errorf("source url should not be empty") 796 | } 797 | 798 | sourceURL, err := utils.NewRepoURL(source) 799 | if err != nil { 800 | return fmt.Errorf("url %s format error: %v", source, err) 801 | } 802 | 803 | // if dest is not specific, use default registry and src repo 804 | if target == "" { 805 | if c.config.FlagConf.Config.DefaultRegistry != "" { 806 | target = c.config.FlagConf.Config.DefaultRegistry + "/" + 807 | sourceURL.GetNamespace() + "/" + sourceURL.GetRepoWithTag() 808 | } else { 809 | return fmt.Errorf("the default registry and namespace should not be nil if you want to use them") 810 | } 811 | } 812 | 813 | targetURL, err := utils.NewRepoURL(target) 814 | if err != nil { 815 | return fmt.Errorf("url %s format error: %v", target, err) 816 | } 817 | 818 | var imageSource *transfer.ImageSource 819 | var imageTarget *transfer.ImageTarget 820 | 821 | sourceSecurity, exist := c.config.GetSecuritySpecific(sourceURL.GetRegistry(), sourceURL.GetNamespace()) 822 | if exist { 823 | log.Infof("Find auth information for %v, username: %v", sourceURL.GetURL(), sourceSecurity.Username) 824 | } else { 825 | log.Infof("Cannot find auth information for %v, pull actions will be anonymous", sourceURL.GetURL()) 826 | } 827 | 828 | targetSecurity, exist := c.config.GetSecuritySpecific(targetURL.GetRegistry(), targetURL.GetNamespace()) 829 | if exist { 830 | log.Infof("Find auth information for %v, username: %v", targetURL.GetURL(), targetSecurity.Username) 831 | 832 | } else { 833 | log.Infof("Cannot find auth information for %v, push actions will be anonymous", targetURL.GetURL()) 834 | } 835 | 836 | // multi-tags config 837 | tags := sourceURL.GetTag() 838 | if moreTag := strings.Split(tags, ","); len(moreTag) > 1 { 839 | if targetURL.GetTag() != "" && targetURL.GetTag() != sourceURL.GetTag() { 840 | return fmt.Errorf("multi-tags source should not correspond to a target with tag: %s:%s", 841 | sourceURL.GetURL(), targetURL.GetURL()) 842 | } 843 | log.Debugf("source %s tags is %s", sourceURL.GetURL(), moreTag) 844 | imageTarget, err = transfer.NewImageTarget(targetURL.GetRegistry(), targetURL.GetRepoWithNamespace(), targetURL.GetTag(), targetSecurity.Username, targetSecurity.Password, targetSecurity.Insecure) 845 | if err != nil { 846 | return fmt.Errorf("generate %s image target error: %v", sourceURL.GetURL(), err) 847 | } 848 | targetTags, err := imageTarget.GetTargetRepoTags() 849 | log.Debugf("target %s tags is %s", targetURL.GetURL(), targetTags) 850 | if err != nil { 851 | return fmt.Errorf("get tags failed from %s error: %v", targetURL.GetURL(), err) 852 | } 853 | c.GenJobFilterTag(moreTag, targetTags, sourceURL, targetURL, sourceSecurity, targetSecurity, wg) 854 | return nil 855 | } 856 | 857 | imageSource, err = transfer.NewImageSource(sourceURL.GetRegistry(), sourceURL.GetRepoWithNamespace(), sourceURL.GetTag(), sourceSecurity.Username, sourceSecurity.Password, sourceSecurity.Insecure) 858 | if err != nil { 859 | return fmt.Errorf("generate %s image source error: %v", sourceURL.GetURL(), err) 860 | } 861 | 862 | imageTarget, err = transfer.NewImageTarget(targetURL.GetRegistry(), targetURL.GetRepoWithNamespace(), targetURL.GetTag(), targetSecurity.Username, targetSecurity.Password, targetSecurity.Insecure) 863 | if err != nil { 864 | return fmt.Errorf("generate %s image target error: %v", sourceURL.GetURL(), err) 865 | } 866 | 867 | // if tag is not specific, return tags 868 | if sourceURL.GetTag() == "" { 869 | if targetURL.GetTag() != "" { 870 | return fmt.Errorf("not allow source tag empty and target tag not empty, both side of the config: %s:%s", 871 | sourceURL.GetURL(), targetURL.GetURL()) 872 | } 873 | 874 | // get all tags of this source repo 875 | sourceTags, err := imageSource.GetSourceRepoTags() 876 | log.Debugf("source %s tags is %s", sourceURL.GetURL(), sourceTags) 877 | if err != nil { 878 | return fmt.Errorf("get tags failed from %s error: %v", sourceURL.GetURL(), err) 879 | } 880 | 881 | targetTags, err := imageTarget.GetTargetRepoTags() 882 | log.Debugf("target %s tags is %s", targetURL.GetURL(), targetTags) 883 | if err != nil { 884 | return fmt.Errorf("get tags failed from %s error: %v", targetURL.GetURL(), err) 885 | } 886 | 887 | c.GenJobFilterTag(sourceTags, targetTags, sourceURL, targetURL, sourceSecurity, targetSecurity, wg) 888 | return nil 889 | } 890 | 891 | // if source tag is set but without destinate tag, use the same tag as source 892 | destTag := targetURL.GetTag() 893 | if destTag == "" { 894 | destTag = sourceURL.GetTag() 895 | } 896 | 897 | imageTarget, err = transfer.NewImageTarget(targetURL.GetRegistry(), targetURL.GetRepoWithNamespace(), 898 | destTag, targetSecurity.Username, targetSecurity.Password, targetSecurity.Insecure) 899 | if err != nil { 900 | return fmt.Errorf("generate %s image target error: %v", sourceURL.GetURL(), err) 901 | } 902 | 903 | sourceDigest, err := imageSource.GetImageDigest() 904 | if err != nil { 905 | log.Errorf("Failed to get source image digest from %s/%s:%s error: %v", imageSource.GetRegistry(), imageSource.GetRepository(), sourceURL.GetTag(), err) 906 | return err 907 | } 908 | targetDigest, err := imageTarget.GetImageDigest() 909 | if err == nil { 910 | if sourceDigest == targetDigest { 911 | log.Infof("Skip push image, target image %s/%s:%s already exist and has same digest %s", imageTarget.GetRegistry(), imageTarget.GetRepository(), imageTarget.GetTag(), sourceDigest) 912 | return nil 913 | } 914 | } else if !utils.IsDigestNotFound(err) { 915 | log.Errorf("Failed to get target image digest from %s/%s:%s error: %v", imageTarget.GetRegistry(), imageTarget.GetRepository(), destTag, err) 916 | return err 917 | } 918 | 919 | c.PutNormalURLPair(&URLPair{ 920 | source: source, 921 | target: target, 922 | }) 923 | log.Infof("put normal url pair source: %s, target: %s", source, target) 924 | return nil 925 | } 926 | 927 | // HandleURLPair put urlPair to normalURLPair 928 | func (c *Client) HandleURLPair() { 929 | routineNum := c.config.FlagConf.Config.RoutineNums 930 | wg := sync.WaitGroup{} 931 | 932 | for i := 0; i < routineNum; i++ { 933 | wg.Add(1) 934 | go func() { 935 | defer wg.Done() 936 | defer log.Infof("exit HandleURLPair main loop") 937 | for { 938 | urlPair, empty := c.GetURLPair() 939 | // no more job to generate 940 | if empty { 941 | log.Infof("HandleURLPair is empty") 942 | break 943 | } 944 | err := c.GenTagURLPair(urlPair.source, urlPair.target, &wg) 945 | if err != nil { 946 | log.Errorf("Generate tag urlPair %s to %s error: %v", urlPair.source, urlPair.target, err) 947 | // put to failedGenNormalURLPair 948 | c.PutAFailedGenNormalURLPair(urlPair) 949 | } 950 | } 951 | }() 952 | } 953 | wg.Wait() 954 | } 955 | 956 | // CcrtoTcrGenTagRetry put urlPair to normalURLPair if job is CcrtoTcr 957 | func (c *Client) CcrtoTcrGenTagRetry() { 958 | routineNum := 5 959 | wg := sync.WaitGroup{} 960 | 961 | for i := 0; i < routineNum; i++ { 962 | wg.Add(1) 963 | go func() { 964 | defer wg.Done() 965 | defer log.Debugf("exit CcrtoTcrGenTagRetry main loop") 966 | for { 967 | urlPair, empty := c.GetURLPair() 968 | // no more job to generate 969 | if empty { 970 | log.Debugf("CcrtoTcrGenTagRetry url pair is empty") 971 | break 972 | } 973 | err := c.GenCcrtoTcrTagURLPair(urlPair.source, urlPair.target, &wg) 974 | if err != nil { 975 | log.Errorf("Generate tag urlPair %s to %s error: %v", urlPair.source, urlPair.target, err) 976 | // put to failedGenNormalURLPair 977 | c.PutAFailedGenNormalURLPair(urlPair) 978 | } 979 | } 980 | }() 981 | } 982 | wg.Wait() 983 | } 984 | 985 | // GenCcrtoTcrTagURLPair is generate normal url pair 986 | func (c *Client) GenCcrtoTcrTagURLPair(source string, target string, wg *sync.WaitGroup) error { 987 | urlPair := &URLPair{ 988 | source: source, 989 | target: target, 990 | } 991 | 992 | sourceURL, err := utils.NewRepoURL(source) 993 | if err != nil { 994 | return fmt.Errorf("url %s format error: %v", source, err) 995 | } 996 | 997 | targetURL, err := utils.NewRepoURL(target) 998 | if err != nil { 999 | return fmt.Errorf("url %s format error: %v", target, err) 1000 | } 1001 | 1002 | sourceSecurity, exist := c.config.GetSecuritySpecific(sourceURL.GetRegistry(), sourceURL.GetNamespace()) 1003 | if exist { 1004 | log.Infof("Find auth information for %v, username: %v", sourceURL.GetURL(), sourceSecurity.Username) 1005 | } else { 1006 | log.Infof("Cannot find auth information for %v, pull actions will be anonymous", sourceURL.GetURL()) 1007 | } 1008 | 1009 | targetSecurity, exist := c.config.GetSecuritySpecific(targetURL.GetRegistry(), targetURL.GetNamespace()) 1010 | if exist { 1011 | log.Infof("Find auth information for %v, username: %v", targetURL.GetURL(), targetSecurity.Username) 1012 | 1013 | } else { 1014 | log.Infof("Cannot find auth information for %v, push actions will be anonymous", targetURL.GetURL()) 1015 | } 1016 | 1017 | if sourceURL.GetTag() == "" { 1018 | ccrClient := ccrapis.NewCCRAPIClient() 1019 | 1020 | ccrSecretID, ccrSecretKey, err := ccrapis.GetCcrSecret(c.config.Secret) 1021 | if err != nil { 1022 | log.Errorf("GetCcrSecret error: %s", err) 1023 | return err 1024 | } 1025 | 1026 | sourceTags, err := ccrClient.GetRepoTags(ccrSecretID, ccrSecretKey, c.config.FlagConf.Config.CCRRegion, sourceURL.GetRepoWithNamespace(), int64(c.config.FlagConf.Config.CCRTagNums)) 1027 | log.Debugf("ccr target %s tags is %s", sourceURL.GetOriginURL(), sourceTags) 1028 | if err != nil { 1029 | log.Errorf("Failed get ccr repo %s tags, error: %s", sourceURL.GetRepoWithNamespace(), err) 1030 | return fmt.Errorf("failed get ccr repo %s tags, error: %s", sourceURL.GetRepoWithNamespace(), err) 1031 | } 1032 | 1033 | imageTarget, err := transfer.NewImageTarget(targetURL.GetRegistry(), targetURL.GetRepoWithNamespace(), targetURL.GetTag(), targetSecurity.Username, targetSecurity.Password, targetSecurity.Insecure) 1034 | if err != nil { 1035 | return fmt.Errorf("generate %s image target error: %v", sourceURL.GetURL(), err) 1036 | } 1037 | 1038 | targetTags, err := imageTarget.GetTargetRepoTags() 1039 | log.Debugf("target %s tags is %s", targetURL.GetURL(), targetTags) 1040 | if err != nil { 1041 | return fmt.Errorf("get tags failed from %s error: %v", targetURL.GetURL(), err) 1042 | } 1043 | 1044 | log.Debugf("GenCcrtoTcrTagURLPair call GenJobFilterTag") 1045 | c.GenJobFilterTag(sourceTags, targetTags, sourceURL, targetURL, sourceSecurity, targetSecurity, wg) 1046 | return nil 1047 | } 1048 | 1049 | // already contain tag 1050 | 1051 | imageSource, err := transfer.NewImageSource(sourceURL.GetRegistry(), sourceURL.GetRepoWithNamespace(), sourceURL.GetTag(), sourceSecurity.Username, sourceSecurity.Password, sourceSecurity.Insecure) 1052 | if err != nil { 1053 | return fmt.Errorf("generate %s image source error: %v", sourceURL.GetURL(), err) 1054 | } 1055 | 1056 | imageTarget, err := transfer.NewImageTarget(targetURL.GetRegistry(), targetURL.GetRepoWithNamespace(), targetURL.GetTag(), targetSecurity.Username, targetSecurity.Password, targetSecurity.Insecure) 1057 | if err != nil { 1058 | return fmt.Errorf("generate %s image target error: %v", sourceURL.GetURL(), err) 1059 | } 1060 | 1061 | sourceDigest, err := imageSource.GetImageDigest() 1062 | if err != nil { 1063 | log.Errorf("Failed to get source image digest from %s/%s:%s error: %v", imageSource.GetRegistry(), imageSource.GetRepository(), sourceURL.GetTag(), err) 1064 | return err 1065 | } 1066 | targetDigest, err := imageTarget.GetImageDigest() 1067 | if err != nil { 1068 | log.Errorf("Failed to get target image digest from %s/%s:%s error: %v", imageTarget.GetRegistry(), imageTarget.GetRepository(), targetURL.GetTag(), err) 1069 | return err 1070 | } 1071 | 1072 | if sourceDigest == targetDigest { 1073 | log.Infof("Skip push image, target image %s/%s:%s already exist and has same digest %s", imageTarget.GetRegistry(), imageTarget.GetRepository(), imageTarget.GetTag(), sourceDigest) 1074 | return nil 1075 | } 1076 | 1077 | c.PutNormalURLPair(urlPair) 1078 | log.Infof("put normal url pair source: %s, target: %s", source, target) 1079 | return nil 1080 | } 1081 | -------------------------------------------------------------------------------- /pkg/log/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making TKEStack 3 | * available. 4 | * 5 | * Copyright (C) 2012-2020 Tencent. All Rights Reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package log 20 | -------------------------------------------------------------------------------- /pkg/log/encoder.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making TKEStack 3 | * available. 4 | * 5 | * Copyright (C) 2012-2020 Tencent. All Rights Reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package log 20 | 21 | import ( 22 | "time" 23 | 24 | "strings" 25 | 26 | "strconv" 27 | 28 | "github.com/skipor/goenv" 29 | "go.uber.org/zap/zapcore" 30 | ) 31 | 32 | func timeEncoder(t time.Time, enc zapcore.PrimitiveArrayEncoder) { 33 | enc.AppendString(t.Format("2006-01-02 15:04:05.000")) 34 | } 35 | 36 | func milliSecondsDurationEncoder(d time.Duration, enc zapcore.PrimitiveArrayEncoder) { 37 | enc.AppendFloat64(float64(d) / float64(time.Millisecond)) 38 | } 39 | 40 | func goPathCallerEncoder(caller zapcore.EntryCaller, enc zapcore.PrimitiveArrayEncoder) { 41 | if !caller.Defined { 42 | enc.AppendString("") 43 | return 44 | } 45 | enc.AppendString(strings.TrimPrefix(goenv.TrimGoPathSrc(caller.File), 46 | "github.com/tencentcloud/tke/") + ":" + strconv.Itoa(caller.Line)) 47 | } 48 | -------------------------------------------------------------------------------- /pkg/log/flag.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making TKEStack 3 | * available. 4 | * 5 | * Copyright (C) 2012-2020 Tencent. All Rights Reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package log 20 | 21 | import ( 22 | "fmt" 23 | "sync" 24 | "time" 25 | 26 | "github.com/spf13/pflag" 27 | "go.uber.org/zap/zapcore" 28 | ) 29 | 30 | const ( 31 | // SamplingFreqFlagName flag name 32 | SamplingFreqFlagName = "log-sampling-frequency" 33 | // LevelFlagName flag name 34 | LevelFlagName = "log-level" 35 | // FormatFlagName flag name 36 | FormatFlagName = "log-format" 37 | // WithColorFlagName flag name 38 | WithColorFlagName = "log-enable-color" 39 | // IgnoreCallerFlagName caller flag 40 | IgnoreCallerFlagName = "log-ignore-caller" 41 | // OutputPathsName path name 42 | OutputPathsName = "log-output-paths" 43 | ) 44 | 45 | var ( 46 | lock = &sync.RWMutex{} 47 | logSamplingFreq = pflag.Duration(SamplingFreqFlagName, 100*time.Millisecond, "Log sampling `INTERVAL`") 48 | logLevel = pflag.String(LevelFlagName, "info", "Minimum log output `LEVEL`") 49 | logFormat = pflag.String(FormatFlagName, "plain", "Log output `FORMAT`") 50 | logWithColor = pflag.Bool(WithColorFlagName, false, "Whether to output colored log") 51 | logIgnoreCaller = pflag.Bool(IgnoreCallerFlagName, false, "Ignore the output of caller information in the log") 52 | logOutputPaths = pflag.StringSlice(OutputPathsName, []string{}, "Log output paths, comma separated.") 53 | ) 54 | 55 | // AddFlags registers this package's flags on arbitrary FlagSets, such that they 56 | // point to the same value as the global flags. 57 | func AddFlags(fs *pflag.FlagSet) { 58 | fs.AddFlag(pflag.Lookup(LevelFlagName)) 59 | fs.AddFlag(pflag.Lookup(FormatFlagName)) 60 | fs.AddFlag(pflag.Lookup(WithColorFlagName)) 61 | fs.AddFlag(pflag.Lookup(IgnoreCallerFlagName)) 62 | fs.AddFlag(pflag.Lookup(SamplingFreqFlagName)) 63 | fs.AddFlag(pflag.Lookup(OutputPathsName)) 64 | } 65 | 66 | // SetLevel to change the log level flag 67 | func SetLevel(level string) error { 68 | lock.Lock() 69 | defer lock.Unlock() 70 | if _, err := parseLevel(); err != nil { 71 | return err 72 | } 73 | logLevel = &level 74 | return nil 75 | } 76 | 77 | // Level returns the current log level 78 | func Level() string { 79 | lock.RLock() 80 | defer lock.RUnlock() 81 | return *logLevel 82 | } 83 | 84 | // SetFormat to change the log format flag 85 | func SetFormat(format string) error { 86 | lock.Lock() 87 | defer lock.Unlock() 88 | if _, err := parseFormat(); err != nil { 89 | return err 90 | } 91 | logFormat = &format 92 | return nil 93 | } 94 | 95 | // Format return the current log format 96 | func Format() string { 97 | lock.RLock() 98 | defer lock.RUnlock() 99 | return *logFormat 100 | } 101 | 102 | func mustLevel() zapcore.Level { 103 | level := MustParseLevel() 104 | zapLevel := zapcore.InfoLevel 105 | if err := zapLevel.UnmarshalText([]byte(level)); err != nil { 106 | panic(err) 107 | } 108 | return zapLevel 109 | } 110 | 111 | func parseFormat() (string, error) { 112 | switch *logFormat { 113 | case "json", "JSON": 114 | return "json", nil 115 | case "console", "plain": 116 | return "console", nil 117 | default: 118 | return "", fmt.Errorf("unable to parse the special format") 119 | } 120 | } 121 | 122 | // MustParseFormat to parse the log format flag 123 | func MustParseFormat() string { 124 | if format, err := parseFormat(); err == nil { 125 | return format 126 | } 127 | return "console" 128 | } 129 | 130 | func parseLevel() (string, error) { 131 | switch *logLevel { 132 | case "DEBUG", "debug", "dbg", "DBG": 133 | return "DEBUG", nil 134 | case "WARN", "warn", "warning", "WARNING": 135 | return "WARN", nil 136 | case "ERROR", "error", "ERR", "err": 137 | return "ERROR", nil 138 | case "FATAL", "fatal": 139 | return "FATAL", nil 140 | case "PANIC", "panic": 141 | return "PANIC", nil 142 | default: 143 | return "", fmt.Errorf("unable to parse the special level") 144 | } 145 | } 146 | 147 | // MustParseLevel to parse the log level flag 148 | func MustParseLevel() string { 149 | if level, err := parseLevel(); err == nil { 150 | return level 151 | } 152 | return "INFO" 153 | } 154 | -------------------------------------------------------------------------------- /pkg/log/log.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making TKEStack 3 | * available. 4 | * 5 | * Copyright (C) 2012-2020 Tencent. All Rights Reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package log 20 | 21 | import ( 22 | "fmt" 23 | "log" 24 | "os" 25 | "strings" 26 | "sync" 27 | "time" 28 | 29 | "go.uber.org/zap" 30 | "go.uber.org/zap/zapcore" 31 | "gopkg.in/natefinch/lumberjack.v2" 32 | ) 33 | 34 | var ( 35 | logger *zap.Logger 36 | once sync.Once 37 | ) 38 | 39 | // InitLogger initializes logger the way we want for tke. 40 | func InitLogger() { 41 | once.Do(func() { 42 | logger = newLogger() 43 | }) 44 | } 45 | 46 | // FlushLogger calls the underlying Core's Sync method, flushing any buffered 47 | // log entries. Applications should take care to call Sync before exiting. 48 | func FlushLogger() { 49 | if logger != nil { 50 | // #nosec 51 | // nolint: errcheck 52 | logger.Sync() 53 | } 54 | 55 | } 56 | 57 | // ZapLogger returns zap logger instance. 58 | func ZapLogger() *zap.Logger { 59 | return getLogger() 60 | } 61 | 62 | // Reset to recreate the logger by changed flag params 63 | func Reset() { 64 | lock.Lock() 65 | defer lock.Unlock() 66 | logger = newLogger() 67 | } 68 | 69 | // Check return if logging a message at the specified level is enabled. 70 | func Check(level int32) bool { 71 | var lvl zapcore.Level 72 | if level < 5 { 73 | lvl = zapcore.InfoLevel 74 | } else { 75 | lvl = zapcore.DebugLevel 76 | } 77 | checkEntry := getLogger().Check(lvl, "") 78 | return checkEntry != nil 79 | } 80 | 81 | // StdErrLogger returns logger of standard library which writes to supplied zap 82 | // logger at error level 83 | func StdErrLogger() *log.Logger { 84 | if l, err := zap.NewStdLogAt(getLogger(), zapcore.ErrorLevel); err == nil { 85 | return l 86 | } 87 | return nil 88 | } 89 | 90 | // StdInfoLogger returns logger of standard library which writes to supplied zap 91 | // logger at info level 92 | func StdInfoLogger() *log.Logger { 93 | if l, err := zap.NewStdLogAt(getLogger(), zapcore.InfoLevel); err == nil { 94 | return l 95 | } 96 | return nil 97 | } 98 | 99 | // Debug method output debug level log. 100 | func Debug(msg string, fields ...Field) { 101 | getLogger().Debug(msg, fields...) 102 | } 103 | 104 | // Info method output info level log. 105 | func Info(msg string, fields ...Field) { 106 | getLogger().Info(msg, fields...) 107 | } 108 | 109 | // Warn method output warning level log. 110 | func Warn(msg string, fields ...Field) { 111 | getLogger().Warn(msg, fields...) 112 | } 113 | 114 | // Error method output error level log. 115 | func Error(msg string, fields ...Field) { 116 | getLogger().Error(msg, fields...) 117 | } 118 | 119 | // Panic method output panic level log and shutdown application. 120 | func Panic(msg string, fields ...Field) { 121 | getLogger().Panic(msg, fields...) 122 | } 123 | 124 | // Fatal method output fatal level log. 125 | func Fatal(msg string, fields ...Field) { 126 | getLogger().Fatal(msg, fields...) 127 | } 128 | 129 | // Debugf uses fmt.Sprintf to log a templated message. 130 | func Debugf(template string, args ...interface{}) { 131 | Debug(fmt.Sprintf(template, args...)) 132 | } 133 | 134 | // Infof uses fmt.Sprintf to log a templated message. 135 | func Infof(template string, args ...interface{}) { 136 | Info(fmt.Sprintf(template, args...)) 137 | } 138 | 139 | // Warnf uses fmt.Sprintf to log a templated message. 140 | func Warnf(template string, args ...interface{}) { 141 | Warn(fmt.Sprintf(template, args...)) 142 | } 143 | 144 | // Errorf uses fmt.Sprintf to log a templated message. 145 | func Errorf(template string, args ...interface{}) { 146 | Error(fmt.Sprintf(template, args...)) 147 | } 148 | 149 | // Panicf uses fmt.Sprintf to log a templated message, then panics. 150 | func Panicf(template string, args ...interface{}) { 151 | Panic(fmt.Sprintf(template, args...)) 152 | } 153 | 154 | // Fatalf uses fmt.Sprintf to log a templated message, then calls os.Exit. 155 | func Fatalf(template string, args ...interface{}) { 156 | Fatal(fmt.Sprintf(template, args...)) 157 | } 158 | 159 | func getLogger() *zap.Logger { 160 | once.Do(func() { 161 | logger = newLogger() 162 | }) 163 | return logger 164 | } 165 | 166 | func newLogger() *zap.Logger { 167 | encoderConfig := zapcore.EncoderConfig{ 168 | TimeKey: "time", 169 | LevelKey: "level", 170 | NameKey: "logger", 171 | CallerKey: "caller", 172 | MessageKey: "msg", 173 | StacktraceKey: "stack", 174 | LineEnding: zapcore.DefaultLineEnding, 175 | EncodeLevel: zapcore.LowercaseLevelEncoder, 176 | EncodeTime: timeEncoder, 177 | EncodeDuration: milliSecondsDurationEncoder, 178 | EncodeCaller: zapcore.ShortCallerEncoder, 179 | } 180 | // when output to local path, with color is forbidden 181 | if *logWithColor && ((logOutputPaths == nil) || (len(*logOutputPaths) == 0)) { 182 | encoderConfig.EncodeLevel = zapcore.CapitalColorLevelEncoder 183 | } 184 | loggerConfig := &zap.Config{ 185 | Level: zap.NewAtomicLevelAt(mustLevel()), 186 | Development: false, 187 | DisableCaller: *logIgnoreCaller, 188 | DisableStacktrace: false, 189 | Sampling: &zap.SamplingConfig{ 190 | Initial: 100, 191 | Thereafter: int(*logSamplingFreq / time.Millisecond), 192 | }, 193 | Encoding: MustParseFormat(), 194 | EncoderConfig: encoderConfig, 195 | OutputPaths: []string{"stdout"}, 196 | ErrorOutputPaths: []string{"stderr"}, 197 | } 198 | if logOutputPaths != nil { 199 | loggerConfig.OutputPaths = append(loggerConfig.OutputPaths, *logOutputPaths...) 200 | } 201 | 202 | //log rolling 203 | w := zapcore.AddSync(&lumberjack.Logger{ 204 | Filename: strings.Join(*logOutputPaths, ""), 205 | MaxSize: 500, // megabytes 206 | MaxBackups: 3, 207 | MaxAge: 30, // days 208 | }) 209 | core := zapcore.NewCore( 210 | zapcore.NewConsoleEncoder(encoderConfig), 211 | zapcore.NewMultiWriteSyncer(zapcore.AddSync(os.Stdout), 212 | w), 213 | zap.NewAtomicLevelAt(mustLevel()), 214 | ) 215 | 216 | l := zap.New(core, zap.AddStacktrace(zapcore.PanicLevel), 217 | zap.AddCaller(), zap.Development(), zap.AddCallerSkip(2)) 218 | 219 | /*l, err := loggerConfig.Build(zap.AddStacktrace(zapcore.PanicLevel), 220 | zap.AddCallerSkip(1)) 221 | if err != nil { 222 | panic(err) 223 | }*/ 224 | initRestfulLogger(l) 225 | return l 226 | } 227 | -------------------------------------------------------------------------------- /pkg/log/log_restful.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making TKEStack 3 | * available. 4 | * 5 | * Copyright (C) 2012-2020 Tencent. All Rights Reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package log 20 | 21 | import ( 22 | "github.com/emicklei/go-restful/log" 23 | "go.uber.org/zap" 24 | "go.uber.org/zap/zapcore" 25 | ) 26 | 27 | func initRestfulLogger(logger *zap.Logger) { 28 | if l, err := zap.NewStdLogAt(logger, zapcore.InfoLevel); err == nil { 29 | log.SetLogger(l) 30 | } else { 31 | logger.Error("Failed to get standard library logger from logger", zap.Error(err)) 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /pkg/log/type.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making TKEStack 3 | * available. 4 | * 5 | * Copyright (C) 2012-2020 Tencent. All Rights Reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package log 20 | 21 | import ( 22 | "go.uber.org/zap" 23 | "go.uber.org/zap/zapcore" 24 | ) 25 | 26 | // Field is an alias for the field structure in the underlying log frame 27 | type Field = zapcore.Field 28 | 29 | // nolint: golint 30 | var ( 31 | Any = zap.Any 32 | Array = zap.Array 33 | Object = zap.Object 34 | Binary = zap.Binary 35 | Bool = zap.Bool 36 | Bools = zap.Bools 37 | ByteString = zap.ByteString 38 | ByteStrings = zap.ByteStrings 39 | Complex64 = zap.Complex64 40 | Complex64s = zap.Complex64s 41 | Complex128 = zap.Complex128 42 | Complex128s = zap.Complex128s 43 | Duration = zap.Duration 44 | Durations = zap.Durations 45 | Err = zap.Error 46 | Errors = zap.Errors 47 | Float32 = zap.Float32 48 | Float32s = zap.Float32s 49 | Float64 = zap.Float64 50 | Float64s = zap.Float64s 51 | Int = zap.Int 52 | Ints = zap.Ints 53 | Int8 = zap.Int8 54 | Int8s = zap.Int8s 55 | Int16 = zap.Int16 56 | Int16s = zap.Int16s 57 | Int32 = zap.Int32 58 | Int32s = zap.Int32s 59 | Int64 = zap.Int64 60 | Int64s = zap.Int64s 61 | Namespace = zap.Namespace 62 | Reflect = zap.Reflect 63 | Stack = zap.Stack 64 | String = zap.String 65 | Stringer = zap.Stringer 66 | Strings = zap.Strings 67 | Time = zap.Time 68 | Times = zap.Times 69 | Uint = zap.Uint 70 | Uints = zap.Uints 71 | Uint8 = zap.Uint8 72 | Uint8s = zap.Uint8s 73 | Uint16 = zap.Uint16 74 | Uint16s = zap.Uint16s 75 | Uint32 = zap.Uint32 76 | Uint32s = zap.Uint32s 77 | Uint64 = zap.Uint64 78 | Uint64s = zap.Uint64s 79 | Uintptr = zap.Uintptr 80 | Uintptrs = zap.Uintptrs 81 | ) 82 | -------------------------------------------------------------------------------- /pkg/transfer/job.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making TKEStack 3 | * available. 4 | * 5 | * Copyright (C) 2012-2020 Tencent. All Rights Reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package transfer 20 | 21 | import ( 22 | "github.com/containers/image/v5/manifest" 23 | "github.com/containers/image/v5/pkg/blobinfocache/memory" 24 | "github.com/containers/image/v5/pkg/blobinfocache/none" 25 | "github.com/pkg/errors" 26 | "tkestack.io/image-transfer/pkg/log" 27 | ) 28 | 29 | var ( 30 | // NoCache used to disable a blobinfocache 31 | NoCache = none.NoCache 32 | // Memory cache to enable a blobinfocache 33 | Memory = memory.New() 34 | ) 35 | 36 | // Job act as a sync action, it will pull a images from source to target 37 | type Job struct { 38 | Source *ImageSource 39 | Target *ImageTarget 40 | } 41 | 42 | // NewJob creates a transfer job 43 | func NewJob(source *ImageSource, target *ImageTarget) *Job { 44 | 45 | return &Job{ 46 | Source: source, 47 | Target: target, 48 | } 49 | } 50 | 51 | // Run is the main function of a transfer job 52 | func (j *Job) Run() error { 53 | // get manifest from source 54 | manifestByte, manifestType, err := j.Source.GetManifest() 55 | if err != nil { 56 | log.Errorf("Failed to get manifest from %s/%s:%s error: %v", 57 | j.Source.GetRegistry(), j.Source.GetRepository(), j.Source.GetTag(), err) 58 | return err 59 | } 60 | log.Infof("Get manifest from %s/%s:%s", j.Source.GetRegistry(), j.Source.GetRepository(), j.Source.GetTag()) 61 | 62 | blobInfos, err := j.Source.GetBlobInfos(manifestByte, manifestType) 63 | if err != nil { 64 | log.Errorf("Get blob info from %s/%s:%s error: %v", 65 | j.Source.GetRegistry(), j.Source.GetRepository(), j.Source.GetTag(), err) 66 | return err 67 | } 68 | 69 | // blob transformation 70 | for _, blobinfo := range blobInfos { 71 | blobExist, err := j.Target.CheckBlobExist(blobinfo) 72 | if err != nil { 73 | log.Errorf("Check blob %s(%v) to %s/%s:%s exist error: %v", 74 | blobinfo.Digest, blobinfo.Size, j.Target.GetRegistry(), j.Target.GetRepository(), j.Target.GetTag(), err) 75 | return err 76 | } 77 | 78 | if !blobExist { 79 | // pull a blob from source 80 | log.Infof("Getting blob from %s/%s:%s ing...", j.Source.GetRegistry(), j.Source.GetRepository(), j.Source.GetTag()) 81 | blob, size, err := j.Source.GetABlob(blobinfo) 82 | if err != nil { 83 | log.Errorf("Get blob %s(%v) from %s/%s:%s failed: %v", blobinfo.Digest, 84 | size, j.Source.GetRegistry(), j.Source.GetRepository(), j.Source.GetTag(), err) 85 | return err 86 | } 87 | 88 | log.Infof("Get a blob %s(%v) from %s/%s:%s success", blobinfo.Digest, size, 89 | j.Source.GetRegistry(), j.Source.GetRepository(), j.Source.GetTag()) 90 | 91 | blobinfo.Size = size 92 | // push a blob to target 93 | log.Infof("Putting blob to %s/%s:%s ing...", j.Target.GetRegistry(), j.Target.GetRepository(), j.Target.GetTag()) 94 | if err := j.Target.PutABlob(blob, blobinfo); err != nil { 95 | log.Errorf("Put blob %s(%v) to %s/%s:%s failed: %v", blobinfo.Digest, blobinfo.Size, 96 | j.Target.GetRegistry(), j.Target.GetRepository(), j.Target.GetTag(), err) 97 | if closeErr := blob.Close(); closeErr != nil { 98 | return errors.Wrapf(err, " (close error: %v)", closeErr) 99 | } 100 | return err 101 | } 102 | 103 | log.Infof("Put blob %s(%v) to %s/%s:%s success", blobinfo.Digest, blobinfo.Size, 104 | j.Target.GetRegistry(), j.Target.GetRepository(), j.Target.GetTag()) 105 | } else { 106 | // print the log of ignored blob 107 | log.Infof("Blob %s(%v) has been pushed to %s, will not be pulled", blobinfo.Digest, 108 | blobinfo.Size, j.Target.GetRegistry()+"/"+j.Target.GetRepository()) 109 | } 110 | } 111 | 112 | //Push manifest list 113 | if manifestType == manifest.DockerV2ListMediaType { 114 | manifestSchemaListInfo, err := manifest.Schema2ListFromManifest(manifestByte) 115 | if err != nil { 116 | return err 117 | } 118 | 119 | var subManifestByte []byte 120 | 121 | // push manifest to target 122 | for _, manifestDescriptorElem := range manifestSchemaListInfo.Manifests { 123 | 124 | log.Infof("handle manifest OS:%s Architecture:%s ", manifestDescriptorElem.Platform.OS, 125 | manifestDescriptorElem.Platform.Architecture) 126 | 127 | subManifestByte, _, err = j.Source.source.GetManifest(j.Source.ctx, &manifestDescriptorElem.Digest) 128 | if err != nil { 129 | log.Errorf("Get manifest %v of OS:%s Architecture:%s for manifest list error: %v", 130 | manifestDescriptorElem.Digest, manifestDescriptorElem.Platform.OS, 131 | manifestDescriptorElem.Platform.Architecture, err) 132 | return err 133 | } 134 | 135 | if err := j.Target.PushManifest(subManifestByte); err != nil { 136 | log.Errorf("Put manifest to %s/%s:%s error: %v", j.Target.GetRegistry(), 137 | j.Target.GetRepository(), j.Target.GetTag(), err) 138 | return err 139 | } 140 | 141 | log.Infof("Put manifest to %s/%s:%s os:%s arch:%s", j.Target.GetRegistry(), j.Target.GetRepository(), 142 | j.Target.GetTag(), manifestDescriptorElem.Platform.OS, manifestDescriptorElem.Platform.Architecture) 143 | 144 | } 145 | 146 | // push manifest list to target 147 | if err := j.Target.PushManifest(manifestByte); err != nil { 148 | log.Errorf("Put manifestList to %s/%s:%s error: %v", j.Target.GetRegistry(), 149 | j.Target.GetRepository(), j.Target.GetTag(), err) 150 | return err 151 | } 152 | 153 | log.Infof("Put manifestList to %s/%s:%s", j.Target.GetRegistry(), j.Target.GetRepository(), j.Target.GetTag()) 154 | 155 | } else { 156 | 157 | // push manifest to target 158 | if err := j.Target.PushManifest(manifestByte); err != nil { 159 | log.Errorf("Put manifest to %s/%s:%s error: %v", j.Target.GetRegistry(), 160 | j.Target.GetRepository(), j.Target.GetTag(), err) 161 | return err 162 | } 163 | 164 | log.Infof("Put manifest to %s/%s:%s", j.Target.GetRegistry(), j.Target.GetRepository(), j.Target.GetTag()) 165 | } 166 | 167 | log.Infof("Synchronization successfully from %s/%s:%s to %s/%s:%s", j.Source.GetRegistry(), j.Source.GetRepository(), 168 | j.Source.GetTag(), j.Target.GetRegistry(), j.Target.GetRepository(), j.Target.GetTag()) 169 | 170 | return nil 171 | } 172 | -------------------------------------------------------------------------------- /pkg/transfer/manifest.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making TKEStack 3 | * available. 4 | * 5 | * Copyright (C) 2012-2020 Tencent. All Rights Reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package transfer 20 | 21 | import ( 22 | "fmt" 23 | 24 | "github.com/containers/image/v5/manifest" 25 | ) 26 | 27 | // ManifestSchemaV2List describes a schema V2 manifest list 28 | type ManifestSchemaV2List struct { 29 | Manifests []ManifestSchemaV2Info `json:"manifests"` 30 | } 31 | 32 | // ManifestSchemaV2Info includes of the imformation needes of a schema V2 manifest file 33 | type ManifestSchemaV2Info struct { 34 | Digest string `json:"digest"` 35 | } 36 | 37 | // ManifestHandler expends the ability of handling manifest list in schema2, but it's not finished yet 38 | // return the digest array of manifests in the manifest list if exist. 39 | func ManifestHandler(m []byte, t string, i *ImageSource) ([]manifest.Manifest, error) { 40 | var manifestInfoSlice []manifest.Manifest 41 | 42 | if t == manifest.DockerV2Schema2MediaType { 43 | manifestInfo, err := manifest.Schema2FromManifest(m) 44 | if err != nil { 45 | return nil, err 46 | } 47 | manifestInfoSlice = append(manifestInfoSlice, manifestInfo) 48 | return manifestInfoSlice, nil 49 | } else if t == manifest.DockerV2Schema1MediaType || t == manifest.DockerV2Schema1SignedMediaType { 50 | manifestInfo, err := manifest.Schema1FromManifest(m) 51 | if err != nil { 52 | return nil, err 53 | } 54 | manifestInfoSlice = append(manifestInfoSlice, manifestInfo) 55 | return manifestInfoSlice, nil 56 | } else if t == manifest.DockerV2ListMediaType { 57 | 58 | manifestSchemaListInfo, err := manifest.Schema2ListFromManifest(m) 59 | if err != nil { 60 | return nil, err 61 | } 62 | 63 | for _, manifestDescriptorElem := range manifestSchemaListInfo.Manifests { 64 | 65 | manifestByte, manifestType, err := i.source.GetManifest(i.ctx, &manifestDescriptorElem.Digest) 66 | if err != nil { 67 | return nil, err 68 | } 69 | 70 | platformSpecManifest, err := ManifestHandler(manifestByte, manifestType, i) 71 | if err != nil { 72 | return nil, err 73 | } 74 | 75 | manifestInfoSlice = append(manifestInfoSlice, platformSpecManifest...) 76 | } 77 | return manifestInfoSlice, nil 78 | } 79 | return nil, fmt.Errorf("unsupported manifest type: %v", t) 80 | } 81 | -------------------------------------------------------------------------------- /pkg/transfer/source.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making TKEStack 3 | * available. 4 | * 5 | * Copyright (C) 2012-2020 Tencent. All Rights Reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package transfer 20 | 21 | import ( 22 | "context" 23 | "fmt" 24 | 25 | "github.com/opencontainers/go-digest" 26 | 27 | "io" 28 | 29 | 30 | "github.com/containers/image/v5/docker" 31 | "github.com/containers/image/v5/types" 32 | "tkestack.io/image-transfer/pkg/utils" 33 | ) 34 | 35 | // ImageSource is a reference to a remote image need to be pulled. 36 | type ImageSource struct { 37 | registry string 38 | repository string 39 | tag string 40 | sourceRef types.ImageReference 41 | source types.ImageSource 42 | ctx context.Context 43 | sysctx *types.SystemContext 44 | } 45 | 46 | // NewImageSource generates a PullJob by repository, the repository string must include "tag", 47 | // if username or password is empty, access to repository will be anonymous. 48 | // a repository string is the rest part of the images url except "tag" and "registry" 49 | func NewImageSource(registry, repository, tag, username, password string, insecure bool) (*ImageSource, error) { 50 | if utils.CheckIfIncludeTag(repository) { 51 | return nil, fmt.Errorf("repository string should not include tag") 52 | } 53 | 54 | // tag may be empty 55 | tagWithColon := "" 56 | if tag != "" { 57 | tagWithColon = ":" + tag 58 | } 59 | 60 | srcRef, err := docker.ParseReference("//" + registry + "/" + repository + tagWithColon) 61 | if err != nil { 62 | return nil, err 63 | } 64 | 65 | var sysctx *types.SystemContext 66 | if insecure { 67 | // destinatoin registry is http service 68 | sysctx = &types.SystemContext{ 69 | DockerInsecureSkipTLSVerify: types.OptionalBoolTrue, 70 | } 71 | } else { 72 | sysctx = &types.SystemContext{} 73 | } 74 | 75 | ctx := context.WithValue(context.Background(), interface{}("ImageSource"), repository) 76 | if username != "" && password != "" { 77 | sysctx.DockerAuthConfig = &types.DockerAuthConfig{ 78 | Username: username, 79 | Password: password, 80 | } 81 | } 82 | 83 | var rawSource types.ImageSource 84 | if tag != "" { 85 | // if tag is empty, will attach to the "latest" tag, and will get a error if "latest" is not exist 86 | rawSource, err = srcRef.NewImageSource(ctx, sysctx) 87 | if err != nil { 88 | return nil, err 89 | } 90 | } 91 | 92 | return &ImageSource{ 93 | sourceRef: srcRef, 94 | source: rawSource, 95 | ctx: ctx, 96 | sysctx: sysctx, 97 | registry: registry, 98 | repository: repository, 99 | tag: tag, 100 | }, nil 101 | } 102 | 103 | // GetManifest get manifest file from source image 104 | func (i *ImageSource) GetManifest() ([]byte, string, error) { 105 | if i.source == nil { 106 | return nil, "", fmt.Errorf("can not get manifest file without specfied a tag") 107 | } 108 | return i.source.GetManifest(i.ctx, nil) 109 | } 110 | 111 | // GetBlobInfos get blobs from source image. 112 | func (i *ImageSource) GetBlobInfos(manifestByte []byte, manifestType string) ([]types.BlobInfo, error) { 113 | if i.source == nil { 114 | return nil, fmt.Errorf("can not get blobs without specfied a tag") 115 | } 116 | 117 | manifestInfoSlice, err := ManifestHandler(manifestByte, manifestType, i) 118 | if err != nil { 119 | return nil, err 120 | } 121 | 122 | // get a manifest 123 | 124 | srcBlobs := []types.BlobInfo{} 125 | 126 | for _, manifestInfo := range manifestInfoSlice { 127 | blobInfos := manifestInfo.LayerInfos() 128 | for _, l := range blobInfos { 129 | srcBlobs = append(srcBlobs, l.BlobInfo) 130 | } 131 | // append config blob info 132 | configBlob := manifestInfo.ConfigInfo() 133 | if configBlob.Digest != "" { 134 | srcBlobs = append(srcBlobs, configBlob) 135 | } 136 | } 137 | 138 | return srcBlobs, nil 139 | } 140 | 141 | // GetABlob gets a blob from remote image 142 | func (i *ImageSource) GetABlob(blobInfo types.BlobInfo) (io.ReadCloser, int64, error) { 143 | return i.source.GetBlob(i.ctx, types.BlobInfo{Digest: blobInfo.Digest, Size: -1}, NoCache) 144 | } 145 | 146 | // Close an ImageSource 147 | func (i *ImageSource) Close() error { 148 | return i.source.Close() 149 | } 150 | 151 | // GetRegistry returns the registry of a ImageSource 152 | func (i *ImageSource) GetRegistry() string { 153 | return i.registry 154 | } 155 | 156 | // GetRepository returns the repository of a ImageSource 157 | func (i *ImageSource) GetRepository() string { 158 | return i.repository 159 | } 160 | 161 | // GetTag returns the tag of a ImageSource 162 | func (i *ImageSource) GetTag() string { 163 | return i.tag 164 | } 165 | 166 | // GetSourceRepoTags gets all the tags of a repository which ImageSource belongs to 167 | func (i *ImageSource) GetSourceRepoTags() ([]string, error) { 168 | return docker.GetRepositoryTags(i.ctx, i.sysctx, i.sourceRef) 169 | } 170 | 171 | // GetImageDigest checks if a tag exist for target, return target tag of digest 172 | func (i *ImageSource) GetImageDigest() (digest.Digest, error) { 173 | return docker.GetDigest(i.ctx, i.sysctx, i.sourceRef) 174 | } 175 | -------------------------------------------------------------------------------- /pkg/transfer/target.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making TKEStack 3 | * available. 4 | * 5 | * Copyright (C) 2012-2020 Tencent. All Rights Reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package transfer 20 | 21 | import ( 22 | "context" 23 | "fmt" 24 | "io" 25 | 26 | "github.com/opencontainers/go-digest" 27 | 28 | "github.com/containers/image/v5/docker" 29 | "github.com/containers/image/v5/types" 30 | "tkestack.io/image-transfer/pkg/utils" 31 | ) 32 | 33 | // ImageTarget is a reference of a remote image we will push to 34 | type ImageTarget struct { 35 | registry string 36 | repository string 37 | tag string 38 | targetRef types.ImageReference 39 | target types.ImageDestination 40 | ctx context.Context 41 | sysctx *types.SystemContext 42 | } 43 | 44 | // NewImageTarget generates a ImageTarget by repository, the repository string must include "tag". 45 | // If username or password is empty, access to repository will be anonymous. 46 | func NewImageTarget(registry, repository, tag, username, password string, insecure bool) (*ImageTarget, error) { 47 | if utils.CheckIfIncludeTag(repository) { 48 | return nil, fmt.Errorf("repository string should not include tag") 49 | } 50 | 51 | // tag may be empty 52 | tagWithColon := "" 53 | if tag != "" { 54 | tagWithColon = ":" + tag 55 | } 56 | 57 | // if tag is empty, will attach to the "latest" tag 58 | destRef, err := docker.ParseReference("//" + registry + "/" + repository + tagWithColon) 59 | if err != nil { 60 | return nil, err 61 | } 62 | 63 | var sysctx *types.SystemContext 64 | if insecure { 65 | // destinatoin registry is http service 66 | sysctx = &types.SystemContext{ 67 | DockerInsecureSkipTLSVerify: types.OptionalBoolTrue, 68 | } 69 | } else { 70 | sysctx = &types.SystemContext{} 71 | } 72 | 73 | ctx := context.WithValue(context.Background(), interface{}("ImageTarget"), repository) 74 | if username != "" && password != "" { 75 | sysctx.DockerAuthConfig = &types.DockerAuthConfig{ 76 | Username: username, 77 | Password: password, 78 | } 79 | } 80 | 81 | rawtarget, err := destRef.NewImageDestination(ctx, sysctx) 82 | if err != nil { 83 | return nil, err 84 | } 85 | 86 | return &ImageTarget{ 87 | targetRef: destRef, 88 | target: rawtarget, 89 | ctx: ctx, 90 | sysctx: sysctx, 91 | registry: registry, 92 | repository: repository, 93 | tag: tag, 94 | }, nil 95 | } 96 | 97 | // PushManifest push a manifest file to target image 98 | func (i *ImageTarget) PushManifest(manifestByte []byte) error { 99 | return i.target.PutManifest(i.ctx, manifestByte, nil) 100 | } 101 | 102 | // PutABlob push a blob to target image 103 | func (i *ImageTarget) PutABlob(blob io.ReadCloser, blobInfo types.BlobInfo) error { 104 | _, err := i.target.PutBlob(i.ctx, blob, types.BlobInfo{ 105 | Digest: blobInfo.Digest, 106 | Size: blobInfo.Size, 107 | }, Memory, true) 108 | 109 | // io.ReadCloser need to be close 110 | defer blob.Close() 111 | 112 | return err 113 | } 114 | 115 | // CheckBlobExist checks if a blob exist for target and reuse exist blobs 116 | func (i *ImageTarget) CheckBlobExist(blobInfo types.BlobInfo) (bool, error) { 117 | exist, _, err := i.target.TryReusingBlob(i.ctx, types.BlobInfo{ 118 | Digest: blobInfo.Digest, 119 | Size: blobInfo.Size, 120 | }, Memory, false) 121 | 122 | return exist, err 123 | } 124 | 125 | // Close a ImageTarget 126 | func (i *ImageTarget) Close() error { 127 | return i.target.Close() 128 | } 129 | 130 | // GetRegistry returns the registry of a ImageTarget 131 | func (i *ImageTarget) GetRegistry() string { 132 | return i.registry 133 | } 134 | 135 | // GetRepository returns the repository of a ImageTarget 136 | func (i *ImageTarget) GetRepository() string { 137 | return i.repository 138 | } 139 | 140 | // GetTag return the tag of a ImageTarget 141 | func (i *ImageTarget) GetTag() string { 142 | return i.tag 143 | } 144 | 145 | // GetImageDigest checks if a tag exist for target, return target tag of digest 146 | func (i *ImageTarget) GetImageDigest() (digest.Digest, error) { 147 | return docker.GetDigest(i.ctx, i.sysctx, i.targetRef) 148 | } 149 | 150 | // GetTargetRepoTags gets all the tags of a repository which ImageTarget belongs to 151 | func (i *ImageTarget) GetTargetRepoTags() ([]string, error) { 152 | tags, err := docker.GetRepositoryTags(i.ctx, i.sysctx, i.targetRef) 153 | if err != nil && utils.IsTagsNotFound(err) { 154 | return nil, nil 155 | } 156 | return tags, err 157 | } -------------------------------------------------------------------------------- /pkg/utils/ratelimiter.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making TKEStack 3 | * available. 4 | * 5 | * Copyright (C) 2012-2020 Tencent. All Rights Reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package utils 20 | 21 | import ( 22 | "net/http" 23 | "sync" 24 | 25 | "go.uber.org/ratelimit" 26 | ) 27 | 28 | type limitTransport struct { 29 | http.RoundTripper 30 | limiter ratelimit.Limiter 31 | } 32 | 33 | var _ http.RoundTripper = limitTransport{} 34 | 35 | func (t limitTransport) RoundTrip(req *http.Request) (*http.Response, error) { 36 | t.limiter.Take() 37 | return t.RoundTripper.RoundTrip(req) 38 | } 39 | 40 | var limiterOnce sync.Once 41 | var limiter ratelimit.Limiter 42 | 43 | // NewLimiter generates a new limiter. 44 | func NewLimiter(rate int) ratelimit.Limiter { 45 | limiterOnce.Do(func() { 46 | limiter = ratelimit.New(rate) 47 | }) 48 | return limiter 49 | } 50 | 51 | // NewRateLimitedTransport generates a new transport with rateLimit. 52 | func NewRateLimitedTransport(rate int, transport http.RoundTripper) http.RoundTripper { 53 | return &limitTransport{ 54 | RoundTripper: transport, 55 | limiter: NewLimiter(rate), 56 | } 57 | } 58 | 59 | var listLimiterOnce sync.Once 60 | var listLimiter ratelimit.Limiter 61 | 62 | // NewListLimiter generates a new limiter. 63 | func NewListLimiter(rate int) ratelimit.Limiter { 64 | listLimiterOnce.Do(func() { 65 | listLimiter = ratelimit.New(rate) 66 | }) 67 | return listLimiter 68 | } 69 | 70 | // NewListRateLimitedTransport generates a new transport with rateLimit. 71 | func NewListRateLimitedTransport(rate int, transport http.RoundTripper) http.RoundTripper { 72 | return &limitTransport{ 73 | RoundTripper: transport, 74 | limiter: NewListLimiter(rate), 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /pkg/utils/utils.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making TKEStack 3 | * available. 4 | * 5 | * Copyright (C) 2012-2020 Tencent. All Rights Reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package utils 20 | 21 | import ( 22 | "fmt" 23 | "strings" 24 | ) 25 | 26 | // The RepoURL will divide a images url to //: 27 | type RepoURL struct { 28 | // origin url 29 | url string 30 | 31 | registry string 32 | namespace string 33 | repo string 34 | tag string 35 | } 36 | 37 | // NewRepoURL creates a RepoURL 38 | func NewRepoURL(url string) (*RepoURL, error) { 39 | // split to registry/namespace/repoAndTag 40 | slice := strings.SplitN(url, "/", 3) 41 | 42 | var tag, repo string 43 | repoAndTag := slice[len(slice)-1] 44 | s := strings.Split(repoAndTag, ":") 45 | if len(s) > 2 { 46 | return nil, fmt.Errorf("invalid repository url: %v", url) 47 | } else if len(s) == 2 { 48 | repo = s[0] 49 | tag = s[1] 50 | } else { 51 | repo = s[0] 52 | tag = "" 53 | } 54 | 55 | if len(slice) == 3 { 56 | return &RepoURL{ 57 | url: url, 58 | registry: slice[0], 59 | namespace: slice[1], 60 | repo: repo, 61 | tag: tag, 62 | }, nil 63 | } else if len(slice) == 2 { 64 | // if first string is a domain 65 | if strings.Contains(slice[0], ".") { 66 | return &RepoURL{ 67 | url: url, 68 | registry: slice[0], 69 | namespace: "", 70 | repo: repo, 71 | tag: tag, 72 | }, nil 73 | } 74 | 75 | return &RepoURL{ 76 | url: url, 77 | registry: "registry.hub.docker.com", 78 | namespace: slice[0], 79 | repo: repo, 80 | tag: tag, 81 | }, nil 82 | } else { 83 | return &RepoURL{ 84 | url: url, 85 | registry: "registry.hub.docker.com", 86 | namespace: "library", 87 | repo: repo, 88 | tag: tag, 89 | }, nil 90 | } 91 | } 92 | 93 | // GetURL returns the whole url 94 | func (r *RepoURL) GetURL() string { 95 | url := r.GetURLWithoutTag() 96 | if r.tag != "" { 97 | url = url + ":" + r.tag 98 | } 99 | return url 100 | } 101 | 102 | // GetOriginURL returns the whole url 103 | func (r *RepoURL) GetOriginURL() string { 104 | return r.url 105 | } 106 | 107 | // GetRegistry returns the registry in a url 108 | func (r *RepoURL) GetRegistry() string { 109 | return r.registry 110 | } 111 | 112 | // GetNamespace returns the namespace in a url 113 | func (r *RepoURL) GetNamespace() string { 114 | return r.namespace 115 | } 116 | 117 | // GetRepo returns the repository in a url 118 | func (r *RepoURL) GetRepo() string { 119 | return r.repo 120 | } 121 | 122 | // GetTag returns the tag in a url 123 | func (r *RepoURL) GetTag() string { 124 | return r.tag 125 | } 126 | 127 | // GetRepoWithNamespace returns namespace/repository in a url 128 | func (r *RepoURL) GetRepoWithNamespace() string { 129 | if r.namespace == "" { 130 | return r.repo 131 | } 132 | return r.namespace + "/" + r.repo 133 | } 134 | 135 | // GetRepoWithTag returns repository:tag in a url 136 | func (r *RepoURL) GetRepoWithTag() string { 137 | if r.tag == "" { 138 | return r.repo 139 | } 140 | return r.repo + ":" + r.tag 141 | } 142 | 143 | // GetURLWithoutTag returns registry/namespace/repository in a url 144 | func (r *RepoURL) GetURLWithoutTag() string { 145 | if r.namespace == "" { 146 | return r.registry + "/" + r.repo 147 | } 148 | return r.registry + "/" + r.namespace + "/" + r.repo 149 | } 150 | 151 | // CheckIfIncludeTag checks if a repository string includes tag 152 | func CheckIfIncludeTag(repository string) bool { 153 | return strings.Contains(repository, ":") 154 | } 155 | 156 | // IsContain judge the item is in items or not 157 | func IsContain(items []string, item string) bool { 158 | for _, eachItem := range items { 159 | if eachItem == item { 160 | return true 161 | } 162 | } 163 | return false 164 | } 165 | 166 | // IsTagsNotFound judge is the error get tags from repo http not found 167 | func IsTagsNotFound(err error) bool { 168 | if err.Error() == "Error fetching tags list: invalid status code from registry 404 (Not Found)" { 169 | return true 170 | } 171 | return false 172 | } 173 | 174 | // IsDigestNotFound judge is the digest exist 175 | func IsDigestNotFound(err error) bool { 176 | if strings.Contains(err.Error(), "StatusCode: 404") { 177 | return true 178 | } 179 | return false 180 | } 181 | --------------------------------------------------------------------------------