├── .github
└── workflows
│ └── go-cross-build.yml
├── .gitignore
├── Dockerfile
├── LICENSE
├── README.md
├── README_cn.md
├── auth.go
├── cmd
└── crproxy
│ └── main.go
├── credentials.go
├── crproxy.go
├── crproxy_blob.go
├── crproxy_control.go
├── crproxy_manifest.go
├── ctx_value.go
├── examples
├── default
│ ├── .gitignore
│ ├── README.md
│ ├── docker-compose.yml
│ ├── html
│ │ └── index.html
│ ├── nginx
│ │ └── default.conf
│ ├── registry
│ │ └── config.yml
│ ├── reload.sh
│ ├── setup-alias.sh
│ ├── setup-gateway.sh
│ ├── start.sh
│ └── update-tls.sh
└── simple
│ └── README.md
├── go.mod
├── go.sum
├── hang_manager.go
├── hang_manager_test.go
├── internal
├── acme
│ └── acme.go
├── maps
│ └── maps.go
└── server
│ └── server.go
├── logo
├── LICENSE
├── crproxy.png
└── crproxy.svg
├── storage
└── driver
│ ├── obs
│ ├── doc.go
│ └── obs.go
│ └── oss
│ ├── doc.go
│ └── oss.go
├── sync.go
├── utils.go
└── utils_test.go
/.github/workflows/go-cross-build.yml:
--------------------------------------------------------------------------------
1 | name: Build
2 | on:
3 | push:
4 | tags:
5 | - v*
6 |
7 | jobs:
8 | build:
9 | runs-on: ubuntu-latest
10 | steps:
11 | - uses: actions/checkout@v2
12 | - name: Set up Go
13 | uses: actions/setup-go@v2
14 | with:
15 | go-version: 1.21
16 | - name: Build Cross Platform
17 | uses: wzshiming/action-go-build-cross-plantform@v1
18 | - name: Upload Release Assets
19 | uses: wzshiming/action-upload-release-assets@v1
20 | env:
21 | GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
22 | - name: Log into registry
23 | run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
24 | - name: Upload Release Images
25 | uses: wzshiming/action-upload-release-images@v1
26 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | # Binaries for programs and plugins
2 | *.exe
3 | *.exe~
4 | *.dll
5 | *.so
6 | *.dylib
7 |
8 | # Test binary, built with `go test -c`
9 | *.test
10 |
11 | # Output of the go coverage tool, specifically when used with LiteIDE
12 | *.out
13 |
14 | # Dependency directories (remove the comment below to include it)
15 | # vendor/
16 |
--------------------------------------------------------------------------------
/Dockerfile:
--------------------------------------------------------------------------------
1 | FROM golang:alpine AS builder
2 | WORKDIR /go/src/github.com/wzshiming/crproxy/
3 | COPY . .
4 | ENV CGO_ENABLED=0
5 | RUN go install ./cmd/crproxy
6 |
7 | FROM alpine
8 | EXPOSE 8080
9 | COPY --from=builder /go/bin/crproxy /usr/local/bin/
10 | ENTRYPOINT [ "/usr/local/bin/crproxy" ]
11 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2022 Shiming Zhang
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # CRProxy (Container Registry Proxy)
2 |
3 | 
4 |
5 | CRProxy is a generic image proxy
6 |
7 | Add the prefix `m.daocloud.io/` to all places that need to use images
8 |
9 | - [English](https://github.com/wzshiming/crproxy/blob/master/README.md)
10 | - [简体中文](https://github.com/wzshiming/crproxy/blob/master/README_cn.md)
11 |
12 | ## m.daocloud.io
13 |
14 | you can deploy your own image proxy server if you need to.
15 |
16 | [Refer to](https://github.com/wzshiming/crproxy/tree/master/examples/default)
17 |
18 | ## crproxy + registry(pull through cache mode)
19 |
20 | - for organizations and companies which need to serve big clusters
21 | - [container-image-mirror](https://blog.geekcity.tech/articles/kubernetes/argocd/container-image-mirror/)
22 | * deploy into k8s with argocd
23 | * storage with pvc, s3(minio/oss compatible)
24 | * bind ssl with cert-manager and ingress
25 |
26 | ## On Docker
27 |
28 | Just add the prefix `m.daocloud.io/`
29 |
30 | ``` bash
31 | docker pull m.daocloud.io/docker.io/library/busybox
32 | ```
33 |
34 | ## On Kubernetes
35 |
36 | Just add the prefix `m.daocloud.io/`
37 |
38 | ``` yaml
39 | image: m.daocloud.io/docker.io/library/busybox
40 | ```
41 |
--------------------------------------------------------------------------------
/README_cn.md:
--------------------------------------------------------------------------------
1 | # CRProxy (Container Registry Proxy)
2 |
3 |
4 |
5 | CRProxy 是一个通用的 Image 代理
6 |
7 | 在所有需要使用镜像的地方加上前缀 `m.daocloud.io/`
8 |
9 | - [English](https://github.com/wzshiming/crproxy/blob/master/README.md)
10 | - [简体中文](https://github.com/wzshiming/crproxy/blob/master/README_cn.md)
11 |
12 | ## m.daocloud.io
13 |
14 | 如有需要您可以部署自己的镜像代理服务器
15 |
16 | [参考](https://github.com/wzshiming/crproxy/tree/master/examples/default)
17 |
18 | ## On Docker
19 |
20 | 只需要添加前缀 `m.daocloud.io/`
21 |
22 | ``` bash
23 | docker pull m.daocloud.io/docker.io/library/busybox
24 | ```
25 |
26 | ## On Kubernetes
27 |
28 | 只需要添加前缀 `m.daocloud.io/`
29 |
30 | ``` yaml
31 | image: m.daocloud.io/docker.io/library/busybox
32 | ```
33 |
--------------------------------------------------------------------------------
/auth.go:
--------------------------------------------------------------------------------
1 | package crproxy
2 |
3 | import (
4 | "bytes"
5 | "crypto/rand"
6 | "crypto/sha256"
7 | "encoding/base64"
8 | "encoding/json"
9 | "fmt"
10 | "hash"
11 | "io"
12 | "net/http"
13 | "net/url"
14 | "strings"
15 | "time"
16 |
17 | "github.com/docker/distribution/registry/api/errcode"
18 | )
19 |
20 | func (c *CRProxy) AuthToken(rw http.ResponseWriter, r *http.Request) {
21 | if r.Method != http.MethodGet {
22 | errcode.ServeJSON(rw, errcode.ErrorCodeUnsupported)
23 | return
24 | }
25 | if !c.simpleAuth {
26 | errcode.ServeJSON(rw, errcode.ErrorCodeUnsupported)
27 | return
28 | }
29 | query := r.URL.Query()
30 | scope := query.Get("scope")
31 | service := query.Get("service")
32 |
33 | if c.simpleAuthUserpassFunc != nil {
34 | authorization := r.Header.Get("Authorization")
35 | auth := strings.SplitN(authorization, " ", 2)
36 | if len(auth) != 2 {
37 | if c.logger != nil {
38 | c.logger.Println("Login failed", authorization)
39 | }
40 | errcode.ServeJSON(rw, errcode.ErrorCodeDenied)
41 | return
42 | }
43 | switch auth[0] {
44 | case "Basic":
45 | user, pass, ok := parseBasicAuth(auth[1])
46 | if user == "" || pass == "" {
47 | errcode.ServeJSON(rw, errcode.ErrorCodeDenied)
48 | return
49 | }
50 |
51 | var u *url.Userinfo
52 | if ok {
53 | u = url.UserPassword(user, pass)
54 | } else {
55 | u = url.User(user)
56 | }
57 | if !c.simpleAuthUserpassFunc(r, u) {
58 | if c.logger != nil {
59 | c.logger.Println("Login failed user and password", u)
60 | }
61 | errcode.ServeJSON(rw, errcode.ErrorCodeDenied)
62 | return
63 | }
64 |
65 | if c.logger != nil {
66 | c.logger.Println("Login succeed user", u.Username())
67 | }
68 | default:
69 | if c.logger != nil {
70 | c.logger.Println("Unsupported authorization", authorization)
71 | }
72 | errcode.ServeJSON(rw, errcode.ErrorCodeDenied)
73 | return
74 | }
75 | }
76 |
77 | rw.Header().Set("Content-Type", "application/json")
78 |
79 | now := time.Now()
80 | expiresIn := 60
81 | token := defaultTokenManager.Encode(Token{
82 | Service: service,
83 | Scope: scope,
84 | ExpiresAt: now.Add(time.Duration(expiresIn) * time.Second),
85 | })
86 |
87 | json.NewEncoder(rw).Encode(tokenInfo{
88 | Token: token,
89 | ExpiresIn: int64(expiresIn),
90 | IssuedAt: now,
91 | })
92 | }
93 |
94 | func (c *CRProxy) authenticate(rw http.ResponseWriter, r *http.Request) {
95 | tokenURL := c.tokenURL
96 | if tokenURL == "" {
97 | var scheme = "http"
98 | if c.tokenAuthForceTLS || r.TLS != nil || r.URL.Scheme == "https" {
99 | scheme = "https"
100 | }
101 | tokenURL = scheme + "://" + r.Host + "/auth/token"
102 | }
103 | header := fmt.Sprintf("Bearer realm=%q,service=%q", tokenURL, r.Host)
104 | rw.Header().Set("WWW-Authenticate", header)
105 | c.errorResponse(rw, r, errcode.ErrorCodeUnauthorized)
106 | }
107 |
108 | func (c *CRProxy) authorization(rw http.ResponseWriter, r *http.Request) bool {
109 | if c.privilegedNoAuth && c.isPrivileged(r, nil) {
110 | r.Header.Del("Authorization")
111 | return true
112 | }
113 |
114 | auth := r.Header.Get("Authorization")
115 | if auth == "" {
116 | return false
117 | }
118 |
119 | if !strings.HasPrefix(auth, "Bearer ") {
120 | return false
121 | }
122 |
123 | token, ok := defaultTokenManager.Decode(auth[7:])
124 | if !ok {
125 | return false
126 | }
127 |
128 | if token.ExpiresAt.Before(time.Now()) {
129 | return false
130 | }
131 |
132 | r.Header.Del("Authorization")
133 | return true
134 | }
135 |
136 | type tokenInfo struct {
137 | Token string `json:"token,omitempty"`
138 | ExpiresIn int64 `json:"expires_in,omitempty"`
139 | IssuedAt time.Time `json:"issued_at,omitempty"`
140 | }
141 |
142 | var defaultTokenManager = &tokenManager{
143 | NewHash: sha256.New,
144 | RandReader: rand.Reader,
145 | HashSize: sha256.Size,
146 | RandSize: 16,
147 | EncodeToString: base64.RawURLEncoding.EncodeToString,
148 | DecodeString: base64.RawURLEncoding.DecodeString,
149 | }
150 |
151 | type tokenManager struct {
152 | NewHash func() hash.Hash
153 | RandReader io.Reader
154 | HashSize int
155 | RandSize int
156 | EncodeToString func([]byte) string
157 | DecodeString func(string) ([]byte, error)
158 | }
159 |
160 | type Token struct {
161 | ExpiresAt time.Time `json:"expires_at,omitempty"`
162 | Scope string `json:"scope,omitempty"`
163 | Service string `json:"service,omitempty"`
164 | }
165 |
166 | func (p *tokenManager) Encode(t Token) (code string) {
167 | sum := make([]byte, p.RandSize+p.HashSize)
168 | io.ReadFull(p.RandReader, sum[:p.RandSize])
169 | hashSum := p.NewHash()
170 | data, _ := json.Marshal(t)
171 | hashSum.Write(data)
172 | hashSum.Write(sum[:p.RandSize])
173 | sum = hashSum.Sum(sum[:p.RandSize])
174 | return p.EncodeToString(sum) + "." + p.EncodeToString(data)
175 | }
176 |
177 | func (p *tokenManager) Decode(code string) (t Token, b bool) {
178 | cs := strings.Split(code, ".")
179 | if len(cs) != 2 {
180 | return t, false
181 | }
182 |
183 | sum, err := p.DecodeString(cs[0])
184 | if err != nil {
185 | return t, false
186 | }
187 | if len(sum) != p.HashSize+p.RandSize {
188 | return t, false
189 | }
190 | data, err := p.DecodeString(cs[1])
191 | if err != nil {
192 | return t, false
193 | }
194 | hashSum := p.NewHash()
195 | hashSum.Write(data)
196 | hashSum.Write(sum[:p.RandSize])
197 | newSum := hashSum.Sum(nil)
198 | if !bytes.Equal(sum[p.RandSize:], newSum) {
199 | return t, false
200 | }
201 |
202 | err = json.Unmarshal(data, &t)
203 | if err != nil {
204 | return t, false
205 | }
206 |
207 | return t, true
208 | }
209 |
210 | func parseBasicAuth(auth string) (username, password string, ok bool) {
211 | c, err := base64.StdEncoding.DecodeString(auth)
212 | if err != nil {
213 | return "", "", false
214 | }
215 | cs := string(c)
216 | username, password, ok = strings.Cut(cs, ":")
217 | if !ok {
218 | return "", "", false
219 | }
220 | return username, password, true
221 | }
222 |
--------------------------------------------------------------------------------
/cmd/crproxy/main.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "bufio"
5 | "bytes"
6 | "context"
7 | "fmt"
8 | "io"
9 | "log"
10 | "net/http"
11 | "net/http/pprof"
12 | "net/url"
13 | "os"
14 | "slices"
15 | "strings"
16 | "sync/atomic"
17 | "time"
18 |
19 | "github.com/docker/distribution/registry/storage/driver/factory"
20 | "github.com/gorilla/handlers"
21 | "github.com/spf13/pflag"
22 | "github.com/wzshiming/geario"
23 | "github.com/wzshiming/hostmatcher"
24 |
25 | _ "github.com/daocloud/crproxy/storage/driver/obs"
26 | _ "github.com/daocloud/crproxy/storage/driver/oss"
27 | _ "github.com/docker/distribution/registry/storage/driver/azure"
28 | _ "github.com/docker/distribution/registry/storage/driver/gcs"
29 | _ "github.com/docker/distribution/registry/storage/driver/s3-aws"
30 |
31 | "github.com/daocloud/crproxy"
32 | "github.com/daocloud/crproxy/internal/server"
33 | )
34 |
35 | var (
36 | behind bool
37 | address string
38 | userpass []string
39 | disableKeepAlives []string
40 | limitDelay bool
41 | blobsSpeedLimit string
42 | ipsSpeedLimit string
43 | totalBlobsSpeedLimit string
44 | allowHostList []string
45 | allowImageListFromFile string
46 | blockImageList []string
47 | blockMessage string
48 | privilegedIPList []string
49 | privilegedImageListFromFile string
50 | privilegedNoAuth bool
51 | retry int
52 | retryInterval time.Duration
53 | storageDriver string
54 | storageParameters map[string]string
55 | linkExpires time.Duration
56 | redirectLinks string
57 | disableTagsList bool
58 | enablePprof bool
59 | defaultRegistry string
60 | overrideDefaultRegistry map[string]string
61 | simpleAuth bool
62 | simpleAuthUserpass map[string]string
63 | tokenURL string
64 | tokenAuthForceTLS bool
65 |
66 | redirectOriginBlobLinks bool
67 |
68 | acmeHosts []string
69 | acmeCacheDir string
70 | certFile string
71 | privateKeyFile string
72 |
73 | enableInternalAPI bool
74 |
75 | readmeURL string
76 |
77 | allowHeadMethod bool
78 |
79 | manifestCacheDuration time.Duration
80 | )
81 |
82 | func init() {
83 | pflag.BoolVar(&behind, "behind", false, "Behind the reverse proxy")
84 | pflag.StringSliceVarP(&userpass, "user", "u", nil, "host and username and password -u user:pwd@host")
85 | pflag.StringVarP(&address, "address", "a", ":8080", "listen on the address")
86 | pflag.StringSliceVar(&disableKeepAlives, "disable-keep-alives", nil, "disable keep alives for the host")
87 | pflag.BoolVar(&limitDelay, "limit-delay", false, "limit with delay")
88 | pflag.StringVar(&blobsSpeedLimit, "blobs-speed-limit", "", "blobs speed limit per second (default unlimited)")
89 | pflag.StringVar(&ipsSpeedLimit, "ips-speed-limit", "", "ips speed limit per second (default unlimited)")
90 | pflag.StringVar(&totalBlobsSpeedLimit, "total-blobs-speed-limit", "", "total blobs speed limit per second (default unlimited)")
91 | pflag.StringSliceVar(&allowHostList, "allow-host-list", nil, "allow host list")
92 | pflag.StringVar(&allowImageListFromFile, "allow-image-list-from-file", "", "allow image list from file")
93 | pflag.StringSliceVar(&blockImageList, "block-image-list", nil, "block image list (deprecated)")
94 | pflag.StringVar(&blockMessage, "block-message", "", "block message")
95 | pflag.StringSliceVar(&privilegedIPList, "privileged-ip-list", nil, "privileged IP list")
96 | pflag.BoolVar(&privilegedNoAuth, "privileged-no-auth", false, "privileged no auth (deprecated)")
97 | pflag.StringVar(&privilegedImageListFromFile, "privileged-image-list-from-file", "", "privileged image list from file")
98 | pflag.IntVar(&retry, "retry", 0, "retry times")
99 | pflag.DurationVar(&retryInterval, "retry-interval", 0, "retry interval")
100 | pflag.StringVar(&storageDriver, "storage-driver", "", "storage driver")
101 | pflag.StringToStringVar(&storageParameters, "storage-parameters", nil, "storage parameters")
102 | pflag.DurationVar(&linkExpires, "link-expires", 0, "link expires")
103 | pflag.StringVar(&redirectLinks, "redirect-links", "", "redirect links")
104 | pflag.BoolVar(&disableTagsList, "disable-tags-list", false, "disable tags list")
105 | pflag.BoolVar(&enablePprof, "enable-pprof", false, "Enable pprof")
106 | pflag.StringVar(&defaultRegistry, "default-registry", "", "default registry used for non full-path docker pull, like:docker.io")
107 | pflag.StringToStringVar(&overrideDefaultRegistry, "override-default-registry", nil, "override default registry")
108 | pflag.BoolVar(&simpleAuth, "simple-auth", false, "enable simple auth")
109 | pflag.StringToStringVar(&simpleAuthUserpass, "simple-auth-user", nil, "simple auth user and password")
110 | pflag.StringVar(&tokenURL, "token-url", "", "token url (deprecated)")
111 | pflag.BoolVar(&tokenAuthForceTLS, "token-auth-force-tls", false, "token auth force TLS (deprecated)")
112 |
113 | pflag.BoolVar(&redirectOriginBlobLinks, "redirect-origin-blob-links", false, "redirect origin blob links")
114 |
115 | pflag.StringSliceVar(&acmeHosts, "acme-hosts", nil, "acme hosts")
116 | pflag.StringVar(&acmeCacheDir, "acme-cache-dir", "", "acme cache dir")
117 | pflag.StringVar(&certFile, "cert-file", "", "cert file")
118 | pflag.StringVar(&privateKeyFile, "private-key-file", "", "private key file")
119 | pflag.BoolVar(&enableInternalAPI, "enable-internal-api", false, "enable internal api")
120 |
121 | pflag.StringVar(&readmeURL, "readme-url", "", "redirect readme url when not found")
122 | pflag.BoolVar(&allowHeadMethod, "allow-head-method", false, "allow head method")
123 |
124 | pflag.DurationVar(&manifestCacheDuration, "manifest-cache-duration", 0, "manifest cache duration")
125 | pflag.Parse()
126 | }
127 |
128 | func toUserAndPass(userpass []string) (map[string]crproxy.Userpass, error) {
129 | bc := map[string]crproxy.Userpass{}
130 | for _, up := range userpass {
131 | s := strings.SplitN(up, "@", 3)
132 | if len(s) != 2 {
133 | return nil, fmt.Errorf("invalid userpass %q", up)
134 | }
135 |
136 | u := strings.SplitN(s[0], ":", 3)
137 | if len(s) != 2 {
138 | return nil, fmt.Errorf("invalid userpass %q", up)
139 | }
140 | host := s[1]
141 | user := u[0]
142 | pwd := u[1]
143 | bc[host] = crproxy.Userpass{
144 | Username: user,
145 | Password: pwd,
146 | }
147 | }
148 | return bc, nil
149 | }
150 |
151 | func main() {
152 | ctx := context.Background()
153 | logger := log.New(os.Stderr, "[cr proxy] ", log.LstdFlags)
154 |
155 | mux := http.NewServeMux()
156 | cli := &http.Client{
157 | CheckRedirect: func(req *http.Request, via []*http.Request) error {
158 | if len(via) > 10 {
159 | return http.ErrUseLastResponse
160 | }
161 | s := make([]string, 0, len(via)+1)
162 | for _, v := range via {
163 | s = append(s, v.URL.String())
164 | }
165 |
166 | lastRedirect := req.URL.String()
167 | s = append(s, lastRedirect)
168 | logger.Println("redirect", s)
169 |
170 | if v := crproxy.GetCtxValue(req.Context()); v != nil {
171 | v.LastRedirect = lastRedirect
172 | }
173 | return nil
174 | },
175 | }
176 |
177 | opts := []crproxy.Option{
178 | crproxy.WithBaseClient(cli),
179 | crproxy.WithLogger(logger),
180 | crproxy.WithMaxClientSizeForEachRegistry(16),
181 | crproxy.WithDomainAlias(map[string]string{
182 | "docker.io": "registry-1.docker.io",
183 | "ollama.ai": "registry.ollama.ai",
184 | }),
185 | crproxy.WithPathInfoModifyFunc(func(info *crproxy.ImageInfo) *crproxy.ImageInfo {
186 | // docker.io/busybox => docker.io/library/busybox
187 | if info.Host == "docker.io" && !strings.Contains(info.Name, "/") {
188 | info.Name = "library/" + info.Name
189 | }
190 | if info.Host == "ollama.ai" && !strings.Contains(info.Name, "/") {
191 | info.Name = "library/" + info.Name
192 | }
193 | return info
194 | }),
195 | crproxy.WithDisableKeepAlives(disableKeepAlives),
196 | }
197 |
198 | if storageDriver != "" {
199 | parameters := map[string]interface{}{}
200 | for k, v := range storageParameters {
201 | parameters[k] = v
202 | }
203 | sd, err := factory.Create(storageDriver, parameters)
204 | if err != nil {
205 | logger.Println("create storage driver failed:", err)
206 | os.Exit(1)
207 | }
208 | opts = append(opts, crproxy.WithStorageDriver(sd))
209 | if linkExpires > 0 {
210 | opts = append(opts, crproxy.WithLinkExpires(linkExpires))
211 | }
212 | if redirectLinks != "" {
213 | u, err := url.Parse(redirectLinks)
214 | if err != nil {
215 | logger.Println("parse redirect links failed:", err)
216 | os.Exit(1)
217 | }
218 | opts = append(opts, crproxy.WithRedirectLinks(u))
219 | }
220 | }
221 |
222 | if allowImageListFromFile != "" {
223 | f, err := os.ReadFile(allowImageListFromFile)
224 | if err != nil {
225 | logger.Println("can't read allow list file", allowImageListFromFile, ":", err)
226 | os.Exit(1)
227 | }
228 |
229 | var matcher atomic.Pointer[hostmatcher.Matcher]
230 | m, err := getListFrom(bytes.NewReader(f))
231 | if err != nil {
232 | logger.Println("can't read allow list file", allowImageListFromFile, ":", err)
233 | os.Exit(1)
234 | }
235 | matcher.Store(&m)
236 |
237 | if enableInternalAPI {
238 | mux.HandleFunc("PUT /internal/api/allows", func(rw http.ResponseWriter, r *http.Request) {
239 | body, err := io.ReadAll(r.Body)
240 | if err != nil {
241 | logger.Println("read body failed:", err)
242 | rw.WriteHeader(http.StatusBadRequest)
243 | rw.Write([]byte(err.Error()))
244 | return
245 | }
246 | m, err := getListFrom(bytes.NewReader(body))
247 | if err != nil {
248 | logger.Println("can't read allow list file", allowImageListFromFile, ":", err)
249 | rw.WriteHeader(http.StatusBadRequest)
250 | rw.Write([]byte(err.Error()))
251 | return
252 | }
253 |
254 | err = os.WriteFile(allowImageListFromFile, body, 0644)
255 | if err != nil {
256 | logger.Println("write file failed:", err)
257 | rw.WriteHeader(http.StatusBadRequest)
258 | rw.Write([]byte(err.Error()))
259 | return
260 | }
261 |
262 | matcher.Store(&m)
263 | })
264 | }
265 | opts = append(opts, crproxy.WithBlockFunc(func(info *crproxy.BlockInfo) (string, bool) {
266 | if (*matcher.Load()).Match(info.Host + "/" + info.Name) {
267 | return "", false
268 | }
269 | return blockMessage, true
270 | }))
271 | } else if len(blockImageList) != 0 || len(allowHostList) != 0 {
272 | allowHostMap := map[string]struct{}{}
273 | for _, host := range allowHostList {
274 | allowHostMap[host] = struct{}{}
275 | }
276 | blockImageMap := map[string]struct{}{}
277 | for _, image := range blockImageList {
278 | blockImageMap[image] = struct{}{}
279 | }
280 | opts = append(opts, crproxy.WithBlockFunc(func(info *crproxy.BlockInfo) (string, bool) {
281 | if len(allowHostMap) != 0 {
282 | _, ok := allowHostMap[info.Host]
283 | if !ok {
284 | return blockMessage, true
285 | }
286 | }
287 |
288 | if len(blockImageMap) != 0 {
289 | image := info.Host + "/" + info.Name
290 | _, ok := blockImageMap[image]
291 | if ok {
292 | return blockMessage, true
293 | }
294 | }
295 |
296 | return "", false
297 | }))
298 | }
299 |
300 | if len(privilegedIPList) != 0 || privilegedImageListFromFile != "" {
301 | var matcher atomic.Pointer[hostmatcher.Matcher]
302 | if privilegedImageListFromFile != "" {
303 | f, err := os.ReadFile(privilegedImageListFromFile)
304 | if err != nil {
305 | logger.Println("can't read privileged list file", privilegedImageListFromFile, ":", err)
306 | os.Exit(1)
307 | }
308 |
309 | m, err := getListFrom(bytes.NewReader(f))
310 | if err != nil {
311 | logger.Println("can't read privileged list file", privilegedImageListFromFile, ":", err)
312 | os.Exit(1)
313 | }
314 | matcher.Store(&m)
315 |
316 | if enableInternalAPI {
317 | mux.HandleFunc("PUT /internal/api/privileged", func(rw http.ResponseWriter, r *http.Request) {
318 | body, err := io.ReadAll(r.Body)
319 | if err != nil {
320 | logger.Println("read body failed:", err)
321 | rw.WriteHeader(http.StatusBadRequest)
322 | rw.Write([]byte(err.Error()))
323 | return
324 | }
325 | m, err := getListFrom(bytes.NewReader(body))
326 | if err != nil {
327 | logger.Println("can't read allow list file", privilegedImageListFromFile, ":", err)
328 | rw.WriteHeader(http.StatusBadRequest)
329 | rw.Write([]byte(err.Error()))
330 | return
331 | }
332 |
333 | err = os.WriteFile(privilegedImageListFromFile, body, 0644)
334 | if err != nil {
335 | logger.Println("write file failed:", err)
336 | rw.WriteHeader(http.StatusBadRequest)
337 | rw.Write([]byte(err.Error()))
338 | return
339 | }
340 |
341 | matcher.Store(&m)
342 | })
343 | }
344 | }
345 |
346 | set := map[string]struct{}{}
347 | for _, ip := range privilegedIPList {
348 | set[ip] = struct{}{}
349 | }
350 | opts = append(opts, crproxy.WithPrivilegedFunc(func(r *http.Request, info *crproxy.ImageInfo) bool {
351 | if len(set) != 0 {
352 | ip := r.RemoteAddr
353 | if _, ok := set[ip]; ok {
354 | return true
355 | }
356 | }
357 | if m := matcher.Load(); m != nil && info != nil {
358 | return (*m).Match(info.Host + "/" + info.Name)
359 | }
360 | return false
361 | }))
362 | }
363 |
364 | if privilegedNoAuth {
365 | opts = append(opts, crproxy.WithPrivilegedNoAuth(true))
366 | }
367 |
368 | if len(userpass) != 0 {
369 | bc, err := toUserAndPass(userpass)
370 | if err != nil {
371 | logger.Println("failed to toUserAndPass:", err)
372 | os.Exit(1)
373 | }
374 | opts = append(opts, crproxy.WithUserAndPass(bc))
375 | }
376 |
377 | if ipsSpeedLimit != "" {
378 | b, d, err := getLimit(ipsSpeedLimit)
379 | if err != nil {
380 | logger.Println("failed to getLimit:", err)
381 | os.Exit(1)
382 | }
383 | opts = append(opts, crproxy.WithIPsSpeedLimit(b, d))
384 | }
385 |
386 | if blobsSpeedLimit != "" {
387 | b, d, err := getLimit(blobsSpeedLimit)
388 | if err != nil {
389 | logger.Println("failed to getLimit:", err)
390 | os.Exit(1)
391 | }
392 | opts = append(opts, crproxy.WithBlobsSpeedLimit(b, d))
393 | }
394 |
395 | if totalBlobsSpeedLimit != "" {
396 | b, err := geario.FromBytesSize(totalBlobsSpeedLimit)
397 | if err != nil {
398 | logger.Println("failed to FromBytesSize:", err)
399 | os.Exit(1)
400 | }
401 | opts = append(opts, crproxy.WithTotalBlobsSpeedLimit(b))
402 | }
403 |
404 | if disableTagsList {
405 | opts = append(opts, crproxy.WithDisableTagsList(true))
406 | }
407 |
408 | if retry > 0 {
409 | opts = append(opts, crproxy.WithRetry(retry, retryInterval))
410 | }
411 | if limitDelay {
412 | opts = append(opts, crproxy.WithLimitDelay(true))
413 | }
414 |
415 | if defaultRegistry != "" {
416 | opts = append(opts, crproxy.WithDefaultRegistry(defaultRegistry))
417 | }
418 |
419 | if len(overrideDefaultRegistry) != 0 {
420 | opts = append(opts, crproxy.WithOverrideDefaultRegistry(overrideDefaultRegistry))
421 | }
422 |
423 | if simpleAuth {
424 | opts = append(opts, crproxy.WithSimpleAuth(true, tokenURL, tokenAuthForceTLS))
425 | }
426 | if len(simpleAuthUserpass) != 0 {
427 |
428 | opts = append(opts, crproxy.WithSimpleAuthUserFunc(func(r *http.Request, userinfo *url.Userinfo) bool {
429 | pass, ok := simpleAuthUserpass[userinfo.Username()]
430 | if !ok {
431 | return false
432 | }
433 | upass, ok := userinfo.Password()
434 | if !ok {
435 | return false
436 | }
437 | if upass != pass {
438 | return false
439 | }
440 | return true
441 | }))
442 | }
443 |
444 | if redirectOriginBlobLinks {
445 | opts = append(opts, crproxy.WithRedirectToOriginBlobFunc(func(r *http.Request, info *crproxy.ImageInfo) bool {
446 | return true
447 | }))
448 | }
449 |
450 | if allowHeadMethod {
451 | opts = append(opts, crproxy.WithAllowHeadMethod(allowHeadMethod))
452 | }
453 |
454 | if manifestCacheDuration != 0 {
455 | opts = append(opts, crproxy.WithManifestCacheDuration(manifestCacheDuration))
456 | }
457 |
458 | crp, err := crproxy.NewCRProxy(opts...)
459 | if err != nil {
460 | logger.Println("failed to NewCRProxy:", err)
461 | os.Exit(1)
462 | }
463 |
464 | mux.Handle("/v2/", crp)
465 | mux.HandleFunc("/auth/token", crp.AuthToken)
466 |
467 | mux.HandleFunc("/internal/api/image/sync", crp.Sync)
468 |
469 | if enablePprof {
470 | mux.HandleFunc("/debug/pprof/", pprof.Index)
471 | mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
472 | mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
473 | mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
474 | mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
475 | }
476 | if readmeURL != "" {
477 | mux.HandleFunc("/", func(rw http.ResponseWriter, r *http.Request) {
478 | http.Redirect(rw, r, readmeURL, http.StatusFound)
479 | })
480 | }
481 |
482 | var handler http.Handler = mux
483 | handler = handlers.LoggingHandler(os.Stderr, handler)
484 | if behind {
485 | handler = handlers.ProxyHeaders(handler)
486 | }
487 |
488 | err = server.Run(ctx, address, handler, acmeHosts, acmeCacheDir, certFile, privateKeyFile)
489 | if err != nil {
490 | logger.Println("failed to ListenAndServe:", err)
491 | os.Exit(1)
492 | }
493 | }
494 |
495 | func getLimit(s string) (geario.B, time.Duration, error) {
496 | i := strings.Index(s, "/")
497 | if i == -1 {
498 | b, err := geario.FromBytesSize(s)
499 | if err != nil {
500 | return 0, 0, err
501 | }
502 | return b, time.Second, nil
503 | }
504 |
505 | b, err := geario.FromBytesSize(s[:i])
506 | if err != nil {
507 | return 0, 0, err
508 | }
509 |
510 | dur := s[i+1:]
511 | if dur[0] < '0' || dur[0] > '9' {
512 | dur = "1" + dur
513 | }
514 |
515 | d, err := time.ParseDuration(dur)
516 | if err != nil {
517 | return 0, 0, err
518 | }
519 |
520 | return b, d, nil
521 | }
522 |
523 | func getListFrom(r io.Reader) (hostmatcher.Matcher, error) {
524 | lines := bufio.NewReader(r)
525 | hosts := []string{}
526 | for {
527 | line, _, err := lines.ReadLine()
528 | if err == io.EOF {
529 | break
530 | }
531 | h := strings.TrimSpace(string(line))
532 | if len(h) == 0 {
533 | continue
534 | }
535 | hosts = append(hosts, h)
536 | }
537 | if len(hosts) == 0 {
538 | return nil, fmt.Errorf("no hosts found")
539 | }
540 | if !slices.IsSorted(hosts) {
541 | return nil, fmt.Errorf("hosts not sorted: %v", hosts)
542 | }
543 | return hostmatcher.NewMatcher(hosts), nil
544 | }
545 |
--------------------------------------------------------------------------------
/credentials.go:
--------------------------------------------------------------------------------
1 | package crproxy
2 |
3 | import (
4 | "net/http"
5 | "net/url"
6 | "strings"
7 |
8 | "github.com/docker/distribution/registry/client/auth/challenge"
9 | )
10 |
11 | type Userpass struct {
12 | Username string
13 | Password string
14 | }
15 |
16 | type basicCredentials struct {
17 | credentials map[string]Userpass
18 | }
19 |
20 | func newBasicCredentials(cred map[string]Userpass, domainAlias func(string) string, hostScheme func(string) string) (*basicCredentials, error) {
21 | bc := &basicCredentials{
22 | credentials: map[string]Userpass{},
23 | }
24 | for domain, c := range cred {
25 | urls, err := getAuthURLs(hostScheme(domain)+"://"+domain, domainAlias)
26 | if err != nil {
27 | return nil, err
28 | }
29 | for _, u := range urls {
30 | bc.credentials[u] = c
31 | }
32 | }
33 | return bc, nil
34 | }
35 |
36 | func (c *basicCredentials) Basic(u *url.URL) (string, string) {
37 | up := c.credentials[u.String()]
38 |
39 | return up.Username, up.Password
40 | }
41 |
42 | func (c *basicCredentials) RefreshToken(u *url.URL, service string) string {
43 | return ""
44 | }
45 |
46 | func (c *basicCredentials) SetRefreshToken(u *url.URL, service, token string) {
47 | }
48 |
49 | func getAuthURLs(remoteURL string, domainAlias func(string) string) ([]string, error) {
50 | authURLs := []string{}
51 |
52 | u, err := url.Parse(remoteURL)
53 | if err != nil {
54 | return nil, err
55 | }
56 | if domainAlias != nil {
57 | u.Host = domainAlias(u.Host)
58 | }
59 | remoteURL = u.String()
60 |
61 | resp, err := http.Get(remoteURL + "/v2/")
62 | if err != nil {
63 | return nil, err
64 | }
65 | defer resp.Body.Close()
66 |
67 | for _, c := range challenge.ResponseChallenges(resp) {
68 | if strings.EqualFold(c.Scheme, "bearer") {
69 | authURLs = append(authURLs, c.Parameters["realm"])
70 | }
71 | }
72 |
73 | return authURLs, nil
74 | }
75 |
--------------------------------------------------------------------------------
/crproxy.go:
--------------------------------------------------------------------------------
1 | package crproxy
2 |
3 | import (
4 | "context"
5 | "errors"
6 | "fmt"
7 | "io"
8 | "net"
9 | "net/http"
10 | "net/textproto"
11 | "net/url"
12 | "strings"
13 | "sync"
14 | "time"
15 |
16 | "github.com/daocloud/crproxy/internal/maps"
17 | "github.com/docker/distribution/registry/api/errcode"
18 | "github.com/docker/distribution/registry/client/auth"
19 | "github.com/docker/distribution/registry/client/auth/challenge"
20 | "github.com/docker/distribution/registry/client/transport"
21 | storagedriver "github.com/docker/distribution/registry/storage/driver"
22 | "github.com/wzshiming/geario"
23 | "github.com/wzshiming/hostmatcher"
24 | "github.com/wzshiming/httpseek"
25 | "github.com/wzshiming/lru"
26 | )
27 |
28 | var (
29 | prefix = "/v2/"
30 | catalog = prefix + "_catalog"
31 | )
32 |
33 | type Logger interface {
34 | Println(v ...interface{})
35 | }
36 |
37 | type ImageInfo struct {
38 | Host string
39 | Name string
40 | }
41 |
42 | type BlockInfo struct {
43 | IP string
44 | Host string
45 | Name string
46 | }
47 |
48 | type CRProxy struct {
49 | baseClient *http.Client
50 | challengeManager challenge.Manager
51 | clientset maps.SyncMap[string, *lru.LRU[string, *http.Client]]
52 | clientSize int
53 | modify func(info *ImageInfo) *ImageInfo
54 | insecureDomain map[string]struct{}
55 | domainDisableKeepAlives map[string]struct{}
56 | domainAlias map[string]string
57 | userAndPass map[string]Userpass
58 | basicCredentials *basicCredentials
59 | mutClientset sync.Mutex
60 | bytesPool sync.Pool
61 | logger Logger
62 | totalBlobsSpeedLimit *geario.Gear
63 | speedLimitRecord maps.SyncMap[string, *geario.BPS]
64 | blobsSpeedLimit *geario.B
65 | blobsSpeedLimitDuration time.Duration
66 | ipsSpeedLimit *geario.B
67 | ipsSpeedLimitDuration time.Duration
68 | blockFunc []func(*BlockInfo) (string, bool)
69 | retry int
70 | retryInterval time.Duration
71 | storageDriver storagedriver.StorageDriver
72 | linkExpires time.Duration
73 | mutCache sync.Map
74 | redirectLinks *url.URL
75 | limitDelay bool
76 | privilegedNoAuth bool
77 | disableTagsList bool
78 | simpleAuth bool
79 | simpleAuthUserpassFunc func(r *http.Request, userinfo *url.Userinfo) bool
80 | tokenURL string
81 | tokenAuthForceTLS bool
82 | matcher hostmatcher.Matcher
83 |
84 | defaultRegistry string
85 | overrideDefaultRegistry map[string]string
86 |
87 | privilegedFunc func(r *http.Request, info *ImageInfo) bool
88 | redirectToOriginBlobFunc func(r *http.Request, info *ImageInfo) bool
89 | allowHeadMethod bool
90 |
91 | manifestCache maps.SyncMap[string, time.Time]
92 | manifestCacheDuration time.Duration
93 | }
94 |
95 | type Option func(c *CRProxy)
96 |
97 | func WithManifestCacheDuration(d time.Duration) Option {
98 | return func(c *CRProxy) {
99 | c.manifestCacheDuration = d
100 | }
101 | }
102 |
103 | func WithPrivilegedFunc(f func(r *http.Request, info *ImageInfo) bool) Option {
104 | return func(c *CRProxy) {
105 | c.privilegedFunc = f
106 | }
107 | }
108 |
109 | func WithRedirectToOriginBlobFunc(f func(r *http.Request, info *ImageInfo) bool) Option {
110 | return func(c *CRProxy) {
111 | c.redirectToOriginBlobFunc = f
112 | }
113 | }
114 |
115 | func WithSimpleAuth(b bool, tokenURL string, forceTLS bool) Option {
116 | return func(c *CRProxy) {
117 | c.simpleAuth = b
118 | c.tokenURL = tokenURL
119 | c.tokenAuthForceTLS = forceTLS
120 | }
121 | }
122 |
123 | func WithSimpleAuthUserFunc(f func(r *http.Request, userinfo *url.Userinfo) bool) Option {
124 | return func(c *CRProxy) {
125 | c.simpleAuthUserpassFunc = f
126 | }
127 | }
128 |
129 | func WithDefaultRegistry(target string) Option {
130 | return func(c *CRProxy) {
131 | c.defaultRegistry = target
132 | }
133 | }
134 |
135 | func WithOverrideDefaultRegistry(overrideDefaultRegistry map[string]string) Option {
136 | return func(c *CRProxy) {
137 | c.overrideDefaultRegistry = overrideDefaultRegistry
138 | }
139 | }
140 |
141 | func WithDisableTagsList(b bool) Option {
142 | return func(c *CRProxy) {
143 | c.disableTagsList = b
144 | }
145 | }
146 |
147 | func WithPrivilegedNoAuth(b bool) Option {
148 | return func(c *CRProxy) {
149 | c.privilegedNoAuth = true
150 | }
151 | }
152 |
153 | func WithLimitDelay(b bool) Option {
154 | return func(c *CRProxy) {
155 | c.limitDelay = b
156 | }
157 | }
158 |
159 | func WithLinkExpires(d time.Duration) Option {
160 | return func(c *CRProxy) {
161 | c.linkExpires = d
162 | }
163 | }
164 |
165 | func WithRedirectLinks(l *url.URL) Option {
166 | return func(c *CRProxy) {
167 | c.redirectLinks = l
168 | }
169 | }
170 |
171 | func WithStorageDriver(storageDriver storagedriver.StorageDriver) Option {
172 | return func(c *CRProxy) {
173 | c.storageDriver = storageDriver
174 | }
175 | }
176 |
177 | func WithBlobsSpeedLimit(limit geario.B, duration time.Duration) Option {
178 | return func(c *CRProxy) {
179 | c.blobsSpeedLimit = &limit
180 | c.blobsSpeedLimitDuration = duration
181 | }
182 | }
183 |
184 | func WithIPsSpeedLimit(limit geario.B, duration time.Duration) Option {
185 | return func(c *CRProxy) {
186 | c.ipsSpeedLimit = &limit
187 | c.ipsSpeedLimitDuration = duration
188 | }
189 | }
190 |
191 | func WithTotalBlobsSpeedLimit(limit geario.B) Option {
192 | return func(c *CRProxy) {
193 | c.totalBlobsSpeedLimit = geario.NewGear(time.Second, limit)
194 | }
195 | }
196 |
197 | func WithBaseClient(baseClient *http.Client) Option {
198 | return func(c *CRProxy) {
199 | c.baseClient = baseClient
200 | }
201 | }
202 |
203 | func WithLogger(logger Logger) Option {
204 | return func(c *CRProxy) {
205 | c.logger = logger
206 | }
207 | }
208 |
209 | func WithUserAndPass(userAndPass map[string]Userpass) Option {
210 | return func(c *CRProxy) {
211 | c.userAndPass = userAndPass
212 | }
213 | }
214 |
215 | func WithDomainAlias(domainAlias map[string]string) Option {
216 | return func(c *CRProxy) {
217 | c.domainAlias = domainAlias
218 | }
219 | }
220 |
221 | func WithPathInfoModifyFunc(modify func(info *ImageInfo) *ImageInfo) Option {
222 | return func(c *CRProxy) {
223 | c.modify = modify
224 | }
225 | }
226 |
227 | func WithMaxClientSizeForEachRegistry(clientSize int) Option {
228 | return func(c *CRProxy) {
229 | c.clientSize = clientSize
230 | }
231 | }
232 |
233 | func WithDisableKeepAlives(disableKeepAlives []string) Option {
234 | return func(c *CRProxy) {
235 | c.domainDisableKeepAlives = map[string]struct{}{}
236 | for _, v := range disableKeepAlives {
237 | c.domainDisableKeepAlives[v] = struct{}{}
238 | }
239 | }
240 | }
241 |
242 | func WithBlockFunc(blockFunc func(info *BlockInfo) (string, bool)) Option {
243 | return func(c *CRProxy) {
244 | c.blockFunc = append(c.blockFunc, blockFunc)
245 | }
246 | }
247 |
248 | func WithRetry(retry int, retryInterval time.Duration) Option {
249 | return func(c *CRProxy) {
250 | c.retry = retry
251 | c.retryInterval = retryInterval
252 | }
253 | }
254 |
255 | func WithAllowHeadMethod(allowHeadMethod bool) Option {
256 | return func(c *CRProxy) {
257 | c.allowHeadMethod = allowHeadMethod
258 | }
259 | }
260 |
261 | func NewCRProxy(opts ...Option) (*CRProxy, error) {
262 | c := &CRProxy{
263 | challengeManager: challenge.NewSimpleManager(),
264 | clientSize: 10240,
265 | baseClient: http.DefaultClient,
266 | bytesPool: sync.Pool{
267 | New: func() interface{} {
268 | return make([]byte, 32*1024)
269 | },
270 | },
271 | }
272 | for _, opt := range opts {
273 | opt(c)
274 | }
275 | if len(c.userAndPass) != 0 {
276 | bc, err := newBasicCredentials(c.userAndPass, c.getDomainAlias, c.getScheme)
277 | if err != nil {
278 | return nil, err
279 | }
280 | c.basicCredentials = bc
281 | }
282 | return c, nil
283 | }
284 |
285 | func (c *CRProxy) hostURL(host string) string {
286 | return c.getScheme(host) + "://" + host
287 | }
288 |
289 | func (c *CRProxy) pingURL(host string) string {
290 | return c.hostURL(host) + prefix
291 | }
292 |
293 | func (c *CRProxy) getScheme(host string) string {
294 | if c.insecureDomain != nil {
295 | _, ok := c.insecureDomain[host]
296 | if ok {
297 | return "http"
298 | }
299 | }
300 | return "https"
301 | }
302 |
303 | func (c *CRProxy) getClientset(host string, image string) *http.Client {
304 | sets, hasSets := c.clientset.Load(host)
305 | if hasSets {
306 | client, ok := sets.Get(image)
307 | if ok {
308 | return client
309 | }
310 | }
311 |
312 | c.mutClientset.Lock()
313 | defer c.mutClientset.Unlock()
314 | if sets == nil {
315 | sets = lru.NewLRU(c.clientSize, func(image string, client *http.Client) {
316 | if c.logger != nil {
317 | c.logger.Println("evicted client", host, image)
318 | }
319 | client.CloseIdleConnections()
320 | })
321 | c.clientset.Store(host, sets)
322 | }
323 |
324 | if c.logger != nil {
325 | c.logger.Println("cache client", host, image)
326 | }
327 | var credentialStore auth.CredentialStore
328 | if c.basicCredentials != nil {
329 | credentialStore = c.basicCredentials
330 | }
331 | authHandler := auth.NewTokenHandler(nil, credentialStore, image, "pull")
332 |
333 | tr := c.baseClient.Transport
334 |
335 | if c.domainDisableKeepAlives != nil {
336 | if _, ok := c.domainDisableKeepAlives[host]; ok {
337 | tr = c.disableKeepAlives(tr)
338 | }
339 | }
340 |
341 | if c.retryInterval > 0 {
342 | if tr == nil {
343 | tr = http.DefaultTransport
344 | }
345 | tr = httpseek.NewMustReaderTransport(tr, func(request *http.Request, retry int, err error) error {
346 | if errors.Is(err, context.Canceled) ||
347 | errors.Is(err, context.DeadlineExceeded) {
348 | return err
349 | }
350 | if c.retry > 0 && retry >= c.retry {
351 | return err
352 | }
353 | if c.logger != nil {
354 | c.logger.Println("Retry", request.URL, retry, err)
355 | }
356 | time.Sleep(c.retryInterval)
357 | return nil
358 | })
359 | }
360 |
361 | tr = transport.NewTransport(tr, auth.NewAuthorizer(c.challengeManager, authHandler))
362 |
363 | client := &http.Client{
364 | Transport: tr,
365 | CheckRedirect: c.baseClient.CheckRedirect,
366 | Timeout: c.baseClient.Timeout,
367 | Jar: c.baseClient.Jar,
368 | }
369 |
370 | sets.Put(image, client)
371 | return client
372 | }
373 |
374 | func (c *CRProxy) disableKeepAlives(rt http.RoundTripper) http.RoundTripper {
375 | if rt == nil {
376 | tr := http.DefaultTransport.(*http.Transport).Clone()
377 | tr.DisableKeepAlives = true
378 | return tr
379 | }
380 | if tr, ok := rt.(*http.Transport); ok {
381 | if !tr.DisableKeepAlives {
382 | tr = tr.Clone()
383 | tr.DisableKeepAlives = true
384 | }
385 | return tr
386 | }
387 | if c.logger != nil {
388 | c.logger.Println("failed to disable keep alives")
389 | }
390 | return rt
391 | }
392 |
393 | func (c *CRProxy) ping(host string) error {
394 | if c.logger != nil {
395 | c.logger.Println("ping", host)
396 | }
397 |
398 | ep := c.pingURL(host)
399 | e, err := url.Parse(ep)
400 | if err != nil {
401 | return err
402 | }
403 | challenges, err := c.challengeManager.GetChallenges(*e)
404 | if err == nil && len(challenges) != 0 {
405 | return nil
406 | }
407 |
408 | resp, err := c.baseClient.Get(ep)
409 | if err != nil {
410 | return err
411 | }
412 | defer resp.Body.Close()
413 | err = c.challengeManager.AddResponse(resp)
414 | if err != nil {
415 | return err
416 | }
417 | return nil
418 | }
419 |
420 | func apiBase(w http.ResponseWriter, r *http.Request) {
421 | const emptyJSON = "{}"
422 | // Provide a simple /v2/ 200 OK response with empty json response.
423 | w.Header().Set("Content-Type", "application/json")
424 | w.Header().Set("Content-Length", fmt.Sprint(len(emptyJSON)))
425 |
426 | fmt.Fprint(w, emptyJSON)
427 | }
428 |
429 | func emptyTagsList(w http.ResponseWriter, r *http.Request) {
430 | const emptyTagsList = `{"name":"disable-list-tags","tags":[]}`
431 |
432 | w.Header().Set("Content-Type", "application/json")
433 | w.Header().Set("Content-Length", fmt.Sprint(len(emptyTagsList)))
434 | fmt.Fprint(w, emptyTagsList)
435 | }
436 |
437 | func (c *CRProxy) do(cli *http.Client, r *http.Request) (resp *http.Response, err error) {
438 | forHead := !c.allowHeadMethod && r.Method == http.MethodHead
439 | if forHead {
440 | r.Method = http.MethodGet
441 | }
442 | resp, err = cli.Do(r)
443 | if err != nil {
444 | return nil, err
445 | }
446 |
447 | if forHead {
448 | r.Method = http.MethodHead
449 | if resp.Body != nil {
450 | resp.Body.Close()
451 | }
452 | resp.Body = http.NoBody
453 | }
454 | return resp, err
455 | }
456 |
457 | func (c *CRProxy) doWithAuth(cli *http.Client, r *http.Request, host string) (*http.Response, error) {
458 | resp, err := c.do(cli, r)
459 | if err != nil {
460 | return nil, err
461 | }
462 |
463 | if resp.StatusCode == http.StatusUnauthorized {
464 | err = c.ping(host)
465 | if err != nil {
466 | if c.logger != nil {
467 | c.logger.Println("failed to ping", host, err)
468 | }
469 | return resp, nil
470 | }
471 |
472 | resp0, err0 := c.do(cli, r)
473 | if err0 != nil {
474 | if c.logger != nil {
475 | c.logger.Println("failed to redo", host, err)
476 | }
477 | return resp, nil
478 | }
479 | resp.Body.Close()
480 | resp = resp0
481 | }
482 | return resp, nil
483 | }
484 |
485 | func getIP(str string) string {
486 | host, _, err := net.SplitHostPort(str)
487 | if err == nil && host != "" {
488 | return host
489 | }
490 | return str
491 | }
492 |
493 | func (c *CRProxy) block(info *BlockInfo) (string, bool) {
494 | for _, blockFunc := range c.blockFunc {
495 | blockMessage, block := blockFunc(info)
496 | if block {
497 | return blockMessage, true
498 | }
499 | }
500 | return "", false
501 | }
502 |
503 | func (c *CRProxy) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
504 | if r.Method != http.MethodGet && r.Method != http.MethodHead {
505 | errcode.ServeJSON(rw, errcode.ErrorCodeUnsupported)
506 | return
507 | }
508 |
509 | r.RemoteAddr = getIP(r.RemoteAddr)
510 |
511 | if c.simpleAuth && !c.authorization(rw, r) {
512 | c.authenticate(rw, r)
513 | return
514 | }
515 |
516 | oriPath := r.URL.Path
517 | if oriPath == prefix {
518 | apiBase(rw, r)
519 | return
520 | }
521 | if !strings.HasPrefix(oriPath, prefix) {
522 | c.notFoundResponse(rw, r)
523 | return
524 | }
525 | if oriPath == catalog {
526 | errcode.ServeJSON(rw, errcode.ErrorCodeUnsupported)
527 | return
528 | }
529 |
530 | defaultRegistry := c.defaultRegistry
531 | if c.overrideDefaultRegistry != nil {
532 | r, ok := c.overrideDefaultRegistry[r.Host]
533 | if ok {
534 | defaultRegistry = r
535 | }
536 | }
537 | info, ok := ParseOriginPathInfo(oriPath, defaultRegistry)
538 | if !ok {
539 | errcode.ServeJSON(rw, errcode.ErrorCodeDenied)
540 | return
541 | }
542 |
543 | if c.modify != nil {
544 | n := c.modify(&ImageInfo{
545 | Host: info.Host,
546 | Name: info.Image,
547 | })
548 | info.Host = n.Host
549 | info.Image = n.Name
550 | }
551 |
552 | imageInfo := &ImageInfo{
553 | Host: info.Host,
554 | Name: info.Image,
555 | }
556 |
557 | if c.blockFunc != nil && !c.isPrivileged(r, nil) {
558 |
559 | blockMessage, block := c.block(&BlockInfo{
560 | IP: r.RemoteAddr,
561 | Host: info.Host,
562 | Name: info.Image,
563 | })
564 | if block {
565 | if blockMessage != "" {
566 | errcode.ServeJSON(rw, errcode.ErrorCodeDenied.WithMessage(blockMessage))
567 | } else {
568 | errcode.ServeJSON(rw, errcode.ErrorCodeDenied)
569 | }
570 | return
571 | }
572 | }
573 |
574 | info.Host = c.getDomainAlias(info.Host)
575 |
576 | if info.TagsList && !c.isPrivileged(r, nil) && c.disableTagsList {
577 | emptyTagsList(rw, r)
578 | return
579 | }
580 |
581 | path, err := info.Path()
582 | if err != nil {
583 | if c.logger != nil {
584 | c.logger.Println("failed to get path", err)
585 | }
586 | errcode.ServeJSON(rw, errcode.ErrorCodeUnknown)
587 | return
588 | }
589 | r.RequestURI = ""
590 | r.Host = info.Host
591 | r.URL.Host = info.Host
592 | r.URL.Scheme = c.getScheme(info.Host)
593 | r.URL.Path = path
594 | r.URL.RawQuery = ""
595 | r.URL.ForceQuery = false
596 | r.Body = http.NoBody
597 | if info.Blobs != "" && c.isRedirectToOriginBlob(r, imageInfo) {
598 | c.redirectBlobResponse(rw, r, info)
599 | return
600 | }
601 |
602 | if !c.isPrivileged(r, imageInfo) {
603 | if !c.checkLimit(rw, r, info) {
604 | return
605 | }
606 | }
607 |
608 | if c.storageDriver != nil {
609 | if info.Blobs != "" {
610 | c.cacheBlobResponse(rw, r, info)
611 | return
612 | } else if info.Manifests != "" {
613 | c.cacheManifestResponse(rw, r, info)
614 | return
615 | }
616 | }
617 | c.directResponse(rw, r, info)
618 | }
619 |
620 | func (c *CRProxy) directResponse(rw http.ResponseWriter, r *http.Request, info *PathInfo) {
621 | cli := c.getClientset(info.Host, info.Image)
622 | resp, err := c.doWithAuth(cli, r, info.Host)
623 | if err != nil {
624 | if c.logger != nil {
625 | c.logger.Println("failed to request", info.Host, info.Image, err)
626 | }
627 | errcode.ServeJSON(rw, errcode.ErrorCodeUnknown)
628 | return
629 | }
630 | defer func() {
631 | resp.Body.Close()
632 | }()
633 |
634 | switch resp.StatusCode {
635 | case http.StatusUnauthorized, http.StatusForbidden:
636 | if c.logger != nil {
637 | c.logger.Println("origin direct response 40x, but hit caches", info.Host, info.Image, err, dumpResponse(resp))
638 | }
639 | errcode.ServeJSON(rw, errcode.ErrorCodeDenied)
640 | return
641 | }
642 |
643 | resp.Header.Del("Docker-Ratelimit-Source")
644 |
645 | if resp.StatusCode == http.StatusOK {
646 | oldLink := resp.Header.Get("Link")
647 | if oldLink != "" {
648 | resp.Header.Set("Link", addPrefixToImageForPagination(oldLink, info.Host))
649 | }
650 | }
651 |
652 | header := rw.Header()
653 | for k, v := range resp.Header {
654 | key := textproto.CanonicalMIMEHeaderKey(k)
655 | header[key] = v
656 | }
657 | rw.WriteHeader(resp.StatusCode)
658 |
659 | if r.Method != http.MethodHead {
660 | buf := c.bytesPool.Get().([]byte)
661 | defer c.bytesPool.Put(buf)
662 | var body io.Reader = resp.Body
663 |
664 | if !c.isPrivileged(r, &ImageInfo{
665 | Host: info.Host,
666 | Name: info.Image,
667 | }) {
668 | c.accumulativeLimit(r, info, resp.ContentLength)
669 |
670 | if c.totalBlobsSpeedLimit != nil && info.Blobs != "" {
671 | body = c.totalBlobsSpeedLimit.Reader(body)
672 | }
673 |
674 | if c.blobsSpeedLimit != nil && info.Blobs != "" {
675 | body = geario.NewGear(c.blobsSpeedLimitDuration, *c.blobsSpeedLimit).Reader(body)
676 | }
677 | }
678 |
679 | io.CopyBuffer(rw, body, buf)
680 | }
681 | }
682 |
683 | func (c *CRProxy) errorResponse(rw http.ResponseWriter, r *http.Request, err error) {
684 | if err != nil {
685 | e := err.Error()
686 | if c.logger != nil {
687 | c.logger.Println("error response", r.RemoteAddr, e)
688 | }
689 | }
690 |
691 | if err == nil {
692 | err = errcode.ErrorCodeUnknown
693 | }
694 |
695 | errcode.ServeJSON(rw, err)
696 | }
697 |
698 | func (c *CRProxy) notFoundResponse(rw http.ResponseWriter, r *http.Request) {
699 | http.NotFound(rw, r)
700 | }
701 |
702 | func (c *CRProxy) redirect(rw http.ResponseWriter, r *http.Request, blobPath string) error {
703 | options := map[string]interface{}{
704 | "method": r.Method,
705 | }
706 | linkExpires := c.linkExpires
707 | if linkExpires > 0 {
708 | options["expiry"] = time.Now().Add(linkExpires)
709 | }
710 | u, err := c.storageDriver.URLFor(r.Context(), blobPath, options)
711 | if err != nil {
712 | return err
713 | }
714 | if c.logger != nil {
715 | c.logger.Println("Cache hit", blobPath, u)
716 | }
717 | if c.redirectLinks != nil {
718 | uri, err := url.Parse(u)
719 | if err == nil {
720 | uri.Scheme = c.redirectLinks.Scheme
721 | uri.Host = c.redirectLinks.Host
722 | u = uri.String()
723 | }
724 | }
725 | http.Redirect(rw, r, u, http.StatusTemporaryRedirect)
726 | return nil
727 | }
728 |
729 | func (c *CRProxy) getDomainAlias(host string) string {
730 | if c.domainAlias == nil {
731 | return host
732 | }
733 | h, ok := c.domainAlias[host]
734 | if !ok {
735 | return host
736 | }
737 | return h
738 | }
739 |
--------------------------------------------------------------------------------
/crproxy_blob.go:
--------------------------------------------------------------------------------
1 | package crproxy
2 |
3 | import (
4 | "context"
5 | "crypto/sha256"
6 | "encoding/hex"
7 | "fmt"
8 | "io"
9 | "net/http"
10 | "path"
11 | "strconv"
12 | "strings"
13 |
14 | "github.com/docker/distribution/registry/api/errcode"
15 | )
16 |
17 | func blobCachePath(blob string) string {
18 | blob = strings.TrimPrefix(blob, "sha256:")
19 | return path.Join("/docker/registry/v2/blobs/sha256", blob[:2], blob, "data")
20 | }
21 |
22 | func (c *CRProxy) cacheBlobResponse(rw http.ResponseWriter, r *http.Request, info *PathInfo) {
23 | ctx := r.Context()
24 |
25 | blobPath := blobCachePath(info.Blobs)
26 |
27 | closeValue, loaded := c.mutCache.LoadOrStore(blobPath, make(chan struct{}))
28 | closeCh := closeValue.(chan struct{})
29 | for loaded {
30 | select {
31 | case <-ctx.Done():
32 | err := ctx.Err().Error()
33 | if c.logger != nil {
34 | c.logger.Println(err)
35 | }
36 | http.Error(rw, err, http.StatusInternalServerError)
37 | return
38 | case <-closeCh:
39 | }
40 | closeValue, loaded = c.mutCache.LoadOrStore(blobPath, make(chan struct{}))
41 | closeCh = closeValue.(chan struct{})
42 | }
43 |
44 | doneCache := func() {
45 | c.mutCache.Delete(blobPath)
46 | close(closeCh)
47 | }
48 |
49 | stat, err := c.storageDriver.Stat(ctx, blobPath)
50 | if err == nil {
51 | doneCache()
52 |
53 | size := stat.Size()
54 | if r.Method == http.MethodHead {
55 | rw.Header().Set("Content-Length", strconv.FormatInt(size, 10))
56 | rw.Header().Set("Content-Type", "application/octet-stream")
57 | return
58 | }
59 |
60 | if !c.isPrivileged(r, &ImageInfo{
61 | Host: info.Host,
62 | Name: info.Image,
63 | }) {
64 | c.accumulativeLimit(r, info, size)
65 | if !c.waitForLimit(r, info, size) {
66 | c.errorResponse(rw, r, nil)
67 | return
68 | }
69 | }
70 |
71 | err = c.redirect(rw, r, blobPath)
72 | if err == nil {
73 | return
74 | }
75 | c.errorResponse(rw, r, ctx.Err())
76 | return
77 | }
78 | if c.logger != nil {
79 | c.logger.Println("Cache miss", blobPath)
80 | }
81 |
82 | type repo struct {
83 | err error
84 | size int64
85 | }
86 | signalCh := make(chan repo, 1)
87 |
88 | go func() {
89 | defer doneCache()
90 | size, err := c.cacheBlobContent(context.Background(), r, blobPath, info)
91 | signalCh <- repo{
92 | err: err,
93 | size: size,
94 | }
95 | }()
96 |
97 | select {
98 | case <-ctx.Done():
99 | c.errorResponse(rw, r, ctx.Err())
100 | return
101 | case signal := <-signalCh:
102 | if signal.err != nil {
103 | c.errorResponse(rw, r, signal.err)
104 | return
105 | }
106 | if r.Method == http.MethodHead {
107 | rw.Header().Set("Content-Length", strconv.FormatInt(signal.size, 10))
108 | rw.Header().Set("Content-Type", "application/octet-stream")
109 | return
110 | }
111 |
112 | if !c.isPrivileged(r, &ImageInfo{
113 | Host: info.Host,
114 | Name: info.Image,
115 | }) {
116 | c.accumulativeLimit(r, info, signal.size)
117 | if !c.waitForLimit(r, info, signal.size) {
118 | c.errorResponse(rw, r, nil)
119 | return
120 | }
121 | }
122 |
123 | err = c.redirect(rw, r, blobPath)
124 | if err != nil {
125 | if c.logger != nil {
126 | c.logger.Println("failed to redirect", blobPath, err)
127 | }
128 | }
129 | return
130 | }
131 | }
132 |
133 | func (c *CRProxy) cacheBlobContent(ctx context.Context, r *http.Request, blobPath string, info *PathInfo) (int64, error) {
134 | cli := c.getClientset(info.Host, info.Image)
135 | resp, err := c.doWithAuth(cli, r.WithContext(ctx), info.Host)
136 | if err != nil {
137 | return 0, err
138 | }
139 | defer func() {
140 | resp.Body.Close()
141 | }()
142 |
143 | switch resp.StatusCode {
144 | case http.StatusUnauthorized, http.StatusForbidden:
145 | return 0, errcode.ErrorCodeDenied
146 | }
147 |
148 | if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
149 | return 0, errcode.ErrorCodeUnknown.WithMessage(fmt.Sprintf("Source response code %d", resp.StatusCode))
150 | }
151 |
152 | buf := c.bytesPool.Get().([]byte)
153 | defer c.bytesPool.Put(buf)
154 |
155 | fw, err := c.storageDriver.Writer(ctx, blobPath, false)
156 | if err != nil {
157 | return 0, err
158 | }
159 |
160 | h := sha256.New()
161 | n, err := io.CopyBuffer(fw, io.TeeReader(resp.Body, h), buf)
162 | if err != nil {
163 | fw.Cancel()
164 | return 0, err
165 | }
166 |
167 | if n != resp.ContentLength {
168 | fw.Cancel()
169 | return 0, fmt.Errorf("expected %d bytes, got %d", resp.ContentLength, n)
170 | }
171 |
172 | hash := hex.EncodeToString(h.Sum(nil)[:])
173 | if info.Blobs[7:] != hash {
174 | fw.Cancel()
175 | return 0, fmt.Errorf("expected %s hash, got %s", info.Blobs[7:], hash)
176 | }
177 |
178 | err = fw.Commit()
179 | if err != nil {
180 | return 0, err
181 | }
182 | return n, nil
183 | }
184 |
185 | func (c *CRProxy) redirectBlobResponse(rw http.ResponseWriter, r *http.Request, info *PathInfo) {
186 | r = r.WithContext(withCtxValue(r.Context()))
187 |
188 | cli := c.getClientset(info.Host, info.Image)
189 | resp, err := c.doWithAuth(cli, r, info.Host)
190 | if err != nil {
191 | if c.logger != nil {
192 | c.logger.Println("failed to request", info.Host, info.Image, err)
193 | }
194 | errcode.ServeJSON(rw, errcode.ErrorCodeUnknown)
195 | return
196 | }
197 | defer func() {
198 | resp.Body.Close()
199 | }()
200 |
201 | switch resp.StatusCode {
202 | default:
203 | if c.logger != nil {
204 | c.logger.Println("failed to redirect blob", info.Host, info.Image, resp.StatusCode)
205 | }
206 | errcode.ServeJSON(rw, errcode.ErrorCodeUnavailable)
207 | return
208 | case http.StatusUnauthorized, http.StatusForbidden:
209 | errcode.ServeJSON(rw, errcode.ErrorCodeDenied)
210 | return
211 | case http.StatusTemporaryRedirect, http.StatusPermanentRedirect, http.StatusMovedPermanently, http.StatusFound:
212 | location := resp.Header.Get("Location")
213 | http.Redirect(rw, r, location, http.StatusFound)
214 | return
215 | case http.StatusOK:
216 | v := GetCtxValue(r.Context())
217 | if v != nil && v.LastRedirect != "" {
218 | http.Redirect(rw, r, v.LastRedirect, http.StatusFound)
219 | return
220 | }
221 | errcode.ServeJSON(rw, errcode.ErrorCodeUnavailable)
222 | return
223 | }
224 | }
225 |
226 | func (c *CRProxy) isRedirectToOriginBlob(r *http.Request, info *ImageInfo) bool {
227 | if c.redirectToOriginBlobFunc == nil {
228 | return false
229 | }
230 |
231 | return c.redirectToOriginBlobFunc(r, info)
232 | }
233 |
--------------------------------------------------------------------------------
/crproxy_control.go:
--------------------------------------------------------------------------------
1 | package crproxy
2 |
3 | import (
4 | "net/http"
5 | "strconv"
6 | "time"
7 |
8 | "github.com/docker/distribution/registry/api/errcode"
9 | "github.com/wzshiming/geario"
10 | )
11 |
12 | func (c *CRProxy) isPrivileged(r *http.Request, info *ImageInfo) bool {
13 | if c.privilegedFunc == nil {
14 | return false
15 | }
16 | return c.privilegedFunc(r, info)
17 | }
18 |
19 | func (c *CRProxy) checkLimit(rw http.ResponseWriter, r *http.Request, info *PathInfo) bool {
20 | if c.ipsSpeedLimit != nil && info.Blobs != "" {
21 | bps, _ := c.speedLimitRecord.LoadOrStore(r.RemoteAddr, geario.NewBPSAver(c.ipsSpeedLimitDuration))
22 | aver := bps.Aver()
23 | if aver > *c.ipsSpeedLimit {
24 | if c.logger != nil {
25 | c.logger.Println("exceed limit", r.RemoteAddr, aver, *c.ipsSpeedLimit)
26 | }
27 | if c.limitDelay {
28 | for bps.Aver() > *c.ipsSpeedLimit {
29 | wait := time.Second
30 | n := bps.Next()
31 | if !n.IsZero() {
32 | wait = bps.Next().Sub(time.Now())
33 | if wait < time.Second {
34 | wait = time.Second
35 | }
36 | }
37 | select {
38 | case <-r.Context().Done():
39 | return false
40 | case <-time.After(wait):
41 | }
42 | }
43 | } else {
44 | err := errcode.ErrorCodeTooManyRequests
45 | rw.Header().Set("X-Retry-After", strconv.FormatInt(bps.Next().Unix(), 10))
46 | errcode.ServeJSON(rw, err)
47 | return false
48 | }
49 | }
50 | }
51 |
52 | return true
53 | }
54 |
55 | func (c *CRProxy) waitForLimit(r *http.Request, info *PathInfo, size int64) bool {
56 | if c.blobsSpeedLimit != nil && info.Blobs != "" {
57 | dur := GetSleepDuration(geario.B(size), *c.blobsSpeedLimit, c.blobsSpeedLimitDuration)
58 | if dur > 0 {
59 | if c.logger != nil {
60 | c.logger.Println("delay request", r.RemoteAddr, geario.B(size), dur)
61 | }
62 | select {
63 | case <-r.Context().Done():
64 | return false
65 | case <-time.After(dur):
66 | }
67 | }
68 | }
69 |
70 | return true
71 | }
72 |
73 | func (c *CRProxy) accumulativeLimit(r *http.Request, info *PathInfo, size int64) {
74 | if c.ipsSpeedLimit != nil && info.Blobs != "" {
75 | bps, ok := c.speedLimitRecord.Load(r.RemoteAddr)
76 | if ok {
77 | bps.Add(geario.B(size))
78 | }
79 | }
80 | }
81 |
--------------------------------------------------------------------------------
/crproxy_manifest.go:
--------------------------------------------------------------------------------
1 | package crproxy
2 |
3 | import (
4 | "context"
5 | "crypto/sha256"
6 | "encoding/hex"
7 | "encoding/json"
8 | "fmt"
9 | "io"
10 | "net/http"
11 | "net/textproto"
12 | "path"
13 | "strconv"
14 | "strings"
15 | "time"
16 |
17 | "github.com/docker/distribution/registry/api/errcode"
18 | )
19 |
20 | func manifestRevisionsCachePath(host, image, tagOrBlob string) string {
21 | return path.Join("/docker/registry/v2/repositories", host, image, "_manifests/revisions/sha256", tagOrBlob, "link")
22 | }
23 |
24 | func manifestTagCachePath(host, image, tagOrBlob string) string {
25 | return path.Join("/docker/registry/v2/repositories", host, image, "_manifests/tags", tagOrBlob, "current/link")
26 | }
27 |
28 | func (c *CRProxy) cacheManifestResponse(rw http.ResponseWriter, r *http.Request, info *PathInfo) {
29 | if c.cachedManifest(rw, r, info, true) {
30 | return
31 | }
32 |
33 | cli := c.getClientset(info.Host, info.Image)
34 | resp, err := c.doWithAuth(cli, r, info.Host)
35 | if err != nil {
36 | if c.cachedManifest(rw, r, info, false) {
37 | return
38 | }
39 | if c.logger != nil {
40 | c.logger.Println("failed to request", info.Host, info.Image, err)
41 | }
42 | errcode.ServeJSON(rw, errcode.ErrorCodeUnknown)
43 | return
44 | }
45 | defer func() {
46 | resp.Body.Close()
47 | }()
48 |
49 | switch resp.StatusCode {
50 | case http.StatusUnauthorized, http.StatusForbidden:
51 | if c.cachedManifest(rw, r, info, false) {
52 | if c.logger != nil {
53 | c.logger.Println("origin manifest response 40x, but hit caches", info.Host, info.Image, err, dumpResponse(resp))
54 | }
55 | return
56 | }
57 | if c.logger != nil {
58 | c.logger.Println("origin manifest response 40x", info.Host, info.Image, err, dumpResponse(resp))
59 | }
60 | errcode.ServeJSON(rw, errcode.ErrorCodeDenied)
61 | return
62 | }
63 |
64 | if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusBadRequest {
65 | if c.cachedManifest(rw, r, info, false) {
66 | if c.logger != nil {
67 | c.logger.Println("origin manifest response 5xx, but hit caches", info.Host, info.Image, err, dumpResponse(resp))
68 | }
69 | return
70 | }
71 | if c.logger != nil {
72 | c.logger.Println("origin manifest response 5xx", info.Host, info.Image, err, dumpResponse(resp))
73 | }
74 | }
75 |
76 | resp.Header.Del("Docker-Ratelimit-Source")
77 |
78 | header := rw.Header()
79 | for k, v := range resp.Header {
80 | key := textproto.CanonicalMIMEHeaderKey(k)
81 | header[key] = v
82 | }
83 |
84 | rw.WriteHeader(resp.StatusCode)
85 |
86 | if r.Method == http.MethodHead {
87 | return
88 | }
89 |
90 | if resp.StatusCode >= http.StatusOK || resp.StatusCode < http.StatusMultipleChoices {
91 | body, err := io.ReadAll(resp.Body)
92 | if err != nil {
93 | c.errorResponse(rw, r, err)
94 | return
95 | }
96 |
97 | err = c.cacheManifestContent(context.Background(), info, body)
98 | if err != nil {
99 | c.errorResponse(rw, r, err)
100 | return
101 | }
102 | rw.Write(body)
103 | } else {
104 | io.Copy(rw, resp.Body)
105 | }
106 | }
107 |
108 | func (c *CRProxy) cacheManifestContent(ctx context.Context, info *PathInfo, content []byte) error {
109 | h := sha256.New()
110 | h.Write(content)
111 | hash := hex.EncodeToString(h.Sum(nil)[:])
112 |
113 | if strings.HasPrefix(info.Manifests, "sha256:") {
114 | if info.Manifests[7:] != hash {
115 | return fmt.Errorf("expected hash %s is not same to %s", info.Manifests[7:], hash)
116 | }
117 | } else {
118 | manifestLinkPath := manifestTagCachePath(info.Host, info.Image, info.Manifests)
119 | err := c.storageDriver.PutContent(ctx, manifestLinkPath, []byte("sha256:"+hash))
120 | if err != nil {
121 | return err
122 | }
123 | }
124 |
125 | manifestLinkPath := manifestRevisionsCachePath(info.Host, info.Image, hash)
126 | err := c.storageDriver.PutContent(ctx, manifestLinkPath, []byte("sha256:"+hash))
127 | if err != nil {
128 | return err
129 | }
130 |
131 | blobCachePath := blobCachePath(hash)
132 | err = c.storageDriver.PutContent(ctx, blobCachePath, content)
133 | if err != nil {
134 | return err
135 | }
136 |
137 | if c.manifestCacheDuration > 0 {
138 | c.manifestCache.Store(manifestLinkPath, time.Now())
139 | }
140 | return nil
141 | }
142 |
143 | func (c *CRProxy) cachedManifest(rw http.ResponseWriter, r *http.Request, info *PathInfo, try bool) bool {
144 | if try && c.manifestCacheDuration == 0 {
145 | return false
146 | }
147 |
148 | ctx := r.Context()
149 | var manifestLinkPath string
150 | if strings.HasPrefix(info.Manifests, "sha256:") {
151 | manifestLinkPath = manifestRevisionsCachePath(info.Host, info.Image, info.Manifests[7:])
152 | } else {
153 | manifestLinkPath = manifestTagCachePath(info.Host, info.Image, info.Manifests)
154 | }
155 |
156 | if try {
157 | last, ok := c.manifestCache.Load(manifestLinkPath)
158 | if !ok {
159 | return false
160 | }
161 |
162 | if time.Since(last) > c.manifestCacheDuration {
163 | return false
164 | }
165 | }
166 |
167 | content, err := c.storageDriver.GetContent(ctx, manifestLinkPath)
168 | if err == nil {
169 | digest := string(content)
170 | blobCachePath := blobCachePath(digest)
171 | content, err := c.storageDriver.GetContent(ctx, blobCachePath)
172 | if err == nil {
173 |
174 | mt := struct {
175 | MediaType string `json:"mediaType"`
176 | }{}
177 | err := json.Unmarshal(content, &mt)
178 | if err != nil {
179 | if c.logger != nil {
180 | c.logger.Println("Manifest blob cache err", blobCachePath, err)
181 | }
182 | return false
183 | }
184 | if c.logger != nil {
185 | c.logger.Println("Manifest blob cache hit", blobCachePath)
186 | }
187 | rw.Header().Set("docker-content-digest", digest)
188 | rw.Header().Set("Content-Type", mt.MediaType)
189 | rw.Header().Set("Content-Length", strconv.FormatInt(int64(len(content)), 10))
190 | if r.Method != http.MethodHead {
191 | rw.Write(content)
192 | }
193 | return true
194 | }
195 | if c.logger != nil {
196 | c.logger.Println("Manifest blob cache missed", blobCachePath, err)
197 | }
198 | } else {
199 | if c.logger != nil {
200 | c.logger.Println("Manifest cache missed", manifestLinkPath, err)
201 | }
202 | }
203 |
204 | return false
205 | }
206 |
--------------------------------------------------------------------------------
/ctx_value.go:
--------------------------------------------------------------------------------
1 | package crproxy
2 |
3 | import (
4 | "context"
5 | )
6 |
7 | type infoCtxKey struct{}
8 | type InfoCtxValue struct {
9 | LastRedirect string
10 | }
11 |
12 | func withCtxValue(ctx context.Context) context.Context {
13 | return context.WithValue(ctx, infoCtxKey{}, &InfoCtxValue{})
14 | }
15 |
16 | func GetCtxValue(ctx context.Context) *InfoCtxValue {
17 | v, ok := ctx.Value(infoCtxKey{}).(*InfoCtxValue)
18 | if !ok {
19 | return nil
20 | }
21 | return v
22 | }
23 |
--------------------------------------------------------------------------------
/examples/default/.gitignore:
--------------------------------------------------------------------------------
1 |
2 | log
3 | certbot
4 | nginx
5 |
--------------------------------------------------------------------------------
/examples/default/README.md:
--------------------------------------------------------------------------------
1 | # 📢 部署须知
2 |
3 | 当前的部署形式其实只适用于小量使用的场景, 用的人多会很卡啊, 适用于大量使用的场景的代码是已经在项目里了还没文档描述（作者精力有限，欢迎大佬提交pr）
4 |
5 | 另外本文中显示的仓库地址：`kubesre.xyz`，只做本文演示使用，不保证其稳定和有效性。
6 |
7 | ## 开始部署
8 |
9 | ### 前提
10 |
11 | - 准备一台访问网络畅通的服务器
12 |
13 | > 推荐腾讯云轻量服务器，区域地域选择亚洲离中国近的地方，最近618特惠，一个月的才26元 [点击直达](https://curl.qcloud.com/RW4e7hIf)
14 |
15 | - 准备一个域名（无需备案）并做好 DNS 解析：添加两条解析记录 `@` 记录 和 `*` 记录到准备好的服务器的 IP
16 |
17 | > 域名推荐选择`xyz`结尾了，首年最低7元.
18 | >
19 | > 如果你想使用二级域名,以`cr.kubesre.xyz`举例，你可以将`cr`和`*.cr`解析到服务器ip.
20 |
21 | - 安装好 docker 和 docker-compose 参考：[菜鸟教程](https://www.runoob.com/docker/centos-docker-install.html)
22 |
23 | ### 拉取代码
24 |
25 | ```
26 | git clone https://github.com/wzshiming/crproxy.git
27 | ```
28 |
29 | ### 进入项目目录
30 |
31 | ```
32 | cd crproxy/examples/default
33 | ```
34 |
35 | ### 修改gateway域名
36 |
37 | vim start.sh 第五行
38 |
39 | ```
40 | 原：gateway=m.daocloud.io
41 | 修改为：gateway=kubesre.xyz #改成你自己的域名
42 | ```
43 |
44 | ### 启动服务
45 |
46 | ```
47 | ./start.sh
48 | ```
49 |
50 | > 如果出现了报错大概率是申请ssl证书时,ca机构检查到没有将域名解析到当前服务器导致的.
51 | >
52 | > 如果刚刚添加了域名解析,等解析生效后重新执行`./start.sh`即可
53 |
54 | 如果一切正常这时候你就可以通过添加前缀的方式拉取镜像了.
55 |
56 | 假如你的域名是：`kubesre.xyz`
57 |
58 | **使用增加前缀拉取镜像，比如：**
59 |
60 | **映射关系如下**
61 |
62 | ```
63 | k8s.gcr.io/coredns/coredns:v1.8.6 => kubesre.xyz/k8s.gcr.io/coredns/coredns:v1.8.6
64 | ```
65 |
66 | **拉取镜像**
67 |
68 | ```
69 | docker pull kubesre.xyz/k8s.gcr.io/coredns/coredns:v1.8.6
70 | ```
71 |
72 | 📢 注意：**如果你想使用前缀替换的方式拉取镜像 (务必域名做好 `*` 解析到服务器)**
73 | **映射关系如下**
74 |
75 | ```
76 | k8s.gcr.io/coredns/coredns:v1.8.6 => k8s-gcr.kubesre.xyz/coredns/coredns:v1.8.6
77 | ```
78 |
79 | 那你就需要执行 `setup-alias.sh` 脚本添加 `k8s-gcr` 作为 `k8s.gcr.io` 别名
80 |
81 | ### 添加别名
82 |
83 | **设置环境变量**
84 |
85 | ```
86 | GETEWAY=kubesre.xyz ##替换成自己的域名
87 | ```
88 |
89 | > 第一个参数前缀替换的域名
90 | > 第二个参数是源站的域名
91 | > 第三个参数是在`start.sh`脚本里配置的网关域名
92 |
93 | ```
94 | ./setup-alias.sh k8s-gcr.${GETEWAY} k8s.gcr.io ${GETEWAY}
95 | ```
96 |
97 | ### 为别名申请证书
98 |
99 | ```
100 | update-tls.sh k8s-gcr.${GETEWAY}
101 | ```
102 |
103 | **重启一下nginx服务**
104 |
105 | ```
106 | ./reload.sh
107 | ```
108 |
109 | 不出意外这时候你就可以使用前缀替换方式拉取镜像了
110 |
111 | **感受一下愉快的拉取镜像吧**
112 |
113 | ```
114 | docker pull k8s-gcr.kubesre.xyz/coredns/coredns:v1.8.6
115 | ```
116 |
117 | ## 扩展
118 |
119 | ### 常用的镜像仓库
120 |
121 | 常用的镜像仓库一般有这些:
122 |
123 | | 源站 | 别名 |
124 | | ----------------------- | ---------------------- |
125 | | cr.l5d.io | l5d.kubesre.xyz |
126 | | docker.elastic.co | elastic.kubesre.xyz |
127 | | docker.io | docker.kubesre.xyz |
128 | | gcr.io | gcr.kubesre.xyz |
129 | | ghcr.io | ghcr.kubesre.xyz |
130 | | k8s.gcr.io | k8s-gcr.kubesre.xyz |
131 | | registry.k8s.io | k8s.kubesre.xyz |
132 | | mcr.microsoft.com | mcr.kubesre.xyz |
133 | | nvcr.io | nvcr.kubesre.xyz |
134 | | quay.io | quay.kubesre.xyz |
135 | | registry.jujucharms.com | jujucharms.kubesre.xyz |
136 |
137 | ### 添加常用镜像仓库别名
138 |
139 | **设置环境变量**
140 |
141 | ```bash
142 | GETEWAY=kubesre.xyz ##替换成自己的域名
143 | ```
144 |
145 | **添加别名**
146 |
147 | ```bash
148 | ./setup-alias.sh l5d.${GETEWAY} cr.l5d.io ${GETEWAY}
149 | ./setup-alias.sh elastic.${GETEWAY} docker.elastic.co ${GETEWAY}
150 | ./setup-alias.sh docker.${GETEWAY} docker.io ${GETEWAY}
151 | ./setup-alias.sh gcr.${GETEWAY} gcr.io ${GETEWAY}
152 | ./setup-alias.sh ghcr.${GETEWAY} ghcr.io ${GETEWAY}
153 | ./setup-alias.sh k8s-gcr.${GETEWAY} k8s.gcr.io ${GETEWAY}
154 | ./setup-alias.sh k8s.${GETEWAY} registry.k8s.io ${GETEWAY}
155 | ./setup-alias.sh mcr.${GETEWAY} mcr.microsoft.com ${GETEWAY}
156 | ./setup-alias.sh nvcr.${GETEWAY} nvcr.io ${GETEWAY}
157 | ./setup-alias.sh quay.${GETEWAY} quay.io ${GETEWAY}
158 | ./setup-alias.sh jujucharms.${GETEWAY} registry.jujucharms.com ${GETEWAY}
159 | ./setup-alias.sh rocks-canonical.${GETEWAY} rocks.canonical.com ${GETEWAY}
160 | ```
161 |
162 | **给别名申请证书**
163 |
164 | ```bash
165 | ./update-tls.sh gcr.${GETEWAY}
166 | ./update-tls.sh ghcr.${GETEWAY}
167 | ./update-tls.sh k8s-gcr.${GETEWAY}
168 | ./update-tls.sh k8s.${GETEWAY}
169 | ./update-tls.sh k8s.${GETEWAY}
170 | ./update-tls.sh mcr.${GETEWAY}
171 | ./update-tls.sh nvcr.${GETEWAY}
172 | ./update-tls.sh quay.${GETEWAY}
173 | ./update-tls.sh jujucharms.${GETEWAY}
174 | ./update-tls.sh rocks-canonical.${GETEWAY}
175 | ```
176 |
177 | 最后重启下就可以了
178 |
179 | ```
180 | ./reload.sh
181 | ```
182 | ## 采用者列表
183 | - kubesre.xyz [docker-registry-mirrors](https://github.com/kubesre/docker-registry-mirrors)
184 | - m.daocloud.io [public-image-mirror](https://github.com/DaoCloud/public-image-mirror)
185 |
--------------------------------------------------------------------------------
/examples/default/docker-compose.yml:
--------------------------------------------------------------------------------
1 | version: '3.4'
2 |
3 | services:
4 | gateway:
5 | image: ghcr.io/wzshiming/nginx-certbot:v1.21.4
6 | container_name: gateway
7 | restart: unless-stopped
8 | volumes:
9 | - ./nginx/:/etc/nginx/conf.d/
10 | - ./certbot/conf/:/etc/letsencrypt/
11 | - ./certbot/www/:/var/www/certbot/
12 | - ./html:/usr/share/nginx/html
13 | - ./log/nginx/:/var/log/nginx/
14 | - ./log/letsencrypt:/var/log/letsencrypt/
15 | ports:
16 | - "80:80"
17 | - "443:443"
18 |
19 | # registry:
20 | # image: docker.io/library/registry:2.8.1
21 | # container_name: registry
22 | # restart: unless-stopped
23 | # command:
24 | # - registry
25 | # - serve
26 | # - /etc/docker/registry/config.yml
27 | # volumes:
28 | # - ./registry:/var/lib/registry:rw
29 | # - ./registry/config.yml:/etc/docker/registry/config.yml:ro
30 |
31 | crproxy:
32 | image: ghcr.io/daocloud/crproxy/crproxy:v0.9.1
33 | container_name: crproxy
34 | restart: unless-stopped
35 | # command: |
36 | # # add docker.io username increase pulls from 100 to 200 per 6 hour period
37 | # -u username:password@docker.io
38 | # # support ignoring prefixes, e.g. docker pull mysql
39 | # --default-registry docker.io
40 | # # cache storage example: aliyunoss
41 | # --storage-driver oss
42 | # --storage-parameters accesskeyid=xxxxx,accesskeysecret=xxxxxxx,region=oss-ap-xxxx-1,encrypt=true,bucket=xxxx
43 | # # user
44 | # --simple-auth
45 | # --simple-auth-user user1=pass1
46 | # --simple-auth-user user2=pass2
47 |
48 | ## For crproxy via a proxy
49 |
50 | # environment:
51 | # - https_proxy=http://proxy:8080
52 | # - http_proxy=http://proxy:8080
53 |
54 | # proxy:
55 | # image: ghcr.io/wzshiming/bridge/bridge:v0.8.6
56 | # container_name: proxy
57 | # restart: unless-stopped
58 | # ports:
59 | # - 8080:8080
60 | # command: -b :8080 -p -
61 |
--------------------------------------------------------------------------------
/examples/default/html/index.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | Container Registry Proxy
6 |
7 |
8 |
9 |
17 |
18 |
19 |
--------------------------------------------------------------------------------
/examples/default/nginx/default.conf:
--------------------------------------------------------------------------------
1 | server {
2 | listen 80;
3 | server_name localhost;
4 |
5 | #access_log /var/log/nginx/host.access.log main;
6 |
7 | location / {
8 | root /usr/share/nginx/html;
9 | index index.html index.htm;
10 | }
11 |
12 | #error_page 404 /404.html;
13 |
14 | # redirect server error pages to the static page /50x.html
15 | #
16 | error_page 500 502 503 504 /50x.html;
17 | location = /50x.html {
18 | root /usr/share/nginx/html;
19 | }
20 |
21 | # proxy the PHP scripts to Apache listening on 127.0.0.1:80
22 | #
23 | #location ~ \.php$ {
24 | # proxy_pass http://127.0.0.1;
25 | #}
26 |
27 | # pass the PHP scripts to FastCGI server listening on 127.0.0.1:9000
28 | #
29 | #location ~ \.php$ {
30 | # root html;
31 | # fastcgi_pass 127.0.0.1:9000;
32 | # fastcgi_index index.php;
33 | # fastcgi_param SCRIPT_FILENAME /scripts$fastcgi_script_name;
34 | # include fastcgi_params;
35 | #}
36 |
37 | # deny access to .htaccess files, if Apache's document root
38 | # concurs with nginx's one
39 | #
40 | #location ~ /\.ht {
41 | # deny all;
42 | #}
43 | }
--------------------------------------------------------------------------------
/examples/default/registry/config.yml:
--------------------------------------------------------------------------------
1 | version: 0.1
2 | log:
3 | accesslog:
4 | disabled: true
5 | storage:
6 | cache:
7 | blobdescriptor: inmemory
8 | filesystem:
9 | rootdirectory: "/var/lib/registry"
10 | maintenance:
11 | uploadpurging:
12 | enabled: false
13 | readonly:
14 | enabled: true
15 | http:
16 | addr: :5000
17 | headers:
18 | X-Content-Type-Options: [nosniff]
19 |
20 | health:
21 | storagedriver:
22 | enabled: false
23 |
24 | validation:
25 | disabled: true
26 |
27 | proxy:
28 | remoteurl: http://crproxy:8080
29 |
--------------------------------------------------------------------------------
/examples/default/reload.sh:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env bash
2 |
3 | docker-compose exec gateway nginx -s reload
4 |
--------------------------------------------------------------------------------
/examples/default/setup-alias.sh:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env bash
2 |
3 | domain=${1:-}
4 |
5 | if [[ -z "${domain}" ]]; then
6 | echo "domain is required"
7 | exit 1
8 | fi
9 |
10 | origin=${2:-}
11 |
12 | if [[ -z "${origin}" ]]; then
13 | echo "origin is required"
14 | exit 1
15 | fi
16 |
17 | gateway=${3:-}
18 |
19 | if [[ -z "${gateway}" ]]; then
20 | echo "gateway is required"
21 | exit 1
22 | fi
23 |
24 | function gen() {
25 | local domain=$1
26 | local origin=$2
27 | local gateway=$3
28 | cat <"${conf}"
53 | fi
54 |
--------------------------------------------------------------------------------
/examples/default/setup-gateway.sh:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env bash
2 |
3 | domain=${1:-}
4 |
5 | if [[ -z "${domain}" ]]; then
6 | echo "domain is required"
7 | exit 1
8 | fi
9 |
10 | endpoint=${2:-}
11 |
12 | if [[ -z "${endpoint}" ]]; then
13 | echo "endpoint is required"
14 | exit 1
15 | fi
16 |
17 | function gen() {
18 | local domain=$1
19 | local endpoint=$2
20 | cat <"nginx/gateway-${domain}.conf"
79 | fi
80 |
--------------------------------------------------------------------------------
/examples/default/start.sh:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env bash
2 |
3 | docker-compose up -d
4 |
5 | gateway=m.daocloud.io
6 |
7 | declare -A mapping=()
8 |
9 | #./setup-gateway.sh "${gateway}" "registry:5000"
10 | ./setup-gateway.sh "${gateway}" "crproxy:8080"
11 | ./update-tls.sh "${gateway}"
12 |
13 | for key in ${!mapping[*]}; do
14 | ./setup-alias.sh "${key}" "${mapping[$key]}" "${gateway}"
15 | ./update-tls.sh "${key}"
16 | done
17 |
--------------------------------------------------------------------------------
/examples/default/update-tls.sh:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env bash
2 |
3 | domain=${1:-}
4 |
5 | if [[ -z "${domain}" ]]; then
6 | echo "domain is required"
7 | exit 1
8 | fi
9 |
10 | function cert_renew() {
11 | local domain=$1
12 | docker-compose exec gateway certbot --nginx -n --rsa-key-size 4096 --agree-tos --register-unsafely-without-email --domains "${domain}"
13 | }
14 |
15 | cert_renew "${domain}"
16 |
--------------------------------------------------------------------------------
/examples/simple/README.md:
--------------------------------------------------------------------------------
1 | # 📢 部署须知
2 |
3 | 当前的部署形式其实只适用于个人小量使用的场景
4 |
5 | ## 快速开始
6 |
7 | ### 前提
8 | - 准备一台服务器, 需要确保 80 和 443 端口打开
9 | - 准备一个域名并做好 DNS 解析到准备好的服务器的 IP
10 | - 安装好 docker 和 docker-compose 参考：[菜鸟教程](https://www.runoob.com/docker/centos-docker-install.html)
11 |
12 | ### 启动
13 |
14 | 在服务器里新建一个文件 `docker-compose.yaml` 内容如下
15 |
16 | ``` yaml
17 | version: '3'
18 | services:
19 | crproxy:
20 | image: ghcr.io/daocloud/crproxy/crproxy:v0.9.1
21 | container_name: crproxy
22 | restart: unless-stopped
23 | ports:
24 | - 80:8080
25 | - 443:8080
26 | command: |
27 | --acme-cache-dir=/tmp/acme
28 | --acme-hosts=*
29 | --default-registry=docker.io
30 | tmpfs:
31 | - /tmp/acme
32 |
33 | # 非必须, 如果这台服务器无法畅通的达到你要的镜像仓库可以尝试配置
34 | environment:
35 | - https_proxy=http://proxy:8080
36 | - http_proxy=http://proxy:8080
37 | ```
38 |
39 | 然后执行 `docker-compose up -d`
40 |
41 |
42 | ## 然后就能愉快的拉取镜像了
43 |
44 | ``` shell
45 | docker pull 你的域名/hello-world
46 | ```
47 |
48 | 也可以添加到 /etc/docker/daemon.json
49 |
50 | ``` json
51 | {
52 | "registry-mirrors": [
53 | "https://你的域名"
54 | ]
55 | }
56 | ```
57 |
58 | ``` shell
59 | docker pull hello-world
60 | ```
61 |
--------------------------------------------------------------------------------
/go.mod:
--------------------------------------------------------------------------------
1 | module github.com/daocloud/crproxy
2 |
3 | go 1.22
4 |
5 | require (
6 | github.com/denverdino/aliyungo v0.0.0-20230411124812-ab98a9173ace
7 | github.com/distribution/reference v0.6.0
8 | github.com/docker/distribution v0.0.0
9 | github.com/gorilla/handlers v1.5.2
10 | github.com/huaweicloud/huaweicloud-sdk-go-obs v3.24.6+incompatible
11 | github.com/opencontainers/go-digest v1.0.0
12 | github.com/spf13/pflag v1.0.5
13 | github.com/wzshiming/cmux v0.4.2
14 | github.com/wzshiming/geario v0.0.0-20240308093553-a996e3817533
15 | github.com/wzshiming/hostmatcher v0.0.3
16 | github.com/wzshiming/httpseek v0.1.0
17 | github.com/wzshiming/lru v0.1.0
18 | golang.org/x/crypto v0.28.0
19 | )
20 |
21 | replace github.com/docker/distribution => github.com/distribution/distribution v2.8.3+incompatible
22 |
23 | require (
24 | cloud.google.com/go/compute/metadata v0.3.0 // indirect
25 | github.com/Azure/azure-sdk-for-go v56.3.0+incompatible // indirect
26 | github.com/Azure/go-autorest v14.2.0+incompatible // indirect
27 | github.com/Azure/go-autorest/autorest v0.11.24 // indirect
28 | github.com/Azure/go-autorest/autorest/adal v0.9.18 // indirect
29 | github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
30 | github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect
31 | github.com/Azure/go-autorest/logger v0.2.1 // indirect
32 | github.com/Azure/go-autorest/tracing v0.6.0 // indirect
33 | github.com/aws/aws-sdk-go v1.48.10 // indirect
34 | github.com/beorn7/perks v1.0.1 // indirect
35 | github.com/cespare/xxhash/v2 v2.2.0 // indirect
36 | github.com/dnaeon/go-vcr v1.2.0 // indirect
37 | github.com/docker/go-metrics v0.0.1 // indirect
38 | github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1 // indirect
39 | github.com/felixge/httpsnoop v1.0.4 // indirect
40 | github.com/gofrs/uuid v4.0.0+incompatible // indirect
41 | github.com/golang-jwt/jwt/v4 v4.5.1 // indirect
42 | github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
43 | github.com/golang/protobuf v1.5.3 // indirect
44 | github.com/google/s2a-go v0.1.4 // indirect
45 | github.com/google/uuid v1.3.1 // indirect
46 | github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect
47 | github.com/googleapis/gax-go/v2 v2.11.0 // indirect
48 | github.com/gorilla/mux v1.8.1 // indirect
49 | github.com/jmespath/go-jmespath v0.4.0 // indirect
50 | github.com/kr/text v0.2.0 // indirect
51 | github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
52 | github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect
53 | github.com/opentracing/opentracing-go v1.2.0 // indirect
54 | github.com/prometheus/client_golang v1.17.0 // indirect
55 | github.com/prometheus/client_model v0.5.0 // indirect
56 | github.com/prometheus/common v0.44.0 // indirect
57 | github.com/prometheus/procfs v0.11.1 // indirect
58 | github.com/sirupsen/logrus v1.9.3 // indirect
59 | github.com/stretchr/testify v1.8.4 // indirect
60 | github.com/wzshiming/trie v0.3.1 // indirect
61 | go.opencensus.io v0.24.0 // indirect
62 | golang.org/x/net v0.30.0 // indirect
63 | golang.org/x/oauth2 v0.23.0 // indirect
64 | golang.org/x/sys v0.26.0 // indirect
65 | golang.org/x/text v0.19.0 // indirect
66 | google.golang.org/api v0.126.0 // indirect
67 | google.golang.org/appengine v1.6.7 // indirect
68 | google.golang.org/cloud v0.0.0-20151119220103-975617b05ea8 // indirect
69 | google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect
70 | google.golang.org/grpc v1.59.0 // indirect
71 | google.golang.org/protobuf v1.33.0 // indirect
72 | )
73 |
--------------------------------------------------------------------------------
/go.sum:
--------------------------------------------------------------------------------
1 | cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
2 | cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
3 | cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc=
4 | cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k=
5 | github.com/Azure/azure-sdk-for-go v56.3.0+incompatible h1:DmhwMrUIvpeoTDiWRDtNHqelNUd3Og8JCkrLHQK795c=
6 | github.com/Azure/azure-sdk-for-go v56.3.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
7 | github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs=
8 | github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
9 | github.com/Azure/go-autorest/autorest v0.11.24 h1:1fIGgHKqVm54KIPT+q8Zmd1QlVsmHqeUGso5qm2BqqE=
10 | github.com/Azure/go-autorest/autorest v0.11.24/go.mod h1:G6kyRlFnTuSbEYkQGawPfsCswgme4iYf6rfSKUDzbCc=
11 | github.com/Azure/go-autorest/autorest/adal v0.9.18 h1:kLnPsRjzZZUF3K5REu/Kc+qMQrvuza2bwSnNdhmzLfQ=
12 | github.com/Azure/go-autorest/autorest/adal v0.9.18/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ=
13 | github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw=
14 | github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74=
15 | github.com/Azure/go-autorest/autorest/mocks v0.4.1 h1:K0laFcLE6VLTOwNgSxaGbUcLPuGXlNkbVvq4cW4nIHk=
16 | github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k=
17 | github.com/Azure/go-autorest/autorest/to v0.4.0 h1:oXVqrxakqqV1UZdSazDOPOLvOIz+XA683u8EctwboHk=
18 | github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE=
19 | github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+ZtXWSmf4Tg=
20 | github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8=
21 | github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo=
22 | github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
23 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
24 | github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
25 | github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
26 | github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
27 | github.com/aws/aws-sdk-go v1.48.10 h1:0LIFG3wp2Dt6PsxKWCg1Y1xRrn2vZnW5/gWdgaBalKg=
28 | github.com/aws/aws-sdk-go v1.48.10/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
29 | github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
30 | github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
31 | github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
32 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
33 | github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
34 | github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
35 | github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
36 | github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
37 | github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
38 | github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
39 | github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
40 | github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI=
41 | github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
42 | github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
43 | github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
44 | github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
45 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
46 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
47 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
48 | github.com/denverdino/aliyungo v0.0.0-20230411124812-ab98a9173ace h1:1SnCTPFh2AADpm7ti864EYaugexyiDFt55BW188+d6k=
49 | github.com/denverdino/aliyungo v0.0.0-20230411124812-ab98a9173ace/go.mod h1:TK05uvk4XXfK2kdvRwfcZ1NaxjDxmm7H3aQLko0mJxA=
50 | github.com/distribution/distribution v2.8.3+incompatible h1:RlpEXBLq/WPXYvBYMDAmBX/SnhD67qwtvW/DzKc8pAo=
51 | github.com/distribution/distribution v2.8.3+incompatible/go.mod h1:EgLm2NgWtdKgzF9NpMzUKgzmR7AMmb0VQi2B+ZzDRjc=
52 | github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
53 | github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
54 | github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI=
55 | github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ=
56 | github.com/docker/go-metrics v0.0.1 h1:AgB/0SvBxihN0X8OR4SjsblXkbMvalQ8cjmtKQ2rQV8=
57 | github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw=
58 | github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1 h1:ZClxb8laGDf5arXfYcAtECDFgAgHklGI8CxgjHnXKJ4=
59 | github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE=
60 | github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
61 | github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
62 | github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
63 | github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
64 | github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0=
65 | github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
66 | github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
67 | github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
68 | github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
69 | github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
70 | github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
71 | github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
72 | github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
73 | github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw=
74 | github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
75 | github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
76 | github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
77 | github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
78 | github.com/golang-jwt/jwt/v4 v4.5.1 h1:JdqV9zKUdtaa9gdPlywC3aeoEsR681PlKC+4F5gQgeo=
79 | github.com/golang-jwt/jwt/v4 v4.5.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
80 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
81 | github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
82 | github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
83 | github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
84 | github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
85 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
86 | github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
87 | github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
88 | github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
89 | github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
90 | github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
91 | github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
92 | github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
93 | github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
94 | github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
95 | github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
96 | github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
97 | github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
98 | github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
99 | github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
100 | github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
101 | github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
102 | github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
103 | github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
104 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
105 | github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
106 | github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
107 | github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
108 | github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
109 | github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
110 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
111 | github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc=
112 | github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A=
113 | github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
114 | github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4=
115 | github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
116 | github.com/googleapis/enterprise-certificate-proxy v0.2.3 h1:yk9/cqRKtT9wXZSsRH9aurXEpJX+U6FLtpYTdC3R06k=
117 | github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k=
118 | github.com/googleapis/gax-go/v2 v2.11.0 h1:9V9PWXEsWnPpQhu/PeQIkS4eGzMlTLGgt80cUUI8Ki4=
119 | github.com/googleapis/gax-go/v2 v2.11.0/go.mod h1:DxmR61SGKkGLa2xigwuZIQpkCI2S5iydzRfb3peWZJI=
120 | github.com/gorilla/handlers v1.5.2 h1:cLTUSsNkgcwhgRqvCNmdbRWG0A3N4F+M2nWKdScwyEE=
121 | github.com/gorilla/handlers v1.5.2/go.mod h1:dX+xVpaxdSw+q0Qek8SSsl3dfMk3jNddUkMzo0GtH0w=
122 | github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
123 | github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
124 | github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
125 | github.com/huaweicloud/huaweicloud-sdk-go-obs v3.24.6+incompatible h1:/2MdLc7zHJqzV7J2uVGaoGymVobB/OHC8wmEyWRaK68=
126 | github.com/huaweicloud/huaweicloud-sdk-go-obs v3.24.6+incompatible/go.mod h1:l7VUhRbTKCzdOacdT4oWCwATKyvZqUOlOqr0Ous3k4s=
127 | github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
128 | github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
129 | github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
130 | github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
131 | github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
132 | github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
133 | github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
134 | github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
135 | github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
136 | github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
137 | github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
138 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
139 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
140 | github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60=
141 | github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
142 | github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
143 | github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
144 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
145 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
146 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
147 | github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
148 | github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8=
149 | github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
150 | github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
151 | github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
152 | github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 h1:rc3tiVYb5z54aKaDfakKn0dDjIyPpTtszkjuMzyt7ec=
153 | github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
154 | github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs=
155 | github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
156 | github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
157 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
158 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
159 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
160 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
161 | github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
162 | github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
163 | github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g=
164 | github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q=
165 | github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY=
166 | github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
167 | github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
168 | github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
169 | github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw=
170 | github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI=
171 | github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
172 | github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc=
173 | github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY=
174 | github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY=
175 | github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
176 | github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
177 | github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
178 | github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI=
179 | github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY=
180 | github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
181 | github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
182 | github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
183 | github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
184 | github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
185 | github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
186 | github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
187 | github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
188 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
189 | github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
190 | github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
191 | github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
192 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
193 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
194 | github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
195 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
196 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
197 | github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
198 | github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
199 | github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
200 | github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
201 | github.com/uber/jaeger-client-go v2.30.0+incompatible h1:D6wyKGCecFaSRUpo8lCVbaOOb6ThwMmTEbhRwtKR97o=
202 | github.com/uber/jaeger-client-go v2.30.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk=
203 | github.com/uber/jaeger-lib v2.4.1+incompatible h1:td4jdvLcExb4cBISKIpHuGoVXh+dVKhn2Um6rjCsSsg=
204 | github.com/uber/jaeger-lib v2.4.1+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U=
205 | github.com/wzshiming/cmux v0.4.2 h1:tI73lL5ztVfiqw7R5m5BkxT1+vQ2PBo/oV6qPbNGPiA=
206 | github.com/wzshiming/cmux v0.4.2/go.mod h1:JgE61QfZAjEyNMX0iZo9zIKY6pr9bHVY132yYPwHW5U=
207 | github.com/wzshiming/geario v0.0.0-20240308093553-a996e3817533 h1:mq74wxgDCz7Q6CqZYExt0DHf7Ze28lyMW/TNsfcuk8M=
208 | github.com/wzshiming/geario v0.0.0-20240308093553-a996e3817533/go.mod h1:Fodw3HJvNUS+/MgqXCRp9iYLQfynAu/LKXGOWoX+D/Q=
209 | github.com/wzshiming/hostmatcher v0.0.3 h1:+JYAq6vUZXDEQ1Ipfdc/D7HmaIMngcc71ftonyCQVQk=
210 | github.com/wzshiming/hostmatcher v0.0.3/go.mod h1:F04RIvIWEvOIrIKOlQlMuR8vQMKAVf2YhpU6l31Wwz4=
211 | github.com/wzshiming/httpseek v0.1.0 h1:lEgL7EBELT/VV9UaTp+m3kw5Pe1KOUdY+IPnKkag6tI=
212 | github.com/wzshiming/httpseek v0.1.0/go.mod h1:YoZhlLIwNjTBDXIT8NpK5zRjOgZouRXPaBfjVXdqMMs=
213 | github.com/wzshiming/lru v0.1.0 h1:937SBBo9lDBRMivJqF46eVK2Q3omRWnN7hWpr2i3xZg=
214 | github.com/wzshiming/lru v0.1.0/go.mod h1:uaI2W/Gx3r1gHIc3FG38PKK1idvMXhgY8lToXaBsDIQ=
215 | github.com/wzshiming/trie v0.3.1 h1:YpuoqmEQFJiW0mns/mM6Qk4kdWrXc8kc28/KR1vn0m8=
216 | github.com/wzshiming/trie v0.3.1/go.mod h1:c9thxXTh4KcGkejt4sUsO4c5GUmWpxeWzOJ7AZJaI+8=
217 | github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
218 | go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
219 | go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
220 | go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
221 | go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE=
222 | go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
223 | golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
224 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
225 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
226 | golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
227 | golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
228 | golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
229 | golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw=
230 | golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U=
231 | golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
232 | golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
233 | golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
234 | golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
235 | golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
236 | golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
237 | golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
238 | golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
239 | golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
240 | golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
241 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
242 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
243 | golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
244 | golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
245 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
246 | golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
247 | golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
248 | golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
249 | golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
250 | golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
251 | golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4=
252 | golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU=
253 | golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
254 | golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
255 | golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs=
256 | golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
257 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
258 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
259 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
260 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
261 | golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
262 | golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
263 | golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
264 | golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
265 | golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
266 | golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
267 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
268 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
269 | golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
270 | golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
271 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
272 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
273 | golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
274 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
275 | golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
276 | golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
277 | golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
278 | golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
279 | golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
280 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
281 | golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
282 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
283 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
284 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
285 | golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
286 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
287 | golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
288 | golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM=
289 | golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
290 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
291 | golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
292 | golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
293 | golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
294 | golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
295 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
296 | golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
297 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
298 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
299 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
300 | google.golang.org/api v0.126.0 h1:q4GJq+cAdMAC7XP7njvQ4tvohGLiSlytuL4BQxbIZ+o=
301 | google.golang.org/api v0.126.0/go.mod h1:mBwVAtz+87bEN6CbA1GtZPDOqY2R5ONPqJeIlvyo4Aw=
302 | google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
303 | google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
304 | google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
305 | google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
306 | google.golang.org/cloud v0.0.0-20151119220103-975617b05ea8 h1:Cpp2P6TPjujNoC5M2KHY6g7wfyLYfIWRZaSdIKfDasA=
307 | google.golang.org/cloud v0.0.0-20151119220103-975617b05ea8/go.mod h1:0H1ncTHf11KCFhTc/+EFRbzSCOZx+VUbRMk55Yv5MYk=
308 | google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
309 | google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
310 | google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
311 | google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
312 | google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d h1:VBu5YqKPv6XiJ199exd8Br+Aetz+o08F+PLMnwJQHAY=
313 | google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d/go.mod h1:yZTlhN0tQnXo3h00fuXNCxJdLdIdnVFVBaRJ5LWBbw4=
314 | google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d h1:DoPTO70H+bcDXcd39vOqb2viZxgqeBeSGtZ55yZU4/Q=
315 | google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d/go.mod h1:KjSP20unUpOx5kyQUFa7k4OJg0qeJ7DEZflGDu2p6Bk=
316 | google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d h1:uvYuEyMHKNt+lT4K3bN6fGswmK8qSvcreM3BwjDh+y4=
317 | google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M=
318 | google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
319 | google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
320 | google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
321 | google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
322 | google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
323 | google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
324 | google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
325 | google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ=
326 | google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk=
327 | google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98=
328 | google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
329 | google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
330 | google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
331 | google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
332 | google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
333 | google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
334 | google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
335 | google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
336 | google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
337 | google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
338 | google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
339 | google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
340 | google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
341 | gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
342 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
343 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
344 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
345 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
346 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
347 | gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
348 | gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
349 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
350 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
351 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
352 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
353 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
354 | honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
355 | honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
356 |
--------------------------------------------------------------------------------
/hang_manager.go:
--------------------------------------------------------------------------------
1 | package crproxy
2 |
3 | import (
4 | "time"
5 |
6 | "github.com/wzshiming/geario"
7 | )
8 |
9 | func GetSleepDuration(s geario.B, limit geario.B, r time.Duration) time.Duration {
10 | return time.Duration(s/(limit/geario.B(r)*geario.B(time.Second))) * time.Second
11 | }
12 |
--------------------------------------------------------------------------------
/hang_manager_test.go:
--------------------------------------------------------------------------------
1 | package crproxy
2 |
3 | import (
4 | "testing"
5 | "time"
6 |
7 | "github.com/wzshiming/geario"
8 | )
9 |
10 | func TestGetSleepDuration(t *testing.T) {
11 | type args struct {
12 | s geario.B
13 | limit geario.B
14 | r time.Duration
15 | }
16 | tests := []struct {
17 | name string
18 | args args
19 | want time.Duration
20 | }{
21 | {
22 | args: args{
23 | s: 100,
24 | limit: 100,
25 | r: time.Second,
26 | },
27 | want: time.Second,
28 | },
29 | {
30 | args: args{
31 | s: 200,
32 | limit: 100,
33 | r: time.Second,
34 | },
35 | want: 2 * time.Second,
36 | },
37 | {
38 | args: args{
39 | s: 100,
40 | limit: 50,
41 | r: time.Second,
42 | },
43 | want: 2 * time.Second,
44 | },
45 | {
46 | args: args{
47 | s: 100,
48 | limit: 100,
49 | r: 2 * time.Second,
50 | },
51 | want: 2 * time.Second,
52 | },
53 | {
54 | args: args{
55 | s: 100 * geario.MiB,
56 | limit: geario.MiB,
57 | r: time.Second,
58 | },
59 | want: 100 * time.Second,
60 | },
61 | }
62 | for _, tt := range tests {
63 | t.Run(tt.name, func(t *testing.T) {
64 | if got := GetSleepDuration(tt.args.s, tt.args.limit, tt.args.r); got != tt.want {
65 | t.Errorf("GetSleepDuration() = %v, want %v", got, tt.want)
66 | }
67 | })
68 | }
69 | }
70 |
--------------------------------------------------------------------------------
/internal/acme/acme.go:
--------------------------------------------------------------------------------
1 | package acme
2 |
3 | import (
4 | "context"
5 | "crypto/tls"
6 | "fmt"
7 | "os"
8 | "path/filepath"
9 | "runtime"
10 |
11 | "github.com/wzshiming/hostmatcher"
12 | "golang.org/x/crypto/acme/autocert"
13 | )
14 |
15 | func hostWhitelist(hosts ...string) autocert.HostPolicy {
16 | matcher := hostmatcher.NewMatcher(hosts)
17 | return func(_ context.Context, host string) error {
18 | if !matcher.Match(host) {
19 | return fmt.Errorf("acme/autocert: host %q not configured in HostWhitelist", host)
20 | }
21 | return nil
22 | }
23 | }
24 |
25 | func NewAcme(domains []string, dir string) *tls.Config {
26 | m := &autocert.Manager{
27 | Prompt: autocert.AcceptTOS,
28 | }
29 | if len(domains) > 0 {
30 | m.HostPolicy = hostWhitelist(domains...)
31 | }
32 | if dir == "" {
33 | dir = cacheDir()
34 | }
35 | m.Cache = autocert.DirCache(dir)
36 |
37 | tlsConfig := m.TLSConfig()
38 | return tlsConfig
39 | }
40 |
41 | func homeDir() string {
42 | if runtime.GOOS == "windows" {
43 | return os.Getenv("HOMEDRIVE") + os.Getenv("HOMEPATH")
44 | }
45 | if h := os.Getenv("HOME"); h != "" {
46 | return h
47 | }
48 | return "/"
49 | }
50 |
51 | func cacheDir() string {
52 | const base = "autocert"
53 | switch runtime.GOOS {
54 | case "darwin":
55 | return filepath.Join(homeDir(), "Library", "Caches", base)
56 | case "windows":
57 | for _, ev := range []string{"APPDATA", "CSIDL_APPDATA", "TEMP", "TMP"} {
58 | if v := os.Getenv(ev); v != "" {
59 | return filepath.Join(v, base)
60 | }
61 | }
62 | // Worst case:
63 | return filepath.Join(homeDir(), base)
64 | }
65 | if xdg := os.Getenv("XDG_CACHE_HOME"); xdg != "" {
66 | return filepath.Join(xdg, base)
67 | }
68 | return filepath.Join(homeDir(), ".cache", base)
69 | }
70 |
--------------------------------------------------------------------------------
/internal/maps/maps.go:
--------------------------------------------------------------------------------
1 | package maps
2 |
3 | import (
4 | "sync"
5 | )
6 |
7 | // SyncMap is a wrapper around sync.Map that provides a few additional methods.
8 | type SyncMap[K comparable, V any] struct {
9 | m sync.Map
10 | }
11 |
12 | // Load returns the value stored in the map for a key,
13 | // or nil if no value is present.
14 | func (m *SyncMap[K, V]) Load(key K) (value V, ok bool) {
15 | v, ok := m.m.Load(key)
16 | if !ok {
17 | return value, false
18 | }
19 | return v.(V), true
20 | }
21 |
22 | // Store sets the value for a key.
23 | func (m *SyncMap[K, V]) Store(key K, value V) {
24 | m.m.Store(key, value)
25 | }
26 |
27 | // Delete deletes the value for a key.
28 | func (m *SyncMap[K, V]) Delete(key K) {
29 | m.m.Delete(key)
30 | }
31 |
32 | // Range calls f sequentially for each key and value present in the map.
33 | func (m *SyncMap[K, V]) Range(f func(key K, value V) bool) {
34 | m.m.Range(func(key, value interface{}) bool {
35 | return f(key.(K), value.(V))
36 | })
37 | }
38 |
39 | // LoadAndDelete deletes the value for a key, returning the previous value if any.
40 | func (m *SyncMap[K, V]) LoadAndDelete(key K) (value V, loaded bool) {
41 | v, loaded := m.m.LoadAndDelete(key)
42 | if !loaded {
43 | return value, loaded
44 | }
45 | return v.(V), loaded
46 | }
47 |
48 | // LoadOrStore returns the existing value for the key if present.
49 | func (m *SyncMap[K, V]) LoadOrStore(key K, value V) (V, bool) {
50 | v, loaded := m.m.LoadOrStore(key, value)
51 | if !loaded {
52 | return value, loaded
53 | }
54 | return v.(V), loaded
55 | }
56 |
57 | // Swap stores value for key and returns the previous value for that key.
58 | func (m *SyncMap[K, V]) Swap(key K, value V) (V, bool) {
59 | v, loaded := m.m.Swap(key, value)
60 | if !loaded {
61 | return value, loaded
62 | }
63 | return v.(V), loaded
64 | }
65 |
66 | // Size returns the number of items in the map.
67 | func (m *SyncMap[K, V]) Size() int {
68 | size := 0
69 | m.m.Range(func(key, value interface{}) bool {
70 | size++
71 | return true
72 | })
73 | return size
74 | }
75 |
76 | // Keys returns all the keys in the map.
77 | func (m *SyncMap[K, V]) Keys() []K {
78 | keys := []K{}
79 | m.m.Range(func(key, value interface{}) bool {
80 | keys = append(keys, key.(K))
81 | return true
82 | })
83 | return keys
84 | }
85 |
86 | // Values returns all the values in the map.
87 | func (m *SyncMap[K, V]) Values() []V {
88 | values := []V{}
89 | m.m.Range(func(key, value interface{}) bool {
90 | values = append(values, value.(V))
91 | return true
92 | })
93 | return values
94 | }
95 |
96 | // IsEmpty returns true if the map is empty.
97 | func (m *SyncMap[K, V]) IsEmpty() bool {
98 | empty := true
99 | m.m.Range(func(key, value interface{}) bool {
100 | empty = false
101 | return false
102 | })
103 | return empty
104 | }
105 |
--------------------------------------------------------------------------------
/internal/server/server.go:
--------------------------------------------------------------------------------
1 | package server
2 |
3 | import (
4 | "context"
5 | "fmt"
6 | "net"
7 | "net/http"
8 | "net/http/httptest"
9 | "time"
10 |
11 | "github.com/daocloud/crproxy/internal/acme"
12 | "github.com/wzshiming/cmux"
13 | "github.com/wzshiming/cmux/pattern"
14 | )
15 |
16 | func Run(ctx context.Context, address string, handler http.Handler, acmeHosts []string, acmeCache string, certFile, privateKeyFile string) error {
17 | listener, err := net.Listen("tcp", address)
18 | if err != nil {
19 | return err
20 | }
21 |
22 | muxListener := cmux.NewMuxListener(listener)
23 | tlsListener, err := muxListener.MatchPrefix(pattern.Pattern[pattern.TLS]...)
24 | if err != nil {
25 | return fmt.Errorf("match tls listener: %w", err)
26 | }
27 | unmatchedListener, err := muxListener.Unmatched()
28 | if err != nil {
29 | return fmt.Errorf("unmatched listener: %w", err)
30 | }
31 |
32 | ctx, cancel := context.WithCancel(ctx)
33 | defer cancel()
34 |
35 | errCh := make(chan error, 1)
36 |
37 | if (certFile != "" && privateKeyFile != "") || len(acmeHosts) != 0 {
38 | go func() {
39 | svc := &http.Server{
40 | ReadHeaderTimeout: 5 * time.Second,
41 | BaseContext: func(_ net.Listener) context.Context {
42 | return ctx
43 | },
44 | Addr: address,
45 | Handler: handler,
46 | }
47 | if len(acmeHosts) != 0 {
48 | svc.TLSConfig = acme.NewAcme(acmeHosts, acmeCache)
49 | }
50 | err = svc.ServeTLS(tlsListener, certFile, privateKeyFile)
51 | if err != nil {
52 | errCh <- fmt.Errorf("serve https: %w", err)
53 | }
54 | }()
55 | } else {
56 | svc := httptest.Server{
57 | Listener: tlsListener,
58 | Config: &http.Server{
59 | ReadHeaderTimeout: 5 * time.Second,
60 | BaseContext: func(_ net.Listener) context.Context {
61 | return ctx
62 | },
63 | Addr: address,
64 | Handler: handler,
65 | },
66 | }
67 | svc.StartTLS()
68 | }
69 |
70 | go func() {
71 | svc := &http.Server{
72 | ReadHeaderTimeout: 5 * time.Second,
73 | BaseContext: func(_ net.Listener) context.Context {
74 | return ctx
75 | },
76 | Addr: address,
77 | Handler: handler,
78 | }
79 | err = svc.Serve(unmatchedListener)
80 | if err != nil {
81 | errCh <- fmt.Errorf("serve http: %w", err)
82 | }
83 | }()
84 |
85 | select {
86 | case err = <-errCh:
87 | case <-ctx.Done():
88 | err = ctx.Err()
89 | }
90 |
91 | return err
92 | }
93 |
--------------------------------------------------------------------------------
/logo/LICENSE:
--------------------------------------------------------------------------------
1 | # The crproxy logo files is licensed under Apache 2.0 license.
2 |
--------------------------------------------------------------------------------
/logo/crproxy.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/DaoCloud/crproxy/6e21fb998b9e6cc59fb52b41a3ec3df83884e436/logo/crproxy.png
--------------------------------------------------------------------------------
/logo/crproxy.svg:
--------------------------------------------------------------------------------
1 |
--------------------------------------------------------------------------------
/storage/driver/obs/doc.go:
--------------------------------------------------------------------------------
1 | // Copy from github.com/docker/distribution/registry/storage/driver/obs
2 |
3 | package obs
4 |
--------------------------------------------------------------------------------
/storage/driver/obs/obs.go:
--------------------------------------------------------------------------------
1 | // Package obs provides a storagedriver.StorageDriver implementation to
2 | // store blobs in HuaweiCloud storage.
3 | //
4 | // This package leverages the huaweicloud/huaweicloud-sdk-go-obs client library
5 | // for interfacing with obs.
6 | //
7 | // Because obs is a key, value store the Stat call does not support last modification
8 | // time for directories (directories are an abstraction for key, value stores)
9 | //
10 | // Note that the contents of incomplete uploads are not accessible even though
11 | // Stat returns their length
12 | //
13 |
14 | package obs
15 |
16 | import (
17 | "bytes"
18 | "context"
19 | "errors"
20 | "fmt"
21 | "io"
22 | "math"
23 | "net/http"
24 | "reflect"
25 | "sort"
26 | "strconv"
27 | "strings"
28 | "time"
29 |
30 | storagedriver "github.com/docker/distribution/registry/storage/driver"
31 | "github.com/docker/distribution/registry/storage/driver/base"
32 | "github.com/docker/distribution/registry/storage/driver/factory"
33 | "github.com/huaweicloud/huaweicloud-sdk-go-obs/obs"
34 | )
35 |
36 | // noStorageClass defines the value to be used if storage class is not supported by the OBS endpoint
37 | const noStorageClass obs.StorageClassType = "NONE"
38 |
39 | // OBSStorageClasses lists all compatible (instant retrieval) OBS storage classes
40 | var OBSStorageClasses = []obs.StorageClassType{
41 | noStorageClass,
42 | obs.StorageClassStandard,
43 | obs.StorageClassWarm,
44 | obs.StorageClassCold,
45 | }
46 |
47 | // validStorageClasses contains known OBS StorageClass
48 | var validStorageClasses = map[obs.StorageClassType]struct{}{}
49 |
50 | var OBSAcls = []obs.AclType{
51 | obs.AclPrivate,
52 | obs.AclPublicRead,
53 | obs.AclPublicReadWrite,
54 | obs.AclAuthenticatedRead,
55 | obs.AclBucketOwnerRead,
56 | obs.AclBucketOwnerFullControl,
57 | obs.AclLogDeliveryWrite,
58 | obs.AclPublicReadDelivery,
59 | obs.AclPublicReadWriteDelivery,
60 | }
61 |
62 | // validObjectACLs contains known OBS object Acls
63 | var validObjectACLs = map[obs.AclType]struct{}{}
64 |
65 | const (
66 | driverName = "obs"
67 |
68 | minChunkSize = 5 << 20
69 | maxChunkSize = 5 << 30
70 | defaultChunkSize = 2 * minChunkSize
71 | listMax = 1000
72 | defaultMultipartCopyChunkSize = 32 << 20
73 | defaultMultipartCopyMaxConcurrency = 100
74 | defaultMultipartCopyThresholdSize = 32 << 20
75 | )
76 |
77 | // DriverParameters A struct that encapsulates all of the driver parameters after all values have been set
78 | type DriverParameters struct {
79 | AccessKey string
80 | SecretKey string
81 | Bucket string
82 | Endpoint string
83 | Encrypt bool
84 | ChunkSize int64
85 | RootDirectory string
86 | KeyID string
87 | MultipartCopyChunkSize int64
88 | MultipartCopyMaxConcurrency int64
89 | MultipartCopyThresholdSize int64
90 | MultipartCombineSmallPart bool
91 | StorageClass obs.StorageClassType
92 | ObjectACL obs.AclType
93 | }
94 |
95 | func init() {
96 | for _, storageClass := range OBSStorageClasses {
97 | validStorageClasses[storageClass] = struct{}{}
98 | }
99 | for _, acl := range OBSAcls {
100 | validObjectACLs[acl] = struct{}{}
101 | }
102 | factory.Register(driverName, &obsDriverFactory{})
103 | }
104 |
105 | // obsDriverFactory implements the factory.StorageDriverFactory interface
106 | type obsDriverFactory struct{}
107 |
108 | func (factory *obsDriverFactory) Create(parameters map[string]interface{}) (storagedriver.StorageDriver, error) {
109 | return FromParameters(parameters)
110 | }
111 |
112 | type driver struct {
113 | Client *obs.ObsClient
114 | Bucket string
115 | ChunkSize int64
116 | Encrypt bool
117 | RootDirectory string
118 | KeyID string
119 | MultipartCopyChunkSize int64
120 | MultipartCopyMaxConcurrency int64
121 | MultipartCopyThresholdSize int64
122 | StorageClass obs.StorageClassType
123 | ObjectACL obs.AclType
124 | }
125 |
126 | type baseEmbed struct {
127 | base.Base
128 | }
129 |
130 | // Driver is a storagedriver.StorageDriver implementation backed by OBS
131 | // Objects are stored at absolute keys in the provided bucket.
132 | type Driver struct {
133 | baseEmbed
134 | }
135 |
136 | // FromParameters constructs a new Driver with a given parameters map
137 | // Required parameters:
138 | // - accesskey
139 | // - secretkey
140 | // - region
141 | // - bucket
142 | // - encrypt
143 | func FromParameters(parameters map[string]interface{}) (*Driver, error) {
144 | // Providing no values for these is valid in case the user is authenticating
145 | accessKey, ok := parameters["accesskey"]
146 | if !ok {
147 | return nil, fmt.Errorf("No accesskeyid parameter provided")
148 | }
149 | secretKey, ok := parameters["secretkey"]
150 | if !ok {
151 | return nil, fmt.Errorf("No accesskeysecret parameter provided")
152 | }
153 |
154 | endpoint, ok := parameters["endpoint"]
155 | if !ok || fmt.Sprint(endpoint) == "" {
156 | return nil, fmt.Errorf("No region endpoint parameter provided")
157 | }
158 |
159 | bucket, ok := parameters["bucket"]
160 | if !ok || fmt.Sprint(bucket) == "" {
161 | return nil, fmt.Errorf("No bucket parameter provided")
162 | }
163 |
164 | storageClass := obs.StorageClassStandard
165 | storageClassParam, ok := parameters["storageclass"]
166 | if ok {
167 | storageClassString, ok := storageClassParam.(obs.StorageClassType)
168 | if !ok {
169 | return nil, fmt.Errorf(
170 | "the storageclass parameter must be one of %v, %v invalid",
171 | OBSStorageClasses,
172 | storageClassParam,
173 | )
174 | }
175 | if _, ok = validStorageClasses[storageClassString]; !ok {
176 | return nil, fmt.Errorf(
177 | "the storageclass parameter must be one of %v, %v invalid",
178 | OBSStorageClasses,
179 | storageClassParam,
180 | )
181 | }
182 | storageClass = storageClassString
183 | }
184 |
185 | objectACL := obs.AclPrivate
186 | objectACLParam, ok := parameters["objectacl"]
187 | if ok {
188 | objectACLString, ok := objectACLParam.(obs.AclType)
189 | if !ok {
190 | return nil, fmt.Errorf(
191 | "the objectacl parameter must be one of %v, %v invalid",
192 | OBSAcls,
193 | objectACLParam,
194 | )
195 | }
196 |
197 | if _, ok = validObjectACLs[objectACLString]; !ok {
198 | return nil, fmt.Errorf(
199 | "the objectacl parameter must be one of %v, %v invalid",
200 | OBSAcls,
201 | objectACLParam,
202 | )
203 | }
204 | objectACL = objectACLString
205 | }
206 |
207 | encryptBool := false
208 | encrypt, ok := parameters["encrypt"]
209 | if ok {
210 | switch b := encrypt.(type) {
211 | case bool:
212 | encryptBool = b
213 | case string:
214 | ib, err := strconv.ParseBool(b)
215 | if err != nil {
216 | return nil, err
217 | }
218 | encryptBool = ib
219 | default:
220 | return nil, fmt.Errorf("The encrypt parameter should be a boolean")
221 | }
222 | }
223 |
224 | keyID, ok := parameters["keyid"]
225 | if !ok {
226 | keyID = ""
227 | }
228 |
229 | chunkSize, err := getParameterAsInt64(parameters, "chunksize", defaultChunkSize, minChunkSize, maxChunkSize)
230 | if err != nil {
231 | return nil, err
232 | }
233 |
234 | multipartCopyChunkSize, err := getParameterAsInt64(parameters, "multipartcopychunksize", defaultMultipartCopyChunkSize, minChunkSize, maxChunkSize)
235 | if err != nil {
236 | return nil, err
237 | }
238 |
239 | multipartCopyMaxConcurrency, err := getParameterAsInt64(parameters, "multipartcopymaxconcurrency", defaultMultipartCopyMaxConcurrency, 1, math.MaxInt64)
240 | if err != nil {
241 | return nil, err
242 | }
243 |
244 | multipartCopyThresholdSize, err := getParameterAsInt64(parameters, "multipartcopythresholdsize", defaultMultipartCopyThresholdSize, 0, maxChunkSize)
245 | if err != nil {
246 | return nil, err
247 | }
248 |
249 | rootDirectory, ok := parameters["rootdirectory"]
250 | if !ok {
251 | rootDirectory = ""
252 | }
253 |
254 | params := DriverParameters{
255 | AccessKey: fmt.Sprint(accessKey),
256 | SecretKey: fmt.Sprint(secretKey),
257 | Bucket: fmt.Sprint(bucket),
258 | Endpoint: fmt.Sprint(endpoint),
259 | ChunkSize: chunkSize,
260 | RootDirectory: fmt.Sprint(rootDirectory),
261 | Encrypt: encryptBool,
262 | KeyID: fmt.Sprint(keyID),
263 | MultipartCopyChunkSize: multipartCopyChunkSize,
264 | MultipartCopyMaxConcurrency: multipartCopyMaxConcurrency,
265 | MultipartCopyThresholdSize: multipartCopyThresholdSize,
266 | StorageClass: storageClass,
267 | ObjectACL: objectACL,
268 | }
269 |
270 | return New(params)
271 | }
272 |
273 | // getParameterAsInt64 converts parameters[name] to an int64 value (using
274 | // defaultt if nil), verifies it is no smaller than min, and returns it.
275 | func getParameterAsInt64(parameters map[string]interface{}, name string, defaultt int64, min int64, max int64) (int64, error) {
276 | rv := defaultt
277 | param, ok := parameters[name]
278 | if ok {
279 | switch v := param.(type) {
280 | case string:
281 | vv, err := strconv.ParseInt(v, 0, 64)
282 | if err != nil {
283 | return 0, fmt.Errorf("%s parameter must be an integer, %v invalid", name, param)
284 | }
285 | rv = vv
286 | case int64:
287 | rv = v
288 | case int, uint, int32, uint32, uint64:
289 | rv = reflect.ValueOf(v).Convert(reflect.TypeOf(rv)).Int()
290 | case nil:
291 | // do nothing
292 | default:
293 | return 0, fmt.Errorf("invalid value for %s: %#v", name, param)
294 | }
295 | }
296 |
297 | if rv < min || rv > max {
298 | return 0, fmt.Errorf("the %s %#v parameter should be a number between %d and %d (inclusive)", name, rv, min, max)
299 | }
300 |
301 | return rv, nil
302 | }
303 |
304 | // New constructs a new Driver with the given Aliyun credentials, region, encryption flag, and
305 | // bucketName
306 | func New(params DriverParameters) (*Driver, error) {
307 | client, err := obs.New(params.AccessKey, params.SecretKey, params.Endpoint)
308 | if err != nil {
309 | return nil, err
310 | }
311 |
312 | d := &driver{
313 | Client: client,
314 | Bucket: params.Bucket,
315 | ChunkSize: params.ChunkSize,
316 | Encrypt: params.Encrypt,
317 | RootDirectory: params.RootDirectory,
318 | KeyID: params.KeyID,
319 | MultipartCopyChunkSize: params.MultipartCopyChunkSize,
320 | MultipartCopyMaxConcurrency: params.MultipartCopyMaxConcurrency,
321 | MultipartCopyThresholdSize: params.MultipartCopyThresholdSize,
322 | ObjectACL: params.ObjectACL,
323 | StorageClass: params.StorageClass,
324 | }
325 |
326 | return &Driver{
327 | baseEmbed: baseEmbed{
328 | Base: base.Base{
329 | StorageDriver: d,
330 | },
331 | },
332 | }, nil
333 | }
334 |
335 | func (d *driver) Name() string {
336 | return driverName
337 | }
338 |
339 | // GetContent retrieves the content stored at "path" as a []byte.
340 | func (d *driver) GetContent(ctx context.Context, path string) ([]byte, error) {
341 | reader, err := d.Reader(ctx, path, 0)
342 | if err != nil {
343 | return nil, err
344 | }
345 | return io.ReadAll(reader)
346 | }
347 |
348 | // PutContent stores the []byte content at a location designated by "path".
349 | func (d *driver) PutContent(ctx context.Context, path string, contents []byte) error {
350 | input := &obs.PutObjectInput{}
351 | input.Bucket = d.Bucket
352 | input.Key = d.OBSPath(path)
353 | input.Body = bytes.NewReader(contents)
354 | input.ContentType = d.getContentType()
355 | input.ACL = d.getACL()
356 | input.StorageClass = d.getStorageClass()
357 | input.SseHeader = d.getEncryptionMode()
358 | _, err := d.Client.PutObject(input)
359 | return d.parseError(path, err)
360 | }
361 |
362 | // Reader retrieves an io.ReadCloser for the content stored at "path" with a
363 | // given byte offset.
364 | func (d *driver) Reader(ctx context.Context, path string, offset int64) (io.ReadCloser, error) {
365 | input := obs.GetObjectInput{}
366 | input.Bucket = d.Bucket
367 | input.Key = d.OBSPath(path)
368 | input.RangeStart = offset
369 | input.RangeEnd = math.MaxInt64
370 | output, err := d.Client.GetObject(&input)
371 | if err != nil {
372 | var obsErr obs.ObsError
373 | if errors.As(err, &obsErr) {
374 | if obsErr.Code == "InvalidRange" {
375 | return io.NopCloser(bytes.NewReader(nil)), nil
376 | }
377 | }
378 | return nil, d.parseError(path, err)
379 | }
380 | return output.Body, nil
381 | }
382 |
383 | // Writer returns a FileWriter which will store the content written to it
384 | // at the location designated by "path" after the call to Commit.
385 | func (d *driver) Writer(ctx context.Context, path string, appendParam bool) (storagedriver.FileWriter, error) {
386 | key := d.OBSPath(path)
387 | if !appendParam {
388 | // TODO (brianbland): cancel other uploads at this path
389 | initInput := &obs.InitiateMultipartUploadInput{}
390 | initInput.Bucket = d.Bucket
391 | initInput.Key = key
392 | initInput.ContentType = d.getContentType()
393 | initInput.ACL = d.getACL()
394 | initInput.StorageClass = d.getStorageClass()
395 | initInput.SseHeader = d.getEncryptionMode()
396 | initOutput, err := d.Client.InitiateMultipartUpload(initInput)
397 | if err != nil {
398 | return nil, err
399 | }
400 | return d.newWriter(key, initOutput.UploadId, nil), nil
401 | }
402 |
403 | listMultipartUploadsInput := &obs.ListMultipartUploadsInput{
404 | Bucket: d.Bucket,
405 | Prefix: key,
406 | }
407 | for {
408 | output, err := d.Client.ListMultipartUploads(listMultipartUploadsInput)
409 | if err != nil {
410 | return nil, d.parseError(path, err)
411 | }
412 |
413 | // output.Uploads can only be empty on the first call
414 | // if there were no more results to return after the first call, output.IsTruncated would have been false
415 | // and the loop would be exited without recalling ListMultipartUploads
416 | if len(output.Uploads) == 0 {
417 | break
418 | }
419 |
420 | var allParts []obs.Part
421 | for _, multi := range output.Uploads {
422 | if key != multi.Key {
423 | continue
424 | }
425 |
426 | partsList, err := d.Client.ListParts(&obs.ListPartsInput{
427 | Bucket: d.Bucket,
428 | Key: key,
429 | UploadId: multi.UploadId,
430 | })
431 | if err != nil {
432 | return nil, d.parseError(path, err)
433 | }
434 | allParts = append(allParts, partsList.Parts...)
435 | for partsList.IsTruncated {
436 | partsList, err = d.Client.ListParts(&obs.ListPartsInput{
437 | Bucket: d.Bucket,
438 | Key: key,
439 | UploadId: multi.UploadId,
440 | PartNumberMarker: partsList.NextPartNumberMarker,
441 | })
442 | if err != nil {
443 | return nil, d.parseError(path, err)
444 | }
445 | allParts = append(allParts, partsList.Parts...)
446 | }
447 | return d.newWriter(key, multi.UploadId, allParts), nil
448 | }
449 |
450 | // output.NextUploadIdMarker must have at least one element or we would have returned not found
451 | listMultipartUploadsInput.UploadIdMarker = output.NextUploadIdMarker
452 |
453 | // IsTruncated "specifies whether (true) or not (false) all of the results were returned"
454 | // if everything has been returned, break
455 | if !output.IsTruncated {
456 | break
457 | }
458 | }
459 | return nil, storagedriver.PathNotFoundError{Path: path}
460 | }
461 |
462 | // Stat retrieves the FileInfo for the given path, including the current size
463 | // in bytes and the creation time.
464 | func (d *driver) Stat(ctx context.Context, path string) (storagedriver.FileInfo, error) {
465 | input := &obs.ListObjectsInput{}
466 | input.Bucket = d.Bucket
467 | input.Prefix = d.OBSPath(path)
468 | input.MaxKeys = 1
469 | output, err := d.Client.ListObjects(input)
470 | if err != nil {
471 | return nil, err
472 | }
473 |
474 | fi := storagedriver.FileInfoFields{
475 | Path: path,
476 | }
477 |
478 | if len(output.Contents) == 1 {
479 | if output.Contents[0].Key != d.OBSPath(path) {
480 | fi.IsDir = true
481 | } else {
482 | fi.IsDir = false
483 | fi.Size = output.Contents[0].Size
484 | fi.ModTime = output.Contents[0].LastModified
485 | }
486 | } else if len(output.CommonPrefixes) == 1 {
487 | fi.IsDir = true
488 | } else {
489 | return nil, storagedriver.PathNotFoundError{Path: path}
490 | }
491 |
492 | return storagedriver.FileInfoInternal{FileInfoFields: fi}, nil
493 | }
494 |
495 | // List returns a list of the objects that are direct descendants of the given path.
496 | func (d *driver) List(ctx context.Context, opath string) ([]string, error) {
497 | path := opath
498 | if path != "/" && path[len(path)-1] != '/' {
499 | path = path + "/"
500 | }
501 |
502 | // This is to cover for the cases when the rootDirectory of the driver is either "" or "/".
503 | // In those cases, there is no root prefix to replace and we must actually add a "/" to all
504 | // results in order to keep them as valid paths as recognized by storagedriver.PathRegexp
505 | prefix := ""
506 | if d.OBSPath("") == "" {
507 | prefix = "/"
508 | }
509 | input := &obs.ListObjectsInput{}
510 | input.Bucket = d.Bucket
511 | input.Prefix = d.OBSPath(path)
512 | input.MaxKeys = listMax
513 | input.Delimiter = "/"
514 | output, err := d.Client.ListObjects(input)
515 | if err != nil {
516 | return nil, d.parseError(opath, err)
517 | }
518 |
519 | files := []string{}
520 | directories := []string{}
521 |
522 | for {
523 | for _, content := range output.Contents {
524 | files = append(files, strings.Replace(content.Key, d.OBSPath(""), prefix, 1))
525 | }
526 |
527 | for _, commonPrefix := range output.CommonPrefixes {
528 | commonPrefix := commonPrefix
529 | directories = append(directories, strings.Replace(commonPrefix[0:len(commonPrefix)-1], d.OBSPath(""), prefix, 1))
530 | }
531 |
532 | if output.IsTruncated {
533 | input.Marker = output.NextMarker
534 | output, err = d.Client.ListObjects(input)
535 | if err != nil {
536 | return nil, err
537 | }
538 | } else {
539 | break
540 | }
541 | }
542 |
543 | // This is to cover for the cases when the first key equal to obsPath.
544 | if len(files) > 0 && files[0] == strings.Replace(d.OBSPath(path), d.OBSPath(""), prefix, 1) {
545 | files = files[1:]
546 | }
547 |
548 | if opath != "/" {
549 | if len(files) == 0 && len(directories) == 0 {
550 | // Treat empty response as missing directory, since we don't actually
551 | // have directories in OBS.
552 | return nil, storagedriver.PathNotFoundError{Path: opath}
553 | }
554 | }
555 |
556 | return append(files, directories...), nil
557 | }
558 |
559 | // Move moves an object stored at sourcePath to destPath, removing the original
560 | // object.
561 | func (d *driver) Move(ctx context.Context, sourcePath string, destPath string) error {
562 | /* This is terrible, but aws doesn't have an actual move. */
563 | if err := d.copy(ctx, sourcePath, destPath); err != nil {
564 | return err
565 | }
566 | return d.Delete(ctx, sourcePath)
567 | }
568 |
569 | // copy copies an object stored at sourcePath to destPath.
570 | func (d *driver) copy(ctx context.Context, sourcePath string, destPath string) error {
571 | // OBS can copy objects up to 5 GB in size with a single PUT Object - Copy
572 | // operation. For larger objects, the multipart upload API must be used.
573 | //
574 | // Empirically, multipart copy is fastest with 32 MB parts and is faster
575 | // than PUT Object - Copy for objects larger than 32 MB.
576 |
577 | fileInfo, err := d.Stat(ctx, sourcePath)
578 | if err != nil {
579 | return d.parseError(sourcePath, err)
580 | }
581 |
582 | if fileInfo.Size() <= d.MultipartCopyThresholdSize {
583 | input := &obs.CopyObjectInput{}
584 | input.Bucket = d.Bucket
585 | input.Key = d.OBSPath(destPath)
586 | input.ContentType = d.getContentType()
587 | input.ACL = d.getACL()
588 | input.SseHeader = d.getEncryptionMode()
589 | input.SourceSseHeader = d.getEncryptionMode()
590 | input.StorageClass = d.getStorageClass()
591 | input.CopySourceBucket = d.Bucket
592 | input.CopySourceKey = d.OBSPath(sourcePath)
593 | _, err := d.Client.CopyObject(input)
594 | if err != nil {
595 | return d.parseError(sourcePath, err)
596 | }
597 | return nil
598 | }
599 | initInput := &obs.InitiateMultipartUploadInput{}
600 | initInput.Bucket = d.Bucket
601 | initInput.Key = d.OBSPath(destPath)
602 | initInput.ContentType = d.getContentType()
603 | initInput.ACL = d.getACL()
604 | initInput.StorageClass = d.getStorageClass()
605 | initInput.SseHeader = d.getEncryptionMode()
606 | initOutput, err := d.Client.InitiateMultipartUpload(initInput)
607 | if err != nil {
608 | return err
609 | }
610 |
611 | numParts := (fileInfo.Size() + d.MultipartCopyChunkSize - 1) / d.MultipartCopyChunkSize
612 | completedParts := make([]obs.Part, numParts)
613 | errChan := make(chan error, numParts)
614 | limiter := make(chan struct{}, d.MultipartCopyMaxConcurrency)
615 |
616 | for i := range completedParts {
617 | i := int64(i)
618 | go func() {
619 | limiter <- struct{}{}
620 | firstByte := i * d.MultipartCopyChunkSize
621 | lastByte := firstByte + d.MultipartCopyChunkSize - 1
622 | if lastByte >= fileInfo.Size() {
623 | lastByte = fileInfo.Size() - 1
624 | }
625 | copyPartOutput, err := d.Client.CopyPart(&obs.CopyPartInput{
626 | Bucket: d.Bucket,
627 | Key: d.OBSPath(destPath),
628 | CopySourceBucket: d.Bucket,
629 | CopySourceKey: d.OBSPath(sourcePath),
630 | PartNumber: int(i + 1),
631 | UploadId: initOutput.UploadId,
632 | CopySourceRangeStart: firstByte,
633 | CopySourceRangeEnd: lastByte,
634 | SseHeader: d.getEncryptionMode(),
635 | SourceSseHeader: d.getEncryptionMode(),
636 | })
637 | if err == nil {
638 | completedParts[i] = obs.Part{
639 | ETag: copyPartOutput.ETag,
640 | PartNumber: int(i + 1),
641 | }
642 | }
643 | errChan <- err
644 | <-limiter
645 | }()
646 | }
647 |
648 | for range completedParts {
649 | err := <-errChan
650 | if err != nil {
651 | return err
652 | }
653 | }
654 | _, err = d.Client.CompleteMultipartUpload(&obs.CompleteMultipartUploadInput{
655 | Bucket: d.Bucket,
656 | Key: d.OBSPath(destPath),
657 | UploadId: initOutput.UploadId,
658 | Parts: completedParts,
659 | })
660 | return err
661 | }
662 |
663 | // Delete recursively deletes all objects stored at "path" and its subpaths.
664 | // We must be careful since obs does not guarantee read after delete consistency
665 | func (d *driver) Delete(ctx context.Context, path string) error {
666 | objects := make([]obs.ObjectToDelete, 0, listMax)
667 | obsPath := d.OBSPath(path)
668 | listObjectsInput := &obs.ListObjectsInput{}
669 | listObjectsInput.Bucket = d.Bucket
670 | listObjectsInput.Prefix = obsPath
671 |
672 | for {
673 | // list all the objects
674 | output, err := d.Client.ListObjects(listObjectsInput)
675 |
676 | // output.Contents can only be empty on the first call
677 | // if there were no more results to return after the first call, resp.IsTruncated would have been false
678 | // and the loop would exit without recalling ListObjects
679 | if err != nil || len(output.Contents) == 0 {
680 | return storagedriver.PathNotFoundError{Path: path}
681 | }
682 |
683 | for _, content := range output.Contents {
684 | // Skip if we encounter a key that is not a subpath (so that deleting "/a" does not delete "/ab").
685 | if len(content.Key) > len(obsPath) && (content.Key)[len(obsPath)] != '/' {
686 | continue
687 | }
688 | objects = append(objects, obs.ObjectToDelete{
689 | Key: content.Key,
690 | })
691 | }
692 |
693 | // Delete objects only if the list is not empty, otherwise obs API returns a cryptic error
694 | if len(objects) > 0 {
695 | output, err := d.Client.DeleteObjects(&obs.DeleteObjectsInput{
696 | Bucket: d.Bucket,
697 | Objects: objects,
698 | Quiet: false,
699 | })
700 | if err != nil {
701 | return err
702 | }
703 |
704 | if len(output.Errors) > 0 {
705 | errs := make([]error, 0, len(output.Errors))
706 | for _, err := range output.Errors {
707 | errs = append(errs, errors.New(err.Message))
708 | }
709 |
710 | return storagedriver.Error{
711 | DriverName: driverName,
712 | Enclosed: errors.Join(errs...),
713 | }
714 | }
715 | }
716 | // NOTE: we don't want to reallocate
717 | // the slice so we simply "reset" it
718 | objects = objects[:0]
719 |
720 | listObjectsInput.Marker = output.NextMarker
721 |
722 | // IsTruncated "specifies whether (true) or not (false) all of the results were returned"
723 | // if everything has been returned, break
724 | if !output.IsTruncated {
725 | break
726 | }
727 | }
728 | return nil
729 | }
730 |
731 | // URLFor returns a URL which may be used to retrieve the content stored at the given path.
732 | // May return an UnsupportedMethodErr in certain StorageDriver implementations.
733 | func (d *driver) URLFor(ctx context.Context, path string, options map[string]interface{}) (string, error) {
734 | methodString := http.MethodGet
735 | method, ok := options["method"]
736 | if ok {
737 | methodString, ok = method.(string)
738 | if !ok || (methodString != http.MethodGet && methodString != http.MethodHead) {
739 | return "", storagedriver.ErrUnsupportedMethod{}
740 | }
741 | }
742 |
743 | expiresIn := 20 * time.Minute.Seconds()
744 | expires, ok := options["expiry"]
745 | if ok {
746 | et, ok := expires.(time.Time)
747 | if ok {
748 | expiresIn = time.Until(et).Seconds()
749 | }
750 | }
751 | output, err := d.Client.CreateSignedUrl(&obs.CreateSignedUrlInput{
752 | Bucket: d.Bucket,
753 | Key: d.OBSPath(path),
754 | Method: obs.HttpMethodType(methodString),
755 | Expires: int(expiresIn),
756 | })
757 | if err != nil {
758 | return "", err
759 | }
760 | return output.SignedUrl, nil
761 | }
762 |
763 | // Walk traverses a filesystem defined within driver, starting
764 | // from the given path, calling f on each file
765 | func (d *driver) Walk(ctx context.Context, path string, f storagedriver.WalkFn) error {
766 | return storagedriver.WalkFallback(ctx, d, path, f)
767 | }
768 |
769 | func (d *driver) OBSPath(path string) string {
770 | return strings.TrimLeft(strings.TrimRight(d.RootDirectory, "/")+path, "/")
771 | }
772 |
773 | func (d *driver) parseError(path string, err error) error {
774 | var obsErr obs.ObsError
775 | if errors.As(err, &obsErr) {
776 | if obsErr.Code == "NoSuchKey" {
777 | return storagedriver.PathNotFoundError{Path: path}
778 | }
779 | }
780 | return err
781 | }
782 |
783 | func (d *driver) getContentType() string {
784 | return "application/octet-stream"
785 | }
786 |
787 | func (d *driver) getACL() obs.AclType {
788 | return d.ObjectACL
789 | }
790 |
791 | func (d *driver) getStorageClass() obs.StorageClassType {
792 | if d.StorageClass == noStorageClass {
793 | return ""
794 | }
795 | return d.StorageClass
796 | }
797 |
798 | func (d *driver) getEncryptionMode() obs.ISseHeader {
799 | if !d.Encrypt {
800 | return nil
801 | }
802 | return obs.SseKmsHeader{Key: d.KeyID}
803 | }
804 |
805 | func (d *driver) getSSEKMSKeyID() string {
806 | return d.KeyID
807 | }
808 |
809 | // writer attempts to upload parts to OBS in a buffered fashion where the last
810 | // part is at least as large as the chunksize, so the multipart upload could be
811 | // cleanly resumed in the future. This is violated if Close is called after less
812 | // than a full chunk is written.
813 | type writer struct {
814 | driver *driver
815 | key string
816 | uploadID string
817 | parts []obs.Part
818 | size int64
819 | readyPart []byte
820 | pendingPart []byte
821 | closed bool
822 | committed bool
823 | cancelled bool
824 | }
825 |
826 | func (d *driver) newWriter(key, uploadID string, parts []obs.Part) storagedriver.FileWriter {
827 | var size int64
828 | for _, part := range parts {
829 | size += part.Size
830 | }
831 | return &writer{
832 | driver: d,
833 | key: key,
834 | uploadID: uploadID,
835 | parts: parts,
836 | size: size,
837 | }
838 | }
839 |
840 | type completedParts []obs.Part
841 |
842 | func (a completedParts) Len() int { return len(a) }
843 | func (a completedParts) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
844 | func (a completedParts) Less(i, j int) bool { return a[i].PartNumber < a[j].PartNumber }
845 |
846 | func (w *writer) Write(p []byte) (n int, err error) {
847 | if w.closed {
848 | return 0, fmt.Errorf("already closed")
849 | } else if w.committed {
850 | return 0, fmt.Errorf("already committed")
851 | } else if w.cancelled {
852 | return 0, fmt.Errorf("already cancelled")
853 | }
854 |
855 | // If the last written part is smaller than minChunkSize, we need to make a
856 | // new multipart upload :sadface:
857 | if len(w.parts) > 0 && int(w.parts[len(w.parts)-1].Size) < minChunkSize {
858 | var completedUploadedParts completedParts
859 | for _, part := range w.parts {
860 | completedUploadedParts = append(completedUploadedParts, obs.Part{
861 | ETag: part.ETag,
862 | PartNumber: part.PartNumber,
863 | })
864 | }
865 |
866 | sort.Sort(completedUploadedParts)
867 |
868 | _, err := w.driver.Client.CompleteMultipartUpload(&obs.CompleteMultipartUploadInput{
869 | Bucket: w.driver.Bucket,
870 | Key: w.key,
871 | UploadId: w.uploadID,
872 | Parts: completedUploadedParts,
873 | })
874 | if err != nil {
875 | w.driver.Client.AbortMultipartUpload(&obs.AbortMultipartUploadInput{
876 | Bucket: w.driver.Bucket,
877 | Key: w.key,
878 | UploadId: w.uploadID,
879 | })
880 | return 0, err
881 | }
882 | initInput := &obs.InitiateMultipartUploadInput{}
883 | initInput.Bucket = w.driver.Bucket
884 | initInput.Key = w.key
885 | initInput.ContentType = w.driver.getContentType()
886 | initInput.ACL = w.driver.getACL()
887 | initInput.StorageClass = w.driver.getStorageClass()
888 | initInput.SseHeader = w.driver.getEncryptionMode()
889 | initOutput, err := w.driver.Client.InitiateMultipartUpload(initInput)
890 | if err != nil {
891 | return 0, err
892 | }
893 | w.uploadID = initOutput.UploadId
894 |
895 | // If the entire written file is smaller than minChunkSize, we need to make
896 | // a new part from scratch :double sad face:
897 | if w.size < minChunkSize {
898 | getObjectInput := &obs.GetObjectInput{}
899 | getObjectInput.Bucket = w.driver.Bucket
900 | getObjectInput.Key = w.key
901 | getObjectOutput, err := w.driver.Client.GetObject(getObjectInput)
902 | if err != nil {
903 | return 0, err
904 | }
905 | defer getObjectOutput.Body.Close()
906 | w.parts = nil
907 | w.readyPart, err = io.ReadAll(getObjectOutput.Body)
908 | if err != nil {
909 | return 0, err
910 | }
911 | } else {
912 | // Otherwise we can use the old file as the new first part
913 | copyPartOutput, err := w.driver.Client.CopyPart(&obs.CopyPartInput{
914 | Bucket: w.driver.Bucket,
915 | Key: w.key,
916 | CopySourceBucket: w.driver.Bucket,
917 | CopySourceKey: w.key,
918 | PartNumber: 1,
919 | UploadId: w.uploadID,
920 | SseHeader: w.driver.getEncryptionMode(),
921 | SourceSseHeader: w.driver.getEncryptionMode(),
922 | })
923 | if err != nil {
924 | return 0, err
925 | }
926 | w.parts = []obs.Part{
927 | {
928 | ETag: copyPartOutput.ETag,
929 | PartNumber: 1,
930 | Size: w.size,
931 | },
932 | }
933 | }
934 | }
935 | for len(p) > 0 {
936 | // If no parts are ready to write, fill up the first part
937 | if neededBytes := int(w.driver.ChunkSize) - len(w.readyPart); neededBytes > 0 {
938 | if len(p) >= neededBytes {
939 | w.readyPart = append(w.readyPart, p[:neededBytes]...)
940 | n += neededBytes
941 | p = p[neededBytes:]
942 | } else {
943 | w.readyPart = append(w.readyPart, p...)
944 | n += len(p)
945 | p = nil
946 | }
947 | }
948 |
949 | if neededBytes := int(w.driver.ChunkSize) - len(w.pendingPart); neededBytes > 0 {
950 | if len(p) >= neededBytes {
951 | w.pendingPart = append(w.pendingPart, p[:neededBytes]...)
952 | n += neededBytes
953 | p = p[neededBytes:]
954 | err := w.flushPart()
955 | if err != nil {
956 | w.size += int64(n)
957 | return n, err
958 | }
959 | } else {
960 | w.pendingPart = append(w.pendingPart, p...)
961 | n += len(p)
962 | p = nil
963 | }
964 | }
965 | }
966 | w.size += int64(n)
967 | return n, nil
968 | }
969 |
970 | // flushPart flushes buffers to write a part to OBS.
971 | // Only called by Write (with both buffers full) and Close/Commit (always)
972 | func (w *writer) flushPart() error {
973 | if len(w.readyPart) == 0 && len(w.pendingPart) == 0 {
974 | // nothing to write
975 | return nil
976 | }
977 | if len(w.pendingPart) < int(w.driver.ChunkSize) {
978 | // closing with a small pending part
979 | // combine ready and pending to avoid writing a small part
980 | w.readyPart = append(w.readyPart, w.pendingPart...)
981 | w.pendingPart = nil
982 | }
983 | ouput, err := w.driver.Client.UploadPart(&obs.UploadPartInput{
984 | Bucket: w.driver.Bucket,
985 | Key: w.key,
986 | PartNumber: len(w.parts) + 1,
987 | UploadId: w.uploadID,
988 | Body: bytes.NewReader(w.readyPart),
989 | SseHeader: w.driver.getEncryptionMode(),
990 | })
991 | if err != nil {
992 | return err
993 | }
994 | w.parts = append(w.parts, obs.Part{
995 | ETag: ouput.ETag,
996 | PartNumber: len(w.parts) + 1,
997 | Size: int64(len(w.readyPart)),
998 | })
999 | w.readyPart = w.pendingPart
1000 | w.pendingPart = nil
1001 | return nil
1002 | }
1003 |
1004 | func (w *writer) Close() error {
1005 | if w.closed {
1006 | return fmt.Errorf("already closed")
1007 | }
1008 | w.closed = true
1009 | return w.flushPart()
1010 | }
1011 |
1012 | func (w *writer) Size() int64 {
1013 | return w.size
1014 | }
1015 |
1016 | func (w *writer) Cancel() error {
1017 | if w.closed {
1018 | return fmt.Errorf("already closed")
1019 | } else if w.committed {
1020 | return fmt.Errorf("already committed")
1021 | }
1022 | w.cancelled = true
1023 | _, err := w.driver.Client.AbortMultipartUpload(&obs.AbortMultipartUploadInput{
1024 | Bucket: w.driver.Bucket,
1025 | Key: w.key,
1026 | UploadId: w.uploadID,
1027 | })
1028 | return err
1029 | }
1030 |
1031 | func (w *writer) Commit() error {
1032 | if w.closed {
1033 | return fmt.Errorf("already closed")
1034 | } else if w.committed {
1035 | return fmt.Errorf("already committed")
1036 | } else if w.cancelled {
1037 | return fmt.Errorf("already cancelled")
1038 | }
1039 | err := w.flushPart()
1040 | if err != nil {
1041 | return err
1042 | }
1043 | w.committed = true
1044 |
1045 | var completedUploadedParts completedParts
1046 | for _, part := range w.parts {
1047 | completedUploadedParts = append(completedUploadedParts, obs.Part{
1048 | ETag: part.ETag,
1049 | PartNumber: part.PartNumber,
1050 | })
1051 | }
1052 |
1053 | sort.Sort(completedUploadedParts)
1054 |
1055 | _, err = w.driver.Client.CompleteMultipartUpload(&obs.CompleteMultipartUploadInput{
1056 | Bucket: w.driver.Bucket,
1057 | Key: w.key,
1058 | UploadId: w.uploadID,
1059 | Parts: completedUploadedParts,
1060 | })
1061 | if err != nil {
1062 | w.driver.Client.AbortMultipartUpload(&obs.AbortMultipartUploadInput{
1063 | Bucket: w.driver.Bucket,
1064 | Key: w.key,
1065 | UploadId: w.uploadID,
1066 | })
1067 | return err
1068 | }
1069 | return nil
1070 | }
1071 |
--------------------------------------------------------------------------------
/storage/driver/oss/doc.go:
--------------------------------------------------------------------------------
1 | // Copy from github.com/docker/distribution/registry/storage/driver/oss
2 |
3 | package oss
4 |
--------------------------------------------------------------------------------
/storage/driver/oss/oss.go:
--------------------------------------------------------------------------------
1 | // Package oss provides a storagedriver.StorageDriver implementation to
2 | // store blobs in Aliyun OSS cloud storage.
3 | //
4 | // This package leverages the denverdino/aliyungo client library for interfacing with
5 | // oss.
6 | //
7 | // Because OSS is a key, value store the Stat call does not support last modification
8 | // time for directories (directories are an abstraction for key, value stores)
9 |
10 | package oss
11 |
12 | import (
13 | "bytes"
14 | "context"
15 | "errors"
16 | "fmt"
17 | "io"
18 | "io/ioutil"
19 | "net/http"
20 | "reflect"
21 | "strconv"
22 | "strings"
23 | "time"
24 |
25 | "github.com/denverdino/aliyungo/oss"
26 | storagedriver "github.com/docker/distribution/registry/storage/driver"
27 | "github.com/docker/distribution/registry/storage/driver/base"
28 | "github.com/docker/distribution/registry/storage/driver/factory"
29 | )
30 |
31 | const driverName = "oss"
32 |
33 | // minChunkSize defines the minimum multipart upload chunk size
34 | // OSS API requires multipart upload chunks to be at least 5MB
35 | const minChunkSize = 5 << 20
36 |
37 | const defaultChunkSize = 2 * minChunkSize
38 | const defaultTimeout = 2 * time.Minute // 2 minute timeout per chunk
39 |
40 | // listMax is the largest amount of objects you can request from OSS in a list call
41 | const listMax = 1000
42 |
43 | // DriverParameters A struct that encapsulates all of the driver parameters after all values have been set
44 | type DriverParameters struct {
45 | AccessKeyID string
46 | AccessKeySecret string
47 | Bucket string
48 | Region oss.Region
49 | Internal bool
50 | Encrypt bool
51 | Secure bool
52 | ChunkSize int64
53 | RootDirectory string
54 | Endpoint string
55 | EncryptionKeyID string
56 | }
57 |
58 | func init() {
59 | factory.Register(driverName, &ossDriverFactory{})
60 | }
61 |
62 | // ossDriverFactory implements the factory.StorageDriverFactory interface
63 | type ossDriverFactory struct{}
64 |
65 | func (factory *ossDriverFactory) Create(parameters map[string]interface{}) (storagedriver.StorageDriver, error) {
66 | return FromParameters(parameters)
67 | }
68 |
69 | type driver struct {
70 | Client *oss.Client
71 | Bucket *oss.Bucket
72 | ChunkSize int64
73 | Encrypt bool
74 | RootDirectory string
75 | EncryptionKeyID string
76 | }
77 |
78 | type baseEmbed struct {
79 | base.Base
80 | }
81 |
82 | // Driver is a storagedriver.StorageDriver implementation backed by Aliyun OSS
83 | // Objects are stored at absolute keys in the provided bucket.
84 | type Driver struct {
85 | baseEmbed
86 | }
87 |
88 | // FromParameters constructs a new Driver with a given parameters map
89 | // Required parameters:
90 | // - accesskey
91 | // - secretkey
92 | // - region
93 | // - bucket
94 | // - encrypt
95 | func FromParameters(parameters map[string]interface{}) (*Driver, error) {
96 | // Providing no values for these is valid in case the user is authenticating
97 |
98 | accessKey, ok := parameters["accesskeyid"]
99 | if !ok {
100 | return nil, fmt.Errorf("No accesskeyid parameter provided")
101 | }
102 | secretKey, ok := parameters["accesskeysecret"]
103 | if !ok {
104 | return nil, fmt.Errorf("No accesskeysecret parameter provided")
105 | }
106 |
107 | regionName, ok := parameters["region"]
108 | if !ok || fmt.Sprint(regionName) == "" {
109 | return nil, fmt.Errorf("No region parameter provided")
110 | }
111 |
112 | bucket, ok := parameters["bucket"]
113 | if !ok || fmt.Sprint(bucket) == "" {
114 | return nil, fmt.Errorf("No bucket parameter provided")
115 | }
116 |
117 | internalBool := false
118 | internal, ok := parameters["internal"]
119 | if ok {
120 | switch b := internal.(type) {
121 | case bool:
122 | internalBool = b
123 | case string:
124 | ib, err := strconv.ParseBool(b)
125 | if err != nil {
126 | return nil, err
127 | }
128 | internalBool = ib
129 | default:
130 | return nil, fmt.Errorf("The internal parameter should be a boolean")
131 | }
132 | }
133 |
134 | encryptBool := false
135 | encrypt, ok := parameters["encrypt"]
136 | if ok {
137 | switch b := encrypt.(type) {
138 | case bool:
139 | encryptBool = b
140 | case string:
141 | ib, err := strconv.ParseBool(b)
142 | if err != nil {
143 | return nil, err
144 | }
145 | encryptBool = ib
146 | default:
147 | return nil, fmt.Errorf("The encrypt parameter should be a boolean")
148 | }
149 | }
150 |
151 | encryptionKeyID, ok := parameters["encryptionkeyid"]
152 | if !ok {
153 | encryptionKeyID = ""
154 | }
155 |
156 | secureBool := true
157 | secure, ok := parameters["secure"]
158 | if ok {
159 | switch b := secure.(type) {
160 | case bool:
161 | secureBool = b
162 | case string:
163 | ib, err := strconv.ParseBool(b)
164 | if err != nil {
165 | return nil, err
166 | }
167 | secureBool = ib
168 | default:
169 | return nil, fmt.Errorf("The secure parameter should be a boolean")
170 | }
171 | }
172 |
173 | chunkSize := int64(defaultChunkSize)
174 | chunkSizeParam, ok := parameters["chunksize"]
175 | if ok {
176 | switch v := chunkSizeParam.(type) {
177 | case string:
178 | vv, err := strconv.ParseInt(v, 0, 64)
179 | if err != nil {
180 | return nil, fmt.Errorf("chunksize parameter must be an integer, %v invalid", chunkSizeParam)
181 | }
182 | chunkSize = vv
183 | case int64:
184 | chunkSize = v
185 | case int, uint, int32, uint32, uint64:
186 | chunkSize = reflect.ValueOf(v).Convert(reflect.TypeOf(chunkSize)).Int()
187 | default:
188 | return nil, fmt.Errorf("invalid valud for chunksize: %#v", chunkSizeParam)
189 | }
190 |
191 | if chunkSize < minChunkSize {
192 | return nil, fmt.Errorf("The chunksize %#v parameter should be a number that is larger than or equal to %d", chunkSize, minChunkSize)
193 | }
194 | }
195 |
196 | rootDirectory, ok := parameters["rootdirectory"]
197 | if !ok {
198 | rootDirectory = ""
199 | }
200 |
201 | endpoint, ok := parameters["endpoint"]
202 | if !ok {
203 | endpoint = ""
204 | }
205 |
206 | params := DriverParameters{
207 | AccessKeyID: fmt.Sprint(accessKey),
208 | AccessKeySecret: fmt.Sprint(secretKey),
209 | Bucket: fmt.Sprint(bucket),
210 | Region: oss.Region(fmt.Sprint(regionName)),
211 | ChunkSize: chunkSize,
212 | RootDirectory: fmt.Sprint(rootDirectory),
213 | Encrypt: encryptBool,
214 | Secure: secureBool,
215 | Internal: internalBool,
216 | Endpoint: fmt.Sprint(endpoint),
217 | EncryptionKeyID: fmt.Sprint(encryptionKeyID),
218 | }
219 |
220 | return New(params)
221 | }
222 |
223 | // New constructs a new Driver with the given Aliyun credentials, region, encryption flag, and
224 | // bucketName
225 | func New(params DriverParameters) (*Driver, error) {
226 |
227 | client := oss.NewOSSClient(params.Region, params.Internal, params.AccessKeyID, params.AccessKeySecret, params.Secure)
228 | client.SetEndpoint(params.Endpoint)
229 | bucket := client.Bucket(params.Bucket)
230 | client.SetDebug(false)
231 |
232 | // Validate that the given credentials have at least read permissions in the
233 | // given bucket scope.
234 | if _, err := bucket.List(strings.TrimRight(params.RootDirectory, "/"), "", "", 1); err != nil {
235 | return nil, err
236 | }
237 |
238 | // TODO(tg123): Currently multipart uploads have no timestamps, so this would be unwise
239 | // if you initiated a new OSS client while another one is running on the same bucket.
240 |
241 | d := &driver{
242 | Client: client,
243 | Bucket: bucket,
244 | ChunkSize: params.ChunkSize,
245 | Encrypt: params.Encrypt,
246 | RootDirectory: params.RootDirectory,
247 | EncryptionKeyID: params.EncryptionKeyID,
248 | }
249 |
250 | return &Driver{
251 | baseEmbed: baseEmbed{
252 | Base: base.Base{
253 | StorageDriver: d,
254 | },
255 | },
256 | }, nil
257 | }
258 |
259 | // Implement the storagedriver.StorageDriver interface
260 |
261 | func (d *driver) Name() string {
262 | return driverName
263 | }
264 |
265 | // GetContent retrieves the content stored at "path" as a []byte.
266 | func (d *driver) GetContent(ctx context.Context, path string) ([]byte, error) {
267 | content, err := d.Bucket.Get(d.ossPath(path))
268 | if err != nil {
269 | return nil, parseError(path, err)
270 | }
271 | return content, nil
272 | }
273 |
274 | // PutContent stores the []byte content at a location designated by "path".
275 | func (d *driver) PutContent(ctx context.Context, path string, contents []byte) error {
276 | return parseError(path, d.Bucket.Put(d.ossPath(path), contents, d.getContentType(), getPermissions(), d.getOptions()))
277 | }
278 |
279 | // Reader retrieves an io.ReadCloser for the content stored at "path" with a
280 | // given byte offset.
281 | func (d *driver) Reader(ctx context.Context, path string, offset int64) (io.ReadCloser, error) {
282 | headers := make(http.Header)
283 | headers.Add("Range", "bytes="+strconv.FormatInt(offset, 10)+"-")
284 |
285 | resp, err := d.Bucket.GetResponseWithHeaders(d.ossPath(path), headers)
286 | if err != nil {
287 | return nil, parseError(path, err)
288 | }
289 |
290 | // Due to Aliyun OSS API, status 200 and whole object will be return instead of an
291 | // InvalidRange error when range is invalid.
292 | //
293 | // OSS sever will always return http.StatusPartialContent if range is acceptable.
294 | if resp.StatusCode != http.StatusPartialContent {
295 | resp.Body.Close()
296 | return ioutil.NopCloser(bytes.NewReader(nil)), nil
297 | }
298 |
299 | return resp.Body, nil
300 | }
301 |
302 | // Writer returns a FileWriter which will store the content written to it
303 | // at the location designated by "path" after the call to Commit.
304 | func (d *driver) Writer(ctx context.Context, path string, append bool) (storagedriver.FileWriter, error) {
305 | key := d.ossPath(path)
306 | if !append {
307 | // TODO (brianbland): cancel other uploads at this path
308 | multi, err := d.Bucket.InitMulti(key, d.getContentType(), getPermissions(), d.getOptions())
309 | if err != nil {
310 | return nil, err
311 | }
312 | return d.newWriter(key, multi, nil), nil
313 | }
314 | multis, _, err := d.Bucket.ListMulti(key, "")
315 | if err != nil {
316 | return nil, parseError(path, err)
317 | }
318 | for _, multi := range multis {
319 | if key != multi.Key {
320 | continue
321 | }
322 | parts, err := multi.ListParts()
323 | if err != nil {
324 | return nil, parseError(path, err)
325 | }
326 | var multiSize int64
327 | for _, part := range parts {
328 | multiSize += part.Size
329 | }
330 | return d.newWriter(key, multi, parts), nil
331 | }
332 | return nil, storagedriver.PathNotFoundError{Path: path}
333 | }
334 |
335 | // Stat retrieves the FileInfo for the given path, including the current size
336 | // in bytes and the creation time.
337 | func (d *driver) Stat(ctx context.Context, path string) (storagedriver.FileInfo, error) {
338 | listResponse, err := d.Bucket.List(d.ossPath(path), "", "", 1)
339 | if err != nil {
340 | return nil, err
341 | }
342 |
343 | fi := storagedriver.FileInfoFields{
344 | Path: path,
345 | }
346 |
347 | if len(listResponse.Contents) == 1 {
348 | if listResponse.Contents[0].Key != d.ossPath(path) {
349 | fi.IsDir = true
350 | } else {
351 | fi.IsDir = false
352 | fi.Size = listResponse.Contents[0].Size
353 |
354 | timestamp, err := time.Parse(time.RFC3339Nano, listResponse.Contents[0].LastModified)
355 | if err != nil {
356 | return nil, err
357 | }
358 | fi.ModTime = timestamp
359 | }
360 | } else if len(listResponse.CommonPrefixes) == 1 {
361 | fi.IsDir = true
362 | } else {
363 | return nil, storagedriver.PathNotFoundError{Path: path}
364 | }
365 |
366 | return storagedriver.FileInfoInternal{FileInfoFields: fi}, nil
367 | }
368 |
369 | // List returns a list of the objects that are direct descendants of the given path.
370 | func (d *driver) List(ctx context.Context, opath string) ([]string, error) {
371 | path := opath
372 | if path != "/" && opath[len(path)-1] != '/' {
373 | path = path + "/"
374 | }
375 |
376 | // This is to cover for the cases when the rootDirectory of the driver is either "" or "/".
377 | // In those cases, there is no root prefix to replace and we must actually add a "/" to all
378 | // results in order to keep them as valid paths as recognized by storagedriver.PathRegexp
379 | prefix := ""
380 | if d.ossPath("") == "" {
381 | prefix = "/"
382 | }
383 |
384 | ossPath := d.ossPath(path)
385 | listResponse, err := d.Bucket.List(ossPath, "/", "", listMax)
386 | if err != nil {
387 | return nil, parseError(opath, err)
388 | }
389 |
390 | files := []string{}
391 | directories := []string{}
392 |
393 | for {
394 | for _, key := range listResponse.Contents {
395 | files = append(files, strings.Replace(key.Key, d.ossPath(""), prefix, 1))
396 | }
397 |
398 | for _, commonPrefix := range listResponse.CommonPrefixes {
399 | directories = append(directories, strings.Replace(commonPrefix[0:len(commonPrefix)-1], d.ossPath(""), prefix, 1))
400 | }
401 |
402 | if listResponse.IsTruncated {
403 | listResponse, err = d.Bucket.List(ossPath, "/", listResponse.NextMarker, listMax)
404 | if err != nil {
405 | return nil, err
406 | }
407 | } else {
408 | break
409 | }
410 | }
411 |
412 | // This is to cover for the cases when the first key equal to ossPath.
413 | if len(files) > 0 && files[0] == strings.Replace(ossPath, d.ossPath(""), prefix, 1) {
414 | files = files[1:]
415 | }
416 |
417 | if opath != "/" {
418 | if len(files) == 0 && len(directories) == 0 {
419 | // Treat empty response as missing directory, since we don't actually
420 | // have directories in s3.
421 | return nil, storagedriver.PathNotFoundError{Path: opath}
422 | }
423 | }
424 |
425 | return append(files, directories...), nil
426 | }
427 |
428 | const maxConcurrency = 10
429 |
430 | // Move moves an object stored at sourcePath to destPath, removing the original
431 | // object.
432 | func (d *driver) Move(ctx context.Context, sourcePath string, destPath string) error {
433 | err := d.Bucket.CopyLargeFileInParallel(d.ossPath(sourcePath), d.ossPath(destPath),
434 | d.getContentType(),
435 | getPermissions(),
436 | d.getOptions(),
437 | maxConcurrency)
438 | if err != nil {
439 | return parseError(sourcePath, err)
440 | }
441 |
442 | return d.Delete(ctx, sourcePath)
443 | }
444 |
445 | // Delete recursively deletes all objects stored at "path" and its subpaths.
446 | func (d *driver) Delete(ctx context.Context, path string) error {
447 | ossPath := d.ossPath(path)
448 | listResponse, err := d.Bucket.List(ossPath, "", "", listMax)
449 | if err != nil || len(listResponse.Contents) == 0 {
450 | return storagedriver.PathNotFoundError{Path: path}
451 | }
452 |
453 | ossObjects := make([]oss.Object, listMax)
454 |
455 | for len(listResponse.Contents) > 0 {
456 | numOssObjects := len(listResponse.Contents)
457 | for index, key := range listResponse.Contents {
458 | // Stop if we encounter a key that is not a subpath (so that deleting "/a" does not delete "/ab").
459 | if len(key.Key) > len(ossPath) && (key.Key)[len(ossPath)] != '/' {
460 | numOssObjects = index
461 | break
462 | }
463 | ossObjects[index].Key = key.Key
464 | }
465 |
466 | err := d.Bucket.DelMulti(oss.Delete{Quiet: false, Objects: ossObjects[0:numOssObjects]})
467 | if err != nil {
468 | return nil
469 | }
470 |
471 | if numOssObjects < len(listResponse.Contents) {
472 | return nil
473 | }
474 |
475 | listResponse, err = d.Bucket.List(d.ossPath(path), "", "", listMax)
476 | if err != nil {
477 | return err
478 | }
479 | }
480 |
481 | return nil
482 | }
483 |
484 | // URLFor returns a URL which may be used to retrieve the content stored at the given path.
485 | // May return an UnsupportedMethodErr in certain StorageDriver implementations.
486 | func (d *driver) URLFor(ctx context.Context, path string, options map[string]interface{}) (string, error) {
487 | methodString := "GET"
488 | method, ok := options["method"]
489 | if ok {
490 | methodString, ok = method.(string)
491 | if !ok || (methodString != "GET") {
492 | return "", storagedriver.ErrUnsupportedMethod{}
493 | }
494 | }
495 |
496 | expiresTime := time.Now().Add(20 * time.Minute)
497 |
498 | expires, ok := options["expiry"]
499 | if ok {
500 | et, ok := expires.(time.Time)
501 | if ok {
502 | expiresTime = et
503 | }
504 | }
505 | signedURL := d.Bucket.SignedURLWithMethod(methodString, d.ossPath(path), expiresTime, nil, nil)
506 | return signedURL, nil
507 | }
508 |
509 | // Walk traverses a filesystem defined within driver, starting
510 | // from the given path, calling f on each file
511 | func (d *driver) Walk(ctx context.Context, path string, f storagedriver.WalkFn) error {
512 | return storagedriver.WalkFallback(ctx, d, path, f)
513 | }
514 |
515 | func (d *driver) ossPath(path string) string {
516 | return strings.TrimLeft(strings.TrimRight(d.RootDirectory, "/")+path, "/")
517 | }
518 |
519 | func parseError(path string, err error) error {
520 | var ossErr *oss.Error
521 | if errors.As(err, &ossErr) && ossErr.StatusCode == http.StatusNotFound && (ossErr.Code == "NoSuchKey" || ossErr.Code == "") {
522 | return storagedriver.PathNotFoundError{Path: path}
523 | }
524 |
525 | return err
526 | }
527 |
528 | func (d *driver) getOptions() oss.Options {
529 | return oss.Options{
530 | ServerSideEncryption: d.Encrypt,
531 | ServerSideEncryptionKeyID: d.EncryptionKeyID,
532 | }
533 | }
534 |
535 | func (d *driver) getCopyOptions() oss.CopyOptions {
536 | return oss.CopyOptions{
537 | ServerSideEncryption: d.Encrypt,
538 | ServerSideEncryptionKeyID: d.EncryptionKeyID,
539 | }
540 | }
541 |
542 | func getPermissions() oss.ACL {
543 | return oss.Private
544 | }
545 |
546 | func (d *driver) getContentType() string {
547 | return "application/octet-stream"
548 | }
549 |
550 | // writer attempts to upload parts to S3 in a buffered fashion where the last
551 | // part is at least as large as the chunksize, so the multipart upload could be
552 | // cleanly resumed in the future. This is violated if Close is called after less
553 | // than a full chunk is written.
554 | type writer struct {
555 | driver *driver
556 | key string
557 | multi *oss.Multi
558 | parts []oss.Part
559 | size int64
560 | readyPart []byte
561 | pendingPart []byte
562 | closed bool
563 | committed bool
564 | cancelled bool
565 | }
566 |
567 | func (d *driver) newWriter(key string, multi *oss.Multi, parts []oss.Part) storagedriver.FileWriter {
568 | var size int64
569 | for _, part := range parts {
570 | size += part.Size
571 | }
572 | return &writer{
573 | driver: d,
574 | key: key,
575 | multi: multi,
576 | parts: parts,
577 | size: size,
578 | }
579 | }
580 |
581 | func (w *writer) Write(p []byte) (int, error) {
582 | if w.closed {
583 | return 0, fmt.Errorf("already closed")
584 | } else if w.committed {
585 | return 0, fmt.Errorf("already committed")
586 | } else if w.cancelled {
587 | return 0, fmt.Errorf("already cancelled")
588 | }
589 |
590 | // If the last written part is smaller than minChunkSize, we need to make a
591 | // new multipart upload :sadface:
592 | if len(w.parts) > 0 && int(w.parts[len(w.parts)-1].Size) < minChunkSize {
593 | err := w.multi.Complete(w.parts)
594 | if err != nil {
595 | w.multi.Abort()
596 | return 0, err
597 | }
598 |
599 | multi, err := w.driver.Bucket.InitMulti(w.key, w.driver.getContentType(), getPermissions(), w.driver.getOptions())
600 | if err != nil {
601 | return 0, err
602 | }
603 | w.multi = multi
604 |
605 | // If the entire written file is smaller than minChunkSize, we need to make
606 | // a new part from scratch :double sad face:
607 | if w.size < minChunkSize {
608 | contents, err := w.driver.Bucket.Get(w.key)
609 | if err != nil {
610 | return 0, err
611 | }
612 | w.parts = nil
613 | w.readyPart = contents
614 | } else {
615 | // Otherwise we can use the old file as the new first part
616 | _, part, err := multi.PutPartCopy(1, w.driver.getCopyOptions(), w.driver.Bucket.Name+"/"+w.key)
617 | if err != nil {
618 | return 0, err
619 | }
620 | w.parts = []oss.Part{part}
621 | }
622 | }
623 |
624 | var n int
625 |
626 | for len(p) > 0 {
627 | // If no parts are ready to write, fill up the first part
628 | if neededBytes := int(w.driver.ChunkSize) - len(w.readyPart); neededBytes > 0 {
629 | if len(p) >= neededBytes {
630 | w.readyPart = append(w.readyPart, p[:neededBytes]...)
631 | n += neededBytes
632 | p = p[neededBytes:]
633 | } else {
634 | w.readyPart = append(w.readyPart, p...)
635 | n += len(p)
636 | p = nil
637 | }
638 | }
639 |
640 | if neededBytes := int(w.driver.ChunkSize) - len(w.pendingPart); neededBytes > 0 {
641 | if len(p) >= neededBytes {
642 | w.pendingPart = append(w.pendingPart, p[:neededBytes]...)
643 | n += neededBytes
644 | p = p[neededBytes:]
645 | err := w.flushPart()
646 | if err != nil {
647 | w.size += int64(n)
648 | return n, err
649 | }
650 | } else {
651 | w.pendingPart = append(w.pendingPart, p...)
652 | n += len(p)
653 | p = nil
654 | }
655 | }
656 | }
657 | w.size += int64(n)
658 | return n, nil
659 | }
660 |
661 | func (w *writer) Size() int64 {
662 | return w.size
663 | }
664 |
665 | func (w *writer) Close() error {
666 | if w.closed {
667 | return fmt.Errorf("already closed")
668 | }
669 | w.closed = true
670 | return w.flushPart()
671 | }
672 |
673 | func (w *writer) Cancel() error {
674 | if w.closed {
675 | return fmt.Errorf("already closed")
676 | } else if w.committed {
677 | return fmt.Errorf("already committed")
678 | }
679 | w.cancelled = true
680 | err := w.multi.Abort()
681 | return err
682 | }
683 |
684 | func (w *writer) Commit() error {
685 | if w.closed {
686 | return fmt.Errorf("already closed")
687 | } else if w.committed {
688 | return fmt.Errorf("already committed")
689 | } else if w.cancelled {
690 | return fmt.Errorf("already cancelled")
691 | }
692 | err := w.flushPart()
693 | if err != nil {
694 | return err
695 | }
696 | w.committed = true
697 | err = w.multi.Complete(w.parts)
698 | if err != nil {
699 | w.multi.Abort()
700 | return err
701 | }
702 | return nil
703 | }
704 |
705 | // flushPart flushes buffers to write a part to S3.
706 | // Only called by Write (with both buffers full) and Close/Commit (always)
707 | func (w *writer) flushPart() error {
708 | if len(w.readyPart) == 0 && len(w.pendingPart) == 0 {
709 | // nothing to write
710 | return nil
711 | }
712 | if len(w.pendingPart) < int(w.driver.ChunkSize) {
713 | // closing with a small pending part
714 | // combine ready and pending to avoid writing a small part
715 | w.readyPart = append(w.readyPart, w.pendingPart...)
716 | w.pendingPart = nil
717 | }
718 |
719 | part, err := w.multi.PutPart(len(w.parts)+1, bytes.NewReader(w.readyPart))
720 | if err != nil {
721 | return err
722 | }
723 | w.parts = append(w.parts, part)
724 | w.readyPart = w.pendingPart
725 | w.pendingPart = nil
726 | return nil
727 | }
728 |
--------------------------------------------------------------------------------
/sync.go:
--------------------------------------------------------------------------------
1 | package crproxy
2 |
3 | import (
4 | "context"
5 | "crypto/sha256"
6 | "encoding/hex"
7 | "encoding/json"
8 | "fmt"
9 | "io"
10 | "net/http"
11 |
12 | "github.com/distribution/reference"
13 | "github.com/docker/distribution"
14 | "github.com/docker/distribution/manifest/manifestlist"
15 | "github.com/docker/distribution/manifest/ocischema"
16 | "github.com/docker/distribution/manifest/schema1"
17 | "github.com/docker/distribution/manifest/schema2"
18 | "github.com/docker/distribution/registry/api/errcode"
19 | "github.com/docker/distribution/registry/client"
20 | "github.com/opencontainers/go-digest"
21 | )
22 |
23 | type namedWithoutDomain struct {
24 | reference.Reference
25 | name string
26 | }
27 |
28 | func (n namedWithoutDomain) Name() string {
29 | return n.name
30 | }
31 |
32 | func newNameWithoutDomain(named reference.Named, name string) reference.Named {
33 | return namedWithoutDomain{
34 | Reference: named,
35 | name: name,
36 | }
37 | }
38 |
39 | func (c *CRProxy) Sync(rw http.ResponseWriter, r *http.Request) {
40 | if r.Method != http.MethodPut || c.storageDriver == nil {
41 | errcode.ServeJSON(rw, errcode.ErrorCodeUnsupported)
42 | return
43 | }
44 |
45 | query := r.URL.Query()
46 |
47 | rw.Header().Set("Content-Type", "application/json")
48 |
49 | images := query["image"]
50 |
51 | flusher, _ := rw.(http.Flusher)
52 |
53 | encoder := json.NewEncoder(rw)
54 | for _, image := range images {
55 | if image == "" {
56 | continue
57 | }
58 | err := c.SyncImageLayer(r.Context(), r.RemoteAddr, image, nil, func(sp SyncProgress) error {
59 | err := encoder.Encode(sp)
60 | if err != nil {
61 | return err
62 | }
63 |
64 | if flusher != nil {
65 | flusher.Flush()
66 | }
67 |
68 | return nil
69 | })
70 | if err != nil {
71 | c.errorResponse(rw, r, err)
72 | return
73 | }
74 | }
75 | }
76 |
77 | type SyncProgress struct {
78 | Digest string `json:"digest,omitempty"`
79 | Size int64 `json:"size,omitempty"`
80 | Status string `json:"status,omitempty"`
81 | Platform *manifestlist.PlatformSpec `json:"platform,omitempty"`
82 | }
83 |
84 | func (c *CRProxy) SyncImageLayer(ctx context.Context, ip, image string, filter func(pf manifestlist.PlatformSpec) bool, cb func(sp SyncProgress) error) error {
85 | ref, err := reference.Parse(image)
86 | if err != nil {
87 | return err
88 | }
89 |
90 | named, ok := ref.(reference.Named)
91 | if !ok {
92 | return fmt.Errorf("%s is not a name", ref)
93 | }
94 |
95 | host := reference.Domain(named)
96 |
97 | var name reference.Named
98 |
99 | info := &ImageInfo{
100 | Host: host,
101 | Name: reference.Path(named),
102 | }
103 | if c.modify != nil {
104 | info = c.modify(info)
105 | name = newNameWithoutDomain(named, info.Name)
106 | } else {
107 | name = newNameWithoutDomain(named, info.Name)
108 | }
109 |
110 | if c.blockFunc != nil {
111 | blockMessage, block := c.block(&BlockInfo{
112 | IP: ip,
113 | Host: info.Host,
114 | Name: info.Name,
115 | })
116 | if block {
117 | if blockMessage != "" {
118 | return errcode.ErrorCodeDenied.WithMessage(blockMessage)
119 | } else {
120 | return errcode.ErrorCodeDenied
121 | }
122 | }
123 | }
124 |
125 | host = c.getDomainAlias(host)
126 | info.Host = host
127 |
128 | err = c.ping(host)
129 | if err != nil {
130 | return err
131 | }
132 |
133 | cli := c.getClientset(host, name.Name())
134 |
135 | repo, err := client.NewRepository(name, c.hostURL(host), cli.Transport)
136 | if err != nil {
137 | return err
138 | }
139 |
140 | ms, err := repo.Manifests(ctx)
141 | if err != nil {
142 | return err
143 | }
144 |
145 | bs := repo.Blobs(ctx)
146 |
147 | buf := c.bytesPool.Get().([]byte)
148 | defer c.bytesPool.Put(buf)
149 |
150 | uniq := map[digest.Digest]struct{}{}
151 | blobCallback := func(dgst digest.Digest, size int64, pf *manifestlist.PlatformSpec) error {
152 | _, ok := uniq[dgst]
153 | if ok {
154 | if cb != nil {
155 | err = cb(SyncProgress{
156 | Digest: dgst.String(),
157 | Size: size,
158 | Status: "SKIP",
159 | Platform: pf,
160 | })
161 | if err != nil {
162 | return err
163 | }
164 | }
165 | return nil
166 | }
167 | uniq[dgst] = struct{}{}
168 |
169 | blobPath := blobCachePath(dgst.String())
170 | stat, err := c.storageDriver.Stat(ctx, blobPath)
171 | if err == nil {
172 | if size > 0 {
173 | gotSize := stat.Size()
174 | if size == gotSize {
175 | if c.logger != nil {
176 | c.logger.Println("skip blob", dgst)
177 | }
178 |
179 | if cb != nil {
180 | err = cb(SyncProgress{
181 | Digest: dgst.String(),
182 | Size: size,
183 | Status: "SKIP",
184 | Platform: pf,
185 | })
186 | if err != nil {
187 | return err
188 | }
189 | }
190 | return nil
191 | }
192 | if c.logger != nil {
193 | c.logger.Println("size is not meeting expectations", dgst, size, gotSize)
194 | }
195 | } else {
196 | if c.logger != nil {
197 | c.logger.Println("skip blob", dgst)
198 | }
199 | if cb != nil {
200 | err = cb(SyncProgress{
201 | Digest: dgst.String(),
202 | Size: -1,
203 | Status: "SKIP",
204 | Platform: pf,
205 | })
206 | if err != nil {
207 | return err
208 | }
209 | }
210 | return nil
211 | }
212 | }
213 |
214 | f, err := bs.Open(ctx, dgst)
215 | if err != nil {
216 | return err
217 | }
218 | defer f.Close()
219 |
220 | fw, err := c.storageDriver.Writer(ctx, blobPath, false)
221 | if err != nil {
222 | return err
223 | }
224 |
225 | h := sha256.New()
226 | n, err := io.CopyBuffer(fw, io.TeeReader(f, h), buf)
227 | if err != nil {
228 | fw.Cancel()
229 | return err
230 | }
231 | if wantSize := fw.Size(); n != wantSize {
232 | fw.Cancel()
233 | return fmt.Errorf("expected %d bytes, got %d", wantSize, n)
234 | }
235 |
236 | hash := hex.EncodeToString(h.Sum(nil)[:])
237 | if hex := dgst.Hex(); hex != hash {
238 | fw.Cancel()
239 | return fmt.Errorf("expected %s hash, got %s", hex, hash)
240 | }
241 |
242 | err = fw.Commit()
243 | if err != nil {
244 | return err
245 | }
246 |
247 | if c.logger != nil {
248 | c.logger.Println("sync blob", dgst)
249 | }
250 |
251 | if cb != nil {
252 | err = cb(SyncProgress{
253 | Digest: dgst.String(),
254 | Size: n,
255 | Status: "CACHE",
256 | Platform: pf,
257 | })
258 | if err != nil {
259 | return err
260 | }
261 | }
262 | return nil
263 | }
264 |
265 | manifestCallback := func(tagOrHash string, m distribution.Manifest) error {
266 | _, playload, err := m.Payload()
267 | if err != nil {
268 | return err
269 | }
270 | return c.cacheManifestContent(ctx, &PathInfo{
271 | Host: info.Host,
272 | Image: info.Name,
273 | Manifests: tagOrHash,
274 | }, playload)
275 | }
276 |
277 | err = getLayerFromManifestList(ctx, ms, ref, filter, blobCallback, manifestCallback)
278 | if err != nil {
279 | return err
280 | }
281 | return nil
282 | }
283 |
284 | func getLayerFromManifestList(ctx context.Context, ms distribution.ManifestService, ref reference.Reference, filter func(pf manifestlist.PlatformSpec) bool,
285 | digestCallback func(dgst digest.Digest, size int64, pf *manifestlist.PlatformSpec) error,
286 | manifestCallback func(tagOrHash string, m distribution.Manifest) error) error {
287 | var (
288 | m distribution.Manifest
289 | err error
290 | )
291 | switch r := ref.(type) {
292 | case reference.Digested:
293 | m, err = ms.Get(ctx, r.Digest())
294 | if err != nil {
295 | return err
296 | }
297 | err = manifestCallback(r.Digest().String(), m)
298 | if err != nil {
299 | return err
300 | }
301 | case reference.Tagged:
302 | tag := r.Tag()
303 | m, err = ms.Get(ctx, "", distribution.WithTag(r.Tag()))
304 | if err != nil {
305 | return err
306 | }
307 | err = manifestCallback(tag, m)
308 | if err != nil {
309 | return err
310 | }
311 | default:
312 | return fmt.Errorf("%s no reference to any source", ref)
313 | }
314 |
315 | switch m := m.(type) {
316 | case *manifestlist.DeserializedManifestList:
317 | for _, mfest := range m.ManifestList.Manifests {
318 | if filter != nil && !filter(mfest.Platform) {
319 | continue
320 | }
321 |
322 | m0, err := ms.Get(ctx, mfest.Digest)
323 | if err != nil {
324 | return err
325 | }
326 | err = manifestCallback(mfest.Digest.String(), m0)
327 | if err != nil {
328 | return err
329 | }
330 | err = getLayerFromManifest(m0, func(dgst digest.Digest, size int64) error {
331 | return digestCallback(dgst, size, &mfest.Platform)
332 | })
333 | if err != nil {
334 | return err
335 | }
336 | }
337 | return nil
338 | default:
339 | return getLayerFromManifest(m, func(dgst digest.Digest, size int64) error {
340 | return digestCallback(dgst, size, nil)
341 | })
342 | }
343 | }
344 |
345 | func getLayerFromManifest(m distribution.Manifest, cb func(dgst digest.Digest, size int64) error) error {
346 | switch m := m.(type) {
347 | case *ocischema.DeserializedManifest:
348 | for _, layer := range m.Layers {
349 | if layer.Size == 0 {
350 | continue
351 | }
352 | err := cb(layer.Digest, layer.Size)
353 | if err != nil {
354 | return err
355 | }
356 | }
357 | case *schema2.DeserializedManifest:
358 | for _, layer := range m.Layers {
359 | if layer.Size == 0 {
360 | continue
361 | }
362 | err := cb(layer.Digest, layer.Size)
363 | if err != nil {
364 | return err
365 | }
366 | }
367 | case *schema1.SignedManifest:
368 | for _, layer := range m.FSLayers {
369 | err := cb(layer.BlobSum, -1)
370 | if err != nil {
371 | return err
372 | }
373 | }
374 | }
375 | return nil
376 | }
377 |
--------------------------------------------------------------------------------
/utils.go:
--------------------------------------------------------------------------------
1 | package crproxy
2 |
3 | import (
4 | "fmt"
5 | "io"
6 | "net/http"
7 | "strings"
8 | )
9 |
10 | func addPrefixToImageForPagination(oldLink string, host string) string {
11 | linkAndRel := strings.SplitN(oldLink, ";", 2)
12 | if len(linkAndRel) != 2 {
13 | return oldLink
14 | }
15 | linkURL := strings.SplitN(strings.Trim(linkAndRel[0], "<>"), "/v2/", 2)
16 | if len(linkURL) != 2 {
17 | return oldLink
18 | }
19 | mirrorPath := prefix + host + "/" + linkURL[1]
20 | return fmt.Sprintf("<%s>;%s", mirrorPath, linkAndRel[1])
21 | }
22 |
23 | type PathInfo struct {
24 | Host string
25 | Image string
26 |
27 | TagsList bool
28 | Manifests string
29 | Blobs string
30 | }
31 |
32 | func (p PathInfo) Path() (string, error) {
33 | if p.TagsList {
34 | return prefix + p.Image + "/tags/list", nil
35 | }
36 | if p.Manifests != "" {
37 | return prefix + p.Image + "/manifests/" + p.Manifests, nil
38 | }
39 | if p.Blobs != "" {
40 | return prefix + p.Image + "/blobs/" + p.Blobs, nil
41 | }
42 | return "", fmt.Errorf("unknow kind %#v", p)
43 | }
44 |
45 | func ParseOriginPathInfo(path string, defaultRegistry string) (*PathInfo, bool) {
46 | path = strings.TrimPrefix(path, prefix)
47 | i := strings.IndexByte(path, '/')
48 | if i <= 0 {
49 | return nil, false
50 | }
51 | host := path[:i]
52 | tail := path[i+1:]
53 |
54 | var tails = []string{}
55 | var image = ""
56 |
57 | if !isDomainName(host) || !strings.Contains(host, ".") {
58 | // disable while non default registry seted.
59 | if defaultRegistry == "" {
60 | return nil, false
61 | }
62 | // if host is not a domain name, it is a image.
63 | tails = strings.Split(tail, "/")
64 | if len(tails) < 2 {
65 | // should be more then 2 parts. like /manifests/latest
66 | return nil, false
67 | }
68 | image = strings.Join(tails[:len(tails)-2], "/")
69 | if image == "" {
70 | // the url looks like /v2/[busybox]/manifests/latest.
71 | image = host
72 | } else {
73 | // the url looks like /v2/[pytorch/pytorch/...]/[manifests/latest].
74 | image = host + "/" + image
75 | }
76 | host = defaultRegistry
77 | } else {
78 |
79 | tails = strings.Split(tail, "/")
80 | if len(tails) < 3 {
81 | return nil, false
82 | }
83 | image = strings.Join(tails[:len(tails)-2], "/")
84 | if image == "" {
85 | return nil, false
86 | }
87 | }
88 |
89 | info := &PathInfo{
90 | Host: host,
91 | Image: image,
92 | }
93 | switch tails[len(tails)-2] {
94 | case "tags":
95 | info.TagsList = tails[len(tails)-1] == "list"
96 | case "manifests":
97 | info.Manifests = tails[len(tails)-1]
98 | case "blobs":
99 | info.Blobs = tails[len(tails)-1]
100 | if len(info.Blobs) != 7+64 {
101 | return nil, false
102 | }
103 | }
104 | return info, true
105 | }
106 |
107 | // isDomainName checks if a string is a presentation-format domain name
108 | // (currently restricted to hostname-compatible "preferred name" LDH labels and
109 | // SRV-like "underscore labels"; see golang.org/issue/12421).
110 | func isDomainName(s string) bool {
111 | // See RFC 1035, RFC 3696.
112 | // Presentation format has dots before every label except the first, and the
113 | // terminal empty label is optional here because we assume fully-qualified
114 | // (absolute) input. We must therefore reserve space for the first and last
115 | // labels' length octets in wire format, where they are necessary and the
116 | // maximum total length is 255.
117 | // So our _effective_ maximum is 253, but 254 is not rejected if the last
118 | // character is a dot.
119 | l := len(s)
120 | if l == 0 || l > 254 || l == 254 && s[l-1] != '.' {
121 | return false
122 | }
123 |
124 | last := byte('.')
125 | nonNumeric := false // true once we've seen a letter or hyphen
126 | partlen := 0
127 | for i := 0; i < len(s); i++ {
128 | c := s[i]
129 | switch {
130 | default:
131 | return false
132 | case 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' || c == '_':
133 | nonNumeric = true
134 | partlen++
135 | case '0' <= c && c <= '9':
136 | // fine
137 | partlen++
138 | case c == '-':
139 | // Byte before dash cannot be dot.
140 | if last == '.' {
141 | return false
142 | }
143 | partlen++
144 | nonNumeric = true
145 | case c == '.':
146 | // Byte before dot cannot be dot, dash.
147 | if last == '.' || last == '-' {
148 | return false
149 | }
150 | if partlen > 63 || partlen == 0 {
151 | return false
152 | }
153 | partlen = 0
154 | }
155 | last = c
156 | }
157 | if last == '-' || partlen > 63 {
158 | return false
159 | }
160 |
161 | return nonNumeric
162 | }
163 |
164 | func dumpResponse(resp *http.Response) string {
165 | body, _ := io.ReadAll(io.LimitReader(resp.Body, 100))
166 | return fmt.Sprintf("%d %d %q", resp.StatusCode, resp.ContentLength, string(body))
167 | }
168 |
--------------------------------------------------------------------------------
/utils_test.go:
--------------------------------------------------------------------------------
1 | package crproxy
2 |
3 | import (
4 | "reflect"
5 | "testing"
6 | )
7 |
8 | func TestParseOriginPathInfo(t *testing.T) {
9 |
10 | testDefaultRegistry := "non_docker.io"
11 |
12 | type args struct {
13 | path string
14 | }
15 | tests := []struct {
16 | name string
17 | args args
18 | defaultRegistry string
19 | want *PathInfo
20 | wantOk bool
21 | }{
22 | {
23 | args: args{
24 | path: "/v2/busybox/manifests/1",
25 | },
26 | defaultRegistry: testDefaultRegistry,
27 | want: &PathInfo{
28 | Host: testDefaultRegistry,
29 | Image: "busybox",
30 | Manifests: "1",
31 | },
32 | wantOk: true,
33 | },
34 | {
35 | args: args{
36 | path: "/v2/pytorch/pytorch/manifests/1",
37 | },
38 | defaultRegistry: testDefaultRegistry,
39 | want: &PathInfo{
40 | Host: testDefaultRegistry,
41 | Image: "pytorch/pytorch",
42 | Manifests: "1",
43 | },
44 | wantOk: true,
45 | },
46 | {
47 | args: args{
48 | path: "/v2/v2/manifests/latest",
49 | },
50 | defaultRegistry: testDefaultRegistry,
51 | want: &PathInfo{
52 | Host: testDefaultRegistry,
53 | Image: "v2",
54 | Manifests: "latest",
55 | },
56 | wantOk: true,
57 | },
58 | {
59 | args: args{
60 | path: "/v2/docker.io/busybox/manifests/1",
61 | },
62 | want: &PathInfo{
63 | Host: "docker.io",
64 | Image: "busybox",
65 | Manifests: "1",
66 | },
67 | wantOk: true,
68 | },
69 | {
70 | args: args{
71 | path: "/v2/docker.io/library/busybox/manifests/1",
72 | },
73 | want: &PathInfo{
74 | Host: "docker.io",
75 | Image: "library/busybox",
76 | Manifests: "1",
77 | },
78 | wantOk: true,
79 | },
80 | }
81 | for _, tt := range tests {
82 | t.Run(tt.name, func(t *testing.T) {
83 | got, gotOk := ParseOriginPathInfo(tt.args.path, tt.defaultRegistry)
84 | if !reflect.DeepEqual(got, tt.want) {
85 | t.Errorf("ParseOriginPathInfo() got = %v, want %v", got, tt.want)
86 | }
87 | if gotOk != tt.wantOk {
88 | t.Errorf("ParseOriginPathInfo() gotOk = %v, want %v", gotOk, tt.wantOk)
89 | }
90 | })
91 | }
92 | }
93 |
94 | func Test_addPrefixToImageForPagination(t *testing.T) {
95 | type args struct {
96 | oldLink string
97 | host string
98 | }
99 | tests := []struct {
100 | name string
101 | args args
102 | want string
103 | }{
104 | {
105 | args: args{
106 | oldLink: "; ref=other",
107 | host: "prefix",
108 | },
109 | want: "; ref=other",
110 | },
111 | {
112 | args: args{
113 | oldLink: "; ref=other",
114 | host: "prefix",
115 | },
116 | want: "; ref=other",
117 | },
118 | }
119 | for _, tt := range tests {
120 | t.Run(tt.name, func(t *testing.T) {
121 | if got := addPrefixToImageForPagination(tt.args.oldLink, tt.args.host); got != tt.want {
122 | t.Errorf("addPrefixToImageForPagination() = %v, want %v", got, tt.want)
123 | }
124 | })
125 | }
126 | }
127 |
--------------------------------------------------------------------------------