├── README.md ├── cmd ├── check.go ├── pull.go └── root.go ├── go.mod ├── go.sum ├── main.go ├── registry ├── image.go ├── manifest.go └── tokentransport.go └── scripts └── build.sh /README.md: -------------------------------------------------------------------------------- 1 | ## demo 2 | 3 | 推荐去使用[skeopeo](https://github.com/containers/skopeo)，此项目已经不维护和造轮子，如果有兴趣可以借鉴下思路，azk8s.cn已经失效，如果借鉴过程可以去掉azk8s的代理 4 | 5 | 6 | 7 | ## Usage 8 | 9 | ### check 10 | 11 | Check if the images belongs to scheme2.Manifest 12 | ``` 13 | $ dp c dduportal/bats:0.4.0 14 | scheme2.Manifest: [] 15 | scheme1.Manifest: [dduportal/bats:0.4.0] 16 | $ dp c dduportal/bats:0.4.0 nginx:alpine --only 17 | scheme2.Manifest: [nginx:alpine] 18 | ``` 19 | 20 | ### pull 21 | 22 | Pull the docker images on a machine without docker, and support pulling images from multiple registry at same time 23 | ``` 24 | $ dp pull 25 | pull all images and write to a tar.gz file without docker daemon. 26 | 27 | Usage: 28 | dp pull [flags] 29 | 30 | Aliases: 31 | pull, p 32 | 33 | Examples: 34 | 35 | # pull a image or set the name to save 36 | dp pull nginx:alpine 37 | dp pull -o nginx.tar.gz nginx:alpine 38 | 39 | # pull image use sha256 40 | dp pull mcr.microsoft.com/windows/nanoserver@sha256:ae443bd9609b9ef06d21d6caab59505cb78f24a725cc24716d4427e36aedabf2 41 | 42 | # pull images and set the name to save 43 | dp pull -o project.tar.gz nginx:alpine nginx:1.17.5-alpine-perl 44 | 45 | # pull from different registry 46 | dp pull -o project.tar.gz nginx:alpine gcr.azk8s.cn/google_containers/pause-amd64:3.1 47 | 48 | 49 | Flags: 50 | -h, --help help for pull 51 | -o, --out-file string the name will write to 52 | ``` 53 | ## attention 54 | 55 | Many of the images on quay.io are still scheme1.manifest, and some of the mirror images of some other domain names are long ago. These images will not be successfully pulled. 56 | 57 | ## todo 58 | 59 | - could retry while failed 60 | - multi process download 61 | - with a nice download progress bar 62 | - support quay.io and harbor(hard for quay.io!!) 63 | -------------------------------------------------------------------------------- /cmd/check.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "dp/registry" 5 | "fmt" 6 | "github.com/spf13/cobra" 7 | ) 8 | 9 | var ( 10 | //strict bool 11 | only bool 12 | ) 13 | var checkCmd = &cobra.Command{ 14 | Use: "check", 15 | Aliases: []string{"c"}, 16 | Short: "Check if the images belongs to scheme2.Manifest", 17 | 18 | Example: ` 19 | dp c gcr.io/google_containers/bustbox 20 | 21 | dp c --only nginx:alpine 22 | 23 | dp check nginx:alpine gcr.io/google_containers/pause-amd64:3.1 24 | 25 | `, 26 | Run: func(cmd *cobra.Command, args []string) { 27 | if len(args) == 0 { 28 | _ = cmd.Help() 29 | return 30 | } 31 | var ( 32 | v2 = make([]string, 0) 33 | v1 = make([]string, 0) 34 | 35 | ) 36 | for _, name := range args { 37 | p := registry.NewPull(name) 38 | manifest, _ := p.Manifests() 39 | if manifest == nil || manifest.SchemaVersion != 2 { 40 | v1 = append(v1, name) 41 | } else { 42 | v2 = append(v2, name) 43 | } 44 | } 45 | fmt.Printf("scheme2.Manifest: %v\n", v2) 46 | if !only { 47 | fmt.Printf("scheme1.Manifest: %v\n", v1) 48 | } 49 | }, 50 | } 51 | 52 | func init() { 53 | rootCmd.AddCommand(checkCmd) 54 | checkCmd.Flags().BoolVarP(&only, "only", "o", false, "only print which is scheme2.Manifest") 55 | } 56 | -------------------------------------------------------------------------------- /cmd/pull.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "dp/registry" 5 | "fmt" 6 | "github.com/spf13/cobra" 7 | "log" 8 | "os" 9 | "strings" 10 | "time" 11 | ) 12 | 13 | var ( 14 | //strict bool 15 | saveName string 16 | ) 17 | var pullCmd = &cobra.Command{ 18 | Use: "pull", 19 | Aliases: []string{"p"}, 20 | Short: "pull images", 21 | Long: ` 22 | pull all images and write to a tar.gz file without docker daemon.`, 23 | Example: ` 24 | # pull a image or set the name to save 25 | dp pull nginx:alpine 26 | dp pull -o nginx.tar.gz nginx:alpine 27 | 28 | # pull image use sha256 29 | dp pull mcr.microsoft.com/windows/nanoserver@sha256:ae443bd9609b9ef06d21d6caab59505cb78f24a725cc24716d4427e36aedabf2 30 | 31 | # pull images and set the name to save 32 | dp pull -o project.tar.gz nginx:alpine nginx:1.17.5-alpine-perl 33 | 34 | # pull from different registry 35 | dp pull -o project.tar.gz nginx:alpine gcr.io/google_containers/pause-amd64:3.1 36 | `, 37 | Run: func(cmd *cobra.Command, args []string) { 38 | if len(args) == 0 { 39 | _ = cmd.Help() 40 | return 41 | } 42 | if len(args) == 1 && saveName == "" { 43 | saveName = strings.ReplaceAll(args[0], "/", "_") 44 | saveName = fmt.Sprintf("%s.tar.gz", strings.Replace(saveName, ":", "@", 1)) 45 | } 46 | // todo regex check 47 | //for _, name := range args { 48 | //https://github.com/docker/distribution/blob/master/reference/regexp.go 49 | //} 50 | if saveName == "" { 51 | saveName = fmt.Sprintf("%s.tar.gz", time.Now().Format("2006-1-2-15:04:05")) 52 | } 53 | if err := registry.Save(args, saveName);err != nil { 54 | _ = os.Remove(saveName) 55 | log.Fatal("Save failed: ", err) 56 | } 57 | log.Printf("Successfully written to file %s", saveName) 58 | 59 | }, 60 | } 61 | 62 | func init() { 63 | rootCmd.AddCommand(pullCmd) 64 | //cpCmd.Flags().BoolVarP(&strict, "strict-mode", "s", false, 65 | // "The image name of the pull is strictly checked. If it is wrong, it will not be pulled.") 66 | pullCmd.Flags().StringVarP(&saveName, "out-file", "o", "", "the name will write to,default use timeformat") 67 | } 68 | -------------------------------------------------------------------------------- /cmd/root.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "github.com/spf13/cobra" 6 | "os" 7 | ) 8 | 9 | 10 | var rootCmd = &cobra.Command{ 11 | Use: "dp", 12 | Short: "a docker images tool which without docker daemon", 13 | Long: ` 14 | could pull image without docker daemon ath ths all os.`, 15 | //Run: func(cmd *cobra.Command, args []string) { 16 | // 17 | //}, 18 | } 19 | 20 | // Execute adds all child commands to the root command and sets flags appropriately. 21 | // This is called by main.main(). It only needs to happen once to the rootCmd. 22 | func Execute() { 23 | if err := rootCmd.Execute(); err != nil { 24 | fmt.Println(err) 25 | os.Exit(1) 26 | } 27 | } 28 | 29 | 30 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module dp 2 | 3 | go 1.13 4 | 5 | require ( 6 | github.com/docker/distribution v2.7.1+incompatible 7 | github.com/opencontainers/go-digest v1.0.0-rc1 8 | github.com/opencontainers/image-spec v1.0.1 // indirect 9 | github.com/spf13/cobra v0.0.5 10 | ) 11 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 2 | github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= 3 | github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= 4 | github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= 5 | github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= 6 | github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= 7 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 8 | github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= 9 | github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= 10 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= 11 | github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= 12 | github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= 13 | github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= 14 | github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= 15 | github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= 16 | github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= 17 | github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ= 18 | github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= 19 | github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI= 20 | github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= 21 | github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= 22 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 23 | github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= 24 | github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= 25 | github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= 26 | github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s= 27 | github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= 28 | github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= 29 | github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= 30 | github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= 31 | github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= 32 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 33 | github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= 34 | github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= 35 | golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 36 | golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 37 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 38 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 39 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 40 | 41 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "dp/cmd" 5 | ) 6 | 7 | 8 | func main() { 9 | 10 | cmd.Execute() 11 | 12 | } 13 | 14 | -------------------------------------------------------------------------------- /registry/image.go: -------------------------------------------------------------------------------- 1 | package registry 2 | 3 | import "time" 4 | 5 | //https://github.com/moby/moby/blob/master/image/image.go 6 | //https://github.com/moby/moby/blob/master/api/types/container/config.go 7 | 8 | 9 | type ImageConfig struct { 10 | Architecture string `json:"architecture"` 11 | Config struct { 12 | AttachStderr bool `json:"AttachStderr"` 13 | AttachStdin bool `json:"AttachStdin"` 14 | AttachStdout bool `json:"AttachStdout"` 15 | OpenStdin bool `json:"OpenStdin"` 16 | StdinOnce bool `json:"StdinOnce"` 17 | Tty bool `json:"Tty"` 18 | Cmd []string `json:"Cmd"` 19 | Domainname string `json:"Domainname"` 20 | Entrypoint interface{} `json:"Entrypoint"` 21 | Env []string `json:"Env"` 22 | Hostname string `json:"Hostname"` 23 | Image string `json:"Image"` 24 | Labels struct { 25 | } `json:"Labels"` 26 | OnBuild []interface{} `json:"OnBuild"` 27 | User string `json:"User"` 28 | Volumes interface{} `json:"Volumes"` 29 | WorkingDir string `json:"WorkingDir"` 30 | } `json:"config"` 31 | Container string `json:"container"` 32 | ContainerConfig struct { 33 | AttachStderr bool `json:"AttachStderr"` 34 | AttachStdin bool `json:"AttachStdin"` 35 | AttachStdout bool `json:"AttachStdout"` 36 | OpenStdin bool `json:"OpenStdin"` 37 | StdinOnce bool `json:"StdinOnce"` 38 | Tty bool `json:"Tty"` 39 | Cmd []string `json:"Cmd"` 40 | Domainname string `json:"Domainname"` 41 | Entrypoint interface{} `json:"Entrypoint"` 42 | Env []string `json:"Env"` 43 | Hostname string `json:"Hostname"` 44 | Image string `json:"Image"` 45 | Labels struct { 46 | } `json:"Labels"` 47 | OnBuild []interface{} `json:"OnBuild"` 48 | User string `json:"User"` 49 | Volumes interface{} `json:"Volumes"` 50 | WorkingDir string `json:"WorkingDir"` 51 | } `json:"container_config"` 52 | Created time.Time `json:"created"` 53 | DockerVersion string `json:"docker_version"` 54 | Os string `json:"os"` 55 | //History []struct { 56 | // Created time.Time `json:"created"` 57 | // CreatedBy string `json:"created_by"` 58 | // EmptyLayer bool `json:"empty_layer,omitempty"` 59 | //} `json:"history"` 60 | //Rootfs struct { 61 | // DiffIds []string `json:"diff_ids"` 62 | // Type string `json:"type"` 63 | //} `json:"rootfs"` 64 | ID string `json:"id"` 65 | Parent string `json:"parent"` 66 | } -------------------------------------------------------------------------------- /registry/manifest.go: -------------------------------------------------------------------------------- 1 | package registry 2 | 3 | import ( 4 | "archive/tar" 5 | "bytes" 6 | "compress/gzip" 7 | "crypto/sha256" 8 | "crypto/tls" 9 | "encoding/json" 10 | "errors" 11 | "fmt" 12 | "github.com/docker/distribution/manifest/schema2" 13 | "github.com/opencontainers/go-digest" 14 | "io" 15 | "io/ioutil" 16 | "net/http" 17 | "os" 18 | "path/filepath" 19 | "strconv" 20 | "time" 21 | "strings" 22 | ) 23 | 24 | const ( 25 | REPO = "library" 26 | DefaultTAG = "latest" 27 | REGISTRY = "registry-1.docker.io" 28 | //github.com/docker/docker/image/tarexport 29 | ManifestFileName = "manifest.json" 30 | LegacyLayerFileName = "layer.tar" 31 | LegacyConfigFileName = "json" 32 | LegacyVersionFileName = "VERSION" 33 | LegacyRepositoriesFileName = "repositories" 34 | ) 35 | 36 | type WriteBarFunc func(downloadName string, length, downLen int64) 37 | 38 | type TarAddfileFunc func(size int64, name string, b interface{}, totalWrite ...int64) (int64, error) 39 | 40 | type Pull struct { 41 | // like registry-1.docker.io 42 | Registry string 43 | //like latest 44 | Tag string 45 | // Registry namespcase 46 | Repository string 47 | // Must be implemented in order to verify `RoundTrip()` 48 | Client *http.Client 49 | // 50 | ImgParts []string 51 | 52 | ImgNameWithoutTag string 53 | } 54 | 55 | // for manifest.json file 56 | type ManifestItem struct { 57 | Config string `json:"Config"` 58 | RepoTags []string `json:"RepoTags"` 59 | Layers []string `json:"Layers"` 60 | } 61 | 62 | func NewPull(pullImg string) *Pull { 63 | p := &Pull{ 64 | Tag: DefaultTAG, 65 | ImgParts: strings.Split(pullImg, "/"), 66 | } 67 | repo := REPO 68 | tempStrSlice := make([]string, 0) 69 | 70 | if strings.Contains(p.ImgParts[len(p.ImgParts)-1], "@") { 71 | p.ImgNameWithoutTag = strings.SplitN(pullImg, "@", 2)[1] 72 | tempStrSlice = strings.Split(p.ImgParts[len(p.ImgParts)-1], "@") 73 | } else if strings.Contains(p.ImgParts[len(p.ImgParts)-1], ":"){ 74 | p.ImgNameWithoutTag = strings.SplitN(pullImg, ":", 2)[1] 75 | tempStrSlice = strings.Split(p.ImgParts[len(p.ImgParts)-1], ":") 76 | } else { 77 | tempStrSlice = []string{p.ImgParts[len(p.ImgParts)-1], DefaultTAG} 78 | } 79 | img := tempStrSlice[0] 80 | p.Tag = tempStrSlice[1] 81 | 82 | //`:` means the port, the first part has `.` means the domain name or ip 83 | if len(p.ImgParts) > 1 && 84 | ( strings.Contains(p.ImgParts[0], ".") || strings.Contains(p.ImgParts[0], ":") ) { 85 | // use domain 86 | switch p.ImgParts[0] { 87 | case "quay.io": 88 | p.Registry = "quay.azk8s.cn" 89 | case "gcr.io": 90 | p.Registry = "gcr.azk8s.cn" 91 | default: 92 | p.Registry = p.ImgParts[0] 93 | } 94 | p.Registry = p.ImgParts[0] 95 | repo = strings.Join(p.ImgParts[1:len(p.ImgParts) - 1], "/") 96 | } else {// dockerhub 97 | //p.Registry = REGISTRY 98 | p.Registry = "dockerhub.azk8s.cn" 99 | if len(p.ImgParts[:len(p.ImgParts)-1]) != 0 { 100 | repo = strings.Join(p.ImgParts[:len(p.ImgParts)-1], "/") 101 | } 102 | } 103 | p.Repository = fmt.Sprintf("%s/%s", repo, img) 104 | 105 | p.Client = &http.Client{ 106 | Transport: NewTokenTransport(&http.Transport{ 107 | Proxy: http.ProxyFromEnvironment, 108 | //DialContext: (&net.Dialer{ 109 | // Timeout: 30 * time.Second, 110 | // KeepAlive: 30 * time.Second, 111 | //}).DialContext, 112 | //ForceAttemptHTTP2: true, 113 | //MaxIdleConns: 150, 114 | //MaxIdleConnsPerHost: -1, 115 | //IdleConnTimeout: 90 * time.Second, 116 | //TLSHandshakeTimeout: 10 * time.Second, 117 | //ExpectContinueTimeout: 5 * time.Second, 118 | TLSClientConfig: &tls.Config{InsecureSkipVerify:true}, 119 | //ResponseHeaderTimeout: time.Second * 8, 120 | }), 121 | Timeout: time.Second * 15, 122 | } 123 | 124 | return p 125 | } 126 | 127 | func (p *Pull) Do(req *http.Request) (*http.Response, error) { 128 | return p.Client.Do(req) 129 | } 130 | 131 | func (p *Pull) Manifests() (*schema2.Manifest, error) { 132 | req, _ := http.NewRequest("GET", 133 | fmt.Sprintf("https://%s/v2/%s/manifests/%s", p.Registry, p.Repository, p.Tag), nil) 134 | req.Header.Set("Accept", schema2.MediaTypeManifest) 135 | resp, err := p.Do(req) 136 | if err != nil { 137 | return nil, fmt.Errorf("while request manifests|%s", err) 138 | } 139 | defer resp.Body.Close() 140 | 141 | respBody,_ := ioutil.ReadAll(resp.Body) 142 | if strings.Contains(string(respBody), `"errors"`) { 143 | return nil, errors.New(string(respBody)) 144 | } 145 | var data schema2.Manifest 146 | if err := json.Unmarshal(respBody, &data); err != nil { 147 | return nil, fmt.Errorf("unmarshal err|%s", err) 148 | } 149 | if data.SchemaVersion != 2 { 150 | return nil, fmt.Errorf( 151 | "is not a schema2.Manifest for %s, maybe is quay.io's old images for https://github.com/moby/buildkit/issues/409", 152 | strings.Join(p.ImgParts, "/")) 153 | } 154 | return &data, nil 155 | } 156 | 157 | 158 | func (p *Pull) Blobs(Digest digest.Digest, Range int64) (int64, io.ReadCloser, error) { 159 | req, _ := http.NewRequest("GET", 160 | fmt.Sprintf("https://%s/v2/%s/blobs/%s", p.Registry, p.Repository, Digest.String()), nil) 161 | req.Header.Set("Accept", schema2.MediaTypeManifest) 162 | req.Header.Set("Range", fmt.Sprintf("bytes=%d-", Range)) 163 | resp, err := p.Do(req) 164 | if err != nil { 165 | return 0, nil, fmt.Errorf("while request blobs|%s", err) 166 | } 167 | 168 | fSize, err := strconv.ParseInt(resp.Header.Get("Content-Length"), 10, 32) 169 | if err != nil { 170 | return 0, nil, fmt.Errorf("Content-Length|%s", err) 171 | } 172 | return fSize, resp.Body, nil 173 | } 174 | 175 | 176 | func Save(names []string, fileName string) (error) { 177 | fw, err := os.Create(fileName) 178 | if err != nil { 179 | return err 180 | } 181 | defer fw.Close() 182 | 183 | gw := gzip.NewWriter(fw) 184 | defer gw.Close() 185 | tw := tar.NewWriter(gw) 186 | defer tw.Close() 187 | 188 | tarAddfile := TarAddfileWithDownBar(tw, WriteBar) 189 | var ( 190 | manifestJsons = make([]ManifestItem, 0) 191 | repositoriesJson = make(map[string]map[string]string, 1) 192 | ) 193 | 194 | for _, name := range names { 195 | parentID := "" 196 | p := NewPull(name) 197 | data, err := p.Manifests() 198 | if err != nil { 199 | return err 200 | } 201 | 202 | fSize, confRespsBody, err := p.Blobs(data.Config.Digest, 0) 203 | if err != nil { 204 | return fmt.Errorf("while the config digest to get conf %s", err) 205 | } 206 | // for id.json file 207 | confResps, err := ioutil.ReadAll(confRespsBody) 208 | defer confRespsBody.Close() 209 | if _, err := tarAddfile(fSize, data.Config.Digest.Hex() + ".json", confResps);err != nil { 210 | return err 211 | } 212 | manifestJson := ManifestItem{ 213 | Config: data.Config.Digest.Hex() + ".json", 214 | RepoTags: []string{name}, 215 | Layers: make([]string, 0), 216 | } 217 | var confCont ImageConfig 218 | err = json.Unmarshal(confResps, &confCont) 219 | if err != nil { 220 | return fmt.Errorf("%s for confCont", err) 221 | } 222 | 223 | for i, layer := range data.Layers { 224 | 225 | // https://github.com/moby/moby/blob/master/image/tarexport/save.go#L294-L329 226 | // https://gist.github.com/aaronlehmann/b42a2eaf633fc949f93b#id-definitions-and-calculations 227 | legacyLayerDir := fmt.Sprintf("%x", 228 | sha256.Sum256([]byte(fmt.Sprintf(parentID + "\n" + layer.Digest.String() + "\n")))) 229 | 230 | // for layer.tar gzip header 231 | if _, err := tarAddfile(layer.Size, filepath.Join(legacyLayerDir, LegacyLayerFileName), nil);err != nil { 232 | return err 233 | } 234 | var ( 235 | written int64 = 0 236 | ) 237 | for { 238 | _, respBody, err := p.Blobs(layer.Digest, written) 239 | if err != nil { 240 | return fmt.Errorf("while get blobSum %s", err) 241 | } 242 | // for layer.tar 243 | written, err = tarAddfile(layer.Size, filepath.Join(legacyLayerDir, LegacyLayerFileName), respBody, written) 244 | if err != nil { 245 | return err 246 | } 247 | if written >= layer.Size { 248 | break 249 | } 250 | } 251 | 252 | manifestJson.Layers = append(manifestJson.Layers, filepath.Join(legacyLayerDir, LegacyLayerFileName)) 253 | 254 | // for VERSION 255 | if _, err := tarAddfile(int64(len([]byte(`1.0`))), 256 | filepath.Join(legacyLayerDir, LegacyVersionFileName), []byte(`1.0`));err != nil { 257 | return err 258 | } 259 | 260 | confCont.ID = legacyLayerDir 261 | 262 | if parentID != "" { 263 | confCont.Parent = parentID 264 | } 265 | parentID = confCont.ID 266 | 267 | confBytes := NewLayerEmptyJson() 268 | if i == len(data.Layers) - 1 { 269 | confBytesFull, _:= json.Marshal(&confCont) 270 | confBytes = confBytesFull 271 | } 272 | // for json 273 | if _, err := tarAddfile(int64(len(confBytes)), 274 | filepath.Join(legacyLayerDir, LegacyConfigFileName), confBytes);err != nil { 275 | return err 276 | } 277 | //layerName append 278 | } 279 | manifestJsons = append(manifestJsons, manifestJson) 280 | if v, ok := repositoriesJson[p.Repository]; ok { 281 | v[p.Tag] = data.Layers[len(data.Layers)-1].Digest.Hex() 282 | } else { 283 | repositoriesJson[p.ImgNameWithoutTag] = map[string]string{ 284 | p.Tag: data.Layers[len(data.Layers)-1].Digest.Hex(), 285 | } 286 | } 287 | } 288 | 289 | //for ManifestFileName = "manifest.json" 290 | manifestBytes, err := json.Marshal(&manifestJsons) 291 | if err != nil { 292 | return fmt.Errorf("while Marshal manifestJsons|%s", err) 293 | } 294 | if _, err := tarAddfile(int64(len(manifestBytes)), ManifestFileName, manifestBytes);err != nil { 295 | return err 296 | } 297 | 298 | //for LegacyRepositoriesFileName = "repositories" 299 | repositoriesBytes, err := json.Marshal(&repositoriesJson) 300 | if err != nil { 301 | return fmt.Errorf("while Marshal repositoriesBytes|%s", err) 302 | } 303 | if _, err := tarAddfile(int64(len(repositoriesBytes)), LegacyRepositoriesFileName, repositoriesBytes);err != nil { 304 | return err 305 | } 306 | 307 | return nil 308 | } 309 | 310 | func WriteBar(downloadName string, length, downLen int64) { 311 | fmt.Printf("\r%-76s CurrentTotalBytes %15d, ConsumedTotalBytes: %15d, %d%%", 312 | downloadName, length, downLen, downLen*100/length) 313 | } 314 | 315 | func TarAddfileWithDownBar(tw *tar.Writer, wb WriteBarFunc) TarAddfileFunc { 316 | // b != nil ====> will write data 317 | // b == nil ====> don't write data 318 | // len(totalWrite) == 0 ===> will write header 319 | return func(size int64, name string, b interface{}, totalWrite ...int64) (int64, error) { 320 | var ( 321 | buf = make([]byte, 32*1024) 322 | written int64 323 | err error 324 | data io.Reader 325 | ) 326 | 327 | if len(totalWrite) == 0 { // for layer.tar while retry 328 | err = tw.WriteHeader(&tar.Header{ 329 | Mode: 0644, 330 | Size: size, 331 | Name: name, 332 | }) 333 | if err != nil { 334 | return 0, fmt.Errorf("%s write header|%s", name, err) 335 | } 336 | } 337 | 338 | if b != nil { 339 | switch v := b.(type) { 340 | case []byte: 341 | data = bytes.NewReader(v) 342 | case io.ReadCloser: 343 | data = v 344 | default: 345 | return 0, fmt.Errorf("invalid type") 346 | } 347 | 348 | if len(totalWrite) >= 1 { 349 | written = totalWrite[0] 350 | } 351 | 352 | for { 353 | numRead, readErr := data.Read(buf) 354 | if numRead > 0 { 355 | numWrite, writeErr := tw.Write(buf[0:numRead]) 356 | if numWrite > 0 { 357 | written += int64(numWrite) 358 | } 359 | if writeErr != nil { 360 | err = io.ErrShortWrite 361 | break 362 | } 363 | } 364 | if readErr != nil { 365 | if readErr != io.EOF { 366 | err = readErr 367 | } 368 | break 369 | } 370 | wb(name, size, written) 371 | } 372 | if written >= size { 373 | fmt.Println() 374 | } 375 | } 376 | return written, nil 377 | } 378 | } 379 | 380 | type EmptyConfig struct { 381 | Created time.Time `json:"created"` 382 | ContainerConfig struct { 383 | Hostname string `json:"Hostname"` 384 | Domainname string `json:"Domainname"` 385 | User string `json:"User"` 386 | AttachStdin bool `json:"AttachStdin"` 387 | AttachStdout bool `json:"AttachStdout"` 388 | AttachStderr bool `json:"AttachStderr"` 389 | Tty bool `json:"Tty"` 390 | OpenStdin bool `json:"OpenStdin"` 391 | StdinOnce bool `json:"StdinOnce"` 392 | Env interface{} `json:"Env"` 393 | Cmd interface{} `json:"Cmd"` 394 | Image string `json:"Image"` 395 | Volumes interface{} `json:"Volumes"` 396 | WorkingDir string `json:"WorkingDir"` 397 | Entrypoint interface{} `json:"Entrypoint"` 398 | OnBuild interface{} `json:"OnBuild"` 399 | Labels interface{} `json:"Labels"` 400 | } `json:"container_config"` 401 | } 402 | 403 | func NewLayerEmptyJson() []byte { 404 | 405 | d, _ := json.Marshal(&EmptyConfig{ 406 | Created: time.Unix(0, 0), 407 | ContainerConfig: struct { 408 | Hostname string `json:"Hostname"` 409 | Domainname string `json:"Domainname"` 410 | User string `json:"User"` 411 | AttachStdin bool `json:"AttachStdin"` 412 | AttachStdout bool `json:"AttachStdout"` 413 | AttachStderr bool `json:"AttachStderr"` 414 | Tty bool `json:"Tty"` 415 | OpenStdin bool `json:"OpenStdin"` 416 | StdinOnce bool `json:"StdinOnce"` 417 | Env interface{} `json:"Env"` 418 | Cmd interface{} `json:"Cmd"` 419 | Image string `json:"Image"` 420 | Volumes interface{} `json:"Volumes"` 421 | WorkingDir string `json:"WorkingDir"` 422 | Entrypoint interface{} `json:"Entrypoint"` 423 | OnBuild interface{} `json:"OnBuild"` 424 | Labels interface{} `json:"Labels"` 425 | }{}, 426 | }) 427 | return d 428 | } 429 | -------------------------------------------------------------------------------- /registry/tokentransport.go: -------------------------------------------------------------------------------- 1 | package registry 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "net/http" 7 | "net/url" 8 | "strings" 9 | ) 10 | 11 | 12 | 13 | type TokenTransport struct { 14 | Transport http.RoundTripper 15 | } 16 | 17 | type authToken struct { 18 | ExpiresIn int `json:"expires_in"` 19 | Token string `json:"token"` 20 | } 21 | 22 | type ResponseError struct { 23 | Errors []struct { 24 | Code string `json:"code"` 25 | Message string `json:"message"` 26 | Detail []struct { 27 | Type string `json:"Type"` 28 | Class string `json:"Class"` 29 | Name string `json:"Name"` 30 | Action string `json:"Action"` 31 | } `json:"detail"` 32 | } `json:"errors"` 33 | } 34 | 35 | func NewTokenTransport(transport http.RoundTripper) *TokenTransport { 36 | return &TokenTransport{Transport:transport} 37 | } 38 | 39 | func (t *TokenTransport) RoundTrip(req *http.Request) (*http.Response, error) { 40 | resp, err := t.Transport.RoundTrip(req) 41 | if err != nil { 42 | return resp, err 43 | } 44 | if authService := isTokenDemand(resp); authService != nil { 45 | defer resp.Body.Close() 46 | resp, err = t.authAndRetry(authService, req) 47 | } 48 | return resp, err 49 | } 50 | 51 | 52 | func (t *TokenTransport) authAndRetry(authService *authService, req *http.Request) (*http.Response, error) { 53 | token, authResp, err := t.auth(authService) 54 | if err != nil { 55 | return authResp, err 56 | } 57 | 58 | req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token)) 59 | resp, err := t.Transport.RoundTrip(req) 60 | return resp, err 61 | } 62 | 63 | func (t *TokenTransport) auth(authService *authService) (string, *http.Response, error) { 64 | authReq, err := authService.Request() 65 | if err != nil { 66 | return "", nil, err 67 | } 68 | 69 | client := http.Client{ 70 | Transport: t.Transport, 71 | } 72 | 73 | response, err := client.Do(authReq) 74 | if err != nil { 75 | return "", nil, err 76 | } 77 | 78 | if response.StatusCode != http.StatusOK { 79 | return "", response, err 80 | } 81 | defer response.Body.Close() 82 | 83 | var authToken authToken 84 | decoder := json.NewDecoder(response.Body) 85 | err = decoder.Decode(&authToken) 86 | if err != nil { 87 | return "", nil, err 88 | } 89 | 90 | return authToken.Token, nil, nil 91 | } 92 | 93 | 94 | type authService struct { 95 | Realm string 96 | Service string 97 | Scope string 98 | } 99 | 100 | func (authService *authService) Request() (*http.Request, error) { 101 | url, err := url.Parse(authService.Realm) 102 | if err != nil { 103 | return nil, err 104 | } 105 | 106 | q := url.Query() 107 | q.Set("service", authService.Service) 108 | if authService.Scope != "" { 109 | q.Set("scope", authService.Scope) 110 | } 111 | url.RawQuery = q.Encode() 112 | 113 | request, err := http.NewRequest("GET", url.String(), nil) 114 | 115 | return request, err 116 | } 117 | 118 | func isTokenDemand(resp *http.Response) *authService { 119 | if resp == nil { 120 | return nil 121 | } 122 | if resp.StatusCode != http.StatusUnauthorized { 123 | return nil 124 | } 125 | return parseOauthHeader(resp) 126 | } 127 | 128 | func parseOauthHeader(resp *http.Response) *authService { 129 | if len(resp.Header["Www-Authenticate"]) > 0 { 130 | result := make(map[string]string) 131 | wantedHeaders := []string{"realm", "service", "scope"} 132 | authHeaderValueSlice := strings.Split(resp.Header["Www-Authenticate"][0], ",") 133 | for _, r := range authHeaderValueSlice { 134 | for _, w := range wantedHeaders { 135 | if strings.Contains(r, w) { 136 | result[w] = strings.Split(r, `"`)[1] 137 | } 138 | } 139 | } 140 | return &authService{ 141 | Realm: result["realm"], 142 | Service: result["service"], 143 | Scope: result["scope"], 144 | } 145 | } 146 | 147 | return nil 148 | } -------------------------------------------------------------------------------- /scripts/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | [ -z "$1" ] && { 4 | echo must use with tag name 5 | exit 1 6 | } 7 | export GO111MODULE=on 8 | export GOPROXY=GOPROXY=https://goproxy.cn,https://mirrors.aliyun.com/goproxy/,https://goproxy.io,https://athens.azurefd.net,direct 9 | export CGO_ENABLED=0 10 | go build -o dp main.go 11 | GOOS=windows go build -o dp.exe main.go 12 | tar zcf dp-windows-amd64-${1}.tar.gz dp.exe 13 | tar zcf dp-linux-amd64-${1}.tar.gz dp 14 | 15 | --------------------------------------------------------------------------------