├── .gitignore ├── .gitmodules ├── .travis.yml ├── Gomfile ├── LICENSE ├── Makefile ├── README.md ├── deb └── go-docker-registry │ ├── DEBIAN │ ├── control │ ├── postinst │ ├── postrm │ ├── preinst │ └── prerm │ └── opt │ └── go-docker-registry │ ├── log │ └── run │ └── run ├── registry.go └── src └── registry ├── api ├── api.go ├── app.go ├── caching.go ├── images.go ├── index.go ├── status.go └── tags.go ├── config └── config.go ├── layers ├── tar.go └── util.go ├── logger └── logger.go └── storage ├── local.go ├── local_test.go ├── s3.go ├── s3_test.go ├── storage.go └── storage_test.go /.gitignore: -------------------------------------------------------------------------------- 1 | bin 2 | config/s3_test.json 3 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brightcove-archive/ooyala_go-docker-registry/88ead7d173eff37ae24b59d3b0be084e7e558704/.gitmodules -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | go: 1.2 4 | 5 | install: 6 | - go get code.google.com/p/go.tools/cmd/cover 7 | -------------------------------------------------------------------------------- /Gomfile: -------------------------------------------------------------------------------- 1 | gom 'github.com/cespare/go-apachelog', :commit => 'bd03c773db22297a309ba6c24b872f51466c6a8d' 2 | gom 'github.com/crowdmob/goamz/aws', :commit => '8c1f9c953b0176803763cb911326e7aad9f02b5b' 3 | gom 'github.com/crowdmob/goamz/s3', :commit => '8c1f9c953b0176803763cb911326e7aad9f02b5b' 4 | gom 'github.com/gorilla/mux', :commit => '9ede152210fa25c1377d33e867cb828c19316445' 5 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2014 Ooyala Inc. 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | 203 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | SEMVER := 0.1.0 2 | 3 | PROJECT_ROOT := $(shell pwd) 4 | VENDOR_PATH := $(PROJECT_ROOT)/vendor 5 | PROJECT_NAME := $(shell pwd | xargs basename) 6 | 7 | GOPATH := $(GOPATH):$(PROJECT_ROOT):$(VENDOR_PATH) 8 | export GOPATH 9 | 10 | GOM := $(VENDOR_PATH)/bin/gom 11 | GOM_VENDOR_NAME := vendor 12 | export GOM_VENDOR_NAME 13 | 14 | all: build 15 | 16 | clean: 17 | @rm -rf bin pkg $(VENDOR_PATH) *.deb 18 | 19 | $(VENDOR_PATH): 20 | @echo "Installing Dependencies..." 21 | @mkdir -p $(VENDOR_PATH) || exit 2 22 | @GOPATH=$(VENDOR_PATH) go get github.com/ghao-ooyala/gom 23 | $(GOM) install 24 | @echo "Done." 25 | 26 | init: clean $(VENDOR_PATH) 27 | @mkdir bin 28 | 29 | build: init 30 | @go build -o bin/registry registry.go 31 | 32 | 33 | test: init 34 | ifdef TEST_PACKAGE 35 | @echo "Testing $$TEST_PACKAGE..." 36 | @go test -cover $$TEST_FLAGS $$TEST_PACKAGE 37 | else 38 | @for p in `find ./src -type f -name "*_test.go" |sed 's-\./src/\(.*\)/.*-\1-' |sort -u`; do \ 39 | echo "Testing $$p..."; \ 40 | go test -cover $$TEST_FLAGS $$p || exit 1; \ 41 | done 42 | @echo 43 | @echo "ok." 44 | endif 45 | 46 | annotate: 47 | ifdef TEST_PACKAGE 48 | @echo "Annotating $$TEST_PACKAGE..." 49 | @go test -coverprofile=cover.out $$TEST_FLAGS $$TEST_PACKAGE 50 | @go tool cover -html=cover.out 51 | @rm -f cover.out 52 | else 53 | @echo "Specify package!" 54 | endif 55 | 56 | fmt: 57 | @gofmt -l -w registry.go 58 | @find src -name \*.go -exec gofmt -l -w {} \; 59 | 60 | DEB_STAGING := $(PROJECT_ROOT)/staging 61 | PKG_INSTALL_DIR := $(DEB_STAGING)/$(PROJECT_NAME)/opt/go-docker-registry 62 | 63 | deb: clean build 64 | @cp -a $(PROJECT_ROOT)/deb $(DEB_STAGING) 65 | @cp -a $(PROJECT_ROOT)/bin $(PKG_INSTALL_DIR)/ 66 | @perl -p -i -e "s/__VERSION__/$(SEMVER)/g" $(DEB_STAGING)/$(PROJECT_NAME)/DEBIAN/control 67 | @cd $(DEB_STAGING) && dpkg --build $(PROJECT_NAME) $(PROJECT_ROOT) 68 | @rm -rf $(DEB_STAGING) 69 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | go-docker-registry [![build status](https://secure.travis-ci.org/ooyala/go-docker-registry.png?branch=master)](http://travis-ci.org/ooyala/go-docker-registry) 2 | ================== 3 | 4 | Docker Registry Written in Go 5 | 6 | Go Clone of https://github.com/dotcloud/docker-registry 7 | 8 | The following is currently unimplemented: 9 | - Storage other than local and S3 10 | - Status API 11 | -------------------------------------------------------------------------------- /deb/go-docker-registry/DEBIAN/control: -------------------------------------------------------------------------------- 1 | Package: go-docker-registry 2 | Version: __VERSION__ 3 | Section: base 4 | Priority: Optional 5 | Architecture: amd64 6 | Depends: runit 7 | Maintainer: appsplat-team@ooyala.com 8 | Description: Go implementation of docker registry. 9 | -------------------------------------------------------------------------------- /deb/go-docker-registry/DEBIAN/postinst: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo "reviving registry..." 4 | cd /etc/service 5 | ln -s /opt/go-docker-registry go-docker-registry 6 | i=0 7 | while [ -z "$(pidof registry)" ] && [ $i -lt 10 ]; do 8 | echo "waiting for registry to revive..." 9 | sleep 1 10 | if [ -p /opt/go-docker-registry/supervise/ok ]; then 11 | sv up go-docker-registry 12 | fi 13 | (( i++ )) 14 | done 15 | if [ $i -eq 10 ]; then 16 | echo "could not revive registry." 17 | exit 1 18 | fi 19 | echo "registry revived." 20 | -------------------------------------------------------------------------------- /deb/go-docker-registry/DEBIAN/postrm: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ "$1" != "upgrade" ]; then 4 | rm -rf /opt/go-docker-registry 5 | fi 6 | -------------------------------------------------------------------------------- /deb/go-docker-registry/DEBIAN/preinst: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | kill_registry() { 4 | echo "killing registry..." 5 | sv down go-docker-registry 6 | rm /etc/service/go-docker-registry 7 | i=0 8 | while [ "$(pidof registry)" ] && [ $i -lt 5 ]; do 9 | echo "waiting for registry to die..." 10 | sleep 1 11 | (( i++ )) 12 | done 13 | if [ "$(pidof registry)" ]; then 14 | echo "registry is proving resilient. bringing out the big guns..." 15 | kill -9 $(pidof registry) 16 | fi 17 | echo "registry killed." 18 | } 19 | 20 | if [ -L '/etc/service/go-docker-registry' ]; then 21 | kill_registry 22 | fi 23 | 24 | mkdir -p /var/log/atlantis/registry 25 | -------------------------------------------------------------------------------- /deb/go-docker-registry/DEBIAN/prerm: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | kill_registry() { 4 | echo "killing registry..." 5 | sv down go-docker-registry 6 | rm /etc/service/go-docker-registry 7 | i=0 8 | while [ "$(pidof registry)" ] && [ $i -lt 5 ]; do 9 | echo "waiting for registry to die..." 10 | sleep 1 11 | (( i++ )) 12 | done 13 | if [ "$(pidof registry)" ]; then 14 | echo "registry is proving resilient. bringing out the big guns..." 15 | kill -9 $(pidof registry) 16 | fi 17 | echo "registry killed." 18 | } 19 | 20 | if [ -L '/etc/service/go-docker-registry' ]; then 21 | kill_registry 22 | fi 23 | -------------------------------------------------------------------------------- /deb/go-docker-registry/opt/go-docker-registry/log/run: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | exec svlogd -ttt /var/log/atlantis/registry 4 | -------------------------------------------------------------------------------- /deb/go-docker-registry/opt/go-docker-registry/run: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | exec ./bin/registry -config /etc/go-docker-registry/config.json 2>&1 4 | -------------------------------------------------------------------------------- /registry.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "registry/api" 6 | "registry/config" 7 | "registry/logger" 8 | "registry/storage" 9 | ) 10 | 11 | func main() { 12 | var cfgFile string 13 | flag.StringVar(&cfgFile, "config", "/etc/go-docker-registry/config.json", "config file") 14 | flag.Parse() 15 | 16 | cfg, err := config.New(cfgFile) 17 | if err != nil { 18 | logger.Fatal(err.Error()) 19 | } 20 | 21 | storage, err := storage.New(cfg.Storage) 22 | if err != nil { 23 | logger.Fatal(err.Error()) 24 | } 25 | 26 | registryAPI := api.New(cfg.API, storage) 27 | logger.Fatal(registryAPI.ListenAndServe().Error()) 28 | } 29 | -------------------------------------------------------------------------------- /src/registry/api/api.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "github.com/cespare/go-apachelog" 7 | "github.com/gorilla/mux" 8 | "io" 9 | "log" 10 | "net/http" 11 | "os" 12 | "regexp" 13 | "registry/storage" 14 | ) 15 | 16 | var USER_AGENT_REGEXP = regexp.MustCompile("([^\\s/]+)/([^\\s/]+)") 17 | var EMPTY_HEADERS = map[string][]string{} 18 | 19 | type Config struct { 20 | Addr string `json:"addr"` 21 | DefaultHeaders map[string][]string `json:"default_headers"` 22 | } 23 | 24 | type RegistryAPI struct { 25 | *Config 26 | Storage storage.Storage 27 | } 28 | 29 | func New(cfg *Config, storage storage.Storage) *RegistryAPI { 30 | return &RegistryAPI{Config: cfg, Storage: storage} 31 | } 32 | 33 | func (a *RegistryAPI) ListenAndServe() error { 34 | r := mux.NewRouter() 35 | r.HandleFunc("/", a.HomeHandler) 36 | 37 | // 38 | // Registry APIs (http://docs.docker.io/en/latest/reference/api/registry_api/) 39 | // 40 | 41 | // http://docs.docker.io/en/latest/reference/api/registry_api/#status 42 | // Documented and implemented in docker-registry 0.6.5 43 | r.HandleFunc("/_ping", a.PingHandler) 44 | r.HandleFunc("/v1/_ping", a.PingHandler) 45 | // Undocumented but implemented in docker-registry 0.6.5 46 | r.HandleFunc("/_status", a.StatusHandler) 47 | r.HandleFunc("/v1/_status", a.StatusHandler) 48 | 49 | // http://docs.docker.io/en/latest/reference/api/registry_api/#images 50 | // Documented and implemented in docker-registry 0.6.5 51 | r.HandleFunc("/v1/images/{imageID}/layer", a.RequireCompletion(a.CheckIfModifiedSince(a.GetImageLayerHandler))).Methods("GET") 52 | r.HandleFunc("/v1/images/{imageID}/layer", a.PutImageLayerHandler).Methods("PUT") 53 | r.HandleFunc("/v1/images/{imageID}/json", a.RequireCompletion(a.CheckIfModifiedSince(a.GetImageJsonHandler))).Methods("GET") 54 | r.HandleFunc("/v1/images/{imageID}/json", a.PutImageJsonHandler).Methods("PUT") 55 | r.HandleFunc("/v1/images/{imageID}/ancestry", a.RequireCompletion(a.CheckIfModifiedSince(a.GetImageAncestryHandler))).Methods("GET") 56 | // Undocumented but implemented in docker-registry 0.6.5 57 | r.HandleFunc("/v1/images/{imageID}/checksum", a.PutImageChecksumHandler).Methods("PUT") 58 | r.HandleFunc("/v1/images/{imageID}/files", a.RequireCompletion(a.CheckIfModifiedSince(a.GetImageFilesHandler))).Methods("GET") 59 | r.HandleFunc("/v1/images/{imageID}/diff", a.RequireCompletion(a.CheckIfModifiedSince(a.GetImageDiffHandler))).Methods("GET") 60 | 61 | // http://docs.docker.io/en/latest/reference/api/registry_api/#tags 62 | // Documented and implemented in docker-registry 0.6.5 63 | r.HandleFunc("/v1/repositories/{repo}/tags", a.GetRepoTagsHandler).Methods("GET") 64 | r.HandleFunc("/v1/repositories/{repo}/tags/{tag}", a.GetRepoTagHandler).Methods("GET") 65 | r.HandleFunc("/v1/repositories/{repo}/tags/{tag}", a.PutRepoTagHandler).Methods("PUT") 66 | r.HandleFunc("/v1/repositories/{repo}/tags/{tag}", a.DeleteRepoTagHandler).Methods("DELETE") 67 | r.HandleFunc("/v1/repositories/{namespace}/{repo}/tags", a.GetRepoTagsHandler).Methods("GET") 68 | r.HandleFunc("/v1/repositories/{namespace}/{repo}/tags/{tag}", a.GetRepoTagHandler).Methods("GET") 69 | r.HandleFunc("/v1/repositories/{namespace}/{repo}/tags/{tag}/json", a.GetRepoTagJsonHandler).Methods("GET") 70 | r.HandleFunc("/v1/repositories/{namespace}/{repo}/tags/{tag}", a.PutRepoTagHandler).Methods("PUT") 71 | r.HandleFunc("/v1/repositories/{namespace}/{repo}/tags/{tag}", a.DeleteRepoTagHandler).Methods("DELETE") 72 | // Undocumented but implemented in docker-registry 0.6.5 73 | r.HandleFunc("/v1/repositories/{repo}/tags", a.DeleteRepoTagsHandler).Methods("DELETE") 74 | r.HandleFunc("/v1/repositories/{repo}/json", a.GetRepoJsonHandler).Methods("GET") 75 | r.HandleFunc("/v1/repositories/{namespace}/{repo}/tags", a.DeleteRepoTagsHandler).Methods("DELETE") 76 | r.HandleFunc("/v1/repositories/{namespace}/{repo}/json", a.GetRepoJsonHandler).Methods("GET") 77 | // Documented and unimplemented in docker-registry 0.6.5 78 | r.HandleFunc("/v1/repositories/{repo}/", a.DeleteRepoHandler).Methods("DELETE") 79 | r.HandleFunc("/v1/repositories/{namespace}/{repo}/", a.DeleteRepoHandler).Methods("DELETE") 80 | // Undocumented and unimplemented (additional) 81 | r.HandleFunc("/v1/repositories/{repo}", a.DeleteRepoHandler).Methods("DELETE") 82 | r.HandleFunc("/v1/repositories/{namespace}/{repo}", a.DeleteRepoHandler).Methods("DELETE") 83 | 84 | // Unused (for private images) 85 | //r.HandleFunc("/v1/private_images/{imageID}/layer", a.GetPrivateImageLayerHandler).Methods("GET") 86 | //r.HandleFunc("/v1/private_images/{imageID}/json", a.GetPrivateImageJsonHandler).Methods("GET") 87 | //r.HandleFunc("/v1/private_images/{imageID}/files", a.GetPrivateImageFilesHandler).Methods("GET") 88 | //r.HandleFunc("/v1/repositories/{repo}/properties", a.GetRepoPropertiesHandler).Methods("GET") 89 | //r.HandleFunc("/v1/repositories/{repo}/properties", a.PutRepoPropertiesHandler).Methods("PUT") 90 | //r.HandleFunc("/v1/repositories/{namespace}/{repo}/properties", a.GetRepoPropertiesHandler).Methods("GET") 91 | //r.HandleFunc("/v1/repositories/{namespace}/{repo}/properties", a.PutRepoPropertiesHandler).Methods("PUT") 92 | 93 | // 94 | // Index APIs (http://docs.docker.io/en/latest/reference/api/index_api/) 95 | // 96 | 97 | // http://docs.docker.io/en/latest/reference/api/index_api/#users 98 | // Documented and implemented in docker-registry 0.6.5 99 | r.HandleFunc("/v1/users", a.LoginHandler).Methods("GET") 100 | r.HandleFunc("/v1/users", a.CreateUserHandler).Methods("POST") 101 | r.HandleFunc("/v1/users/", a.LoginHandler).Methods("GET") 102 | r.HandleFunc("/v1/users/", a.CreateUserHandler).Methods("POST") 103 | r.HandleFunc("/v1/users/{username}/", a.UpdateUserHandler).Methods("PUT") 104 | 105 | // http://docs.docker.io/en/latest/reference/api/index_api/#repository 106 | // Documented and implemented in docker-registry 0.6.5 107 | r.HandleFunc("/v1/repositories/{repo}/", a.PutRepoHandler).Methods("PUT") 108 | r.HandleFunc("/v1/repositories/{repo}/images", a.GetRepoImagesHandler).Methods("GET") 109 | r.HandleFunc("/v1/repositories/{repo}/images", a.PutRepoImagesHandler).Methods("PUT") 110 | r.HandleFunc("/v1/repositories/{repo}/auth", a.PutRepoAuthHandler).Methods("PUT") 111 | r.HandleFunc("/v1/repositories/{namespace}/{repo}/", a.PutRepoHandler).Methods("PUT") 112 | r.HandleFunc("/v1/repositories/{namespace}/{repo}/images", a.GetRepoImagesHandler).Methods("GET") 113 | r.HandleFunc("/v1/repositories/{namespace}/{repo}/images", a.PutRepoImagesHandler).Methods("PUT") 114 | r.HandleFunc("/v1/repositories/{namespace}/{repo}/auth", a.PutRepoAuthHandler).Methods("PUT") 115 | // Undocumented but implemented in docker-registry 0.6.5 116 | r.HandleFunc("/v1/repositories/{repo}", a.PutRepoHandler).Methods("PUT") 117 | r.HandleFunc("/v1/repositories/{repo}/images", a.DeleteRepoImagesHandler).Methods("DELETE") 118 | r.HandleFunc("/v1/repositories/{namespace}/{repo}", a.PutRepoHandler).Methods("PUT") 119 | r.HandleFunc("/v1/repositories/{namespace}/{repo}/images", a.DeleteRepoImagesHandler).Methods("DELETE") 120 | 121 | // http://docs.docker.io/en/latest/reference/api/index_api/#search 122 | // Documented and implemented in docker-registry 0.6.5 123 | r.HandleFunc("/v1/search", a.SearchHandler).Methods("GET") 124 | 125 | log.Printf("Listening on %s", a.Config.Addr) 126 | return http.ListenAndServe(a.Config.Addr, apachelog.NewHandler(r, os.Stderr)) 127 | } 128 | 129 | func (a *RegistryAPI) response(w http.ResponseWriter, data interface{}, code int, headers map[string][]string) { 130 | for name, values := range a.Config.DefaultHeaders { 131 | w.Header()[name] = append(w.Header()[name], values...) 132 | } 133 | for name, values := range headers { 134 | w.Header()[name] = append(w.Header()[name], values...) 135 | } 136 | switch typedData := data.(type) { 137 | case nil: 138 | w.WriteHeader(code) 139 | w.Write([]byte{}) 140 | case bool: 141 | w.WriteHeader(code) 142 | fmt.Fprintf(w, "%t", typedData) 143 | case int: 144 | w.WriteHeader(code) 145 | fmt.Fprintf(w, "%d", typedData) 146 | case string: 147 | w.WriteHeader(code) 148 | if code >= 400 { 149 | // if error, jsonify 150 | w.Write([]byte("{\"error\":\"" + typedData + "\"}")) 151 | } else { 152 | w.Write([]byte(typedData)) 153 | } 154 | case []byte: 155 | // no need to wrap error here because if data comes in as a []byte it is meant to be raw data 156 | w.WriteHeader(code) 157 | w.Write(typedData) 158 | case io.Reader: 159 | w.WriteHeader(code) 160 | io.Copy(w, typedData) 161 | default: 162 | // write json 163 | if encoded, err := json.Marshal(data); err != nil { 164 | http.Error(w, err.Error(), http.StatusInternalServerError) 165 | } else { 166 | w.WriteHeader(code) 167 | w.Write(encoded) 168 | } 169 | } 170 | } 171 | 172 | func (a *RegistryAPI) internalError(w http.ResponseWriter, text string) { 173 | a.response(w, "Internal Error: "+text, http.StatusInternalServerError, EMPTY_HEADERS) 174 | } 175 | 176 | func NotImplementedHandler(w http.ResponseWriter, r *http.Request) { 177 | http.Error(w, "Not Implemented", http.StatusNotImplemented) 178 | } 179 | 180 | func parseRepo(r *http.Request, extra string) (string, string, string) { 181 | vars := mux.Vars(r) 182 | namespace := vars["namespace"] 183 | if vars["namespace"] == "" { 184 | namespace = "library" 185 | } 186 | return namespace, vars["repo"], vars[extra] 187 | } 188 | -------------------------------------------------------------------------------- /src/registry/api/app.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | ) 7 | 8 | func (a *RegistryAPI) HomeHandler(w http.ResponseWriter, r *http.Request) { 9 | fmt.Fprintln(w, "go-docker-registry server") 10 | } 11 | 12 | func (a *RegistryAPI) PingHandler(w http.ResponseWriter, r *http.Request) { 13 | a.response(w, true, http.StatusOK, map[string][]string{"X-Docker-Registry-Standalone": []string{"true"}}) 14 | } 15 | -------------------------------------------------------------------------------- /src/registry/api/caching.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "time" 7 | ) 8 | 9 | func DefaultCacheHeaders() map[string][]string { 10 | expires := time.Now().UTC().Add(365 * 24 * time.Hour) 11 | return map[string][]string{ 12 | "Cache-Control": []string{fmt.Sprintf("public, max-age=%d", int64(365*24*time.Hour.Seconds()))}, 13 | "Expires": []string{expires.Format("Thu, 01 Jan 1970 00:00:00 GMT")}, 14 | "Last-Modified": []string{"Thu, 01 Jan 1970 00:00:00 GMT"}, 15 | } 16 | } 17 | 18 | func (a *RegistryAPI) CheckIfModifiedSince(handler http.HandlerFunc) http.HandlerFunc { 19 | return func(w http.ResponseWriter, r *http.Request) { 20 | // check If-Modified-Since (if it exists, just send back a 304 because it will never change) 21 | if len(r.Header["If-Modified-Since"]) > 0 { 22 | a.response(w, true, 304, DefaultCacheHeaders()) 23 | return 24 | } 25 | handler(w, r) 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/registry/api/images.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "crypto/sha256" 5 | "encoding/hex" 6 | "encoding/json" 7 | "fmt" 8 | "github.com/gorilla/mux" 9 | "io" 10 | "io/ioutil" 11 | "net/http" 12 | "registry/layers" 13 | "registry/logger" 14 | "registry/storage" 15 | "strconv" 16 | "strings" 17 | ) 18 | 19 | const COOKIE_SEPARATOR = "|" 20 | 21 | func (a *RegistryAPI) RequireCompletion(handler http.HandlerFunc) http.HandlerFunc { 22 | return func(w http.ResponseWriter, r *http.Request) { 23 | vars := mux.Vars(r) 24 | imageID := vars["imageID"] 25 | if exists, _ := a.Storage.Exists(storage.ImageMarkPath(imageID)); exists { 26 | a.response(w, "Image is being uploaded, retry later", http.StatusBadRequest, EMPTY_HEADERS) 27 | return 28 | } 29 | handler(w, r) 30 | } 31 | } 32 | 33 | // Must be wrapped by: RequiresCompletion, CheckIfModifiedSince 34 | // Sets: DefaultCacheHeaders 35 | func (a *RegistryAPI) GetImageLayerHandler(w http.ResponseWriter, r *http.Request) { 36 | vars := mux.Vars(r) 37 | imageID := vars["imageID"] 38 | headers := DefaultCacheHeaders() 39 | reader, err := a.Storage.GetReader(storage.ImageLayerPath(imageID)) 40 | if err != nil { 41 | // every "Image not found" response in this file. 42 | a.response(w, "Image not found: "+err.Error(), http.StatusNotFound, EMPTY_HEADERS) 43 | return 44 | } 45 | a.response(w, reader, http.StatusOK, headers) 46 | } 47 | 48 | func (a *RegistryAPI) PutImageLayerHandler(w http.ResponseWriter, r *http.Request) { 49 | vars := mux.Vars(r) 50 | imageID := vars["imageID"] 51 | jsonContent, err := a.Storage.Get(storage.ImageJsonPath(imageID)) 52 | if err != nil { 53 | a.response(w, "Image not found: "+err.Error(), http.StatusNotFound, EMPTY_HEADERS) 54 | return 55 | } 56 | layerPath := storage.ImageLayerPath(imageID) 57 | markPath := storage.ImageMarkPath(imageID) 58 | layerExists, _ := a.Storage.Exists(layerPath) 59 | markExists, _ := a.Storage.Exists(markPath) 60 | if layerExists && !markExists { 61 | a.response(w, "Image already exists", http.StatusConflict, EMPTY_HEADERS) 62 | return 63 | } 64 | // This next section reads the tarball from the body while computing various checksums. sha256Writer is used 65 | // to compute a checksum of the entire tarball using a TeeReader which will read from the body while 66 | // simultaneously writing what it read to sha256Writer. tarInfo will read the tar after it is put into the 67 | // storage and checksum each individual file within it (and checksum those checksums with the jsonContent) 68 | sha256Writer := sha256.New() 69 | sha256Writer.Write(append(jsonContent, '\n')) 70 | teeReader := io.TeeReader(r.Body, sha256Writer) 71 | // this will create the checksums for a tar and the json for tar file info 72 | tarInfo := layers.NewTarInfo() 73 | // PutReader takes a function that will run after the write finishes: 74 | err = a.Storage.PutReader(layerPath, teeReader, tarInfo.Load) 75 | if err != nil { 76 | a.response(w, "Internal Error: "+err.Error(), http.StatusInternalServerError, EMPTY_HEADERS) 77 | return 78 | } 79 | 80 | checksums := []string{"sha256:" + hex.EncodeToString(sha256Writer.Sum(nil))} 81 | 82 | docker_version, err := layers.DockerVersion(r.Header["User-Agent"]) 83 | if err != nil { 84 | a.response(w, err.Error(), http.StatusBadRequest, EMPTY_HEADERS) 85 | return 86 | } 87 | version_numbers := strings.Split(docker_version, ".") 88 | if version_numbers[0] < "1" { 89 | if minor, _ := strconv.Atoi(version_numbers[1]); minor < 10 { 90 | if tarInfo.Error == nil { 91 | filesJson, err := tarInfo.TarFilesInfo.Json() 92 | if err != nil { 93 | a.response(w, "Internal Error: "+err.Error(), http.StatusInternalServerError, EMPTY_HEADERS) 94 | return 95 | } 96 | layers.SetImageFilesCache(a.Storage, imageID, filesJson) 97 | } 98 | // computing tarsum even if tarinfo.Error is nil as per python docker-registry 99 | tarsum := tarInfo.TarSum.Compute(jsonContent) 100 | checksums = append(checksums, tarsum) 101 | } 102 | } 103 | 104 | if err := layers.StoreChecksum(a.Storage, imageID, checksums); err != nil { 105 | a.response(w, "Error storing Checksum: "+err.Error(), http.StatusInternalServerError, EMPTY_HEADERS) 106 | return 107 | } 108 | a.response(w, true, http.StatusOK, EMPTY_HEADERS) 109 | } 110 | 111 | // Must be wrapped by: RequiresCompletion, CheckIfModifiedSince 112 | // Sets: DefaultCacheHeaders 113 | func (a *RegistryAPI) GetImageJsonHandler(w http.ResponseWriter, r *http.Request) { 114 | vars := mux.Vars(r) 115 | imageID := vars["imageID"] 116 | headers := DefaultCacheHeaders() 117 | data, err := a.Storage.Get(storage.ImageJsonPath(imageID)) 118 | if err != nil { 119 | a.response(w, "Image not found: "+err.Error(), http.StatusNotFound, EMPTY_HEADERS) 120 | return 121 | } 122 | size, err := a.Storage.Size(storage.ImageLayerPath(imageID)) 123 | if err != nil { 124 | a.response(w, "Unable to Compute Layer Size: "+err.Error(), http.StatusInternalServerError, EMPTY_HEADERS) 125 | return 126 | } 127 | headers["X-Docker-Size"] = []string{fmt.Sprintf("%d", size)} 128 | checksumPath := storage.ImageChecksumPath(imageID) 129 | if _, err := a.Storage.Exists(checksumPath); err != nil { 130 | a.response(w, "Checksum Not Found: "+err.Error(), http.StatusNotFound, EMPTY_HEADERS) 131 | return 132 | } 133 | 134 | var parsed_checksum []string 135 | checksum, err := a.Storage.Get(checksumPath) 136 | if err != nil { 137 | a.response(w, "Error Reading Checksum: "+err.Error(), http.StatusInternalServerError, EMPTY_HEADERS) 138 | return 139 | } 140 | json.Unmarshal(checksum, &parsed_checksum) 141 | headers["X-Docker-Checksum-Payload"] = parsed_checksum 142 | // check and compute header checksum for docker < 0.10 143 | docker_version, err := layers.DockerVersion(r.Header["User-Agent"]) 144 | if err != nil { 145 | a.response(w, err.Error(), http.StatusBadRequest, EMPTY_HEADERS) 146 | return 147 | } 148 | version_numbers := strings.Split(docker_version, ".") 149 | if version_numbers[0] < "1" { 150 | if minor, _ := strconv.Atoi(version_numbers[1]); minor < 10 { 151 | headers["X-Docker-Checksum"] = parsed_checksum 152 | delete(headers, "X-Docker-Checksum-Payload") 153 | } 154 | } 155 | a.response(w, data, http.StatusOK, headers) 156 | } 157 | 158 | func (a *RegistryAPI) PutImageJsonHandler(w http.ResponseWriter, r *http.Request) { 159 | vars := mux.Vars(r) 160 | imageID := vars["imageID"] 161 | // decode json from request body 162 | bodyBytes, err := ioutil.ReadAll(r.Body) 163 | if err != nil { 164 | a.response(w, "Error Reading Body: "+err.Error(), http.StatusBadRequest, EMPTY_HEADERS) 165 | return 166 | } 167 | var data map[string]interface{} 168 | err = json.Unmarshal(bodyBytes, &data) 169 | if err != nil { 170 | a.response(w, "Invalid JSON: "+err.Error(), http.StatusBadRequest, EMPTY_HEADERS) 171 | return 172 | } 173 | logger.Debug("[PutImageJson] body:\n%s", bodyBytes) 174 | if _, exists := data["id"]; !exists { 175 | a.response(w, "Missing key 'id' in JSON", http.StatusBadRequest, EMPTY_HEADERS) 176 | return 177 | } 178 | dataID, ok := data["id"].(string) 179 | if !ok { 180 | a.response(w, "Invalid JSON: 'id' is not a string", http.StatusBadRequest, EMPTY_HEADERS) 181 | return 182 | } 183 | if imageID != dataID { 184 | a.response(w, "JSON image id != image id specified in path", http.StatusBadRequest, EMPTY_HEADERS) 185 | return 186 | } 187 | var parentID string 188 | if _, exists := data["parent"]; exists { 189 | parentID, ok = data["parent"].(string) 190 | if !ok { 191 | a.response(w, "Invalid JSON: 'parent' is not a string", http.StatusBadRequest, EMPTY_HEADERS) 192 | return 193 | } 194 | if exists, _ := a.Storage.Exists(storage.ImageJsonPath(parentID)); !exists { 195 | a.response(w, "Image depends on non-existant parent", http.StatusBadRequest, EMPTY_HEADERS) 196 | return 197 | } 198 | } 199 | jsonPath := storage.ImageJsonPath(imageID) 200 | markPath := storage.ImageMarkPath(imageID) 201 | if exists, _ := a.Storage.Exists(jsonPath); exists { 202 | if markExists, _ := a.Storage.Exists(markPath); !markExists { 203 | a.response(w, "Image already exists", http.StatusConflict, EMPTY_HEADERS) 204 | return 205 | } 206 | } 207 | err = a.Storage.Put(markPath, []byte("true")) 208 | if err != nil { 209 | a.response(w, "Put Mark Error: "+err.Error(), http.StatusInternalServerError, EMPTY_HEADERS) 210 | return 211 | } 212 | // We cleanup any old checksum in case it's a retry after a fail 213 | a.Storage.Remove(storage.ImageChecksumPath(imageID)) 214 | err = a.Storage.Put(jsonPath, bodyBytes) 215 | if err != nil { 216 | a.response(w, "Put Json Error: "+err.Error(), http.StatusInternalServerError, EMPTY_HEADERS) 217 | return 218 | } 219 | if err := layers.GenerateAncestry(a.Storage, imageID, parentID); err != nil { 220 | a.response(w, "Generate Ancestry Error: "+err.Error(), http.StatusInternalServerError, EMPTY_HEADERS) 221 | return 222 | } 223 | a.response(w, "true", http.StatusOK, EMPTY_HEADERS) 224 | } 225 | 226 | // Must be wrapped by: RequiresCompletion, CheckIfModifiedSince 227 | // Sets: DefaultCacheHeaders 228 | func (a *RegistryAPI) GetImageAncestryHandler(w http.ResponseWriter, r *http.Request) { 229 | vars := mux.Vars(r) 230 | imageID := vars["imageID"] 231 | headers := DefaultCacheHeaders() 232 | data, err := a.Storage.Get(storage.ImageAncestryPath(imageID)) 233 | if err != nil { 234 | a.response(w, "Image not found: "+err.Error(), http.StatusNotFound, EMPTY_HEADERS) 235 | return 236 | } 237 | a.response(w, data, http.StatusOK, headers) 238 | } 239 | 240 | func (a *RegistryAPI) PutImageChecksumHandler(w http.ResponseWriter, r *http.Request) { 241 | vars := mux.Vars(r) 242 | imageID := vars["imageID"] 243 | 244 | checksum := r.Header.Get("X-Docker-Checksum-Payload") 245 | 246 | logger.Debug("X-Docker-Checksum-Payload " + checksum) 247 | logger.Debug("X-Docker-Checksum " + r.Header.Get("X-Docker-Checksum")) 248 | 249 | // compute checksum for docker < 0.10 250 | docker_version, err := layers.DockerVersion(r.Header["User-Agent"]) 251 | if err != nil { 252 | a.response(w, err.Error(), http.StatusBadRequest, EMPTY_HEADERS) 253 | return 254 | } 255 | version_numbers := strings.Split(docker_version, ".") 256 | if version_numbers[0] < "1" { 257 | if minor, _ := strconv.Atoi(version_numbers[1]); minor < 10 { 258 | checksum = r.Header.Get("X-Docker-Checksum") 259 | } 260 | } 261 | 262 | if checksum == "" { 263 | a.response(w, "Missing Image's checksum", http.StatusBadRequest, EMPTY_HEADERS) 264 | return 265 | } 266 | // check if image json exists 267 | if exists, _ := a.Storage.Exists(storage.ImageJsonPath(imageID)); !exists { 268 | a.response(w, "Image not found", http.StatusNotFound, EMPTY_HEADERS) 269 | return 270 | } 271 | 272 | markPath := storage.ImageMarkPath(imageID) 273 | if exists, _ := a.Storage.Exists(markPath); !exists { 274 | a.response(w, "Cannot set this image checksum (mark path does not exist)", http.StatusConflict, EMPTY_HEADERS) 275 | return 276 | } 277 | 278 | checksums := loadChecksums(a, imageID) 279 | if !stringInSlice(checksum, checksums) { 280 | logger.Debug("[PutImageLayer]["+imageID+"] Wrong checksum:"+string(checksum)+" not in %#v", checksums) 281 | a.response(w, "Checksum mismatch, ignoring the layer", http.StatusBadRequest, EMPTY_HEADERS) 282 | return 283 | } 284 | 285 | if err := a.Storage.Remove(markPath); err != nil { 286 | a.response(w, "Error removing Mark Path: "+err.Error(), http.StatusInternalServerError, EMPTY_HEADERS) 287 | return 288 | } 289 | a.response(w, true, http.StatusOK, EMPTY_HEADERS) 290 | } 291 | 292 | // Must be wrapped by: RequiresCompletion, CheckIfModifiedSince 293 | // Sets: DefaultCacheHeaders 294 | func (a *RegistryAPI) GetImageFilesHandler(w http.ResponseWriter, r *http.Request) { 295 | vars := mux.Vars(r) 296 | imageID := vars["imageID"] 297 | headers := DefaultCacheHeaders() 298 | data, err := layers.GetImageFilesJson(a.Storage, imageID) 299 | if err != nil { 300 | switch err.(type) { 301 | case layers.TarError: 302 | a.response(w, "Layer format not supported", http.StatusBadRequest, EMPTY_HEADERS) 303 | return 304 | default: 305 | a.response(w, "Image not found: "+err.Error(), http.StatusNotFound, EMPTY_HEADERS) 306 | return 307 | } 308 | } 309 | a.response(w, data, http.StatusOK, headers) 310 | } 311 | 312 | // Must be wrapped by: RequiresCompletion, CheckIfModifiedSince 313 | // Sets: DefaultCacheHeaders 314 | func (a *RegistryAPI) GetImageDiffHandler(w http.ResponseWriter, r *http.Request) { 315 | vars := mux.Vars(r) 316 | imageID := vars["imageID"] 317 | headers := DefaultCacheHeaders() 318 | diffJson, err := layers.GetImageDiffCache(a.Storage, imageID) 319 | if err != nil { 320 | // not cache miss. actual error 321 | a.response(w, "Internal Error: "+err.Error(), http.StatusInternalServerError, EMPTY_HEADERS) 322 | return 323 | } 324 | if diffJson == nil { 325 | // cache miss spawn goroutine to generate the diff and push it to S3 326 | go layers.GenDiff(a.Storage, imageID) 327 | diffJson = []byte{} 328 | } 329 | // copied from docker-registry. not sure why we would return StatusOK when the cache missed... 330 | a.response(w, diffJson, http.StatusOK, headers) 331 | } 332 | 333 | func loadChecksums(a *RegistryAPI, imageID string) []string { 334 | var data []string 335 | checksumPath := storage.ImageChecksumPath(imageID) 336 | if exists, _ := a.Storage.Exists(checksumPath); exists { 337 | content, _ := a.Storage.Get(checksumPath) 338 | json.Unmarshal(content, &data) 339 | } 340 | return data 341 | } 342 | 343 | func stringInSlice(element string, list []string) bool { 344 | for _, item := range list { 345 | if item == element { 346 | return true 347 | } 348 | } 349 | return false 350 | } 351 | -------------------------------------------------------------------------------- /src/registry/api/index.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "encoding/json" 5 | "registry/layers" 6 | "registry/logger" 7 | "registry/storage" 8 | "io/ioutil" 9 | "net/http" 10 | ) 11 | 12 | func IndexHeaders(r *http.Request, namespace, repo, access string) map[string][]string { 13 | fakeToken := []string{"Token signature=FAKESIGNATURE123,repository=\"" + namespace + "/" + repo + "\",access=" + access} 14 | return map[string][]string{ 15 | "X-Docker-Endpoints": []string{r.Host}, 16 | "WWW-Authenticate": fakeToken, 17 | "X-Docker-Token": fakeToken, 18 | } 19 | } 20 | 21 | func (a *RegistryAPI) putRepoImageHandler(w http.ResponseWriter, r *http.Request, successStatus int) { 22 | namespace, repo, _ := parseRepo(r, "") 23 | bodyBytes, err := ioutil.ReadAll(r.Body) 24 | if err != nil { 25 | a.response(w, "Internal Error: "+err.Error(), http.StatusInternalServerError, EMPTY_HEADERS) 26 | return 27 | } 28 | logger.Debug("[PutRepoImage] body:\n%s", bodyBytes) 29 | var body []map[string]interface{} 30 | if err := json.Unmarshal(bodyBytes, &body); err != nil { 31 | a.response(w, "Error Decoding JSON: "+err.Error(), http.StatusBadRequest, EMPTY_HEADERS) 32 | return 33 | } 34 | if err := layers.UpdateIndexImages(a.Storage, namespace, repo, bodyBytes, body); err != nil { 35 | a.response(w, "Internal Error: "+err.Error(), http.StatusInternalServerError, EMPTY_HEADERS) 36 | return 37 | } 38 | a.response(w, "", successStatus, IndexHeaders(r, namespace, repo, "write")) 39 | } 40 | 41 | func (a *RegistryAPI) LoginHandler(w http.ResponseWriter, r *http.Request) { 42 | // Empty Shell 43 | a.response(w, "OK", http.StatusOK, EMPTY_HEADERS) 44 | } 45 | 46 | func (a *RegistryAPI) CreateUserHandler(w http.ResponseWriter, r *http.Request) { 47 | // Empty Shell 48 | a.response(w, "User Created (lies)", http.StatusCreated, EMPTY_HEADERS) 49 | } 50 | 51 | func (a *RegistryAPI) UpdateUserHandler(w http.ResponseWriter, r *http.Request) { 52 | // Empty Shell 53 | a.response(w, "", http.StatusNoContent, EMPTY_HEADERS) 54 | } 55 | 56 | func (a *RegistryAPI) PutRepoHandler(w http.ResponseWriter, r *http.Request) { 57 | a.putRepoImageHandler(w, r, http.StatusOK) 58 | } 59 | 60 | func (a *RegistryAPI) PutRepoAuthHandler(w http.ResponseWriter, r *http.Request) { 61 | // Empty Shell 62 | a.response(w, "OK", http.StatusOK, EMPTY_HEADERS) 63 | } 64 | 65 | func (a *RegistryAPI) GetRepoImagesHandler(w http.ResponseWriter, r *http.Request) { 66 | namespace, repo, _ := parseRepo(r, "") 67 | data, err := a.Storage.Get(storage.RepoIndexImagesPath(namespace, repo)) 68 | if err != nil { 69 | a.response(w, "Image Not Found", http.StatusNotFound, EMPTY_HEADERS) 70 | return 71 | } 72 | a.response(w, data, http.StatusOK, IndexHeaders(r, namespace, repo, "read")) 73 | } 74 | 75 | func (a *RegistryAPI) PutRepoImagesHandler(w http.ResponseWriter, r *http.Request) { 76 | a.putRepoImageHandler(w, r, http.StatusNoContent) 77 | } 78 | 79 | func (a *RegistryAPI) DeleteRepoImagesHandler(w http.ResponseWriter, r *http.Request) { 80 | namespace, repo, _ := parseRepo(r, "") 81 | // from docker-registry 0.6.5: Does nothing, this file will be removed when DELETE on repos 82 | a.response(w, "", http.StatusNoContent, IndexHeaders(r, namespace, repo, "delete")) 83 | } 84 | 85 | func (a *RegistryAPI) SearchHandler(w http.ResponseWriter, r *http.Request) { 86 | a.response(w, map[string]string{}, http.StatusOK, EMPTY_HEADERS) 87 | } 88 | -------------------------------------------------------------------------------- /src/registry/api/status.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "net/http" 5 | ) 6 | 7 | func (a *RegistryAPI) StatusHandler(w http.ResponseWriter, r *http.Request) { 8 | // TODO[jigish] implement this 9 | // informative data about the server should go here =P 10 | NotImplementedHandler(w, r) 11 | } 12 | -------------------------------------------------------------------------------- /src/registry/api/tags.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "encoding/json" 5 | "io/ioutil" 6 | "net/http" 7 | "path" 8 | "registry/logger" 9 | "registry/storage" 10 | "strings" 11 | "time" 12 | ) 13 | 14 | var EMPTY_REPO_JSON = map[string]interface{}{ 15 | "last_update": nil, 16 | "docker_version": nil, 17 | "docker_go_version": nil, 18 | "arch": "amd64", 19 | "os": "linux", 20 | "kernel": nil, 21 | } 22 | 23 | func (a *RegistryAPI) GetRepoTagsHandler(w http.ResponseWriter, r *http.Request) { 24 | namespace, repo, _ := parseRepo(r, "") 25 | logger.Debug("[GetRepoTags] namespace=%s; repository=%s", namespace, repo) 26 | names, err := a.Storage.List(storage.RepoTagPath(namespace, repo, "")) 27 | if err != nil { 28 | a.response(w, "Repository not found: "+err.Error(), http.StatusNotFound, EMPTY_HEADERS) 29 | return 30 | } 31 | data := map[string]string{} 32 | for _, name := range names { 33 | base := path.Base(name) 34 | if !strings.HasPrefix(base, storage.TAG_PREFIX) { 35 | continue 36 | } 37 | // this is a tag 38 | tagName := strings.TrimPrefix(base, storage.TAG_PREFIX) 39 | content, err := a.Storage.Get(name) 40 | if err != nil { 41 | a.internalError(w, err.Error()) 42 | return 43 | } 44 | data[tagName] = string(content) 45 | } 46 | a.response(w, data, http.StatusOK, EMPTY_HEADERS) 47 | } 48 | 49 | func (a *RegistryAPI) DeleteRepoTagsHandler(w http.ResponseWriter, r *http.Request) { 50 | namespace, repo, _ := parseRepo(r, "") 51 | logger.Debug("[DeleteRepoTags] namespace=%s; repository=%s", namespace, repo) 52 | if err := a.Storage.RemoveAll(storage.RepoTagPath(namespace, repo, "")); err != nil { 53 | a.response(w, "Repository not found: "+err.Error(), http.StatusNotFound, EMPTY_HEADERS) 54 | return 55 | } 56 | a.response(w, true, http.StatusOK, EMPTY_HEADERS) 57 | } 58 | 59 | func (a *RegistryAPI) GetRepoTagHandler(w http.ResponseWriter, r *http.Request) { 60 | namespace, repo, tag := parseRepo(r, "tag") 61 | logger.Debug("[GetRepoTag] namespace=%s; repository=%s; tag=%s", namespace, repo, tag) 62 | content, err := a.Storage.Get(storage.RepoTagPath(namespace, repo, tag)) 63 | if err != nil { 64 | a.response(w, "Tag not found: "+err.Error(), http.StatusNotFound, EMPTY_HEADERS) 65 | return 66 | } 67 | a.response(w, content, http.StatusOK, EMPTY_HEADERS) 68 | } 69 | 70 | func (a *RegistryAPI) PutRepoTagHandler(w http.ResponseWriter, r *http.Request) { 71 | namespace, repo, tag := parseRepo(r, "tag") 72 | logger.Debug("[PutRepoTag] namespace=%s; repository=%s; tag=%s", namespace, repo, tag) 73 | data, err := ioutil.ReadAll(r.Body) 74 | if err != nil { 75 | a.response(w, "Error reading request body: "+err.Error(), http.StatusBadRequest, EMPTY_HEADERS) 76 | return 77 | } else if len(data) == 0 { 78 | a.response(w, "Empty data", http.StatusBadRequest, EMPTY_HEADERS) 79 | return 80 | } 81 | logger.Debug("[PutRepoTag] body:\n%s", data) 82 | imageID := strings.Trim(string(data), "\"") // trim quotes 83 | if exists, err := a.Storage.Exists(storage.ImageJsonPath(imageID)); err != nil || !exists { 84 | a.response(w, "Image not found: "+err.Error(), http.StatusNotFound, EMPTY_HEADERS) 85 | return 86 | } 87 | err = a.Storage.Put(storage.RepoTagPath(namespace, repo, tag), []byte(imageID)) 88 | if err != nil { 89 | a.internalError(w, err.Error()) 90 | return 91 | } 92 | uaStrings := r.Header["User-Agent"] 93 | uaString := "" 94 | if len(uaStrings) > 0 { 95 | // just use the first one. there *should* only be one to begin with. 96 | uaString = uaStrings[0] 97 | } 98 | dataMap := CreateRepoJson(uaString) 99 | jsonData, err := json.Marshal(&dataMap) 100 | if err != nil { 101 | a.internalError(w, err.Error()) 102 | return 103 | } 104 | a.Storage.Put(storage.RepoTagJsonPath(namespace, repo, tag), jsonData) 105 | if tag == "latest" { 106 | a.Storage.Put(storage.RepoJsonPath(namespace, repo), jsonData) 107 | } 108 | a.response(w, true, http.StatusOK, EMPTY_HEADERS) 109 | } 110 | 111 | func (a *RegistryAPI) DeleteRepoTagHandler(w http.ResponseWriter, r *http.Request) { 112 | namespace, repo, tag := parseRepo(r, "tag") 113 | logger.Debug("[DeleteRepoTag] namespace=%s; repository=%s; tag=%s", namespace, repo, tag) 114 | if err := a.Storage.Remove(storage.RepoTagPath(namespace, repo, tag)); err != nil { 115 | a.response(w, "Tag not found: "+err.Error(), http.StatusNotFound, EMPTY_HEADERS) 116 | return 117 | } 118 | a.response(w, true, http.StatusOK, EMPTY_HEADERS) 119 | } 120 | 121 | func (a *RegistryAPI) GetRepoJsonHandler(w http.ResponseWriter, r *http.Request) { 122 | namespace, repo, _ := parseRepo(r, "") 123 | logger.Debug("[GetRepoJson] namespace=%s; repository=%s", namespace, repo) 124 | content, err := a.Storage.Get(storage.RepoJsonPath(namespace, repo)) 125 | if err != nil { 126 | // docker-registry has this error ignored. so i guess we will too... 127 | a.response(w, EMPTY_REPO_JSON, http.StatusOK, EMPTY_HEADERS) 128 | return 129 | } 130 | var data map[string]interface{} 131 | if err := json.Unmarshal(content, &data); err != nil { 132 | // docker-registry has this error ignored. so i guess we will too... 133 | a.response(w, EMPTY_REPO_JSON, http.StatusOK, EMPTY_HEADERS) 134 | return 135 | } 136 | a.response(w, data, http.StatusOK, EMPTY_HEADERS) 137 | return 138 | } 139 | 140 | func CreateRepoJson(userAgent string) map[string]interface{} { 141 | props := map[string]interface{}{ 142 | "last_update": time.Now().Unix(), 143 | } 144 | matches := USER_AGENT_REGEXP.FindAllStringSubmatch(userAgent, -1) 145 | uaMap := map[string]string{} 146 | for _, match := range matches { 147 | if len(match) < 3 { 148 | continue 149 | } 150 | uaMap[match[1]] = match[2] 151 | } 152 | if val, exists := uaMap["docker"]; exists { 153 | props["docker_version"] = val 154 | } 155 | if val, exists := uaMap["go"]; exists { 156 | props["docker_go_version"] = val 157 | } 158 | if val, exists := uaMap["arch"]; exists { 159 | props["arch"] = strings.ToLower(val) 160 | } 161 | if val, exists := uaMap["kernel"]; exists { 162 | props["kernel"] = strings.ToLower(val) 163 | } 164 | if val, exists := uaMap["os"]; exists { 165 | props["os"] = strings.ToLower(val) 166 | } 167 | return props 168 | } 169 | 170 | func (a *RegistryAPI) DeleteRepoHandler(w http.ResponseWriter, r *http.Request) { 171 | namespace, repo, _ := parseRepo(r, "") 172 | err := a.Storage.RemoveAll(storage.RepoPath(namespace,repo)) 173 | if err != nil{ 174 | a.response(w, err.Error(), http.StatusNotFound, EMPTY_HEADERS) 175 | return 176 | } 177 | a.response(w, true, http.StatusOK, EMPTY_HEADERS) 178 | return 179 | } 180 | 181 | func (a *RegistryAPI) GetRepoTagJsonHandler(w http.ResponseWriter, r *http.Request) { 182 | namespace, repo, tag := parseRepo(r, "tag") 183 | data := map[string]string{ 184 | "last_update": "", 185 | "docker_version": "", 186 | "docker_go_version": "", 187 | "arch": "amd64", 188 | "os": "linux", 189 | "kernel": "", 190 | } 191 | content, err := a.Storage.Get(storage.RepoTagJsonPath(namespace, repo, tag)) 192 | if err != nil { 193 | a.response(w, data, http.StatusNotFound, EMPTY_HEADERS) 194 | return 195 | } 196 | a.response(w, content, http.StatusOK, EMPTY_HEADERS) 197 | return 198 | } 199 | -------------------------------------------------------------------------------- /src/registry/config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "encoding/json" 5 | "registry/api" 6 | "registry/storage" 7 | "os" 8 | ) 9 | 10 | type Config struct { 11 | API *api.Config `json:"api"` 12 | Storage *storage.Config `json:"storage"` 13 | } 14 | 15 | func New(filename string) (*Config, error) { 16 | // read in config 17 | var cfg Config 18 | if cfgFile, err := os.Open(filename); err != nil { 19 | return nil, err 20 | } else { 21 | dec := json.NewDecoder(cfgFile) 22 | if err := dec.Decode(&cfg); err != nil { 23 | return nil, err 24 | } 25 | } 26 | return &cfg, nil 27 | } 28 | -------------------------------------------------------------------------------- /src/registry/layers/tar.go: -------------------------------------------------------------------------------- 1 | package layers 2 | 3 | import ( 4 | "archive/tar" 5 | "compress/gzip" 6 | "crypto/sha256" 7 | "encoding/hex" 8 | "encoding/json" 9 | "fmt" 10 | "registry/logger" 11 | "hash" 12 | "io" 13 | "sort" 14 | "strings" 15 | ) 16 | 17 | const TAR_FILES_INFO_SIZE = 8 18 | 19 | type TarError string 20 | 21 | func (e TarError) Error() string { 22 | return string(e) 23 | } 24 | 25 | type TarInfo struct { 26 | TarSum *TarSum 27 | TarFilesInfo *TarFilesInfo 28 | Error error 29 | } 30 | 31 | func NewTarInfo() *TarInfo { 32 | return &TarInfo{ 33 | TarSum: NewTarSum(), 34 | TarFilesInfo: NewTarFilesInfo(), 35 | Error: nil, 36 | } 37 | } 38 | 39 | func (t *TarInfo) Load(file io.ReadSeeker) { 40 | var reader *tar.Reader 41 | file.Seek(0, 0) 42 | gzipReader, err := gzip.NewReader(file) 43 | if err != nil { 44 | // likely not a gzip compressed file 45 | file.Seek(0, 0) 46 | reader = tar.NewReader(file) 47 | } else { 48 | reader = tar.NewReader(gzipReader) 49 | } 50 | for { 51 | header, err := reader.Next() 52 | if err == io.EOF { 53 | // end of tar file 54 | break 55 | } else if err != nil { 56 | // error occured 57 | logger.Debug("[TarInfoLoad] Error when reading tar stream tarsum. Disabling TarSum, TarFilesInfo. Error: %s", err.Error()) 58 | t.Error = TarError(err.Error()) 59 | return 60 | } 61 | t.TarSum.Append(header, reader) 62 | t.TarFilesInfo.Append(header) 63 | } 64 | } 65 | 66 | type TarSum struct { 67 | hashes []string 68 | sha hash.Hash 69 | } 70 | 71 | func NewTarSum() *TarSum { 72 | return (&TarSum{}).init() 73 | } 74 | 75 | func (t *TarSum) init() *TarSum { 76 | t.hashes = []string{} 77 | t.sha = sha256.New() 78 | t.sha.Reset() 79 | return t 80 | } 81 | 82 | func (t *TarSum) Append(header *tar.Header, reader io.Reader) { 83 | headerStr := "name" + header.Name 84 | headerStr += fmt.Sprintf("mode%d", header.Mode) 85 | headerStr += fmt.Sprintf("uid%d", header.Uid) 86 | headerStr += fmt.Sprintf("gid%d", header.Gid) 87 | headerStr += fmt.Sprintf("size%d", header.Size) 88 | headerStr += fmt.Sprintf("mtime%d", header.ModTime.UTC().Unix()) 89 | headerStr += fmt.Sprintf("typeflag%c", header.Typeflag) 90 | headerStr += "linkname" + header.Linkname 91 | headerStr += "uname" + header.Uname 92 | headerStr += "gname" + header.Gname 93 | headerStr += fmt.Sprintf("devmajor%d", header.Devmajor) 94 | headerStr += fmt.Sprintf("devminor%d", header.Devminor) 95 | t.sha.Reset() 96 | if header.Size > int64(0) { 97 | t.sha.Write([]byte(headerStr)) 98 | _, err := io.Copy(t.sha, reader) 99 | if err != nil { 100 | logger.Debug("[TarSumAppend] error copying to sha: %s", err.Error()) 101 | t.sha.Reset() 102 | t.sha.Write([]byte(headerStr)) 103 | } 104 | } else { 105 | t.sha.Write([]byte(headerStr)) 106 | } 107 | t.hashes = append(t.hashes, hex.EncodeToString(t.sha.Sum(nil))) 108 | } 109 | 110 | func (t *TarSum) Compute(seed []byte) string { 111 | logger.Debug("[TarSumCompute] seed:\n<<%s>>", seed) 112 | sort.Strings(t.hashes) 113 | t.sha.Reset() 114 | t.sha.Write(seed) 115 | for _, hash := range t.hashes { 116 | t.sha.Write([]byte(hash)) 117 | } 118 | tarsum := "tarsum+sha256:" + hex.EncodeToString(t.sha.Sum(nil)) 119 | logger.Debug("[TarSumCompute] return %s", tarsum) 120 | return tarsum 121 | } 122 | 123 | type TarFilesInfo struct { 124 | headers []*tar.Header 125 | } 126 | 127 | func NewTarFilesInfo() *TarFilesInfo { 128 | return &TarFilesInfo{headers: []*tar.Header{}} 129 | } 130 | 131 | func (t *TarFilesInfo) Load(file io.Reader) error { 132 | reader := tar.NewReader(file) 133 | for { 134 | header, err := reader.Next() 135 | if err == io.EOF { 136 | // end of tar file 137 | break 138 | } else if err != nil { 139 | // error occured 140 | return TarError(err.Error()) 141 | } 142 | t.Append(header) 143 | } 144 | return nil 145 | } 146 | 147 | func (t *TarFilesInfo) Append(h *tar.Header) { 148 | t.headers = append(t.headers, h) 149 | } 150 | 151 | // This function returns json containing a slice of file info objects 152 | // file info is a weird tuple (why it isn't just a map i have no idea) 153 | // file info: 154 | // [ 155 | // filename, 156 | // file type, 157 | // is deleted, 158 | // size, 159 | // mod time, 160 | // mode, 161 | // uid, 162 | // gid 163 | // ] 164 | func (t *TarFilesInfo) Json() ([]byte, error) { 165 | // convert to the weird tuple docker-registry 0.6.5 uses (why wasn't this just a map!?) 166 | tupleSlice := [][]interface{}{} 167 | for _, header := range t.headers { 168 | filename := header.Name 169 | isDeleted := false 170 | if filename == "." { 171 | filename = "/" 172 | } 173 | if strings.HasPrefix(filename, "./") { 174 | filename = "/" + strings.TrimPrefix(filename, "./") 175 | } 176 | if strings.HasPrefix(filename, "/.wh.") { 177 | filename = "/" + strings.TrimPrefix(filename, "/.wh.") 178 | isDeleted = true 179 | } 180 | // NOTE(edanaher): Well, if filename started with /.wh..wh., this could trigger. Presumably, .wh is a 181 | // tombstone (WHiteout) indicating the file was deleted, and if it was recreated, you "delete" the 182 | // tombstone, which could conceivable mean a double-tombstone is a no-op. I feel like it would take more 183 | // logic to make that work, but it makes some semblance of sense. 184 | if strings.HasPrefix(filename, "/.wh.") { 185 | continue 186 | } 187 | 188 | filetype := "u" 189 | switch header.Typeflag { 190 | case tar.TypeReg: 191 | fallthrough 192 | case tar.TypeRegA: 193 | filetype = "f" 194 | case tar.TypeLink: 195 | filetype = "l" 196 | case tar.TypeSymlink: 197 | filetype = "s" 198 | case tar.TypeChar: 199 | filetype = "c" 200 | case tar.TypeBlock: 201 | filetype = "b" 202 | case tar.TypeDir: 203 | filetype = "d" 204 | case tar.TypeFifo: 205 | filetype = "i" 206 | case tar.TypeCont: 207 | filetype = "t" 208 | case tar.TypeGNULongName: 209 | fallthrough 210 | case tar.TypeGNULongLink: 211 | fallthrough 212 | case 'S': // GNU Sparse (for some reason archive/tar doesn't have a constant for it) 213 | filetype = string([]byte{header.Typeflag}) 214 | } 215 | 216 | tupleSlice = append(tupleSlice, []interface{}{ 217 | filename, 218 | filetype, 219 | isDeleted, 220 | header.Size, 221 | header.ModTime.Unix(), 222 | header.Mode, 223 | header.Uid, 224 | header.Gid, 225 | }) 226 | } 227 | return json.Marshal(&tupleSlice) 228 | } 229 | -------------------------------------------------------------------------------- /src/registry/layers/util.go: -------------------------------------------------------------------------------- 1 | package layers 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "regexp" 7 | "registry/logger" 8 | "registry/storage" 9 | "strings" 10 | ) 11 | 12 | // this function takes both []byte and []map[string]interface{} to shortcut in some cases. 13 | func UpdateIndexImages(s storage.Storage, namespace, repo string, additionalBytes []byte, 14 | additional []map[string]interface{}) error { 15 | path := storage.RepoIndexImagesPath(namespace, repo) 16 | // get previous content 17 | previousData, err := s.Get(path) 18 | if err != nil { 19 | // doesn't yet exist, just put the data 20 | return s.Put(path, additionalBytes) 21 | } 22 | var previous []map[string]interface{} 23 | if err := json.Unmarshal(previousData, &previous); err != nil { 24 | return err 25 | } 26 | if len(previous) == 0 { 27 | // nothing in previous, just put the data 28 | return s.Put(path, additionalBytes) 29 | } 30 | // Merge existing images with the incoming images. if the image ID exists in the existing, check to see if 31 | // the checksum is the same. if it is just continue, if it isn't replace it with the incoming image 32 | newImagesMap := map[string]map[string]interface{}{} 33 | for _, value := range additional { 34 | id, ok := value["id"].(string) 35 | if !ok { 36 | // json was screwed up 37 | return errors.New("Invalid Data") 38 | } 39 | if imageData, ok := newImagesMap[id]; ok { 40 | if _, ok := imageData["checksum"]; ok { 41 | continue 42 | } 43 | } 44 | newImagesMap[id] = value 45 | } 46 | for _, value := range previous { 47 | id, ok := value["id"].(string) 48 | if !ok { 49 | // json was screwed up 50 | return errors.New("Invalid Data") 51 | } 52 | if imageData, ok := newImagesMap[id]; ok { 53 | if _, ok := imageData["checksum"]; ok { 54 | continue 55 | } 56 | } 57 | newImagesMap[id] = value 58 | } 59 | newImagesArr := make([]map[string]interface{}, len(newImagesMap)) 60 | i := 0 61 | for _, image := range newImagesMap { 62 | newImagesArr[i] = image 63 | i++ 64 | } 65 | data, err := json.Marshal(&newImagesArr) 66 | if err != nil { 67 | return err 68 | } 69 | return s.Put(path, data) 70 | } 71 | 72 | func GetImageFilesCache(s storage.Storage, imageID string) ([]byte, error) { 73 | return s.Get(storage.ImageFilesPath(imageID)) 74 | } 75 | 76 | func SetImageFilesCache(s storage.Storage, imageID string, filesJson []byte) error { 77 | return s.Put(storage.ImageFilesPath(imageID), filesJson) 78 | } 79 | 80 | // return json file listing for given image id 81 | // Download the specified layer and determine the file contents. If the cache already exists, just return it. 82 | func GetImageFilesJson(s storage.Storage, imageID string) ([]byte, error) { 83 | // if the files json exists in the cache, return it 84 | filesJson, err := GetImageFilesCache(s, imageID) 85 | if err != nil { 86 | return filesJson, nil 87 | } 88 | 89 | // cache doesn't exist. download remote layer 90 | // docker-registry 0.6.5 has an lzma decompress here. it actually doesn't seem to be used so i've omitted it 91 | // will add it later if need be. 92 | tarFilesInfo := NewTarFilesInfo() 93 | if reader, err := s.GetReader(storage.ImageLayerPath(imageID)); err != nil { 94 | return nil, err 95 | } else if err := tarFilesInfo.Load(reader); err != nil { 96 | return nil, err 97 | } 98 | return tarFilesInfo.Json() 99 | } 100 | 101 | func StoreChecksum(s storage.Storage, imageID string, checksums []string) error { 102 | for _, checksum := range checksums { 103 | parts := strings.Split(checksum, ":") 104 | if len(parts) != 2 { 105 | return errors.New("Invalid checksum format") 106 | } 107 | } 108 | content, err := json.Marshal(checksums) 109 | if err != nil { 110 | return err 111 | } 112 | return s.Put(storage.ImageChecksumPath(imageID), content) 113 | } 114 | 115 | func GenerateAncestry(s storage.Storage, imageID, parentID string) (err error) { 116 | logger.Debug("[GenerateAncestry] imageID=" + imageID + " parentID=" + parentID) 117 | path := storage.ImageAncestryPath(imageID) 118 | if parentID == "" { 119 | return s.Put(path, []byte(`["`+imageID+`"]`)) 120 | } 121 | var content []byte 122 | if content, err = s.Get(storage.ImageAncestryPath(parentID)); err != nil { 123 | return err 124 | } 125 | var ancestry []string 126 | if err := json.Unmarshal(content, &ancestry); err != nil { 127 | return err 128 | } 129 | ancestry = append([]string{imageID}, ancestry...) 130 | if content, err = json.Marshal(&ancestry); err != nil { 131 | return err 132 | } 133 | return s.Put(path, content) 134 | } 135 | 136 | func GetImageDiffCache(s storage.Storage, imageID string) ([]byte, error) { 137 | path := storage.ImageDiffPath(imageID) 138 | if exists, _ := s.Exists(path); exists { 139 | return s.Get(storage.ImageDiffPath(imageID)) 140 | } 141 | // that indicates miss/successful hit/cache error... 142 | // weird that we have no way of knowing that this is a cache miss outside of this function, but this is how 143 | // docker-registry does it so we'll follow... 144 | return nil, nil // nil error, because cache missed 145 | } 146 | 147 | func SetImageDiffCache(s storage.Storage, imageID string, diffJson []byte) error { 148 | return s.Put(storage.ImageDiffPath(imageID), diffJson) 149 | } 150 | 151 | func GenDiff(s storage.Storage, imageID string) { 152 | // Comment from docker-registry 0.6.5 153 | // get json describing file differences in layer 154 | // Calculate the diff information for the files contained within 155 | // the layer. Return a dictionary of lists grouped by whether they 156 | // were deleted, changed or created in this layer. 157 | // To determine what happened to a file in a layer we walk backwards 158 | // through the ancestry until we see the file in an older layer. Based 159 | // on whether the file was previously deleted or not we know whether 160 | // the file was created or modified. If we do not find the file in an 161 | // ancestor we know the file was just created. 162 | // - File marked as deleted by union fs tar: DELETED 163 | // - Ancestor contains non-deleted file: CHANGED 164 | // - Ancestor contains deleted marked file: CREATED 165 | // - No ancestor contains file: CREATED 166 | 167 | diffJson, err := GetImageDiffCache(s, imageID) 168 | if err == nil && diffJson != nil { 169 | // cache hit, just return 170 | logger.Debug("[GenDiff][" + imageID + "] already exists") 171 | return 172 | } 173 | 174 | anPath := storage.ImageAncestryPath(imageID) 175 | anContent, err := s.Get(anPath) 176 | if err != nil { 177 | // error fetching ancestry, just return 178 | logger.Error("[GenDiff][" + imageID + "] error fetching ancestry: " + err.Error()) 179 | return 180 | } 181 | var ancestry []string 182 | if err := json.Unmarshal(anContent, &ancestry); err != nil { 183 | // json unmarshal fail, just return 184 | logger.Error("[GenDiff][" + imageID + "] error unmarshalling ancestry json: " + err.Error()) 185 | return 186 | } 187 | // get map of file infos 188 | infoMap, err := fileInfoMap(s, imageID) 189 | if err != nil { 190 | // error getting file info, just return 191 | logger.Error("[GenDiff][" + imageID + "] error getting files info: " + err.Error()) 192 | return 193 | } 194 | 195 | deleted := map[string][]interface{}{} 196 | changed := map[string][]interface{}{} 197 | created := map[string][]interface{}{} 198 | 199 | for _, anID := range ancestry { 200 | anInfoMap, err := fileInfoMap(s, anID) 201 | if err != nil { 202 | // error getting file info, just return 203 | logger.Error("[GenDiff][" + imageID + "] error getting ancestor " + anID + " files info: " + err.Error()) 204 | return 205 | } 206 | for fname, info := range infoMap { 207 | isDeleted, isBool := (info[1]).(bool) 208 | // if the file info is in a bad format (isDeleted is not a bool), we should just assume it is deleted. 209 | // technically isBool should never be false. 210 | if !isBool || isDeleted { 211 | if !isBool { 212 | logger.Error("[GenDiff][" + imageID + "] file info is in a bad format") 213 | } 214 | deleted[fname] = info 215 | delete(infoMap, fname) 216 | continue 217 | } 218 | anInfo := anInfoMap[fname] 219 | if err != nil || anInfo == nil { 220 | // doesn't exist, must be created. do nothing. 221 | continue 222 | } 223 | isDeleted, isBool = anInfo[1].(bool) 224 | if !isBool || isDeleted { 225 | if !isBool { 226 | logger.Error("[GenDiff][" + imageID + "] file info is in a bad format") 227 | } 228 | // deleted in ancestor, must be created now. 229 | created[fname] = info 230 | } else { 231 | // not deleted in ancestor, must have just changed now. 232 | changed[fname] = info 233 | } 234 | delete(infoMap, fname) 235 | } 236 | } 237 | // dump all created stuff from infoMap 238 | for fname, info := range infoMap { 239 | created[fname] = info 240 | } 241 | 242 | diff := map[string]map[string][]interface{}{ 243 | "deleted": deleted, 244 | "changed": changed, 245 | "created": created, 246 | } 247 | if diffJson, err = json.Marshal(&diff); err != nil { 248 | // json marshal fail. just return 249 | logger.Error("[GenDiff][" + imageID + "] error marshalling new diff json: " + err.Error()) 250 | return 251 | } 252 | if err := SetImageDiffCache(s, imageID, diffJson); err != nil { 253 | // json marshal fail. just return 254 | logger.Error("[GenDiff][" + imageID + "] error setting new diff cache: " + err.Error()) 255 | return 256 | } 257 | } 258 | 259 | // This function returns a map of file name -> file info for all files found in the image imageID. 260 | // file info is a weird tuple (why it isn't just a map i have no idea) 261 | // file info: 262 | // [ 263 | // filename, 264 | // file type, 265 | // is deleted, 266 | // size, 267 | // mod time, 268 | // mode, 269 | // uid, 270 | // gid 271 | // ] 272 | // this function also strips filename out of the fileinfo before it sets it as a value in the map. 273 | func fileInfoMap(s storage.Storage, imageID string) (map[string][]interface{}, error) { 274 | fContent, err := GetImageFilesJson(s, imageID) 275 | if err != nil { 276 | return nil, err 277 | } 278 | var infoArr [][]interface{} 279 | if err := json.Unmarshal(fContent, &infoArr); err != nil { 280 | return nil, err 281 | } 282 | m := make(map[string][]interface{}, len(infoArr)) 283 | for _, info := range infoArr { 284 | if len(info) != TAR_FILES_INFO_SIZE { 285 | continue 286 | } 287 | if nameStr, ok := info[0].(string); ok { 288 | m[nameStr] = info[1:] 289 | } 290 | } 291 | return m, nil 292 | } 293 | 294 | func DockerVersion(ua []string) (string, error) { 295 | docker_version_pattern := "docker/([^\\s]+)" 296 | re := regexp.MustCompile(docker_version_pattern) 297 | match := re.FindStringSubmatch(ua[0]) 298 | if len(match) != 0 { 299 | return match[1], nil 300 | } 301 | return "", errors.New("Cannot parse Docker version") 302 | } 303 | -------------------------------------------------------------------------------- /src/registry/logger/logger.go: -------------------------------------------------------------------------------- 1 | package logger 2 | 3 | import ( 4 | "log" 5 | ) 6 | 7 | var debug = false 8 | 9 | func DebugOn() { 10 | debug = true 11 | } 12 | 13 | func DebugOff() { 14 | debug = false 15 | } 16 | 17 | func Info(fmt string, args ...interface{}) { 18 | log.Printf("[INFO] "+fmt, args...) 19 | } 20 | 21 | func Error(fmt string, args ...interface{}) { 22 | log.Printf("[ERROR] "+fmt, args...) 23 | } 24 | 25 | func Fatal(fmt string, args ...interface{}) { 26 | log.Fatalf("[FATAL] "+fmt, args...) 27 | } 28 | 29 | func Debug(fmt string, args ...interface{}) { 30 | if debug { 31 | log.Printf("[DEBUG] "+fmt, args...) 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/registry/storage/local.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "errors" 5 | "io" 6 | "io/ioutil" 7 | "os" 8 | "path" 9 | "strings" 10 | ) 11 | 12 | type Local struct { 13 | Root string `json:"root"` 14 | } 15 | 16 | func (s *Local) init() error { 17 | return os.MkdirAll(s.Root, 0755) 18 | } 19 | 20 | func (s *Local) createFile(relpath string) (*os.File, error) { 21 | abspath := path.Join(s.Root, relpath) 22 | if err := os.MkdirAll(path.Dir(abspath), 0755); err != nil { 23 | return nil, err 24 | } 25 | return os.OpenFile(abspath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) 26 | } 27 | 28 | func (s *Local) Get(relpath string) ([]byte, error) { 29 | return ioutil.ReadFile(path.Join(s.Root, relpath)) 30 | } 31 | 32 | func (s *Local) Put(relpath string, data []byte) (err error) { 33 | var file *os.File 34 | if file, err = s.createFile(relpath); err != nil { 35 | return err 36 | } 37 | defer file.Close() 38 | _, err = file.Write(data) 39 | return err 40 | } 41 | 42 | func (s *Local) GetReader(relpath string) (io.ReadCloser, error) { 43 | return os.Open(path.Join(s.Root, relpath)) 44 | } 45 | 46 | func (s *Local) PutReader(relpath string, r io.Reader, afterWrite func(io.ReadSeeker)) error { 47 | file, err := s.createFile(relpath) 48 | if err != nil { 49 | return err 50 | } 51 | defer func() { 52 | file.Seek(0, 0) 53 | afterWrite(file) 54 | file.Close() 55 | }() 56 | _, err = io.Copy(file, r) 57 | return err 58 | } 59 | 60 | func (s *Local) List(relpath string) ([]string, error) { 61 | abspath := path.Join(s.Root, relpath) 62 | infos, err := ioutil.ReadDir(abspath) 63 | if err != nil { 64 | return nil, err 65 | } 66 | if len(infos) == 0 { 67 | // to be consistent with S3, return no such file or directory here. from docker-registry 0.6.5 68 | return nil, errors.New("open " + abspath + ": no such file or directory") 69 | } 70 | list := make([]string, len(infos)) 71 | for i, info := range infos { 72 | list[i] = path.Join(relpath, info.Name()) 73 | if !strings.HasPrefix(list[i], "/") { 74 | list[i] = "/" + list[i] 75 | } 76 | } 77 | return list, nil 78 | } 79 | 80 | func (s *Local) Exists(relpath string) (bool, error) { 81 | info, err := os.Stat(path.Join(s.Root, relpath)) 82 | if os.IsNotExist(err) { 83 | return false, nil 84 | } 85 | return info != nil, err 86 | } 87 | 88 | func (s *Local) Size(relpath string) (int64, error) { 89 | info, err := os.Stat(path.Join(s.Root, relpath)) 90 | if info == nil || err != nil { 91 | // dunno size 92 | return -1, err 93 | } 94 | return info.Size(), nil 95 | } 96 | 97 | func (s *Local) Remove(relpath string) error { 98 | // this is not abspath because Exists uses relpath 99 | if ok, err := s.Exists(relpath); !ok || err != nil { 100 | return errors.New("no such file or directory: " + relpath) 101 | } 102 | abspath := path.Join(s.Root, relpath) 103 | err := os.Remove(abspath) 104 | if err != nil { 105 | return err 106 | } 107 | for absdir := path.Dir(abspath); s.removeIfEmpty(absdir); absdir = path.Dir(absdir) { 108 | // loop over parent directories and remove them if empty 109 | // we do this because that is how s3 looks since it is purely a key-value store 110 | } 111 | return nil 112 | } 113 | 114 | func (s *Local) RemoveAll(relpath string) error { 115 | // this is not abspath because Exists uses relpath 116 | if ok, err := s.Exists(relpath); !ok || err != nil { 117 | return errors.New("no such file or directory: " + relpath) 118 | } 119 | abspath := path.Join(s.Root, relpath) 120 | err := os.RemoveAll(abspath) 121 | if err != nil { 122 | return err 123 | } 124 | for absdir := path.Dir(abspath); s.removeIfEmpty(absdir); absdir = path.Dir(absdir) { 125 | // loop over parent directires and remove them if empty 126 | // we do this because that is how s3 looks since it is purely a key-value store 127 | } 128 | return nil 129 | } 130 | 131 | func (s *Local) removeIfEmpty(dir string) bool { 132 | infos, err := ioutil.ReadDir(dir) 133 | if err != nil { 134 | // eh. something weird happened, don't do anything 135 | return false 136 | } 137 | if len(infos) != 0 { 138 | // not empty, don't delete 139 | return false 140 | } 141 | return os.Remove(dir) == nil 142 | } 143 | -------------------------------------------------------------------------------- /src/registry/storage/local_test.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestLocal(t *testing.T) { 8 | testStorage(t, &Local{ 9 | Root: "/tmp/go-docker-registry-test", 10 | }) 11 | } 12 | -------------------------------------------------------------------------------- /src/registry/storage/s3.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "crypto/sha256" 5 | "errors" 6 | "fmt" 7 | "github.com/crowdmob/goamz/aws" 8 | "github.com/crowdmob/goamz/s3" 9 | "io" 10 | "os" 11 | "path" 12 | "strings" 13 | "sync" 14 | "time" 15 | ) 16 | 17 | const S3_CONTENT_TYPE = "application/binary" 18 | 19 | var S3_OPTIONS = s3.Options{} 20 | var EMPTY_HEADERS = map[string][]string{} 21 | 22 | type S3 struct { 23 | auth aws.Auth 24 | authLock sync.RWMutex // lock for the auth so we can update it when we need to 25 | region aws.Region 26 | s3 *s3.S3 27 | bucket *s3.Bucket 28 | bufferDir *BufferDir // used to buffer content if length is unknown 29 | root string // sanitized root (no leading slash) 30 | 31 | Region string `json:"region"` 32 | Bucket string `json:"bucket"` 33 | Root string `json:"root"` 34 | BufferDir string `json:"buffer_dir"` 35 | AccessKey string `json:"access_key"` 36 | SecretKey string `json:"secret_key"` 37 | } 38 | 39 | func (s *S3) getAuth() (err error) { 40 | s.auth, err = aws.GetAuth(s.AccessKey, s.SecretKey, "", time.Time{}) 41 | if s.s3 != nil { 42 | s.s3.Auth = s.auth 43 | } 44 | return 45 | } 46 | 47 | func (s *S3) updateAuth() { 48 | s.authLock.Lock() 49 | defer s.authLock.Unlock() 50 | for err := s.getAuth(); err != nil; err = s.getAuth() { 51 | time.Sleep(5 * time.Second) 52 | } 53 | } 54 | 55 | func (s *S3) updateAuthLoop() { 56 | // this function just updates the auth. s.auth should be set before this is called 57 | // this is primarily used for role tagged ec2 instances who get an expiry with their auth. 58 | // if you set the access key and secret in environment variables, this will exit immediately. 59 | for { 60 | if s.auth.Expiration().IsZero() { 61 | // no reason to update, expiration is zero. 62 | return 63 | } 64 | if diff := s.auth.Expiration().Sub(time.Now()); diff < 0 { 65 | // if we're past the expiration time, update the auth 66 | s.updateAuth() 67 | } else { 68 | // if we're not past the expiration time, sleep until the expiration time is up 69 | time.Sleep(diff) 70 | } 71 | } 72 | } 73 | 74 | func (s *S3) init() error { 75 | if s.Bucket == "" { 76 | return errors.New("Please Specify an S3 Bucket") 77 | } 78 | if s.Region == "" { 79 | return errors.New("Please Specify an S3 Region") 80 | } 81 | if s.Root == "" { 82 | return errors.New("Please Specify an S3 Root Path") 83 | } 84 | if s.BufferDir == "" { 85 | return errors.New("Please Specify a Buffer Directory to use for Uploads") 86 | } 87 | 88 | var ok bool 89 | if s.region, ok = aws.Regions[s.Region]; !ok { 90 | return errors.New("Invalid Region: " + s.Region) 91 | } 92 | err := s.getAuth() 93 | if err != nil { 94 | return err 95 | } 96 | s.s3 = s3.New(s.auth, s.region) 97 | s.bucket = s.s3.Bucket(s.Bucket) 98 | if err := os.MkdirAll(s.BufferDir, 0755); err != nil && !os.IsExist(err) { 99 | // there was an error and it wasn't that the directory already exists 100 | return err 101 | } 102 | s.bufferDir = &BufferDir{Mutex: sync.Mutex{}, root: s.BufferDir} 103 | s.root = strings.TrimPrefix(s.Root, "/") 104 | go s.updateAuthLoop() 105 | return nil 106 | } 107 | 108 | func (s *S3) key(relpath string) string { 109 | return path.Join(s.root, relpath) // s3 expects no leading slash in some operations 110 | } 111 | 112 | func (s *S3) Get(relpath string) ([]byte, error) { 113 | s.authLock.RLock() 114 | defer s.authLock.RUnlock() 115 | return s.bucket.Get(s.key(relpath)) 116 | } 117 | 118 | func (s *S3) Put(relpath string, data []byte) error { 119 | s.authLock.RLock() 120 | defer s.authLock.RUnlock() 121 | return s.bucket.Put(s.key(relpath), data, S3_CONTENT_TYPE, s3.Private, S3_OPTIONS) 122 | } 123 | 124 | func (s *S3) GetReader(relpath string) (io.ReadCloser, error) { 125 | s.authLock.RLock() 126 | defer s.authLock.RUnlock() 127 | return s.bucket.GetReader(s.key(relpath)) 128 | } 129 | 130 | func (s *S3) PutReader(relpath string, r io.Reader, afterWrite func(io.ReadSeeker)) error { 131 | key := s.key(relpath) 132 | buffer, err := s.bufferDir.reserve(key) 133 | if err != nil { 134 | return err 135 | } 136 | defer buffer.release(afterWrite) 137 | // don't know the length, buffer to file first 138 | length, err := io.Copy(buffer, r) 139 | if err != nil { 140 | return err 141 | } 142 | buffer.Seek(0, 0) // seek to the beginning of the file 143 | // we know the length, write to s3 from file now 144 | return s.bucket.PutReader(s.key(relpath), buffer, length, S3_CONTENT_TYPE, s3.Private, S3_OPTIONS) 145 | } 146 | 147 | func (s *S3) List(relpath string) ([]string, error) { 148 | s.authLock.RLock() 149 | defer s.authLock.RUnlock() 150 | result, err := s.bucket.List(s.key(relpath)+"/", "/", "", 0) 151 | if err != nil { 152 | return nil, err 153 | } 154 | names := make([]string, len(result.Contents)+len(result.CommonPrefixes)) 155 | for i, key := range result.Contents { 156 | names[i] = strings.TrimPrefix(key.Key, s.root) 157 | if !strings.HasPrefix(names[i], "/") { 158 | names[i] = "/" + names[i] 159 | } 160 | } 161 | for i, prefix := range result.CommonPrefixes { 162 | prefixIdx := i+len(result.Contents) 163 | // trim trailing "/" and preceeding s.root 164 | names[prefixIdx] = strings.TrimPrefix(strings.TrimSuffix(prefix, "/"), s.root) 165 | // if there is no preceeding / then add it 166 | if !strings.HasPrefix(names[prefixIdx], "/") { 167 | names[prefixIdx] = "/" + names[prefixIdx] 168 | } 169 | } 170 | if len(names) == 0 { 171 | // nothing there. return an error. 172 | return nil, errors.New("No keys exist in " + s.key(relpath)) 173 | } 174 | return names, nil 175 | } 176 | 177 | func (s *S3) Exists(relpath string) (bool, error) { 178 | s.authLock.RLock() 179 | defer s.authLock.RUnlock() 180 | return s.bucket.Exists(s.key(relpath)) 181 | } 182 | 183 | func (s *S3) Size(relpath string) (int64, error) { 184 | s.authLock.RLock() 185 | defer s.authLock.RUnlock() 186 | resp, err := s.bucket.Head(s.key(relpath), EMPTY_HEADERS) 187 | if err != nil { 188 | return -1, err 189 | } 190 | return resp.ContentLength, nil 191 | } 192 | 193 | func (s *S3) Remove(relpath string) error { 194 | s.authLock.RLock() 195 | defer s.authLock.RUnlock() 196 | if exists, err := s.bucket.Exists(s.key(relpath)); !exists || err != nil { 197 | return errors.New("no such file or directory: " + relpath) 198 | } 199 | return s.bucket.Del(s.key(relpath)) 200 | } 201 | 202 | func (s *S3) RemoveAll(relpath string) error { 203 | // find and remove everything "under" it 204 | s.authLock.RLock() 205 | defer s.authLock.RUnlock() 206 | result, err := s.bucket.List(s.key(relpath)+"/", "", "", 0) 207 | if err != nil { 208 | return err 209 | } 210 | if len(result.Contents) == 0 { 211 | // nothing under it, return error 212 | return errors.New("no such file or directory " + relpath) 213 | } 214 | for _, key := range result.Contents { 215 | s.bucket.Del(key.Key) 216 | } 217 | // finally, remove it if needed 218 | return s.bucket.Del(s.key(relpath)) 219 | } 220 | 221 | // This will ensure that we don't try to upload the same thing from two different requests at the same time 222 | type BufferDir struct { 223 | sync.Mutex 224 | root string 225 | } 226 | 227 | func (b *BufferDir) reserve(key string) (*Buffer, error) { 228 | b.Lock() 229 | defer b.Unlock() 230 | // sha key path and create temporary file 231 | filepath := path.Join(b.root, fmt.Sprintf("%x", sha256.Sum256([]byte(key)))) 232 | if _, err := os.Stat(filepath); !os.IsNotExist(err) { 233 | // buffer file already exists 234 | return nil, errors.New("Upload already in progress for key " + key) 235 | } 236 | // if not exist, create buffer file 237 | file, err := os.Create(filepath) 238 | if err != nil { 239 | return nil, err 240 | } 241 | return &Buffer{File: *file, dir: b}, nil 242 | } 243 | 244 | type Buffer struct { 245 | os.File 246 | dir *BufferDir 247 | } 248 | 249 | func (b *Buffer) release(beforeRelease func(io.ReadSeeker)) { 250 | b.dir.Lock() 251 | defer b.dir.Unlock() 252 | b.Seek(0, 0) 253 | beforeRelease(&b.File) 254 | b.Close() 255 | os.Remove(b.Name()) 256 | } 257 | -------------------------------------------------------------------------------- /src/registry/storage/s3_test.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | ) 7 | 8 | func TestS3(t *testing.T) { 9 | // read test config. has sensitive data so pass filename in as env variable 10 | var s3 S3 11 | err := storageFromFile(os.Getenv("TEST_S3_CONFIG"), &s3) 12 | if err != nil { 13 | t.Fatal(err) 14 | } 15 | testStorage(t, &s3) 16 | } 17 | -------------------------------------------------------------------------------- /src/registry/storage/storage.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "io" 7 | "path" 8 | ) 9 | 10 | const TAG_PREFIX = "tag_" 11 | 12 | type Storage interface { 13 | init() error 14 | 15 | Get(string) ([]byte, error) 16 | Put(string, []byte) error 17 | GetReader(string) (io.ReadCloser, error) 18 | PutReader(string, io.Reader, func(io.ReadSeeker)) error 19 | List(string) ([]string, error) 20 | Exists(string) (bool, error) 21 | Size(string) (int64, error) 22 | Remove(string) error 23 | RemoveAll(string) error 24 | } 25 | 26 | type Config struct { 27 | Type string `json:"type"` 28 | Local *Local `json:"local"` 29 | S3 *S3 `json:"s3"` 30 | } 31 | 32 | func New(cfg *Config) (Storage, error) { 33 | switch cfg.Type { 34 | case "local": 35 | if cfg.Local != nil { 36 | return cfg.Local, cfg.Local.init() 37 | } 38 | return nil, errors.New("No config for storage type 'local' found") 39 | case "s3": 40 | if cfg.S3 != nil { 41 | return cfg.S3, cfg.S3.init() 42 | } 43 | return nil, errors.New("No config for storage type 's3' found") 44 | default: 45 | return nil, errors.New("Invalid storage type: " + cfg.Type) 46 | } 47 | } 48 | 49 | func ImageJsonPath(id string) string { 50 | return fmt.Sprintf("images/%s/json", id) 51 | } 52 | 53 | func ImageMarkPath(id string) string { 54 | return fmt.Sprintf("images/%s/_inprogress", id) 55 | } 56 | 57 | func ImageChecksumPath(id string) string { 58 | return fmt.Sprintf("images/%s/_checksum", id) 59 | } 60 | 61 | func ImageLayerPath(id string) string { 62 | return fmt.Sprintf("images/%s/layer", id) 63 | } 64 | 65 | func ImageAncestryPath(id string) string { 66 | return fmt.Sprintf("images/%s/ancestry", id) 67 | } 68 | 69 | func ImageFilesPath(id string) string { 70 | return fmt.Sprintf("images/%s/_files", id) 71 | } 72 | 73 | func ImageDiffPath(id string) string { 74 | return fmt.Sprintf("images/%s/_diff", id) 75 | } 76 | 77 | func RepoImagesListPath(namespace, repo string) string { 78 | return fmt.Sprintf("repositories/%s/_images_list", path.Join(namespace, repo)) 79 | } 80 | 81 | func RepoTagPath(namespace, repo, tag string) string { 82 | if tag == "" { 83 | return fmt.Sprintf("repositories/%s", path.Join(namespace, repo)) 84 | } 85 | return fmt.Sprintf("repositories/%s/%s", path.Join(namespace, repo), TAG_PREFIX+tag) 86 | } 87 | 88 | func RepoJsonPath(namespace, repo string) string { 89 | return fmt.Sprintf("repositories/%s/json", path.Join(namespace, repo)) 90 | } 91 | 92 | func RepoIndexImagesPath(namespace, repo string) string { 93 | return fmt.Sprintf("repositories/%s/_index_images", path.Join(namespace, repo)) 94 | } 95 | 96 | func RepoPrivatePath(namespace, repo string) string { 97 | return fmt.Sprintf("repositories/%s/_private", path.Join(namespace, repo)) 98 | } 99 | 100 | func RepoTagJsonPath(namespace, repo, tag string) string { 101 | tag = "tag" + tag + "_json" 102 | return fmt.Sprintf("repositories/%s", path.Join(namespace, repo, tag)) 103 | } 104 | 105 | func RepoPath(namespace, repo string) string { 106 | return fmt.Sprintf("repositories/%s", path.Join(namespace, repo)) 107 | } -------------------------------------------------------------------------------- /src/registry/storage/storage_test.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "io/ioutil" 7 | "os" 8 | "testing" 9 | ) 10 | 11 | func checkSlices(t *testing.T, got, expected []string) { 12 | diffMap := map[string]int{} 13 | for _, val := range got { 14 | diffMap[val]++ 15 | } 16 | for _, val := range expected { 17 | diffMap[val]-- 18 | if diffMap[val] == 0 { 19 | delete(diffMap, val) 20 | } 21 | } 22 | if len(diffMap) == 0 { 23 | return 24 | } 25 | t.Fatalf("Slices not equal. got %+v, expected %+v", got, expected) 26 | } 27 | 28 | func storageFromFile(filename string, storage Storage) error { 29 | file, err := os.Open(filename) 30 | if err != nil { 31 | return err 32 | } 33 | defer file.Close() 34 | dec := json.NewDecoder(file) 35 | if err := dec.Decode(storage); err != nil { 36 | return err 37 | } 38 | return storage.init() 39 | } 40 | 41 | func testStorage(t *testing.T, storage Storage) { 42 | // remove all to initialize 43 | storage.RemoveAll("/") 44 | if _, err := storage.List("/"); err == nil { 45 | t.Fatal("According to docker 0.6.5, listing an empty directory should return an error") 46 | } 47 | 48 | testGetPutExistsSizeRemove(t, storage) 49 | testGetPutReaders(t, storage) 50 | testListRemoveAll(t, storage) 51 | 52 | // cleanup 53 | storage.RemoveAll("/") 54 | if _, err := storage.List("/"); err == nil { 55 | t.Fatal("According to docker 0.6.5, listing an empty directory should return an error") 56 | } 57 | } 58 | 59 | func testGetPutExistsSizeRemove(t *testing.T, storage Storage) { 60 | if exists, _ := storage.Exists("/1"); exists == true { 61 | t.Fatal("Key should not exist yet") 62 | } 63 | if _, err := storage.Get("/1"); err == nil { 64 | t.Fatal("Getting something that doesn't exist should cause an error") 65 | } 66 | if err := storage.Remove("/1"); err == nil { 67 | t.Fatal("Removing something that doesn't exist should cause an error") 68 | } 69 | if err := storage.Put("/1", []byte("lolwtf")); err != nil { 70 | t.Fatal(err) 71 | } 72 | if exists, _ := storage.Exists("/1"); exists == false { 73 | t.Fatal("Key should exist now") 74 | } 75 | if size, err := storage.Size("/1"); err != nil { 76 | t.Fatal("Size should not result in an error") 77 | } else if size != int64(len("lolwtf")) { 78 | t.Fatalf("Size should be %d", len("lolwtf")) 79 | } 80 | if content, err := storage.Get("/1"); err != nil { 81 | t.Fatal(err) 82 | } else if string(content) != "lolwtf" { 83 | t.Log("the content should be 'lolwtf' was '" + string(content) + "'") 84 | t.FailNow() 85 | } 86 | if err := storage.Remove("/1"); err != nil { 87 | t.Fatal(err) 88 | } 89 | if _, err := storage.List("/"); err == nil { 90 | t.Fatal("According to docker 0.6.5, listing an empty directory should return an error") 91 | } 92 | } 93 | 94 | func testGetPutReaders(t *testing.T, storage Storage) { 95 | if exists, _ := storage.Exists("/dir/1"); exists == true { 96 | t.Fatal("Key should not exist yet") 97 | } 98 | if _, err := storage.GetReader("/dir/1"); err == nil { 99 | t.Fatal("Getting something that doesn't exist should cause an error") 100 | } 101 | if err := storage.Remove("/dir/1"); err == nil { 102 | t.Fatal("Removing something that doesn't exist should cause an error") 103 | } 104 | fileSize := int64(-1) 105 | afterWrite := func(file *os.File) { 106 | info, err := file.Stat() 107 | if err != nil { 108 | fileSize = -2 109 | return 110 | } 111 | fileSize = info.Size() 112 | } 113 | if err := storage.PutReader("/dir/1", bytes.NewBufferString("lolwtfdir"), afterWrite); err != nil { 114 | t.Fatal(err) 115 | } 116 | if fileSize == -1 { 117 | t.Fatal("afterWrite should have been called!") 118 | } else if fileSize == -2 { 119 | t.Fatal("afterWrite should have a proper handle on a file!") 120 | } else if fileSize != int64(len("lolwtfdir")) { 121 | t.Fatal("afterWrite should have the correct file size!") 122 | } 123 | if size, err := storage.Size("/dir/1"); err != nil { 124 | t.Fatal("Size should not result in an error") 125 | } else if size != int64(len("lolwtfdir")) { 126 | t.Fatalf("Size should be %d", len("lolwtfdir")) 127 | } 128 | if exists, _ := storage.Exists("/dir/1"); exists == false { 129 | t.Fatal("Key should exist now") 130 | } 131 | if reader, err := storage.GetReader("/dir/1"); err != nil { 132 | t.Fatal(err) 133 | } else { 134 | content, err := ioutil.ReadAll(reader) 135 | if err != nil { 136 | t.Fatal(err) 137 | } 138 | if string(content) != "lolwtfdir" { 139 | t.Log("the content should be 'lolwtfdir' was '" + string(content) + "'") 140 | t.FailNow() 141 | } 142 | } 143 | if err := storage.Remove("/dir/1"); err != nil { 144 | t.Fatal(err) 145 | } 146 | if _, err := storage.List("/dir"); err == nil { 147 | t.Fatal("According to docker 0.6.5, listing an empty directory should return an error") 148 | } 149 | if names, err := storage.List("/"); err == nil { 150 | // this tests to make sure empty directories are removed (s3 behavior exists on all storages) 151 | t.Fatalf("According to docker 0.6.5, listing an empty directory should return an error, got %+v", names) 152 | } 153 | } 154 | 155 | func testListRemoveAll(t *testing.T, storage Storage) { 156 | if err := storage.Put("/dir/1", []byte("lolwtfdir1")); err != nil { 157 | t.Fatal(err) 158 | } 159 | if err := storage.Put("/dir/2", []byte("lolwtfdir2")); err != nil { 160 | t.Fatal(err) 161 | } 162 | if err := storage.Put("/dir/3", []byte("lolwtfdir3")); err != nil { 163 | t.Fatal(err) 164 | } 165 | if err := storage.Put("/anotherdir/1", []byte("lolwtfanotherdir1")); err != nil { 166 | t.Fatal(err) 167 | } 168 | if names, err := storage.List("/"); err != nil { 169 | t.Fatal(err) 170 | } else if len(names) != 2 { 171 | t.Fatal("There should be two names in the directory list") 172 | } else { 173 | checkSlices(t, names, []string{"/dir", "/anotherdir"}) 174 | } 175 | if names, err := storage.List("/dir"); err != nil { 176 | t.Fatal(err) 177 | } else if len(names) != 3 { 178 | t.Fatal("There should be three names in the directory list") 179 | } else { 180 | checkSlices(t, names, []string{"/dir/1", "/dir/2", "/dir/3"}) 181 | } 182 | if names, err := storage.List("/anotherdir/"); err != nil { 183 | t.Fatal(err) 184 | } else if len(names) != 1 { 185 | t.Fatal("There should be one name in the directory list") 186 | } else { 187 | checkSlices(t, names, []string{"/anotherdir/1"}) 188 | } 189 | if err := storage.RemoveAll("/dir"); err != nil { 190 | t.Fatal(err) 191 | } 192 | if names, err := storage.List("/"); err != nil { 193 | t.Fatal(err) 194 | } else if len(names) != 1 { 195 | t.Fatal("There should be one name in the directory list") 196 | } else { 197 | checkSlices(t, names, []string{"/anotherdir"}) 198 | } 199 | if _, err := storage.List("/dir"); err == nil { 200 | t.Fatal("According to docker 0.6.5, listing an empty directory should return an error") 201 | } 202 | if names, err := storage.List("/anotherdir"); err != nil { 203 | t.Fatal(err) 204 | } else if len(names) != 1 { 205 | t.Fatal("There should be one name in the directory list") 206 | } else { 207 | checkSlices(t, names, []string{"/anotherdir/1"}) 208 | } 209 | if err := storage.RemoveAll("/anotherdir"); err != nil { 210 | t.Fatal(err) 211 | } 212 | if _, err := storage.List("/"); err == nil { 213 | t.Fatal("According to docker 0.6.5, listing an empty directory should return an error") 214 | } 215 | if _, err := storage.List("/dir"); err == nil { 216 | t.Fatal("According to docker 0.6.5, listing an empty directory should return an error") 217 | } 218 | if _, err := storage.List("/anotherdir"); err == nil { 219 | t.Fatal("According to docker 0.6.5, listing an empty directory should return an error") 220 | } 221 | } 222 | --------------------------------------------------------------------------------