├── .gitignore ├── .travis.yml ├── LICENSE ├── Makefile ├── README.md ├── cmd ├── get.go ├── list.go ├── put.go └── root.go ├── examples └── k8s_contacts │ ├── README.md │ ├── main.go │ └── slack.png ├── glide.lock ├── glide.yaml ├── grafeas ├── client.go ├── client_test.go ├── storage.go └── storage_test.go ├── main.go └── registry ├── blob.go ├── blob_test.go ├── names.go ├── names_test.go ├── registry.go ├── registry_test.go └── storage.go /.gitignore: -------------------------------------------------------------------------------- 1 | manifesto 2 | bin/ 3 | vendor/ 4 | releases/ 5 | *.json 6 | examples/k8s_contacts/k8s_contacts 7 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | sudo: false 3 | go: 4 | - 1.8 5 | jobs: 6 | include: 7 | - script: make all 8 | deploy: 9 | provider: releases 10 | api_key: 11 | secure: QrixD87Gida5K3xaU1+lRDTxZZh3744GQD4XKlvoDOxxHC/buQBqGGL0MiDK7MU3s4Csvkmj+tH1sVMvrBwXBbT6q2zEmc+wv4LXF8svYMquvli8phEteifOYTSh4SSj6UiWAXQ/T5G/qr/rfZ/S2JdMMU9/ShSsvSyIimxryuW3tmQ0rZDlnmhL/mQKbNbJLmOBP8xtl8+i/gwRQwJNizxKH17XpoLNzPGrGu+BiZKIIlicVsqvPzKn7FLkh8g1qEGDHq4nnCUtHiyyei5XW+5+sIdRr5d9lydIvYM54B8upw/BDLbkpNzER9JNlF0MNg0WTk6xss8HHFYxvszlQ5i6QYMNTGL8rJz36oDo1oHSlVoS7hMSpgNvOYdMPjFwh3PXWzFaSLZNEhy7rqAqcynpuZW81qMNEQgmWkOJ+u9fLkUSR2Vkl+p6MogXAyqMw/qpJUPeDmgmLo55pnmUI/TaaweFcrI+80rptNIcglNcAuui7zP5bCOl8iFmfDnaenC5MX2RsVuFAP8kWii1bc/HHtr7myjm/P/hIRORP1Ota4L+2TlRmDP4n0EC22JCP81xX3Ph0eriOF/f6aiFV2aATRxroQxjeormIqlir/H17Xbbry8GSgR8W8sjLoXPI/v9LJKl/MlPtYQ5V6DTrgkee/r4gIhQindYyQTT8hU= 12 | file: 13 | - releases/manifesto-darwin-amd64.tar.gz 14 | - releases/manifesto-windows-amd64.tar.gz 15 | - releases/manifesto-linux-amd64.tar.gz 16 | skip_cleanup: true 17 | on: 18 | repo: aquasecurity/manifesto 19 | tags: true -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | NAME=manifesto 2 | PACKAGE_NAME=github.com/aquasecurity/$(NAME) 3 | TAG=$$(git describe --abbrev=0 --tags) 4 | 5 | LDFLAGS += -X "$(PACKAGE_NAME)/version.BuildTime=$(shell date -u '+%Y-%m-%d %I:%M:%S %Z')" 6 | LDFLAGS += -X "$(PACKAGE_NAME)/version.BuildVersion=$(shell git describe --abbrev=0 --tags)" 7 | LDFLAGS += -X "$(PACKAGE_NAME)/version.BuildSHA=$(shell git rev-parse HEAD)" 8 | # Strip debug information 9 | LDFLAGS += -s 10 | 11 | ifeq ($(OS),Windows_NT) 12 | suffix := .exe 13 | endif 14 | 15 | all: build test 16 | 17 | $(GOPATH)/bin/glide$(suffix): 18 | go get github.com/Masterminds/glide 19 | 20 | $(GOPATH)/bin/manifesto$(suffix): 21 | go get github.com/aquasecurity/manifesto 22 | 23 | glide.lock: glide.yaml $(GOPATH)/bin/glide$(suffix) 24 | glide update 25 | @touch $@ 26 | 27 | vendor: glide.lock 28 | glide install 29 | @touch $@ 30 | 31 | releases: 32 | mkdir -p releases 33 | 34 | bin/linux/amd64: 35 | mkdir -p bin/linux/amd64 36 | 37 | bin/windows/amd64: 38 | mkdir -p bin/windows/amd64 39 | 40 | bin/darwin/amd64: 41 | mkdir -p bin/darwin/amd64 42 | 43 | build: darwin linux windows 44 | 45 | test: 46 | go test -v $(shell go list ./... | grep -v /vendor/) 47 | 48 | darwin: vendor releases bin/darwin/amd64 49 | env GOOS=darwin GOAARCH=amd64 go build -ldflags '$(LDFLAGS)' -v -o $(CURDIR)/bin/darwin/amd64/$(NAME) 50 | tar -cvzf releases/$(NAME)-darwin-amd64.tar.gz bin/darwin/amd64/$(NAME) 51 | 52 | linux: vendor releases bin/linux/amd64 53 | env GOOS=linux GOAARCH=amd64 go build -ldflags '$(LDFLAGS)' -v -o $(CURDIR)/bin/linux/amd64/$(NAME) 54 | tar -cvzf releases/$(NAME)-linux-amd64.tar.gz bin/linux/amd64/$(NAME) 55 | 56 | windows: vendor releases bin/windows/amd64 57 | env GOOS=windows GOAARCH=amd64 go build -ldflags '$(LDFLAGS)' -v -o $(CURDIR)/bin/windows/amd64/$(NAME).exe 58 | tar -cvzf releases/$(NAME)-windows-amd64.tar.gz bin/windows/amd64/$(NAME).exe 59 | 60 | clean: 61 | rm -fr releases bin 62 | 63 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # manifesto 2 | Manifesto lets users store and query metadata for Docker images. This metadata can be information that you want to store about an image *post-build* - where labels are not sufficient. 3 | 4 | [![Build Status](https://travis-ci.org/aquasecurity/manifesto.svg?branch=master)](https://travis-ci.org/aquasecurity/manifesto) 5 | 6 | ## Use cases 7 | * **Managing QA approval status** After an image has been built, it needs to go through various testing and approval processes before your organization is ready to use it in production. Keep track of approval status, and who has given sign-off by storing it alongside the image itself. 8 | * **Storing security profiles for an image** Manifesto makes it easy to associate a Seccomp or AppArmor profile with an image, so that you can automatically retrieve the correct profile at the point you want to run a container. 9 | * **Storing vulnerability scan reports** Images should be scanned regularly for vulnerabilities as new ones may be found in existing code. Manifesto enables storing the latest scan report for an image without modifying the image itself. 10 | * **Support contacts** Store the phone number or Slack channel to contact in the event this container image starts causing problems in your live deployment. Update these details without needing to update the image. 11 | * **Tracking active images** With CI/CD it's easy to end up with hundreds of thousands of container images in your registry. Use manifesto to store whether an image is actively being used in production - or to indicate the images that can safely be pruned. 12 | 13 | The intention is that each piece of metadata could be signed using Notary. This means you can reliably get back the most recent version of that piece of metadata and know that it was put in place by someone with the authority to do so. 14 | 15 | At the moment this is a Proof of Concept - feedback and ideas very welcome. 16 | 17 | ## Demo 18 | 19 | [![asciicast](https://asciinema.org/a/128283.png)](https://asciinema.org/a/128283) 20 | 21 | ## Installation 22 | 23 | We automatically build binary executables for Mac, Linux and Windows, or you can rebuild from source. Whichever approach you take, **you'll also need to set up credentials** - see below. 24 | 25 | ### Installing binaries 26 | Download the latest binary for your platform from the [releases tab](https://github.com/aquasecurity/manifesto/releases) and unzip it. 27 | 28 | You may find it easiest to move the binary into your path once you have downloaded it. For example 29 | 30 | ``` 31 | $ wget https://github.com/aquasecurity/manifesto/releases/download//manifesto-darwin-amd64.tar.gz 32 | $ tar xf manifesto-darwin-amd64.tar.gz 33 | $ cp bin/darwin/amd64/manifesto /us/local/bin 34 | ``` 35 | 36 | ### Building from source 37 | * Clone this repo (or `go get github.com/aquasecurity/manifesto`) 38 | * Go to the directory and `go build .` 39 | 40 | ### Set up credentials 41 | In this release you will need to be logged in to the Docker Registry - do this with [`docker login`](https://docs.docker.com/engine/reference/commandline/login/). This means that manifesto can execute docker commands directly. 42 | 43 | In addition, since we are now using the Registry API directly to store and retrieve metadata in blobs, you need to pass in your username and password to manifesto itself. You can do this with the command line, environment variables REGISTRY\_USERNAME and REGISTRY\_PASSWORD, or by responding to prompts. 44 | 45 | ## Usage 46 | 47 | ``` 48 | $ ./manifesto --help 49 | Store, retrieve and list pieces of metadata alongside your container images in the registry. 50 | Metadata is associated with specific images (by hash). 51 | 52 | Usage: 53 | manifesto [command] 54 | 55 | Available Commands: 56 | get Show metadata for the container image 57 | help Help about any command 58 | list List currently stored metadata for the container image 59 | put Put metadata for the container image 60 | 61 | Flags: 62 | -h, --help help for manifesto 63 | -p, --password string Registry password (can also be passed in with the env var REGISTRY_PASSWORD) 64 | -u, --username string Registry username (can also be passed in with the env var REGISTRY_USERNAME) 65 | -v, --verbose Send debug output to stderr 66 | 67 | Use "manifesto [command] --help" for more information about a command. 68 | ``` 69 | 70 | By default (like Docker images) manifesto assumes the 'latest' tag if a tag is not given. 71 | 72 | ### Example 73 | 74 | ``` 75 | $ ./manifesto put myorg/imagetest something ~/temp.json 76 | Storing metadata 'something' for 'myorg/imagetest:latest' 77 | Metadata 'something' for 'myorg/imagetest:latest' stored at sha256:7be34480285971f16eed284b13fa7d417649f18c7d1af9b2de6970ce99e3cbbd 78 | Updating manifesto for myorg/imagetest 79 | Replacing 'something' metadata in manifesto for 'myorg/imagetest:latest' 80 | 81 | $ ./manifesto list myorg/imagetest 82 | Metadata types stored for image 'myorg/imagetest:latest': 83 | something 84 | 85 | $ ./manifesto get myorg/imagetest something 86 | { 87 | "key" : "value", 88 | "createdBy" : "liz", 89 | "number" : 56 90 | } 91 | ``` 92 | 93 | # Proof of concept status 94 | 95 | In this proof of concept: 96 | 97 | * we store arbitrary metadata within the Docker registry 98 | * we store a "manifesto" for the repository with the fixed tag "_manifesto" 99 | * the manifesto is a json file with references to all the metadata stored for this repository 100 | 101 | ![Manifesto is stored as an image, which references the data blobs for individual pieces of metadata](https://docs.google.com/drawings/d/1IGm4WnhL3J0hp2hdELrevyn3SMbgs0tlKNHjYIQHqtM/pub?w=960&h=720) 102 | 103 | ### Note - use of image tags in this prototype 104 | In v 0.0.1 the metadata was stored as separate images, tagged as \_manifesto\_*metadata-type*. This allowed us to build the initial prototype very easily, but it meant there were additional repository tags for each piece of metadata, which seems undesirable. 105 | 106 | In this version we use the Registry API to store metadata directly in blobs, as shown in the diagram above. This will mean there will just be one additional tag in the repository, \_manifesto. 107 | 108 | ## Can I store metadata for any image? 109 | 110 | As the metadata is stored in the same repository as the image itself, you can only store metadata for images you own. You could of course save a copy of a third-party image in your own repository, and store metadata alongside it. 111 | 112 | ## How is this better than labels pointing to some arbitrary location where information is stored? 113 | 114 | Putting metadata into the registry itself allows us to leverage both existing technology and existing infrastructure. An organisation that stores images in its own on-premise registry can simply store metadata in it, and if they are using Notary again they can simply re-use the same deployment. The signing of metadata uses exactly the same process as the signing of images, so any existing audits and controls can be re-used. 115 | 116 | ## Manifesto data 117 | 118 | The manifesto associated with a repository is built using a Dockerfile like this: 119 | 120 | ``` 121 | FROM scratch 122 | ADD /data 123 | ``` 124 | 125 | Where contains all the metadata references for this repository. An example might look like this: 126 | 127 | ``` 128 | { 129 | "images": [{ 130 | "image_digest": "70d2f067eb94ec8ab0530068a414d8dbe8c203244ae5d5ad4ba6eb1babd1c1c1", 131 | "manifesto": [{ 132 | "type": "seccomp", 133 | "digest": "sha256:a2fe22a6d44aa86432adad99481c3ad526ba35af2223df126620d20e38c70fac" 134 | }, 135 | { 136 | "type": "approvals", 137 | "digest": "sha256:6ced8eb4e6a61639601e7073963ec04a80f70a11442157e1dd825f042879a6da" 138 | }, 139 | { 140 | "type": "contact", 141 | "digest": "sha256:e896a0012a3450d9cef7e040eea8bed3fe06188957439fea501a65b62c65b4f1" 142 | } 143 | ] 144 | }, 145 | { 146 | "image_digest": "51d2f067eb94ec8ab0531987a414d8dbe8c203244ae5d5ad4ba6eb1babd1d54a", 147 | "manifesto": [{ 148 | "type": "seccomp", 149 | "digest": "sha256:b2f72296d04ea36435adae99481c3ad526ba35af2223df126620d20e38c9763c" 150 | }, 151 | { 152 | "type": "documentation", 153 | "digest": "sha256:9ce18eb4e6a66639601e7073963ec04aa0f70a11442157e1d9825f042879abb1" 154 | } 155 | ] 156 | }] 157 | } 158 | ``` 159 | 160 | The *type* of each piece of metadata is simply an arbitrary string to identify that type of data. One possibility is to use standardized names (possibly as defined in the OCI image spec or similar) for the type to indicate that the associated data blob contains JSON in a standardized format (such as the vulnerability scanning report format). 161 | 162 | ## Wait - in Docker, it's *tags* that get signed in Notary. Why is metadata associated with image digests? 163 | When you pull a Docker image tagged, say, 1.4 you'll get whatever the latest signed version tagged 1.4 is. The tag can move between different images (i.e. with different digests) as updates are made, for example to update from 1.4.3 to 1.4.4. 164 | 165 | For many types of image metadata (e.g. approval status, vulnerability scan report), a piece of metadata must be associated with a particular build i.e. identified by the digest. 166 | 167 | When metadata is added, if signing is enabled the metadata blob gets signed in Notary, as does the manifesto with the references to the metadata blobs. 168 | 169 | # To Do's 170 | 171 | * Add data signing capabilities and verification with Notary. 172 | * Code currently execs out to the docker client executable - would be better to use the go client and call the API directly. 173 | * You need to enter your Docker Hub username and password to put metadata (or pass them in as env vars or parameters). Other commands currently exec out to docker, which means an existing `docker login` is sufficient for the credentials. It would be better to be consistent, and even better if there were a way of using the credentials from that `docker login` 174 | 175 | -------------------------------------------------------------------------------- /cmd/get.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2017 Aqua Security Software Ltd. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package cmd 16 | 17 | import ( 18 | "fmt" 19 | "os" 20 | 21 | "github.com/spf13/cobra" 22 | ) 23 | 24 | // MetadataManifesto gives the type of a piece of arbitrary manifesto data, and the digest where it can be found 25 | // A given image can only have one current piece of data of each type. 26 | // Example types might include: "seccomp", "approvals", "contact" 27 | type MetadataManifesto struct { 28 | Type string `json:"type"` 29 | Digest string `json:"digest"` 30 | } 31 | 32 | // ImageMetadataManifesto associates a piece of manifesto data with a particular image 33 | type ImageMetadataManifesto struct { 34 | ImageDigest string `json:"image_digest"` 35 | MetadataManifesto []MetadataManifesto `json:"manifesto"` 36 | } 37 | 38 | // MetadataManifestoList holds all the metadata for a given image repository 39 | type MetadataManifestoList struct { 40 | Images []ImageMetadataManifesto `json:"images"` 41 | } 42 | 43 | // getCmd gets manifesto data 44 | var getCmd = &cobra.Command{ 45 | Use: "get [IMAGE] [metadata]", 46 | Short: "Show metadata for the container image", 47 | Long: `Display metadata information about the container image.`, 48 | Run: func(cmd *cobra.Command, args []string) { 49 | var err error 50 | 51 | if len(args) < 2 { 52 | cmd.Help() 53 | return 54 | } 55 | 56 | metadata := args[1] 57 | data, imageName, err := storageBackend.GetMetadata(args[0], metadata) 58 | if err != nil { 59 | fmt.Printf("Error getting metadata for image %s: %v\n", imageName, err) 60 | os.Exit(1) 61 | } 62 | if len(data) == 0 { 63 | fmt.Printf("Could not find '%s' metadata for image '%s'\n", metadata, imageName) 64 | os.Exit(0) 65 | } 66 | 67 | fmt.Printf("%s\n", string(data)) 68 | }, 69 | } 70 | -------------------------------------------------------------------------------- /cmd/list.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2017 Aqua Security Software Ltd. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package cmd 16 | 17 | import ( 18 | "fmt" 19 | "os" 20 | 21 | "github.com/spf13/cobra" 22 | ) 23 | 24 | // listCmd gets a list of available manifesto data for this image 25 | var listCmd = &cobra.Command{ 26 | Use: "list [IMAGE]", 27 | Short: "List currently stored metadata for the container image", 28 | Long: `Display a list of the metadata stored for the specified container image.`, 29 | Run: func(cmd *cobra.Command, args []string) { 30 | var err error 31 | if len(args) < 1 { 32 | cmd.Help() 33 | return 34 | } 35 | 36 | metadataTypes, imageName, err := storageBackend.ListMetadata(args[0]) 37 | if err != nil { 38 | fmt.Printf(err.Error()) 39 | os.Exit(1) 40 | } 41 | 42 | if len(metadataTypes) == 0 { 43 | fmt.Printf("No metadata stored for image '%s'\n", imageName) 44 | } 45 | 46 | for _, v := range metadataTypes { 47 | fmt.Printf(" %s\n", v) 48 | } 49 | }, 50 | } 51 | -------------------------------------------------------------------------------- /cmd/put.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2017 Aqua Security Software Ltd. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package cmd 16 | 17 | import ( 18 | "fmt" 19 | "os" 20 | 21 | "github.com/spf13/cobra" 22 | ) 23 | 24 | type Stream struct { 25 | Stream string `json:"stream"` 26 | } 27 | 28 | // putCmd stores manifesto data for this image 29 | var putCmd = &cobra.Command{ 30 | Use: "put [IMAGE] [metadata] [data]", 31 | Short: "Put metadata for the container image", 32 | Long: `Store data as metadata associated with the image. 33 | For Grafeas proof-of-concept support 34 | - the data field is a directory, containing a notes and occurrences directory, each containing JSON files 35 | - the only supported metadata type is PACKAGE_VULNERABILITY`, 36 | Run: func(cmd *cobra.Command, args []string) { 37 | var err error 38 | 39 | if len(args) < 3 { 40 | cmd.Help() 41 | return 42 | } 43 | 44 | name := args[0] 45 | metadataName := args[1] 46 | datafile := args[2] 47 | 48 | imageName, err := storageBackend.PutMetadata(name, metadataName, datafile) 49 | if err != nil { 50 | fmt.Printf("Error putting metadata for image %s: %v\n", imageName, err) 51 | os.Exit(1) 52 | } 53 | }, 54 | } 55 | -------------------------------------------------------------------------------- /cmd/root.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2017 Aqua Security Software Ltd. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package cmd 16 | 17 | import ( 18 | "fmt" 19 | "os" 20 | 21 | "github.com/aquasecurity/manifesto/grafeas" 22 | "github.com/aquasecurity/manifesto/registry" 23 | 24 | "github.com/op/go-logging" 25 | "github.com/spf13/cobra" 26 | "github.com/spf13/viper" 27 | ) 28 | 29 | type metadataStorage interface { 30 | GetMetadata(image string, metadata string) ([]byte, string, error) 31 | ListMetadata(image string) ([]string, string, error) 32 | PutMetadata(image string, metadata string, data string) (string, error) 33 | } 34 | 35 | var ( 36 | username string 37 | password string 38 | storage string 39 | verbose bool 40 | grafeasURL string 41 | grafeasProjID string 42 | storageBackend metadataStorage 43 | log = logging.MustGetLogger("") 44 | ) 45 | 46 | // RootCmd represents the base command when called without any subcommands 47 | var RootCmd = &cobra.Command{ 48 | Use: "manifesto", 49 | Short: "Manage metadata associated with your container images", 50 | Long: `Store, retrieve and list pieces of metadata about your container images. 51 | Storage options: 52 | - store in the container registry alongside your images 53 | - use the Grafeas API 54 | This tool is currently a proof-of-concept so please expect changes.`, 55 | } 56 | 57 | // Execute adds all child commands to the root command sets flags appropriately. 58 | // This is called by main.main(). It only needs to happen once to the rootCmd. 59 | func Execute() { 60 | if err := RootCmd.Execute(); err != nil { 61 | fmt.Println(err) 62 | os.Exit(-1) 63 | } 64 | } 65 | 66 | func init() { 67 | cobra.OnInitialize(initConfig) 68 | 69 | // Here you will define your flags and configuration settings. 70 | // Cobra supports Persistent Flags, which, if defined here, 71 | // will be global for your application. 72 | 73 | RootCmd.PersistentFlags().StringVarP(&username, "username", "u", "", "Registry username (can also be passed in with the env var REGISTRY_USERNAME)") 74 | RootCmd.PersistentFlags().StringVarP(&password, "password", "p", "", "Registry password (can also be passed in with the env var REGISTRY_PASSWORD)") 75 | RootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "Send debug output to stderr") 76 | 77 | RootCmd.PersistentFlags().StringVarP(&storage, "storage", "s", "", "Storage type to use (default is 'registry', also supported: 'grafeas'") 78 | 79 | RootCmd.PersistentFlags().StringVarP(&grafeasURL, "grafeas_url", "", "http://grafeas:8080", "URL of Grafeas server (only used if storage type is grafeas)") 80 | RootCmd.PersistentFlags().StringVarP(&grafeasProjID, "grafeas_proj_id", "", "", "Grafeas project ID (only used if storage type is grafeas). This would typically be a customer project name.") 81 | 82 | // Cobra also supports local flags, which will only run 83 | // when this action is called directly. 84 | 85 | RootCmd.AddCommand(getCmd) 86 | RootCmd.AddCommand(listCmd) 87 | RootCmd.AddCommand(putCmd) 88 | } 89 | 90 | // initConfig reads in config file and ENV variables if set. 91 | func initConfig() { 92 | viper.SetConfigName(".manifesto") // name of config file (without extension) 93 | viper.AddConfigPath(".") // adding current directory as first search path 94 | viper.AddConfigPath("$HOME") // adding home directory 95 | 96 | viper.BindEnv("username", "REGISTRY_USERNAME") 97 | viper.BindEnv("password", "REGISTRY_PASSWORD") 98 | viper.BindEnv("storage", "MANIFESTO_STORAGE") 99 | viper.BindEnv("verbose", "MANIFESTO_VERBOSE") 100 | viper.BindEnv("grafeas_url", "GRAFEAS_URL") 101 | viper.BindEnv("grafeas_proj_id", "GRAFEAS_PROJ_ID") 102 | 103 | viper.AutomaticEnv() // read in environment variables that match 104 | 105 | viper.BindPFlag("username", RootCmd.Flags().Lookup("username")) 106 | viper.BindPFlag("password", RootCmd.Flags().Lookup("password")) 107 | viper.BindPFlag("storage", RootCmd.Flags().Lookup("storage")) 108 | viper.BindPFlag("verbose", RootCmd.Flags().Lookup("verbose")) 109 | viper.BindPFlag("grafeas_url", RootCmd.Flags().Lookup("grafeas_url")) 110 | viper.BindPFlag("grafeas_proj_id", RootCmd.Flags().Lookup("grafeas_proj_id")) 111 | 112 | // If a config file is found, read it in. 113 | if err := viper.ReadInConfig(); err == nil { 114 | log.Debugf("Using config file %s", viper.ConfigFileUsed()) 115 | } 116 | 117 | username = viper.GetString("username") 118 | password = viper.GetString("password") 119 | verbose = viper.GetBool("verbose") 120 | 121 | storage = viper.GetString("storage") 122 | grafeasURL = viper.GetString("grafeas_url") 123 | grafeasProjID = viper.GetString("grafeas_proj_id") 124 | 125 | // Set up logging 126 | if verbose { 127 | logging.SetLevel(logging.DEBUG, "") 128 | } else { 129 | logging.SetLevel(logging.INFO, "") 130 | } 131 | 132 | // Backend storage type 133 | switch storage { 134 | case "grafeas": 135 | if grafeasProjID == "" { 136 | fmt.Println("You need to specify a Grafeas project ID") 137 | os.Exit(1) 138 | } 139 | storageBackend = grafeas.NewStorage(grafeasURL, grafeasProjID, verbose) 140 | default: 141 | log.Debug("Registry storage") 142 | storageBackend = registry.NewStorage(username, password, verbose) 143 | } 144 | } 145 | -------------------------------------------------------------------------------- /examples/k8s_contacts/README.md: -------------------------------------------------------------------------------- 1 | # k8s_contacts 2 | This demonstrates using container metadata to make it easy to generate automated alerts 3 | for a piece of code running under Kubernetes, for example if it goes wrong. 4 | 5 | ## Setting up the metadata 6 | Prepare the contact metadata by creating `contact.file` containing JSON data: 7 | 8 | ``` 9 | { 10 | "slack": "https://hooks.slack.com/services/12345/67890/1234567890", 11 | "slack_channel": "#alerts" 12 | ...other data as required 13 | } 14 | ``` 15 | 16 | Add the metadata to the container image (you might have this done by a CI/CD job whenever 17 | a new version of the image is added to the container registry): 18 | 19 | ``` 20 | manifesto put companyx/componenta contact ../contact.file 21 | ``` 22 | 23 | Run the component in Kubernetes: 24 | 25 | ``` 26 | kubectl run componenta --image=companyx/componenta 27 | deployment "componenta" created 28 | ``` 29 | 30 | Add the annotation to the running deployment: 31 | ``` 32 | kubectl annotate deployment componenta contact="$(manifesto get companyx/componenta contact)" --overwrite=true 33 | ``` 34 | 35 | ## When something goes wrong with component A 36 | If something detects a problem with the running component, it can generate the appropriate slack alert: 37 | ``` 38 | k8s_contacts componentA 39 | ``` 40 | 41 | This generates a Slack alert: 42 | 43 | ![Slack alert](slack.png) -------------------------------------------------------------------------------- /examples/k8s_contacts/main.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2017 Aqua Security Software Ltd. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // This example code shows how to use a Kubernetes annotation to generate 16 | // a Slack alert. 17 | // The annotation is expected be called "contact" and contain a JSON struct: 18 | // 19 | // { 20 | // "slack": "", 21 | // "slack_channel": "" (optional) 22 | // } 23 | // 24 | package main 25 | 26 | import ( 27 | "encoding/json" 28 | "fmt" 29 | "os" 30 | "os/exec" 31 | 32 | slack "github.com/ashwanthkumar/slack-go-webhook" 33 | ) 34 | 35 | type Contact struct { 36 | Slack string 37 | SlackChannel string `json:"slack_channel"` 38 | } 39 | 40 | func main() { 41 | component := os.Args[1] 42 | channel := "#general" 43 | 44 | cmd := exec.Command("kubectl", "get", "deployment", component, "-o=custom-columns=:.metadata.annotations.contact", "--no-headers") 45 | out, _ := cmd.Output() 46 | 47 | var contact Contact 48 | err := json.Unmarshal(out, &contact) 49 | if err != nil { 50 | panic(err) 51 | } 52 | 53 | if contact.Slack == "" { 54 | fmt.Println("No Slack channel found") 55 | os.Exit(1) 56 | } 57 | 58 | if contact.SlackChannel != "" { 59 | channel = contact.SlackChannel 60 | } 61 | 62 | payload := slack.Payload{ 63 | Channel: channel, 64 | Text: "Hey, " + component + " needs your help", 65 | Username: "Kubebot", 66 | IconEmoji: ":kubernetes:", 67 | } 68 | 69 | errs := slack.Send(contact.Slack, "", payload) 70 | if len(errs) > 0 { 71 | fmt.Printf("error: %s\n", errs) 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /examples/k8s_contacts/slack.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aquasecurity/manifesto/5ff99105e1e9b20a4277aa4af64e69c44a77482c/examples/k8s_contacts/slack.png -------------------------------------------------------------------------------- /glide.lock: -------------------------------------------------------------------------------- 1 | hash: adcaa254f0195c4d199d8a35201c28c91f28b0867e632b323e745b9cf0e21376 2 | updated: 2017-11-30T15:39:48.571863735Z 3 | imports: 4 | - name: github.com/fsnotify/fsnotify 5 | version: 4da3e2cfbabc9f751898f250b49f2439785783a1 6 | - name: github.com/go-resty/resty 7 | version: 13d77a7f95fd8d1bad97c56bf1b8562633c2f01a 8 | - name: github.com/Grafeas/client-go 9 | version: d30b35402e7c6dfc9427d967ce12af59683e206a 10 | subpackages: 11 | - v1alpha1 12 | - name: github.com/grafeas/grafeas 13 | version: e61c8332bcfc34d72a9e44906499247d0286ed96 14 | subpackages: 15 | - samples/server/go-server/api/server/name 16 | - server-go/errors 17 | - name: github.com/hashicorp/hcl 18 | version: 392dba7d905ed5d04a5794ba89f558b27e2ba1ca 19 | subpackages: 20 | - hcl/ast 21 | - hcl/parser 22 | - hcl/scanner 23 | - hcl/strconv 24 | - hcl/token 25 | - json/parser 26 | - json/scanner 27 | - json/token 28 | - name: github.com/inconshreveable/mousetrap 29 | version: 76626ae9c91c4f2a10f34cad8ce83ea42c93bb75 30 | - name: github.com/labstack/gommon 31 | version: 57409ada9da0f2afad6664c49502f8c50fbd8476 32 | subpackages: 33 | - color 34 | - log 35 | - name: github.com/magiconair/properties 36 | version: be5ece7dd465ab0765a9682137865547526d1dfb 37 | - name: github.com/mattn/go-colorable 38 | version: 6fcc0c1fd9b620311d821b106a400b35dc95c497 39 | - name: github.com/mattn/go-isatty 40 | version: 6ca4dbf54d38eea1a992b3c722a76a5d1c4cb25c 41 | - name: github.com/mitchellh/mapstructure 42 | version: d0303fe809921458f417bcf828397a65db30a7e4 43 | - name: github.com/op/go-logging 44 | version: b2cb9fa56473e98db8caba80237377e83fe44db5 45 | - name: github.com/pelletier/go-toml 46 | version: 69d355db5304c0f7f809a2edc054553e7142f016 47 | - name: github.com/spf13/afero 48 | version: 9be650865eab0c12963d8753212f4f9c66cdcf12 49 | subpackages: 50 | - mem 51 | - name: github.com/spf13/cast 52 | version: acbeb36b902d72a7a4c18e8f3241075e7ab763e4 53 | - name: github.com/spf13/cobra 54 | version: 161584fc2e266ca7969b456171a0bca2ce915ed4 55 | - name: github.com/spf13/jwalterweatherman 56 | version: 0efa5202c04663c757d84f90f5219c1250baf94f 57 | - name: github.com/spf13/pflag 58 | version: e57e3eeb33f795204c1ca35f56c44f83227c6e66 59 | - name: github.com/spf13/viper 60 | version: 25b30aa063fc18e48662b86996252eabdcf2f0c7 61 | - name: github.com/valyala/bytebufferpool 62 | version: e746df99fe4a3986f4d4f79e13c1e0117ce9c2f7 63 | - name: github.com/valyala/fasttemplate 64 | version: dcecefd839c4193db0d35b88ec65b4c12d360ab0 65 | - name: golang.org/x/crypto 66 | version: 558b6879de74bc843225cde5686419267ff707ca 67 | subpackages: 68 | - ssh/terminal 69 | - name: golang.org/x/net 70 | version: c8c74377599bd978aee1cf3b9b63a8634051cec2 71 | subpackages: 72 | - publicsuffix 73 | - name: golang.org/x/sys 74 | version: 0f826bdd13b500be0f1d4004938ad978fcc6031e 75 | subpackages: 76 | - unix 77 | - name: golang.org/x/text 78 | version: 3bd178b88a8180be2df394a1fbb81313916f0e7b 79 | subpackages: 80 | - transform 81 | - unicode/norm 82 | - name: gopkg.in/yaml.v2 83 | version: 25c4ec802a7d637f88d584ab26798e94ad14c13b 84 | testImports: [] 85 | -------------------------------------------------------------------------------- /glide.yaml: -------------------------------------------------------------------------------- 1 | package: github.com/aquasecurity/manifesto 2 | import: 3 | - package: github.com/op/go-logging 4 | version: v1 5 | - package: github.com/spf13/cobra 6 | - package: github.com/spf13/viper 7 | - package: golang.org/x/crypto 8 | subpackages: 9 | - ssh/terminal 10 | -------------------------------------------------------------------------------- /grafeas/client.go: -------------------------------------------------------------------------------- 1 | // Package grafeas allows manifesto to interact with a Grafeas server 2 | // 3 | // Copyright © 2017 Aqua Security Software Ltd. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | package grafeas 18 | 19 | import ( 20 | "bytes" 21 | "encoding/json" 22 | "fmt" 23 | "io/ioutil" 24 | "net/http" 25 | "net/url" 26 | "strconv" 27 | 28 | grafeas "github.com/Grafeas/client-go/v1alpha1" 29 | ) 30 | 31 | func (s *Storage) createNote(projectsID string, noteID string, note grafeas.Note) (*grafeas.Note, error) { 32 | n, err := json.Marshal(note) 33 | if err != nil { 34 | return nil, fmt.Errorf("marshalling note: %v", err) 35 | } 36 | 37 | return s.createNoteFromBytes(projectsID, noteID, n) 38 | } 39 | 40 | func (s *Storage) createNoteFromBytes(projectsID string, noteID string, n []byte) (*grafeas.Note, error) { 41 | path := s.url + "/v1alpha1/projects/" + projectsID + "/notes" 42 | 43 | queryParams := url.Values{} 44 | queryParams.Add("noteId", noteID) 45 | path = path + "?" + queryParams.Encode() 46 | 47 | log.Debugf("CreateNote at URL %s", path) 48 | req, err := http.NewRequest("POST", path, bytes.NewBuffer(n)) 49 | if err != nil { 50 | log.Errorf("http request: %v", err) 51 | return nil, err 52 | } 53 | 54 | var successPayload = new(grafeas.Note) 55 | rsp, err := s.client.Do(req) 56 | if err != nil { 57 | return successPayload, err 58 | } 59 | 60 | b, err := ioutil.ReadAll(rsp.Body) 61 | if err != nil { 62 | return successPayload, err 63 | } 64 | 65 | if rsp.StatusCode != http.StatusOK { 66 | log.Debugf("CreateNote response: %s %s", rsp.Status, string(b)) 67 | return successPayload, err 68 | } 69 | 70 | err = json.Unmarshal(b, &successPayload) 71 | return successPayload, err 72 | } 73 | 74 | func (s *Storage) createOccurrence(projectsID string, occurrence grafeas.Occurrence) (*grafeas.Occurrence, error) { 75 | o, err := json.Marshal(occurrence) 76 | if err != nil { 77 | return nil, fmt.Errorf("marshalling occurrence: %v", err) 78 | } 79 | 80 | return s.createOccurrenceFromBytes(projectsID, o) 81 | } 82 | 83 | func (s *Storage) createOccurrenceFromBytes(projectsID string, o []byte) (*grafeas.Occurrence, error) { 84 | path := s.url + "/v1alpha1/projects/" + projectsID + "/occurrences" 85 | 86 | log.Debugf("CreateOccurrence at URL %s", path) 87 | req, err := http.NewRequest("POST", path, bytes.NewBuffer(o)) 88 | if err != nil { 89 | log.Errorf("http request: %v", err) 90 | return nil, err 91 | } 92 | 93 | var successPayload = new(grafeas.Occurrence) 94 | rsp, err := s.client.Do(req) 95 | if err != nil { 96 | return successPayload, err 97 | } 98 | 99 | b, err := ioutil.ReadAll(rsp.Body) 100 | if err != nil { 101 | return successPayload, err 102 | } 103 | 104 | if rsp.StatusCode != http.StatusOK { 105 | log.Debugf("CreateOccurrence response: %s %s", rsp.Status, string(b)) 106 | return successPayload, err 107 | } 108 | 109 | err = json.Unmarshal(b, &successPayload) 110 | return successPayload, err 111 | } 112 | 113 | func (s *Storage) listOccurrences(projectsID string, filter string, pageSize int32, pageToken string) (*grafeas.ListOccurrencesResponse, error) { 114 | path := s.url + "/v1alpha1/projects/" + projectsID + "/occurrences" 115 | 116 | queryParams := url.Values{} 117 | if filter != "" { 118 | queryParams.Add("filter", filter) 119 | } 120 | if pageSize != 0 { 121 | queryParams.Add("pageSize", strconv.Itoa(int(pageSize))) 122 | } 123 | if pageToken != "" { 124 | queryParams.Add("pageToken", pageToken) 125 | } 126 | 127 | if len(queryParams) > 0 { 128 | path = path + "?" + queryParams.Encode() 129 | } 130 | 131 | req, err := http.NewRequest("GET", path, nil) 132 | if err != nil { 133 | log.Errorf("http request: %v", err) 134 | return nil, err 135 | } 136 | 137 | var successPayload = new(grafeas.ListOccurrencesResponse) 138 | rsp, err := s.client.Do(req) 139 | if err != nil { 140 | return successPayload, err 141 | } 142 | 143 | if rsp.StatusCode != http.StatusOK { 144 | log.Debugf("ListOccurrences response: %s %s", rsp.Status, rsp.Body) 145 | return successPayload, err 146 | } 147 | 148 | b, err := ioutil.ReadAll(rsp.Body) 149 | if err != nil { 150 | return successPayload, err 151 | } 152 | 153 | err = json.Unmarshal(b, &successPayload) 154 | return successPayload, err 155 | } 156 | 157 | func (s *Storage) getNote(projectsID string, notesID string) (*grafeas.Note, error) { 158 | path := s.url + "/v1alpha1/projects/" + projectsID + "/notes/" + notesID 159 | 160 | log.Debugf("GetNote from URL %s", path) 161 | req, err := http.NewRequest("GET", path, nil) 162 | if err != nil { 163 | log.Errorf("http request: %v", err) 164 | return nil, err 165 | } 166 | 167 | var successPayload = new(grafeas.Note) 168 | rsp, err := s.client.Do(req) 169 | if err != nil { 170 | return successPayload, err 171 | } 172 | 173 | if rsp.StatusCode != http.StatusOK { 174 | log.Debugf("ListOccurrences response: %s %s", rsp.Status, rsp.Body) 175 | return successPayload, err 176 | } 177 | 178 | b, err := ioutil.ReadAll(rsp.Body) 179 | if err != nil { 180 | return successPayload, err 181 | } 182 | 183 | log.Debugf("Note: %s", string(b)) 184 | err = json.Unmarshal(b, &successPayload) 185 | return successPayload, err 186 | } 187 | 188 | func (s *Storage) listNotes(projectsID string, filter string, pageSize int32, pageToken string) (*grafeas.ListNotesResponse, error) { 189 | path := s.url + "/v1alpha1/projects/" + projectsID + "/notes" 190 | 191 | queryParams := url.Values{} 192 | if filter != "" { 193 | queryParams.Add("filter", filter) 194 | } 195 | if pageSize != 0 { 196 | queryParams.Add("pageSize", strconv.Itoa(int(pageSize))) 197 | } 198 | if pageToken != "" { 199 | queryParams.Add("pageToken", pageToken) 200 | } 201 | if len(queryParams) > 0 { 202 | path = path + "?" + queryParams.Encode() 203 | } 204 | 205 | log.Debugf("ListNotes from URL %s", path) 206 | req, err := http.NewRequest("GET", path, nil) 207 | if err != nil { 208 | log.Errorf("http request: %v", err) 209 | return nil, err 210 | } 211 | 212 | var successPayload = new(grafeas.ListNotesResponse) 213 | rsp, err := s.client.Do(req) 214 | if err != nil { 215 | return successPayload, err 216 | } 217 | 218 | if rsp.StatusCode != http.StatusOK { 219 | log.Debugf("ListOccurrences response: %s %s", rsp.Status, rsp.Body) 220 | return successPayload, err 221 | } 222 | 223 | b, err := ioutil.ReadAll(rsp.Body) 224 | if err != nil { 225 | return successPayload, err 226 | } 227 | 228 | err = json.Unmarshal(b, &successPayload) 229 | return successPayload, err 230 | } 231 | -------------------------------------------------------------------------------- /grafeas/client_test.go: -------------------------------------------------------------------------------- 1 | // TEMPORARILY FAKE SOME DATA 2 | package grafeas 3 | 4 | import ( 5 | "net/http" 6 | "net/http/httptest" 7 | "testing" 8 | 9 | grafeas "github.com/Grafeas/client-go/v1alpha1" 10 | ) 11 | 12 | func TestCreateNote(t *testing.T) { 13 | tests := []struct { 14 | name string 15 | URL string 16 | method string 17 | response string 18 | }{ 19 | { 20 | name: "createNote", 21 | URL: "/v1alpha1/projects/testProjID/notes?noteId=testNoteID", 22 | method: "POST", 23 | response: `{ "name": "projects/aqua-scan/notes/CVE-2014-9911"}`, 24 | }, 25 | { 26 | name: "createOccurrence", 27 | URL: "/v1alpha1/projects/testProjID/occurrences", 28 | method: "POST", 29 | response: `{ "name": "projects/aqua-scan/occurrences/12345"}`, 30 | }, 31 | { 32 | name: "getNote", 33 | URL: "/v1alpha1/projects/testProjID/notes/testNoteID", 34 | method: "GET", 35 | response: `{ "name": "projects/aqua-scan/notes/CVE-2014-9911"}`, 36 | }, 37 | { 38 | name: "listOccurrences", 39 | URL: "/v1alpha1/projects/testProjID/occurrences", 40 | method: "GET", 41 | response: `{}`, 42 | }, 43 | { 44 | name: "listOccurrences1", 45 | URL: "/v1alpha1/projects/testProjID/occurrences?filter=x%3Dy", 46 | method: "GET", 47 | response: `{"occurrences": [ 48 | { "name": "hello" } 49 | ]}`, 50 | }, 51 | { 52 | name: "listNotes", 53 | URL: "/v1alpha1/projects/testProjID/notes", 54 | method: "GET", 55 | response: `{}`, 56 | }, 57 | { 58 | name: "listNotes1", 59 | URL: "/v1alpha1/projects/testProjID/notes?filter=x%3Dy", 60 | method: "GET", 61 | response: `{"notes": [ 62 | { "name": "hello" } 63 | ]}`, 64 | }, 65 | } 66 | 67 | for _, test := range tests { 68 | t.Run(test.name, func(t *testing.T) { 69 | server := httptest.NewServer( 70 | http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 71 | t.Logf("%s", r.URL) 72 | if r.URL.String() != test.URL { 73 | t.Fatalf("Unexpected URL %s\nexpected %s", r.URL.String(), test.URL) 74 | } 75 | if r.Method != test.method { 76 | t.Fatalf("Unexpected Method %s expected %s", r.Method, test.method) 77 | } 78 | w.Write([]byte(test.response)) 79 | })) 80 | 81 | defer server.Close() 82 | s := &Storage{ 83 | projID: "testProjID", 84 | url: server.URL, 85 | client: http.DefaultClient, 86 | } 87 | 88 | switch test.name { 89 | case "createNote": 90 | notes := tempNotes("lizrice/hello:1") 91 | n, err := s.createNote("testProjID", "testNoteID", notes[0]) 92 | if err != nil { 93 | t.Fatalf("%v", err) 94 | } 95 | if n.Name != "projects/aqua-scan/notes/CVE-2014-9911" { 96 | t.Fatalf("Unexpected name: %s", n.Name) 97 | } 98 | case "createOccurrence": 99 | occurrences := tempOccurrences("lizrice/hello:1") 100 | o, err := s.createOccurrence("testProjID", occurrences[0]) 101 | if err != nil { 102 | t.Fatalf("%v", err) 103 | } 104 | if o.Name != "projects/aqua-scan/occurrences/12345" { 105 | t.Fatalf("Unexpected name: %s", o.Name) 106 | } 107 | case "getNote": 108 | n, err := s.getNote("testProjID", "testNoteID") 109 | if err != nil { 110 | t.Fatalf("%v", err) 111 | } 112 | if n.Name != "projects/aqua-scan/notes/CVE-2014-9911" { 113 | t.Fatalf("Unexpected name: %s", n.Name) 114 | } 115 | case "getOccurrence": 116 | t.Fatalf("Not implemented") 117 | case "listOccurrences": 118 | l, err := s.listOccurrences("testProjID", "", 0, "") 119 | if err != nil { 120 | t.Fatalf("%v", err) 121 | } 122 | if len(l.Occurrences) > 0 { 123 | t.Fatalf("Unexpected occurrences") 124 | } 125 | case "listOccurrences1": 126 | l, err := s.listOccurrences("testProjID", "x=y", 0, "") 127 | if err != nil { 128 | t.Fatalf("%v", err) 129 | } 130 | if len(l.Occurrences) != 1 { 131 | t.Fatalf("Unexpected occurrences") 132 | } 133 | if l.Occurrences[0].Name != "hello" { 134 | t.Fatalf("Unexpected name %s", l.Occurrences[0].Name) 135 | } 136 | case "listNotes": 137 | l, err := s.listNotes("testProjID", "", 0, "") 138 | if err != nil { 139 | t.Fatalf("%v", err) 140 | } 141 | if len(l.Notes) > 0 { 142 | t.Fatalf("Unexpected notes") 143 | } 144 | case "listNotes1": 145 | l, err := s.listNotes("testProjID", "x=y", 0, "") 146 | if err != nil { 147 | t.Fatalf("%v", err) 148 | } 149 | if len(l.Notes) != 1 { 150 | t.Fatalf("Unexpected notes") 151 | } 152 | if l.Notes[0].Name != "hello" { 153 | t.Fatalf("Unexpected name %s", l.Notes[0].Name) 154 | } 155 | } 156 | }) 157 | } 158 | } 159 | 160 | func tempOccurrences(image string) []grafeas.Occurrence { 161 | 162 | occurrence0 := grafeas.Occurrence{ 163 | ResourceUrl: "registry-1.docker.io/lizrice/hello@sha256:fb19fad1d75d467310fb4962787431acfecf17a32b29cd64d7d586c547446ba5", 164 | NoteName: "projects/aqua-scan/notes/CVE-2014-9911", 165 | Kind: "PACKAGE_VULNERABILITY", 166 | VulnerabilityDetails: grafeas.VulnerabilityDetails{ 167 | Severity: "HIGH", 168 | CvssScore: 7.5, 169 | PackageIssue: []grafeas.PackageIssue{ 170 | grafeas.PackageIssue{ 171 | SeverityName: "HIGH", 172 | AffectedLocation: grafeas.VulnerabilityLocation{ 173 | CpeUri: "cpe:/o:debian:debian_linux:8", 174 | Package_: "icu", 175 | Version: grafeas.Version{ 176 | Name: "52.1", 177 | Revision: "8+deb8u3", 178 | }, 179 | }, 180 | FixedLocation: grafeas.VulnerabilityLocation{ 181 | CpeUri: "cpe:/o:debian:debian_linux:8", 182 | Package_: "icu", 183 | Version: grafeas.Version{ 184 | Name: "52.1", 185 | Revision: "8+deb8u4", 186 | }, 187 | }, 188 | }, 189 | }, 190 | }, 191 | } 192 | 193 | occurrence1 := grafeas.Occurrence{ 194 | ResourceUrl: "registry-1.docker.io/lizrice/hello@sha256:2a1b47e618e712fd95680091cac56468a9e7e2fe60bf4224fbea3613f4a64cea", 195 | NoteName: "projects/aqua-scan/notes/CVE-2014-9911", 196 | Kind: "PACKAGE_VULNERABILITY", 197 | VulnerabilityDetails: grafeas.VulnerabilityDetails{ 198 | Severity: "HIGH", 199 | CvssScore: 7.5, 200 | PackageIssue: []grafeas.PackageIssue{ 201 | grafeas.PackageIssue{ 202 | SeverityName: "HIGH", 203 | AffectedLocation: grafeas.VulnerabilityLocation{ 204 | CpeUri: "cpe:/o:debian:debian_linux:8", 205 | Package_: "icu", 206 | Version: grafeas.Version{ 207 | Name: "52.1", 208 | Revision: "8+deb8u3", 209 | }, 210 | }, 211 | FixedLocation: grafeas.VulnerabilityLocation{ 212 | CpeUri: "cpe:/o:debian:debian_linux:8", 213 | Package_: "icu", 214 | Version: grafeas.Version{ 215 | Name: "52.1", 216 | Revision: "8+deb8u4", 217 | }, 218 | }, 219 | }, 220 | }, 221 | }, 222 | } 223 | 224 | occurrence2 := grafeas.Occurrence{ 225 | ResourceUrl: "registry-1.docker.io/lizrice/hello@sha256:2a1b47e618e712fd95680091cac56468a9e7e2fe60bf4224fbea3613f4a64cea", 226 | NoteName: "projects/aqua-scan/notes/CVE-2017-I-made-this-up", 227 | Kind: "PACKAGE_VULNERABILITY", 228 | VulnerabilityDetails: grafeas.VulnerabilityDetails{ 229 | Severity: "HIGH", 230 | CvssScore: 7.5, 231 | PackageIssue: []grafeas.PackageIssue{ 232 | grafeas.PackageIssue{ 233 | SeverityName: "HIGH", 234 | AffectedLocation: grafeas.VulnerabilityLocation{ 235 | CpeUri: "cpe:/o:debian:debian_linux:8", 236 | Package_: "icu", 237 | Version: grafeas.Version{ 238 | Name: "52.1", 239 | Revision: "8+deb8u3", 240 | }, 241 | }, 242 | FixedLocation: grafeas.VulnerabilityLocation{ 243 | CpeUri: "cpe:/o:debian:debian_linux:8", 244 | Package_: "icu", 245 | Version: grafeas.Version{ 246 | Name: "52.1", 247 | Revision: "8+deb8u4", 248 | }, 249 | }, 250 | }, 251 | }, 252 | }, 253 | } 254 | 255 | switch image { 256 | case "lizrice/hello:1": 257 | return []grafeas.Occurrence{occurrence0} 258 | case "lizrice/hello:2": 259 | return []grafeas.Occurrence{occurrence1, occurrence2} 260 | } 261 | 262 | return []grafeas.Occurrence{} 263 | } 264 | 265 | func tempNotes(image string) []grafeas.Note { 266 | note0 := grafeas.Note{ 267 | Name: "projects/aqua-scan/notes/CVE-2014-9911", 268 | ShortDescription: "CVE-2014-9911", 269 | LongDescription: "NIST vectors: AV:N/AC:L/Au:N/C:P/I:P", 270 | Kind: "PACKAGE_VULNERABILITY", 271 | VulnerabilityType: grafeas.VulnerabilityType{ 272 | CvssScore: 7.5, 273 | Severity: "HIGH", 274 | Details: []grafeas.Detail{ 275 | { 276 | CpeUri: "cpe:/o:debian:debian_linux:7", 277 | Package_: "icu", 278 | Description: "Stack-based buffer overflow in the ures_getByKeyWithFallback function in " + 279 | "common/uresbund.cpp in International Components for Unicode (ICU) before 54.1 for C/C++ allows " + 280 | "remote attackers to cause a denial of service or possibly have unspecified other impact via a crafted uloc_getDisplayName call.", 281 | MinAffectedVersion: grafeas.Version{ 282 | Kind: "MINIMUM", 283 | }, 284 | SeverityName: "HIGH", 285 | 286 | FixedLocation: grafeas.VulnerabilityLocation{ 287 | CpeUri: "cpe:/o:debian:debian_linux:7", 288 | Package_: "icu", 289 | Version: grafeas.Version{ 290 | Name: "4.8.1.1", 291 | Revision: "12+deb7u6", 292 | }, 293 | }, 294 | }, 295 | { 296 | CpeUri: "cpe:/o:debian:debian_linux:8", 297 | Package_: "icu", 298 | Description: "Stack-based buffer overflow in the ures_getByKeyWithFallback function in " + 299 | "common/uresbund.cpp in International Components for Unicode (ICU) before 54.1 for C/C++ allows " + 300 | "remote attackers to cause a denial of service or possibly have unspecified other impact via a crafted uloc_getDisplayName call.", 301 | MinAffectedVersion: grafeas.Version{ 302 | Kind: "MINIMUM", 303 | }, 304 | SeverityName: "HIGH", 305 | 306 | FixedLocation: grafeas.VulnerabilityLocation{ 307 | CpeUri: "cpe:/o:debian:debian_linux:8", 308 | Package_: "icu", 309 | Version: grafeas.Version{ 310 | Name: "52.1", 311 | Revision: "8+deb8u4", 312 | }, 313 | }, 314 | }, 315 | { 316 | CpeUri: "cpe:/o:debian:debian_linux:9", 317 | Package_: "icu", 318 | Description: "Stack-based buffer overflow in the ures_getByKeyWithFallback function in " + 319 | "common/uresbund.cpp in International Components for Unicode (ICU) before 54.1 for C/C++ allows " + 320 | "remote attackers to cause a denial of service or possibly have unspecified other impact via a crafted uloc_getDisplayName call.", 321 | MinAffectedVersion: grafeas.Version{ 322 | Kind: "MINIMUM", 323 | }, 324 | SeverityName: "HIGH", 325 | 326 | FixedLocation: grafeas.VulnerabilityLocation{ 327 | CpeUri: "cpe:/o:debian:debian_linux:9", 328 | Package_: "icu", 329 | Version: grafeas.Version{ 330 | Name: "55.1", 331 | Revision: "3", 332 | }, 333 | }, 334 | }, 335 | { 336 | CpeUri: "cpe:/o:canonical:ubuntu_linux:14.04", 337 | Package_: "andriod", 338 | Description: "Stack-based buffer overflow in the ures_getByKeyWithFallback function in " + 339 | "common/uresbund.cpp in International Components for Unicode (ICU) before 54.1 for C/C++ allows " + 340 | "remote attackers to cause a denial of service or possibly have unspecified other impact via a crafted uloc_getDisplayName call.", 341 | MinAffectedVersion: grafeas.Version{ 342 | Kind: "MINIMUM", 343 | }, 344 | SeverityName: "MEDIUM", 345 | 346 | FixedLocation: grafeas.VulnerabilityLocation{ 347 | CpeUri: "cpe:/o:canonical:ubuntu_linux:14.04", 348 | Package_: "andriod", 349 | Version: grafeas.Version{ 350 | Kind: "MAXIMUM", 351 | }, 352 | }, 353 | }, 354 | }, 355 | }, 356 | RelatedUrl: []grafeas.RelatedUrl{ 357 | { 358 | Url: "https://security-tracker.debian.org/tracker/CVE-2014-9911", 359 | Label: "More Info", 360 | }, 361 | { 362 | Url: "http://people.ubuntu.com/~ubuntu-security/cve/CVE-2014-9911", 363 | Label: "More Info", 364 | }, 365 | }, 366 | } 367 | 368 | note1 := grafeas.Note{ 369 | // Name: "projects/aqua-scan/notes/CVE-2017-I-made-this-up", 370 | ShortDescription: "CVE-2017-I-made-this-up", 371 | LongDescription: "NIST vectors: AV:N/AC:L/Au:N/C:P/I:P", 372 | Kind: "PACKAGE_VULNERABILITY", 373 | VulnerabilityType: grafeas.VulnerabilityType{ 374 | CvssScore: 7.5, 375 | Severity: "HIGH", 376 | Details: []grafeas.Detail{ 377 | { 378 | CpeUri: "cpe:/o:debian:debian_linux:7", 379 | Package_: "icu", 380 | Description: "Stack-based buffer overflow in the ures_getByKeyWithFallback function in " + 381 | "common/uresbund.cpp in International Components for Unicode (ICU) before 54.1 for C/C++ allows " + 382 | "remote attackers to cause a denial of service or possibly have unspecified other impact via a crafted uloc_getDisplayName call.", 383 | MinAffectedVersion: grafeas.Version{ 384 | Kind: "MINIMUM", 385 | }, 386 | SeverityName: "HIGH", 387 | 388 | FixedLocation: grafeas.VulnerabilityLocation{ 389 | CpeUri: "cpe:/o:debian:debian_linux:7", 390 | Package_: "icu", 391 | Version: grafeas.Version{ 392 | Name: "4.8.1.1", 393 | Revision: "12+deb7u6", 394 | }, 395 | }, 396 | }, 397 | { 398 | CpeUri: "cpe:/o:debian:debian_linux:8", 399 | Package_: "icu", 400 | Description: "Stack-based buffer overflow in the ures_getByKeyWithFallback function in " + 401 | "common/uresbund.cpp in International Components for Unicode (ICU) before 54.1 for C/C++ allows " + 402 | "remote attackers to cause a denial of service or possibly have unspecified other impact via a crafted uloc_getDisplayName call.", 403 | MinAffectedVersion: grafeas.Version{ 404 | Kind: "MINIMUM", 405 | }, 406 | SeverityName: "HIGH", 407 | 408 | FixedLocation: grafeas.VulnerabilityLocation{ 409 | CpeUri: "cpe:/o:debian:debian_linux:8", 410 | Package_: "icu", 411 | Version: grafeas.Version{ 412 | Name: "52.1", 413 | Revision: "8+deb8u4", 414 | }, 415 | }, 416 | }, 417 | }, 418 | }, 419 | RelatedUrl: []grafeas.RelatedUrl{ 420 | { 421 | Url: "https://security-tracker.debian.org/tracker/CVE-2017-I-made-this-up", 422 | Label: "More Info", 423 | }, 424 | { 425 | Url: "http://people.ubuntu.com/~ubuntu-security/cve/CVE-2017-I-made-this-up", 426 | Label: "More Info", 427 | }, 428 | }, 429 | } 430 | 431 | switch image { 432 | case "lizrice/hello:1": 433 | return []grafeas.Note{note0} 434 | case "lizrice/hello:2": 435 | return []grafeas.Note{note0, note1} 436 | } 437 | 438 | return []grafeas.Note{} 439 | } 440 | -------------------------------------------------------------------------------- /grafeas/storage.go: -------------------------------------------------------------------------------- 1 | // Package grafeas allows manifesto to interact with a Grafeas server 2 | // 3 | // Copyright © 2017 Aqua Security Software Ltd. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License.package grafeas 16 | package grafeas 17 | 18 | import ( 19 | "context" 20 | "encoding/json" 21 | "fmt" 22 | "io/ioutil" 23 | "net/http" 24 | "os" 25 | "os/exec" 26 | "path/filepath" 27 | "strings" 28 | 29 | "golang.org/x/oauth2/google" 30 | 31 | grafeas "github.com/Grafeas/client-go/v1alpha1" 32 | "github.com/op/go-logging" 33 | ) 34 | 35 | var log = logging.MustGetLogger("") 36 | 37 | // Storage is a storage backend for metadata that uses the Grafeas API 38 | // 39 | // For the time being it only supports vulnerability metadata in Aqua format 40 | // which it converts into Grafeas Notes & Occurrences. 41 | // 42 | // manifesto metadata type must be aqua_vulnerability_scan 43 | type Storage struct { 44 | verbose bool 45 | projID string 46 | client *http.Client 47 | url string 48 | } 49 | 50 | // NewStorage returns a metadata storage backend using Grafeas 51 | func NewStorage(url string, projID string, verbose bool) *Storage { 52 | var err error 53 | var c *http.Client 54 | log.Debugf("Grafeas backend at %s for project %s", url, projID) 55 | 56 | switch os.Getenv("GOOGLE_APPLICATION_CREDENTIALS") { 57 | case "": 58 | fmt.Printf("Set GOOGLE_APPLICATION_CREDENTIALS to the service account json file\n") 59 | os.Exit(1) 60 | case "LOCAL": 61 | log.Debugf("No credentials") 62 | c = http.DefaultClient 63 | default: 64 | c, err = google.DefaultClient(context.Background(), "https://www.googleapis.com/auth/cloud-platform") 65 | if err != nil { 66 | fmt.Printf("Error getting google API client: %v", err) 67 | os.Exit(1) 68 | } 69 | } 70 | 71 | // Test that we can access this API 72 | u := url + "/v1alpha1/projects/" + projID + "/occurrences" 73 | req, err := http.NewRequest("GET", u, nil) 74 | if err != nil { 75 | log.Errorf("http request: %v", err) 76 | os.Exit(1) 77 | } 78 | 79 | resp, err := c.Do(req) 80 | if err != nil { 81 | log.Errorf("API error %v", err) 82 | os.Exit(1) 83 | } 84 | 85 | if resp.StatusCode != http.StatusOK { 86 | log.Errorf("API Status %s", resp.Status) 87 | os.Exit(1) 88 | } 89 | 90 | log.Debugf("Can access API") 91 | 92 | return &Storage{ 93 | verbose: verbose, 94 | projID: projID, 95 | client: c, 96 | url: url, 97 | } 98 | } 99 | 100 | // urlForImage gets the resourceUrl for an image 101 | func urlForImage(image string) (string, error) { 102 | ex := exec.Command("docker", "inspect", image, "-f", "{{.RepoDigests}}") 103 | digestOut, err := ex.Output() 104 | if err != nil { 105 | return "", fmt.Errorf("error reading inspect output: %v", err) 106 | } 107 | 108 | hh := strings.Split(string(digestOut), "@") 109 | if len(hh) < 2 { 110 | return "", fmt.Errorf("digest not found in %s", digestOut) 111 | } 112 | 113 | digest := strings.TrimSpace(hh[1]) 114 | digest = strings.TrimRight(digest, "]") 115 | return image + ":" + digest, nil 116 | } 117 | 118 | // GetMetadata gets a named piece of metadata. 119 | // Only PACKAGE_VULNERABILITY is currently supported for Grafeas backend 120 | func (s *Storage) GetMetadata(image string, metadata string) ([]byte, string, error) { 121 | var data []byte 122 | var occurrences []grafeas.Occurrence 123 | 124 | if metadata != "PACKAGE_VULNERABILITY" { 125 | return nil, image, fmt.Errorf("metadata type must be PACKAGE_VULNERABILITY for Grafeas") 126 | } 127 | 128 | // ListOccurrences filtering on the container image URL 129 | u, err := urlForImage(image) 130 | if err != nil { 131 | return nil, image, err 132 | } 133 | 134 | // TODO!! Need to figure out what to do with the gcr.io prefix 135 | filter := "resource_url=" + u 136 | occurrenceRsp, err := s.listOccurrences(s.projID, filter, 0, "") 137 | if err != nil { 138 | return []byte{}, image, fmt.Errorf("list occurrences failed: %v", err) 139 | } 140 | 141 | for _, occurrence := range occurrenceRsp.Occurrences { 142 | log.Debugf("Occurrence for %s", occurrence.ResourceUrl) 143 | log.Debugf(" kind: %#v", occurrence.Kind) 144 | if occurrence.Kind == metadata { 145 | occurrences = append(occurrences, occurrence) 146 | } 147 | } 148 | 149 | // TODO!! Turn the occurrences list into an Aqua format report? 150 | data, err = json.Marshal(occurrences) 151 | return data, image, err 152 | } 153 | 154 | // ListMetadata gets a list of metadata for an image 155 | func (s *Storage) ListMetadata(image string) ([]string, string, error) { 156 | metadataTypes := make(map[string]struct{}, 0) 157 | var metadata []string 158 | 159 | // ListOccurrences filtering on the container image URL 160 | u, err := urlForImage(image) 161 | if err != nil { 162 | return nil, image, err 163 | } 164 | 165 | // TODO!! Need to figure out what to do with the gcr.io prefix 166 | filter := "resource_url=" + u 167 | log.Debugf("Filter %s", filter) 168 | occurrenceRsp, err := s.listOccurrences(s.projID, filter, 0, "") 169 | if err != nil { 170 | return []string{}, image, fmt.Errorf("list occurrences failed: %v", err) 171 | } 172 | 173 | for _, occurrence := range occurrenceRsp.Occurrences { 174 | if _, ok := metadataTypes[occurrence.Kind]; !ok { 175 | metadataTypes[occurrence.Kind] = struct{}{} 176 | } 177 | } 178 | 179 | for k := range metadataTypes { 180 | metadata = append(metadata, k) 181 | } 182 | 183 | return metadata, image, nil 184 | } 185 | 186 | // PutMetadata stores metadata about an image 187 | func (s *Storage) PutMetadata(image string, metadata string, dirName string) (string, error) { 188 | // We only support PACKAGE_VULNERABILITY for the metadata type in Grafeas 189 | if metadata != "PACKAGE_VULNERABILITY" { 190 | return image, fmt.Errorf("metadata type must be PACKAGE_VULNERABILITY for Grafeas") 191 | } 192 | image, err := s.load(dirName, "notes", s.projID) 193 | if err == nil { 194 | image, err = s.load(dirName, "occurrences", s.projID) 195 | } 196 | 197 | return image, err 198 | 199 | } 200 | 201 | // load reads JSON files representing Notes and Occurrences, and stores them using Grafeas 202 | // - dirName is the parent directory (this would typically be the name of the container image) 203 | // - objType must be either "notes" or "occurrences" 204 | // 205 | // load expects to find files in //.json 206 | // For notes the name will be taken from the . 207 | func (s *Storage) load(dirName string, objType string, projID string) (image string, err error) { 208 | d := filepath.Join(dirName, objType) 209 | if _, err := os.Stat(d); err != nil { 210 | return "", fmt.Errorf("no directory found at %s", d) 211 | } 212 | 213 | files, err := ioutil.ReadDir(d) 214 | if err != nil { 215 | return "", fmt.Errorf("error reading directory %s: %v", d, err) 216 | } 217 | 218 | for _, f := range files { 219 | if !strings.HasSuffix(f.Name(), ".json") { 220 | continue 221 | } 222 | b, err := ioutil.ReadFile(filepath.Join(d, f.Name())) 223 | if err != nil { 224 | return "", fmt.Errorf("error reading file %s: %v", f.Name(), err) 225 | } 226 | 227 | switch objType { 228 | case "notes": 229 | // TODO!! Should check first to see if Note already exists 230 | id := strings.TrimSuffix(f.Name(), ".json") 231 | log.Debugf("Creating note projectID %s, note name %s", projID, id) 232 | _, err = s.createNoteFromBytes(projID, id, b) 233 | if err != nil { 234 | return image, fmt.Errorf("create note %s failed: %v", id, err) 235 | } 236 | case "occurrences": 237 | // TODO!! Check that the image name in each Occurrence matches the image name we were given 238 | log.Debugf("Creating occurrence in projectID %s", s.projID) 239 | _, err = s.createOccurrenceFromBytes(s.projID, b) 240 | if err != nil { 241 | return image, fmt.Errorf("create occurrence for %s failed: %v", projID, err) 242 | } 243 | default: 244 | return "", fmt.Errorf("unexpected object type %s", objType) 245 | } 246 | } 247 | 248 | return image, nil 249 | } 250 | -------------------------------------------------------------------------------- /grafeas/storage_test.go: -------------------------------------------------------------------------------- 1 | package grafeas 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io/ioutil" 7 | "net/http" 8 | "net/http/httptest" 9 | "os" 10 | "path/filepath" 11 | "testing" 12 | ) 13 | 14 | func TestLoadAndPutMetadata(t *testing.T) { 15 | dir := "temptest" 16 | err := os.Mkdir(dir, 0777) 17 | if err != nil { 18 | t.Fatalf("Can't create temp dir") 19 | } 20 | defer os.RemoveAll(dir) 21 | fmt.Printf("temp dir is %s\n", dir) 22 | 23 | oPath := filepath.Join(dir, "occurrences") 24 | if err := os.Mkdir(oPath, 0777); err != nil { 25 | t.Fatalf("mkdir %s %v", oPath, err) 26 | } 27 | nPath := filepath.Join(dir, "notes") 28 | if err := os.Mkdir(nPath, 0777); err != nil { 29 | t.Fatalf("mkdir %s %v", nPath, err) 30 | } 31 | 32 | occurrences := tempOccurrences("lizrice/hello:1") 33 | b, err := json.Marshal(occurrences[0]) 34 | oFile := filepath.Join(oPath, "occurrence1.json") 35 | fmt.Printf("writing file at %s\n", oFile) 36 | if err := ioutil.WriteFile(oFile, b, 0666); err != nil { 37 | log.Fatal(err) 38 | } 39 | 40 | notes := tempNotes("lizrice/hello:1") 41 | b, err = json.Marshal(notes[0]) 42 | nFile := filepath.Join(nPath, "note1.json") 43 | if err := ioutil.WriteFile(nFile, b, 0666); err != nil { 44 | log.Fatal(err) 45 | } 46 | 47 | server := httptest.NewServer( 48 | http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 49 | w.Write([]byte(`{"name": "test"}`)) 50 | })) 51 | 52 | defer server.Close() 53 | s := &Storage{ 54 | projID: "testProjID", 55 | url: server.URL, 56 | client: http.DefaultClient, 57 | } 58 | 59 | _, err = s.PutMetadata("lizrice/hello:1", "PACKAGE_VULNERABILITY", dir) 60 | if err != nil { 61 | t.Fatalf("%v", err) 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2017 Aqua Security Software Ltd. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import ( 18 | "fmt" 19 | "os" 20 | 21 | "github.com/aquasecurity/manifesto/cmd" 22 | ) 23 | 24 | func main() { 25 | if err := cmd.RootCmd.Execute(); err != nil { 26 | fmt.Println(err) 27 | os.Exit(-1) 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /registry/blob.go: -------------------------------------------------------------------------------- 1 | // Package registry is a cut-down Registry V2 client for manifesto 2 | // 3 | // Copyright © 2017 Aqua Security Software Ltd. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | package registry 18 | 19 | import ( 20 | "crypto/sha256" 21 | "encoding/hex" 22 | "fmt" 23 | "io" 24 | "io/ioutil" 25 | "net/http" 26 | "strings" 27 | ) 28 | 29 | func computeDigest(data []byte) string { 30 | hasher := sha256.New() 31 | hasher.Write(data) 32 | digest := hasher.Sum(nil) 33 | return "sha256:" + hex.EncodeToString(digest) 34 | } 35 | 36 | // UploadBlob pushes a blob of data to the registry 37 | func (r *V2) UploadBlob(repoName string, data io.Reader) (string, error) { 38 | 39 | // Post to get the location / UUID for this upload 40 | URL := "/v2/" + repoName + "/blobs/uploads/" 41 | res, err := r.call("POST", URL, []byte{}, "") 42 | if err != nil { 43 | return "", fmt.Errorf("post to %s failed: %v", URL, err) 44 | } 45 | 46 | // We don't need the body at all so discard and close it now 47 | io.Copy(ioutil.Discard, res.Body) 48 | res.Body.Close() 49 | 50 | if res.StatusCode != http.StatusAccepted { 51 | return "", fmt.Errorf("post to %s not accepted: %s", URL, res.Status) 52 | } 53 | 54 | // The post gives us the location for the blob upload 55 | location := res.Header.Get("Location") 56 | 57 | // Read the data so we can calculate its digest 58 | b, err := ioutil.ReadAll(data) 59 | if err != nil { 60 | return "", fmt.Errorf("couldn't read data: %v", err) 61 | } 62 | 63 | digest := computeDigest(b) 64 | if strings.Contains(location, "?") { 65 | location += ("&digest=" + digest) 66 | } else { 67 | location += ("?digest=" + digest) 68 | } 69 | 70 | // Upload the data monolithically 71 | res, err = r.call("PUT", location, b, "application/octet-stream") 72 | if err != nil { 73 | return "", fmt.Errorf("upload blob failed: %v", err) 74 | } 75 | 76 | io.Copy(ioutil.Discard, res.Body) 77 | res.Body.Close() 78 | 79 | if res.StatusCode != http.StatusCreated { 80 | return "", fmt.Errorf("unexpected upload status: %s", res.Status) 81 | } 82 | 83 | return digest, nil 84 | } 85 | 86 | // GetBlob downloads a blob specified by repo name and digest 87 | func (r *V2) GetBlob(repoName string, digest string) ([]byte, error) { 88 | res, err := r.get("/v2/" + repoName + "/blobs/" + digest) 89 | if err != nil { 90 | return []byte{}, fmt.Errorf("get blob failed: %v", err) 91 | } 92 | 93 | defer res.Body.Close() 94 | if res.StatusCode != http.StatusOK { 95 | io.Copy(ioutil.Discard, res.Body) 96 | return []byte{}, fmt.Errorf("unexpected get blob status: %s", res.Status) 97 | } 98 | 99 | data, err := ioutil.ReadAll(res.Body) 100 | if err != nil { 101 | return []byte{}, fmt.Errorf("error reading get blob: %v", err) 102 | } 103 | 104 | return data, nil 105 | } 106 | -------------------------------------------------------------------------------- /registry/blob_test.go: -------------------------------------------------------------------------------- 1 | package registry 2 | 3 | import ( 4 | "strings" 5 | "testing" 6 | ) 7 | 8 | func TestComputeDigest(t *testing.T) { 9 | const shaLen = 64 10 | d := computeDigest([]byte("hello world")) 11 | dd := strings.TrimPrefix(d, "sha256:") 12 | if len(dd) == len(d) { 13 | t.Fatalf("Digest %s didn't start with 'sha256:'", d) 14 | } 15 | 16 | if len(dd) != shaLen { 17 | t.Fatalf("Expected SHA to have %d chars, it's %d\n%s", shaLen, len(dd), dd) 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /registry/names.go: -------------------------------------------------------------------------------- 1 | package registry 2 | 3 | import "strings" 4 | 5 | // registryName - dockerHub if omitted 6 | // repoName - hostname/org/repo - hostname is omitted if it's dockerHub 7 | // imageName - hostname/org/repo:tag - latest is used for the tag if not specified; hostname omitted if it's dockerHub 8 | // repoNameNoHost - org/repo - hostname always omitted 9 | func getNameComponents(name string) (registryName string, repoName string, imageName string, repoNameNoHost string, tagName string, digestName string) { 10 | 11 | // reference := name [ ":" tag ] [ "@" digest ] 12 | nameSlice := strings.Split(name, "@") 13 | if len(nameSlice) > 1 { 14 | digestName = nameSlice[1] 15 | name = nameSlice[0] 16 | } 17 | 18 | // name := [ hostname "/" ] component [ "/" component ]* 19 | // name can include : as part of the host name, so we look for hostname and components before 20 | // looking for the tag 21 | nameSlice = strings.Split(name, "/") 22 | registryName = dockerHub 23 | if len(nameSlice) > 2 { 24 | name = strings.Join(nameSlice[1:], "/") 25 | registryName = nameSlice[0] 26 | 27 | // Include registry name in repo name if it's not Docker Hub 28 | repoName = registryName + "/" 29 | } 30 | 31 | // Now look for a tag 32 | nameSlice = strings.Split(name, ":") 33 | repoName += nameSlice[0] 34 | repoNameNoHost = nameSlice[0] 35 | tagName = "latest" 36 | if len(nameSlice) > 1 { 37 | tagName = nameSlice[1] 38 | } 39 | 40 | imageName = repoName + ":" + tagName 41 | return registryName, repoName, imageName, repoNameNoHost, tagName, digestName 42 | } 43 | 44 | func imageNameForManifest(imageName string) string { 45 | return imageName + ":_manifesto" 46 | } 47 | -------------------------------------------------------------------------------- /registry/names_test.go: -------------------------------------------------------------------------------- 1 | package registry 2 | 3 | import "testing" 4 | 5 | func TestGetNameComponents(t *testing.T) { 6 | cases := []struct { 7 | input string 8 | reg string 9 | repo string 10 | img string 11 | repoNoHost string 12 | tag string 13 | digest string 14 | }{ 15 | {"lizrice/imagetest", "registry-1.docker.io", "lizrice/imagetest", "lizrice/imagetest:latest", "lizrice/imagetest", "latest", ""}, 16 | {"lizrice/imagetest:v1.0", "registry-1.docker.io", "lizrice/imagetest", "lizrice/imagetest:v1.0", "lizrice/imagetest", "v1.0", ""}, 17 | {"quay.io/lizrice/imagetest:v1.0", "quay.io", "quay.io/lizrice/imagetest", "quay.io/lizrice/imagetest:v1.0", "lizrice/imagetest", "v1.0", ""}, 18 | {"localhost:5000/lizrice/imagetest:v1.0", "localhost:5000", "localhost:5000/lizrice/imagetest", "localhost:5000/lizrice/imagetest:v1.0", "lizrice/imagetest", "v1.0", ""}, 19 | {"lizrice/imagetest@12345", "registry-1.docker.io", "lizrice/imagetest", "lizrice/imagetest:latest", "lizrice/imagetest", "latest", "12345"}, 20 | } 21 | 22 | for _, c := range cases { 23 | t.Run(c.input, func(t *testing.T) { 24 | reg, repo, img, repoNoHost, tag, digest := getNameComponents(c.input) 25 | if reg != c.reg { 26 | t.Fatalf("registry name: got %s expected %s", reg, c.reg) 27 | } 28 | if repo != c.repo { 29 | t.Fatalf("repo name: got %s expected %s", repo, c.repo) 30 | } 31 | if img != c.img { 32 | t.Fatalf("image name: got %s expected %s", img, c.img) 33 | } 34 | if repoNoHost != c.repoNoHost { 35 | t.Fatalf("repo name without host: got %s expected %s", repoNoHost, c.repoNoHost) 36 | } 37 | if tag != c.tag { 38 | t.Fatalf("tag name: got %s expected %s", tag, c.tag) 39 | } 40 | if digest != c.digest { 41 | t.Fatalf("digest: got %s expected %s", digest, c.digest) 42 | } 43 | }) 44 | } 45 | } 46 | 47 | func TestImageNameForManifest(t *testing.T) { 48 | cases := []struct { 49 | input string 50 | img string 51 | }{ 52 | {"lizrice/imagetest", "lizrice/imagetest:_manifesto"}, 53 | {"lizrice/imagetest:v1.0", "lizrice/imagetest:_manifesto"}, 54 | {"quay.io/lizrice/imagetest:v1.0", "quay.io/lizrice/imagetest:_manifesto"}, 55 | {"localhost:5000/lizrice/imagetest", "localhost:5000/lizrice/imagetest:_manifesto"}, 56 | } 57 | 58 | for _, c := range cases { 59 | t.Run(c.input, func(t *testing.T) { 60 | _, repo, _, _, _, _ := getNameComponents(c.input) 61 | img := imageNameForManifest(repo) 62 | if img != c.img { 63 | t.Fatalf("manifesto image name: got %s expected %s", img, c.img) 64 | } 65 | }) 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /registry/registry.go: -------------------------------------------------------------------------------- 1 | // Package registry is a cut-down Registry V2 client for manifesto 2 | // 3 | // Copyright © 2017 Aqua Security Software Ltd. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | package registry 17 | 18 | import ( 19 | "bytes" 20 | "encoding/json" 21 | "errors" 22 | "fmt" 23 | "net/http" 24 | "net/url" 25 | "os" 26 | "regexp" 27 | "strings" 28 | "time" 29 | 30 | "golang.org/x/crypto/ssh/terminal" 31 | ) 32 | 33 | const dockerHub = "registry-1.docker.io" 34 | const tempFileName = "_manifesto.out" 35 | const tempContainerName = "manifesto.temp" 36 | 37 | // AuthType is a simple int type used for enumerations of the authentication 38 | // types supported by version 2 registries 39 | type AuthType int 40 | 41 | const ( 42 | // Insecure denotes a registry with no authentication at all 43 | Insecure AuthType = iota 44 | // Token denotes a registry with token-based (OAuth2) authentication 45 | Token 46 | // Htpasswd denotes a registry with Basic HTTP authentication 47 | Htpasswd 48 | ) 49 | 50 | type token string 51 | type scope string 52 | 53 | // V2 is the registry structure for registries supporting the V2 API 54 | type V2 struct { 55 | Client *http.Client 56 | URL string 57 | AuthType AuthType 58 | Username string 59 | Password string 60 | Tokens map[scope]token // map from scope to token 61 | Realm string 62 | Service string 63 | } 64 | 65 | type AuthChallenge struct { 66 | Scheme, Realm string 67 | Params map[string]string 68 | } 69 | 70 | // MetadataManifesto gives the type of a piece of arbitrary manifesto data, and the digest where it can be found 71 | // A given image can only have one current piece of data of each type. 72 | // Example types might include: "seccomp", "approvals", "contact" 73 | type MetadataManifesto struct { 74 | Type string `json:"type"` 75 | Digest string `json:"digest"` 76 | } 77 | 78 | // ImageMetadataManifesto associates a piece of manifesto data with a particular image 79 | type ImageMetadataManifesto struct { 80 | ImageDigest string `json:"image_digest"` 81 | MetadataManifesto []MetadataManifesto `json:"manifesto"` 82 | } 83 | 84 | // MetadataManifestoList holds all the metadata for a given image repository 85 | type MetadataManifestoList struct { 86 | Images []ImageMetadataManifesto `json:"images"` 87 | } 88 | 89 | // New creates a new instance of the V2 structure for the registry located 90 | // in the provided URL, and checks that the registry supports V2 91 | func New(URL, username, password string) (*V2, error) { 92 | // make sure URL does not have a trailing slash 93 | URL = strings.TrimSpace(URL) 94 | URL = strings.TrimSuffix(URL, "/") 95 | 96 | // make sure URL is not empty 97 | if URL == "" { 98 | return nil, errors.New("The registry URL must be provided") 99 | } 100 | 101 | if !strings.HasPrefix(URL, "http") { 102 | URL = "https://" + URL 103 | } 104 | 105 | // If we don't already have a username and password, prompt for them 106 | if username == "" { 107 | fmt.Fprintf(os.Stderr, "Username: ") 108 | fmt.Scanf("%s", &username) 109 | } 110 | if password == "" { 111 | fmt.Fprintf(os.Stderr, "Password: ") 112 | pwd, err := terminal.ReadPassword(0) 113 | fmt.Fprintf(os.Stderr, "\n") 114 | if err != nil { 115 | return nil, fmt.Errorf("error reading password: %v", err) 116 | } 117 | password = string(pwd) 118 | } 119 | 120 | r := &V2{ 121 | URL: URL, 122 | Client: &http.Client{ 123 | Timeout: 10 * time.Second, 124 | }, 125 | Username: username, 126 | Password: password, 127 | Tokens: make(map[scope]token), 128 | } 129 | 130 | if username != "" { 131 | r.AuthType = Htpasswd 132 | } 133 | 134 | return r, nil 135 | } 136 | 137 | // Get is shorthand for call, without having to pass in unneeded parameters 138 | func (r *V2) get(path string) (*http.Response, error) { 139 | return r.call("GET", path, []byte{}, "") 140 | } 141 | 142 | // call makes an HTTP request; if it fails authentication it tries to get the right authentication token 143 | // and tries again 144 | func (r *V2) call(method string, path string, data []byte, contentType string) (*http.Response, error) { 145 | // Try making the request 146 | res, err := r.makeRequest(method, path, data, contentType, "") 147 | if err != nil { 148 | return nil, fmt.Errorf("failed request: %v", err) 149 | } 150 | 151 | // If authorization wasn't a problem we can return 152 | if res.StatusCode != http.StatusUnauthorized { 153 | return res, nil 154 | } 155 | 156 | // If this wasn't authorized we should have a challenge describing the token we need to get 157 | auth, err := parseWWWAuthenticate(res.Header.Get("WWW-Authenticate")) 158 | if err != nil { 159 | return nil, fmt.Errorf("failed to parse www-authenticate: %v", err) 160 | } 161 | 162 | // We might have a token we can try 163 | s, ok := auth.Params["scope"] 164 | if ok { 165 | // We know the scope, do we have a token? 166 | t, ok := r.Tokens[scope(s)] 167 | if ok { 168 | res, err = r.makeRequest(method, path, data, contentType, t) 169 | if err != nil { 170 | return res, fmt.Errorf("failed request with existing token: %v", err) 171 | } 172 | 173 | // If authorization wasn't a problem we can return successfully 174 | if res.StatusCode != http.StatusUnauthorized { 175 | return res, nil 176 | } 177 | } 178 | } 179 | 180 | // We didn't have the right token, or if we had a token it didn't work, perhaps because it has expired. 181 | t, err := r.getToken(auth) 182 | if err != nil { 183 | return nil, fmt.Errorf("failed to get token: %v", err) 184 | } 185 | 186 | // Try the call again 187 | res, err = r.makeRequest(method, path, data, contentType, t) 188 | return res, err 189 | } 190 | 191 | // makeRequest makes an HTTP request, setting headers and authentication 192 | func (r *V2) makeRequest(method string, path string, data []byte, contentType string, t token) (*http.Response, error) { 193 | url := path 194 | if strings.HasPrefix(url, "/") { 195 | url = r.URL + path 196 | } 197 | 198 | req, err := http.NewRequest(method, url, bytes.NewReader(data)) 199 | if err != nil { 200 | return nil, fmt.Errorf("failed to make request: %v", err) 201 | } 202 | 203 | req.Header.Set("User-Agent", "curl") 204 | if contentType != "" { 205 | req.Header.Set("Content-Type", contentType) 206 | } 207 | 208 | switch r.AuthType { 209 | case Htpasswd: 210 | req.SetBasicAuth(r.Username, r.Password) 211 | case Token: 212 | if t != "" { 213 | req.Header.Set("Authorization", "Bearer "+string(t)) 214 | } 215 | } 216 | 217 | return r.Client.Do(req) 218 | } 219 | 220 | // AuthResponse contains the token 221 | type AuthResponse struct { 222 | Token string 223 | } 224 | 225 | func (r *V2) getToken(auth AuthChallenge) (token, error) { 226 | // Construct a request for a token, using the information we parsed out of an authenticate challenge 227 | query := url.Values{} 228 | for k, v := range auth.Params { 229 | query.Add(k, v) 230 | } 231 | 232 | tokenURL := auth.Realm + "?" + query.Encode() 233 | req, err := http.NewRequest("GET", tokenURL, nil) 234 | if err != nil { 235 | return "", fmt.Errorf("failed to create request: %s", err) 236 | } 237 | 238 | req.SetBasicAuth(r.Username, r.Password) 239 | 240 | res, err := r.Client.Do(req) 241 | if err != nil { 242 | return "", fmt.Errorf("failed to send request: %s", err) 243 | } 244 | defer res.Body.Close() 245 | 246 | if res.StatusCode != http.StatusOK { 247 | return "", fmt.Errorf("token request returned status %s", res.Status) 248 | } 249 | 250 | // Get the token out of the response 251 | var authRsp AuthResponse 252 | err = json.NewDecoder(res.Body).Decode(&authRsp) 253 | if err != nil { 254 | return "", fmt.Errorf("error decoding token response: %v", err) 255 | } 256 | 257 | s, ok := auth.Params["scope"] 258 | if !ok { 259 | return "", fmt.Errorf("no scope for token") 260 | } 261 | r.Tokens[scope(s)] = token(authRsp.Token) 262 | r.AuthType = Token // We'll use tokens from now on 263 | return token(authRsp.Token), nil 264 | } 265 | 266 | var authChallengeRegexp = regexp.MustCompile("^([A-Za-z0-9]+) realm=\"([^\"]+)\"(.*)") 267 | var authParamRegexp = regexp.MustCompile(",([^=]+)=\"([^\"]+)\"") 268 | 269 | // parseWWWAuthenticate parses the contents of a "WWW-Authenticate" HTTP header 270 | func parseWWWAuthenticate(header string) (auth AuthChallenge, err error) { 271 | matches := authChallengeRegexp.FindStringSubmatch(header) 272 | 273 | if len(matches) < 3 { 274 | return auth, errors.New("Empty or invalid WWW-Authenticate header") 275 | } 276 | 277 | auth.Scheme = matches[1] 278 | auth.Realm = matches[2] 279 | 280 | if len(matches) == 4 { 281 | // we also have parameters, let's parse them 282 | auth.Params = make(map[string]string) 283 | paramMatches := authParamRegexp.FindAllStringSubmatch(matches[3], -1) 284 | for _, p := range paramMatches { 285 | // Each of these matches should have 3 entries 286 | // [0] = whole match e.g. ,service="registry.docker.io" 287 | // [1] = parameter name e.g. service 288 | // [2] = parameter value e.g. registry.docker.io 289 | if len(p) != 3 { 290 | return auth, errors.New("Failed to parse WWW-Authenticate header params") 291 | } 292 | auth.Params[p[1]] = p[2] 293 | } 294 | } 295 | 296 | return auth, nil 297 | } 298 | -------------------------------------------------------------------------------- /registry/registry_test.go: -------------------------------------------------------------------------------- 1 | package registry 2 | 3 | import "testing" 4 | 5 | func TestParseWWWAuth(t *testing.T) { 6 | testcases := []struct { 7 | input string 8 | scheme string 9 | realm string 10 | scope string 11 | service string 12 | }{ 13 | {input: "Basic realm=\"secure\"", scheme: "Basic", realm: "secure"}, 14 | {input: "Bearer realm=\"https://auth.docker.io/token\",service=\"registry.docker.io\",scope=\"repository:library/mongo:pull\"", 15 | scheme: "Bearer", realm: "https://auth.docker.io/token", service: "registry.docker.io", scope: "repository:library/mongo:pull"}, 16 | {input: "Bearer realm=\"https://auth.docker.io/token\",service=\"registry.docker.io\",scope=\"repository:library/mongo:push,pull\"", 17 | scheme: "Bearer", realm: "https://auth.docker.io/token", service: "registry.docker.io", scope: "repository:library/mongo:push,pull"}, 18 | } 19 | 20 | for _, tc := range testcases { 21 | t.Run(tc.input, func(t *testing.T) { 22 | auth, err := parseWWWAuthenticate(tc.input) 23 | if err != nil { 24 | t.Fatalf("Failed but shouldn't have: %v", err) 25 | } 26 | if auth.Scheme != tc.scheme { 27 | t.Fatalf("Scheme: expected %s, got %s", tc.scheme, auth.Scheme) 28 | } 29 | if auth.Realm != tc.realm { 30 | t.Fatalf("Realm: expected %s, got %s", tc.realm, auth.Realm) 31 | } 32 | 33 | compare(t, auth, "service", tc.service) 34 | compare(t, auth, "scope", tc.scope) 35 | }) 36 | } 37 | } 38 | 39 | func compare(t *testing.T, auth AuthChallenge, paramName string, expected string) { 40 | param, ok := auth.Params[paramName] 41 | if expected != "" { 42 | if !ok { 43 | t.Fatal("Params: expected to have '%s' key, did not have one", paramName) 44 | } 45 | if param != expected { 46 | t.Fatalf("Params.%s: expected %s, got %s", paramName, expected, param) 47 | } 48 | } else { 49 | if ok { 50 | t.Fatal("Params: didn't expect to have '%s' key", paramName) 51 | } 52 | } 53 | } 54 | 55 | func TestNew(t *testing.T) { 56 | r, err := New("example.com", "user", "pass") 57 | if err != nil { 58 | t.Fatalf("Failed to create registry: %v", err) 59 | } 60 | 61 | if r.URL != "https://example.com" { 62 | t.Fatalf("Unexpected registry URL %s", r.URL) 63 | } 64 | 65 | if r.Username != "user" { 66 | t.Fatalf("Unexpected user %s", r.Username) 67 | } 68 | 69 | if r.Password != "pass" { 70 | t.Fatalf("Unexpected password %s", r.Password) 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /registry/storage.go: -------------------------------------------------------------------------------- 1 | package registry 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io" 7 | "io/ioutil" 8 | "os" 9 | "os/exec" 10 | "strings" 11 | 12 | "github.com/labstack/gommon/log" 13 | ) 14 | 15 | // Storage is a backend for storing metadata that uses the Registry V2 API 16 | type Storage struct { 17 | username string 18 | password string 19 | verbose bool 20 | registry *V2 21 | } 22 | 23 | // NewStorage returns a metadata storage backend using a Registry 24 | func NewStorage(username, password string, verbose bool) *Storage { 25 | return &Storage{ 26 | username: username, 27 | password: password, 28 | verbose: verbose, 29 | } 30 | } 31 | 32 | // GetMetadata returns the data stored for this image with this metadata type 33 | func (s *Storage) GetMetadata(name string, metadata string) (data []byte, imageName string, err error) { 34 | registryURL, repoName, imageName, repoNameNoHost, _, imageDigest := getNameComponents(name) 35 | metadataImageName := imageNameForManifest(repoName) 36 | 37 | // Get the digest for the image 38 | if imageDigest == "" { 39 | imageDigest, err = s.dockerGetDigest(imageName) 40 | if err != nil { 41 | err = fmt.Errorf("image '%s' not found: %v", imageName, err) 42 | return 43 | } 44 | } 45 | 46 | log.Debugf("Image has digest %s", imageDigest) 47 | 48 | // Get the metadata manifest for this image 49 | raw, err := s.dockerGetData(metadataImageName) 50 | if err != nil { 51 | fmt.Printf("No manifesto data stored for image '%s'", imageName) 52 | os.Exit(1) 53 | } 54 | var mml MetadataManifestoList 55 | json.Unmarshal(raw, &mml) 56 | log.Debug("Repo metadata index retrieved") 57 | 58 | // We'll need the registry API from here on 59 | r, err := New(registryURL, s.username, s.password) 60 | if err != nil { 61 | fmt.Fprintf(os.Stderr, "Error connecting to registry: %v\n", err) 62 | os.Exit(1) 63 | } 64 | 65 | for _, v := range mml.Images { 66 | if v.ImageDigest == imageDigest { 67 | log.Debug("Image metadata retrieved") 68 | for _, m := range v.MetadataManifesto { 69 | if m.Type == metadata { 70 | log.Debugf("'%s' metadata identified", metadata) 71 | contents, err := r.GetBlob(repoNameNoHost, m.Digest) 72 | if err != nil { 73 | // Maybe this metadata was stored as an image by a previous version of manifesto 74 | // so try getting it that way 75 | // TODO!! Retire this one day 76 | log.Debugf("This metadata may be stored in an image rather than a blob: %v", err) 77 | contents, err = s.dockerGetData(repoName + "@" + m.Digest) 78 | if err != nil { 79 | fmt.Printf("Couldn't find %s data from manifesto: %v\n", metadata, err) 80 | os.Exit(1) 81 | } 82 | } 83 | 84 | // We should have found the data so we may as well quit now 85 | return contents, imageName, nil 86 | } 87 | } 88 | } 89 | } 90 | return []byte{}, imageName, nil 91 | } 92 | 93 | // ListMetadata lists the types of metadata currently stored for this image 94 | func (s *Storage) ListMetadata(image string) (metadataTypes []string, imageName string, err error) { 95 | _, repoName, imageName, _, _, imageDigest := getNameComponents(image) 96 | 97 | metadataImageName := imageNameForManifest(repoName) 98 | 99 | // Get the digest for the image 100 | if imageDigest == "" { 101 | imageDigest, err = s.dockerGetDigest(imageName) 102 | if err != nil { 103 | fmt.Printf("Image '%s' not found: %v\n", imageName, err) 104 | os.Exit(1) 105 | } 106 | } 107 | 108 | log.Debugf("Image has digest %s", imageDigest) 109 | 110 | // Get the manifesto data for this repo 111 | raw, err := s.dockerGetData(metadataImageName) 112 | if err != nil { 113 | fmt.Printf("No manifesto data stored for image '%s'\n", imageName) 114 | log.Debugf("%v", err) 115 | os.Exit(1) 116 | } 117 | var mml MetadataManifestoList 118 | json.Unmarshal(raw, &mml) 119 | 120 | log.Debugf("Metadata index: %v", mml) 121 | 122 | for _, v := range mml.Images { 123 | if v.ImageDigest == imageDigest { 124 | fmt.Printf("Metadata types stored for image '%s':\n", imageName) 125 | for _, m := range v.MetadataManifesto { 126 | metadataTypes = append(metadataTypes, m.Type) 127 | } 128 | } 129 | } 130 | 131 | return 132 | } 133 | 134 | // PutMetadata stores metadata under a type for an image 135 | func (s *Storage) PutMetadata(image string, metadata string, datafile string) (imageName string, err error) { 136 | f, err := os.Open(datafile) 137 | if err != nil { 138 | return "", fmt.Errorf("error opening file %s: %v", datafile, err) 139 | } 140 | 141 | registryURL, repoName, imageName, repoNameNoHost, _, imageDigest := getNameComponents(image) 142 | metadataImageName := imageNameForManifest(repoName) 143 | 144 | // Get the digest for this image 145 | if imageDigest == "" { 146 | imageDigest, err = s.dockerGetDigest(imageName) 147 | if err != nil { 148 | return imageName, fmt.Errorf("image '%s' not found: %v", imageName, err) 149 | } 150 | } 151 | 152 | log.Debugf("Image has digest %s", imageDigest) 153 | 154 | // We'll need the registry API from here on 155 | r, err := New(registryURL, s.username, s.password) 156 | if err != nil { 157 | return imageName, fmt.Errorf("error connecting to registry: %v", err) 158 | } 159 | 160 | // Store the piece of metadata we've been given 161 | digest, err := r.UploadBlob(repoNameNoHost, f) 162 | if err != nil { 163 | return imageName, fmt.Errorf("error uploading metadata to registry: %v", err) 164 | } 165 | 166 | fmt.Printf("Metadata '%s' for image '%s' stored at %s\n", metadata, imageName, digest) 167 | 168 | // Read the current manifesto if it exists 169 | var mml MetadataManifestoList 170 | raw, err := s.dockerGetData(metadataImageName) 171 | if err != nil { 172 | fmt.Printf("Creating new manifesto for %s\n", repoName) 173 | } else { 174 | json.Unmarshal(raw, &mml) 175 | } 176 | 177 | replaced := false 178 | found := false 179 | for k, v := range mml.Images { 180 | if v.ImageDigest == imageDigest { 181 | found = true 182 | for kk, m := range v.MetadataManifesto { 183 | if m.Type == metadata { 184 | // Replace this with the new blob 185 | fmt.Printf("Updating '%s' metadata in manifesto for '%s'\n", metadata, imageName) 186 | mml.Images[k].MetadataManifesto[kk].Digest = digest 187 | replaced = true 188 | } 189 | } 190 | 191 | // A new piece of metadata for this image 192 | if !replaced { 193 | fmt.Printf("Adding '%s' metadata to manifesto for '%s'\n", metadata, imageName) 194 | newMetadata := MetadataManifesto{ 195 | Type: metadata, 196 | Digest: digest, 197 | } 198 | mml.Images[k].MetadataManifesto = append(mml.Images[k].MetadataManifesto, newMetadata) 199 | } 200 | } 201 | } 202 | 203 | // Metadata for a new image 204 | if !found { 205 | fmt.Printf("Adding '%s' metadata to manifesto for '%s'\n", metadata, imageName) 206 | newImm := ImageMetadataManifesto{ 207 | ImageDigest: imageDigest, 208 | MetadataManifesto: []MetadataManifesto{ 209 | { 210 | Type: metadata, 211 | Digest: digest, 212 | }, 213 | }, 214 | } 215 | mml.Images = append(mml.Images, newImm) 216 | } 217 | 218 | // Write the manifesto file 219 | data, err := json.Marshal(mml) 220 | if err != nil { 221 | return imageName, fmt.Errorf("couldn't marshal manifesto data: %v", err) 222 | } 223 | 224 | err = ioutil.WriteFile(tempFileName, []byte(data), 0644) 225 | if err != nil { 226 | return imageName, fmt.Errorf("couldn't write temporary manifesto file: %v", err) 227 | } 228 | 229 | // Store the manifesto file in the registry 230 | s.dockerPutData(metadataImageName, "manifesto", tempFileName) 231 | err = os.Remove(tempFileName) 232 | if err != nil { 233 | return imageName, fmt.Errorf("couldn't remove temporary manifesto file: %v", err) 234 | } 235 | 236 | return 237 | } 238 | 239 | func (s *Storage) dockerGetDigest(imageName string) (digest string, err error) { 240 | // Make sure we have an up-to-date version of this image 241 | s.execCommand("docker", "pull", imageName) 242 | ex := exec.Command("docker", "inspect", imageName, "-f", "{{.RepoDigests}}") 243 | digestOut, err := ex.Output() 244 | if err != nil { 245 | return "", fmt.Errorf("error reading inspect output: %v", err) 246 | } 247 | 248 | hh := strings.Split(string(digestOut), "@") 249 | if len(hh) < 2 { 250 | return "", fmt.Errorf("digest not found in %s", digestOut) 251 | } 252 | 253 | digest = strings.TrimSpace(hh[1]) 254 | digest = strings.TrimRight(digest, "]") 255 | return digest, nil 256 | } 257 | 258 | func (s *Storage) dockerGetData(imageName string) ([]byte, error) { 259 | err := s.execCommand("docker", "pull", imageName) 260 | if err != nil { 261 | return []byte{}, err 262 | } 263 | 264 | s.execCommand("docker", "create", "--name="+tempContainerName, imageName, "x") 265 | s.execCommand("docker", "cp", tempContainerName+":/data", tempFileName) 266 | s.execCommand("docker", append([]string{"rm"}, tempContainerName)...) 267 | raw, err := ioutil.ReadFile(tempFileName) 268 | if err != nil { 269 | return raw, err 270 | } 271 | err = os.Remove(tempFileName) 272 | if err != nil { 273 | return raw, err 274 | } 275 | 276 | return raw, err 277 | } 278 | 279 | // imageName is the name we'll store this data under, including the tag e.g. myorg/myrepo:mytag or myorg/myrepo@sha256:12345... 280 | // datafile is the name of the file we get the data from 281 | func (s *Storage) dockerPutData(imageName string, metadataName string, datafile string) (string, error) { 282 | // Copy file locally so that it's going to be in the build context 283 | metadata, err := os.Open(datafile) 284 | if err != nil { 285 | return "", fmt.Errorf("couldn't open file %s: %v", datafile, err) 286 | } 287 | 288 | defer metadata.Close() 289 | tf, err := ioutil.TempFile(".", "metadata") 290 | if err != nil { 291 | return "", fmt.Errorf("error creating temporary file: %v", err) 292 | } 293 | 294 | _, err = io.Copy(tf, metadata) 295 | if err != nil { 296 | return "", fmt.Errorf("error copying to temporary file: %v", err) 297 | } 298 | 299 | if err = tf.Close(); err != nil { 300 | os.Remove(tf.Name()) 301 | return "", fmt.Errorf("error closing temporary file: %v", err) 302 | } 303 | 304 | df, err := ioutil.TempFile(".", "Dockerfile") 305 | dockerfile := fmt.Sprintf("FROM scratch \nADD %s /data\n", tf.Name()) 306 | _, err = df.Write([]byte(dockerfile)) 307 | if err != nil { 308 | return "", fmt.Errorf("couldn't create Dockerfile: %v", err) 309 | } 310 | 311 | s.execCommand("docker", "build", "-f", df.Name(), "-t", imageName, ".") 312 | 313 | // Delete the Dockerfile and the temporary file 314 | err = os.Remove(df.Name()) 315 | if err != nil { 316 | return "", fmt.Errorf("couldn't delete Dockerfile: %v", err) 317 | } 318 | 319 | err = os.Remove(tf.Name()) 320 | if err != nil { 321 | return "", fmt.Errorf("couldn't delete temporary file: %v", err) 322 | } 323 | 324 | s.execCommand("docker", "push", imageName) 325 | 326 | digest, err := s.dockerGetDigest(imageName) 327 | if err != nil { 328 | return "", fmt.Errorf("couldn't get digest: %v", err) 329 | } 330 | 331 | return digest, nil 332 | } 333 | 334 | func (s *Storage) execCommand(name string, arg ...string) error { 335 | ex := exec.Command(name, arg...) 336 | ex.Stdin = os.Stdin 337 | ex.Stderr = os.Stderr 338 | if s.verbose { 339 | ex.Stdout = os.Stdout 340 | } 341 | return ex.Run() 342 | } 343 | --------------------------------------------------------------------------------