├── .bumpversion.cfg ├── .dockerignore ├── .gcloudignore ├── .github └── workflows │ └── codeql-analysis.yml ├── .gitignore ├── .vscode ├── launch.json └── settings.json ├── Autologin.md ├── Dockerfile.autologin ├── Dockerfile.boltdb ├── Dockerfile.dynamodb ├── LICENSE ├── README.md ├── app.yaml ├── cloudbuild.yaml ├── cmd ├── autologin │ └── main.go ├── boltdb │ └── main.go ├── dynamodb │ └── main.go ├── gae │ └── main.go └── routes.go ├── dev.bat ├── go.mod ├── go.sum ├── index.yaml ├── misc ├── config │ ├── bench.yaml │ └── metahub-demo.yaml ├── docker │ ├── caddy │ │ └── Caddyfile │ └── docker-compose.yml ├── metahub-demo │ ├── Dockerfile │ ├── bin │ │ └── entry.sh │ ├── build.sh │ └── yml │ │ └── template.yml └── pics │ ├── metahub-overview.png │ └── metahub-proxy.png ├── pkg ├── accounts │ ├── auth_middleware.go │ ├── github_handler.go │ ├── google_handler.go │ ├── handler_base.go │ ├── identity_handler.go │ ├── jwt_encoding.go │ └── router.go ├── daemon │ └── service.go ├── machinetypes │ ├── add_handler.go │ ├── auth_middleware.go │ ├── delete_handler.go │ ├── get_handler.go │ ├── list_handler.go │ ├── password_generator.go │ ├── router.go │ └── update_handler.go ├── registry │ ├── blob.go │ ├── filter │ │ └── service.go │ ├── http │ │ ├── client │ │ │ ├── backend_auth.go │ │ │ └── service.go │ │ └── server │ │ │ ├── base_handler.go │ │ │ ├── blobs_handler.go │ │ │ ├── manifests_handler.go │ │ │ └── router.go │ ├── manifest.go │ ├── manifest_descriptor.go │ ├── manifest_list.go │ └── service.go ├── storage │ ├── accesstoken.go │ ├── accesstoken_service.go │ ├── account.go │ ├── account_service.go │ ├── boltdb │ │ ├── accesstoken_model.go │ │ ├── accesstoken_service.go │ │ ├── account_model.go │ │ ├── account_service.go │ │ ├── dummies.go │ │ ├── machinetype_model.go │ │ ├── machinetype_service.go │ │ ├── service.go │ │ ├── types.go │ │ └── users.go │ ├── clouddatastore │ │ ├── accesstoken_model.go │ │ ├── accesstoken_service.go │ │ ├── account_model.go │ │ ├── account_service.go │ │ ├── machinetype_model.go │ │ ├── machinetype_service.go │ │ └── service.go │ ├── dynamodb │ │ ├── accesstoken_model.go │ │ ├── accesstoken_service.go │ │ ├── account_model.go │ │ ├── account_service.go │ │ ├── dummies.go │ │ ├── machinetype_model.go │ │ ├── machinetype_service.go │ │ ├── service.go │ │ ├── types.go │ │ └── users.go │ ├── machinetype.go │ ├── machinetype_service.go │ └── service.go └── tooling │ ├── config.go │ ├── config_test.go │ ├── docker.go │ ├── ec2meta.go │ └── ec2ssm.go ├── static └── .gitignore ├── templates └── .gitignore └── ui ├── .gitignore ├── babel.config.js ├── package-lock.json ├── package.json ├── public ├── favicon.ico └── index.html ├── src ├── App.vue ├── components │ └── LoginDialog.vue ├── main.js ├── mixins │ └── features.js ├── plugins │ ├── account │ │ ├── Account.vue │ │ └── index.js │ ├── axios.js │ ├── fakeAccount │ │ ├── Account.vue │ │ └── index.js │ └── vuetify.js ├── router.js └── views │ ├── MachineType.vue │ ├── MachineTypes.vue │ └── Welcome.vue └── vue.config.js /.bumpversion.cfg: -------------------------------------------------------------------------------- 1 | [bumpversion] 2 | current_version = 0.2.11 3 | commit = True 4 | tag = True 5 | 6 | [bumpversion:file:cmd/boltdb/main.go] 7 | search = fmt.Println(`v{current_version}`) 8 | replace = fmt.Println(`v{new_version}`) 9 | 10 | [bumpversion:file:cmd/dynamodb/main.go] 11 | search = fmt.Println(`v{current_version}`) 12 | replace = fmt.Println(`v{new_version}`) 13 | 14 | [bumpversion:file:cmd/autologin/main.go] 15 | search = fmt.Println(`v{current_version}`) 16 | replace = fmt.Println(`v{new_version}`) 17 | 18 | [bumpversion:file:ui/package.json] 19 | search = "version": "{current_version}" 20 | replace = "version": "{new_version}" 21 | 22 | [bumpversion:file:Autologin.md] 23 | search = metahub-autologin:v{current_version} 24 | replace = metahub-autologin:v{new_version} 25 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | Dockerfile 2 | .git 3 | ui/node_modules -------------------------------------------------------------------------------- /.gcloudignore: -------------------------------------------------------------------------------- 1 | .gcloudignore 2 | # If you would like to upload your .git directory, .gitignore file or 3 | # files from your .gitignore file, remove the corresponding line below: 4 | .git 5 | .gitignore 6 | #!include:.gitignore 7 | ui/ -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | # You may wish to alter this file to override the set of languages analyzed, 5 | # or to provide custom queries or build logic. 6 | # 7 | # ******** NOTE ******** 8 | # We have attempted to detect the languages in your repository. Please check 9 | # the `language` matrix defined below to confirm you have the correct set of 10 | # supported CodeQL languages. 11 | # 12 | name: "CodeQL" 13 | 14 | on: 15 | push: 16 | branches: [ main ] 17 | pull_request: 18 | # The branches below must be a subset of the branches above 19 | branches: [ main ] 20 | schedule: 21 | - cron: '24 10 * * 2' 22 | 23 | jobs: 24 | analyze: 25 | name: Analyze 26 | runs-on: ubuntu-latest 27 | 28 | strategy: 29 | fail-fast: false 30 | matrix: 31 | language: [ 'go', 'javascript' ] 32 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] 33 | # Learn more: 34 | # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed 35 | 36 | steps: 37 | - name: Checkout repository 38 | uses: actions/checkout@v2 39 | 40 | # Initializes the CodeQL tools for scanning. 41 | - name: Initialize CodeQL 42 | uses: github/codeql-action/init@v1 43 | with: 44 | languages: ${{ matrix.language }} 45 | # If you wish to specify custom queries, you can do so here or in a config file. 46 | # By default, queries listed here will override any specified in a config file. 47 | # Prefix the list here with "+" to use these queries and those in the config file. 48 | # queries: ./path/to/local/query, your-org/your-repo/queries@main 49 | 50 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 51 | # If this step fails, then you should remove it and run the build manually (see below) 52 | - name: Autobuild 53 | uses: github/codeql-action/autobuild@v1 54 | 55 | # ℹ️ Command-line programs to run using the OS shell. 56 | # 📚 https://git.io/JvXDl 57 | 58 | # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines 59 | # and modify them (or add more) to build your code if your project 60 | # uses a compiled language 61 | 62 | #- run: | 63 | # make bootstrap 64 | # make release 65 | 66 | - name: Perform CodeQL Analysis 67 | uses: github/codeql-action/analyze@v1 68 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, build with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | .theia/ 14 | debug 15 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "name": "Launch", 9 | "type": "go", 10 | "request": "launch", 11 | "mode": "auto", 12 | "program": "${fileDirname}", 13 | "env": {}, 14 | "args": [] 15 | }, 16 | { 17 | "name": "Debug", 18 | "type": "go", 19 | "request": "launch", 20 | "mode": "debug", 21 | "program": "${workspaceFolder}/main.go", 22 | "env": {}, 23 | "args": [] 24 | } 25 | ] 26 | } -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "go.inferGopath": false 3 | } -------------------------------------------------------------------------------- /Autologin.md: -------------------------------------------------------------------------------- 1 | ## Autologin 2 | 3 | ```bash 4 | docker pull -q public.ecr.aws/a4y4t9s0/metahub-autologin:v0.2.11 5 | export MH_USER=$(docker run --rm --network=host public.ecr.aws/a4y4t9s0/metahub-autologin:v0.2.11 -get-user) 6 | echo "> MH_USER: ${MH_USER}" 7 | export MH_PASS=$(docker run --rm --network=host public.ecr.aws/a4y4t9s0/metahub-autologin:v0.2.11 -get-pass -aws-region=eu-west-1) 8 | echo "> MH_PASS: ${MH_PASS}" 9 | echo ${MH_PASS} |docker login --username ${MH_USER} --password-stdin mh.qnib.org 10 | ``` 11 | 12 | ## Autologin on AWS instance 13 | 14 | First we pull the latest image. 15 | 16 | ```bash 17 | $ docker pull -q public.ecr.aws/a4y4t9s0/metahub-autologin:v0.2.11 18 | ``` 19 | 20 | As we are running on an `c5.9xlarge` instance with hyperthreading enabled, the username generated is `metahub--c59xl-ht`. 21 | 22 | ```bash 23 | $ docker run -ti --rm --network=host public.ecr.aws/a4y4t9s0/metahub-autologin:v0.2.11 -get-user 24 | metahub-c59xl-ht 25 | ``` 26 | 27 | To login to metahub, we need to present the docker socket. As the login information are local to the user, we'll map in the `.docker` directory to persist the login. 28 | The `AWS_REGION` defines the parameter store to fetch the password from `/metahub/password`. 29 | 30 | ```bash 31 | $ docker run -e AWS_REGION=eu-west-1 -ti --rm --network=host -v ${HOME}/.docker:/root/.docker \ 32 | -v /var/run/docker.sock:/var/run/docker.sock public.ecr.aws/a4y4t9s0/metahub-autologin:v0.2.11 33 | 2021/01/05 16:30:06 Use docker login via client 34 | 2021/01/05 16:30:08 Login Succeeded 35 | ``` 36 | 37 | From now on we'll be able to fetch images from metahub. 38 | 39 | ```bash 40 | $ cat .docker/config.json 41 | { 42 | "auths": { 43 | "mh.qnib.org": { 44 | "auth": "bWV0YWh1Yi1jNTl4bC1odDpncm9tYWNzLXNhcnVzMTIz" 45 | } 46 | }, 47 | "HttpHeaders": { 48 | "User-Agent": "Docker-Client/19.03.12 (linux)" 49 | } 50 | } 51 | $ docker pull -q mh.qnib.org/qnib/metahub-demo:2021-01-04.2 52 | mh.qnib.org/qnib/metahub-demo:2021-01-04.2 53 | $ docker run -ti --rm mh.qnib.org/qnib/metahub-demo:2021-01-04.2 54 | >> This image is optimized for 'c5.9xl' with hyperthreading turned 'on' 55 | $ 56 | ``` 57 | 58 | Instead of autogenerate the login, the type can also be enforced. 59 | 60 | ```bash 61 | $ docker run -ti --rm --network=host public.ecr.aws/a4y4t9s0/metahub-autologin:v0.2.11 -get-user -type c524xl 62 | metahub-c524xl 63 | $ docker run -e AWS_REGION=eu-west-1 -ti --rm --network=host -v ${HOME}/.docker:/root/.docker \ 64 | -v /var/run/docker.sock:/var/run/docker.sock public.ecr.aws/a4y4t9s0/metahub-autologin:v0.2.11 -docker-login -type=c524xl 65 | $ docker rmi mh.qnib.org/qnib/metahub-demo:2021-01-04.2 66 | $ docker pull -q mh.qnib.org/qnib/metahub-demo:2021-01-04.2 67 | mh.qnib.org/qnib/metahub-demo:2021-01-04.2 68 | $ docker run -ti --rm mh.qnib.org/qnib/metahub-demo:2021-01-04.2 69 | >> This image is optimized for 'c5.24xl' with hyperthreading turned 'off' 70 | ``` -------------------------------------------------------------------------------- /Dockerfile.autologin: -------------------------------------------------------------------------------- 1 | FROM golang:1.12-alpine AS go 2 | RUN apk add git 3 | WORKDIR /go/metahub 4 | COPY ./go.mod . 5 | COPY ./go.sum . 6 | RUN go mod download 7 | COPY ./cmd ./cmd 8 | COPY ./pkg ./pkg 9 | WORKDIR /go/metahub/cmd/autologin 10 | ENV CGO_ENABLED=0 GOOS=linux 11 | RUN go build -a -ldflags '-extldflags "-static"' . 12 | 13 | 14 | # Go binary serves the ui web content 15 | FROM alpine:3.12.3 16 | RUN apk add --no-cache --update docker util-linux 17 | COPY --from=go /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ 18 | COPY --from=go /go/metahub/cmd/autologin/autologin /usr/bin/autologin 19 | ENTRYPOINT ["/usr/bin/autologin"] 20 | CMD ["-docker-login"] -------------------------------------------------------------------------------- /Dockerfile.boltdb: -------------------------------------------------------------------------------- 1 | ARG MH_CONFIG=bench.yaml 2 | FROM node:11.4.0 AS ui 3 | RUN npm install -g npm@7.3.0 4 | WORKDIR /go/metahub 5 | COPY ./ui/package* ./ui/ 6 | RUN cd ui && npm install 7 | COPY ./ui ./ui 8 | COPY ./static ./static 9 | COPY ./templates ./templates 10 | WORKDIR /go/metahub/ui/ 11 | RUN npm run build 12 | 13 | FROM golang:1.12 AS go 14 | WORKDIR /go/metahub 15 | COPY ./go.mod . 16 | COPY ./go.sum . 17 | RUN go mod download 18 | COPY ./cmd ./cmd 19 | COPY ./pkg ./pkg 20 | WORKDIR /go/metahub/cmd/boltdb 21 | # static build 22 | ENV CGO_ENABLED=0 GOOS=linux 23 | RUN go build -a -ldflags '-extldflags "-static"' . 24 | EXPOSE 8080 25 | 26 | FROM golang:1.12 AS boltq 27 | WORKDIR /go 28 | RUN git clone https://github.com/schmichael/boltq.git \ 29 | && cd boltq \ 30 | && go get -d \ 31 | && go build 32 | 33 | # Go binary serves the ui web content 34 | FROM ubuntu:18.04 35 | ENV PORT=80 36 | RUN apt update \ 37 | && apt install -y vim 38 | ARG MH_CONFIG=${MH_CONFIG} 39 | RUN echo "metahub -config /data/metahub.yaml" > /root/.bash_history 40 | COPY --from=go /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ 41 | COPY --from=boltq /go/boltq/boltq /usr/bin/ 42 | COPY --from=ui /go/metahub/static /srv/html/static 43 | COPY --from=ui /go/metahub/templates/gen/index.html /srv/html/ 44 | COPY --from=go /go/metahub/cmd/boltdb/boltdb /usr/bin/metahub 45 | COPY ./misc/config/${MH_CONFIG} /data/metahub.yaml 46 | VOLUME /data/ 47 | WORKDIR /data/ 48 | CMD ["/usr/bin/metahub", "-config", "/data/metahub.yaml"] 49 | -------------------------------------------------------------------------------- /Dockerfile.dynamodb: -------------------------------------------------------------------------------- 1 | FROM node:11.4.0 AS ui 2 | RUN npm install -g npm@7.3.0 3 | WORKDIR /go/metahub 4 | COPY ./ui/package* ./ui/ 5 | RUN cd ui && npm install 6 | COPY ./ui ./ui 7 | COPY ./static ./static 8 | COPY ./templates ./templates 9 | WORKDIR /go/metahub/ui/ 10 | RUN npm run build 11 | 12 | FROM golang:1.12 AS go 13 | WORKDIR /go/metahub 14 | COPY ./go.mod . 15 | COPY ./go.sum . 16 | RUN go mod download 17 | COPY ./cmd ./cmd 18 | COPY ./pkg ./pkg 19 | WORKDIR /go/metahub/cmd/dynamodb 20 | # static build 21 | ENV CGO_ENABLED=0 GOOS=linux 22 | RUN go build -a -ldflags '-extldflags "-static"' . 23 | EXPOSE 8080 24 | 25 | # Go binary serves the ui web content 26 | FROM ubuntu:18.04 27 | ENV PORT=80 28 | COPY --from=go /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ 29 | COPY --from=ui /go/metahub/static /srv/html/static 30 | COPY --from=ui /go/metahub/templates/gen/index.html /srv/html/ 31 | COPY --from=go /go/metahub/cmd/dynamodb/dynamodb /usr/bin/metahub 32 | VOLUME /data/ 33 | WORKDIR /data/ 34 | CMD ["/usr/bin/metahub"] 35 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | # This repository represents the initial version of MetaHub from 2019 3 | 4 | **Please check out the current version with much more features here**: [metahub-registry.org](https://metahub-registry.org/) 5 | 6 | 7 | ### Old README version 8 | 9 | Announcement: [qnib.org](http://www.qnib.org/2019/06/12/metahub/) 10 | 11 | The MetaHub project is meta-data registry, which serves images filtered via login so that a machine gets the image that 12 | fits the specifics of the host the image is going to run on. 13 | 14 | That could be picking an image that not only fits the CPU Architecture (x86-64, ppcle, arm) but is optimized for the microarchitecture 15 | of the host (Broadwell, Skylake, ...). And it does not stop there - any host specific attribute can be use: 16 | Accelerators, network, configuration or the full depth of gcc options. 17 | 18 | ![](/misc/pics/metahub-proxy.png) 19 | 20 | ### Example 21 | 22 | A machine that logs in with the user `qnib-type1` will get a Broadwell image, `qnib-type2` serves a Skylake optimized image. 23 | 24 | ``` 25 | $ docker login -u qnib-type1 -p qnib-type1 metahub.qnib.org 26 | $ docker run metahub.qnib.org/qnib/bench && docker run -ti --rm metahub.qnib.org/qnib/bench 27 | Unable to find image 'metahub.qnib.org/qnib/bench:latest' locally 28 | latest: Pulling from qnib/bench 29 | Status: Downloaded newer image for metahub.qnib.org/qnib/bench:latest 30 | >> This container is optimised for: cpu:broadwell 31 | $ docker inspect metahub.qnib.org/qnib/bench |jq '.[].RepoDigests[1] 32 | "metahub.qnib.org/qnib/bench@sha256:f972f05f4ff0c5df22c1f4fc339b14068b8cee96d0525f4fb13dbea84a900c89" 33 | $ docker login -u qnib-type2 -p qnib-type2 metahub.qnib.org 34 | Login Succeeded 35 | $ docker run metahub.qnib.org/qnib/bench && docker run -ti --rm metahub.qnib.org/qnib/bench 36 | Unable to find image 'metahub.qnib.org/qnib/bench:latest' locally 37 | latest: Pulling from qnib/bench 38 | Status: Downloaded newer image for metahub.qnib.org/qnib/bench:latest 39 | >> This container is optimised for: cpu:skylake 40 | $ docker inspect metahub.qnib.org/qnib/bench |jq '.[].RepoDigests[1] 41 | "metahub.qnib.org/qnib/bench@sha256:c62049e0707d3461b0a5b69d0ebe322f16a17b4439b5d9844e24cf97d40faa64" 42 | ``` 43 | 44 | It does that by filtering the manifest list and only present the image that fits the demanded features. 45 | 46 | ``` 47 | $cat manifest.yml 48 | image: docker.io/qnib/bench 49 | manifests: 50 | - 51 | image: docker.io/qnib/plain-featuretest:cpu-skylake 52 | platform: 53 | architecture: amd64 54 | os: linux 55 | features: 56 | - cpu:skylake 57 | - 58 | image: docker.io/qnib/plain-featuretest:cpu-broadwell 59 | platform: 60 | architecture: amd64 61 | os: linux 62 | features: 63 | - cpu:broadwell 64 | ``` 65 | 66 | ![](/misc/pics/metahub-overview.png) 67 | 68 | -------------------------------------------------------------------------------- /app.yaml: -------------------------------------------------------------------------------- 1 | runtime: go111 2 | main: ./cmd/gae 3 | default_expiration: "1m" 4 | 5 | handlers: 6 | - url: /static 7 | static_dir: static 8 | - url: /$ 9 | static_files: templates/gen/index.html 10 | upload: templates/gen/index.html 11 | secure: always 12 | 13 | env_variables: 14 | DATASTORE_PROJECT_ID: metahub 15 | -------------------------------------------------------------------------------- /cloudbuild.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | #- name: "ubuntu" 3 | # args: ["sed", "-i", "-e", "s/__CACHE_NAME__/$BUILD_ID/", "client/worker.js"] 4 | - name: "gcr.io/cloud-builders/npm" 5 | dir: "ui" 6 | args: ["install"] 7 | - name: "gcr.io/cloud-builders/npm" 8 | dir: "ui" 9 | args: ["run", "build"] 10 | - name: "gcr.io/cloud-builders/gcloud" 11 | args: ["app", "deploy", "app.yaml", "index.yaml", "--version=1"] 12 | timeout: "1600s" -------------------------------------------------------------------------------- /cmd/autologin/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "os" 7 | 8 | "log" 9 | 10 | "github.com/qnib/metahub/pkg/tooling" 11 | ) 12 | 13 | var ( 14 | version = flag.Bool("version", false, "print version") 15 | username = flag.String("user", "metahub", "The username to login (default: metahub)") 16 | typename = flag.String("type", "", "Define the machine type, will be generated based on instance info otherwise") 17 | region = flag.String("aws-region", "", "AWS REGION") 18 | regname = flag.String("registry", "mh.qnib.org", "Metahub registry name") 19 | getPw = flag.Bool("get-pass", false, "fetch password from SSM") 20 | getUser = flag.Bool("get-user", false, "generate metahub login-user") 21 | dockerLogin = flag.Bool("docker-login", false, "Use the docker-client to login directly (e.g. /var/run/docker.sock)") 22 | ) 23 | 24 | func main() { 25 | flag.Parse() 26 | if *version { 27 | fmt.Println(`v0.2.11`) 28 | os.Exit(0) 29 | } 30 | awsRegion := "" 31 | switch { 32 | case *region == "" && os.Getenv("AWS_REGION") != "": 33 | awsRegion = os.Getenv("AWS_REGION") 34 | case *region != "" && os.Getenv("AWS_REGION") != "": 35 | log.Printf("--aws-region is set while AWS_REGION is also present; we'll go with the CLI argument") 36 | awsRegion = *region 37 | case *region == "" && os.Getenv("AWS_REGION") == "" && !*dockerLogin: 38 | log.Printf("--aws-region and $AWS_REGION are both empty") 39 | default: 40 | awsRegion = *region 41 | } 42 | if !*getPw && !*getUser && !*dockerLogin { 43 | fmt.Println("Use: -get-pass, -docker-login or -get-user") 44 | os.Exit(0) 45 | } 46 | if *dockerLogin { 47 | md, err := tooling.GetMetaData() 48 | if err != nil { 49 | panic(err) 50 | } 51 | uname := fmt.Sprintf("%s-%s", *username, md.GetMetahubTypename(*typename)) 52 | passwd, err := tooling.GetSSMPassword(awsRegion, "/metahub/password") 53 | if err != nil { 54 | panic(err) 55 | } 56 | err = tooling.DockerLogin(string(passwd), uname, *regname) 57 | if err != nil { 58 | log.Print(err.Error()) 59 | os.Exit(1) 60 | } 61 | os.Exit(0) 62 | } 63 | if *getUser { 64 | md, err := tooling.GetMetaData() 65 | if err != nil { 66 | panic(err) 67 | } 68 | fmt.Printf("%s-%s\n", *username, md.GetMetahubTypename(*typename)) 69 | os.Exit(0) 70 | } 71 | if *getPw { 72 | passwd, err := tooling.GetSSMPassword(awsRegion, "/metahub/password") 73 | if err != nil { 74 | panic(err) 75 | } 76 | fmt.Printf("%s", passwd) 77 | os.Exit(0) 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /cmd/boltdb/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "fmt" 7 | "log" 8 | "net/http" 9 | "os" 10 | 11 | "github.com/qnib/metahub/pkg/daemon" 12 | registry "github.com/qnib/metahub/pkg/registry/http/client" 13 | "github.com/qnib/metahub/pkg/storage/boltdb" 14 | 15 | "github.com/qnib/metahub/cmd" 16 | ) 17 | 18 | var ( 19 | version = flag.Bool("version", false, "print version") 20 | config = flag.String("config", "", "Config holding the initial machine types and users") 21 | genHash = flag.String("genhash", "", "Create password hash and exit") 22 | ) 23 | 24 | // Log intercepts each requests and writes it out 25 | func Log(handler http.Handler) http.Handler { 26 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 27 | log.Printf("%s %s %s", r.RemoteAddr, r.Method, r.URL) 28 | handler.ServeHTTP(w, r) 29 | }) 30 | } 31 | 32 | func main() { 33 | flag.Parse() 34 | if *version { 35 | fmt.Println(`v0.2.11`) 36 | os.Exit(0) 37 | } 38 | port := os.Getenv("PORT") 39 | if port == "" { 40 | port = "8080" 41 | } 42 | 43 | storageService := boltdb.NewService(*config) 44 | if *genHash != "" { 45 | mtScv, _ := storageService.MachineTypeService(context.Background()) 46 | fmt.Println(mtScv.GenPasswordHash(*genHash)) 47 | os.Exit(0) 48 | } 49 | 50 | registryService := registry.NewService() 51 | daemonService := daemon.NewService(storageService, registryService) 52 | 53 | router := cmd.RegisterRoutes(daemonService) 54 | 55 | log.Fatal(http.ListenAndServe(fmt.Sprintf(":%s", port), Log(router))) 56 | } 57 | -------------------------------------------------------------------------------- /cmd/dynamodb/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "log" 7 | "net/http" 8 | "os" 9 | 10 | "github.com/qnib/metahub/pkg/daemon" 11 | 12 | "github.com/qnib/metahub/cmd" 13 | registry "github.com/qnib/metahub/pkg/registry/http/client" 14 | "github.com/qnib/metahub/pkg/storage/dynamodb" 15 | ) 16 | 17 | var ( 18 | version = flag.Bool("version", false, "print version") 19 | ) 20 | 21 | // Log intercepts each requests and writes it out 22 | func Log(handler http.Handler) http.Handler { 23 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 24 | log.Printf("%s %s %s", r.RemoteAddr, r.Method, r.URL) 25 | handler.ServeHTTP(w, r) 26 | }) 27 | } 28 | 29 | func main() { 30 | flag.Parse() 31 | if *version { 32 | fmt.Println(`v0.2.11`) 33 | os.Exit(0) 34 | } 35 | port := os.Getenv("PORT") 36 | if port == "" { 37 | port = "8080" 38 | } 39 | 40 | storageService := dynamodb.NewService() 41 | registryService := registry.NewService() 42 | daemonService := daemon.NewService(storageService, registryService) 43 | 44 | router := cmd.RegisterRoutes(daemonService) 45 | 46 | log.Fatal(http.ListenAndServe(fmt.Sprintf(":%s", port), Log(router))) 47 | } 48 | -------------------------------------------------------------------------------- /cmd/gae/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "net/http" 7 | "os" 8 | 9 | "github.com/qnib/metahub/cmd" 10 | "github.com/qnib/metahub/pkg/daemon" 11 | registry "github.com/qnib/metahub/pkg/registry/http/client" 12 | "github.com/qnib/metahub/pkg/storage/clouddatastore" 13 | ) 14 | 15 | func main() { 16 | port := os.Getenv("PORT") 17 | if port == "" { 18 | port = "8080" 19 | } 20 | 21 | storageService := clouddatastore.NewService() 22 | registryService := registry.NewService() 23 | daemonService := daemon.NewService(storageService, registryService) 24 | 25 | cmd.RegisterRoutes(daemonService) 26 | 27 | log.Fatal(http.ListenAndServe(fmt.Sprintf(":%s", port), nil)) 28 | } 29 | -------------------------------------------------------------------------------- /cmd/routes.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "net/http" 5 | 6 | "github.com/qnib/metahub/pkg/daemon" 7 | 8 | "github.com/qnib/metahub/pkg/accounts" 9 | "github.com/qnib/metahub/pkg/machinetypes" 10 | "github.com/qnib/metahub/pkg/registry/http/server" 11 | ) 12 | 13 | func RegisterRoutes(service daemon.Service) *http.ServeMux { 14 | router := http.NewServeMux() 15 | RegisterStaticRoutes(service, router) 16 | RegisterAPIRoutes(service, router) 17 | return router 18 | } 19 | 20 | // RegisterRoutes registers handlers/routers 21 | func RegisterAPIRoutes(service daemon.Service, router *http.ServeMux) { 22 | handleRouter(service, router, "/v2", server.NewRouter) 23 | handleRouter(service, router, "/auth", accounts.NewRouter) 24 | handleRouter(service, router, "/machinetypes", machinetypes.NewRouter) 25 | } 26 | 27 | func RegisterStaticRoutes(service daemon.Service, router *http.ServeMux) { 28 | router.Handle("/", http.FileServer(http.Dir("/srv/html/"))) 29 | } 30 | 31 | func handleRouter(service daemon.Service, router *http.ServeMux, prefix string, h func(service daemon.Service, prefix string) http.Handler) { 32 | router.Handle(prefix+"/", h(service, prefix)) 33 | } 34 | -------------------------------------------------------------------------------- /dev.bat: -------------------------------------------------------------------------------- 1 | dev_appserver.py --support_datastore_emulator=true --datastore_emulator_port=9090 --clear_datastore=yes --automatic_restart=1 --require_indexes=yes --env_var=DATASTORE_EMULATOR_HOST=localhost:9090 . -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/qnib/metahub 2 | 3 | go 1.12 4 | 5 | require ( 6 | cloud.google.com/go v0.37.4 7 | github.com/aws/aws-sdk-go v1.36.19 8 | github.com/boltdb/bolt v1.3.1 9 | github.com/docker/distribution v2.7.0+incompatible 10 | github.com/docker/go-metrics v0.0.0-20181218153428-b84716841b82 // indirect 11 | github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect 12 | github.com/google/go-github v17.0.0+incompatible 13 | github.com/google/go-querystring v1.0.0 // indirect 14 | github.com/gorilla/context v1.1.1 15 | github.com/gorilla/mux v1.7.1 16 | github.com/klauspost/cpuid v1.3.1 17 | github.com/opencontainers/go-digest v1.0.0-rc1 18 | github.com/opencontainers/image-spec v1.0.1 // indirect 19 | github.com/prometheus/common v0.2.0 20 | github.com/sirupsen/logrus v1.4.1 // indirect 21 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 22 | golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421 23 | gopkg.in/yaml.v2 v2.2.8 24 | ) 25 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 2 | cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 3 | cloud.google.com/go v0.37.4 h1:glPeL3BQJsbF6aIIYfZizMwc5LTYz250bDMjttbBGAU= 4 | cloud.google.com/go v0.37.4/go.mod h1:NHPJ89PdicEuT9hdPXMROBD91xc5uRDxsMtSB16k7hw= 5 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 6 | github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= 7 | github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= 8 | github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU= 9 | github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= 10 | github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY= 11 | github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= 12 | github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= 13 | github.com/aws/aws-sdk-go v1.36.19 h1:zbJZKkxeDiYxUYFjymjWxPye+qa1G2gRVyhIzZrB9zA= 14 | github.com/aws/aws-sdk-go v1.36.19/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= 15 | github.com/aws/aws-sdk-go-v2 v0.31.0 h1:TNTDsz+Xq80nYzZPUFS4a2Oyjz9jKHKcTuNAXtcW8b8= 16 | github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= 17 | github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= 18 | github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4= 19 | github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= 20 | github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= 21 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 22 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 23 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 24 | github.com/docker/distribution v2.7.0+incompatible h1:neUDAlf3wX6Ml4HdqTrbcOHXtfRN0TFIwt6YFL7N9RU= 25 | github.com/docker/distribution v2.7.0+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= 26 | github.com/docker/go-metrics v0.0.0-20181218153428-b84716841b82 h1:X0fj836zx99zFu83v/M79DuBn84IL/Syx1SY6Y5ZEMA= 27 | github.com/docker/go-metrics v0.0.0-20181218153428-b84716841b82/go.mod h1:/u0gXw0Gay3ceNrsHubL3BtdOL2fHf93USgMTe0W5dI= 28 | github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 h1:UhxFibDNY/bfvqU5CAUmr9zpesgbU6SWc8/B4mflAE4= 29 | github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= 30 | github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= 31 | github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= 32 | github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= 33 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= 34 | github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= 35 | github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= 36 | github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= 37 | github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= 38 | github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= 39 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= 40 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= 41 | github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 42 | github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 43 | github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= 44 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 45 | github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 46 | github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= 47 | github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= 48 | github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= 49 | github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY= 50 | github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= 51 | github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= 52 | github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= 53 | github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= 54 | github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= 55 | github.com/googleapis/gax-go/v2 v2.0.4 h1:hU4mGcQI4DaAYW+IbTun+2qEZVFxK0ySjQLTbS0VQKc= 56 | github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= 57 | github.com/gorilla/context v1.1.1 h1:AWwleXJkX/nhcU9bZSnZoi3h/qGYqQAGhq6zZe/aQW8= 58 | github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= 59 | github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= 60 | github.com/gorilla/mux v1.7.1 h1:Dw4jY2nghMMRsh1ol8dv1axHkDwMQK2DHerMNJsIpJU= 61 | github.com/gorilla/mux v1.7.1/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= 62 | github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo= 63 | github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= 64 | github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= 65 | github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= 66 | github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= 67 | github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= 68 | github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= 69 | github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= 70 | github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= 71 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 72 | github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s= 73 | github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4= 74 | github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= 75 | github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 76 | github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= 77 | github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= 78 | github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= 79 | github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= 80 | github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 81 | github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 82 | github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= 83 | github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ= 84 | github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= 85 | github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI= 86 | github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= 87 | github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= 88 | github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= 89 | github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 90 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 91 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 92 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 93 | github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= 94 | github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829 h1:D+CiwcpGTW6pL6bv6KI3KbyEyCKyS+1JWS2h8PNDnGA= 95 | github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= 96 | github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8= 97 | github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= 98 | github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f h1:BVwpUVJDADN2ufcGik7W992pyps0wZ888b/y9GXcLTU= 99 | github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= 100 | github.com/prometheus/common v0.2.0 h1:kUZDBDTdBVBYBj5Tmh2NZLlF60mfjA27rM34b+cVwNU= 101 | github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= 102 | github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= 103 | github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1 h1:/K3IL0Z1quvmJ7X0A1AwNEK7CRkVK3YwfOU/QAL4WGg= 104 | github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= 105 | github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= 106 | github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= 107 | github.com/sirupsen/logrus v1.4.1 h1:GL2rEmy6nsikmW0r8opw9JIRScdMF5hA8cOYLH7In1k= 108 | github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= 109 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 110 | github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 111 | github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= 112 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 113 | go.opencensus.io v0.20.1 h1:pMEjRZ1M4ebWGikflH7nQpV6+Zr88KBMA2XJD3sbijw= 114 | go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= 115 | golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 116 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 117 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= 118 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 119 | golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 120 | golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 121 | golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= 122 | golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 123 | golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 124 | golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 125 | golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 126 | golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 127 | golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 128 | golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 129 | golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 130 | golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628= 131 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 132 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 133 | golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME= 134 | golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= 135 | golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= 136 | golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421 h1:Wo7BWFiOk0QRFMLYMqJGFMd9CgUAcGx7V+qEg/h5IBI= 137 | golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= 138 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 139 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 140 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 141 | golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6 h1:bjcUS9ztw9kFmmIxJInhon/0Is3p+EHBKNgquIzo1OI= 142 | golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 143 | golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 144 | golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 145 | golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 146 | golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 147 | golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 148 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU= 149 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 150 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 151 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA= 152 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 153 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 154 | golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2 h1:z99zHgr7hKfrUcX/KsoJk5FJfjTceCKIp96+biqP4To= 155 | golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 156 | golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= 157 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 158 | golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 159 | golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 160 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 161 | golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 162 | golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= 163 | golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 164 | google.golang.org/api v0.3.1 h1:oJra/lMfmtm13/rgY/8i3MzjFWYXvQIAKjQ3HqofMk8= 165 | google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= 166 | google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= 167 | google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= 168 | google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 169 | google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= 170 | google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 171 | google.golang.org/genproto v0.0.0-20190404172233-64821d5d2107 h1:xtNn7qFlagY2mQNFHMSRPjT2RkOV4OXM7P5TVy9xATo= 172 | google.golang.org/genproto v0.0.0-20190404172233-64821d5d2107/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 173 | google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= 174 | google.golang.org/grpc v1.19.0 h1:cfg4PD8YEdSFnm7qLV4++93WcmhH2nIUhMjhdCvl3j8= 175 | google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= 176 | gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= 177 | gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= 178 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 179 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 180 | gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= 181 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= 182 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 183 | gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= 184 | gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 185 | honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 186 | honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 187 | honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 188 | -------------------------------------------------------------------------------- /index.yaml: -------------------------------------------------------------------------------- 1 | indexes: 2 | # AUTOGENERATED 3 | 4 | # This index.yaml is automatically updated whenever the Cloud Datastore 5 | # emulator detects that a new type of query is run. If you want to manage the 6 | # index.yaml file manually, remove the "# AUTOGENERATED" marker line above. 7 | # If you want to manage some indexes manually, move them above the marker line. 8 | 9 | -------------------------------------------------------------------------------- /misc/config/bench.yaml: -------------------------------------------------------------------------------- 1 | # Meant to be used with qnib/bench 2 | user: user 3 | password: hash 4 | types: 5 | - name: skylake 6 | features: 7 | - cpu:skylake 8 | - name: broadwell 9 | features: 10 | - cpu:broadwell 11 | - name: nv_broadwell 12 | features: 13 | - cpu:broadwell 14 | - nvcap:5.2 -------------------------------------------------------------------------------- /misc/config/metahub-demo.yaml: -------------------------------------------------------------------------------- 1 | # Meant to be used with qnib/metahub-demo:2021-01-06.2 2 | user: metahub 3 | # passwd: gromacs-sarus123 4 | password: "$2a$14$gb2QajF6cioofNOFzEVjguCVbxVbn1Pez/nJ69IVd98iHi3s/3Mc." 5 | types: 6 | - name: c59xl 7 | features: 8 | - instType:c5 9 | - instSize:9xl 10 | - instHT:off 11 | - name: c59xl-ht 12 | features: 13 | - instType:c5 14 | - instSize:9xl 15 | - instHT:on 16 | - name: c512xl 17 | features: 18 | - instType:c5 19 | - instSize:12xl 20 | - instHT:off 21 | - name: c512xl-ht 22 | features: 23 | - instType:c5 24 | - instSize:12xl 25 | - instHT:on 26 | - name: c516xl 27 | features: 28 | - instType:c5 29 | - instSize:16xl 30 | - instHT:off 31 | - name: c516xl-ht 32 | features: 33 | - instType:c5 34 | - instSize:16xl 35 | - instHT:on 36 | - name: c518xl 37 | features: 38 | - instType:c5 39 | - instSize:18xl 40 | - instHT:off 41 | - name: c518xl-ht 42 | features: 43 | - instType:c5 44 | - instSize:18xl 45 | - instHT:on 46 | - name: c524xl 47 | features: 48 | - instType:c5 49 | - instSize:24xl 50 | - instHT:off 51 | - name: c524xl-ht 52 | features: 53 | - instType:c5 54 | - instSize:24xl 55 | - instHT:on 56 | # c5n 57 | - name: c5n9xl 58 | features: 59 | - instType:c5n 60 | - instSize:9xl 61 | - instHT:off 62 | - name: c5n9xl-ht 63 | features: 64 | - instType:c5n 65 | - instSize:9xl 66 | - instHT:on 67 | - name: c5n18xl 68 | features: 69 | - instType:c5n 70 | - instSize:18xl 71 | - instHT:off 72 | - name: c5n18xl-ht 73 | features: 74 | - instType:c5n 75 | - instSize:18xl 76 | - instHT:on 77 | # c5a 78 | - name: c5a16xl 79 | features: 80 | - instType:c5a 81 | - instSize:16xl 82 | - instHT:off 83 | - name: c5a16xl-ht 84 | features: 85 | - instType:c5a 86 | - instSize:16xl 87 | - instHT:on 88 | - name: c5a24xl 89 | features: 90 | - instType:c5a 91 | - instSize:24xl 92 | - instHT:off 93 | - name: c5a24xl-ht 94 | features: 95 | - instType:c5a 96 | - instSize:24xl 97 | - instHT:on -------------------------------------------------------------------------------- /misc/docker/caddy/Caddyfile: -------------------------------------------------------------------------------- 1 | metahub.qnib.org { 2 | reverse_proxy metahub:8080 3 | } 4 | 5 | metahub-static.qnib.org { 6 | reverse_proxy metahub-static:8080 7 | } -------------------------------------------------------------------------------- /misc/docker/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3.7" 2 | services: 3 | caddy: 4 | image: caddy:2.0.0-alpine 5 | ports: 6 | - 80:80 7 | - 443:443 8 | configs: 9 | - source: caddyfile 10 | target: /etc/caddy/Caddyfile 11 | metahub: 12 | image: public.ecr.aws/a4y4t9s0/metahub:latest 13 | environment: 14 | - PORT=8080 15 | - PASSWORD_IGNORE=true 16 | metahub-static: 17 | image: public.ecr.aws/a4y4t9s0/metahub:latest 18 | environment: 19 | - PORT=8080 20 | - PASSWORD_IGNORE=true 21 | - STATIC_MACHINES=true 22 | 23 | configs: 24 | caddyfile: 25 | file: ./caddy/Caddyfile -------------------------------------------------------------------------------- /misc/metahub-demo/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine 2 | 3 | ARG INST 4 | ARG SIZE 5 | ARG HT 6 | ENV INST=${INST} 7 | ENV SIZE=${SIZE} 8 | ENV HT=${HT} 9 | ENV SLEEP_TIME=0 10 | COPY bin/entry.sh /usr/local/bin/ 11 | CMD ["/usr/local/bin/entry.sh"] 12 | -------------------------------------------------------------------------------- /misc/metahub-demo/bin/entry.sh: -------------------------------------------------------------------------------- 1 | #!/bin/ash 2 | 3 | if [[ "X${INST}" == "X" ]];then 4 | echo ">> This image is not optimized." 5 | else 6 | echo ">> This image is optimized for '${INST}.${SIZE}' with hyperthreading turned '${HT}'" 7 | fi 8 | if [[ "X${SLEEP_TIME}" != "X0" ]];then 9 | echo -n ">> Sleeping for '${SLEEP_TIME}'" 10 | sleep ${SLEEP_TIME} 11 | echo " DONE!" 12 | fi -------------------------------------------------------------------------------- /misc/metahub-demo/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | tmp_dir=$(mktemp -d -t build-XXXXXXXXXX) 4 | echo ">> TEMPDIR: $tmp_dir" 5 | yq d yml/template.yml manifests >${tmp_dir}/mh-test.yml 6 | yq w -i ${tmp_dir}/mh-test.yml image docker.io/qnib/metahub-demo:$(date +%F).2 7 | INSTS="c5 c5n c5a c6g" 8 | SIZES="9xl 12xl 16xl 18xl 24xl" 9 | HTS="on off" 10 | INSTS="c5" 11 | SIZES="18xl" 12 | HTS="on off" 13 | for i in $(echo $INSTS);do 14 | for s in $(echo ${SIZES});do 15 | for ht in ${HTS};do 16 | IMG_TAG=${i}_${s}_${ht} 17 | IMG_NAME=docker.io/qnib/metahub-single:${IMG_TAG} 18 | echo ">>> Build ${IMG_NAME}" 19 | docker build --quiet -t ${IMG_NAME} \ 20 | --build-arg=INST=${i} \ 21 | --build-arg=SIZE=${s} \ 22 | --build-arg=HT=${ht} \ 23 | . 24 | docker push ${IMG_NAME} >/dev/null 25 | yq w -i ${tmp_dir}/mh-test.yml 'manifests[+].image' ${IMG_NAME} 26 | echo ">> ADD {image: 'IMG_NAME', platform: {architecture: 'amd64',os: 'linux', features: ['instance:$i','size:$s','ht:$ht']}}" 27 | yq r -j ${tmp_dir}/mh-test.yml \ 28 | |jq -r --arg IMGNAME "${IMG_NAME}" --arg INSTR "instance:$i" --arg SIZESTR "size:$s" --arg HTSTR "ht:$ht" \ 29 | '.manifests |= map(select(.image==$IMGNAME)+= {platform: {architecture: "amd64",os: "linux", features: [$INSTR,$SIZESTR, $HTSTR]}})' \ 30 | |yq r --prettyPrint - >${tmp_dir}/mh-test_${IMG_TAG}.yml 31 | sleep 0.2 32 | done 33 | done 34 | done 35 | echo ">> Merge temp files" 36 | yq merge ${tmp_dir}/mh-test_* > ${tmp_dir}/mh-test.yml 37 | echo ">> Push manifest: ${tmp_dir}/mh-test.yml" 38 | manifest-tool push from-spec ${tmp_dir}/mh-test.yml 39 | #rm -rf ${tmp_dir} 40 | -------------------------------------------------------------------------------- /misc/metahub-demo/yml/template.yml: -------------------------------------------------------------------------------- 1 | image: docker.io/qnib/metahub-demo 2 | manifests: 3 | -------------------------------------------------------------------------------- /misc/pics/metahub-overview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/qnib/metahub/f57aa1d9fb8b05564efa233a663c499bb6d46b12/misc/pics/metahub-overview.png -------------------------------------------------------------------------------- /misc/pics/metahub-proxy.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/qnib/metahub/f57aa1d9fb8b05564efa233a663c499bb6d46b12/misc/pics/metahub-proxy.png -------------------------------------------------------------------------------- /pkg/accounts/auth_middleware.go: -------------------------------------------------------------------------------- 1 | package accounts 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "net/http" 7 | "strings" 8 | 9 | "github.com/qnib/metahub/pkg/daemon" 10 | 11 | "github.com/gorilla/context" 12 | ) 13 | 14 | type oauthError string 15 | 16 | var ( 17 | invalidRequest oauthError = "invalid_request" 18 | invalidToken oauthError = "invalid_token" 19 | insufficientScope oauthError = "insufficient_scope" 20 | ) 21 | 22 | // AuthMiddleware checks user 23 | func AuthMiddleware(service daemon.Service) func(http.Handler) http.Handler { 24 | storage := service.Storage() 25 | 26 | realm := "MetaHub" 27 | 28 | return func(next http.Handler) http.Handler { 29 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 30 | ctx := r.Context() 31 | 32 | authorizationHeader := r.Header.Get("authorization") 33 | if authorizationHeader == "" { 34 | unauthorized(w, realm, invalidRequest, "missing authorization header") 35 | return 36 | } 37 | bearerToken := strings.Split(authorizationHeader, " ") 38 | if len(bearerToken) != 2 { 39 | unauthorized(w, realm, invalidRequest, "invalid authorization header: %q", authorizationHeader) 40 | return 41 | } 42 | accessTokenString := bearerToken[1] 43 | 44 | accessTokenService, err := storage.AccessTokenService(ctx) 45 | if err != nil { 46 | log.Printf("failed to create AccessTokenService: %v", err) 47 | w.WriteHeader(http.StatusInternalServerError) 48 | return 49 | } 50 | 51 | at, err := accessTokenService.Get(accessTokenString) 52 | if err != nil { 53 | log.Printf("error getting access token: %v", err) 54 | w.WriteHeader(http.StatusInternalServerError) 55 | return 56 | } 57 | if at == nil { 58 | log.Printf("unknown access token") 59 | unauthorized(w, realm, invalidToken, "unknown access token") 60 | return 61 | } 62 | 63 | accountService, err := storage.AccountService(ctx) 64 | if err != nil { 65 | log.Printf("error getting account service: %v", err) 66 | w.WriteHeader(http.StatusInternalServerError) 67 | return 68 | } 69 | 70 | account, err := accountService.Get(at.AccountName) 71 | if err != nil { 72 | log.Printf("error getting account: %v", err) 73 | w.WriteHeader(http.StatusInternalServerError) 74 | return 75 | } 76 | if account == nil { 77 | log.Printf("unknown account") 78 | unauthorized(w, realm, invalidToken, "unknown account") 79 | return 80 | } 81 | 82 | context.Set(r, "accountName", at.AccountName) 83 | context.Set(r, "account", account) 84 | 85 | next.ServeHTTP(w, r) 86 | }) 87 | } 88 | } 89 | 90 | func unauthorized(w http.ResponseWriter, realm string, err oauthError, format string, v ...interface{}) { 91 | description := fmt.Sprintf(format, v...) 92 | w.Header().Add("WWW-Authenticate", fmt.Sprintf("Bearer realm=%q,error=%q,error_description=%q", realm, err, description)) 93 | w.WriteHeader(http.StatusUnauthorized) 94 | } 95 | -------------------------------------------------------------------------------- /pkg/accounts/github_handler.go: -------------------------------------------------------------------------------- 1 | package accounts 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "log" 7 | "net/http" 8 | 9 | "github.com/qnib/metahub/pkg/daemon" 10 | 11 | "github.com/google/go-github/github" 12 | "golang.org/x/oauth2" 13 | githuboauth "golang.org/x/oauth2/github" 14 | ) 15 | 16 | func getGithubHandler(service daemon.Service) http.Handler { 17 | 18 | config := oauth2.Config{ 19 | ClientID: "65d9c15a3eb4e0afdd01", 20 | ClientSecret: "7d9c3f1e3ee87a912f2748a8161621c64e724509", 21 | Scopes: []string{"user:email"}, 22 | Endpoint: githuboauth.Endpoint, 23 | } 24 | 25 | return getBaseHandler(service, "github", config, func(ctx context.Context, token *oauth2.Token) (*user, error) { 26 | oauthClient := config.Client(ctx, token) 27 | client := github.NewClient(oauthClient) 28 | u, _, err := client.Users.Get(ctx, "") 29 | if err != nil { 30 | return nil, fmt.Errorf("error client.Users.Get(): %v", err) 31 | } 32 | log.Printf("Logged in as GitHub user: %v", *u) 33 | 34 | return &user{ 35 | ID: fmt.Sprintf("%d", u.GetID()), 36 | Name: u.GetEmail(), 37 | }, nil 38 | }) 39 | 40 | } 41 | -------------------------------------------------------------------------------- /pkg/accounts/google_handler.go: -------------------------------------------------------------------------------- 1 | package accounts 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "io/ioutil" 8 | "net/http" 9 | "strings" 10 | 11 | "github.com/qnib/metahub/pkg/daemon" 12 | 13 | "golang.org/x/oauth2" 14 | googleAuth "golang.org/x/oauth2/google" 15 | ) 16 | 17 | func getGoogleHandler(service daemon.Service) http.Handler { 18 | 19 | config := oauth2.Config{ 20 | ClientID: "936040293434-i3m9p4km8it5np2bs253a7rvedchofs6.apps.googleusercontent.com", 21 | ClientSecret: "E0wI5Bb0KbZ1__DgztogDu1O", 22 | Scopes: []string{"profile", "email", "openid"}, 23 | Endpoint: googleAuth.Endpoint, 24 | } 25 | 26 | return getBaseHandler(service, "google", config, func(ctx context.Context, token *oauth2.Token) (*user, error) { 27 | response, err := http.Get("https://www.googleapis.com/oauth2/v2/userinfo?access_token=" + token.AccessToken) 28 | if err != nil { 29 | return nil, fmt.Errorf("failed getting user info: %v", err) 30 | } 31 | defer response.Body.Close() 32 | contents, err := ioutil.ReadAll(response.Body) 33 | if err != nil { 34 | return nil, fmt.Errorf("failed read response: %v", err) 35 | } 36 | 37 | userInfoReader := strings.NewReader(string(contents)) 38 | userInfoDecoder := json.NewDecoder(userInfoReader) 39 | var userInfo struct { 40 | ID string `json:"id"` 41 | Email string `json:"email"` 42 | Name string `json:"name"` 43 | } 44 | if err := userInfoDecoder.Decode(&userInfo); err != nil { 45 | return nil, fmt.Errorf("error decoding user info: %v", err) 46 | } 47 | 48 | return &user{ 49 | ID: userInfo.ID, 50 | Name: userInfo.Name, 51 | }, nil 52 | }) 53 | 54 | } 55 | -------------------------------------------------------------------------------- /pkg/accounts/handler_base.go: -------------------------------------------------------------------------------- 1 | package accounts 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "log" 7 | "net/http" 8 | 9 | "github.com/qnib/metahub/pkg/daemon" 10 | "github.com/qnib/metahub/pkg/storage" 11 | 12 | "golang.org/x/oauth2" 13 | ) 14 | 15 | type user struct { 16 | ID string 17 | Name string 18 | } 19 | 20 | type getUserFunc func(ctx context.Context, token *oauth2.Token) (*user, error) 21 | 22 | func getBaseHandler(service daemon.Service, provider string, config oauth2.Config, getUser getUserFunc) http.Handler { 23 | storageService := service.Storage() 24 | 25 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 26 | ctx := r.Context() 27 | 28 | decoder := json.NewDecoder(r.Body) 29 | var body struct { 30 | Code string `json:"code"` 31 | ClientID string `json:"clientId"` 32 | RedirectURI string `json:"redirectUri"` 33 | } 34 | err := decoder.Decode(&body) 35 | if err != nil { 36 | log.Printf("error decoding code: %v", err) 37 | w.WriteHeader(http.StatusInternalServerError) 38 | return 39 | } 40 | 41 | config.RedirectURL = body.RedirectURI 42 | 43 | if body.ClientID != config.ClientID { 44 | log.Printf("invalid client ID %q", body.ClientID) 45 | w.WriteHeader(http.StatusInternalServerError) 46 | return 47 | } 48 | 49 | token, err := config.Exchange(ctx, body.Code) 50 | if err != nil { 51 | log.Printf("Exchange failed: %v", err) 52 | w.WriteHeader(http.StatusInternalServerError) 53 | return 54 | } 55 | if !token.Valid() { 56 | log.Printf("token invalid") 57 | w.WriteHeader(http.StatusInternalServerError) 58 | return 59 | } 60 | 61 | user, err := getUser(ctx, token) 62 | if err != nil { 63 | log.Printf("failed getting user id: %v", err) 64 | w.WriteHeader(http.StatusInternalServerError) 65 | return 66 | } 67 | 68 | accountService, err := storageService.AccountService(ctx) 69 | if err != nil { 70 | log.Printf("failed to create AccountService: %v", err) 71 | w.WriteHeader(http.StatusInternalServerError) 72 | return 73 | } 74 | 75 | accountName := storage.GetAccountName(provider, user.ID) 76 | if err := accountService.Upsert(accountName, storage.Account{ 77 | DisplayName: user.Name, 78 | }); err != nil { 79 | log.Printf("error updating account: %v", err) 80 | w.WriteHeader(http.StatusInternalServerError) 81 | return 82 | } 83 | 84 | accessTokenService, err := storageService.AccessTokenService(ctx) 85 | if err != nil { 86 | log.Printf("failed to create AccessTokenService: %v", err) 87 | w.WriteHeader(http.StatusInternalServerError) 88 | return 89 | } 90 | 91 | if err := accessTokenService.Put(token.AccessToken, storage.AccessToken{ 92 | AccountName: accountName, 93 | Expiry: token.Expiry, 94 | }); err != nil { 95 | log.Printf("error storing access token: %v", err) 96 | w.WriteHeader(http.StatusInternalServerError) 97 | return 98 | } 99 | 100 | jwt, err := tokenToJSON(token) 101 | if err != nil { 102 | log.Printf("error creating JWT: %v", err) 103 | w.WriteHeader(http.StatusInternalServerError) 104 | return 105 | } 106 | 107 | //log.Print(jwt) 108 | 109 | w.Write([]byte(jwt)) 110 | }) 111 | } 112 | -------------------------------------------------------------------------------- /pkg/accounts/identity_handler.go: -------------------------------------------------------------------------------- 1 | package accounts 2 | 3 | import ( 4 | "encoding/json" 5 | "log" 6 | "net/http" 7 | 8 | "github.com/qnib/metahub/pkg/daemon" 9 | "github.com/qnib/metahub/pkg/storage" 10 | 11 | "github.com/gorilla/context" 12 | ) 13 | 14 | func getIdentityHandler(service daemon.Service) http.Handler { 15 | authMiddleware := AuthMiddleware(service) 16 | 17 | return authMiddleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 18 | //ctx := r.Context() 19 | accountName := context.Get(r, "accountName").(string) 20 | account := context.Get(r, "account").(*storage.Account) 21 | 22 | info := struct { 23 | AccountName string `json:"accountName"` 24 | DisplayName string `json:"displayName"` 25 | }{ 26 | AccountName: accountName, 27 | DisplayName: account.DisplayName, 28 | } 29 | 30 | d, err := json.Marshal(info) 31 | if err != nil { 32 | log.Printf("error marshaling response data: %v", err) 33 | w.WriteHeader(http.StatusInternalServerError) 34 | return 35 | } 36 | w.Write(d) 37 | })) 38 | } 39 | -------------------------------------------------------------------------------- /pkg/accounts/jwt_encoding.go: -------------------------------------------------------------------------------- 1 | package accounts 2 | 3 | import ( 4 | "encoding/json" 5 | 6 | "golang.org/x/oauth2" 7 | ) 8 | 9 | func tokenToJSON(token *oauth2.Token) (string, error) { 10 | d, err := json.Marshal(token) 11 | if err != nil { 12 | return "", err 13 | } 14 | return string(d), nil 15 | } 16 | 17 | func tokenFromJSON(jsonStr string) (*oauth2.Token, error) { 18 | var token oauth2.Token 19 | if err := json.Unmarshal([]byte(jsonStr), &token); err != nil { 20 | return nil, err 21 | } 22 | return &token, nil 23 | } 24 | -------------------------------------------------------------------------------- /pkg/accounts/router.go: -------------------------------------------------------------------------------- 1 | package accounts 2 | 3 | import ( 4 | "net/http" 5 | 6 | "github.com/qnib/metahub/pkg/daemon" 7 | 8 | "github.com/gorilla/mux" 9 | ) 10 | 11 | // NewRouter returns a router for auth callbacks 12 | func NewRouter(service daemon.Service, pathPrefix string) http.Handler { 13 | router := mux.NewRouter() 14 | router.Handle(pathPrefix+"/github", getGithubHandler(service)).Methods("POST") 15 | router.Handle(pathPrefix+"/google", getGoogleHandler(service)).Methods("POST") 16 | router.Handle(pathPrefix+"/identity", getIdentityHandler(service)).Methods("GET") 17 | return router 18 | } 19 | -------------------------------------------------------------------------------- /pkg/daemon/service.go: -------------------------------------------------------------------------------- 1 | package daemon 2 | 3 | import ( 4 | "github.com/qnib/metahub/pkg/registry" 5 | "github.com/qnib/metahub/pkg/storage" 6 | ) 7 | 8 | // Service is the main interface to other execution services 9 | type Service interface { 10 | Storage() storage.Service 11 | Registry() registry.Service 12 | } 13 | 14 | // NewService returns a new environment 15 | func NewService(s storage.Service, r registry.Service) Service { 16 | return service{ 17 | s: s, 18 | r: r, 19 | } 20 | } 21 | 22 | type service struct { 23 | s storage.Service 24 | r registry.Service 25 | } 26 | 27 | func (s service) Storage() storage.Service { 28 | return s.s 29 | } 30 | 31 | func (s service) Registry() registry.Service { 32 | return s.r 33 | } 34 | -------------------------------------------------------------------------------- /pkg/machinetypes/add_handler.go: -------------------------------------------------------------------------------- 1 | package machinetypes 2 | 3 | import ( 4 | "encoding/json" 5 | "log" 6 | "net/http" 7 | 8 | "github.com/qnib/metahub/pkg/daemon" 9 | 10 | "github.com/qnib/metahub/pkg/storage" 11 | 12 | "github.com/gorilla/context" 13 | ) 14 | 15 | func getAddHandler(service daemon.Service) http.Handler { 16 | storageService := service.Storage() 17 | 18 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 19 | ctx := r.Context() 20 | 21 | accountName := context.Get(r, "accountName").(string) 22 | 23 | decoder := json.NewDecoder(r.Body) 24 | var mt storage.MachineType 25 | err := decoder.Decode(&mt) 26 | if err != nil { 27 | log.Printf("error decoding request data: %v", err) 28 | w.WriteHeader(http.StatusInternalServerError) 29 | return 30 | } 31 | 32 | machineTypeService, err := storageService.MachineTypeService(ctx) 33 | if err != nil { 34 | log.Printf("failed to create MachineTypeService: %v", err) 35 | w.WriteHeader(http.StatusInternalServerError) 36 | return 37 | } 38 | 39 | if err := machineTypeService.Add(accountName, &mt); err != nil { 40 | log.Printf("failed adding machine type: %v", err) 41 | w.WriteHeader(http.StatusInternalServerError) 42 | return 43 | } 44 | 45 | d, err := json.Marshal(mt) 46 | if err != nil { 47 | log.Printf("error marshaling response data: %v", err) 48 | w.WriteHeader(http.StatusInternalServerError) 49 | return 50 | } 51 | w.Header().Set("content-type", "application/json") 52 | w.Write(d) 53 | }) 54 | } 55 | -------------------------------------------------------------------------------- /pkg/machinetypes/auth_middleware.go: -------------------------------------------------------------------------------- 1 | package machinetypes 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "net/http" 7 | 8 | "github.com/qnib/metahub/pkg/daemon" 9 | "github.com/qnib/metahub/pkg/registry/filter" 10 | 11 | "github.com/gorilla/context" 12 | ) 13 | 14 | // AuthMiddleware checks machine type credentials 15 | func AuthMiddleware(service daemon.Service) func(http.Handler) http.Handler { 16 | storage := service.Storage() 17 | 18 | return func(next http.Handler) http.Handler { 19 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 20 | ctx := r.Context() 21 | 22 | username, password, ok := r.BasicAuth() 23 | if !ok { 24 | unauthorized(w) 25 | return 26 | } 27 | 28 | mediaTypeService, err := storage.MachineTypeService(ctx) 29 | if err != nil { 30 | log.Printf("failed to create MachineTypeService: %v", err) 31 | w.WriteHeader(http.StatusInternalServerError) 32 | return 33 | } 34 | 35 | mt, err := mediaTypeService.GetByUsername(username) 36 | if err != nil { 37 | log.Printf("error getting machine type: %v", err) 38 | w.WriteHeader(http.StatusInternalServerError) 39 | return 40 | } 41 | if mt == nil { 42 | log.Printf("unknown login (machine type is empty)") 43 | unauthorized(w) 44 | return 45 | } 46 | if !mediaTypeService.CheckPassword(password, mt.Password) { 47 | log.Printf("invalid password") 48 | unauthorized(w) 49 | return 50 | } 51 | 52 | backendRegistryService := service.Registry() 53 | filterRegistryService := filter.NewService(backendRegistryService, *mt) 54 | context.Set(r, "registryService", filterRegistryService) 55 | 56 | next.ServeHTTP(w, r) 57 | }) 58 | } 59 | } 60 | 61 | func unauthorized(w http.ResponseWriter) { 62 | w.Header().Add("WWW-Authenticate", fmt.Sprintf(`Basic realm="%s"`, "MetaHub")) 63 | w.WriteHeader(http.StatusUnauthorized) 64 | } 65 | -------------------------------------------------------------------------------- /pkg/machinetypes/delete_handler.go: -------------------------------------------------------------------------------- 1 | package machinetypes 2 | 3 | import ( 4 | "encoding/json" 5 | "log" 6 | "net/http" 7 | 8 | "github.com/qnib/metahub/pkg/daemon" 9 | 10 | "github.com/gorilla/context" 11 | ) 12 | 13 | func getDeleteHandler(service daemon.Service) http.Handler { 14 | storageService := service.Storage() 15 | 16 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 17 | ctx := r.Context() 18 | 19 | accountName := context.Get(r, "accountName").(string) 20 | 21 | decoder := json.NewDecoder(r.Body) 22 | var requestParams struct { 23 | ID int64 `json:"id"` 24 | } 25 | err := decoder.Decode(&requestParams) 26 | if err != nil { 27 | log.Printf("error decoding request data: %v", err) 28 | w.WriteHeader(http.StatusInternalServerError) 29 | return 30 | } 31 | 32 | machineTypeService, err := storageService.MachineTypeService(ctx) 33 | if err != nil { 34 | log.Printf("failed to create MachineTypeService: %v", err) 35 | w.WriteHeader(http.StatusInternalServerError) 36 | return 37 | } 38 | 39 | if err := machineTypeService.Delete(accountName, requestParams.ID); err != nil { 40 | log.Printf("failed deleting machine type: %v", err) 41 | w.WriteHeader(http.StatusInternalServerError) 42 | return 43 | } 44 | }) 45 | } 46 | -------------------------------------------------------------------------------- /pkg/machinetypes/get_handler.go: -------------------------------------------------------------------------------- 1 | package machinetypes 2 | 3 | import ( 4 | "encoding/json" 5 | "log" 6 | "net/http" 7 | "strconv" 8 | 9 | "github.com/qnib/metahub/pkg/daemon" 10 | 11 | "github.com/gorilla/context" 12 | ) 13 | 14 | func getGetHandler(service daemon.Service) http.Handler { 15 | storageService := service.Storage() 16 | 17 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 18 | ctx := r.Context() 19 | 20 | accountName := context.Get(r, "accountName").(string) 21 | 22 | machineTypeIDString := r.URL.Query().Get("id") 23 | machineTypeID, err := strconv.ParseInt(machineTypeIDString, 10, 64) 24 | if err != nil { 25 | log.Printf("failed parsing machine type id: %v", err) 26 | w.WriteHeader(http.StatusInternalServerError) 27 | return 28 | } 29 | 30 | machineTypeService, err := storageService.MachineTypeService(ctx) 31 | if err != nil { 32 | log.Printf("failed to create MachineTypeService: %v", err) 33 | w.WriteHeader(http.StatusInternalServerError) 34 | return 35 | } 36 | 37 | mt, err := machineTypeService.GetByID(accountName, machineTypeID) 38 | if err != nil { 39 | log.Printf("failed getting machine type: %v", err) 40 | w.WriteHeader(http.StatusInternalServerError) 41 | return 42 | } 43 | 44 | d, err := json.Marshal(mt) 45 | if err != nil { 46 | log.Printf("error marshaling response data: %v", err) 47 | w.WriteHeader(http.StatusInternalServerError) 48 | return 49 | } 50 | w.Write(d) 51 | }) 52 | } 53 | -------------------------------------------------------------------------------- /pkg/machinetypes/list_handler.go: -------------------------------------------------------------------------------- 1 | package machinetypes 2 | 3 | import ( 4 | "encoding/json" 5 | "log" 6 | "net/http" 7 | 8 | "github.com/qnib/metahub/pkg/daemon" 9 | "github.com/qnib/metahub/pkg/storage" 10 | 11 | "github.com/gorilla/context" 12 | ) 13 | 14 | func getListHandler(service daemon.Service) http.Handler { 15 | storageService := service.Storage() 16 | 17 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 18 | ctx := r.Context() 19 | 20 | accountName := context.Get(r, "accountName").(string) 21 | 22 | machineTypeService, err := storageService.MachineTypeService(ctx) 23 | if err != nil { 24 | log.Printf("failed to create MachineTypeService: %v", err) 25 | w.WriteHeader(http.StatusInternalServerError) 26 | return 27 | } 28 | 29 | machineTypes, err := machineTypeService.List(accountName) 30 | if err != nil { 31 | log.Printf("error querying machine types: %v", err) 32 | w.WriteHeader(http.StatusInternalServerError) 33 | return 34 | } 35 | log.Printf("Get %d machineTypes back", len(machineTypes)) 36 | response := struct { 37 | MachineTypes []storage.MachineType `json:"machineTypes,omitempty"` 38 | }{ 39 | MachineTypes: machineTypes, 40 | } 41 | 42 | d, err := json.Marshal(response) 43 | if err != nil { 44 | log.Printf("error marshaling response data: %v", err) 45 | w.WriteHeader(http.StatusInternalServerError) 46 | return 47 | } 48 | w.Header().Set("content-type", "application/json") 49 | w.Write(d) 50 | }) 51 | } 52 | -------------------------------------------------------------------------------- /pkg/machinetypes/password_generator.go: -------------------------------------------------------------------------------- 1 | package machinetypes 2 | 3 | import ( 4 | "crypto/rand" 5 | "fmt" 6 | "io" 7 | ) 8 | 9 | // Adapted from https://elithrar.github.io/article/generating-secure-random-numbers-crypto-rand/ 10 | 11 | func init() { 12 | assertAvailablePRNG() 13 | } 14 | 15 | func assertAvailablePRNG() { 16 | // Assert that a cryptographically secure PRNG is available. 17 | // Panic otherwise. 18 | buf := make([]byte, 1) 19 | 20 | _, err := io.ReadFull(rand.Reader, buf) 21 | if err != nil { 22 | panic(fmt.Sprintf("crypto/rand is unavailable: Read() failed with %#v", err)) 23 | } 24 | } 25 | 26 | // generateRandomBytes returns securely generated random bytes. 27 | // It will return an error if the system's secure random 28 | // number generator fails to function correctly, in which 29 | // case the caller should not continue. 30 | func generateRandomBytes(n int) ([]byte, error) { 31 | b := make([]byte, n) 32 | _, err := rand.Read(b) 33 | // Note that err == nil only if we read len(b) bytes. 34 | if err != nil { 35 | return nil, err 36 | } 37 | 38 | return b, nil 39 | } 40 | 41 | // generateRandomString returns a securely generated random string. 42 | // It will return an error if the system's secure random 43 | // number generator fails to function correctly, in which 44 | // case the caller should not continue. 45 | func generateRandomString(n int) (string, error) { 46 | const letters = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz-" 47 | bytes, err := generateRandomBytes(n) 48 | if err != nil { 49 | return "", err 50 | } 51 | for i, b := range bytes { 52 | bytes[i] = letters[b%byte(len(letters))] 53 | } 54 | return string(bytes), nil 55 | } 56 | 57 | func generateLoginPassword() (string, error) { 58 | return generateRandomString(32) 59 | } 60 | -------------------------------------------------------------------------------- /pkg/machinetypes/router.go: -------------------------------------------------------------------------------- 1 | package machinetypes 2 | 3 | import ( 4 | "net/http" 5 | 6 | auth "github.com/qnib/metahub/pkg/accounts" 7 | "github.com/qnib/metahub/pkg/daemon" 8 | 9 | "github.com/gorilla/mux" 10 | ) 11 | 12 | // NewRouter returns a router for feature sets 13 | func NewRouter(service daemon.Service, pathPrefix string) http.Handler { 14 | router := mux.NewRouter() 15 | router.Use(auth.AuthMiddleware(service)) 16 | router.Handle(pathPrefix+"/get", getGetHandler(service)).Methods("GET") 17 | router.Handle(pathPrefix+"/add", getAddHandler(service)).Methods("POST") 18 | router.Handle(pathPrefix+"/list", getListHandler(service)).Methods("GET") 19 | router.Handle(pathPrefix+"/delete", getDeleteHandler(service)).Methods("POST") 20 | router.Handle(pathPrefix+"/update", getUpdateHandler(service)).Methods("POST") 21 | return router 22 | } 23 | -------------------------------------------------------------------------------- /pkg/machinetypes/update_handler.go: -------------------------------------------------------------------------------- 1 | package machinetypes 2 | 3 | import ( 4 | "encoding/json" 5 | "log" 6 | "net/http" 7 | 8 | "github.com/qnib/metahub/pkg/daemon" 9 | "github.com/qnib/metahub/pkg/storage" 10 | 11 | "github.com/gorilla/context" 12 | ) 13 | 14 | func getUpdateHandler(service daemon.Service) http.Handler { 15 | storageService := service.Storage() 16 | 17 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 18 | ctx := r.Context() 19 | 20 | accountName := context.Get(r, "accountName").(string) 21 | 22 | decoder := json.NewDecoder(r.Body) 23 | var mt storage.MachineType 24 | err := decoder.Decode(&mt) 25 | if err != nil { 26 | log.Printf("error decoding request data: %v", err) 27 | w.WriteHeader(http.StatusInternalServerError) 28 | return 29 | } 30 | 31 | machineTypeService, err := storageService.MachineTypeService(ctx) 32 | if err != nil { 33 | log.Printf("failed to create MachineTypeService: %v", err) 34 | w.WriteHeader(http.StatusInternalServerError) 35 | return 36 | } 37 | 38 | if err := machineTypeService.Update(accountName, mt); err != nil { 39 | log.Printf("failed updating machine type: %v", err) 40 | w.WriteHeader(http.StatusInternalServerError) 41 | return 42 | } 43 | 44 | d, err := json.Marshal(mt) 45 | if err != nil { 46 | log.Printf("error marshaling response data: %v", err) 47 | w.WriteHeader(http.StatusInternalServerError) 48 | return 49 | } 50 | w.Header().Set("content-type", "application/json") 51 | w.Write(d) 52 | }) 53 | } 54 | -------------------------------------------------------------------------------- /pkg/registry/blob.go: -------------------------------------------------------------------------------- 1 | package registry 2 | 3 | // Blob describes a 4 | type Blob struct { 5 | // MediaType describe the type of the content. All text based formats are 6 | // encoded as utf-8. 7 | MediaType string 8 | 9 | // Size in bytes of content. 10 | Size int64 11 | } 12 | -------------------------------------------------------------------------------- /pkg/registry/filter/service.go: -------------------------------------------------------------------------------- 1 | package filter 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | "log" 8 | 9 | "github.com/qnib/metahub/pkg/registry" 10 | "github.com/qnib/metahub/pkg/storage" 11 | 12 | "github.com/docker/distribution" 13 | "github.com/opencontainers/go-digest" 14 | 15 | manifestListSchema "github.com/docker/distribution/manifest/manifestlist" 16 | manifestSchema "github.com/docker/distribution/manifest/schema2" 17 | ) 18 | 19 | // https://docs.docker.com/registry/spec/manifest-v2-2/#image-manifest-field-descriptions 20 | // https://docs.docker.com/registry/spec/api/#digest-parameter 21 | 22 | func init() { 23 | _ = manifestListSchema.SchemaVersion 24 | _ = manifestSchema.SchemaVersion 25 | } 26 | 27 | type service struct { 28 | inner registry.Service 29 | machineType storage.MachineType 30 | } 31 | 32 | // NewService returns a new registry service, wraping another registry Service, to filter image manifests 33 | func NewService(inner registry.Service, machineType storage.MachineType) registry.Service { 34 | return &service{ 35 | inner: inner, 36 | machineType: machineType, 37 | } 38 | } 39 | 40 | func (s *service) GetBlob(ctx context.Context, repositoryString string, d digest.Digest) (io.ReadCloser, registry.Blob, error) { 41 | return s.inner.GetBlob(ctx, repositoryString, d) 42 | } 43 | 44 | func (s *service) GetManifest(ctx context.Context, repositoryString string, referenceString string) (registry.Manifest, error) { 45 | log.Println("> In GetManifest()") 46 | m, err := s.inner.GetManifest(ctx, repositoryString, referenceString) 47 | if err != nil { 48 | log.Printf(" >> s.inner.GetManifest() returned without error") 49 | return m, err 50 | } 51 | unmarshaledManifest, _, err := distribution.UnmarshalManifest(m.ContentType, m.Data) 52 | if err != nil { 53 | return m, fmt.Errorf("error unmarshaling manifest: %v", err) 54 | } 55 | manifestList, ok := unmarshaledManifest.(*manifestListSchema.DeserializedManifestList) 56 | if !ok { 57 | log.Println("unmarshaledManifest.(*manifestListSchema.DeserializedManifestList) was not ok") 58 | return m, nil 59 | } 60 | manifestList, err = s.filterManifestsFromList(manifestList) 61 | if err != nil { 62 | return m, fmt.Errorf("error filtering manifest list: %v", err) 63 | } 64 | mediaType, payload, err := manifestList.Payload() 65 | if err != nil { 66 | return m, fmt.Errorf("error getting manifest payload: %v", err) 67 | } 68 | m = registry.Manifest{ 69 | Data: payload, 70 | ContentType: mediaType, 71 | } 72 | return m, nil 73 | } 74 | 75 | func (s *service) filterManifestsFromList(manifestList *manifestListSchema.DeserializedManifestList) (*manifestListSchema.DeserializedManifestList, error) { 76 | machineType := s.machineType 77 | log.Printf("machine type features: %v", machineType.Features) 78 | 79 | machineFeatureSet := make(map[string]struct{}, 0) 80 | for _, f := range machineType.Features { 81 | machineFeatureSet[f] = struct{}{} 82 | } 83 | 84 | filteredManifests := make([]manifestListSchema.ManifestDescriptor, 0) 85 | skipped := 0 86 | for _, m := range manifestList.Manifests { 87 | if len(m.Platform.Features) != len(machineFeatureSet) { 88 | skipped++ 89 | //log.Printf("skipping manifest features %v", m.Platform.Features) 90 | continue 91 | } 92 | featureMismatch := false 93 | for i := 0; i < len(machineFeatureSet); i++ { 94 | platformFeature := m.Platform.Features[i] 95 | if _, ok := machineFeatureSet[platformFeature]; !ok { 96 | featureMismatch = true 97 | break 98 | } 99 | } 100 | if featureMismatch { 101 | skipped++ 102 | log.Printf("skipping manifest features %v", m.Platform.Features) 103 | continue 104 | } 105 | //log.Printf("allow manifest features %v", m.Platform.Features) 106 | filteredManifests = append(filteredManifests, m) 107 | } 108 | log.Printf("skipped %d and left %d manifest features", skipped, len(filteredManifests)) 109 | newManifestList, err := manifestListSchema.FromDescriptors(filteredManifests) 110 | if err != nil { 111 | return nil, fmt.Errorf("error generating new manifest list: %v", err) 112 | } 113 | return newManifestList, nil 114 | } 115 | -------------------------------------------------------------------------------- /pkg/registry/http/client/backend_auth.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "net/http" 5 | "time" 6 | 7 | "github.com/docker/distribution/registry/client/auth" 8 | "github.com/docker/distribution/registry/client/auth/challenge" 9 | "github.com/docker/distribution/registry/client/transport" 10 | ) 11 | 12 | var serverBase = "https://registry-1.docker.io" 13 | 14 | func backendAuthTransport(server, image string) http.RoundTripper { 15 | base := http.DefaultTransport 16 | 17 | modifiers := []transport.RequestModifier{ 18 | transport.NewHeaderRequestModifier(http.Header{ 19 | "User-Agent": []string{"my-client"}, 20 | }), 21 | } 22 | 23 | authTransport := transport.NewTransport(base, modifiers...) 24 | pingClient := &http.Client{ 25 | Transport: authTransport, 26 | Timeout: 5 * time.Second, 27 | } 28 | req, err := http.NewRequest("GET", server+"/v2/", nil) 29 | if err != nil { 30 | panic(err) 31 | } 32 | 33 | challengeManager := challenge.NewSimpleManager() 34 | resp, err := pingClient.Do(req) 35 | if err != nil { 36 | panic(err) 37 | } 38 | defer resp.Body.Close() 39 | if err := challengeManager.AddResponse(resp); err != nil { 40 | panic(err) 41 | } 42 | tokenHandler := auth.NewTokenHandler(base, nil, image, "pull") 43 | modifiers = append(modifiers, auth.NewAuthorizer(challengeManager, tokenHandler, auth.NewBasicHandler(nil))) 44 | 45 | return transport.NewTransport(base, modifiers...) 46 | } 47 | -------------------------------------------------------------------------------- /pkg/registry/http/client/service.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | 8 | "github.com/qnib/metahub/pkg/registry" 9 | 10 | "github.com/docker/distribution" 11 | "github.com/docker/distribution/reference" 12 | registryClient "github.com/docker/distribution/registry/client" 13 | "github.com/opencontainers/go-digest" 14 | 15 | manifestListSchema "github.com/docker/distribution/manifest/manifestlist" 16 | manifestSchema "github.com/docker/distribution/manifest/schema2" 17 | ) 18 | 19 | // https://docs.docker.com/registry/spec/manifest-v2-2/#image-manifest-field-descriptions 20 | 21 | func init() { 22 | _ = manifestListSchema.SchemaVersion 23 | _ = manifestSchema.SchemaVersion 24 | } 25 | 26 | type service struct { 27 | serverBase string 28 | } 29 | 30 | // NewService returns a new HTTP client registry service 31 | func NewService() registry.Service { 32 | return &service{ 33 | serverBase: "https://registry-1.docker.io", 34 | } 35 | } 36 | 37 | func (s *service) newRepositoryClient(repositoryString string) (distribution.Repository, error) { 38 | 39 | //TODO: add "library" segment? 40 | repositoryName, err := reference.WithName(repositoryString) 41 | if err != nil { 42 | return nil, fmt.Errorf("error parsing repository name: %v", err) 43 | } 44 | 45 | // get backend blob service 46 | transportAuth := backendAuthTransport(serverBase, repositoryString) 47 | repositoryClient, err := registryClient.NewRepository(repositoryName, s.serverBase, transportAuth) 48 | if err != nil { 49 | return nil, fmt.Errorf("error creating repository object: %v", err) 50 | } 51 | 52 | return repositoryClient, nil 53 | } 54 | 55 | func (s *service) GetBlob(ctx context.Context, repositoryString string, d digest.Digest) (io.ReadCloser, registry.Blob, error) { 56 | var blob registry.Blob 57 | 58 | // get backend repository blobs service 59 | repositoryClient, err := s.newRepositoryClient(repositoryString) 60 | if err != nil { 61 | return nil, blob, fmt.Errorf("error loading blob stats from backend: %v", err) 62 | } 63 | blobService := repositoryClient.Blobs(ctx) 64 | 65 | // get blob stats 66 | blobStats, err := blobService.Stat(ctx, d) 67 | if err != nil { 68 | return nil, blob, fmt.Errorf("error loading blob stats from backend: %v", err) 69 | } 70 | 71 | blob.Size = blobStats.Size 72 | blob.MediaType = blobStats.MediaType 73 | 74 | // open blob content stream 75 | blobContentReader, err := blobService.Open(ctx, d) 76 | if err != nil { 77 | return nil, blob, fmt.Errorf("error getting blob stream from backend: %v", err) 78 | } 79 | 80 | return blobContentReader, blob, nil 81 | } 82 | 83 | func (s *service) GetManifest(ctx context.Context, repositoryString string, referenceString string) (registry.Manifest, error) { 84 | var m registry.Manifest 85 | 86 | // get image reference 87 | var tag distribution.ManifestServiceOption 88 | var d digest.Digest 89 | { 90 | dgst, err := digest.Parse(referenceString) 91 | if err != nil { 92 | tag = distribution.WithTag(referenceString) 93 | } else { 94 | d = dgst 95 | } 96 | } 97 | 98 | // get backend manifest service 99 | repositoryClient, err := s.newRepositoryClient(repositoryString) 100 | if err != nil { 101 | return m, fmt.Errorf("error loading blob stats from backend: %v", err) 102 | } 103 | manifestService, err := repositoryClient.Manifests(ctx) 104 | if err != nil { 105 | return m, fmt.Errorf("error creating repository object: %v", err) 106 | } 107 | 108 | // call backend manifest 109 | var manifest distribution.Manifest 110 | if tag == nil { 111 | manifest, err = manifestService.Get(ctx, d) 112 | } else { 113 | manifest, err = manifestService.Get(ctx, d, tag) 114 | } 115 | if err != nil { 116 | return m, fmt.Errorf("error getting backend manifest: %v", err) 117 | } 118 | 119 | mediaType, payload, err := manifest.Payload() 120 | if err != nil { 121 | return m, fmt.Errorf("error getting manifest payload: %v", err) 122 | } 123 | m = registry.Manifest{ 124 | Data: payload, 125 | ContentType: mediaType, 126 | } 127 | return m, nil 128 | } 129 | -------------------------------------------------------------------------------- /pkg/registry/http/server/base_handler.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "net/http" 5 | 6 | "github.com/qnib/metahub/pkg/daemon" 7 | ) 8 | 9 | func getBaseHandler(service daemon.Service) http.Handler { 10 | //storageService := env.Storage() 11 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 12 | 13 | }) 14 | } 15 | -------------------------------------------------------------------------------- /pkg/registry/http/server/blobs_handler.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "io" 5 | "log" 6 | "net/http" 7 | "strconv" 8 | 9 | "github.com/qnib/metahub/pkg/daemon" 10 | "github.com/qnib/metahub/pkg/registry" 11 | 12 | "github.com/gorilla/context" 13 | "github.com/gorilla/mux" 14 | digest "github.com/opencontainers/go-digest" 15 | ) 16 | 17 | func getBlobsHandler(service daemon.Service) http.Handler { 18 | //storageService := env.Storage() 19 | 20 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 21 | ctx := r.Context() 22 | vars := mux.Vars(r) 23 | 24 | // get repository and blob digest 25 | repository := getRepository(r) 26 | digest, err := digest.Parse(vars["reference"]) 27 | if err != nil { 28 | log.Printf("error parsing blob reference: %v", err) 29 | w.WriteHeader(http.StatusInternalServerError) 30 | return 31 | } 32 | 33 | // get manifest from registry service 34 | registryService := context.Get(r, "registryService").(registry.Service) 35 | blobReader, blob, err := registryService.GetBlob(ctx, repository, digest) 36 | if err != nil { 37 | log.Printf("error getting blob: %v", err) 38 | w.WriteHeader(http.StatusInternalServerError) 39 | return 40 | } 41 | defer blobReader.Close() 42 | 43 | // set response headers 44 | w.Header().Set("Content-Length", strconv.FormatInt(blob.Size, 10)) 45 | w.Header().Set("Content-Type", blob.MediaType) 46 | w.Header().Set("Docker-Content-Digest", digest.String()) 47 | w.Header().Set("Etag", digest.String()) 48 | 49 | // stream blob content to client 50 | _, err = io.CopyN(w, blobReader, blob.Size) 51 | if err != nil { 52 | log.Printf("error getting blob stream from backend: %v", err) 53 | w.WriteHeader(http.StatusInternalServerError) 54 | return 55 | } 56 | }) 57 | } 58 | -------------------------------------------------------------------------------- /pkg/registry/http/server/manifests_handler.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "net/http" 7 | 8 | "github.com/qnib/metahub/pkg/daemon" 9 | "github.com/qnib/metahub/pkg/registry" 10 | 11 | "github.com/gorilla/context" 12 | 13 | manifestListSchema "github.com/docker/distribution/manifest/manifestlist" 14 | manifestSchema "github.com/docker/distribution/manifest/schema2" 15 | "github.com/gorilla/mux" 16 | digestLib "github.com/opencontainers/go-digest" 17 | ) 18 | 19 | // https://docs.docker.com/registry/spec/api/#digest-parameter 20 | 21 | func init() { 22 | _ = manifestListSchema.SchemaVersion 23 | _ = manifestSchema.SchemaVersion 24 | } 25 | 26 | func getRegistryHandler(service daemon.Service) http.Handler { 27 | 28 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 29 | ctx := r.Context() 30 | vars := mux.Vars(r) 31 | 32 | // get repository and image reference 33 | repository := getRepository(r) 34 | reference := vars["reference"] 35 | 36 | // get manifest from registry service 37 | registryService := context.Get(r, "registryService").(registry.Service) 38 | manifest, err := registryService.GetManifest(ctx, repository, reference) 39 | if err != nil { 40 | log.Printf("error getting manifest: %v", err) 41 | w.WriteHeader(http.StatusInternalServerError) 42 | return 43 | } 44 | 45 | // send response 46 | w.Header().Set("Content-Type", manifest.ContentType) 47 | w.Header().Set("Content-Length", fmt.Sprint(len(manifest.Data))) 48 | digest, _ := digestLib.Parse(reference) 49 | w.Header().Set("Docker-Content-Digest", digest.String()) 50 | w.Header().Set("Etag", fmt.Sprintf(`"%s"`, digest)) 51 | w.Write(manifest.Data) 52 | }) 53 | } 54 | -------------------------------------------------------------------------------- /pkg/registry/http/server/router.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "net/http" 5 | 6 | "github.com/qnib/metahub/pkg/daemon" 7 | "github.com/qnib/metahub/pkg/machinetypes" 8 | 9 | "github.com/gorilla/mux" 10 | ) 11 | 12 | // NewRouter returns a router for the registry API endpoints 13 | func NewRouter(service daemon.Service, pathPrefix string) http.Handler { 14 | router := mux.NewRouter() 15 | router.Use(machinetypes.AuthMiddleware(service)) 16 | router.Handle(pathPrefix+"/{image}/manifests/{reference}", getRegistryHandler(service)).Methods("GET") 17 | router.Handle(pathPrefix+"/{repo}/{image}/manifests/{reference}", getRegistryHandler(service)).Methods("GET") 18 | router.Handle(pathPrefix+"/{image}/blobs/{reference}", getBlobsHandler(service)).Methods("GET") 19 | router.Handle(pathPrefix+"/{repo}/{image}/blobs/{reference}", getBlobsHandler(service)).Methods("GET") 20 | router.Handle(pathPrefix+"/", getBaseHandler(service)).Methods("GET") 21 | return router 22 | } 23 | 24 | func getRepository(r *http.Request) string { 25 | vars := mux.Vars(r) 26 | image := vars["image"] 27 | repo := vars["repo"] 28 | if repo == "" { 29 | repo = "library" 30 | } 31 | name := repo + "/" + image 32 | return name 33 | } 34 | -------------------------------------------------------------------------------- /pkg/registry/manifest.go: -------------------------------------------------------------------------------- 1 | package registry 2 | 3 | // Manifest describes a 4 | type Manifest struct { 5 | Data []byte 6 | ContentType string 7 | } 8 | -------------------------------------------------------------------------------- /pkg/registry/manifest_descriptor.go: -------------------------------------------------------------------------------- 1 | package registry 2 | 3 | // ManifestDescriptor describes a 4 | /*type ManifestDescriptor struct { 5 | Data []byte 6 | PlatformFeatures []string 7 | }*/ 8 | -------------------------------------------------------------------------------- /pkg/registry/manifest_list.go: -------------------------------------------------------------------------------- 1 | package registry 2 | 3 | // ManifestList describes a 4 | /*type ManifestList struct { 5 | Manifests []ManifestDescriptor 6 | ContentType string 7 | }*/ 8 | -------------------------------------------------------------------------------- /pkg/registry/service.go: -------------------------------------------------------------------------------- 1 | package registry 2 | 3 | import ( 4 | "context" 5 | "io" 6 | 7 | digest "github.com/opencontainers/go-digest" 8 | ) 9 | 10 | // Service provides access to a remote registry image repositories 11 | type Service interface { 12 | GetBlob(ctx context.Context, repository string, d digest.Digest) (io.ReadCloser, Blob, error) 13 | GetManifest(ctx context.Context, repositoryString string, referenceString string) (Manifest, error) 14 | } 15 | -------------------------------------------------------------------------------- /pkg/storage/accesstoken.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import "time" 4 | 5 | // AccessToken are used to make API requests on behalf of a user. 6 | // The access token represents the authorization of a specific application to access 7 | // specific parts of a user’s data. 8 | type AccessToken struct { 9 | AccountName string 10 | Expiry time.Time 11 | } 12 | -------------------------------------------------------------------------------- /pkg/storage/accesstoken_service.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | // AccessTokenService provides access to AccessToken objects. 4 | type AccessTokenService interface { 5 | Get(token string) (*AccessToken, error) 6 | Put(token string, at AccessToken) error 7 | } 8 | -------------------------------------------------------------------------------- /pkg/storage/account.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import "fmt" 4 | 5 | // Account represents a user account 6 | type Account struct { 7 | DisplayName string 8 | } 9 | 10 | //GetAccountName returns a account name from a provider name and provider ID 11 | func GetAccountName(provider string, id string) string { 12 | return fmt.Sprintf("%s-%s", provider, id) 13 | } 14 | -------------------------------------------------------------------------------- /pkg/storage/account_service.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | // AccountService provides access to Account objects. 4 | type AccountService interface { 5 | Upsert(name string, a Account) error 6 | Get(name string) (*Account, error) 7 | } 8 | -------------------------------------------------------------------------------- /pkg/storage/boltdb/accesstoken_model.go: -------------------------------------------------------------------------------- 1 | package boltdb 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | var accessTokenEntityKind = "access_token" 8 | 9 | type accessToken struct { 10 | AccountName string `datastore:"account,noindex"` 11 | Expiry time.Time `datastore:"expiry,omitempty"` 12 | } 13 | -------------------------------------------------------------------------------- /pkg/storage/boltdb/accesstoken_service.go: -------------------------------------------------------------------------------- 1 | package boltdb 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | "github.com/qnib/metahub/pkg/storage" 8 | ) 9 | 10 | type accessTokenService struct { 11 | ctx context.Context 12 | } 13 | 14 | func (s *accessTokenService) Get(token string) (*storage.AccessToken, error) { 15 | //TODO: check at.Expiry? 16 | return &storage.AccessToken{ 17 | AccountName: token, 18 | Expiry: time.Time{}, 19 | }, nil 20 | } 21 | 22 | func (s *accessTokenService) Put(token string, at storage.AccessToken) error { 23 | return nil 24 | } 25 | -------------------------------------------------------------------------------- /pkg/storage/boltdb/account_model.go: -------------------------------------------------------------------------------- 1 | package boltdb 2 | 3 | var accountEntityKind = "account" 4 | 5 | type account struct { 6 | DisplayName string `datastore:"name,noindex"` 7 | } 8 | -------------------------------------------------------------------------------- /pkg/storage/boltdb/account_service.go: -------------------------------------------------------------------------------- 1 | package boltdb 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/qnib/metahub/pkg/storage" 7 | ) 8 | 9 | type accountService struct { 10 | ctx context.Context 11 | } 12 | 13 | func (s *accountService) Upsert(name string, a storage.Account) error { 14 | return nil 15 | } 16 | 17 | func (s *accountService) Get(name string) (*storage.Account, error) { 18 | return &storage.Account{ 19 | DisplayName: name, 20 | }, nil 21 | } 22 | -------------------------------------------------------------------------------- /pkg/storage/boltdb/dummies.go: -------------------------------------------------------------------------------- 1 | package boltdb 2 | 3 | import "github.com/qnib/metahub/pkg/storage" 4 | 5 | // Consts for protoype 6 | const ( 7 | user = "qnib" 8 | accountName = user 9 | ) 10 | 11 | // Dummy MachineTypes 12 | var ( 13 | mType1 = storage.MachineType{ 14 | ID: 1, 15 | DisplayName: "type1", 16 | Features: []string{"cpu:broadwell"}, 17 | Login: user + "-type1", 18 | Password: user + "-type1", 19 | } 20 | mType2 = storage.MachineType{ 21 | ID: 2, 22 | DisplayName: "type2", 23 | Features: []string{"cpu:skylake"}, 24 | Login: user + "-type2", 25 | Password: user + "-type2", 26 | } 27 | mType3 = storage.MachineType{ 28 | ID: 3, 29 | DisplayName: "type3", 30 | Features: []string{"cpu:coffelake"}, 31 | Login: user + "-type3", 32 | Password: user + "-type3", 33 | } 34 | mType4 = storage.MachineType{ 35 | ID: 4, 36 | DisplayName: "type4", 37 | Features: []string{"cpu:broadwell", "nvcap:5.2"}, 38 | Login: user + "-type4", 39 | Password: user + "-type4", 40 | } 41 | ) 42 | -------------------------------------------------------------------------------- /pkg/storage/boltdb/machinetype_model.go: -------------------------------------------------------------------------------- 1 | package boltdb 2 | 3 | var machineTypeEntityKind = "MachineType" 4 | 5 | type machineTypeModel struct { 6 | DisplayName string `datastore:"name,noindex"` 7 | Features []string `datastore:"features,noindex"` 8 | Login string `datastore:"login"` 9 | Password string `datastore:"password,noindex"` 10 | } 11 | 12 | -------------------------------------------------------------------------------- /pkg/storage/boltdb/machinetype_service.go: -------------------------------------------------------------------------------- 1 | package boltdb 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "log" 8 | "os" 9 | "strings" 10 | 11 | "github.com/boltdb/bolt" 12 | "github.com/qnib/metahub/pkg/storage" 13 | "github.com/qnib/metahub/pkg/tooling" 14 | "golang.org/x/crypto/bcrypt" 15 | ) 16 | 17 | type machineTypeService struct { 18 | ctx context.Context 19 | ConfigPath string 20 | UserConfig string 21 | } 22 | 23 | func formatLogin(accountName string, login string) { 24 | 25 | } 26 | 27 | // Init runs when a new MachineTypeService is started, overkill! 28 | func (s *machineTypeService) Init() (err error) { 29 | err = s.InitTypes() 30 | if err != nil { 31 | return 32 | } 33 | err = s.InitUsers() 34 | return 35 | } 36 | 37 | func (s *machineTypeService) InitUsers() (err error) { 38 | log.Println("InitUsers()") 39 | result := []string{} 40 | db.View(func(tx *bolt.Tx) error { 41 | b := tx.Bucket([]byte("USERS")) 42 | c := b.Cursor() 43 | for k, _ := c.First(); k != nil; k, _ = c.Next() { 44 | result = append(result, string(k)) 45 | } 46 | return err 47 | }) 48 | if len(result) == 0 { 49 | log.Printf("Fill the bucket with the user found in config file: %s", s.ConfigPath) 50 | cfg, err := tooling.CreateConfigFromFile(s.ConfigPath) 51 | err = db.Update(func(tx *bolt.Tx) error { 52 | b := tx.Bucket([]byte("USERS")) 53 | id, _ := b.NextSequence() 54 | uItem := UserItem{ 55 | ID: int64(id), 56 | Login: cfg.User, 57 | Password: cfg.Password, // hash me? 58 | } 59 | 60 | err = b.Put([]byte(cfg.User), uItem.ToBytes()) 61 | if err != nil { 62 | log.Printf("Error putting user '%s' in: %s", cfg.User, err.Error()) 63 | } 64 | return err 65 | }) 66 | } 67 | return 68 | } 69 | 70 | func (s *machineTypeService) InitTypes() (err error) { 71 | log.Println("InitTypes()") 72 | result := []string{} 73 | db.View(func(tx *bolt.Tx) error { 74 | b := tx.Bucket([]byte("TYPES")) 75 | c := b.Cursor() 76 | for k, _ := c.First(); k != nil; k, _ = c.Next() { 77 | result = append(result, string(k)) 78 | } 79 | return err 80 | }) 81 | if len(result) == 0 { 82 | log.Printf("Fill the bucket with types found in config file: %s", s.ConfigPath) 83 | cfg, err := tooling.CreateConfigFromFile(s.ConfigPath) 84 | for _, mt := range cfg.Types { 85 | err = db.Update(func(tx *bolt.Tx) error { 86 | b := tx.Bucket([]byte("TYPES")) 87 | id, _ := b.NextSequence() 88 | tItem := TypeItem{ 89 | ID: int64(id), 90 | Type: mt.DisplayName, 91 | Features: strings.Join(mt.Features, ","), 92 | } 93 | err = b.Put([]byte(mt.DisplayName), tItem.ToBytes()) 94 | if err != nil { 95 | log.Printf("Error putting type in: %s", err.Error()) 96 | } 97 | return err 98 | }) 99 | } 100 | 101 | } 102 | return 103 | } 104 | 105 | func (s *machineTypeService) GetByID(accountName string, id int64) (mt *storage.MachineType, err error) { 106 | log.Printf("GetByID(%s, %d)\n", accountName, id) 107 | if _, b := os.LookupEnv("STATIC_MACHINES"); b { 108 | log.Println("Environment STATIC_MACHINES is set: Hardcoded types are served") 109 | switch id { 110 | case 1: 111 | mt = &mType1 112 | case 2: 113 | mt = &mType2 114 | case 3: 115 | mt = &mType3 116 | case 4: 117 | mt = &mType4 118 | } 119 | return mt, nil 120 | } 121 | var foundID bool 122 | db.View(func(tx *bolt.Tx) error { 123 | // Assume bucket exists and has keys 124 | b := tx.Bucket([]byte("TYPES")) 125 | c := b.Cursor() 126 | var mType storage.MachineType 127 | for k, v := c.First(); k != nil; k, v = c.Next() { 128 | if err := json.Unmarshal(v, &mType); err != nil { 129 | panic(err) 130 | } 131 | if mType.ID == id { 132 | log.Printf("Found the entry in TYPES: %v", mType) 133 | foundID = true 134 | mt = &mType 135 | return err 136 | } 137 | } 138 | return err 139 | }) 140 | if !foundID { 141 | msg := fmt.Sprintf("Could not find MachineType with ID: %d", id) 142 | log.Printf(msg) 143 | err = fmt.Errorf(msg) 144 | 145 | } 146 | return mt, err 147 | } 148 | 149 | func (s *machineTypeService) GetByUsername(username string) (mt *storage.MachineType, err error) { 150 | log.Printf("GetByUsername(%s)\n", username) 151 | userSplit := strings.Split(username, "-") 152 | usern := userSplit[0] 153 | var user UserItem 154 | foundUser := false 155 | db.View(func(tx *bolt.Tx) error { 156 | // Assume bucket exists and has keys 157 | b := tx.Bucket([]byte("USERS")) 158 | var tmpUsr UserItem 159 | c := b.Cursor() 160 | for k, v := c.First(); k != nil; k, v = c.Next() { 161 | if err := json.Unmarshal(v, &tmpUsr); err != nil { 162 | panic(err) 163 | } 164 | if tmpUsr.Login == usern { 165 | log.Printf("Found user '%s'", usern) 166 | foundUser = true 167 | user = tmpUsr 168 | break 169 | } 170 | } 171 | return nil 172 | }) 173 | if !foundUser { 174 | return nil, fmt.Errorf("Could not find user '%s'", usern) 175 | } 176 | mt = &storage.MachineType{ 177 | Login: usern, 178 | Password: user.Password, 179 | } 180 | typen := strings.Join(userSplit[1:], "-") 181 | foundType := false 182 | db.View(func(tx *bolt.Tx) error { 183 | // Assume bucket exists and has keys 184 | b := tx.Bucket([]byte("TYPES")) 185 | var tmpType TypeItem 186 | c := b.Cursor() 187 | for k, v := c.First(); k != nil; k, v = c.Next() { 188 | if err := json.Unmarshal(v, &tmpType); err != nil { 189 | panic(err) 190 | } 191 | if tmpType.Type == typen { 192 | log.Printf("Found mType: %s\n", typen) 193 | foundType = true 194 | mt.Features = strings.Split(tmpType.Features, ",") 195 | mt.DisplayName = tmpType.Type 196 | break 197 | } 198 | } 199 | return nil 200 | }) 201 | if !foundType { 202 | return nil, fmt.Errorf("Could not find type '%s' for user '%s'", typen, usern) 203 | } 204 | return mt, nil 205 | } 206 | 207 | func (s *machineTypeService) Add(accountName string, mt *storage.MachineType) (err error) { 208 | dbSync.Lock() 209 | defer dbSync.Unlock() 210 | err = db.Update(func(tx *bolt.Tx) error { 211 | b := tx.Bucket([]byte("TYPES")) 212 | id, _ := b.NextSequence() 213 | tItem := TypeItem{ 214 | ID: int64(id), 215 | Type: mt.DisplayName, 216 | Features: strings.Join(mt.Features, ","), 217 | } 218 | err := b.Put([]byte(mt.DisplayName), tItem.ToBytes()) 219 | if err != nil { 220 | return fmt.Errorf("could not set machine-type: %v", err) 221 | } 222 | log.Printf(" ADD: Added Machine %s (ID:%d)\n", tItem.Type, tItem.ID) 223 | return nil 224 | }) 225 | return err 226 | } 227 | 228 | func (s *machineTypeService) Delete(accountName string, id int64) error { 229 | return nil 230 | } 231 | 232 | func (s *machineTypeService) List(accountName string) (mt []storage.MachineType, err error) { 233 | log.Printf("mt.List(accountName=%s)", accountName) 234 | result := []storage.MachineType{} 235 | if _, b := os.LookupEnv("STATIC_MACHINES"); b { 236 | log.Println("Environment STATIC_MACHINES is set: Serve static machine type") 237 | return []storage.MachineType{ 238 | mType1, 239 | mType2, 240 | mType3, 241 | mType4, 242 | }, nil 243 | } 244 | db.View(func(tx *bolt.Tx) error { 245 | // Assume bucket exists and has keys 246 | b := tx.Bucket([]byte("TYPES")) 247 | c := b.Cursor() 248 | var tItem TypeItem 249 | var mType storage.MachineType 250 | for k, v := c.First(); k != nil; k, v = c.Next() { 251 | log.Printf("Found key '%v'", k) 252 | if err := json.Unmarshal(v, &tItem); err != nil { 253 | panic(err) 254 | } 255 | mType.ID = tItem.ID 256 | mType.DisplayName = tItem.Type 257 | mType.Features = strings.Split(tItem.Features, ",") 258 | log.Printf(">> Add MT to result: %v", mType) 259 | result = append(result, mType) 260 | } 261 | return nil 262 | }) 263 | return result, nil 264 | } 265 | 266 | func (s *machineTypeService) Update(accountName string, mt storage.MachineType) (err error) { 267 | dbSync.Lock() 268 | defer dbSync.Unlock() 269 | err = db.Update(func(tx *bolt.Tx) error { 270 | b := tx.Bucket([]byte("TYPES")) 271 | err := b.Put([]byte(mt.Login), mt.ToBytes()) 272 | if err != nil { 273 | return fmt.Errorf("could not set machine-type: %v", err) 274 | } 275 | log.Printf(" Updated Machine %s (ID:%d)\n", mt.Login, mt.ID) 276 | return nil 277 | }) 278 | return err 279 | } 280 | 281 | // GenPasswordHash creates a hash from a plain password 282 | func (s *machineTypeService) GenPasswordHash(passwd string) string { 283 | bytes, _ := bcrypt.GenerateFromPassword([]byte(passwd), 14) 284 | return string(bytes) 285 | } 286 | 287 | // CheckPasswordHash compares the hash of the plain password with what is in the storage 288 | func (s *machineTypeService) CheckPassword(password, hash string) bool { 289 | err := bcrypt.CompareHashAndPassword([]byte(hash), []byte(password)) 290 | return err == nil 291 | } 292 | -------------------------------------------------------------------------------- /pkg/storage/boltdb/service.go: -------------------------------------------------------------------------------- 1 | package boltdb 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "log" 7 | "sync" 8 | 9 | "github.com/boltdb/bolt" 10 | "github.com/qnib/metahub/pkg/storage" 11 | ) 12 | 13 | var db *bolt.DB 14 | var dbSync sync.Mutex 15 | 16 | func init() { 17 | err := setupDB() 18 | if err != nil { 19 | log.Fatal(err.Error()) 20 | } 21 | } 22 | 23 | // NewService returns a new storage.Service for boltdb 24 | func NewService(cpath string) storage.Service { 25 | return &service{ 26 | ConfigPath: cpath, 27 | } 28 | } 29 | 30 | type service struct { 31 | ConfigPath string 32 | } 33 | 34 | func (s *service) MachineTypeService(ctx context.Context) (mt storage.MachineTypeService, err error) { 35 | mt = &machineTypeService{ 36 | ctx: ctx, 37 | ConfigPath: s.ConfigPath, 38 | } 39 | err = mt.Init() 40 | return 41 | } 42 | 43 | func (s *service) AccessTokenService(ctx context.Context) (storage.AccessTokenService, error) { 44 | return &accessTokenService{ 45 | ctx: ctx, 46 | }, nil 47 | } 48 | 49 | func (s *service) AccountService(ctx context.Context) (storage.AccountService, error) { 50 | return &accountService{ 51 | ctx: ctx, 52 | }, nil 53 | } 54 | 55 | func setupDB() error { 56 | var err error 57 | dbSync.Lock() 58 | defer dbSync.Unlock() 59 | db, err = bolt.Open("my.db", 0600, nil) 60 | if err != nil { 61 | return fmt.Errorf("could not open db, %v", err) 62 | } 63 | err = db.Update(func(tx *bolt.Tx) error { 64 | _, err := tx.CreateBucketIfNotExists([]byte("TOKENS")) 65 | if err != nil { 66 | return fmt.Errorf("could not create TOKENS bucket: %v", err) 67 | } 68 | _, err = tx.CreateBucketIfNotExists([]byte("USERS")) 69 | if err != nil { 70 | return fmt.Errorf("could not create USERS bucket: %v", err) 71 | } 72 | _, err = tx.CreateBucketIfNotExists([]byte("TYPES")) 73 | if err != nil { 74 | return fmt.Errorf("could not create TYPES bucket: %v", err) 75 | } 76 | return nil 77 | }) 78 | if err != nil { 79 | return fmt.Errorf("could not set up buckets, %v", err) 80 | } 81 | 82 | fmt.Println("DB Setup Done") 83 | return nil 84 | } 85 | -------------------------------------------------------------------------------- /pkg/storage/boltdb/types.go: -------------------------------------------------------------------------------- 1 | package boltdb 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | ) 7 | 8 | // TypeItem holds the features for a particular type of client 9 | type TypeItem struct { 10 | ID int64 11 | Type string 12 | Features string 13 | } 14 | 15 | // ToBytes encodes everything to be stored in boltdb 16 | func (ti TypeItem) ToBytes() []byte { 17 | reqBodyBytes := new(bytes.Buffer) 18 | json.NewEncoder(reqBodyBytes).Encode(ti) 19 | return reqBodyBytes.Bytes() 20 | } 21 | -------------------------------------------------------------------------------- /pkg/storage/boltdb/users.go: -------------------------------------------------------------------------------- 1 | package boltdb 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | ) 7 | 8 | // UserItem holds the features for a particular login 9 | type UserItem struct { 10 | ID int64 11 | Login string 12 | Password string 13 | } 14 | 15 | // ToBytes encodes everything to be stored in boltdb 16 | func (ui UserItem) ToBytes() []byte { 17 | reqBodyBytes := new(bytes.Buffer) 18 | json.NewEncoder(reqBodyBytes).Encode(ui) 19 | return reqBodyBytes.Bytes() 20 | } 21 | -------------------------------------------------------------------------------- /pkg/storage/clouddatastore/accesstoken_model.go: -------------------------------------------------------------------------------- 1 | package clouddatastore 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | var accessTokenEntityKind = "access_token" 8 | 9 | type accessToken struct { 10 | AccountName string `datastore:"account,noindex"` 11 | Expiry time.Time `datastore:"expiry,omitempty"` 12 | } 13 | -------------------------------------------------------------------------------- /pkg/storage/clouddatastore/accesstoken_service.go: -------------------------------------------------------------------------------- 1 | package clouddatastore 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/qnib/metahub/pkg/storage" 8 | 9 | "cloud.google.com/go/datastore" 10 | ) 11 | 12 | type accessTokenService struct { 13 | ctx context.Context 14 | client *datastore.Client 15 | } 16 | 17 | func (s *accessTokenService) Get(token string) (*storage.AccessToken, error) { 18 | accessTokenKey := datastore.NameKey(accessTokenEntityKind, token, nil) 19 | var at accessToken 20 | err := s.client.Get(s.ctx, accessTokenKey, &at) 21 | if err == datastore.ErrNoSuchEntity { 22 | return nil, nil 23 | } 24 | if err != nil { 25 | return nil, fmt.Errorf("error getting datastore entity: %v", err) 26 | } 27 | //TODO: check at.Expiry? 28 | return &storage.AccessToken{ 29 | AccountName: at.AccountName, 30 | Expiry: at.Expiry, 31 | }, nil 32 | } 33 | 34 | func (s *accessTokenService) Put(token string, at storage.AccessToken) error { 35 | accessTokenKey := datastore.NameKey(accessTokenEntityKind, token, nil) 36 | e := accessToken{ 37 | AccountName: at.AccountName, 38 | Expiry: at.Expiry, 39 | } 40 | if _, err := s.client.Put(s.ctx, accessTokenKey, &e); err != nil { 41 | return fmt.Errorf("error putting access token: %v", err) 42 | } 43 | return nil 44 | } 45 | -------------------------------------------------------------------------------- /pkg/storage/clouddatastore/account_model.go: -------------------------------------------------------------------------------- 1 | package clouddatastore 2 | 3 | var accountEntityKind = "account" 4 | 5 | type account struct { 6 | DisplayName string `datastore:"name,noindex"` 7 | } 8 | -------------------------------------------------------------------------------- /pkg/storage/clouddatastore/account_service.go: -------------------------------------------------------------------------------- 1 | package clouddatastore 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/qnib/metahub/pkg/storage" 8 | 9 | "cloud.google.com/go/datastore" 10 | ) 11 | 12 | type accountService struct { 13 | ctx context.Context 14 | client *datastore.Client 15 | } 16 | 17 | func (s *accountService) Upsert(name string, a storage.Account) error { 18 | k := datastore.NameKey(accountEntityKind, name, nil) 19 | 20 | e := account{ 21 | DisplayName: a.DisplayName, 22 | } 23 | 24 | var tmp account 25 | if err := s.client.Get(s.ctx, k, &tmp); err == datastore.ErrNoSuchEntity { 26 | if _, err := s.client.Put(s.ctx, k, &e); err != nil { 27 | return fmt.Errorf("error putting account: %v", err) 28 | } 29 | } else if err != nil { 30 | return fmt.Errorf("error getting account: %v", err) 31 | } else { 32 | if tmp.DisplayName != e.DisplayName { 33 | if _, err := s.client.Put(s.ctx, k, &e); err != nil { 34 | return fmt.Errorf("error putting account: %v", err) 35 | } 36 | } 37 | } 38 | return nil 39 | } 40 | 41 | func (s *accountService) Get(name string) (*storage.Account, error) { 42 | k := datastore.NameKey(accountEntityKind, name, nil) 43 | var e account 44 | if err := s.client.Get(s.ctx, k, &e); err == datastore.ErrNoSuchEntity { 45 | return nil, nil 46 | } else if err != nil { 47 | return nil, fmt.Errorf("error getting entity: %v", err) 48 | } 49 | return &storage.Account{ 50 | DisplayName: e.DisplayName, 51 | }, nil 52 | } 53 | -------------------------------------------------------------------------------- /pkg/storage/clouddatastore/machinetype_model.go: -------------------------------------------------------------------------------- 1 | package clouddatastore 2 | 3 | var machineTypeEntityKind = "MachineType" 4 | 5 | type machineTypeModel struct { 6 | DisplayName string `datastore:"name,noindex"` 7 | Features []string `datastore:"features,noindex"` 8 | Login string `datastore:"login"` 9 | Password string `datastore:"password,noindex"` 10 | } 11 | 12 | -------------------------------------------------------------------------------- /pkg/storage/clouddatastore/machinetype_service.go: -------------------------------------------------------------------------------- 1 | package clouddatastore 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "sort" 7 | 8 | "cloud.google.com/go/datastore" 9 | "github.com/qnib/metahub/pkg/storage" 10 | ) 11 | 12 | type machineTypeService struct { 13 | ctx context.Context 14 | client *datastore.Client 15 | } 16 | 17 | func formatLogin(accountName string, login string) { 18 | 19 | } 20 | 21 | func (s *machineTypeService) Init() (err error) { 22 | return 23 | } 24 | 25 | func (s *machineTypeService) GetByID(accountName string, id int64) (*storage.MachineType, error) { 26 | accountKey := datastore.NameKey(accountEntityKind, accountName, nil) 27 | machineTypeKey := datastore.IDKey(machineTypeEntityKind, id, accountKey) 28 | var mt machineTypeModel 29 | err := s.client.Get(s.ctx, machineTypeKey, &mt) 30 | if _, ok := err.(*datastore.ErrFieldMismatch); ok { 31 | err = nil 32 | } 33 | if err == datastore.ErrNoSuchEntity { 34 | return nil, nil 35 | } 36 | if err != nil { 37 | return nil, fmt.Errorf("error getting datastore entity: %v", err) 38 | } 39 | return &storage.MachineType{ 40 | ID: machineTypeKey.ID, 41 | DisplayName: mt.DisplayName, 42 | Features: mt.Features, 43 | Password: mt.Password, 44 | Login: mt.Login, 45 | }, nil 46 | } 47 | 48 | func (s *machineTypeService) GetByUsername(username string) (*storage.MachineType, error) { 49 | 50 | var machineTypes []machineTypeModel 51 | q := datastore.NewQuery(machineTypeEntityKind) 52 | q = q.Filter("login =", username) 53 | machineTypeKeys, err := s.client.GetAll(s.ctx, q, &machineTypes) 54 | if _, ok := err.(*datastore.ErrFieldMismatch); ok { 55 | err = nil 56 | } 57 | if err != nil { 58 | return nil, fmt.Errorf("error querying feature sets: %v", err) 59 | } 60 | 61 | if len(machineTypeKeys) == 0 { 62 | return nil, nil 63 | } 64 | if len(machineTypeKeys) > 1 { 65 | return nil, fmt.Errorf("found %d entities", len(machineTypeKeys)) 66 | } 67 | 68 | mt := machineTypes[0] 69 | machineTypeKey := machineTypeKeys[0] 70 | 71 | /* machineTypeKey, err := datastore.DecodeKey(username) 72 | var mt machineTypeModel 73 | err = s.client.Get(s.ctx, machineTypeKey, &mt) 74 | if _, ok := err.(*datastore.ErrFieldMismatch); ok { 75 | err = nil 76 | } 77 | if err == datastore.ErrNoSuchEntity { 78 | return nil, nil 79 | } 80 | if err != nil { 81 | return nil, fmt.Errorf("error getting machine type: %v", err) 82 | }*/ 83 | return &storage.MachineType{ 84 | ID: machineTypeKey.ID, 85 | DisplayName: mt.DisplayName, 86 | Features: mt.Features, 87 | Password: mt.Password, 88 | Login: mt.Login, 89 | }, nil 90 | } 91 | 92 | func (s *machineTypeService) Add(accountName string, mt *storage.MachineType) error { 93 | 94 | if existingMt, err := s.GetByUsername(mt.Login); err != nil { 95 | return fmt.Errorf("failed to check for existing login: %v", err) 96 | } else if existingMt != nil { 97 | return fmt.Errorf("login already exist: %v", err) 98 | } 99 | 100 | accountKey := datastore.NameKey(accountEntityKind, accountName, nil) 101 | machineTypeKey := datastore.IncompleteKey(machineTypeEntityKind, accountKey) 102 | 103 | entity := machineTypeModel{ 104 | DisplayName: mt.DisplayName, 105 | Features: mt.Features, 106 | Login: mt.Login, 107 | Password: mt.Password, 108 | } 109 | machineTypeKey, err := s.client.Put(s.ctx, machineTypeKey, &entity) 110 | if err != nil { 111 | return fmt.Errorf("error putting machine type entity: %v", err) 112 | } 113 | 114 | mt.ID = machineTypeKey.ID 115 | 116 | return nil 117 | } 118 | 119 | func (s *machineTypeService) Delete(accountName string, id int64) error { 120 | accountKey := datastore.NameKey(accountEntityKind, accountName, nil) 121 | machineTypeKey := datastore.IDKey(machineTypeEntityKind, id, accountKey) 122 | 123 | err := s.client.Delete(s.ctx, machineTypeKey) 124 | if err != nil { 125 | return fmt.Errorf("error deleting entity: %v", err) 126 | } 127 | return nil 128 | } 129 | 130 | func (s *machineTypeService) List(accountName string) ([]storage.MachineType, error) { 131 | accountKey := datastore.NameKey(accountEntityKind, accountName, nil) 132 | var machineTypes []machineTypeModel 133 | q := datastore.NewQuery(machineTypeEntityKind) 134 | q = q.Ancestor(accountKey) 135 | machineTypeKeys, err := s.client.GetAll(s.ctx, q, &machineTypes) 136 | if _, ok := err.(*datastore.ErrFieldMismatch); ok { 137 | err = nil 138 | } 139 | if err != nil { 140 | return nil, fmt.Errorf("error querying feature sets: %v", err) 141 | } 142 | //log.Printf("%d feature sets", len(featureSets)) 143 | 144 | result := make([]storage.MachineType, len(machineTypes)) 145 | 146 | sort.Slice(machineTypes, func(i, j int) bool { 147 | return machineTypes[i].DisplayName < machineTypes[j].DisplayName 148 | }) 149 | sort.Slice(machineTypeKeys, func(i, j int) bool { 150 | return machineTypes[i].DisplayName < machineTypes[j].DisplayName 151 | }) 152 | 153 | for i, mt := range machineTypes { 154 | k := machineTypeKeys[i] 155 | result[i] = storage.MachineType{ 156 | ID: k.ID, 157 | DisplayName: mt.DisplayName, 158 | Features: mt.Features, 159 | Login: mt.Login, 160 | Password: mt.Password, 161 | } 162 | } 163 | return result, nil 164 | } 165 | 166 | func (s *machineTypeService) Update(accountName string, mt storage.MachineType) error { 167 | accountKey := datastore.NameKey(accountEntityKind, accountName, nil) 168 | machineTypeKey := datastore.IDKey(machineTypeEntityKind, mt.ID, accountKey) 169 | 170 | var tmp machineTypeModel 171 | err := s.client.Get(s.ctx, machineTypeKey, &tmp) 172 | if _, ok := err.(*datastore.ErrFieldMismatch); ok { 173 | err = nil 174 | } 175 | if err != nil { 176 | return fmt.Errorf("error getting entity: %v", err) 177 | } 178 | 179 | if tmp.Login != mt.Login { 180 | if existingMt, err := s.GetByUsername(mt.Login); err != nil { 181 | return fmt.Errorf("failed to check for existing login: %v", err) 182 | } else if existingMt != nil { 183 | return fmt.Errorf("login already exist: %v", err) 184 | } 185 | } 186 | 187 | tmp.DisplayName = mt.DisplayName 188 | tmp.Features = mt.Features 189 | tmp.Login = mt.Login 190 | tmp.Password = mt.Password 191 | 192 | _, err = s.client.Put(s.ctx, machineTypeKey, &tmp) 193 | if err != nil { 194 | return fmt.Errorf("error putting machine type entity: %v", err) 195 | } 196 | 197 | return nil 198 | } 199 | -------------------------------------------------------------------------------- /pkg/storage/clouddatastore/service.go: -------------------------------------------------------------------------------- 1 | package clouddatastore 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/qnib/metahub/pkg/storage" 8 | 9 | "cloud.google.com/go/datastore" 10 | ) 11 | 12 | // NewService returns a new storage.Service for GCP Cloud Datastore 13 | func NewService() storage.Service { 14 | return &service{} 15 | } 16 | 17 | type service struct { 18 | } 19 | 20 | func (s *service) newClient(ctx context.Context) (*datastore.Client, error) { 21 | return datastore.NewClient(ctx, "") 22 | } 23 | 24 | func (s *service) MachineTypeService(ctx context.Context) (storage.MachineTypeService, error) { 25 | datastoreClient, err := s.newClient(ctx) 26 | if err != nil { 27 | return nil, fmt.Errorf("failed to create client: %v", err) 28 | } 29 | return &machineTypeService{ 30 | ctx: ctx, 31 | client: datastoreClient, 32 | }, nil 33 | } 34 | 35 | func (s *service) AccessTokenService(ctx context.Context) (storage.AccessTokenService, error) { 36 | datastoreClient, err := s.newClient(ctx) 37 | if err != nil { 38 | return nil, fmt.Errorf("failed to create client: %v", err) 39 | } 40 | return &accessTokenService{ 41 | ctx: ctx, 42 | client: datastoreClient, 43 | }, nil 44 | } 45 | 46 | func (s *service) AccountService(ctx context.Context) (storage.AccountService, error) { 47 | datastoreClient, err := s.newClient(ctx) 48 | if err != nil { 49 | return nil, fmt.Errorf("failed to create client: %v", err) 50 | } 51 | return &accountService{ 52 | ctx: ctx, 53 | client: datastoreClient, 54 | }, nil 55 | } 56 | -------------------------------------------------------------------------------- /pkg/storage/dynamodb/accesstoken_model.go: -------------------------------------------------------------------------------- 1 | package dynamodb 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | var accessTokenEntityKind = "access_token" 8 | 9 | type accessToken struct { 10 | AccountName string `datastore:"account,noindex"` 11 | Expiry time.Time `datastore:"expiry,omitempty"` 12 | } 13 | -------------------------------------------------------------------------------- /pkg/storage/dynamodb/accesstoken_service.go: -------------------------------------------------------------------------------- 1 | package dynamodb 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | "github.com/qnib/metahub/pkg/storage" 8 | ) 9 | 10 | type accessTokenService struct { 11 | ctx context.Context 12 | } 13 | 14 | func (s *accessTokenService) Get(token string) (*storage.AccessToken, error) { 15 | //TODO: check at.Expiry? 16 | return &storage.AccessToken{ 17 | AccountName: token, 18 | Expiry: time.Time{}, 19 | }, nil 20 | } 21 | 22 | func (s *accessTokenService) Put(token string, at storage.AccessToken) error { 23 | return nil 24 | } 25 | -------------------------------------------------------------------------------- /pkg/storage/dynamodb/account_model.go: -------------------------------------------------------------------------------- 1 | package dynamodb 2 | 3 | var accountEntityKind = "account" 4 | 5 | type account struct { 6 | DisplayName string `datastore:"name,noindex"` 7 | } 8 | -------------------------------------------------------------------------------- /pkg/storage/dynamodb/account_service.go: -------------------------------------------------------------------------------- 1 | package dynamodb 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/qnib/metahub/pkg/storage" 7 | ) 8 | 9 | type accountService struct { 10 | ctx context.Context 11 | } 12 | 13 | func (s *accountService) Upsert(name string, a storage.Account) error { 14 | return nil 15 | } 16 | 17 | func (s *accountService) Get(name string) (*storage.Account, error) { 18 | return &storage.Account{ 19 | DisplayName: name, 20 | }, nil 21 | } 22 | -------------------------------------------------------------------------------- /pkg/storage/dynamodb/dummies.go: -------------------------------------------------------------------------------- 1 | package dynamodb 2 | 3 | import "github.com/qnib/metahub/pkg/storage" 4 | 5 | // Consts for protoype 6 | const ( 7 | user = "qnib" 8 | accountName = user 9 | ) 10 | 11 | // Dummy MachineTypes 12 | var ( 13 | mType1 = storage.MachineType{ 14 | ID: 1, 15 | DisplayName: "type1", 16 | Features: []string{"cpu:broadwell"}, 17 | Login: user + "-type1", 18 | Password: user + "-type1", 19 | } 20 | mType2 = storage.MachineType{ 21 | ID: 2, 22 | DisplayName: "type2", 23 | Features: []string{"cpu:skylake"}, 24 | Login: user + "-type2", 25 | Password: user + "-type2", 26 | } 27 | mType3 = storage.MachineType{ 28 | ID: 3, 29 | DisplayName: "type3", 30 | Features: []string{"cpu:coffelake"}, 31 | Login: user + "-type3", 32 | Password: user + "-type3", 33 | } 34 | mType4 = storage.MachineType{ 35 | ID: 4, 36 | DisplayName: "type4", 37 | Features: []string{"cpu:broadwell", "nvcap:5.2"}, 38 | Login: user + "-type4", 39 | Password: user + "-type4", 40 | } 41 | ) 42 | -------------------------------------------------------------------------------- /pkg/storage/dynamodb/machinetype_model.go: -------------------------------------------------------------------------------- 1 | package dynamodb 2 | 3 | var machineTypeEntityKind = "MachineType" 4 | 5 | type machineTypeModel struct { 6 | DisplayName string `datastore:"name,noindex"` 7 | Features []string `datastore:"features,noindex"` 8 | Login string `datastore:"login"` 9 | Password string `datastore:"password,noindex"` 10 | } 11 | -------------------------------------------------------------------------------- /pkg/storage/dynamodb/machinetype_service.go: -------------------------------------------------------------------------------- 1 | package dynamodb 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "log" 7 | "strings" 8 | 9 | "github.com/qnib/metahub/pkg/storage" 10 | ) 11 | 12 | type machineTypeService struct { 13 | ctx context.Context 14 | } 15 | 16 | func formatLogin(accountName string, login string) { 17 | 18 | } 19 | func (s *machineTypeService) Init() (err error) { 20 | return 21 | } 22 | 23 | func (s *machineTypeService) GetByID(accountName string, id int64) (mt *storage.MachineType, err error) { 24 | log.Printf("GetByID(%s, %d)\n", accountName, id) 25 | return mt, err 26 | } 27 | 28 | /*** GetByUsername will be used to authenticate a client. 29 | We'll chop off the prefix (e.g. qnib from qnib-c518xl-shp2) and chat the prefix against the users table 30 | */ 31 | func (s *machineTypeService) GetByUsername(username string) (mt *storage.MachineType, err error) { 32 | log.Printf("GetByUsername(%s)\n", username) 33 | // Chop of the first part of the usrename, without the type specific suffixes 34 | userSplit := strings.Split(username, "-") 35 | if len(userSplit) == 1 { 36 | err = fmt.Errorf("username should contain the actual user seperated by a dash (e.g. qnib-type1). Got: %s", username) 37 | return 38 | } 39 | usern := userSplit[0] 40 | user, err := mhTableUserScan(svc, fmt.Sprintf("%s_users", mhDbTablePrefix), usern) 41 | if err != nil { 42 | return 43 | } 44 | mt = &storage.MachineType{ 45 | Login: user.Login, 46 | Password: user.Password, 47 | } 48 | typen := strings.Join(userSplit[1:], "-") 49 | typ, err := mhTableTypeScan(svc, fmt.Sprintf("%s_types", mhDbTablePrefix), typen) 50 | if err != nil { 51 | return 52 | } 53 | mt.Features = strings.Split(typ.Features, ",") 54 | mt.DisplayName = typ.Type 55 | return 56 | } 57 | 58 | func (s *machineTypeService) Add(accountName string, mt *storage.MachineType) (err error) { 59 | return err 60 | } 61 | 62 | func (s *machineTypeService) Delete(accountName string, id int64) error { 63 | return nil 64 | } 65 | 66 | func (s *machineTypeService) List(accountName string) (mts []storage.MachineType, err error) { 67 | log.Printf("mt.List(accountName=%s)", accountName) 68 | typeItems, err := mhTableTypeList(svc, fmt.Sprintf("%s_types", mhDbTablePrefix)) 69 | if err != nil { 70 | return 71 | } 72 | for _, tItem := range typeItems { 73 | mt := storage.MachineType{ 74 | DisplayName: tItem.Type, 75 | Features: strings.Split(tItem.Features, ","), 76 | } 77 | mts = append(mts, mt) 78 | } 79 | return 80 | } 81 | 82 | func (s *machineTypeService) Update(accountName string, mt storage.MachineType) (err error) { 83 | return err 84 | } 85 | -------------------------------------------------------------------------------- /pkg/storage/dynamodb/service.go: -------------------------------------------------------------------------------- 1 | package dynamodb 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "log" 7 | "sync" 8 | 9 | "github.com/qnib/metahub/pkg/storage" 10 | 11 | "github.com/aws/aws-sdk-go/aws" 12 | "github.com/aws/aws-sdk-go/aws/awserr" 13 | "github.com/aws/aws-sdk-go/aws/session" 14 | "github.com/aws/aws-sdk-go/service/dynamodb" 15 | "github.com/aws/aws-sdk-go/service/dynamodb/dynamodbattribute" 16 | ) 17 | 18 | var ( 19 | svc *dynamodb.DynamoDB 20 | dbSync sync.Mutex 21 | ) 22 | 23 | const ( 24 | mhDbTablePrefix = "metahub-v1" 25 | ) 26 | 27 | func init() { 28 | err := setupDB() 29 | if err != nil { 30 | log.Fatal(err.Error()) 31 | } 32 | } 33 | 34 | // NewService returns a new storage.Service for boltdb 35 | func NewService() storage.Service { 36 | return &service{} 37 | } 38 | 39 | type service struct{} 40 | 41 | func (s *service) MachineTypeService(ctx context.Context) (storage.MachineTypeService, error) { 42 | return &machineTypeService{ 43 | ctx: ctx, 44 | }, nil 45 | } 46 | 47 | func (s *service) AccessTokenService(ctx context.Context) (storage.AccessTokenService, error) { 48 | return &accessTokenService{ 49 | ctx: ctx, 50 | }, nil 51 | } 52 | 53 | func (s *service) AccountService(ctx context.Context) (storage.AccountService, error) { 54 | return &accountService{ 55 | ctx: ctx, 56 | }, nil 57 | } 58 | 59 | func setupDB() (err error) { 60 | dbSync.Lock() 61 | defer dbSync.Unlock() 62 | sess := session.Must(session.NewSessionWithOptions(session.Options{ 63 | SharedConfigState: session.SharedConfigEnable, 64 | })) 65 | 66 | // Create DynamoDB client 67 | svc = dynamodb.New(sess) 68 | if !mhTableExists(svc, fmt.Sprintf("%s_types", mhDbTablePrefix)) { 69 | err = mhTableTypesCreate(svc) 70 | if err != nil { 71 | log.Fatal(err.Error()) 72 | } 73 | } 74 | if !mhTableExists(svc, fmt.Sprintf("%s_users", mhDbTablePrefix)) { 75 | err = mhTableUsersCreate(svc) 76 | if err != nil { 77 | log.Fatal(err.Error()) 78 | } 79 | err = mhTableInit(svc) 80 | } 81 | 82 | fmt.Println("DB Setup Done") 83 | return 84 | } 85 | 86 | func mhTableExists(db *dynamodb.DynamoDB, mhDbTable string) bool { 87 | input := &dynamodb.ListTablesInput{} 88 | for { 89 | // Get the list of tables 90 | result, err := db.ListTables(input) 91 | if err != nil { 92 | if aerr, ok := err.(awserr.Error); ok { 93 | switch aerr.Code() { 94 | case dynamodb.ErrCodeInternalServerError: 95 | log.Println(dynamodb.ErrCodeInternalServerError, aerr.Error()) 96 | default: 97 | log.Println(aerr.Error()) 98 | } 99 | } else { 100 | // Print the error, cast err to awserr.Error to get the Code and 101 | // Message from an error. 102 | fmt.Println(err.Error()) 103 | } 104 | } 105 | for _, n := range result.TableNames { 106 | // if we already find the table we are looking for - awesome 107 | // -> otherwise rember we need to create it. 108 | log.Printf("Table found: %s\n", *n) 109 | if *n == mhDbTable { 110 | return true 111 | } 112 | 113 | } 114 | // assign the last read tablename as the start for our next call to the ListTables function 115 | // the maximum number of table names returned in a call is 100 (default), which requires us to make 116 | // multiple calls to the ListTables function to retrieve all table names 117 | input.ExclusiveStartTableName = result.LastEvaluatedTableName 118 | if result.LastEvaluatedTableName == nil { 119 | break 120 | } 121 | } 122 | return false 123 | } 124 | 125 | func mhTableInit(db *dynamodb.DynamoDB) (err error) { 126 | return 127 | } 128 | 129 | func mhTableUsersCreate(db *dynamodb.DynamoDB) (err error) { 130 | mhDbTable := fmt.Sprintf("%s_users", mhDbTablePrefix) 131 | input := &dynamodb.CreateTableInput{ 132 | AttributeDefinitions: GetUsersAttrDefs(), 133 | KeySchema: GetUsersAttrSchemas(), 134 | ProvisionedThroughput: &dynamodb.ProvisionedThroughput{ 135 | ReadCapacityUnits: aws.Int64(10), 136 | WriteCapacityUnits: aws.Int64(10), 137 | }, 138 | TableName: aws.String(mhDbTable), 139 | } 140 | 141 | _, err = db.CreateTable(input) 142 | if err != nil { 143 | fmt.Println("Got error calling CreateTable:") 144 | fmt.Println(err.Error()) 145 | } 146 | fmt.Println("Created the table", mhDbTable) 147 | return 148 | } 149 | 150 | func mhTableTypesCreate(db *dynamodb.DynamoDB) (err error) { 151 | mhDbTable := fmt.Sprintf("%s_types", mhDbTablePrefix) 152 | input := &dynamodb.CreateTableInput{ 153 | AttributeDefinitions: GetTypesAttrDefs(), 154 | KeySchema: GetTypesAttrSchemas(), 155 | ProvisionedThroughput: &dynamodb.ProvisionedThroughput{ 156 | ReadCapacityUnits: aws.Int64(10), 157 | WriteCapacityUnits: aws.Int64(10), 158 | }, 159 | TableName: aws.String(mhDbTable), 160 | } 161 | 162 | _, err = db.CreateTable(input) 163 | if err != nil { 164 | fmt.Println("Got error calling CreateTable:") 165 | fmt.Println(err.Error()) 166 | } 167 | fmt.Println("Created the table", mhDbTable) 168 | return 169 | } 170 | 171 | func mhTableUserScan(db *dynamodb.DynamoDB, tableName, usern string) (user UsersItem, err error) { 172 | log.Printf("Search for username '%s' in '%s'", usern, tableName) 173 | var queryInput = &dynamodb.QueryInput{ 174 | TableName: aws.String(tableName), 175 | KeyConditions: map[string]*dynamodb.Condition{ 176 | "Login": { 177 | ComparisonOperator: aws.String("EQ"), 178 | AttributeValueList: []*dynamodb.AttributeValue{ 179 | { 180 | S: aws.String(usern), 181 | }, 182 | }, 183 | }, 184 | }, 185 | } 186 | result, err := db.Query(queryInput) 187 | if err != nil { 188 | fmt.Println(err.Error()) 189 | return 190 | } 191 | userObj := []UsersItem{} 192 | err = dynamodbattribute.UnmarshalListOfMaps(result.Items, &userObj) 193 | 194 | switch len(userObj) { 195 | case 0: 196 | err = fmt.Errorf("Could not find user: '%s'", usern) 197 | case 1: 198 | user = userObj[0] 199 | default: 200 | err = fmt.Errorf("Found multiple users for '%s'? WTF?", usern) 201 | } 202 | return 203 | } 204 | 205 | func mhTableTypeList(db *dynamodb.DynamoDB, tableName string) (typeItems []TypeItem, err error) { 206 | log.Printf("List machine types in '%s'", tableName) 207 | params := &dynamodb.ScanInput{ 208 | TableName: aws.String(tableName), 209 | } 210 | result, err := svc.Scan(params) 211 | if err != nil { 212 | fmt.Println(err.Error()) 213 | return 214 | } 215 | err = dynamodbattribute.UnmarshalListOfMaps(result.Items, &typeItems) 216 | return 217 | } 218 | 219 | func mhTableTypeScan(db *dynamodb.DynamoDB, tableName, typen string) (typeItem TypeItem, err error) { 220 | log.Printf("Search for type '%s' in '%s'", typen, tableName) 221 | var queryInput = &dynamodb.QueryInput{ 222 | TableName: aws.String(tableName), 223 | KeyConditions: map[string]*dynamodb.Condition{ 224 | "Type": { 225 | ComparisonOperator: aws.String("EQ"), 226 | AttributeValueList: []*dynamodb.AttributeValue{ 227 | { 228 | S: aws.String(typen), 229 | }, 230 | }, 231 | }, 232 | }, 233 | } 234 | result, err := db.Query(queryInput) 235 | if err != nil { 236 | fmt.Println(err.Error()) 237 | return 238 | } 239 | typeObj := []TypeItem{} 240 | err = dynamodbattribute.UnmarshalListOfMaps(result.Items, &typeObj) 241 | 242 | switch len(typeObj) { 243 | case 0: 244 | err = fmt.Errorf("Could not find type: '%s'", typen) 245 | case 1: 246 | typeItem = typeObj[0] 247 | default: 248 | err = fmt.Errorf("Found multiple types for '%s'? WTF?", typen) 249 | } 250 | return 251 | } 252 | -------------------------------------------------------------------------------- /pkg/storage/dynamodb/types.go: -------------------------------------------------------------------------------- 1 | package dynamodb 2 | 3 | import ( 4 | "github.com/aws/aws-sdk-go/aws" 5 | "github.com/aws/aws-sdk-go/service/dynamodb" 6 | ) 7 | 8 | // GetTypesAttrDefs returns the definitions 9 | func GetTypesAttrDefs() []*dynamodb.AttributeDefinition { 10 | return []*dynamodb.AttributeDefinition{{ 11 | AttributeName: aws.String("Type"), 12 | AttributeType: aws.String("S"), 13 | }, { 14 | AttributeName: aws.String("Features"), 15 | AttributeType: aws.String("S"), 16 | }} 17 | } 18 | 19 | // GetTypesAttrSchemas returns the schemas 20 | func GetTypesAttrSchemas() []*dynamodb.KeySchemaElement { 21 | return []*dynamodb.KeySchemaElement{{ 22 | AttributeName: aws.String("Type"), 23 | KeyType: aws.String("HASH"), 24 | }, { 25 | AttributeName: aws.String("Features"), 26 | KeyType: aws.String("RANGE"), 27 | }} 28 | } 29 | 30 | // TypesItem holds the features for a particular type of client 31 | type TypeItem struct { 32 | Type string 33 | Features string 34 | } 35 | -------------------------------------------------------------------------------- /pkg/storage/dynamodb/users.go: -------------------------------------------------------------------------------- 1 | package dynamodb 2 | 3 | import ( 4 | "github.com/aws/aws-sdk-go/aws" 5 | "github.com/aws/aws-sdk-go/service/dynamodb" 6 | ) 7 | 8 | // GetUsersAttrDefs returns the definitions 9 | func GetUsersAttrDefs() []*dynamodb.AttributeDefinition { 10 | return []*dynamodb.AttributeDefinition{{ 11 | AttributeName: aws.String("Login"), 12 | AttributeType: aws.String("S"), 13 | }, { 14 | AttributeName: aws.String("Password"), 15 | AttributeType: aws.String("S"), 16 | }} 17 | } 18 | 19 | // GetUsersAttrSchemas returns the schemas 20 | func GetUsersAttrSchemas() []*dynamodb.KeySchemaElement { 21 | return []*dynamodb.KeySchemaElement{{ 22 | AttributeName: aws.String("Login"), 23 | KeyType: aws.String("HASH"), 24 | }, { 25 | AttributeName: aws.String("Password"), 26 | KeyType: aws.String("RANGE"), 27 | }} 28 | } 29 | 30 | // UsersItem holds the features for a particular login 31 | type UsersItem struct { 32 | Login string 33 | Password string 34 | } 35 | -------------------------------------------------------------------------------- /pkg/storage/machinetype.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | ) 7 | 8 | // MachineType describes a set of machines/hosts sharing same hardware specs 9 | type MachineType struct { 10 | ID int64 `json:"id"` 11 | DisplayName string `json:"name"` 12 | Features []string `json:"features,omitempty"` 13 | Login string `json:"login"` 14 | Password string `json:"password"` 15 | } 16 | 17 | func (mtm MachineType) ToBytes() []byte { 18 | reqBodyBytes := new(bytes.Buffer) 19 | json.NewEncoder(reqBodyBytes).Encode(mtm) 20 | return reqBodyBytes.Bytes() 21 | } -------------------------------------------------------------------------------- /pkg/storage/machinetype_service.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | // MachineTypeService provides access to machine types objects. 4 | type MachineTypeService interface { 5 | GetByID(accountName string, id int64) (*MachineType, error) 6 | GetByUsername(username string) (*MachineType, error) 7 | Add(accountName string, machineType *MachineType) error 8 | Delete(accountName string, id int64) error 9 | List(accountName string) ([]MachineType, error) 10 | Update(accountName string, machineType MachineType) error 11 | Init() error 12 | CheckPassword(plain, hash string) bool 13 | GenPasswordHash(pwd string) string 14 | } 15 | -------------------------------------------------------------------------------- /pkg/storage/service.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import "context" 4 | 5 | // Service provides access to all storage objects. 6 | type Service interface { 7 | MachineTypeService(ctx context.Context) (MachineTypeService, error) 8 | AccessTokenService(ctx context.Context) (AccessTokenService, error) 9 | AccountService(ctx context.Context) (AccountService, error) 10 | } 11 | -------------------------------------------------------------------------------- /pkg/tooling/config.go: -------------------------------------------------------------------------------- 1 | package tooling 2 | 3 | import ( 4 | "io/ioutil" 5 | "log" 6 | 7 | "gopkg.in/yaml.v2" 8 | ) 9 | 10 | // MachineType stores the name and the list of features 11 | type MachineType struct { 12 | DisplayName string `yaml:"name"` 13 | Features []string `yaml:"features"` 14 | } 15 | 16 | // Config holds a list of MachineTypes 17 | type Config struct { 18 | //TODO: Password should be a hash 19 | User string `yaml:"user"` 20 | Password string `yaml:"password"` 21 | Types []MachineType `yaml:"types"` 22 | } 23 | 24 | // ParseConfig takes a bytearray and unmarshals it 25 | func ParseConfig(data []byte) (cfg Config, err error) { 26 | err = yaml.Unmarshal(data, &cfg) 27 | return 28 | } 29 | 30 | // CreateConfigFromFile parses a file and returns a workshop 31 | func CreateConfigFromFile(fpath string) (cfg Config, err error) { 32 | log.Printf("Reading file: %s", fpath) 33 | yData, err := ioutil.ReadFile(fpath) 34 | if err != nil { 35 | log.Printf("yamlFile.Get err #%v ", err) 36 | } 37 | cfg, err = ParseConfig(yData) 38 | return 39 | } 40 | -------------------------------------------------------------------------------- /pkg/tooling/config_test.go: -------------------------------------------------------------------------------- 1 | package tooling 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | ) 7 | 8 | var testConfig1 = ` 9 | user: user 10 | password: hash 11 | types: 12 | - name: broadwell 13 | features: 14 | - cpu:broadwell 15 | - name: skylake 16 | features: 17 | - cpu:skylake 18 | ` 19 | 20 | func Test_ParseConfig_testConfig1(t *testing.T) { 21 | cfg, err := ParseConfig([]byte(testConfig1)) 22 | if err != nil { 23 | t.Error(err.Error()) 24 | } 25 | if cfg.User != "user" { 26 | t.Errorf("User should be 'user'; got '%s'", cfg.User) 27 | } 28 | if len(cfg.Types) != 2 { 29 | t.Error("Length of testConfig1 should be 2") 30 | } 31 | for _, mt := range cfg.Types { 32 | fmt.Println(mt) 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /pkg/tooling/docker.go: -------------------------------------------------------------------------------- 1 | package tooling 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "os" 7 | "os/exec" 8 | "time" 9 | ) 10 | 11 | // DockerLogin uses a docker client to login to metahub 12 | func DockerLogin(pass, user, registry string) (err error) { 13 | log.Println("Use docker login via client") 14 | cmd := fmt.Sprintf("docker --log-level=debug --debug login --password %s --username %s %s", pass, user, registry) 15 | out, err := exec.Command("ash", "-c", cmd).Output() 16 | if err != nil { 17 | log.Printf("CMD: %s", cmd) 18 | log.Printf("Err: %s", err.Error()) 19 | return fmt.Errorf("Failed to execute command: %s", cmd) 20 | } 21 | time.Sleep(2) 22 | // TODO: That's rather ugly, but the login data is stored per user... 23 | err = os.Chmod("/root/.docker/config.json", 0644) 24 | if err != nil { 25 | return fmt.Errorf("Failed to chmod /root/.docker/config.json: %s", err.Error()) 26 | } 27 | log.Println(string(out)) 28 | return 29 | } 30 | -------------------------------------------------------------------------------- /pkg/tooling/ec2meta.go: -------------------------------------------------------------------------------- 1 | package tooling 2 | 3 | import ( 4 | "fmt" 5 | "os/exec" 6 | "regexp" 7 | "strings" 8 | 9 | "github.com/aws/aws-sdk-go/aws/ec2metadata" 10 | "github.com/aws/aws-sdk-go/aws/session" 11 | ) 12 | 13 | // EC2Meta holds the infos for the instance 14 | type EC2Meta struct { 15 | InstanceType string 16 | InstanceSize string 17 | HyperThreading string 18 | } 19 | 20 | // GetMetaData fetches the metadata of an instance 21 | func GetMetaData() (md EC2Meta, err error) { 22 | sess := session.Must(session.NewSessionWithOptions(session.Options{ 23 | SharedConfigState: session.SharedConfigEnable, 24 | })) 25 | svc := ec2metadata.New(sess) 26 | instType, err := svc.GetMetadata("instance-type") 27 | iTypeSlice := strings.Split(instType, ".") 28 | md.InstanceType = iTypeSlice[0] 29 | r, _ := regexp.Compile("[0-9]*xl") 30 | md.InstanceSize = r.FindString(iTypeSlice[1]) 31 | tpc := getThreadsPerCore() 32 | switch tpc { 33 | case "1": 34 | md.HyperThreading = "off" 35 | case "2": 36 | md.HyperThreading = "on" 37 | default: 38 | md.HyperThreading = "na" 39 | } 40 | return 41 | } 42 | 43 | func getThreadsPerCore() (res string) { 44 | cmd := exec.Command("lscpu") 45 | cmd.Wait() 46 | out, err := cmd.Output() 47 | if err != nil { 48 | panic(err) 49 | } 50 | outstring := strings.TrimSpace(string(out)) 51 | lines := strings.Split(outstring, "\n") 52 | for _, line := range lines { 53 | fields := strings.Split(line, ":") 54 | if len(fields) < 2 { 55 | continue 56 | } 57 | key := strings.TrimSpace(fields[0]) 58 | value := strings.TrimSpace(fields[1]) 59 | 60 | switch key { 61 | case "Thread(s) per core": 62 | res = value 63 | } 64 | } 65 | return 66 | } 67 | 68 | // GetMetahubFeatures returns the features 69 | func (md *EC2Meta) GetMetahubFeatures() []string { 70 | return []string{ 71 | fmt.Sprintf("instType:%s", md.InstanceType), 72 | fmt.Sprintf("instSize:%s", md.InstanceSize), 73 | fmt.Sprintf("instHT:%s", md.HyperThreading), 74 | } 75 | } 76 | 77 | // GetMetahubTypename returns the typename to login to metahub 78 | func (md *EC2Meta) GetMetahubTypename(tname string) string { 79 | if tname != "" { 80 | return tname 81 | } 82 | res := []string{fmt.Sprintf("%s%s", md.InstanceType, md.InstanceSize)} 83 | if md.HyperThreading == "on" { 84 | res = append(res, "ht") 85 | } 86 | return strings.Join(res, "-") 87 | } 88 | -------------------------------------------------------------------------------- /pkg/tooling/ec2ssm.go: -------------------------------------------------------------------------------- 1 | package tooling 2 | 3 | import ( 4 | "github.com/aws/aws-sdk-go/aws" 5 | "github.com/aws/aws-sdk-go/aws/session" 6 | "github.com/aws/aws-sdk-go/service/ssm" 7 | ) 8 | 9 | // GetSSMPassword fetches the metahub password from SSM 10 | func GetSSMPassword(region, path string) (passwd string, err error) { 11 | sess := session.Must(session.NewSessionWithOptions(session.Options{ 12 | SharedConfigState: session.SharedConfigEnable, 13 | })) 14 | ssmsvc := ssm.New(sess, aws.NewConfig().WithRegion(region)) 15 | param, err := ssmsvc.GetParameter(&ssm.GetParameterInput{ 16 | Name: aws.String(path), 17 | WithDecryption: aws.Bool(false), 18 | }) 19 | if err != nil { 20 | panic(err) 21 | } 22 | 23 | passwd = *param.Parameter.Value 24 | return 25 | } 26 | -------------------------------------------------------------------------------- /static/.gitignore: -------------------------------------------------------------------------------- 1 | gen -------------------------------------------------------------------------------- /templates/.gitignore: -------------------------------------------------------------------------------- 1 | gen -------------------------------------------------------------------------------- /ui/.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | node_modules 3 | /dist 4 | 5 | # local env files 6 | .env.local 7 | .env.*.local 8 | 9 | # Log files 10 | npm-debug.log* 11 | yarn-debug.log* 12 | yarn-error.log* 13 | 14 | # Editor directories and files 15 | .idea 16 | .vscode 17 | *.suo 18 | *.ntvs* 19 | *.njsproj 20 | *.sln 21 | *.sw* 22 | -------------------------------------------------------------------------------- /ui/babel.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | presets: [ 3 | '@vue/app' 4 | ] 5 | } 6 | -------------------------------------------------------------------------------- /ui/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "metahub", 3 | "version": "0.2.11", 4 | "private": true, 5 | "scripts": { 6 | "serve": "vue-cli-service serve", 7 | "build": "vue-cli-service build", 8 | "lint": "vue-cli-service lint" 9 | }, 10 | "dependencies": { 11 | "axios": "^0.18.1", 12 | "core-js": "^2.6.5", 13 | "vue": "^2.6.6", 14 | "vue-authenticate": "^1.3.4", 15 | "vue-router": "^3.0.3", 16 | "vuetify": "^1.5.5" 17 | }, 18 | "devDependencies": { 19 | "@vue/cli-plugin-babel": "^3.5.0", 20 | "@vue/cli-plugin-eslint": "^3.5.0", 21 | "@vue/cli-service": "^3.5.0", 22 | "babel-eslint": "^10.0.1", 23 | "eslint": "^5.8.0", 24 | "eslint-plugin-vue": "^5.0.0", 25 | "optimist": "*", 26 | "stylus": "^0.54.5", 27 | "stylus-loader": "^3.0.1", 28 | "vue-cli-plugin-vuetify": "^0.5.0", 29 | "vue-template-compiler": "^2.5.21", 30 | "vuetify-loader": "^1.0.5" 31 | }, 32 | "eslintConfig": { 33 | "root": true, 34 | "env": { 35 | "node": true 36 | }, 37 | "extends": [ 38 | "plugin:vue/essential", 39 | "eslint:recommended" 40 | ], 41 | "rules": {}, 42 | "parserOptions": { 43 | "parser": "babel-eslint" 44 | } 45 | }, 46 | "postcss": { 47 | "plugins": { 48 | "autoprefixer": {} 49 | } 50 | }, 51 | "browserslist": [ 52 | "> 1%", 53 | "last 2 versions", 54 | "not ie <= 8" 55 | ] 56 | } 57 | -------------------------------------------------------------------------------- /ui/public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/qnib/metahub/f57aa1d9fb8b05564efa233a663c499bb6d46b12/ui/public/favicon.ico -------------------------------------------------------------------------------- /ui/public/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | MetaHub 9 | 10 | 11 | 12 | 13 | 16 |
17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /ui/src/App.vue: -------------------------------------------------------------------------------- 1 | 58 | 59 | 99 | -------------------------------------------------------------------------------- /ui/src/components/LoginDialog.vue: -------------------------------------------------------------------------------- 1 | 25 | 26 | 61 | 62 | 64 | -------------------------------------------------------------------------------- /ui/src/main.js: -------------------------------------------------------------------------------- 1 | import Vue from 'vue' 2 | Vue.config.productionTip = false 3 | 4 | import './plugins/vuetify' 5 | import './plugins/axios' 6 | // if ENV without-accounts==false 7 | // per default is accounts included 8 | //import './plugins/account' 9 | import './plugins/fakeAccount' 10 | // endif 11 | import './mixins/features' 12 | 13 | import router from './router' 14 | import App from './App.vue' 15 | new Vue({ 16 | router, 17 | render: h => h(App), 18 | }).$mount('#app') 19 | -------------------------------------------------------------------------------- /ui/src/mixins/features.js: -------------------------------------------------------------------------------- 1 | import Vue from 'vue' 2 | 3 | Vue.mixin({ 4 | methods: { 5 | getCommonFeatures() { 6 | return [ 7 | { header: "Size" }, 8 | { name: "size:9xl", title: "9xlarge", group: "Size" }, 9 | { name: "size:12xl", title: "12xlarge", group: "Size" }, 10 | { name: "size:16xl", title: "16xlarge", group: "Size" }, 11 | { name: "size:18xl", title: "18xlarge", group: "Size" }, 12 | { name: "size:24xl", title: "24xlarge", group: "Size" }, 13 | { header: "Instance" }, 14 | { name: "instance:c5", title: "c5", group: "Instances" }, 15 | { name: "instance:c5n", title: "c5n", group: "Instances" }, 16 | { name: "instance:c5a", title: "c5a", group: "Instances" }, 17 | { name: "instance:c6g", title: "c6", group: "Instances" }, 18 | { header: "Hyperthreading" }, 19 | { name: "ht:on", title: "On", group: "Hyperthreading" }, 20 | { name: "ht:off", title: "Off", group: "Hyperthreading" }, 21 | { header: "MPI" }, 22 | { name: "mpi:mpich", title: "MPICH", group: "MPI" }, 23 | { name: "mpi:intelmpi", title: "IntelMPI", group: "MPI" }, 24 | { name: "mpi:openmpi", title: "OpenMPI", group: "MPI" }, 25 | ] 26 | }, 27 | formatFeature(name) { 28 | const list = this.getCommonFeatures(); 29 | for (var i in list) { 30 | const f = list[i]; 31 | if (f.name == name) { 32 | return f.title; 33 | } 34 | } 35 | return name; 36 | }, 37 | } 38 | }); 39 | -------------------------------------------------------------------------------- /ui/src/plugins/account/Account.vue: -------------------------------------------------------------------------------- 1 | 25 | 26 | 106 | 107 | 109 | -------------------------------------------------------------------------------- /ui/src/plugins/account/index.js: -------------------------------------------------------------------------------- 1 | import Vue from 'vue' 2 | import VueAuthenticate from 'vue-authenticate' 3 | 4 | Vue.use(VueAuthenticate, { 5 | providers: { 6 | github: { 7 | clientId: '65d9c15a3eb4e0afdd01', 8 | }, 9 | google: { 10 | clientId: '936040293434-i3m9p4km8it5np2bs253a7rvedchofs6.apps.googleusercontent.com', 11 | } 12 | } 13 | }) 14 | 15 | import Account from "./Account"; 16 | 17 | Vue.use({ 18 | install(vue) { 19 | const root = new Vue({ 20 | data: {}, 21 | render: createElement => createElement(Account), 22 | mounted() { 23 | vue.prototype.$account = this.$root.$children[0]; 24 | }, 25 | }) 26 | root.$mount(document.body.appendChild(document.createElement('div'))) 27 | } 28 | }) -------------------------------------------------------------------------------- /ui/src/plugins/axios.js: -------------------------------------------------------------------------------- 1 | import Vue from 'vue' 2 | import VueAxios from 'vue-axios' 3 | import axios from 'axios' 4 | 5 | Vue.use(VueAxios, axios) 6 | -------------------------------------------------------------------------------- /ui/src/plugins/fakeAccount/Account.vue: -------------------------------------------------------------------------------- 1 | 4 | 5 | 26 | 27 | 29 | -------------------------------------------------------------------------------- /ui/src/plugins/fakeAccount/index.js: -------------------------------------------------------------------------------- 1 | import Vue from 'vue' 2 | 3 | //import axios from '../axios.js' 4 | import axios from 'axios'; 5 | 6 | import Account from "./Account"; 7 | 8 | Vue.use({ 9 | install(vue) { 10 | axios.interceptors.request.use(function (config) { 11 | config.headers.Authorization = 'Bearer metahub'; 12 | return config; 13 | }, function (error) { 14 | return Promise.reject(error); 15 | }); 16 | 17 | const root = new Vue({ 18 | data: {}, 19 | render: createElement => createElement(Account), 20 | mounted() { 21 | vue.prototype.$account = this.$root.$children[0]; 22 | }, 23 | }) 24 | root.$mount(document.body.appendChild(document.createElement('div'))) 25 | } 26 | }) -------------------------------------------------------------------------------- /ui/src/plugins/vuetify.js: -------------------------------------------------------------------------------- 1 | import Vue from 'vue' 2 | import Vuetify from 'vuetify/lib' 3 | import 'vuetify/src/stylus/app.styl' 4 | 5 | Vue.use(Vuetify, { 6 | iconfont: 'md', 7 | }) 8 | -------------------------------------------------------------------------------- /ui/src/router.js: -------------------------------------------------------------------------------- 1 | import Vue from 'vue' 2 | import Router from 'vue-router' 3 | 4 | Vue.use(Router) 5 | 6 | import Welcome from "./views/Welcome"; 7 | import MachineTypes from "./views/MachineTypes"; 8 | import MachineType from "./views/MachineType" 9 | 10 | const router = new Router({ 11 | mode: 'hash', 12 | routes: [ 13 | { 14 | name: "welcome", path: '/', component: Welcome, meta: { 15 | title: "Welcome", 16 | } 17 | }, 18 | { 19 | name: "machine-types", 20 | path: '/machinetypes', 21 | components: { 22 | default: MachineTypes, 23 | }, 24 | meta: { 25 | title: "Machine Types", 26 | requiresAuth: true, 27 | } 28 | }, 29 | { 30 | name: "edit-machine-type", 31 | path: '/machinetypes/edit/:id', 32 | component: MachineType, 33 | meta: { 34 | title: "Machine Type details", 35 | requiresAuth: true, 36 | } 37 | }, 38 | { 39 | name: "new-machine-type", 40 | path: '/machinetypes/new', 41 | component: MachineType, 42 | meta: { 43 | title: "Machine Type details", 44 | requiresAuth: true, 45 | } 46 | }, 47 | ] 48 | }) 49 | router.beforeEach((to, from, next) => { 50 | if (router.app.$account == undefined) { 51 | next(); 52 | return; 53 | } 54 | if (to.matched.some(record => record.meta.requiresAuth)) { 55 | const account = router.app.$account; 56 | if (account.isLoggedIn()) { 57 | next(); 58 | } else { 59 | account.login(function () { 60 | if (!account.isLoggedIn()) { 61 | next(false); 62 | return; 63 | } 64 | next(); 65 | }); 66 | } 67 | } else { 68 | next(); 69 | } 70 | }) 71 | export default router; -------------------------------------------------------------------------------- /ui/src/views/MachineType.vue: -------------------------------------------------------------------------------- 1 | 107 | 108 | 294 | 295 | 314 | -------------------------------------------------------------------------------- /ui/src/views/MachineTypes.vue: -------------------------------------------------------------------------------- 1 | 44 | 45 | 84 | 85 | 87 | -------------------------------------------------------------------------------- /ui/src/views/Welcome.vue: -------------------------------------------------------------------------------- 1 | 4 | 5 | 10 | 11 | 13 | -------------------------------------------------------------------------------- /ui/vue.config.js: -------------------------------------------------------------------------------- 1 | const path = require('path') 2 | const argv = require('optimist').argv 3 | let config = {} 4 | let command = argv['_'][0] 5 | switch (command) { 6 | case 'build': 7 | config = { 8 | publicPath: '/static/gen', 9 | assetsDir: './', 10 | outputDir: '../static/gen', 11 | pages: { 12 | app: { 13 | entry: 'src/main.js', 14 | template: 'public/index.html', 15 | filename: path.resolve('../templates/gen', 'index.html') 16 | } 17 | } 18 | } 19 | break 20 | case 'serve': 21 | config = { 22 | devServer: { 23 | proxy: 'http://localhost:8080' 24 | } 25 | } 26 | } 27 | module.exports = config --------------------------------------------------------------------------------