├── .commitsar.yml ├── .dockerignore ├── .gitattributes ├── .github └── workflows │ ├── ci.yml │ └── clean.yml ├── .gitignore ├── .golangci.yaml ├── LICENSE ├── Makefile ├── README.md ├── cmd └── server │ └── server.go ├── deploy └── manifests │ ├── docker-compose.yaml │ └── kubernetes.yaml ├── go.mod ├── go.sum ├── hack ├── boilerplate │ └── go.txt ├── build.sh ├── ci.sh ├── deps.sh ├── lib │ ├── image.sh │ ├── init.sh │ ├── log.sh │ ├── style.sh │ ├── target.sh │ ├── util.sh │ └── version.sh ├── lint.sh ├── package.sh └── test.sh ├── pack └── server │ └── image │ └── Dockerfile └── pkg ├── apis ├── config │ └── config.go ├── debug │ └── handler.go ├── logger.go ├── measure │ └── handler.go ├── provider │ ├── handler.go │ ├── handler_view.go │ └── handler_view_test.go ├── runtime │ ├── bind │ │ ├── binder.go │ │ └── mapping.go │ ├── handler.go │ ├── metrics.go │ ├── middleware_error.go │ ├── middleware_filter.go │ ├── middleware_flowcontrol.go │ ├── middleware_observation.go │ ├── middleware_recovery.go │ ├── middleware_route.go │ ├── openapi │ │ └── extension.go │ ├── request.go │ ├── request_stream.go │ ├── response.go │ ├── router.go │ ├── router_advice.go │ ├── router_options.go │ ├── router_route.go │ ├── router_route_test.go │ ├── router_static.go │ ├── router_stream.go │ ├── router_validation.go │ ├── scheme_route.go │ ├── scheme_route_extension.go │ ├── scheme_route_openapi.go │ ├── scheme_route_test.go │ └── swagger-ui │ │ ├── favicon-16x16.png │ │ ├── favicon-32x32.png │ │ ├── swagger-ui-bundle.js │ │ ├── swagger-ui-bundle.js.map │ │ ├── swagger-ui.css │ │ └── swagger-ui.css.map ├── server.go └── setup.go ├── consts └── consts.go ├── database ├── bolt.go ├── bolt_linux.go ├── bolt_nolinux.go ├── driver.go ├── health.go └── metrics.go ├── download ├── client.go └── http.go ├── health ├── registry.go └── validate.go ├── metric ├── index.go └── registry.go ├── provider ├── metadata │ └── service.go ├── service.go └── storage │ └── service.go ├── registry └── service.go ├── server ├── cmd.go ├── init.go ├── init_health_checkers.go ├── init_metric_collectors.go ├── init_tasks.go ├── init_test.go ├── runner.go └── start_apis.go └── tasks └── provider └── task.go /.commitsar.yml: -------------------------------------------------------------------------------- 1 | commits: 2 | disabled: false 3 | strict: false 4 | limit: 100 5 | all: false 6 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | # Files 2 | .DS_Store 3 | *.lock 4 | *.test 5 | *.out 6 | *.swp 7 | *.swo 8 | # *.db 9 | *.exe 10 | *.exe~ 11 | *.dll 12 | *.so 13 | *.dylib 14 | *.log 15 | 16 | # Dirs 17 | .idea/ 18 | .vscode/ 19 | .kube/ 20 | .terraform/ 21 | .vagrant/ 22 | .bundle/ 23 | .cache/ 24 | .docker/ 25 | .entc/ 26 | #.sbin/ 27 | #.dist/ 28 | log/ 29 | certs/ 30 | tmp/ 31 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | * text=auto eol=lf 2 | 3 | **/*.pb.go linguist-generated=true 4 | staging/**/go.sum linguist-generated=true 5 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | permissions: 4 | contents: read 5 | pull-requests: read 6 | actions: read 7 | 8 | env: 9 | REPO: "sealio" 10 | VERSION: "${{ github.ref_name }}" 11 | BUILD_PLATFORMS: "linux/amd64,linux/arm64" 12 | PARALLELIZE: "false" 13 | GO_VERSION: "1.21.13" 14 | 15 | defaults: 16 | run: 17 | shell: bash 18 | 19 | on: 20 | workflow_dispatch: { } 21 | push: 22 | tags: 23 | - "v*.*.*" 24 | branches: 25 | - "main" 26 | paths-ignore: 27 | - "docs/**" 28 | - "**.md" 29 | - "**.mdx" 30 | - "**.png" 31 | - "**.jpg" 32 | pull_request: 33 | branches: 34 | - "main" 35 | paths-ignore: 36 | - "docs/**" 37 | - "**.md" 38 | - "**.mdx" 39 | - "**.png" 40 | - "**.jpg" 41 | 42 | jobs: 43 | build: 44 | timeout-minutes: 60 45 | runs-on: ubuntu-22.04 46 | steps: 47 | - name: Checkout 48 | uses: actions/checkout@v3 49 | with: 50 | # checkout the whole histories for commitsar, 51 | # currently commitsar needs full git objects to work correctly. 52 | fetch-depth: 0 53 | persist-credentials: false 54 | - name: Setup Go 55 | timeout-minutes: 15 56 | uses: actions/setup-go@v4 57 | with: 58 | go-version: "${{ env.GO_VERSION }}" 59 | cache-dependency-path: | 60 | **/go.sum 61 | - name: Setup Toolbox 62 | timeout-minutes: 5 63 | uses: actions/cache@v4 64 | with: 65 | # restore/save service binaries, e.g. goimports, golangci-lint, commitsar. 66 | key: toolbox-${{ runner.os }} 67 | path: | 68 | ${{ github.workspace }}/.sbin 69 | - name: Build 70 | run: make ci 71 | env: 72 | LINT_DIRTY: "true" 73 | PACKAGE_BUILD: "false" 74 | - name: Archive Publish Result 75 | uses: actions/cache/save@v4 76 | with: 77 | # save package resources, e.g. go build result, downloaded UI, entrypoint script. 78 | key: archive-${{ runner.os }}-${{ github.sha }} 79 | path: | 80 | ${{ github.workspace }}/.dist/package 81 | 82 | publish: 83 | needs: 84 | - build 85 | permissions: 86 | contents: write 87 | actions: read 88 | id-token: write 89 | timeout-minutes: 60 90 | runs-on: ubuntu-22.04 91 | strategy: 92 | matrix: 93 | include: 94 | - target: hermitcrab 95 | task: server 96 | steps: 97 | - name: Checkout 98 | uses: actions/checkout@v4 99 | with: 100 | fetch-depth: 1 101 | persist-credentials: false 102 | - name: Setup QEMU 103 | uses: docker/setup-qemu-action@v3 104 | with: 105 | image: tonistiigi/binfmt:qemu-v7.0.0 106 | platforms: "arm64" 107 | - name: Setup Buildx 108 | uses: docker/setup-buildx-action@v3 109 | - name: Login DockerHub 110 | if: ${{ github.event_name != 'pull_request' }} 111 | uses: docker/login-action@v3 112 | with: 113 | username: ${{ secrets.CI_DOCKERHUB_USERNAME }} 114 | password: ${{ secrets.CI_DOCKERHUB_PASSWORD }} 115 | - name: Unarchive Publish Result 116 | timeout-minutes: 5 117 | uses: actions/cache/restore@v4 118 | with: 119 | # restore package resources, e.g. go build result, downloaded UI, entrypoint script. 120 | key: archive-${{ runner.os }}-${{ github.sha }} 121 | path: | 122 | ${{ github.workspace }}/.dist/package 123 | - name: Get Metadata 124 | id: metadata 125 | uses: docker/metadata-action@v5 126 | with: 127 | images: ${{ env.REPO }}/${{ matrix.target }} 128 | - name: Package 129 | uses: docker/build-push-action@v5 130 | id: package 131 | with: 132 | push: ${{ github.event_name != 'pull_request' }} 133 | file: .dist/package/${{ matrix.target }}/${{ matrix.task }}/image/Dockerfile 134 | context: .dist/package/${{ matrix.target }}/${{ matrix.task }}/ 135 | platforms: ${{ env.BUILD_PLATFORMS }} 136 | tags: ${{ steps.metadata.outputs.tags }} 137 | labels: ${{ steps.metadata.outputs.labels }} 138 | # configure build cache, 139 | # ref to https://github.com/moby/buildkit/tree/v0.11.5#registry-push-image-and-cache-separately. 140 | cache-from: | 141 | type=registry,ref=${{ env.REPO }}/build-cache:${{ matrix.target }}-${{ matrix.task }} 142 | cache-to: | 143 | ${{ github.event_name != 'pull_request' && format('type=registry,mode=max,oci-mediatypes=false,compression=gzip,ref={0}/build-cache:{1}-{2},ignore-error=true', env.REPO, matrix.target, matrix.task) || '' }} 144 | - name: Setup Cosign 145 | if: ${{ github.event_name != 'pull_request' }} 146 | uses: sigstore/cosign-installer@v3.3.0 147 | with: 148 | cosign-release: v2.2.2 149 | - name: Prove 150 | if: ${{ github.event_name != 'pull_request' }} 151 | run: | 152 | set -euo pipefail 153 | 154 | # login 155 | cosign login "docker.io" -u "${DOCKERHUB_USERNAME}" -p "${DOCKERHUB_PASSWORD}" 156 | 157 | # prove 158 | curl -o slsa-generator --retry 3 --retry-all-errors --retry-delay 3 -sSfL \ 159 | "https://github.com/slsa-framework/slsa-github-generator/releases/download/${SLSA_GITHUB_GENERATOR_VERSION}/${SLSA_GITHUB_GENERATOR}" 160 | chmod a+x slsa-generator 161 | predicate_name="predicate.json" 162 | ./slsa-generator generate --predicate="${predicate_name}" 163 | cosign attest --predicate="${predicate_name}" \ 164 | --type slsaprovenance \ 165 | --yes \ 166 | "${UNTRUSTED_IMAGE}@${UNTRUSTED_DIGEST}" 167 | env: 168 | COSIGN_EXPERIMENTAL: "1" 169 | SLSA_GITHUB_GENERATOR: "slsa-generator-container-linux-amd64" 170 | SLSA_GITHUB_GENERATOR_VERSION: "v1.5.0" 171 | GITHUB_CONTEXT: "${{ toJSON(github) }}" 172 | UNTRUSTED_IMAGE: "${{ env.REPO }}/${{ matrix.target }}" 173 | UNTRUSTED_DIGEST: "${{ steps.package.outputs.digest }}" 174 | DOCKERHUB_USERNAME: "${{ secrets.CI_DOCKERHUB_USERNAME }}" 175 | DOCKERHUB_PASSWORD: "${{ secrets.CI_DOCKERHUB_PASSWORD }}" 176 | continue-on-error: true 177 | -------------------------------------------------------------------------------- /.github/workflows/clean.yml: -------------------------------------------------------------------------------- 1 | name: Clean 2 | 3 | permissions: 4 | contents: write 5 | pull-requests: read 6 | actions: write 7 | 8 | defaults: 9 | run: 10 | shell: bash 11 | 12 | on: 13 | schedule: 14 | - cron: '0 */12 * * *' 15 | workflow_dispatch: { } 16 | 17 | jobs: 18 | clean: 19 | timeout-minutes: 5 20 | runs-on: ubuntu-22.04 21 | steps: 22 | - name: Checkout 23 | uses: actions/checkout@v3 24 | with: 25 | fetch-depth: 1 26 | persist-credentials: false 27 | - name: Remove Cache 28 | uses: actions/github-script@v6 29 | with: 30 | # clean up caches, 31 | # ref to https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#force-deleting-cache-entries, 32 | # and https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#restrictions-for-accessing-a-cache. 33 | script: | 34 | const owner = context.repo.owner 35 | const repo = context.repo.repo 36 | var deleteCaches = new Array() 37 | 38 | // get candidate items. 39 | const { data: cs } = await github.rest.actions.getActionsCacheList({ 40 | owner: owner, 41 | repo: repo, 42 | }); 43 | for (const c of cs.actions_caches) { 44 | // clean closed pull request's caches. 45 | if (c.ref.match(/^refs\/pull\/.*$/)) { 46 | var prNum = c.ref.replace(/[^\d]/g, "") 47 | const { data: pr } = await github.rest.pulls.get({ 48 | owner: owner, 49 | repo: repo, 50 | pull_number: prNum, 51 | }) 52 | if (pr.state === 'closed') { 53 | deleteCaches.push(c) 54 | } 55 | continue 56 | } 57 | // do not clean toolbox caches. 58 | if (c.key.match(/^toolbox-.*$/)) { 59 | continue 60 | } 61 | // clean push archived caches. 62 | if (c.key.match(/^archive-.*$/)) { 63 | deleteCaches.push(c) 64 | continue 65 | } 66 | // clean stale built caches. 67 | if (!c.key.match(/^setup-go-.*-${{ hashFiles('**/go.sum') }}$/)) { 68 | deleteCaches.push(c) 69 | continue 70 | } 71 | } 72 | 73 | // delete 74 | for (const c of deleteCaches) { 75 | await github.rest.actions.deleteActionsCacheById({ 76 | owner: owner, 77 | repo: repo, 78 | cache_id: c.id, 79 | }) 80 | console.log(`cleaned cache "${c.key}"`) 81 | } 82 | continue-on-error: true 83 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Files 2 | .DS_Store 3 | *.lock 4 | *.test 5 | *.out 6 | *.swp 7 | *.swo 8 | *.db 9 | *.exe 10 | *.exe~ 11 | *.dll 12 | *.so 13 | *.dylib 14 | *.log 15 | go.work 16 | go.work.* 17 | 18 | # Dirs 19 | .idea/ 20 | .vscode/ 21 | .kube/ 22 | .terraform/ 23 | .vagrant/ 24 | .bundle/ 25 | .cache/ 26 | .docker/ 27 | .entc/ 28 | .sbin/ 29 | .dist/ 30 | log/ 31 | certs/ 32 | tmp/ 33 | -------------------------------------------------------------------------------- /.golangci.yaml: -------------------------------------------------------------------------------- 1 | run: 2 | timeout: 10m 3 | tests: true 4 | skip-files: 5 | - "doc.go" 6 | modules-download-mode: readonly 7 | 8 | # output configuration options 9 | output: 10 | # Format: colored-line-number|line-number|json|tab|checkstyle|code-climate|junit-xml|github-actions 11 | # 12 | # Multiple can be specified by separating them by comma, output can be provided 13 | # for each of them by separating format name and path by colon symbol. 14 | # Output path can be either `stdout`, `stderr` or path to the file to write to. 15 | # Example: "checkstyle:report.json,colored-line-number" 16 | # 17 | # Default: colored-line-number 18 | format: colored-line-number 19 | # Print lines of code with issue. 20 | # Default: true 21 | print-issued-lines: true 22 | # Print linter name in the end of issue text. 23 | # Default: true 24 | print-linter-name: true 25 | # Make issues output unique by line. 26 | # Default: true 27 | uniq-by-line: true 28 | # Add a prefix to the output file references. 29 | # Default is no prefix. 30 | path-prefix: "" 31 | # Sort results by: filepath, line and column. 32 | sort-results: true 33 | 34 | linters: 35 | disable-all: true 36 | enable: 37 | - asciicheck 38 | - bidichk 39 | - decorder 40 | - durationcheck 41 | - errcheck 42 | - errname 43 | - errorlint 44 | - exportloopref 45 | - godot 46 | - goconst 47 | - gofumpt 48 | - gocritic 49 | - gosimple 50 | - gosec 51 | - govet 52 | - ineffassign 53 | - lll 54 | - makezero 55 | - misspell 56 | - misspell 57 | - nakedret 58 | - nilerr 59 | - prealloc 60 | - predeclared 61 | - revive 62 | - staticcheck 63 | - stylecheck 64 | - typecheck 65 | - unconvert 66 | - unparam 67 | - unused 68 | - usestdlibvars 69 | - whitespace 70 | 71 | # enable complexity linters 72 | # - dupl 73 | # - gocognit 74 | # - gocyclo 75 | # - funlen 76 | 77 | linters-settings: 78 | staticcheck: 79 | checks: ["all", "-SA1019", "-SA2002", "-SA5008"] 80 | stylecheck: 81 | checks: ["all", "-ST1003"] 82 | gosec: 83 | severity: "low" 84 | confidence: "low" 85 | excludes: 86 | - G101 87 | - G112 88 | revive: 89 | rules: 90 | - name: var-naming 91 | disabled: true 92 | arguments: 93 | - ["HTTP", "ID", "TLS", "TCP", "UDP", "API", "CA", "URL", "DNS"] 94 | godot: 95 | # Comments to be checked: `declarations`, `toplevel`, or `all`. 96 | # Default: declarations 97 | scope: all 98 | # List of regexps for excluding particular comment lines from check. 99 | # Default: [] 100 | exclude: 101 | # Exclude todo and fixme comments. 102 | - "^fixme:" 103 | - "^todo:" 104 | # Check that each sentence ends with a period. 105 | # Default: true 106 | period: true 107 | # Check that each sentence starts with a capital letter. 108 | # Default: false 109 | capital: true 110 | lll: 111 | # max line length, lines longer will be reported. Default is 120. 112 | # '\t' is counted as 1 character by default, and can be changed with the tab-width option 113 | line-length: 150 114 | # tab width in spaces. Default to 1. 115 | tab-width: 1 116 | goconst: 117 | # Minimal length of string constant. 118 | # Default: 3 119 | min-len: 3 120 | # Minimum occurrences of constant string count to trigger issue. 121 | # Default: 3 122 | min-occurrences: 3 123 | misspell: 124 | # Correct spellings using locale preferences for US or UK. 125 | # Default is to use a neutral variety of English. 126 | # Setting locale to US will correct the British spelling of 'colour' to 'color'. 127 | locale: US 128 | unparam: 129 | # Inspect exported functions, default is false. Set to true if no external program/library imports your code. 130 | # XXX: if you enable this setting, unparam will report a lot of false-positives in text editors: 131 | # if it's called for subdir of a project it can't find external interfaces. All text editor integrations 132 | # with golangci-lint call it on a directory with the changed file. 133 | check-exported: false 134 | unused: 135 | # Mark all struct fields that have been written to as used. 136 | # Default: true 137 | field-writes-are-uses: true 138 | # Treat IncDec statement (e.g. `i++` or `i--`) as both read and write operation instead of just write. 139 | # Default: false 140 | post-statements-are-reads: true 141 | # Mark all exported identifiers as used. 142 | # Default: true 143 | exported-is-used: true 144 | # Mark all exported fields as used. 145 | # default: true 146 | exported-fields-are-used: true 147 | # Mark all function parameters as used. 148 | # default: true 149 | parameters-are-used: true 150 | # Mark all local variables as used. 151 | # default: true 152 | local-variables-are-used: true 153 | # Mark all identifiers inside generated files as used. 154 | # Default: true 155 | generated-is-used: true 156 | errorlint: 157 | # Check whether fmt.Errorf uses the %w verb for formatting errors. See the readme for caveats 158 | errorf: true 159 | # Check for plain type assertions and type switches 160 | asserts: true 161 | # Check for plain error comparisons 162 | comparison: true 163 | makezero: 164 | always: false 165 | gosimple: 166 | go: "1.19" 167 | checks: ["all"] 168 | nakedret: 169 | max-func-lines: 60 170 | usestdlibvars: 171 | # Suggest the use of http.MethodXX 172 | # Default: true 173 | http-method: true 174 | # Suggest the use of http.StatusXX 175 | # Default: true 176 | http-status-code: true 177 | # Suggest the use of time.Weekday 178 | # Default: true 179 | time-weekday: true 180 | # Suggest the use of time.Month 181 | # Default: false 182 | time-month: true 183 | # Suggest the use of time.Layout 184 | # Default: false 185 | time-layout: true 186 | # Suggest the use of crypto.Hash 187 | # Default: false 188 | crypto-hash: true 189 | decorder: 190 | dec-order: 191 | - const 192 | - var 193 | - func 194 | disable-init-func-first-check: false 195 | disable-dec-order-check: true 196 | 197 | issues: 198 | exclude-rules: 199 | - path: _test\.go 200 | linters: 201 | - errcheck 202 | - gosec 203 | - rowserrcheck 204 | - makezero 205 | - lll 206 | - funlen 207 | - wsl 208 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | SHELL := /bin/bash 2 | 3 | # Borrowed from https://stackoverflow.com/questions/18136918/how-to-get-current-relative-directory-of-your-makefile 4 | curr_dir := $(patsubst %/,%,$(dir $(abspath $(lastword $(MAKEFILE_LIST))))) 5 | 6 | # Borrowed from https://stackoverflow.com/questions/2214575/passing-arguments-to-make-run 7 | rest_args := $(wordlist 2, $(words $(MAKECMDGOALS)), $(MAKECMDGOALS)) 8 | $(eval $(rest_args):;@:) 9 | 10 | targets := $(shell ls $(curr_dir)/hack | grep '.sh' | sed 's/\.sh//g') 11 | $(targets): 12 | @$(curr_dir)/hack/$@.sh $(rest_args) 13 | 14 | help: 15 | # 16 | # Usage: 17 | # 18 | # * [dev] `make deps`, get dependencies. 19 | # 20 | # * [dev] `make lint`, check style. 21 | # - `BUILD_TAGS="jsoniter" make lint` check with specified tags. 22 | # - `LINT_DIRTY=true make lint` verify whether the code tree is dirty. 23 | # 24 | # * [dev] `make test`, execute unit testing. 25 | # - `BUILD_TAGS="jsoniter" make test` test with specified tags. 26 | # 27 | # * [dev] `make build`, execute cross building. 28 | # - `VERSION=vX.y.z+l.m make build` build all targets with vX.y.z+l.m version. 29 | # - `OS=linux ARCH=arm64 make build` build all targets run on linux/arm64 arch. 30 | # - `BUILD_TAGS="jsoniter" make build` build with specified tags. 31 | # - `BUILD_PLATFORMS="linux/amd64,linux/arm64" make build` do multiple platforms go build. 32 | # 33 | # * [dev] `make package`, embed running resources into a Docker image on one platform. 34 | # - `REPO=xyz make package` package all targets named with xyz repository. 35 | # - `VERSION=vX.y.z+l.m make package` package all targets named with vX.y.z-l.m tag. 36 | # - `TAG=main make package` package all targets named with main tag. 37 | # - `OS=linux ARCH=arm64 make package` package all targets run on linux/arm64 arch. 38 | # - `PACKAGE_BUILD=false make package` prepare build resource but disable docker build. 39 | # - `DOCKER_USERNAME=... DOCKER_PASSWORD=... PACKAGE_PUSH=true make package` execute docker push after build. 40 | # 41 | # * [ci] `make ci`, execute `make deps`, `make lint`, `make test`, `make build` and `make package`. 42 | # - `CI_CHECK=false make ci` only execute `make build` and `make package`. 43 | # - `CI_PUBLISH=false make ci` only execute `make deps`, `make lint` and `make test`. 44 | # 45 | @echo 46 | 47 | .DEFAULT_GOAL := build 48 | .PHONY: $(targets) 49 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Hermit Crab 2 | 3 | > tl;dr: Available Terraform/OpenTofu Provider network mirroring service. 4 | 5 | [![](https://goreportcard.com/badge/github.com/seal-io/hermitcrab)](https://goreportcard.com/report/github.com/seal-io/hermitcrab) 6 | [![](https://img.shields.io/github/actions/workflow/status/seal-io/hermitcrab/ci.yml?label=ci)](https://github.com/seal-io/hermitcrab/actions) 7 | [![](https://img.shields.io/docker/image-size/sealio/hermitcrab/main?label=docker)](https://hub.docker.com/r/sealio/hermitcrab/tags) 8 | [![](https://img.shields.io/github/v/tag/seal-io/hermitcrab?label=release)](https://github.com/seal-io/hermitcrab/releases) 9 | [![](https://img.shields.io/github/license/seal-io/hermitcrab?label=license)](https://github.com/seal-io/hermitcrab#license) 10 | 11 | Hermit Crab provides a stable and reliable [Terraform](https://registry.terraform.io/browse/providers)/[OpenTofu](https://opentofu.org/registry/) Provider network mirror service. 12 | 13 | This tool is maintained by [Seal](https://github.com/seal-io). 14 | 15 | ```mermaid 16 | sequenceDiagram 17 | actor tf as terraform init 18 | participant hc as Hermit Crab 19 | participant tfreg as Terraform Registry 20 | participant stg as Provider Package Storage 21 | 22 | tf ->> hc: list available versions 23 | alt not found 24 | hc ->> tfreg: list available versions 25 | tfreg -->> hc: 26 | hc ->> hc: save 27 | end 28 | hc -->> tf: {"versions":{"2.0.0":{},"2.0.1":{}}} 29 | 30 | tf ->> hc: list available installation packages 31 | alt not found 32 | par 33 | hc ->> tfreg: find darwin/amd64 provider package 34 | tfreg -->> hc: 35 | hc ->> hc: save 36 | and 37 | hc ->> tfreg: find linux/amd64 provider package 38 | tfreg -->> hc: 39 | hc ->> hc: save 40 | end 41 | end 42 | hc -->> tf: {"archives": {"darwin_amd64":{},"linux_amd64":{}} 43 | 44 | tf ->> hc: download platform provider package, like darwin/amd64 45 | alt not found 46 | par not downloading 47 | hc ->> stg: download 48 | stg -->> hc: 49 | hc ->> hc: store 50 | and downloading 51 | hc ->> hc: wait until downloading finished 52 | end 53 | end 54 | hc -->> tf: ***.zip 55 | 56 | ``` 57 | 58 | ## Background 59 | 60 | When we drive [Terraform](https://www.terraform.io/) at some automation scenarios, like CI, automatic deployment, etc., we need to download the Provider plugins from the internet by `init` command. 61 | 62 | Depending on [Terraform Provider Registry Protocol](https://developer.hashicorp.com/terraform/internals/provider-registry-protocol), we may download a plugin that is not cached from an unstable networking remote. 63 | 64 | To mitigate the effect of unstable networking, there are two ways to solve this: [Implied Mirroring](https://developer.hashicorp.com/terraform/cli/config/config-file#implied-local-mirror-directories) and [Network Mirroring](https://developer.hashicorp.com/terraform/cli/config/config-file#network_mirror). 65 | 66 | As far as **Implied Mirroring** is concerned, it works well when a Provider matches the [Version Constraints](https://developer.hashicorp.com/terraform/language/expressions/version-constraints). However, this mode fails when the version is not cached in the local file directory. 67 | 68 | ``` 69 | ╷ 70 | │ Error: Failed to query available provider packages 71 | │ 72 | │ Could not retrieve the list of available versions for provider 73 | ``` 74 | 75 | What's even more troublesome is that if the version changes, we need to continuously maintain this local file directory. 76 | 77 | **Network Mirroring**, different from **Implied Mirroring**, can maintain all versions(including the latest) at a nearby network and allows distributed Terraform working agents to share the same mirroring package. 78 | 79 | ## Usage 80 | 81 | Hermit Crab implements the [Terraform](https://developer.hashicorp.com/terraform/internals/provider-registry-protocol)/[OpenTofu](https://opentofu.org/docs/internals/provider-network-mirror-protocol/) Provider Registry Protocol and acts as a mirroring service. 82 | 83 | Hermit Crab can be easily served through [Docker](https://www.docker.com/). 84 | 85 | ```shell 86 | docker run -d --restart=always -p 80:80 -p 443:443 sealio/hermitcrab 87 | ``` 88 | 89 | Hermit Crab saves the mirrored(downloaded) packages in the `/var/run/hermitcrab` directory by default, which can be persisted by mounting a volume. 90 | 91 | ```shell 92 | docker run -d --restart=always -p 80:80 -p 443:443 \ 93 | -v /tmp/hermitcrab:/var/run/hermitcrab \ 94 | sealio/hermitcrab 95 | ``` 96 | 97 | Hermit Crab manages the archives as the following layer structure, which is absolutely compatible with the output of [`terraform providers mirror`](https://developer.hashicorp.com/terraform/cli/commands/providers/mirror)/[`tofu providers mirror`](https://opentofu.org/docs/cli/commands/providers/mirror). 98 | 99 | ``` 100 | /var/run/hermitcrab/data/providers 101 | ├── / 102 | │ ├── / 103 | │ │ ├── / 104 | │ │ │ ├── terraform-provider-___.zip 105 | ``` 106 | 107 | Hermit Crab also can reuse the mirroring providers prepared by `terraform providers mirror`/`tofo providers mirror`. 108 | 109 | ```shell 110 | terraform providers mirror /tmp/providers-plugins 111 | 112 | docker run -d --restart=always -p 80:80 -p 443:443 \ 113 | -v /tmp/providers-plugins:/usr/share/terraform/providers/plugins \ 114 | sealio/hermitcrab 115 | ``` 116 | 117 | Terraform/OpenTofu Provider Network Mirror protocol wants [HTTPS](https://en.wikipedia.org/wiki/HTTPS) access, Hermit Crab provides multiple ways to achieve this. 118 | 119 | - Use the default self-signed certificate, no additional configuration is required. 120 | 121 | Since Terraform always verifies the certificate insecure or not, under this mode, we need to import the self-signed certificate into the trusted certificate store. 122 | 123 | ```shell 124 | # download the self-signed certificate 125 | echo quit | openssl s_client -showcerts -servername -connect 2>/dev/null | openssl x509 -outform PEM >server.pem 126 | ``` 127 | 128 | - Using [ACME](https://en.wikipedia.org/wiki/Automatic_Certificate_Management_Environment) to [gain a trusted certificate](https://letsencrypt.org/docs/challenge-types/), need a domain name and a DNS configuration. 129 | 130 | ```shell 131 | docker run -d --restart=always -p 80:80 -p 443:443 \ 132 | -e SERVER_TLS_AUTO_CERT_DOMAINS= \ 133 | sealio/hermitcrab 134 | ``` 135 | 136 | - Use a custom certificate. 137 | 138 | ```shell 139 | docker run -d --restart=always -p 80:80 -p 443:443 \ 140 | -v /:/etc/hermitcrab/ssl/key.pem \ 141 | -v /:/etc/hermitcrab/ssl/cert.pem \ 142 | -e SERVER_TLS_PRIVATE_KEY_FILE=/etc/hermitcrab/ssl/key.pem \ 143 | -e SERVER_TLS_CERT_FILE=/etc/hermitcrab/ssl/cert.pem \ 144 | sealio/hermitcrab 145 | ``` 146 | 147 | Also support to launch from Helm Chart. 148 | 149 | ```shell 150 | # latest version 151 | helm install my-release oci://ghcr.io/seal-io/helm-charts/hermitcrab 152 | # with specific version 153 | helm install my-release oci://ghcr.io/seal-io/helm-charts/hermitcrab --version 154 | ``` 155 | 156 | After setting up Hermit Crab, you can make the [Terraform](https://developer.hashicorp.com/terraform/cli/config/config-file)/[OpenTofu](https://opentofu.org/docs/cli/config/config-file/) CLI Configuration File as below to use the mirroring service. 157 | 158 | ```hcl 159 | provider_installation { 160 | network_mirror { 161 | url = "https://
/v1/providers/" 162 | } 163 | } 164 | ``` 165 | 166 | ## Notice 167 | 168 | Hermit Crab is not a [Terraform Registry](https://registry.terraform.io), although implementing these protocols is not difficult, there are many options that you can choose from, like [HashiCorp Terraform Enterprise](https://www.hashicorp.com/products/terraform/pricing/), [JFrog Artifactory](https://jfrog.com/help/r/jfrog-artifactory-documentation/terraform-registry), etc. 169 | 170 | Hermit Crab cannot mirror [Terraform Module](https://developer.hashicorp.com/terraform/internals/module-registry-protocol), since obtaining Terraform modules is diverse, like [Git](https://developer.hashicorp.com/terraform/language/modules/sources#generic-git-repository), [HTTP URLs](https://developer.hashicorp.com/terraform/language/modules/sources#http-urls), [S3 Bucket](https://developer.hashicorp.com/terraform/language/modules/sources#gcs-bucket) and so on, it's hard to provide a unified way to mirror them. 171 | 172 | Hermit Crab doesn't support rewriting the provider [hostname](https://developer.hashicorp.com/terraform/internals/provider-network-mirror-protocol#hostname), which is a rare case and may make template/module reusing difficult. One possible scenario is that there is a private Terraform Registry in your network, and you need to use the community template/module without any modification. 173 | 174 | Hermit Crab automatically synchronizes the in-use versions per 30 minutes, if the information update occurs during sleep, we can manually trigger the synchronization by sending a `PUT` request to `/v1/providers/sync`. 175 | 176 | Hermit Crab only performs a checksum verification on the downloaded archives. For archives that already exist in the implied or explicit directory, checksum verification is not performed. 177 | 178 | Hermit Crab allows downloading the archives matching `^terraform-provider-(?P[\w-]+)[_-](?P[\w|\\.]+)[_-](?P[a-z]+)[_-](?P[a-z0-9]+)([_-].*)?\.zip$`, but it is recommended to follow the [Terraform Release Rules](https://developer.hashicorp.com/terraform/registry/providers/publishing#manually-preparing-a-release). 179 | 180 | # License 181 | 182 | Copyright (c) 2023 [Seal, Inc.](https://seal.io) 183 | 184 | Licensed under the Apache License, Version 2.0 (the "License"); 185 | you may not use this file except in compliance with the License. 186 | You may obtain a copy of the License at [LICENSE](./LICENSE) file for details. 187 | 188 | Unless required by applicable law or agreed to in writing, software 189 | distributed under the License is distributed on an "AS IS" BASIS, 190 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 191 | See the License for the specific language governing permissions and 192 | limitations under the License. 193 | -------------------------------------------------------------------------------- /cmd/server/server.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | 6 | "github.com/seal-io/walrus/utils/clis" 7 | "github.com/seal-io/walrus/utils/log" 8 | "github.com/seal-io/walrus/utils/signals" 9 | 10 | "github.com/seal-io/hermitcrab/pkg/server" 11 | ) 12 | 13 | func main() { 14 | cmd := server.Command() 15 | 16 | app := clis.AsApp(cmd) 17 | if err := app.RunContext(signals.Handler(), os.Args); err != nil { 18 | log.Fatal(err) 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /deploy/manifests/docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: '3.6' 2 | 3 | services: 4 | hermitcrab: 5 | image: "sealio/hermitcrab:main" 6 | restart: always 7 | container_name: hermitcrab 8 | command: 9 | - "hermitcrab" 10 | - "--log-debug" 11 | - "--log-verbosity=4" 12 | volumes: 13 | - hermitcrab-data:/var/run/hermitcrab 14 | ports: 15 | - "80:80" 16 | - "443:443" 17 | 18 | volumes: 19 | hermitcrab-data: { } 20 | -------------------------------------------------------------------------------- /deploy/manifests/kubernetes.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: PersistentVolumeClaim 4 | metadata: 5 | namespace: default 6 | name: hermitcrab 7 | labels: 8 | "app.kubernetes.io/part-of": "hermitcrab" 9 | "app.kubernetes.io/component": "hermitcrab-server" 10 | spec: 11 | # When a PVC does not specify a storageClassName, 12 | # the default StorageClass is used. 13 | accessModes: 14 | - ReadWriteOnce 15 | resources: 16 | requests: 17 | storage: 500Mi 18 | --- 19 | apiVersion: v1 20 | kind: Service 21 | metadata: 22 | namespace: default 23 | name: hermitcrab 24 | spec: 25 | selector: 26 | "app.kubernetes.io/part-of": "hermitcrab" 27 | "app.kubernetes.io/component": "hermitcrab-server" 28 | ports: 29 | - name: http 30 | port: 80 31 | targetPort: http 32 | - name: https 33 | port: 443 34 | targetPort: https 35 | --- 36 | apiVersion: apps/v1 37 | kind: Deployment 38 | metadata: 39 | namespace: walrus-system 40 | name: terraform-provider-mirror 41 | labels: 42 | "app.kubernetes.io/part-of": "hermitcrab" 43 | "app.kubernetes.io/component": "hermitcrab-server" 44 | "app.kubernetes.io/name": "hermitcrab" 45 | spec: 46 | replicas: 1 47 | selector: 48 | matchLabels: 49 | "app.kubernetes.io/part-of": "hermitcrab" 50 | "app.kubernetes.io/component": "hermitcrab-server" 51 | "app.kubernetes.io/name": "hermitcrab" 52 | template: 53 | metadata: 54 | labels: 55 | "app.kubernetes.io/part-of": "hermitcrab" 56 | "app.kubernetes.io/component": "hermitcrab-server" 57 | "app.kubernetes.io/name": "hermitcrab" 58 | spec: 59 | automountServiceAccountToken: false 60 | restartPolicy: Always 61 | containers: 62 | - name: hermitcrab 63 | image: sealio/hermitcrab:main 64 | imagePullPolicy: Always 65 | resources: 66 | limits: 67 | cpu: '2' 68 | memory: '4Gi' 69 | requests: 70 | cpu: '500m' 71 | memory: '512Mi' 72 | ports: 73 | - name: http 74 | containerPort: 80 75 | - name: https 76 | containerPort: 443 77 | startupProbe: 78 | failureThreshold: 10 79 | periodSeconds: 5 80 | httpGet: 81 | port: 80 82 | path: /readyz 83 | readinessProbe: 84 | failureThreshold: 3 85 | timeoutSeconds: 5 86 | periodSeconds: 5 87 | httpGet: 88 | port: 80 89 | path: /readyz 90 | livenessProbe: 91 | failureThreshold: 10 92 | timeoutSeconds: 5 93 | periodSeconds: 10 94 | httpGet: 95 | # Redirect the liveness probe request. 96 | httpHeaders: 97 | - name: "User-Agent" 98 | value: "" 99 | port: 80 100 | path: /livez 101 | volumeMounts: 102 | - name: data 103 | mountPath: /var/run/hermitcrab 104 | volumes: 105 | - name: data 106 | persistentVolumeClaim: 107 | claimName: hermitcrab 108 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/seal-io/hermitcrab 2 | 3 | go 1.21 4 | 5 | replace github.com/seal-io/walrus/utils => github.com/seal-io/walrus/staging/utils v0.0.0-20240318015456-5aef1b7525ec 6 | 7 | require ( 8 | github.com/Masterminds/semver/v3 v3.2.1 9 | github.com/dustin/go-humanize v1.0.1 10 | github.com/getkin/kin-openapi v0.122.0 11 | github.com/gin-gonic/gin v1.9.1 12 | github.com/google/uuid v1.6.0 13 | github.com/gorilla/websocket v1.5.1 14 | github.com/prometheus/client_golang v1.19.0 15 | github.com/seal-io/walrus/utils v0.0.0-00010101000000-000000000000 16 | github.com/sirupsen/logrus v1.9.3 17 | github.com/stretchr/testify v1.9.0 18 | github.com/tidwall/gjson v1.17.1 19 | github.com/urfave/cli/v2 v2.27.1 20 | go.etcd.io/bbolt v1.3.9 21 | go.uber.org/multierr v1.11.0 22 | golang.org/x/crypto v0.26.0 23 | golang.org/x/exp v0.0.0-20240404231335-c0f41cb1a7a0 24 | golang.org/x/time v0.5.0 25 | k8s.io/apimachinery v0.29.3 26 | k8s.io/klog/v2 v2.120.1 27 | ) 28 | 29 | require ( 30 | github.com/akerl/go-indefinite-article v0.0.2-0.20221219154354-6280c92263d6 // indirect 31 | github.com/akerl/timber v0.0.3 // indirect 32 | github.com/alitto/pond v1.8.3 // indirect 33 | github.com/andybalholm/brotli v1.1.0 // indirect 34 | github.com/beorn7/perks v1.0.1 // indirect 35 | github.com/bytedance/sonic v1.11.3 // indirect 36 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 37 | github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect 38 | github.com/chenzhuoyu/iasm v0.9.1 // indirect 39 | github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect 40 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect 41 | github.com/evanphx/json-patch v5.9.0+incompatible // indirect 42 | github.com/gabriel-vasile/mimetype v1.4.3 // indirect 43 | github.com/gin-contrib/sse v0.1.0 // indirect 44 | github.com/go-co-op/gocron v1.37.0 // indirect 45 | github.com/go-logr/logr v1.4.1 // indirect 46 | github.com/go-openapi/inflect v0.21.0 // indirect 47 | github.com/go-openapi/jsonpointer v0.21.0 // indirect 48 | github.com/go-openapi/swag v0.23.0 // indirect 49 | github.com/go-playground/locales v0.14.1 // indirect 50 | github.com/go-playground/universal-translator v0.18.1 // indirect 51 | github.com/go-playground/validator/v10 v10.19.0 // indirect 52 | github.com/goccy/go-json v0.10.2 // indirect 53 | github.com/invopop/yaml v0.3.1 // indirect 54 | github.com/josharian/intern v1.0.0 // indirect 55 | github.com/json-iterator/go v1.1.12 // indirect 56 | github.com/klauspost/compress v1.17.7 // indirect 57 | github.com/klauspost/cpuid/v2 v2.2.7 // indirect 58 | github.com/leodido/go-urn v1.4.0 // indirect 59 | github.com/mailru/easyjson v0.7.7 // indirect 60 | github.com/mattn/go-isatty v0.0.20 // indirect 61 | github.com/mattn/go-runewidth v0.0.15 // indirect 62 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 63 | github.com/modern-go/reflect2 v1.0.2 // indirect 64 | github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect 65 | github.com/pelletier/go-toml/v2 v2.2.0 // indirect 66 | github.com/perimeterx/marshmallow v1.1.5 // indirect 67 | github.com/pkg/errors v0.9.1 // indirect 68 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect 69 | github.com/prometheus/client_model v0.6.1 // indirect 70 | github.com/prometheus/common v0.52.2 // indirect 71 | github.com/prometheus/procfs v0.13.0 // indirect 72 | github.com/rivo/uniseg v0.4.7 // indirect 73 | github.com/robfig/cron/v3 v3.0.1 // indirect 74 | github.com/russross/blackfriday/v2 v2.1.0 // indirect 75 | github.com/tidwall/match v1.1.1 // indirect 76 | github.com/tidwall/pretty v1.2.1 // indirect 77 | github.com/tidwall/sjson v1.2.5 // indirect 78 | github.com/twitchyliquid64/golang-asm v0.15.1 // indirect 79 | github.com/ugorji/go/codec v1.2.12 // indirect 80 | github.com/valyala/bytebufferpool v1.0.0 // indirect 81 | github.com/valyala/fasthttp v1.52.0 // indirect 82 | github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 // indirect 83 | go.uber.org/atomic v1.11.0 // indirect 84 | go.uber.org/automaxprocs v1.5.3 // indirect 85 | go.uber.org/zap v1.27.0 // indirect 86 | golang.org/x/arch v0.7.0 // indirect 87 | golang.org/x/mod v0.17.0 // indirect 88 | golang.org/x/net v0.24.0 // indirect 89 | golang.org/x/sys v0.23.0 // indirect 90 | golang.org/x/text v0.17.0 // indirect 91 | google.golang.org/protobuf v1.33.0 // indirect 92 | gopkg.in/yaml.v3 v3.0.1 // indirect 93 | k8s.io/utils v0.0.0-20240310230437-4693a0247e57 // indirect 94 | ) 95 | -------------------------------------------------------------------------------- /hack/boilerplate/go.txt: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2023 Seal, Inc 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | // Code generated by "hermitcrab". DO NOT EDIT. 5 | -------------------------------------------------------------------------------- /hack/build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -o errexit 4 | set -o nounset 5 | set -o pipefail 6 | 7 | ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)" 8 | source "${ROOT_DIR}/hack/lib/init.sh" 9 | 10 | BUILD_DIR="${ROOT_DIR}/.dist/build" 11 | mkdir -p "${BUILD_DIR}" 12 | 13 | function build() { 14 | local target="$1" 15 | local task="$2" 16 | local path="$3" 17 | 18 | local ldflags=( 19 | "-X github.com/seal-io/walrus/utils/version.Version=${GIT_VERSION}" 20 | "-X github.com/seal-io/walrus/utils/version.GitCommit=${GIT_COMMIT}" 21 | "-w -s" 22 | "-extldflags '-static'" 23 | ) 24 | 25 | local tags=() 26 | # shellcheck disable=SC2086 27 | IFS=" " read -r -a tags <<<"$(seal::target::build_tags ${target})" 28 | 29 | local platforms=() 30 | # shellcheck disable=SC2086 31 | IFS=" " read -r -a platforms <<<"$(seal::target::build_platforms ${target} ${task})" 32 | 33 | for platform in "${platforms[@]}"; do 34 | local os_arch 35 | IFS="/" read -r -a os_arch <<<"${platform}" 36 | local os="${os_arch[0]}" 37 | local arch="${os_arch[1]}" 38 | GOOS=${os} GOARCH=${arch} CGO_ENABLED=0 go build \ 39 | -trimpath \ 40 | -ldflags="${ldflags[*]}" \ 41 | -tags="${os} ${tags[*]}" \ 42 | -o="${BUILD_DIR}/${target}/${task}-${os}-${arch}" \ 43 | "${path}" 44 | done 45 | } 46 | 47 | function dispatch() { 48 | local target="$1" 49 | local path="$2" 50 | 51 | shift 2 52 | local specified_targets="$*" 53 | if [[ -n ${specified_targets} ]] && [[ ! ${specified_targets} =~ ${target} ]]; then 54 | return 55 | fi 56 | 57 | local tasks=() 58 | # shellcheck disable=SC2086 59 | IFS=" " read -r -a tasks <<<"$(seal::util::find_subdirs ${path}/cmd)" 60 | 61 | for task in "${tasks[@]}"; do 62 | seal::log::debug "building ${target} ${task}" 63 | if [[ "${PARALLELIZE:-true}" == "false" ]]; then 64 | build "${target}" "${task}" "${path}/cmd/${task}" 65 | else 66 | build "${target}" "${task}" "${path}/cmd/${task}" & 67 | fi 68 | done 69 | } 70 | 71 | # 72 | # main 73 | # 74 | 75 | seal::log::info "+++ BUILD +++" "info: ${GIT_VERSION},${GIT_COMMIT:0:7},${GIT_TREE_STATE},${BUILD_DATE}" 76 | 77 | dispatch "hermitcrab" "${ROOT_DIR}" "$@" 78 | 79 | if [[ "${PARALLELIZE:-true}" == "true" ]]; then 80 | seal::util::wait_jobs || seal::log::fatal "--- BUILD ---" 81 | fi 82 | seal::log::info "--- BUILD ---" 83 | -------------------------------------------------------------------------------- /hack/ci.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -o errexit 4 | set -o nounset 5 | set -o pipefail 6 | 7 | ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)" 8 | pushd "${ROOT_DIR}" >/dev/null 2>&1 9 | 10 | # check phase 11 | if [[ "${CI_CHECK:-true}" == "true" ]]; then 12 | make deps "$@" 13 | make lint "$@" 14 | make test "$@" 15 | fi 16 | 17 | # publish phase 18 | if [[ "${CI_PUBLISH:-true}" == "true" ]]; then 19 | make build "$@" 20 | make package "$@" 21 | fi 22 | 23 | popd >/dev/null 2>&1 24 | -------------------------------------------------------------------------------- /hack/deps.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -o errexit 4 | set -o nounset 5 | set -o pipefail 6 | 7 | ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)" 8 | source "${ROOT_DIR}/hack/lib/init.sh" 9 | 10 | function mod() { 11 | local path="$1" 12 | shift 1 13 | 14 | [[ "${path}" == "${ROOT_DIR}" ]] || pushd "${path}" >/dev/null 2>&1 15 | 16 | if [[ -n "$*" ]] && [[ "$*" =~ update$ ]]; then 17 | go get -u ./... 18 | fi 19 | 20 | go mod tidy 21 | go mod download 22 | 23 | [[ "${path}" == "${ROOT_DIR}" ]] || popd >/dev/null 2>&1 24 | } 25 | 26 | function dispatch() { 27 | local target="$1" 28 | local path="$2" 29 | 30 | shift 2 31 | local specified_targets="$*" 32 | if [[ -n ${specified_targets} ]] && [[ ${specified_targets} != "update" ]] && [[ ! ${specified_targets} =~ ${target} ]]; then 33 | return 34 | fi 35 | 36 | seal::log::debug "modding ${target}" 37 | if [[ "${PARALLELIZE:-true}" == "false" ]]; then 38 | mod "${path}" "$@" 39 | else 40 | mod "${path}" "$@" & 41 | fi 42 | } 43 | 44 | # 45 | # main 46 | # 47 | 48 | seal::log::info "+++ MOD +++" 49 | 50 | dispatch "hermitcrab" "${ROOT_DIR}" "$@" 51 | 52 | if [[ "${PARALLELIZE:-true}" == "true" ]]; then 53 | seal::util::wait_jobs || seal::log::fatal "--- MOD ---" 54 | fi 55 | seal::log::info "--- MOD ---" 56 | -------------------------------------------------------------------------------- /hack/lib/image.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # ----------------------------------------------------------------------------- 4 | # Image variables helpers. These functions need the 5 | # following variables: 6 | # 7 | # DOCKER_VERSION - The Docker version for running, default is 20.10. 8 | # DOCKER_USERNAME - The username of image registry. 9 | # DOCKER_PASSWORD - The password of image registry. 10 | 11 | docker_version=${DOCKER_VERSION:-"20.10"} 12 | docker_username=${DOCKER_USERNAME:-} 13 | docker_password=${DOCKER_PASSWORD:-} 14 | 15 | function seal::image::docker::install() { 16 | curl --retry 3 --retry-all-errors --retry-delay 3 \ 17 | -sSfL "https://get.docker.com" | sh -s VERSION="${docker_version}" 18 | } 19 | 20 | function seal::image::docker::validate() { 21 | # shellcheck disable=SC2046 22 | if [[ -n "$(command -v $(seal::image::docker::bin))" ]]; then 23 | return 0 24 | fi 25 | 26 | seal::log::info "installing docker" 27 | if seal::image::docker::install; then 28 | seal::log::info "docker: $($(seal::image::docker::bin) version --format '{{.Server.Version}}' 2>&1)" 29 | return 0 30 | fi 31 | seal::log::error "no docker available" 32 | return 1 33 | } 34 | 35 | function seal::image::docker::bin() { 36 | echo -n "docker" 37 | } 38 | 39 | function seal::image::name() { 40 | if [[ -n "${IMAGE:-}" ]]; then 41 | echo -n "${IMAGE}" 42 | else 43 | echo -n "$(basename "${ROOT_DIR}")" 2>/dev/null 44 | fi 45 | } 46 | 47 | function seal::image::tag() { 48 | echo -n "${TAG:-${GIT_VERSION}}" | sed -E 's/[^a-zA-Z0-9\.]+/-/g' 2>/dev/null 49 | } 50 | 51 | function seal::image::login() { 52 | if seal::image::docker::validate; then 53 | if [[ -n ${docker_username} ]] && [[ -n ${docker_password} ]]; then 54 | seal::log::debug "docker login ${*:-} -u ${docker_username} -p ***" 55 | if ! docker login "${*:-}" -u "${docker_username}" -p "${docker_password}" >/dev/null 2>&1; then 56 | seal::log::fatal "failed: docker login ${*:-} -u ${docker_username} -p ***" 57 | fi 58 | fi 59 | return 0 60 | fi 61 | 62 | seal::log::fatal "cannot execute image login as client is not found" 63 | } 64 | 65 | function seal::image::build::within_container() { 66 | if seal::image::docker::validate; then 67 | if ! $(seal::image::docker::bin) buildx inspect --builder="seal"; then 68 | seal::log::debug "setting up qemu" 69 | $(seal::image::docker::bin) run \ 70 | --rm \ 71 | --privileged \ 72 | tonistiigi/binfmt:qemu-v7.0.0 --install all 73 | seal::log::debug "setting up buildx" 74 | $(seal::image::docker::bin) buildx create \ 75 | --name="seal" \ 76 | --driver="docker-container" \ 77 | --buildkitd-flags="--allow-insecure-entitlement security.insecure --allow-insecure-entitlement network.host" \ 78 | --use \ 79 | --bootstrap 80 | fi 81 | 82 | return 0 83 | fi 84 | 85 | seal::log::fatal "cannot execute image build as client is not found" 86 | } 87 | 88 | function seal::image::build() { 89 | if seal::image::docker::validate; then 90 | seal::log::debug "docker buildx build $*" 91 | $(seal::image::docker::bin) buildx build "$@" 92 | 93 | return 0 94 | fi 95 | 96 | seal::log::fatal "cannot execute image build as client is not found" 97 | } 98 | -------------------------------------------------------------------------------- /hack/lib/init.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -o errexit 4 | set -o nounset 5 | set -o pipefail 6 | 7 | unset CDPATH 8 | 9 | # Set no_proxy for localhost if behind a proxy, otherwise, 10 | # the connections to localhost in scripts will time out. 11 | export no_proxy=127.0.0.1,localhost 12 | 13 | ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd -P)" 14 | mkdir -p "${ROOT_DIR}/.sbin" 15 | 16 | for file in "${ROOT_DIR}/hack/lib/"*; do 17 | if [[ -f "${file}" ]] && [[ "${file}" != *"init.sh" ]]; then 18 | # shellcheck disable=SC1090 19 | source "${file}" 20 | fi 21 | done 22 | 23 | seal::log::install_errexit 24 | seal::version::get_version_vars 25 | -------------------------------------------------------------------------------- /hack/lib/log.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | ## 4 | # Borrowed from github.com/kubernetes/kubernetes/hack/lib/logging.sh 5 | ## 6 | 7 | # ----------------------------------------------------------------------------- 8 | # Logger variables helpers. These functions need the 9 | # following variables: 10 | # 11 | # LOG_LEVEL - The level of logger, default is "debug". 12 | 13 | log_level="${LOG_LEVEL:-"debug"}" 14 | log_colorful="${LOG_COLORFUL:-"true"}" 15 | 16 | # Handler for when we exit automatically on an error. 17 | seal::log::errexit() { 18 | local err="${PIPESTATUS[*]}" 19 | 20 | # if the shell we are in doesn't have errexit set (common in subshells) then 21 | # don't dump stacks. 22 | set +o | grep -qe "-o errexit" || return 23 | 24 | set +o xtrace 25 | seal::log::panic "${BASH_SOURCE[1]}:${BASH_LINENO[0]} '${BASH_COMMAND}' exited with status ${err}" "${1:-1}" 26 | } 27 | 28 | seal::log::install_errexit() { 29 | # trap ERR to provide an error handler whenever a command exits nonzero, this 30 | # is a more verbose version of set -o errexit 31 | trap 'seal::log::errexit' ERR 32 | 33 | # setting errtrace allows our ERR trap handler to be propagated to functions, 34 | # expansions and subshells 35 | set -o errtrace 36 | } 37 | 38 | # Debug level logging. 39 | seal::log::debug() { 40 | [[ ${log_level} == "debug" ]] || return 0 41 | local message="${2:-}" 42 | 43 | local timestamp 44 | timestamp="$(date +"[%m%d %H:%M:%S]")" 45 | echo -e "[DEBG] ${timestamp} ${1-}" >&2 46 | shift 1 47 | for message; do 48 | echo -e " ${message}" >&2 49 | done 50 | } 51 | 52 | # Info level logging. 53 | seal::log::info() { 54 | [[ ${log_level} == "debug" ]] || [[ ${log_level} == "info" ]] || return 0 55 | local message="${2:-}" 56 | 57 | local timestamp 58 | timestamp="$(date +"[%m%d %H:%M:%S]")" 59 | if [[ ${log_colorful} == "true" ]]; then 60 | echo -e "\033[34m[INFO]\033[0m ${timestamp} ${1-}" >&2 61 | else 62 | echo -e "[INFO] ${timestamp} ${1-}" >&2 63 | fi 64 | shift 1 65 | for message; do 66 | echo -e " ${message}" >&2 67 | done 68 | } 69 | 70 | # Warn level logging. 71 | seal::log::warn() { 72 | local message="${2:-}" 73 | 74 | local timestamp 75 | timestamp="$(date +"[%m%d %H:%M:%S]")" 76 | if [[ ${log_colorful} == "true" ]]; then 77 | echo -e "\033[33m[WARN]\033[0m ${timestamp} ${1-}" >&2 78 | else 79 | echo -e "[WARN] ${timestamp} ${1-}" >&2 80 | fi 81 | shift 1 82 | for message; do 83 | echo -e " ${message}" >&2 84 | done 85 | } 86 | 87 | # Error level logging, log an error but keep going, don't dump the stack or exit. 88 | seal::log::error() { 89 | local message="${2:-}" 90 | 91 | local timestamp 92 | timestamp="$(date +"[%m%d %H:%M:%S]")" 93 | if [[ ${log_colorful} == "true" ]]; then 94 | echo -e "\033[31m[ERRO]\033[0m ${timestamp} ${1-}" >&2 95 | else 96 | echo -e "[ERRO] ${timestamp} ${1-}" >&2 97 | fi 98 | shift 1 99 | for message; do 100 | echo -e " ${message}" >&2 101 | done 102 | } 103 | 104 | # Fatal level logging, log an error but exit with 1, don't dump the stack or exit. 105 | seal::log::fatal() { 106 | local message="${2:-}" 107 | 108 | local timestamp 109 | timestamp="$(date +"[%m%d %H:%M:%S]")" 110 | if [[ ${log_colorful} == "true" ]]; then 111 | echo -e "\033[41;33m[FATA]\033[0m ${timestamp} ${1-}" >&2 112 | else 113 | echo -e "[FATA] ${timestamp} ${1-}" >&2 114 | fi 115 | shift 1 116 | for message; do 117 | echo -e " ${message}" >&2 118 | done 119 | 120 | exit 1 121 | } 122 | 123 | # Panic level logging, dump the error stack and exit. 124 | # Args: 125 | # $1 Message to log with the error 126 | # $2 The error code to return 127 | # $3 The number of stack frames to skip when printing. 128 | seal::log::panic() { 129 | local message="${1:-}" 130 | local code="${2:-1}" 131 | 132 | local timestamp 133 | timestamp="$(date +"[%m%d %H:%M:%S]")" 134 | if [[ ${log_colorful} == "true" ]]; then 135 | echo -e "\033[41;33m[FATA]\033[0m ${timestamp} ${message}" >&2 136 | else 137 | echo -e "[FATA] ${timestamp} ${message}" >&2 138 | fi 139 | 140 | # print out the stack trace described by $function_stack 141 | if [[ ${#FUNCNAME[@]} -gt 2 ]]; then 142 | if [[ ${log_colorful} == "true" ]]; then 143 | echo -e "\033[31m call stack:\033[0m" >&2 144 | else 145 | echo -e " call stack:" >&2 146 | fi 147 | local i 148 | for ((i = 1; i < ${#FUNCNAME[@]} - 2; i++)); do 149 | echo -e " ${i}: ${BASH_SOURCE[${i} + 2]}:${BASH_LINENO[${i} + 1]} ${FUNCNAME[${i} + 1]}(...)" >&2 150 | done 151 | fi 152 | 153 | if [[ ${log_colorful} == "true" ]]; then 154 | echo -e "\033[41;33m[FATA]\033[0m ${timestamp} exiting with status ${code}" >&2 155 | else 156 | echo -e "[FATA] ${timestamp} exiting with status ${code}" >&2 157 | fi 158 | 159 | popd >/dev/null 2>&1 || exit "${code}" 160 | exit "${code}" 161 | } 162 | -------------------------------------------------------------------------------- /hack/lib/style.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # ----------------------------------------------------------------------------- 4 | # Lint variables helpers. These functions need the 5 | # following variables: 6 | # 7 | # GOLANGCI_LINT_VERSION - The Golangci-lint version, default is v1.55.2. 8 | # COMMITSAR_VERSION - The Commitsar version, default is v0.20.2. 9 | 10 | golangci_lint_version=${GOLANGCI_LINT_VERSION:-"v1.55.2"} 11 | commitsar_version=${COMMITSAR_VERSION:-"v0.20.2"} 12 | 13 | function seal::lint::golangci_lint::install() { 14 | curl --retry 3 --retry-all-errors --retry-delay 3 -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b "${ROOT_DIR}/.sbin" "${golangci_lint_version}" 15 | } 16 | 17 | function seal::lint::golangci_lint::validate() { 18 | # shellcheck disable=SC2046 19 | if [[ -n "$(command -v $(seal::lint::golangci_lint::bin))" ]]; then 20 | if [[ $($(seal::lint::golangci_lint::bin) --version 2>&1 | cut -d " " -f 4 2>&1 | head -n 1) == "${golangci_lint_version#v}" ]]; then 21 | return 0 22 | fi 23 | fi 24 | 25 | seal::log::info "installing golangci-lint ${golangci_lint_version}" 26 | if seal::lint::golangci_lint::install; then 27 | seal::log::info "golangci_lint $($(seal::lint::golangci_lint::bin) --version 2>&1 | cut -d " " -f 4 2>&1 | head -n 1)" 28 | return 0 29 | fi 30 | seal::log::error "no golangci-lint available" 31 | return 1 32 | } 33 | 34 | function seal::lint::golangci_lint::bin() { 35 | local bin="golangci-lint" 36 | if [[ -f "${ROOT_DIR}/.sbin/golangci-lint" ]]; then 37 | bin="${ROOT_DIR}/.sbin/golangci-lint" 38 | fi 39 | echo -n "${bin}" 40 | } 41 | 42 | function seal::lint::run() { 43 | if ! seal::lint::golangci_lint::validate; then 44 | seal::log::warn "using go fmt/vet instead golangci-lint" 45 | shift 1 46 | local fmt_args=() 47 | local vet_args=() 48 | for arg in "$@"; do 49 | if [[ "${arg}" == "--build-tags="* ]]; then 50 | arg="${arg//--build-/-}" 51 | vet_args+=("${arg}") 52 | continue 53 | fi 54 | fmt_args+=("${arg}") 55 | vet_args+=("${arg}") 56 | done 57 | seal::log::debug "go fmt ${fmt_args[*]}" 58 | go fmt "${fmt_args[@]}" 59 | seal::log::debug "go vet ${vet_args[*]}" 60 | go vet "${vet_args[@]}" 61 | return 0 62 | fi 63 | 64 | seal::log::debug "golangci-lint run --fix $*" 65 | $(seal::lint::golangci_lint::bin) run --fix "$@" 66 | } 67 | 68 | function seal::format::goimports::install() { 69 | GOBIN="${ROOT_DIR}/.sbin" go install github.com/incu6us/goimports-reviser/v3@latest 70 | } 71 | 72 | function seal::format::goimports::validate() { 73 | # shellcheck disable=SC2046 74 | if [[ -n "$(command -v $(seal::format::goimports::bin))" ]]; then 75 | return 0 76 | fi 77 | 78 | seal::log::info "installing goimports" 79 | if seal::format::goimports::install; then 80 | return 0 81 | fi 82 | seal::log::error "no goimports-reviser available" 83 | return 1 84 | } 85 | 86 | function seal::format::goimports::bin() { 87 | local bin="goimports-reviser" 88 | if [[ -f "${ROOT_DIR}/.sbin/goimports-reviser" ]]; then 89 | bin="${ROOT_DIR}/.sbin/goimports-reviser" 90 | fi 91 | echo -n "${bin}" 92 | } 93 | 94 | function seal::format::gofumpt::install() { 95 | GOBIN="${ROOT_DIR}/.sbin" go install mvdan.cc/gofumpt@latest 96 | } 97 | 98 | function seal::format::gofumpt::validate() { 99 | # shellcheck disable=SC2046 100 | if [[ -n "$(command -v $(seal::format::gofumpt::bin))" ]]; then 101 | return 0 102 | fi 103 | 104 | seal::log::info "installing gofumpt" 105 | if seal::format::gofumpt::install; then 106 | return 0 107 | fi 108 | seal::log::error "no gofumpt available" 109 | return 1 110 | } 111 | 112 | function seal::format::gofumpt::bin() { 113 | local bin="gofumpt" 114 | if [[ -f "${ROOT_DIR}/.sbin/gofumpt" ]]; then 115 | bin="${ROOT_DIR}/.sbin/gofumpt" 116 | fi 117 | echo -n "${bin}" 118 | } 119 | 120 | # install golines 121 | function seal::format::golines::install() { 122 | GOBIN="${ROOT_DIR}/.sbin" go install github.com/segmentio/golines@latest 123 | } 124 | 125 | function seal::format::golines::validate() { 126 | # shellcheck disable=SC2046 127 | if [[ -n "$(command -v $(seal::format::golines::bin))" ]]; then 128 | return 0 129 | fi 130 | 131 | seal::log::info "installing golines" 132 | if seal::format::golines::install; then 133 | return 0 134 | fi 135 | seal::log::error "no golines available" 136 | return 1 137 | } 138 | 139 | function seal::format::golines::bin() { 140 | local bin="golines" 141 | if [[ -f "${ROOT_DIR}/.sbin/golines" ]]; then 142 | bin="${ROOT_DIR}/.sbin/golines" 143 | fi 144 | echo -n "${bin}" 145 | } 146 | 147 | # install wsl(Whitespace Linter) 148 | function seal::format::wsl::install() { 149 | GOBIN="${ROOT_DIR}/.sbin" go install github.com/bombsimon/wsl/v4/cmd...@master 150 | } 151 | 152 | function seal::format::wsl::validate() { 153 | # shellcheck disable=SC2046 154 | if [[ -n "$(command -v $(seal::format::wsl::bin))" ]]; then 155 | return 0 156 | fi 157 | 158 | seal::log::info "installing wsl" 159 | if seal::format::wsl::install; then 160 | return 0 161 | fi 162 | seal::log::error "no wsl available" 163 | return 1 164 | } 165 | 166 | function seal::format::wsl::bin() { 167 | local bin="wsl" 168 | if [[ -f "${ROOT_DIR}/.sbin/wsl" ]]; then 169 | bin="${ROOT_DIR}/.sbin/wsl" 170 | fi 171 | echo -n "${bin}" 172 | } 173 | 174 | function seal::format::run() { 175 | local path=$1 176 | shift 1 177 | # shellcheck disable=SC2206 178 | local path_ignored=(${*}) 179 | 180 | # goimports 181 | if ! seal::format::goimports::validate; then 182 | seal::log::fatal "cannot execute goimports as client is not found" 183 | fi 184 | 185 | # shellcheck disable=SC2155 186 | local goimports_opts=( 187 | "-rm-unused" 188 | "-set-alias" 189 | "-use-cache" 190 | "-imports-order=std,general,company,project,blanked,dotted" 191 | "-output=file" 192 | ) 193 | set +e 194 | if [[ ${#path_ignored[@]} -gt 0 ]]; then 195 | seal::log::debug "pushd ${path}; go list -f \"{{.Dir}}\" ./... | grep -v -E \"$(seal::util::join_array "|" "${path_ignored[@]}")\" | xargs goimports-reviser ${goimports_opts[*]}; popd" 196 | [[ "${path}" == "${ROOT_DIR}" ]] || pushd "${path}" >/dev/null 2>&1 197 | go list -f "{{.Dir}}" ./... | grep -v -E "$(seal::util::join_array "|" "${path_ignored[@]}")" | xargs "$(seal::format::goimports::bin)" "${goimports_opts[@]}" 198 | [[ "${path}" == "${ROOT_DIR}" ]] || popd >/dev/null 2>&1 199 | else 200 | seal::log::debug "pushd ${path}; go list -f \"{{.Dir}}\" ./... | xargs goimports-reviser ${goimports_opts[*]}; popd" 201 | [[ "${path}" == "${ROOT_DIR}" ]] || pushd "${path}" >/dev/null 2>&1 202 | go list -f "{{.Dir}}" ./... | xargs "$(seal::format::goimports::bin)" "${goimports_opts[@]}" 203 | [[ "${path}" == "${ROOT_DIR}" ]] || popd >/dev/null 2>&1 204 | fi 205 | set -e 206 | 207 | # gofmt interface{} -> any 208 | local gofmt_opts=( 209 | "-w" 210 | "-r" 211 | "interface{} -> any" 212 | "${path}" 213 | ) 214 | 215 | seal::log::debug "gofmt ${gofmt_opts[*]}" 216 | gofmt "${gofmt_opts[@]}" 217 | 218 | # golines 219 | if ! seal::format::golines::validate; then 220 | seal::log::fatal "cannot execute golines as client is not found" 221 | fi 222 | 223 | # gofumpt for golines base-formatter 224 | if ! seal::format::gofumpt::validate; then 225 | seal::log::fatal "cannot execute gofumpt as client is not found" 226 | fi 227 | 228 | local golines_opts=( 229 | "-w" 230 | "--max-len=120" 231 | "--no-reformat-tags" 232 | "--ignore-generated" # file start with generated_ 233 | "--ignored-dirs=.git" 234 | "--ignored-dirs=node_modules" 235 | "--ignored-dirs=vendor" 236 | ) 237 | for ig in "${path_ignored[@]}"; do 238 | golines_opts+=("--ignored-dirs=${ig}") 239 | done 240 | golines_opts+=( 241 | "--base-formatter=$(seal::format::gofumpt::bin) -extra" # format by gofumpt 242 | "${path}" 243 | ) 244 | seal::log::debug "golines ${golines_opts[*]}" 245 | $(seal::format::golines::bin) "${golines_opts[@]}" 246 | 247 | # wsl 248 | if ! seal::format::wsl::validate; then 249 | seal::log::fatal "cannot execute wsl as client is not found" 250 | fi 251 | 252 | local wsl_opts=( 253 | "--allow-assign-and-anything" 254 | "--allow-trailing-comment" 255 | "--force-short-decl-cuddling=false" 256 | "--fix" 257 | ) 258 | set +e 259 | if [[ ${#path_ignored[@]} -gt 0 ]]; then 260 | seal::log::debug "pushd ${path}; go list ./... | grep -v -E \"$(seal::util::join_array "|" "${path_ignored[@]}")\" | xargs wsl ${wsl_opts[*]}; popd" 261 | [[ "${path}" == "${ROOT_DIR}" ]] || pushd "${path}" >/dev/null 2>&1 262 | go list ./... | grep -v -E "$(seal::util::join_array "|" "${path_ignored[@]}")" | xargs "$(seal::format::wsl::bin)" "${wsl_opts[@]}" >/dev/null 2>&1 263 | [[ "${path}" == "${ROOT_DIR}" ]] || popd >/dev/null 2>&1 264 | else 265 | seal::log::debug "pushd ${path}; go list ./... | xargs wsl ${wsl_opts[*]}; popd" 266 | [[ "${path}" == "${ROOT_DIR}" ]] || pushd "${path}" >/dev/null 2>&1 267 | go list ./... | xargs "$(seal::format::wsl::bin)" "${wsl_opts[@]}" 268 | [[ "${path}" == "${ROOT_DIR}" ]] || popd >/dev/null 2>&1 269 | fi 270 | set -e 271 | } 272 | 273 | function seal::commit::commitsar::install() { 274 | local os 275 | os="$(seal::util::get_raw_os)" 276 | local arch 277 | arch="$(seal::util::get_raw_arch)" 278 | curl --retry 3 --retry-all-errors --retry-delay 3 \ 279 | -o /tmp/commitsar.tar.gz \ 280 | -sSfL "https://github.com/aevea/commitsar/releases/download/${commitsar_version}/commitsar_${commitsar_version#v}_${os}_${arch}.tar.gz" 281 | tar -zxvf /tmp/commitsar.tar.gz \ 282 | --directory "${ROOT_DIR}/.sbin" \ 283 | --no-same-owner \ 284 | --exclude ./LICENSE \ 285 | --exclude ./README.md 286 | chmod a+x "${ROOT_DIR}/.sbin/commitsar" 287 | } 288 | 289 | function seal::commit::commitsar::validate() { 290 | # shellcheck disable=SC2046 291 | if [[ -n "$(command -v $(seal::commit::commitsar::bin))" ]]; then 292 | if [[ $($(seal::commit::commitsar::bin) version 2>&1 | cut -d " " -f 7 2>&1 | head -n 1 | xargs echo -n) == "${commitsar_version#v}" ]]; then 293 | return 0 294 | fi 295 | fi 296 | 297 | seal::log::info "installing commitsar ${commitsar_version}" 298 | if seal::commit::commitsar::install; then 299 | seal::log::info "commitsar $($(seal::commit::commitsar::bin) version 2>&1 | cut -d " " -f 7 2>&1 | head -n 1 | xargs echo -n)" 300 | return 0 301 | fi 302 | seal::log::error "no commitsar available" 303 | return 1 304 | } 305 | 306 | function seal::commit::commitsar::bin() { 307 | local bin="commitsar" 308 | if [[ -f "${ROOT_DIR}/.sbin/commitsar" ]]; then 309 | bin="${ROOT_DIR}/.sbin/commitsar" 310 | fi 311 | echo -n "${bin}" 312 | } 313 | 314 | function seal::commit::lint() { 315 | if ! seal::commit::commitsar::validate; then 316 | seal::log::fatal "cannot execute commitsar as client is not found" 317 | fi 318 | 319 | seal::log::debug "commitsar $*" 320 | $(seal::commit::commitsar::bin) "$@" 321 | } 322 | -------------------------------------------------------------------------------- /hack/lib/target.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | function seal::target::build_prefix() { 4 | local prefix 5 | prefix="$(basename "${ROOT_DIR}")" 6 | 7 | if [[ -n "${BUILD_PREFIX:-}" ]]; then 8 | echo -n "${BUILD_PREFIX}" 9 | else 10 | echo -n "${prefix}" 11 | fi 12 | } 13 | 14 | readonly DEFAULT_BUILD_TAGS=( 15 | "netgo" 16 | "urfave_cli_no_docs" 17 | "jsoniter" 18 | "ginx" 19 | ) 20 | 21 | function seal::target::build_tags() { 22 | local target="${1:-}" 23 | 24 | local tags 25 | if [[ -n "${BUILD_TAGS:-}" ]]; then 26 | IFS="," read -r -a tags <<<"${BUILD_TAGS}" 27 | else 28 | case "${target}" in 29 | utils) 30 | tags=() 31 | ;; 32 | *) 33 | tags=("${DEFAULT_BUILD_TAGS[@]}") 34 | ;; 35 | esac 36 | fi 37 | 38 | if [[ ${#tags[@]} -ne 0 ]]; then 39 | echo -n "${tags[@]}" 40 | fi 41 | } 42 | 43 | readonly DEFAULT_BUILD_PLATFORMS=( 44 | linux/amd64 45 | linux/arm64 46 | ) 47 | 48 | readonly DEFAULT_CLI_BUILD_PLATFORMS=( 49 | linux/amd64 50 | linux/arm64 51 | darwin/amd64 52 | darwin/arm64 53 | ) 54 | 55 | function seal::target::build_platforms() { 56 | local target="${1:-}" 57 | local task="$2" 58 | 59 | local platforms 60 | if [[ -z "${OS:-}" ]] && [[ -z "${ARCH:-}" ]]; then 61 | if [[ -n "${task}" ]] && [[ "${task}" = "cli" ]]; then 62 | platforms=("${DEFAULT_CLI_BUILD_PLATFORMS[@]}") 63 | elif [[ -n "${BUILD_PLATFORMS:-}" ]]; then 64 | IFS="," read -r -a platforms <<<"${BUILD_PLATFORMS}" 65 | else 66 | case "${target}" in 67 | utils) 68 | platforms=() 69 | ;; 70 | *) 71 | platforms=("${DEFAULT_BUILD_PLATFORMS[@]}") 72 | ;; 73 | esac 74 | fi 75 | else 76 | local os="${OS:-$(seal::util::get_raw_os)}" 77 | local arch="${ARCH:-$(seal::util::get_raw_arch)}" 78 | platforms=("${os}/${arch}") 79 | fi 80 | 81 | if [[ ${#platforms[@]} -ne 0 ]]; then 82 | echo -n "${platforms[@]}" 83 | fi 84 | } 85 | 86 | function seal::target::package_platform() { 87 | if [[ -z "${OS:-}" ]] && [[ -z "${ARCH:-}" ]]; then 88 | echo -n "linux/$(seal::util::get_raw_arch)" 89 | else 90 | echo -n "${OS}/${ARCH}" 91 | fi 92 | } 93 | -------------------------------------------------------------------------------- /hack/lib/util.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | function seal::util::find_subdirs() { 4 | local path="$1" 5 | if [[ -z "$path" ]]; then 6 | path="./" 7 | fi 8 | # shellcheck disable=SC2010 9 | ls -l "$path" | grep "^d" | awk '{print $NF}' | xargs echo 10 | } 11 | 12 | function seal::util::is_empty_dir() { 13 | local path="$1" 14 | if [[ ! -d "${path}" ]]; then 15 | return 0 16 | fi 17 | 18 | # shellcheck disable=SC2012 19 | if [[ $(ls "${path}" | wc -l) -eq 0 ]]; then 20 | return 0 21 | fi 22 | return 1 23 | } 24 | 25 | function seal::util::join_array() { 26 | local IFS="$1" 27 | shift 1 28 | echo "$*" 29 | } 30 | 31 | function seal::util::get_os() { 32 | local os 33 | if go env GOOS >/dev/null 2>&1; then 34 | os=$(go env GOOS) 35 | else 36 | os=$(echo -n "$(uname -s)" | tr '[:upper:]' '[:lower:]') 37 | fi 38 | 39 | case ${os} in 40 | cygwin_nt*) os="windows" ;; 41 | mingw*) os="windows" ;; 42 | msys_nt*) os="windows" ;; 43 | esac 44 | 45 | echo -n "${os}" 46 | } 47 | 48 | function seal::util::get_raw_os() { 49 | local os 50 | os=$(echo -n "$(uname -s)" | tr '[:upper:]' '[:lower:]') 51 | 52 | case ${os} in 53 | cygwin_nt*) os="windows" ;; 54 | mingw*) os="windows" ;; 55 | msys_nt*) os="windows" ;; 56 | esac 57 | 58 | echo -n "${os}" 59 | } 60 | 61 | function seal::util::get_arch() { 62 | local arch 63 | if go env GOARCH >/dev/null 2>&1; then 64 | arch=$(go env GOARCH) 65 | if [[ "${arch}" == "arm" ]]; then 66 | arch="${arch}v$(go env GOARM)" 67 | fi 68 | else 69 | arch=$(uname -m) 70 | fi 71 | 72 | case ${arch} in 73 | armv5*) arch="armv5" ;; 74 | armv6*) arch="armv6" ;; 75 | armv7*) 76 | if [[ "${1:-}" == "--full-name" ]]; then 77 | arch="armv7" 78 | else 79 | arch="arm" 80 | fi 81 | ;; 82 | aarch64) arch="arm64" ;; 83 | x86) arch="386" ;; 84 | i686) arch="386" ;; 85 | i386) arch="386" ;; 86 | x86_64) arch="amd64" ;; 87 | esac 88 | 89 | echo -n "${arch}" 90 | } 91 | 92 | function seal::util::get_raw_arch() { 93 | local arch 94 | arch=$(uname -m) 95 | 96 | case ${arch} in 97 | armv5*) arch="armv5" ;; 98 | armv6*) arch="armv6" ;; 99 | armv7*) 100 | if [[ "${1:-}" == "--full-name" ]]; then 101 | arch="armv7" 102 | else 103 | arch="arm" 104 | fi 105 | ;; 106 | aarch64) arch="arm64" ;; 107 | x86) arch="386" ;; 108 | i686) arch="386" ;; 109 | i386) arch="386" ;; 110 | x86_64) arch="amd64" ;; 111 | esac 112 | 113 | echo -n "${arch}" 114 | } 115 | 116 | function seal::util::get_random_port_start() { 117 | local offset="${1:-1}" 118 | if [[ ${offset} -le 0 ]]; then 119 | offset=1 120 | fi 121 | 122 | while true; do 123 | random_port=$((RANDOM % 10000 + 50000)) 124 | for ((i = 0; i < offset; i++)); do 125 | if nc -z 127.0.0.1 $((random_port + i)); then 126 | random_port=0 127 | break 128 | fi 129 | done 130 | 131 | if [[ ${random_port} -ne 0 ]]; then 132 | echo -n "${random_port}" 133 | break 134 | fi 135 | done 136 | } 137 | 138 | function seal::util::sed() { 139 | if ! sed -i "$@" >/dev/null 2>&1; then 140 | # back off none GNU sed 141 | sed -i "" "$@" 142 | fi 143 | } 144 | 145 | function seal::util::decode64() { 146 | if [[ $# -eq 0 ]]; then 147 | cat | base64 --decode 148 | else 149 | printf '%s' "$1" | base64 --decode 150 | fi 151 | } 152 | 153 | function seal::util::encode64() { 154 | if [[ $# -eq 0 ]]; then 155 | cat | base64 156 | else 157 | printf '%s' "$1" | base64 158 | fi 159 | } 160 | 161 | function seal::util::kill_jobs() { 162 | for job in $(jobs -p); do 163 | kill -9 "$job" 164 | done 165 | } 166 | 167 | function seal::util::wait_jobs() { 168 | trap seal::util::kill_jobs TERM INT 169 | local fail=0 170 | local job 171 | for job in $(jobs -p); do 172 | wait "${job}" || fail=$((fail + 1)) 173 | done 174 | return ${fail} 175 | } 176 | 177 | function seal::util::dismiss() { 178 | echo "" 1>/dev/null 2>&1 179 | } 180 | -------------------------------------------------------------------------------- /hack/lib/version.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | ## 4 | # Inspired by github.com/kubernetes/kubernetes/hack/lib/version.sh 5 | ## 6 | 7 | # ----------------------------------------------------------------------------- 8 | # Version management helpers. These functions help to set the 9 | # following variables: 10 | # 11 | # GIT_TREE_STATE - "clean" indicates no changes since the git commit id. 12 | # "dirty" indicates source code changes after the git commit id. 13 | # "archive" indicates the tree was produced by 'git archive'. 14 | # "unknown" indicates cannot find out the git tree. 15 | # GIT_COMMIT - The git commit id corresponding to this 16 | # source code. 17 | # GIT_VERSION - "vX.Y" used to indicate the last release version, 18 | # it can be specified via "VERSION". 19 | # BUILD_DATE - The build date of the version. 20 | 21 | function seal::version::get_version_vars() { 22 | BUILD_DATE=$(date -u '+%Y-%m-%dT%H:%M:%SZ') 23 | GIT_TREE_STATE="unknown" 24 | GIT_COMMIT="unknown" 25 | GIT_VERSION="unknown" 26 | 27 | # get the git tree state if the source was exported through git archive. 28 | # shellcheck disable=SC2016,SC2050 29 | if [[ '$Format:%%$' == "%" ]]; then 30 | GIT_TREE_STATE="archive" 31 | GIT_COMMIT='$Format:%H$' 32 | # when a 'git archive' is exported, the '$Format:%D$' below will look 33 | # something like 'HEAD -> release-1.8, tag: v1.8.3' where then 'tag: ' 34 | # can be extracted from it. 35 | if [[ '$Format:%D$' =~ tag:\ (v[^ ,]+) ]]; then 36 | GIT_VERSION="${BASH_REMATCH[1]}" 37 | else 38 | GIT_VERSION="${GIT_COMMIT:0:7}" 39 | fi 40 | # respect specified version. 41 | GIT_VERSION="${VERSION:-${GIT_VERSION}}" 42 | return 43 | fi 44 | 45 | # return directly if not found git client. 46 | if [[ -z "$(command -v git)" ]]; then 47 | # respect specified version. 48 | GIT_VERSION=${VERSION:-${GIT_VERSION}} 49 | return 50 | fi 51 | 52 | # find out git info via git client. 53 | if GIT_COMMIT=$(git rev-parse "HEAD^{commit}" 2>/dev/null); then 54 | # specify as dirty if the tree is not clean. 55 | if git_status=$(git status --porcelain 2>/dev/null) && [[ -n ${git_status} ]]; then 56 | GIT_TREE_STATE="dirty" 57 | else 58 | GIT_TREE_STATE="clean" 59 | fi 60 | 61 | # specify with the tag if the head is tagged. 62 | if GIT_VERSION="$(git rev-parse --abbrev-ref HEAD 2>/dev/null)"; then 63 | if git_tag=$(git tag -l --contains HEAD 2>/dev/null | head -n 1 2>/dev/null) && [[ -n ${git_tag} ]]; then 64 | GIT_VERSION="${git_tag}" 65 | fi 66 | fi 67 | 68 | # specify to dev if the tree is dirty. 69 | if [[ "${GIT_TREE_STATE:-dirty}" == "dirty" ]]; then 70 | GIT_VERSION="dev" 71 | fi 72 | 73 | # respect specified version 74 | GIT_VERSION=${VERSION:-${GIT_VERSION}} 75 | 76 | if ! [[ "${GIT_VERSION}" =~ ^v([0-9]+)\.([0-9]+)(\.[0-9]+)?(-[0-9A-Za-z.-]+)?(\+[0-9A-Za-z.-]+)?$ ]]; then 77 | GIT_VERSION="dev" 78 | fi 79 | 80 | fi 81 | } 82 | -------------------------------------------------------------------------------- /hack/lint.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -o errexit 4 | set -o nounset 5 | set -o pipefail 6 | 7 | ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)" 8 | source "${ROOT_DIR}/hack/lib/init.sh" 9 | 10 | function check_dirty() { 11 | [[ "${LINT_DIRTY:-false}" == "true" ]] || return 0 12 | 13 | if [[ -n "$(command -v git)" ]]; then 14 | if git_status=$(git status --porcelain 2>/dev/null) && [[ -n ${git_status} ]]; then 15 | seal::log::fatal "the git tree is dirty:\n$(git status --porcelain)" 16 | fi 17 | fi 18 | } 19 | 20 | function lint() { 21 | local path="$1" 22 | local path_ignored="$2" 23 | shift 2 24 | # shellcheck disable=SC2206 25 | local build_tags=(${*}) 26 | 27 | [[ "${path}" == "${ROOT_DIR}" ]] || pushd "${path}" >/dev/null 2>&1 28 | 29 | seal::format::run "${path}" "${path_ignored}" 30 | if [[ ${#build_tags[@]} -gt 0 ]]; then 31 | GOLANGCI_LINT_CACHE="$(go env GOCACHE)/golangci-lint" seal::lint::run --build-tags="\"${build_tags[*]}\"" "${path}/..." 32 | else 33 | GOLANGCI_LINT_CACHE="$(go env GOCACHE)/golangci-lint" seal::lint::run "${path}/..." 34 | fi 35 | 36 | [[ "${path}" == "${ROOT_DIR}" ]] || popd >/dev/null 2>&1 37 | } 38 | 39 | function dispatch() { 40 | local target="$1" 41 | local path="$2" 42 | local path_ignored="$3" 43 | 44 | shift 3 45 | local specified_targets="$*" 46 | if [[ -n ${specified_targets} ]] && [[ ! ${specified_targets} =~ ${target} ]]; then 47 | return 48 | fi 49 | 50 | seal::log::debug "linting ${target}" 51 | if [[ "${PARALLELIZE:-false}" != "true" ]]; then 52 | lint "${path}" "${path_ignored}" "$(seal::target::build_tags "${target}")" 53 | else 54 | lint "${path}" "${path_ignored}" "$(seal::target::build_tags "${target}")" & 55 | fi 56 | } 57 | 58 | function after() { 59 | check_dirty 60 | } 61 | 62 | # 63 | # main 64 | # 65 | 66 | seal::log::info "+++ LINT +++" 67 | 68 | seal::commit::lint "${ROOT_DIR}" 69 | 70 | dispatch "hermitcrab" "${ROOT_DIR}" "" "$@" 71 | 72 | after 73 | 74 | if [[ "${PARALLELIZE:-false}" == "true" ]]; then 75 | seal::util::wait_jobs || seal::log::fatal "--- LINT ---" 76 | fi 77 | seal::log::info "--- LINT ---" 78 | -------------------------------------------------------------------------------- /hack/package.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -o errexit 4 | set -o nounset 5 | set -o pipefail 6 | 7 | ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)" 8 | source "${ROOT_DIR}/hack/lib/init.sh" 9 | 10 | PACKAGE_DIR="${ROOT_DIR}/.dist/package" 11 | mkdir -p "${PACKAGE_DIR}" 12 | PACKAGE_TMP_DIR="${PACKAGE_DIR}/tmp" 13 | mkdir -p "${PACKAGE_TMP_DIR}" 14 | 15 | function setup_image_package() { 16 | if [[ "${PACKAGE_PUSH:-false}" == "true" ]]; then 17 | seal::image::login 18 | fi 19 | } 20 | 21 | function setup_image_package_context() { 22 | local target="$1" 23 | local task="$2" 24 | local path="$3" 25 | 26 | local context="${PACKAGE_DIR}/${target}/${task}" 27 | # create targeted dist 28 | rm -rf "${context}" 29 | mkdir -p "${context}" 30 | # copy targeted source to dist 31 | cp -rf "${path}/image" "${context}/image" 32 | # copy built result to dist 33 | cp -rf "${ROOT_DIR}/.dist/build/${target}" "${context}/build" 34 | 35 | echo -n "${context}" 36 | } 37 | 38 | function package() { 39 | local target="$1" 40 | local task="$2" 41 | local path="$3" 42 | 43 | # shellcheck disable=SC2155 44 | local tag="${REPO:-sealio}/${target}:$(seal::image::tag)" 45 | # shellcheck disable=SC2155 46 | local platform="$(seal::target::package_platform)" 47 | 48 | # shellcheck disable=SC2155 49 | local context="$(setup_image_package_context "${target}" "${task}" "${path}")" 50 | 51 | if [[ "${PACKAGE_BUILD:-true}" == "true" ]]; then 52 | # shellcheck disable=SC2086 53 | local no_cache_filter="" 54 | if [[ "${tag##*:}" != "dev" ]]; then 55 | task="${task}-release" 56 | else 57 | no_cache_filter="fetch" 58 | fi 59 | 60 | local cache="type=registry,ref=sealio/build-cache:${target}-${task}" 61 | local output="type=image,push=${PACKAGE_PUSH:-false}" 62 | 63 | seal::image::build \ 64 | --tag="${tag}" \ 65 | --platform="${platform}" \ 66 | --cache-from="${cache}" \ 67 | --output="${output}" \ 68 | --progress="plain" \ 69 | --no-cache-filter="${no_cache_filter}" \ 70 | --file="${context}/image/Dockerfile" \ 71 | "${context}" 72 | fi 73 | } 74 | 75 | function before() { 76 | setup_image_package 77 | } 78 | 79 | function dispatch() { 80 | local target="$1" 81 | local path="$2" 82 | 83 | shift 2 84 | local specified_targets="$*" 85 | if [[ -n ${specified_targets} ]] && [[ ! ${specified_targets} =~ ${target} ]]; then 86 | return 87 | fi 88 | 89 | local tasks=() 90 | # shellcheck disable=SC2086 91 | IFS=" " read -r -a tasks <<<"$(seal::util::find_subdirs ${path}/pack)" 92 | 93 | for task in "${tasks[@]}"; do 94 | seal::log::debug "packaging ${target} ${task}" 95 | if [[ "${PARALLELIZE:-true}" == "false" ]]; then 96 | package "${target}" "${task}" "${path}/pack/${task}" 97 | else 98 | package "${target}" "${task}" "${path}/pack/${task}" & 99 | fi 100 | done 101 | } 102 | 103 | # 104 | # main 105 | # 106 | 107 | seal::log::info "+++ PACKAGE +++" "tag: $(seal::image::tag)" 108 | 109 | before 110 | 111 | dispatch "hermitcrab" "${ROOT_DIR}" "$@" 112 | 113 | if [[ "${PARALLELIZE:-true}" == "true" ]]; then 114 | seal::util::wait_jobs || seal::log::fatal "--- PACKAGE ---" 115 | fi 116 | seal::log::info "--- PACKAGE ---" 117 | -------------------------------------------------------------------------------- /hack/test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -o errexit 4 | set -o nounset 5 | set -o pipefail 6 | 7 | ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)" 8 | source "${ROOT_DIR}/hack/lib/init.sh" 9 | 10 | TEST_DIR="${ROOT_DIR}/.dist/test" 11 | mkdir -p "${TEST_DIR}" 12 | 13 | function test() { 14 | local target="$1" 15 | local path="$2" 16 | 17 | [[ "${path}" == "${ROOT_DIR}" ]] || pushd "${path}" >/dev/null 2>&1 18 | 19 | local tags=() 20 | # shellcheck disable=SC2086 21 | IFS=" " read -r -a tags <<<"$(seal::target::build_tags ${target})" 22 | 23 | CGO_ENABLED=1 go test \ 24 | -v \ 25 | -failfast \ 26 | -race \ 27 | -cover \ 28 | -timeout=10m \ 29 | -tags="${tags[*]}" \ 30 | -coverprofile="${TEST_DIR}/${target}-coverage.out" \ 31 | "${path}/..." 32 | 33 | [[ "${path}" == "${ROOT_DIR}" ]] || popd >/dev/null 2>&1 34 | } 35 | 36 | function dispatch() { 37 | local target="$1" 38 | local path="$2" 39 | 40 | shift 2 41 | local specified_targets="$*" 42 | if [[ -n ${specified_targets} ]] && [[ ! ${specified_targets} =~ ${target} ]]; then 43 | return 44 | fi 45 | 46 | seal::log::debug "testing ${target}" 47 | if [[ "${PARALLELIZE:-true}" == "false" ]]; then 48 | test "${target}" "${path}" 49 | else 50 | test "${target}" "${path}" & 51 | fi 52 | } 53 | 54 | # 55 | # main 56 | # 57 | 58 | seal::log::info "+++ TEST +++" 59 | 60 | dispatch "hermitcrab" "${ROOT_DIR}" "$@" 61 | 62 | if [[ "${PARALLELIZE:-true}" == "true" ]]; then 63 | seal::util::wait_jobs || seal::log::fatal "--- TEST ---" 64 | fi 65 | seal::log::info "--- TEST ---" 66 | -------------------------------------------------------------------------------- /pack/server/image/Dockerfile: -------------------------------------------------------------------------------- 1 | # 2 | # Release 3 | # 4 | FROM --platform=$TARGETPLATFORM alpine:3.19.0 5 | 6 | ARG TARGETPLATFORM 7 | ARG TARGETOS 8 | ARG TARGETARCH 9 | 10 | ENV DEBIAN_FRONTEND=noninteractive 11 | RUN set -eo pipefail; \ 12 | apk add -U --no-cache \ 13 | ca-certificates \ 14 | openssl \ 15 | curl unzip \ 16 | git \ 17 | ; \ 18 | rm -rf /var/cache/apk/* 19 | 20 | # set locale 21 | RUN set -eo pipefail; \ 22 | apk add -U --no-cache \ 23 | tzdata \ 24 | ; \ 25 | rm -rf /var/cache/apk/* 26 | ENV LANG='en_US.UTF-8' \ 27 | LANGUAGE='en_US:en' \ 28 | LC_ALL='en_US.UTF-8' 29 | 30 | EXPOSE 80 443 31 | VOLUME /var/run/hermitcrab 32 | COPY /image/ / 33 | COPY /build/server-${TARGETOS}-${TARGETARCH} /usr/bin/hermitcrab 34 | ENV _RUNNING_INSIDE_CONTAINER_="true" \ 35 | TF_PLUGIN_MIRROR_DIR="/usr/share/terraform/providers/plugins" 36 | CMD ["hermitcrab", "--log-debug", "--log-verbosity=4"] 37 | -------------------------------------------------------------------------------- /pkg/apis/config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import "github.com/seal-io/walrus/utils/vars" 4 | 5 | // TlsCertified indicates whether the server is TLS certified. 6 | var TlsCertified = vars.SetOnce[bool]{} 7 | -------------------------------------------------------------------------------- /pkg/apis/debug/handler.go: -------------------------------------------------------------------------------- 1 | package debug 2 | 3 | import ( 4 | "net/http" 5 | "net/http/pprof" 6 | 7 | "github.com/gin-gonic/gin" 8 | "github.com/gin-gonic/gin/binding" 9 | "github.com/seal-io/walrus/utils/errorx" 10 | "github.com/seal-io/walrus/utils/log" 11 | "github.com/seal-io/walrus/utils/version" 12 | 13 | "github.com/seal-io/hermitcrab/pkg/apis/runtime" 14 | ) 15 | 16 | func Version() runtime.Handle { 17 | info := gin.H{ 18 | "version": version.Version, 19 | "commit": version.GitCommit, 20 | } 21 | 22 | return func(c *gin.Context) { 23 | c.JSON(http.StatusOK, info) 24 | } 25 | } 26 | 27 | func PProf() runtime.HTTPHandler { 28 | // NB(thxCode): init from net/http/pprof. 29 | m := http.NewServeMux() 30 | m.Handle("/debug/pprof/", http.HandlerFunc(pprof.Index)) 31 | m.Handle("/debug/pprof/cmdline", http.HandlerFunc(pprof.Cmdline)) 32 | m.Handle("/debug/pprof/profile", http.HandlerFunc(pprof.Profile)) 33 | m.Handle("/debug/pprof/symbol", http.HandlerFunc(pprof.Symbol)) 34 | m.Handle("/debug/pprof/trace", http.HandlerFunc(pprof.Trace)) 35 | 36 | return m 37 | } 38 | 39 | func SetFlags() runtime.ErrorHandle { 40 | return func(ctx *gin.Context) error { 41 | // Support set flags log-debug and log-verbosity. 42 | var input struct { 43 | LogDebug *bool `query:"log-debug"` 44 | LogVerbosity *uint64 `query:"log-verbosity"` 45 | } 46 | 47 | if err := binding.MapFormWithTag(&input, ctx.Request.URL.Query(), "query"); err != nil { 48 | return errorx.WrapHttpError(http.StatusBadRequest, err, "invalid query params") 49 | } 50 | 51 | resp := map[string]any{} 52 | 53 | if input.LogDebug != nil { 54 | level := log.InfoLevel 55 | if *input.LogDebug { 56 | level = log.DebugLevel 57 | } 58 | 59 | log.SetLevel(level) 60 | resp["log-debug"] = *input.LogDebug 61 | } 62 | 63 | if input.LogVerbosity != nil { 64 | log.SetVerbosity(*input.LogVerbosity) 65 | resp["log-verbosity"] = *input.LogVerbosity 66 | } 67 | 68 | ctx.JSON(http.StatusOK, resp) 69 | 70 | return nil 71 | } 72 | } 73 | 74 | func GetFlags() runtime.ErrorHandle { 75 | return func(ctx *gin.Context) error { 76 | resp := map[string]any{ 77 | "log-debug": log.GetLevel() == log.DebugLevel, 78 | "log-verbosity": log.GetVerbosity(), 79 | } 80 | 81 | ctx.JSON(http.StatusOK, resp) 82 | 83 | return nil 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /pkg/apis/logger.go: -------------------------------------------------------------------------------- 1 | package apis 2 | 3 | import ( 4 | stdlog "log" 5 | "strings" 6 | 7 | "github.com/seal-io/walrus/utils/log" 8 | ) 9 | 10 | func newStdErrorLogger(delegate log.Logger) *stdlog.Logger { 11 | return stdlog.New(logWriter{logger: delegate}, "", 0) 12 | } 13 | 14 | type logWriter struct { 15 | logger log.Logger 16 | } 17 | 18 | func (l logWriter) Write(p []byte) (int, error) { 19 | // Trim the trailing newline. 20 | s := strings.TrimSuffix(string(p), "\n") 21 | 22 | ok := true 23 | 24 | switch { 25 | case strings.HasPrefix(s, "http: TLS handshake error from"): 26 | switch { 27 | case strings.HasSuffix(s, "tls: unknown certificate"): 28 | // Ignore self-generated certificate errors from client. 29 | ok = false 30 | case strings.HasSuffix(s, "connection reset by peer"): 31 | // Reset TLS handshake errors from client. 32 | ok = false 33 | case strings.HasSuffix(s, "EOF"): 34 | // Terminate TLS handshake errors by client. 35 | ok = false 36 | } 37 | case strings.Contains(s, "broken pipe"): 38 | // Ignore the underlying error of broken pipe. 39 | ok = false 40 | } 41 | 42 | if ok { 43 | l.logger.Warn(s) 44 | } 45 | 46 | return len(p), nil 47 | } 48 | -------------------------------------------------------------------------------- /pkg/apis/measure/handler.go: -------------------------------------------------------------------------------- 1 | package measure 2 | 3 | import ( 4 | "net/http" 5 | "time" 6 | 7 | "github.com/gin-gonic/gin" 8 | 9 | "github.com/seal-io/hermitcrab/pkg/apis/runtime" 10 | "github.com/seal-io/hermitcrab/pkg/health" 11 | "github.com/seal-io/hermitcrab/pkg/metric" 12 | ) 13 | 14 | func Readyz() runtime.Handle { 15 | return func(c *gin.Context) { 16 | d, ok := health.MustValidate(c, []string{"database"}) 17 | if !ok { 18 | c.String(http.StatusServiceUnavailable, d) 19 | return 20 | } 21 | 22 | c.String(http.StatusOK, d) 23 | } 24 | } 25 | 26 | func Livez() runtime.Handle { 27 | return func(c *gin.Context) { 28 | d, ok := health.Validate(c, c.QueryArray("exclude")...) 29 | if !ok { 30 | c.String(http.StatusServiceUnavailable, d) 31 | return 32 | } 33 | 34 | c.String(http.StatusOK, d) 35 | } 36 | } 37 | 38 | func Metrics() runtime.HTTPHandler { 39 | return metric.Index(5, 30*time.Second) 40 | } 41 | -------------------------------------------------------------------------------- /pkg/apis/provider/handler.go: -------------------------------------------------------------------------------- 1 | package provider 2 | 3 | import ( 4 | "context" 5 | "net/http" 6 | "sync" 7 | "time" 8 | 9 | "github.com/gin-gonic/gin/render" 10 | "github.com/seal-io/walrus/utils/errorx" 11 | "github.com/seal-io/walrus/utils/gopool" 12 | "github.com/seal-io/walrus/utils/log" 13 | "k8s.io/apimachinery/pkg/util/sets" 14 | 15 | "github.com/seal-io/hermitcrab/pkg/provider" 16 | "github.com/seal-io/hermitcrab/pkg/provider/metadata" 17 | "github.com/seal-io/hermitcrab/pkg/provider/storage" 18 | ) 19 | 20 | func Handle(service *provider.Service) *Handler { 21 | return &Handler{ 22 | s: service, 23 | } 24 | } 25 | 26 | type Handler struct { 27 | m sync.Mutex 28 | 29 | s *provider.Service 30 | } 31 | 32 | func (h *Handler) GetMetadata(req GetMetadataRequest) (GetMetadataResponse, error) { 33 | version := req.Version() 34 | 35 | if version == "index" { 36 | opts := metadata.GetVersionsOptions{ 37 | Hostname: req.Hostname, 38 | Namespace: req.Namespace, 39 | Type: req.Type, 40 | } 41 | 42 | mr, err := h.s.Metadata.GetVersions(req.Context, opts) 43 | if err != nil { 44 | return GetMetadataResponse{}, err 45 | } 46 | 47 | resp := GetMetadataResponse{ 48 | Versions: sets.New[string](), 49 | } 50 | for _, v := range mr { 51 | resp.Versions.Insert(v.Version) 52 | } 53 | 54 | return resp, nil 55 | } 56 | 57 | opts := metadata.GetVersionOptions{ 58 | Hostname: req.Hostname, 59 | Namespace: req.Namespace, 60 | Type: req.Type, 61 | Version: version, 62 | } 63 | 64 | mr, err := h.s.Metadata.GetVersion(req.Context, opts) 65 | if err != nil { 66 | return GetMetadataResponse{}, err 67 | } 68 | 69 | resp := GetMetadataResponse{ 70 | Archives: map[string]Archive{}, 71 | } 72 | 73 | for _, v := range mr.Platforms { 74 | archiveName := v.OS + "_" + v.Arch 75 | 76 | archive := Archive{ 77 | URL: "download/" + v.Filename, 78 | } 79 | if v.Shasum != "" { 80 | archive.Hashes = []string{ 81 | "zh:" + v.Shasum, 82 | } 83 | } 84 | 85 | resp.Archives[archiveName] = archive 86 | } 87 | 88 | return resp, nil 89 | } 90 | 91 | func (h *Handler) DownloadArchive(req DownloadArchiveRequest) (render.Render, error) { 92 | getPlatformOpts := metadata.GetPlatformOptions{ 93 | Hostname: req.Hostname, 94 | Namespace: req.Namespace, 95 | Type: req.Type, 96 | Version: req.Version, 97 | OS: req.OS, 98 | Arch: req.Arch, 99 | } 100 | 101 | mr, err := h.s.Metadata.GetPlatform(req.Context, getPlatformOpts) 102 | if err != nil { 103 | return nil, err 104 | } 105 | 106 | loadOrFetchOpts := storage.LoadArchiveOptions{ 107 | Hostname: req.Hostname, 108 | Namespace: req.Namespace, 109 | Type: req.Type, 110 | Filename: mr.Filename, 111 | Shasum: mr.Shasum, 112 | DownloadURL: mr.DownloadURL, 113 | } 114 | 115 | return h.s.Storage.LoadArchive(req.Context, loadOrFetchOpts) 116 | } 117 | 118 | func (h *Handler) SyncMetadata(req SyncMetadataRequest) error { 119 | if !h.m.TryLock() { 120 | return errorx.HttpErrorf(http.StatusLocked, "previous sync is not finished") 121 | } 122 | 123 | gopool.Go(func() { 124 | defer h.m.Unlock() 125 | 126 | logger := log.WithName("apis").WithName("provider").WithName("sync_metadata") 127 | 128 | timeout := req.Timeout 129 | if timeout == 0 { 130 | timeout = 2 * time.Minute 131 | } 132 | 133 | ctx, cancel := context.WithTimeout(context.Background(), timeout) 134 | defer cancel() 135 | 136 | err := h.s.Metadata.Sync(ctx) 137 | if err != nil { 138 | logger.Warnf("error syncing: %v", err) 139 | } 140 | }) 141 | 142 | return nil 143 | } 144 | -------------------------------------------------------------------------------- /pkg/apis/provider/handler_view.go: -------------------------------------------------------------------------------- 1 | package provider 2 | 3 | import ( 4 | "errors" 5 | "regexp" 6 | "strings" 7 | "time" 8 | 9 | "github.com/gin-gonic/gin" 10 | "k8s.io/apimachinery/pkg/util/sets" 11 | ) 12 | 13 | type ( 14 | GetMetadataRequest struct { 15 | _ struct{} `route:"GET=/:hostname/:namespace/:type/:action"` 16 | 17 | Hostname string `path:"hostname"` 18 | Namespace string `path:"namespace"` 19 | Type string `path:"type"` 20 | Action string `path:"action"` // Eg. Index.json for list versions, {version}.json for list versioned package. 21 | 22 | Context *gin.Context 23 | } 24 | 25 | GetMetadataResponse struct { 26 | Versions sets.Set[string] `json:"versions,omitempty"` 27 | Archives map[string]Archive `json:"archives,omitempty"` 28 | } 29 | 30 | Archive struct { 31 | URL string `json:"url"` 32 | Hashes []string `json:"hashes"` 33 | } 34 | ) 35 | 36 | func (r *GetMetadataRequest) SetGinContext(ctx *gin.Context) { 37 | r.Context = ctx 38 | } 39 | 40 | func (r *GetMetadataRequest) Validate() error { 41 | if len(r.Action) <= 5 { 42 | return errors.New("invalid action") 43 | } 44 | 45 | return nil 46 | } 47 | 48 | func (r *GetMetadataRequest) Version() string { 49 | return strings.TrimPrefix(r.Action[:len(r.Action)-5], "v") 50 | } 51 | 52 | type ( 53 | DownloadArchiveRequest struct { 54 | _ struct{} `route:"GET=/:hostname/:namespace/:type/download/:archive"` 55 | 56 | Hostname string `path:"hostname"` 57 | Namespace string `path:"namespace"` 58 | Type string `path:"type"` 59 | Archive string `path:"archive"` 60 | 61 | Version string 62 | OS string 63 | Arch string 64 | 65 | Context *gin.Context 66 | } 67 | ) 68 | 69 | func (r *DownloadArchiveRequest) SetGinContext(ctx *gin.Context) { 70 | r.Context = ctx 71 | } 72 | 73 | var regexValidArchive = regexp.MustCompile( 74 | `^terraform-provider-(?P[\w-]+)[_-](?P[\w|\\.]+)[_-](?P[a-z]+)[_-](?P[a-z0-9]+)([_-].*)?\.zip$`, 75 | ) 76 | 77 | func (r *DownloadArchiveRequest) Validate() error { 78 | ps := regexValidArchive.FindStringSubmatch(r.Archive) 79 | if len(ps) < 5 { 80 | return errors.New("invalid archive") 81 | } 82 | ps = ps[1:] 83 | 84 | if r.Type != ps[0] { 85 | return errors.New("invalid type") 86 | } 87 | 88 | r.Version = strings.TrimPrefix(ps[1], "v") 89 | r.OS = ps[2] 90 | r.Arch = ps[3] 91 | 92 | return nil 93 | } 94 | 95 | type ( 96 | SyncMetadataRequest struct { 97 | _ struct{} `route:"PUT=/sync"` 98 | 99 | Timeout time.Duration `query:"timeout,default=2m"` 100 | 101 | Context *gin.Context 102 | } 103 | ) 104 | 105 | func (r *SyncMetadataRequest) SetGinContext(ctx *gin.Context) { 106 | r.Context = ctx 107 | } 108 | -------------------------------------------------------------------------------- /pkg/apis/provider/handler_view_test.go: -------------------------------------------------------------------------------- 1 | package provider 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func Test_regexValidArchive(t *testing.T) { 10 | testCases := []struct { 11 | given string 12 | expected bool 13 | }{ 14 | { 15 | given: "terraform-provider-foo_1.2.3_darwin_amd64.zip", 16 | expected: true, 17 | }, 18 | { 19 | given: "terraform-provider-foo_1.2.3_darwin_amd64", 20 | expected: false, 21 | }, 22 | { 23 | given: "terraform-provider-foo_1.2.3_darwin_amd64.zip.zip", 24 | expected: false, 25 | }, 26 | { 27 | given: "terraform-provider-foo_darwin_amd64.zip.zip", 28 | expected: false, 29 | }, 30 | { 31 | given: "terraform-provider-foo__darwin_amd64.zip.zip", 32 | expected: false, 33 | }, 34 | // See https://github.com/seal-io/hermitcrab/issues/15. 35 | { 36 | given: "terraform-provider-teleport-v14.3.3-darwin-arm64-bin.zip", 37 | expected: true, 38 | }, 39 | } 40 | for _, tc := range testCases { 41 | t.Run(tc.given, func(t *testing.T) { 42 | ps := regexValidArchive.FindStringSubmatch(tc.given) 43 | assert.Equal(t, tc.expected, len(ps) >= 5) 44 | }) 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /pkg/apis/runtime/bind/binder.go: -------------------------------------------------------------------------------- 1 | package bind 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "net/http" 7 | 8 | "github.com/gin-gonic/gin" 9 | ) 10 | 11 | const _32MiB = 32 << 20 12 | 13 | // WithForm binds the passed struct pointer using the request form params, 14 | // fails with false returning and aborts the request with 400 status code. 15 | func WithForm(c *gin.Context, r any) bool { 16 | if c.Request == nil { 17 | return abortWithError(c, errors.New("invalid request")) 18 | } 19 | 20 | if err := parseRequestForm(c.Request); err != nil { 21 | return abortWithError(c, err) 22 | } 23 | 24 | err := MapFormWithTag(r, c.Request.Form, "form") 25 | 26 | return abortWithError(c, err) 27 | } 28 | 29 | func parseRequestForm(req *http.Request) error { 30 | if err := req.ParseForm(); err != nil { 31 | return err 32 | } 33 | 34 | if err := req.ParseMultipartForm(_32MiB); err != nil && !errors.Is(err, http.ErrNotMultipart) { 35 | return err 36 | } 37 | 38 | return nil 39 | } 40 | 41 | // WithJSON binds the passed struct pointer using the request json params, 42 | // fails with false returning and aborts the request with 400 status code. 43 | func WithJSON(c *gin.Context, r any) bool { 44 | if c.Request == nil || c.Request.Body == nil { 45 | return abortWithError(c, errors.New("invalid request")) 46 | } 47 | 48 | err := json.NewDecoder(c.Request.Body).Decode(r) 49 | 50 | return abortWithError(c, err) 51 | } 52 | 53 | // WithHeader binds the passed struct pointer using the request header params, 54 | // fails with false returning and aborts the request with 400 status code. 55 | func WithHeader(c *gin.Context, r any) bool { 56 | m := c.Request.Header 57 | 58 | err := MapFormWithTag(r, m, "header") 59 | 60 | return abortWithError(c, err) 61 | } 62 | 63 | // WithQuery binds the passed struct pointer using the request query params, 64 | // fails with false returning and aborts the request with 400 status code. 65 | func WithQuery(c *gin.Context, r any) bool { 66 | m := c.Request.URL.Query() 67 | 68 | err := MapFormWithTag(r, m, "query") 69 | 70 | return abortWithError(c, err) 71 | } 72 | 73 | // WithPath binds the passed struct pointer using the request path params, 74 | // fails with false returning and aborts the request with 400 status code. 75 | func WithPath(c *gin.Context, r any) bool { 76 | m := make(map[string][]string) 77 | for _, v := range c.Params { 78 | m[v.Key] = []string{v.Value} 79 | } 80 | 81 | err := MapFormWithTag(r, m, "path") 82 | 83 | return abortWithError(c, err) 84 | } 85 | 86 | // abortWithError breaks the call chain if found error, 87 | // and returns false. 88 | func abortWithError(c *gin.Context, err error) bool { 89 | if err != nil { 90 | _ = c.AbortWithError(http.StatusBadRequest, err). 91 | SetType(gin.ErrorTypeBind) 92 | return false 93 | } 94 | 95 | return true 96 | } 97 | -------------------------------------------------------------------------------- /pkg/apis/runtime/handler.go: -------------------------------------------------------------------------------- 1 | package runtime 2 | 3 | import ( 4 | "net/http" 5 | 6 | "github.com/gin-gonic/gin" 7 | "github.com/seal-io/walrus/utils/errorx" 8 | "github.com/seal-io/walrus/utils/log" 9 | ) 10 | 11 | type Handle = func(*gin.Context) 12 | 13 | type ErrorHandle = func(c *gin.Context) error 14 | 15 | func wrapErrorHandle(f ErrorHandle) Handle { 16 | return func(c *gin.Context) { 17 | if f == nil { 18 | c.Next() 19 | return 20 | } 21 | 22 | err := f(c) 23 | if err != nil { 24 | _ = c.Error(errorx.Wrap(err, "")) 25 | 26 | c.Abort() 27 | } 28 | } 29 | } 30 | 31 | type Handler interface { 32 | Handle(c *gin.Context) 33 | } 34 | 35 | func wrapHandler(h Handler) Handle { 36 | return func(c *gin.Context) { 37 | if h == nil { 38 | c.Next() 39 | return 40 | } 41 | 42 | h.Handle(c) 43 | } 44 | } 45 | 46 | type ErrorHandler interface { 47 | Handle(c *gin.Context) error 48 | } 49 | 50 | func wrapErrorHandler(h ErrorHandler) Handle { 51 | return func(c *gin.Context) { 52 | if h == nil { 53 | c.Next() 54 | return 55 | } 56 | 57 | err := h.Handle(c) 58 | if err != nil { 59 | _ = c.Error(errorx.Wrap(err, "")) 60 | 61 | c.Abort() 62 | } 63 | } 64 | } 65 | 66 | type HTTPHandler = http.Handler 67 | 68 | func wrapHTTPHandler(h http.Handler) Handle { 69 | return gin.WrapH(h) 70 | } 71 | 72 | type HTTPHandle = http.HandlerFunc 73 | 74 | func wrapHTTPHandle(h http.HandlerFunc) Handle { 75 | return gin.WrapF(h) 76 | } 77 | 78 | func asHandle(h IHandler) Handle { 79 | if h != nil { 80 | switch t := h.(type) { 81 | case Handle: 82 | return t 83 | case ErrorHandle: 84 | return wrapErrorHandle(t) 85 | case Handler: 86 | return wrapHandler(t) 87 | case ErrorHandler: 88 | return wrapErrorHandler(t) 89 | case HTTPHandle: 90 | return wrapHTTPHandle(t) 91 | case HTTPHandler: 92 | return wrapHTTPHandler(t) 93 | } 94 | } 95 | 96 | log.WithName("api"). 97 | Errorf("unknown handle type: %T", h) 98 | 99 | return func(c *gin.Context) { 100 | c.AbortWithStatus(http.StatusInternalServerError) 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /pkg/apis/runtime/metrics.go: -------------------------------------------------------------------------------- 1 | package runtime 2 | 3 | import ( 4 | "github.com/prometheus/client_golang/prometheus" 5 | ) 6 | 7 | var _statsCollector = newStatsCollector() 8 | 9 | func NewStatsCollector() prometheus.Collector { 10 | return _statsCollector 11 | } 12 | 13 | func newStatsCollector() *statsCollector { 14 | ns := "api" 15 | 16 | return &statsCollector{ 17 | requestInflight: prometheus.NewGaugeVec( 18 | prometheus.GaugeOpts{ 19 | Namespace: ns, 20 | Name: "request_inflight", 21 | Help: "The number of inflight request.", 22 | }, 23 | []string{"proto", "path", "method"}, 24 | ), 25 | requestCounter: prometheus.NewCounterVec( 26 | prometheus.CounterOpts{ 27 | Namespace: ns, 28 | Name: "request_total", 29 | Help: "The total number of requests.", 30 | }, 31 | []string{"proto", "path", "method", "code"}, 32 | ), 33 | requestDurations: prometheus.NewHistogramVec( 34 | prometheus.HistogramOpts{ 35 | Namespace: ns, 36 | Name: "request_duration_seconds", 37 | Help: "The response latency distribution in seconds.", 38 | Buckets: []float64{ 39 | 0.005, 40 | 0.025, 41 | 0.05, 42 | 0.1, 43 | 0.2, 44 | 0.4, 45 | 0.6, 46 | 0.8, 47 | 1.0, 48 | 1.25, 49 | 1.5, 50 | 2, 51 | 3, 52 | 4, 53 | 5, 54 | 6, 55 | 8, 56 | 10, 57 | 15, 58 | 20, 59 | 30, 60 | 45, 61 | 60, 62 | }, 63 | }, 64 | []string{"proto", "path", "method", "code"}, 65 | ), 66 | requestSizes: prometheus.NewHistogramVec( 67 | prometheus.HistogramOpts{ 68 | Namespace: ns, 69 | Name: "request_sizes", 70 | Help: "The request size distribution in bytes.", 71 | Buckets: prometheus.ExponentialBuckets(128, 2.0, 15), // 128B, 256B, ..., 2M. 72 | }, 73 | []string{"proto", "path", "method"}, 74 | ), 75 | responseSizes: prometheus.NewHistogramVec( 76 | prometheus.HistogramOpts{ 77 | Namespace: ns, 78 | Name: "response_sizes", 79 | Help: "The response size distribution in bytes.", 80 | Buckets: prometheus.ExponentialBuckets(128, 2.0, 15), // 128B, 256B, ..., 2M. 81 | }, 82 | []string{"proto", "path", "method"}, 83 | ), 84 | } 85 | } 86 | 87 | type statsCollector struct { 88 | requestInflight *prometheus.GaugeVec 89 | requestCounter *prometheus.CounterVec 90 | requestDurations *prometheus.HistogramVec 91 | requestSizes *prometheus.HistogramVec 92 | responseSizes *prometheus.HistogramVec 93 | } 94 | 95 | func (c *statsCollector) Describe(ch chan<- *prometheus.Desc) { 96 | c.requestInflight.Describe(ch) 97 | c.requestCounter.Describe(ch) 98 | c.requestDurations.Describe(ch) 99 | c.requestSizes.Describe(ch) 100 | c.responseSizes.Describe(ch) 101 | } 102 | 103 | func (c *statsCollector) Collect(ch chan<- prometheus.Metric) { 104 | c.requestInflight.Collect(ch) 105 | c.requestCounter.Collect(ch) 106 | c.requestDurations.Collect(ch) 107 | c.requestSizes.Collect(ch) 108 | c.responseSizes.Collect(ch) 109 | } 110 | -------------------------------------------------------------------------------- /pkg/apis/runtime/middleware_error.go: -------------------------------------------------------------------------------- 1 | package runtime 2 | 3 | import ( 4 | "errors" 5 | "net/http" 6 | "strings" 7 | 8 | "github.com/gin-gonic/gin" 9 | "github.com/seal-io/walrus/utils/errorx" 10 | "github.com/seal-io/walrus/utils/log" 11 | ) 12 | 13 | // erroring is a gin middleware, 14 | // which converts the chain calling error into response. 15 | func erroring(c *gin.Context) { 16 | c.Next() 17 | 18 | if len(c.Errors) == 0 { 19 | if c.Writer.Status() >= http.StatusBadRequest && c.Writer.Size() == 0 { 20 | // Detail the error status message. 21 | _ = c.Error(errorx.NewHttpError(c.Writer.Status(), "")) 22 | } else { 23 | // No errors. 24 | return 25 | } 26 | } 27 | 28 | // Get errors from chain and parse into response. 29 | he := getHttpError(c) 30 | 31 | // Log errors. 32 | if len(he.errs) != 0 && withinStacktraceStatus(he.Status) { 33 | reqMethod := c.Request.Method 34 | 35 | reqPath := c.Request.URL.Path 36 | if raw := c.Request.URL.RawQuery; raw != "" { 37 | reqPath = reqPath + "?" + raw 38 | } 39 | 40 | log.WithName("api"). 41 | Errorf("error requesting %s %s: %v", reqMethod, reqPath, errorx.Format(he.errs)) 42 | } 43 | 44 | c.AbortWithStatusJSON(he.Status, he) 45 | } 46 | 47 | func getHttpError(c *gin.Context) (he ErrorResponse) { 48 | var errs []error 49 | 50 | for i := range c.Errors { 51 | if c.Errors[i].Err != nil { 52 | errs = append(errs, c.Errors[i].Err) 53 | } 54 | } 55 | he.errs = errs 56 | 57 | if len(errs) == 0 { 58 | he.Status = http.StatusInternalServerError 59 | } else { 60 | // Get the public error. 61 | he.Status, he.Message = errorx.Public(errs) 62 | 63 | // Get the last error. 64 | if he.Status == 0 { 65 | st, msg := diagnoseError(c.Errors.Last()) 66 | he.Status = st 67 | 68 | if he.Message == "" { 69 | he.Message = msg 70 | } 71 | } 72 | } 73 | 74 | // Correct the code if already write within context. 75 | if c.Writer.Written() { 76 | he.Status = c.Writer.Status() 77 | } 78 | 79 | he.StatusText = http.StatusText(he.Status) 80 | 81 | return he 82 | } 83 | 84 | type ErrorResponse struct { 85 | Message string `json:"message"` 86 | Status int `json:"status"` 87 | StatusText string `json:"statusText"` 88 | 89 | // Errs is the all errors from gin context errors. 90 | errs []error 91 | } 92 | 93 | func diagnoseError(ge *gin.Error) (int, string) { 94 | c := http.StatusInternalServerError 95 | if ge.Type == gin.ErrorTypeBind { 96 | c = http.StatusBadRequest 97 | } 98 | 99 | var b strings.Builder 100 | 101 | if ge.Meta != nil { 102 | m, ok := ge.Meta.(string) 103 | if ok { 104 | b.WriteString("failed to ") 105 | b.WriteString(m) 106 | } 107 | } 108 | 109 | err := ge.Err 110 | if ue := errors.Unwrap(err); ue != nil { 111 | err = ue 112 | } 113 | 114 | // TODO: distinguish between internal and external errors. 115 | b.WriteString(err.Error()) 116 | 117 | return c, b.String() 118 | } 119 | 120 | func withinStacktraceStatus(status int) bool { 121 | return (status < http.StatusOK || status >= http.StatusInternalServerError) && 122 | status != http.StatusSwitchingProtocols 123 | } 124 | -------------------------------------------------------------------------------- /pkg/apis/runtime/middleware_filter.go: -------------------------------------------------------------------------------- 1 | package runtime 2 | 3 | import ( 4 | "net/http" 5 | "sync" 6 | 7 | "github.com/gin-gonic/gin" 8 | ) 9 | 10 | // Only is a gin middleware, 11 | // which is used for judging the incoming request, 12 | // aborts with 403 if not match. 13 | func Only(match func(*gin.Context) bool) Handle { 14 | if match == nil { 15 | return next() 16 | } 17 | 18 | return func(c *gin.Context) { 19 | if !match(c) { 20 | c.AbortWithStatus(http.StatusForbidden) 21 | return 22 | } 23 | 24 | c.Next() 25 | } 26 | } 27 | 28 | // OnlyLocalIP judges the incoming request whether is from localhost, 29 | // aborts with 403 if not match. 30 | func OnlyLocalIP() Handle { 31 | isLocalIP := func(c *gin.Context) bool { 32 | host := c.Request.Host 33 | if host == "127.0.0.1" || host == "localhost" || host == "::1" { 34 | ip := c.RemoteIP() 35 | return ip == "::1" || host == ip 36 | } 37 | 38 | return false 39 | } 40 | 41 | return Only(isLocalIP) 42 | } 43 | 44 | // If is a gin middleware, 45 | // which is used for judging the incoming request, 46 | // execute given handle if matched. 47 | func If(match func(*gin.Context) bool, then Handle) Handle { 48 | if match == nil || then == nil { 49 | return next() 50 | } 51 | 52 | return func(c *gin.Context) { 53 | if !match(c) { 54 | c.Next() 55 | return 56 | } 57 | 58 | then(c) 59 | } 60 | } 61 | 62 | // Per is a gin middleware, 63 | // which is used for providing new handler for different incoming request. 64 | func Per(hashRequest func(*gin.Context) string, provideHandler func() Handle) Handle { 65 | if hashRequest == nil || provideHandler == nil { 66 | return next() 67 | } 68 | 69 | var m sync.Map 70 | 71 | return func(c *gin.Context) { 72 | k := hashRequest(c) 73 | 74 | var h Handle 75 | 76 | v, ok := m.LoadOrStore(k, nil) 77 | if !ok { 78 | h = provideHandler() 79 | m.Store(k, h) 80 | } else { 81 | h = v.(Handle) 82 | } 83 | 84 | h(c) 85 | } 86 | } 87 | 88 | // PerIP provides new handler according to incoming request IP. 89 | func PerIP(provideHandler func() Handle) Handle { 90 | hashRequestByIP := func(c *gin.Context) string { 91 | return c.ClientIP() 92 | } 93 | 94 | return Per(hashRequestByIP, provideHandler) 95 | } 96 | 97 | func next() Handle { 98 | return func(c *gin.Context) { c.Next() } 99 | } 100 | -------------------------------------------------------------------------------- /pkg/apis/runtime/middleware_flowcontrol.go: -------------------------------------------------------------------------------- 1 | package runtime 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "net/http" 7 | "sync/atomic" 8 | "time" 9 | "unsafe" 10 | 11 | "github.com/gin-gonic/gin" 12 | "golang.org/x/time/rate" 13 | ) 14 | 15 | // RequestCounting limits the request count in the given maximum, 16 | // returns 429 if the new request has waited with the given duration. 17 | func RequestCounting(max int, wait time.Duration) Handle { 18 | if max <= 0 { 19 | return func(c *gin.Context) { 20 | c.AbortWithStatus(http.StatusTooManyRequests) 21 | } 22 | } 23 | 24 | limiter := make(chan struct{}, max) 25 | 26 | var token struct{} 27 | 28 | if wait <= 0 { 29 | return func(c *gin.Context) { 30 | select { 31 | default: 32 | if c.Err() == nil { 33 | c.AbortWithStatus(http.StatusTooManyRequests) 34 | return 35 | } 36 | 37 | c.Abort() 38 | case limiter <- token: 39 | defer func() { <-limiter }() 40 | c.Next() 41 | } 42 | } 43 | } 44 | 45 | return func(c *gin.Context) { 46 | ctx, cancel := context.WithTimeout(c, wait) 47 | defer cancel() 48 | select { 49 | case <-ctx.Done(): 50 | if errors.Is(ctx.Err(), context.DeadlineExceeded) { 51 | c.AbortWithStatus(http.StatusTooManyRequests) 52 | return 53 | } 54 | 55 | c.Abort() 56 | case limiter <- token: 57 | defer func() { <-limiter }() 58 | c.Next() 59 | } 60 | } 61 | } 62 | 63 | // RequestThrottling controls the request count per second and allows bursting, 64 | // returns 429 if the new request is not allowed. 65 | func RequestThrottling(qps, burst int) Handle { 66 | if qps <= 0 || burst <= 0 { 67 | return func(c *gin.Context) { 68 | c.AbortWithStatus(http.StatusTooManyRequests) 69 | } 70 | } 71 | 72 | limiter := rate.NewLimiter(rate.Limit(qps), burst) 73 | 74 | return func(c *gin.Context) { 75 | if !limiter.Allow() { 76 | if c.Err() == nil { 77 | c.AbortWithStatus(http.StatusTooManyRequests) 78 | return 79 | } 80 | 81 | c.Abort() 82 | 83 | return 84 | } 85 | 86 | c.Next() 87 | } 88 | } 89 | 90 | // RequestShaping arranges all requests to be received on the given qps, 91 | // returns 429 if the new request can be allowed within the given latency, 92 | // if the given latency is not positive, RequestShaping will never return 429. 93 | func RequestShaping(qps, slack int, latency time.Duration) Handle { 94 | if qps <= 0 { 95 | return func(c *gin.Context) { 96 | c.AbortWithStatus(http.StatusTooManyRequests) 97 | } 98 | } 99 | 100 | type state struct { 101 | arrival time.Time 102 | sleep time.Duration 103 | } 104 | window := time.Second / time.Duration(qps) 105 | maxSleep := -1 * time.Duration(slack) * window 106 | statePointer := func() unsafe.Pointer { 107 | var s state 108 | return unsafe.Pointer(&s) 109 | }() 110 | 111 | return func(c *gin.Context) { 112 | for { 113 | select { 114 | case <-c.Done(): 115 | c.Abort() 116 | return 117 | default: 118 | } 119 | 120 | prevStatePointer := atomic.LoadPointer(&statePointer) 121 | prevState := (*state)(prevStatePointer) 122 | currState := state{ 123 | arrival: time.Now(), 124 | sleep: prevState.sleep, 125 | } 126 | 127 | // For first request. 128 | if prevState.arrival.IsZero() { 129 | taken := atomic.CompareAndSwapPointer(&statePointer, 130 | prevStatePointer, unsafe.Pointer(&currState)) 131 | if !taken { 132 | continue 133 | } 134 | // Allow it immediately. 135 | c.Next() 136 | 137 | return 138 | } 139 | 140 | // For subsequent requests. 141 | currState.sleep += window - currState.arrival.Sub(prevState.arrival) 142 | if currState.sleep < maxSleep { 143 | currState.sleep = maxSleep 144 | } 145 | 146 | var wait time.Duration 147 | 148 | if currState.sleep > 0 { 149 | currState.arrival = currState.arrival.Add(currState.sleep) 150 | wait, currState.sleep = currState.sleep, 0 151 | } 152 | 153 | if latency > 0 && wait > latency { 154 | c.AbortWithStatus(http.StatusTooManyRequests) 155 | return 156 | } 157 | 158 | taken := atomic.CompareAndSwapPointer(&statePointer, prevStatePointer, unsafe.Pointer(&currState)) 159 | if !taken { 160 | continue 161 | } 162 | // Allow it after waiting. 163 | t := time.NewTimer(wait) 164 | select { 165 | case <-t.C: 166 | t.Stop() 167 | c.Next() 168 | case <-c.Done(): 169 | t.Stop() 170 | c.Abort() 171 | } 172 | 173 | return 174 | } 175 | } 176 | } 177 | -------------------------------------------------------------------------------- /pkg/apis/runtime/middleware_observation.go: -------------------------------------------------------------------------------- 1 | package runtime 2 | 3 | import ( 4 | "strconv" 5 | "strings" 6 | "time" 7 | 8 | humanize "github.com/dustin/go-humanize" 9 | "github.com/gin-gonic/gin" 10 | "github.com/seal-io/walrus/utils/log" 11 | "k8s.io/apimachinery/pkg/util/sets" 12 | ) 13 | 14 | // SkipLoggingPaths is a RouterOption to ignore logging for the given paths. 15 | func SkipLoggingPaths(paths ...string) RouterOption { 16 | return routerOption(func(r *Router) { 17 | for i := range paths { 18 | skipLoggingPath(paths[i]) 19 | } 20 | }) 21 | } 22 | 23 | var ( 24 | pathSkipLogging = sets.New[string]() 25 | pathPrefixSkipLogging = sets.New[string]() 26 | ) 27 | 28 | func skipLoggingPath(p string) { 29 | pathSkipLogging.Insert(p) 30 | 31 | lastIdx := strings.LastIndex(p, "/") + 1 32 | if lastIdx <= 0 { 33 | return 34 | } 35 | 36 | lastSeg := p[lastIdx:] 37 | if !strings.HasPrefix(lastSeg, "*") { 38 | return 39 | } 40 | 41 | pathPrefixSkipLogging.Insert(p[:lastIdx]) 42 | } 43 | 44 | // observing is a gin middleware, 45 | // it measures the http request/response with logging and monitoring. 46 | func observing(c *gin.Context) { 47 | logger := log.WithName("api") 48 | 49 | // Validate to skip logging or not. 50 | skipLogging := !logger.Enabled(log.DebugLevel) 51 | 52 | reqPath := c.FullPath() 53 | if reqPath == "" { 54 | reqPath = c.Request.URL.Path 55 | } 56 | 57 | if pathSkipLogging.Has(reqPath) { 58 | skipLogging = true 59 | } else if i := strings.LastIndex(reqPath, "/") + 1; i > 0 { 60 | if pathPrefixSkipLogging.Has(reqPath[:i]) { 61 | skipLogging = true 62 | } 63 | } 64 | 65 | reqProto := c.Request.Proto 66 | reqMethod := c.Request.Method 67 | 68 | switch { 69 | case IsUnidiStreamRequest(c): 70 | reqMethod = "US" 71 | case IsBidiStreamRequest(c): 72 | reqMethod = "BS" 73 | } 74 | 75 | // Record inflight request. 76 | _statsCollector.requestInflight. 77 | WithLabelValues(reqProto, reqPath, reqMethod). 78 | Inc() 79 | 80 | defer func() { 81 | _statsCollector.requestInflight. 82 | WithLabelValues(reqProto, reqPath, reqMethod). 83 | Dec() 84 | }() 85 | 86 | start := time.Now() 87 | 88 | c.Next() 89 | 90 | reqLatency := time.Since(start) 91 | 92 | reqSize := c.Request.ContentLength 93 | if v := c.GetInt64("request_size"); v != 0 { 94 | reqSize = v 95 | } 96 | 97 | respStatus := strconv.Itoa(c.Writer.Status()) 98 | if v := c.GetInt("response_status"); v != 0 { 99 | respStatus = strconv.Itoa(v) 100 | } 101 | 102 | var respSize int64 103 | if c.Writer.Written() { 104 | respSize = int64(c.Writer.Size()) 105 | } else if v := c.GetInt64("response_size"); v != 0 { 106 | respSize = v 107 | } 108 | 109 | // Record request latency. 110 | _statsCollector.requestDurations. 111 | WithLabelValues(reqProto, reqPath, reqMethod, respStatus). 112 | Observe(reqLatency.Seconds()) 113 | 114 | // Record request time. 115 | _statsCollector.requestCounter. 116 | WithLabelValues(reqProto, reqPath, reqMethod, respStatus). 117 | Inc() 118 | 119 | // Record request size. 120 | _statsCollector.requestSizes. 121 | WithLabelValues(reqProto, reqPath, reqMethod). 122 | Observe(float64(reqSize)) 123 | 124 | // Record response size. 125 | _statsCollector.responseSizes. 126 | WithLabelValues(reqProto, reqPath, reqMethod). 127 | Observe(float64(respSize)) 128 | 129 | if !skipLogging { 130 | // Complete logging info. 131 | reqSize := humanize.IBytes(uint64(reqSize)) 132 | respSize := humanize.IBytes(uint64(respSize)) 133 | 134 | reqLatency := reqLatency 135 | if reqLatency > time.Minute { 136 | reqLatency -= reqLatency % time.Second 137 | } 138 | 139 | reqClientIP := c.ClientIP() 140 | 141 | reqPath := c.Request.URL.Path 142 | if raw := c.Request.URL.RawQuery; raw != "" { 143 | reqPath = reqPath + "?" + raw 144 | } 145 | 146 | logger.Debugf("%s | %8s | %10s | %10s | %13v | %15s | %-7s %s", 147 | respStatus, 148 | reqProto, 149 | reqSize, 150 | respSize, 151 | reqLatency, 152 | reqClientIP, 153 | reqMethod, 154 | reqPath, 155 | ) 156 | } 157 | } 158 | -------------------------------------------------------------------------------- /pkg/apis/runtime/middleware_recovery.go: -------------------------------------------------------------------------------- 1 | package runtime 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "net/http" 7 | "runtime" 8 | 9 | "github.com/gin-gonic/gin" 10 | "github.com/seal-io/walrus/utils/log" 11 | ) 12 | 13 | // recovering is a gin middleware, 14 | // which is the same as gin.Recovery, 15 | // but friendly message information can be provided according to the request header. 16 | func recovering(c *gin.Context) { 17 | defer func() { 18 | if r := recover(); r != nil { 19 | err, ok := r.(error) 20 | if !ok { 21 | err = fmt.Errorf("panic observing: %v", r) 22 | } 23 | 24 | log.WithName("api"). 25 | Errorf("panic observing: %v, callstack: \n%s", 26 | err, getPanicCallstack(3)) 27 | 28 | if isStreamRequest(c) { 29 | // Stream request always send header at first, 30 | // so we don't need to rewrite. 31 | return 32 | } 33 | 34 | c.AbortWithStatus(http.StatusInternalServerError) 35 | } 36 | }() 37 | 38 | c.Next() 39 | } 40 | 41 | func getPanicCallstack(skip int) []byte { 42 | var buf bytes.Buffer 43 | 44 | for i := skip; ; i++ { 45 | pc, file, line, ok := runtime.Caller(i) 46 | if !ok { 47 | break 48 | } 49 | fn := "???" 50 | f := runtime.FuncForPC(pc) 51 | 52 | if f != nil { 53 | fn = f.Name() 54 | } 55 | _, _ = fmt.Fprintf(&buf, "%s\n\t%s:%d (0x%x)\n", fn, file, line, pc) 56 | } 57 | 58 | return buf.Bytes() 59 | } 60 | -------------------------------------------------------------------------------- /pkg/apis/runtime/middleware_route.go: -------------------------------------------------------------------------------- 1 | package runtime 2 | 3 | import ( 4 | "net/http" 5 | 6 | "github.com/gin-gonic/gin" 7 | ) 8 | 9 | // noMethod is a gin middleware, 10 | // it aborts the incoming request of not method implementation. 11 | func noMethod(c *gin.Context) { 12 | c.AbortWithStatus(http.StatusMethodNotAllowed) 13 | } 14 | 15 | // noRoute is a gin middleware, 16 | // it aborts the incoming request of not found route. 17 | func noRoute(c *gin.Context) { 18 | c.AbortWithStatus(http.StatusNotFound) 19 | } 20 | -------------------------------------------------------------------------------- /pkg/apis/runtime/openapi/extension.go: -------------------------------------------------------------------------------- 1 | package openapi 2 | 3 | // OpenAPI Extensions. 4 | const ( 5 | // ExtCliOperationName define the extension key to set the CLI operation name. 6 | ExtCliOperationName = "x-cli-operation-name" 7 | 8 | // ExtCliSchemaTypeName define the extension key to set the CLI operation params schema type. 9 | ExtCliSchemaTypeName = "x-cli-schema-type" 10 | 11 | // ExtCliIgnore define the extension key to ignore generate the api operation to the cli used api.json. 12 | ExtCliIgnore = "x-cli-ignore" 13 | 14 | // ExtCliCmdIgnore define the extension key to generate the operation to api.json but will not generate cli command. 15 | ExtCliCmdIgnore = "x-cli-cmd-ignore" 16 | 17 | // ExtCliOutputFormat define the output format set the CLI operation command. 18 | ExtCliOutputFormat = "x-cli-output-format" 19 | 20 | // ExtVersionGitCommit define the git commit hash version for openapi. 21 | ExtVersionGitCommit = "x-version-git-commit" 22 | 23 | // ExtCliTableColumn define the extension key to use in table format. 24 | ExtCliTableColumn = "x-cli-table-column" 25 | ) 26 | -------------------------------------------------------------------------------- /pkg/apis/runtime/request.go: -------------------------------------------------------------------------------- 1 | package runtime 2 | 3 | // RequestPagination holds the requesting pagination data. 4 | type RequestPagination struct { 5 | // Page specifies the page number for querying, 6 | // i.e. /v1/repositories?page=1&perPage=10. 7 | Page int `query:"page,default=1"` 8 | 9 | // PerPage specifies the page size for querying, 10 | // i.e. /v1/repositories?page=1&perPage=10. 11 | PerPage int `query:"perPage,default=100"` 12 | } 13 | 14 | // Limit returns the limit of paging. 15 | func (r RequestPagination) Limit() int { 16 | limit := r.PerPage 17 | if limit <= 0 { 18 | limit = 100 19 | } 20 | 21 | return limit 22 | } 23 | 24 | // Offset returns the offset of paging. 25 | func (r RequestPagination) Offset() int { 26 | offset := r.Limit() * (r.Page - 1) 27 | if offset < 0 { 28 | offset = 0 29 | } 30 | 31 | return offset 32 | } 33 | 34 | // Paging returns the limit and offset of paging, 35 | // returns false if there is no pagination requesting. 36 | func (r RequestPagination) Paging() (limit, offset int, request bool) { 37 | request = r.Page > 0 38 | if !request { 39 | return limit, offset, request 40 | } 41 | limit = r.Limit() 42 | offset = r.Offset() 43 | 44 | return limit, offset, request 45 | } 46 | -------------------------------------------------------------------------------- /pkg/apis/runtime/request_stream.go: -------------------------------------------------------------------------------- 1 | package runtime 2 | 3 | import ( 4 | "context" 5 | "io" 6 | "sync" 7 | "sync/atomic" 8 | "time" 9 | 10 | "github.com/gin-gonic/gin" 11 | "github.com/gorilla/websocket" 12 | "github.com/seal-io/walrus/utils/json" 13 | ) 14 | 15 | // RequestUnidiStream holds the request for single-direction stream. 16 | type RequestUnidiStream struct { 17 | ctx context.Context 18 | ctxCancel func() 19 | conn gin.ResponseWriter 20 | } 21 | 22 | // Write implements io.Writer. 23 | func (r RequestUnidiStream) Write(p []byte) (n int, err error) { 24 | n, err = r.conn.Write(p) 25 | if err != nil { 26 | return n, err 27 | } 28 | 29 | r.conn.Flush() 30 | 31 | return n, err 32 | } 33 | 34 | // SendMsg sends the given data to client. 35 | func (r RequestUnidiStream) SendMsg(data []byte) error { 36 | _, err := r.Write(data) 37 | return err 38 | } 39 | 40 | // SendJSON marshals the given object as JSON and sends to client. 41 | func (r RequestUnidiStream) SendJSON(i any) error { 42 | bs, err := json.Marshal(i) 43 | if err != nil { 44 | return err 45 | } 46 | 47 | return r.SendMsg(bs) 48 | } 49 | 50 | // Cancel cancels the underlay context.Context. 51 | func (r RequestUnidiStream) Cancel() { 52 | r.ctxCancel() 53 | } 54 | 55 | // Deadline implements context.Context. 56 | func (r RequestUnidiStream) Deadline() (deadline time.Time, ok bool) { 57 | return r.ctx.Deadline() 58 | } 59 | 60 | // Done implements context.Context. 61 | func (r RequestUnidiStream) Done() <-chan struct{} { 62 | return r.ctx.Done() 63 | } 64 | 65 | // Err implements context.Context. 66 | func (r RequestUnidiStream) Err() error { 67 | return r.ctx.Err() 68 | } 69 | 70 | // Value implements context.Context. 71 | func (r RequestUnidiStream) Value(key any) any { 72 | return r.ctx.Value(key) 73 | } 74 | 75 | // RequestBidiStream holds the request for dual-directions stream. 76 | type RequestBidiStream struct { 77 | firstReadOnce *sync.Once 78 | firstReadChan <-chan struct { 79 | t int 80 | r io.Reader 81 | e error 82 | } 83 | ctx context.Context 84 | ctxCancel func() 85 | conn *websocket.Conn 86 | connReadBytes *atomic.Int64 87 | connWriteBytes *atomic.Int64 88 | } 89 | 90 | // Read implements io.Reader. 91 | func (r RequestBidiStream) Read(p []byte) (n int, err error) { 92 | var ( 93 | firstRead bool 94 | msgType int 95 | msgReader io.Reader 96 | ) 97 | 98 | r.firstReadOnce.Do(func() { 99 | fr, ok := <-r.firstReadChan 100 | if !ok { 101 | return 102 | } 103 | firstRead = true 104 | msgType, msgReader, err = fr.t, fr.r, fr.e 105 | }) 106 | 107 | if !firstRead { 108 | msgType, msgReader, err = r.conn.NextReader() 109 | } 110 | 111 | if err != nil { 112 | return n, err 113 | } 114 | 115 | switch msgType { 116 | default: 117 | err = &websocket.CloseError{ 118 | Code: websocket.CloseUnsupportedData, 119 | Text: "unresolved message type: binary", 120 | } 121 | 122 | return n, err 123 | case websocket.TextMessage: 124 | } 125 | 126 | n, err = msgReader.Read(p) 127 | if err == nil { 128 | // Measure read bytes. 129 | r.connReadBytes.Add(int64(n)) 130 | } 131 | 132 | return n, err 133 | } 134 | 135 | // Write implements io.Writer. 136 | func (r RequestBidiStream) Write(p []byte) (n int, err error) { 137 | msgWriter, err := r.conn.NextWriter(websocket.TextMessage) 138 | if err != nil { 139 | return n, err 140 | } 141 | 142 | defer func() { _ = msgWriter.Close() }() 143 | 144 | n, err = msgWriter.Write(p) 145 | if err == nil { 146 | // Measure write bytes. 147 | r.connWriteBytes.Add(int64(n)) 148 | } 149 | 150 | return n, err 151 | } 152 | 153 | // RecvMsg receives message from client. 154 | func (r RequestBidiStream) RecvMsg() ([]byte, error) { 155 | return io.ReadAll(r) 156 | } 157 | 158 | // SendMsg sends the given data to client. 159 | func (r RequestBidiStream) SendMsg(data []byte) error { 160 | _, err := r.Write(data) 161 | return err 162 | } 163 | 164 | // RecvJSON receives JSON message from client and unmarshals into the given object. 165 | func (r RequestBidiStream) RecvJSON(i any) error { 166 | bs, err := r.RecvMsg() 167 | if err != nil { 168 | return err 169 | } 170 | 171 | return json.Unmarshal(bs, i) 172 | } 173 | 174 | // SendJSON marshals the given object as JSON and sends to client. 175 | func (r RequestBidiStream) SendJSON(i any) error { 176 | bs, err := json.Marshal(i) 177 | if err != nil { 178 | return err 179 | } 180 | 181 | return r.SendMsg(bs) 182 | } 183 | 184 | // Cancel cancels the underlay context.Context. 185 | func (r RequestBidiStream) Cancel() { 186 | r.ctxCancel() 187 | } 188 | 189 | // Deadline implements context.Context. 190 | func (r RequestBidiStream) Deadline() (deadline time.Time, ok bool) { 191 | return r.ctx.Deadline() 192 | } 193 | 194 | // Done implements context.Context. 195 | func (r RequestBidiStream) Done() <-chan struct{} { 196 | return r.ctx.Done() 197 | } 198 | 199 | // Err implements context.Context. 200 | func (r RequestBidiStream) Err() error { 201 | return r.ctx.Err() 202 | } 203 | 204 | // Value implements context.Context. 205 | func (r RequestBidiStream) Value(key any) any { 206 | return r.ctx.Value(key) 207 | } 208 | -------------------------------------------------------------------------------- /pkg/apis/runtime/response.go: -------------------------------------------------------------------------------- 1 | package runtime 2 | 3 | import ( 4 | "io" 5 | "math" 6 | "net/http" 7 | "reflect" 8 | "strconv" 9 | 10 | "github.com/gin-gonic/gin" 11 | 12 | "github.com/seal-io/hermitcrab/pkg/apis/runtime/bind" 13 | ) 14 | 15 | // NoPageResponse returns the given data without pagination. 16 | func NoPageResponse(data any) *ResponseCollection { 17 | return &ResponseCollection{ 18 | Items: data, 19 | } 20 | } 21 | 22 | // PageResponse returns the given data in a pagination, 23 | // which calculates the pagination info with the given request page and perPage. 24 | func PageResponse(page, perPage int, data any, dataTotalSize int) *ResponseCollection { 25 | var currentSize int 26 | 27 | dataRef := reflect.ValueOf(data) 28 | if dataRef.Kind() == reflect.Slice { 29 | currentSize = dataRef.Len() 30 | } 31 | 32 | var totalPage int 33 | 34 | if page > 0 { 35 | if perPage <= 0 { 36 | perPage = 100 37 | } 38 | totalPage = int(math.Ceil(float64(dataTotalSize) / float64(perPage))) 39 | } else { 40 | page = 1 41 | perPage = dataTotalSize 42 | totalPage = 1 43 | } 44 | 45 | partial := currentSize < dataTotalSize 46 | 47 | nextPage := page + 1 48 | if !partial || nextPage > totalPage { 49 | nextPage = 0 50 | } 51 | 52 | return &ResponseCollection{ 53 | Items: data, 54 | Pagination: &ResponsePagination{ 55 | Page: page, 56 | PerPage: perPage, 57 | Total: dataTotalSize, 58 | TotalPage: totalPage, 59 | Partial: partial, 60 | NextPage: nextPage, 61 | }, 62 | } 63 | } 64 | 65 | // FullPageResponse returns the given data in a pagination, 66 | // which treats the given data as a full page. 67 | func FullPageResponse(data any, dataTotalSize int) *ResponseCollection { 68 | return PageResponse(-1, 0, data, dataTotalSize) 69 | } 70 | 71 | // TypedResponse returns the given data in typed. 72 | func TypedResponse(typ string, data any) *ResponseCollection { 73 | return &ResponseCollection{ 74 | Type: typ, 75 | Items: data, 76 | } 77 | } 78 | 79 | // getPageResponse gains the request pagination from request, 80 | // and returns the given data in a pagination. 81 | func getPageResponse(c *gin.Context, data any, dataTotalSize int) *ResponseCollection { 82 | reqPagination := RequestPagination{ 83 | Page: -1, 84 | } 85 | bind.WithQuery(c, &reqPagination) 86 | 87 | return PageResponse(reqPagination.Page, reqPagination.PerPage, data, dataTotalSize) 88 | } 89 | 90 | // ResponseCollection holds the response data of collection with a pagination. 91 | type ResponseCollection struct { 92 | Type string `json:"type,omitempty"` 93 | Items any `json:"items"` 94 | Pagination *ResponsePagination `json:"pagination,omitempty"` 95 | } 96 | 97 | // ResponsePagination holds the pagination data. 98 | type ResponsePagination struct { 99 | Page int `json:"page"` 100 | PerPage int `json:"perPage"` 101 | Total int `json:"total"` 102 | TotalPage int `json:"totalPage"` 103 | Partial bool `json:"partial"` 104 | NextPage int `json:"nextPage,omitempty"` 105 | } 106 | 107 | // ResponseFile is similar to render.Reader, 108 | // but be able to close the file reader out of the handler processing. 109 | type ResponseFile struct { 110 | ContentType string 111 | ContentLength int64 112 | Headers map[string]string 113 | Reader io.ReadCloser 114 | } 115 | 116 | func (r ResponseFile) Render(w http.ResponseWriter) (err error) { 117 | r.WriteContentType(w) 118 | 119 | if r.ContentLength > 0 { 120 | if r.Headers == nil { 121 | r.Headers = map[string]string{} 122 | } 123 | r.Headers["Content-Length"] = strconv.FormatInt(r.ContentLength, 10) 124 | } 125 | 126 | header := w.Header() 127 | for k, v := range r.Headers { 128 | if header.Get(k) == "" { 129 | header.Set(k, v) 130 | } 131 | } 132 | _, err = io.Copy(w, r.Reader) 133 | 134 | return err 135 | } 136 | 137 | func (r ResponseFile) WriteContentType(w http.ResponseWriter) { 138 | header := w.Header() 139 | if vs := header["Content-Type"]; len(vs) == 0 { 140 | contentType := "application/octet-stream" 141 | if r.ContentType != "" { 142 | contentType = r.ContentType 143 | } 144 | header["Content-Type"] = []string{contentType} 145 | } 146 | } 147 | 148 | func (r ResponseFile) Close() error { 149 | if r.Reader == nil { 150 | return nil 151 | } 152 | 153 | return r.Reader.Close() 154 | } 155 | -------------------------------------------------------------------------------- /pkg/apis/runtime/router.go: -------------------------------------------------------------------------------- 1 | package runtime 2 | 3 | import ( 4 | "net/http" 5 | "path" 6 | 7 | "github.com/gin-gonic/gin" 8 | "github.com/gin-gonic/gin/binding" 9 | ) 10 | 11 | func init() { 12 | // Disable gin default binding. 13 | binding.Validator = nil 14 | } 15 | 16 | type ( 17 | IHandler any 18 | 19 | IResourceHandler interface { 20 | IHandler 21 | 22 | Kind() string 23 | } 24 | 25 | IRouter interface { 26 | http.Handler 27 | 28 | // Use attaches a global middleware to the router. 29 | Use(...IHandler) IRouter 30 | // Group creates a new router group with the given string, 31 | // and returns the new router group. 32 | Group(string) IRouter 33 | // GroupIn creates a new router group with the given string, 34 | // but returns the original router group, 35 | // the new router group is passed to the given function. 36 | GroupIn(string, func(groupRouter IRouter)) IRouter 37 | // GroupRelativePath returns the relative path of router group. 38 | GroupRelativePath() string 39 | 40 | // Static registers GET/HEAD routers to serve the handler. 41 | Static(string, http.FileSystem) IRouter 42 | // Get registers GET router to serve the handler. 43 | Get(string, IHandler) IRouter 44 | // Post registers POST router to serve the handler. 45 | Post(string, IHandler) IRouter 46 | // Delete registers DELETE router to serve the handler. 47 | Delete(string, IHandler) IRouter 48 | // Patch registers PATCH router to serve the handler. 49 | Patch(string, IHandler) IRouter 50 | // Put registers PUT router to serve the handler. 51 | Put(string, IHandler) IRouter 52 | 53 | // Routes registers the reflected routes of a IHandler. 54 | // 55 | // Routes reflects the function descriptors as the below rules, 56 | // if the handler implements IResourceHandler as well. 57 | // 58 | // Input : struct type. 59 | // Output: any types. 60 | // 61 | // * Basic APIs 62 | // 63 | // func Create() (, error) 64 | // -> POST / 65 | // func Get() (, error) 66 | // -> GET //:id(?watch=true) 67 | // func Update() error 68 | // -> PUT //:id 69 | // func Delete() error 70 | // -> DELETE //:id 71 | // func CollectionCreate() (, error) 72 | // -> POST //_/batch 73 | // func CollectionGet() (, (int,) error) 74 | // -> GET /(?watch=true) 75 | // func CollectionUpdate() error 76 | // -> PUT / 77 | // func CollectionDelete() error 78 | // -> DELETE / 79 | // 80 | // * Extensional APIs 81 | // 82 | // func Route() ((), (int,) error) 83 | // -> method //:id/(?watch=true) 84 | // func CollectionRoute() ((), (int,) error) 85 | // -> method //_/(?watch=true) 86 | // 87 | // Otherwise, Routes tries to reflect the function descriptors as the below rules. 88 | // 89 | // Input : struct type. 90 | // Output: any types. 91 | // 92 | // func () ((), (int,) error) 93 | // -> method /(?watch=true) 94 | // 95 | Routes(IHandler) IRouter 96 | } 97 | ) 98 | 99 | type ( 100 | RouterOption interface { 101 | isOption() 102 | } 103 | 104 | RouterOptions []RouterOption 105 | 106 | Router struct { 107 | options RouterOptions 108 | engine *gin.Engine 109 | router gin.IRouter 110 | 111 | adviceProviders []RouteAdviceProvider 112 | authorizer RouteAuthorizer 113 | } 114 | ) 115 | 116 | // Apply applies the options one by one, 117 | // and returns the residual options. 118 | func (opts RouterOptions) Apply(fn func(o RouterOption) bool) (rOpts RouterOptions) { 119 | for i := range opts { 120 | if opts[i] == nil { 121 | continue 122 | } 123 | 124 | if !fn(opts[i]) { 125 | rOpts = append(rOpts, opts[i]) 126 | } 127 | } 128 | 129 | return rOpts 130 | } 131 | 132 | func NewRouter(options ...RouterOption) IRouter { 133 | opts := RouterOptions(options) 134 | 135 | // Apply global options. 136 | opts = opts.Apply(func(o RouterOption) bool { 137 | op, ok := o.(ginGlobalOption) 138 | if ok { 139 | op() 140 | } 141 | 142 | return ok 143 | }) 144 | 145 | e := gin.New() 146 | e.NoMethod(noMethod) 147 | e.NoRoute(noRoute) 148 | 149 | // Apply engine options. 150 | opts = opts.Apply(func(o RouterOption) bool { 151 | op, ok := o.(ginEngineOption) 152 | if ok { 153 | op(e) 154 | } 155 | 156 | return ok 157 | }) 158 | 159 | rt := &Router{ 160 | options: opts, 161 | engine: e, 162 | router: e, 163 | } 164 | 165 | // Apply router options. 166 | rt.options = opts.Apply(func(o RouterOption) bool { 167 | op, ok := o.(routerOption) 168 | if ok { 169 | op(rt) 170 | } 171 | 172 | return ok 173 | }) 174 | 175 | e.Use(observing, recovering, erroring) 176 | 177 | // Apply route options. 178 | rt.options = rt.options.Apply(func(o RouterOption) bool { 179 | op, ok := o.(ginRouteOption) 180 | if ok { 181 | op(rt.engine) 182 | } 183 | 184 | return ok 185 | }) 186 | 187 | return rt 188 | } 189 | 190 | func (rt *Router) ServeHTTP(resp http.ResponseWriter, req *http.Request) { 191 | rt.engine.ServeHTTP(resp, req) 192 | } 193 | 194 | func (rt *Router) Use(handlers ...IHandler) IRouter { 195 | hs := make([]gin.HandlerFunc, 0, len(handlers)) 196 | for i := range handlers { 197 | hs = append(hs, asHandle(handlers[i])) 198 | } 199 | 200 | rt.router.Use(hs...) 201 | 202 | return rt 203 | } 204 | 205 | func (rt *Router) Group(relativePath string) IRouter { 206 | grt := *rt 207 | grt.router = rt.router.Group(relativePath) 208 | 209 | return &grt 210 | } 211 | 212 | func (rt *Router) GroupIn(relativePath string, doGroupRoute func(IRouter)) IRouter { 213 | grt := *rt 214 | grt.router = rt.router.Group(relativePath) 215 | 216 | if doGroupRoute != nil { 217 | doGroupRoute(&grt) 218 | } 219 | 220 | return rt 221 | } 222 | 223 | func (rt *Router) GroupRelativePath() string { 224 | if t, ok := rt.router.(interface{ BasePath() string }); ok { 225 | return t.BasePath() 226 | } 227 | 228 | return "/" 229 | } 230 | 231 | func (rt *Router) Static(p string, fs http.FileSystem) IRouter { 232 | skipLoggingPath(path.Join(p, "/*filepath")) 233 | rt.engine.StaticFS(p, fs) 234 | 235 | return rt 236 | } 237 | 238 | func (rt *Router) Get(path string, handler IHandler) IRouter { 239 | rt.router.GET(path, asHandle(handler)) 240 | return rt 241 | } 242 | 243 | func (rt *Router) Post(path string, handler IHandler) IRouter { 244 | rt.router.POST(path, asHandle(handler)) 245 | return rt 246 | } 247 | 248 | func (rt *Router) Delete(path string, handler IHandler) IRouter { 249 | rt.router.DELETE(path, asHandle(handler)) 250 | return rt 251 | } 252 | 253 | func (rt *Router) Patch(path string, handler IHandler) IRouter { 254 | rt.router.PATCH(path, asHandle(handler)) 255 | return rt 256 | } 257 | 258 | func (rt *Router) Put(path string, handler IHandler) IRouter { 259 | rt.router.PUT(path, asHandle(handler)) 260 | return rt 261 | } 262 | -------------------------------------------------------------------------------- /pkg/apis/runtime/router_advice.go: -------------------------------------------------------------------------------- 1 | package runtime 2 | 3 | import ( 4 | "github.com/gin-gonic/gin" 5 | ) 6 | 7 | type ( 8 | // RouteAdviceReceiver represents the type that can receive advice. 9 | RouteAdviceReceiver any 10 | 11 | // RouteAdviceProvider is a provider to provide advice to the request 12 | // of the reflected routes of a IHandler. 13 | RouteAdviceProvider interface { 14 | // CanSet validates the given RouteAdviceReceiver can be set or not in prepare phase, 15 | // returns true if the given RouteAdviceReceiver can be injected. 16 | // The given RouteAdviceReceiver is stateless, 17 | // please do not perform additional operations on it. 18 | CanSet(RouteAdviceReceiver) bool 19 | 20 | // Set injects the valid RouteAdviceReceiver by this provider before validating, 21 | // the provider should set the corresponding advice to the target. 22 | Set(RouteAdviceReceiver) 23 | } 24 | ) 25 | 26 | // Built-in advice receivers. 27 | type ( 28 | // GinContextAdviceReceiver sets gin.Context 29 | // if the given request type implements this interface before validating. 30 | ginContextAdviceReceiver interface { 31 | // SetGinContext injects the session context before validating. 32 | SetGinContext(*gin.Context) 33 | } 34 | 35 | // UnidiStreamAdviceReceiver sets runtime.RequestUnidiStream 36 | // if the given request type implements this interface after validating. 37 | unidiStreamAdviceReceiver interface { 38 | // SetStream injects the runtime.RequestUnidiStream after validating. 39 | SetStream(RequestUnidiStream) 40 | } 41 | 42 | // BidiStreamAdviceReceiver sets inject runtime.RequestBidiStream 43 | // if the given request type implements this interface after validating. 44 | bidiStreamAdviceReceiver interface { 45 | // SetStream injects the runtime.RequestBidiStream after validating. 46 | SetStream(RequestBidiStream) 47 | } 48 | ) 49 | 50 | // WithRouteAdviceProviders is a RouterOption to configure the advice providers 51 | // for the reflected routes of a IHandler. 52 | func WithRouteAdviceProviders(providers ...RouteAdviceProvider) RouterOption { 53 | return routerOption(func(r *Router) { 54 | for i := range providers { 55 | if providers[i] == nil { 56 | continue 57 | } 58 | 59 | r.adviceProviders = append(r.adviceProviders, providers[i]) 60 | } 61 | }) 62 | } 63 | -------------------------------------------------------------------------------- /pkg/apis/runtime/router_options.go: -------------------------------------------------------------------------------- 1 | package runtime 2 | 3 | import ( 4 | "io" 5 | 6 | "github.com/gin-gonic/gin" 7 | ) 8 | 9 | // ginGlobalOption is the function type of RouterOption, 10 | // which is used to set global options for gin. 11 | type ginGlobalOption func() 12 | 13 | func (ginGlobalOption) isOption() {} 14 | 15 | // ginEngineOption is the function type of RouterOption, 16 | // which is used to set engine for gin. 17 | type ginEngineOption func(*gin.Engine) 18 | 19 | func (ginEngineOption) isOption() {} 20 | 21 | // routerOption is the function type of RouterOption, 22 | // which is used to set Router. 23 | type routerOption func(*Router) 24 | 25 | func (routerOption) isOption() {} 26 | 27 | // ginRouteOption is the function type of RouterOption, 28 | // which is used to register the routes within raw gin method. 29 | type ginRouteOption func(gin.IRouter) 30 | 31 | func (ginRouteOption) isOption() {} 32 | 33 | // WithDefaultWriter is a RouterOption to configure the default writer for gin. 34 | func WithDefaultWriter(w io.Writer) RouterOption { 35 | return ginGlobalOption(func() { 36 | gin.DefaultWriter = w 37 | gin.DefaultErrorWriter = w 38 | }) 39 | } 40 | 41 | // WithDefaultHandler is a RouterOption to configure the default handler for gin. 42 | func WithDefaultHandler(handler IHandler) RouterOption { 43 | return ginEngineOption(func(eng *gin.Engine) { 44 | eng.NoRoute( 45 | asHandle(handler), 46 | noRoute) 47 | }) 48 | } 49 | -------------------------------------------------------------------------------- /pkg/apis/runtime/router_static.go: -------------------------------------------------------------------------------- 1 | package runtime 2 | 3 | import ( 4 | "io/fs" 5 | "net/http" 6 | "os" 7 | "time" 8 | ) 9 | 10 | type StaticHttpFileSystem struct { 11 | http.FileSystem 12 | 13 | Listable bool 14 | Embedded bool 15 | } 16 | 17 | func (fs StaticHttpFileSystem) Open(name string) (http.File, error) { 18 | f, err := fs.FileSystem.Open(name) 19 | if err != nil { 20 | return nil, err 21 | } 22 | 23 | return StaticHttpFile{File: f, Listable: fs.Listable, Embedded: fs.Embedded}, nil 24 | } 25 | 26 | type StaticHttpFile struct { 27 | http.File 28 | 29 | Listable bool 30 | Embedded bool 31 | } 32 | 33 | func (f StaticHttpFile) Readdir(count int) ([]os.FileInfo, error) { 34 | if f.Listable { 35 | return f.File.Readdir(count) 36 | } 37 | 38 | return nil, nil 39 | } 40 | 41 | func (f StaticHttpFile) Stat() (fs.FileInfo, error) { 42 | i, err := f.File.Stat() 43 | if err != nil { 44 | return nil, err 45 | } 46 | 47 | if f.Embedded { 48 | return embeddedFileInfo{FileInfo: i}, nil 49 | } 50 | 51 | return i, nil 52 | } 53 | 54 | type embeddedFileInfo struct { 55 | fs.FileInfo 56 | } 57 | 58 | var embeddedAt = time.Now() 59 | 60 | func (i embeddedFileInfo) ModTime() time.Time { 61 | return embeddedAt 62 | } 63 | -------------------------------------------------------------------------------- /pkg/apis/runtime/router_stream.go: -------------------------------------------------------------------------------- 1 | package runtime 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "io" 7 | "net/http" 8 | "reflect" 9 | "strings" 10 | "sync" 11 | "sync/atomic" 12 | "time" 13 | 14 | "github.com/gin-gonic/gin" 15 | "github.com/gorilla/websocket" 16 | "github.com/seal-io/walrus/utils/gopool" 17 | "github.com/seal-io/walrus/utils/log" 18 | ) 19 | 20 | // isStreamRequest returns true if the incoming request is a stream request. 21 | func isStreamRequest(c *gin.Context) bool { 22 | return IsUnidiStreamRequest(c) || IsBidiStreamRequest(c) 23 | } 24 | 25 | // IsUnidiStreamRequest returns true if the incoming request is a watching request. 26 | func IsUnidiStreamRequest(c *gin.Context) bool { 27 | return c.Request.Method == http.MethodGet && 28 | strings.EqualFold(c.Query("watch"), "true") 29 | } 30 | 31 | // doUnidiStreamRequest handles the unidirectional stream request. 32 | func doUnidiStreamRequest(c *gin.Context, route Route, routeInput reflect.Value) { 33 | logger := log.WithName("api") 34 | 35 | // Ensure chunked request. 36 | protoMajor, protoMinor := c.Request.ProtoMajor, c.Request.ProtoMinor 37 | if protoMajor == 1 && protoMinor == 0 { 38 | // Do not support http/1.0. 39 | c.AbortWithStatus(http.StatusUpgradeRequired) 40 | return 41 | } 42 | 43 | // Flush response headers. 44 | c.Header("Cache-Control", "no-store") 45 | c.Header("Content-Type", "application/octet-stream; charset=UTF-8") 46 | c.Header("X-Content-Type-Options", "nosniff") 47 | 48 | if protoMajor == 1 { 49 | c.Header("Transfer-Encoding", "chunked") 50 | } 51 | 52 | c.Writer.Flush() 53 | 54 | ctx, cancel := context.WithTimeout(c.Request.Context(), 10*time.Minute) 55 | defer cancel() 56 | 57 | stream := RequestUnidiStream{ 58 | ctx: ctx, 59 | ctxCancel: cancel, 60 | conn: c.Writer, 61 | } 62 | 63 | // Discard the underlay context in avoid of conflict using. 64 | if route.RequestAttributes.HasAll(RequestWithGinContext) { 65 | routeInput.Interface().(ginContextAdviceReceiver).SetGinContext(nil) 66 | } 67 | 68 | // Inject request with stream. 69 | routeInput.Interface().(unidiStreamAdviceReceiver).SetStream(stream) 70 | 71 | // Handle stream request. 72 | if route.RequestType.Kind() != reflect.Pointer { 73 | routeInput = routeInput.Elem() 74 | } 75 | routeOutputs := route.GoCaller.Call([]reflect.Value{routeInput}) 76 | 77 | // Handle error if found. 78 | if errObj := routeOutputs[len(routeOutputs)-1].Interface(); errObj != nil { 79 | err := errObj.(error) 80 | if !isUnidiDownstreamCloseError(err) { 81 | logger.Errorf("error processing unidirectional stream request: %v", err) 82 | } 83 | } 84 | } 85 | 86 | // isUnidiDownstreamCloseError returns true if the error is caused by the downstream closing the connection. 87 | func isUnidiDownstreamCloseError(err error) bool { 88 | if errors.Is(err, context.Canceled) || 89 | errors.Is(err, context.DeadlineExceeded) { 90 | return true 91 | } 92 | errMsg := err.Error() 93 | 94 | return strings.Contains(errMsg, "client disconnected") || 95 | strings.Contains(errMsg, "stream closed") 96 | } 97 | 98 | // IsBidiStreamRequest returns true if the incoming request is a websocket request. 99 | func IsBidiStreamRequest(c *gin.Context) bool { 100 | return c.Request.Method == http.MethodGet && 101 | c.IsWebsocket() 102 | } 103 | 104 | // doBidiStreamRequest handles the bidirectional stream request. 105 | func doBidiStreamRequest(c *gin.Context, route Route, routeInput reflect.Value) { 106 | logger := log.WithName("api") 107 | 108 | const ( 109 | // Time allowed to read the next pong message from the peer. 110 | pongWait = 5 * time.Second 111 | // Send pings to peer with this period, must be less than `pongWait`, 112 | // it is also the timeout to write a ping message to the peer. 113 | pingPeriod = (pongWait * 9) / 10 114 | ) 115 | 116 | // Ensure websocket request. 117 | up := websocket.Upgrader{ 118 | HandshakeTimeout: 5 * time.Second, 119 | ReadBufferSize: 4096, 120 | WriteBufferSize: 4096, 121 | } 122 | 123 | conn, err := up.Upgrade(c.Writer, c.Request, nil) 124 | if err != nil { 125 | logger.Errorf("error upgrading bidirectional stream request: %v", err) 126 | return 127 | } 128 | 129 | defer func() { 130 | _ = conn.Close() 131 | }() 132 | 133 | ctx, cancel := context.WithTimeout(c.Request.Context(), 10*time.Minute) 134 | defer cancel() 135 | 136 | // In order to avoid downstream connection leaking, 137 | // we need configuring a handler to close the upstream context. 138 | // To trigger the close handler, 139 | // we have to cut out a goroutine to received downstream, 140 | // if downstream closes, the close handler will be triggered. 141 | conn.SetCloseHandler(func(int, string) (err error) { 142 | cancel() 143 | return err 144 | }) 145 | 146 | frc := make(chan struct { 147 | t int 148 | r io.Reader 149 | e error 150 | }) 151 | 152 | gopool.Go(func() { 153 | var fr struct { 154 | t int 155 | r io.Reader 156 | e error 157 | } 158 | fr.t, fr.r, fr.e = conn.NextReader() 159 | select { 160 | case frc <- fr: 161 | case <-ctx.Done(): 162 | close(frc) 163 | } 164 | }) 165 | 166 | // Ping downstream asynchronously. 167 | gopool.Go(func() { 168 | ping := func() error { 169 | _ = conn.SetReadDeadline(getDeadline(pongWait)) 170 | conn.SetPongHandler(func(string) error { 171 | return conn.SetReadDeadline(getDeadline(pongWait)) 172 | }) 173 | 174 | return conn.WriteControl(websocket.PingMessage, 175 | []byte{}, 176 | getDeadline(pingPeriod)) 177 | } 178 | 179 | t := time.NewTicker(pingPeriod) 180 | defer t.Stop() 181 | 182 | for { 183 | select { 184 | case <-t.C: 185 | if ping() != nil { 186 | // Cancel upstream if failed to touch downstream. 187 | cancel() 188 | return 189 | } 190 | case <-ctx.Done(): 191 | return 192 | } 193 | } 194 | }) 195 | 196 | stream := RequestBidiStream{ 197 | firstReadOnce: &sync.Once{}, 198 | firstReadChan: frc, 199 | ctx: ctx, 200 | ctxCancel: cancel, 201 | conn: conn, 202 | connReadBytes: &atomic.Int64{}, 203 | connWriteBytes: &atomic.Int64{}, 204 | } 205 | 206 | defer func() { 207 | c.Set("request_size", stream.connReadBytes.Load()) 208 | c.Set("response_size", stream.connWriteBytes.Load()) 209 | }() 210 | 211 | // Discard the underlay context in avoid of conflict using. 212 | if route.RequestAttributes.HasAll(RequestWithGinContext) { 213 | routeInput.Interface().(ginContextAdviceReceiver).SetGinContext(nil) 214 | } 215 | 216 | // Inject request with stream. 217 | routeInput.Interface().(bidiStreamAdviceReceiver).SetStream(stream) 218 | 219 | // Handle stream request. 220 | if route.RequestType.Kind() != reflect.Pointer { 221 | routeInput = routeInput.Elem() 222 | } 223 | routeOutputs := route.GoCaller.Call([]reflect.Value{routeInput}) 224 | 225 | // Handle error if found. 226 | closeMsg := websocket.FormatCloseMessage(websocket.CloseNormalClosure, "closed") 227 | 228 | if errObj := routeOutputs[len(routeOutputs)-1].Interface(); errObj != nil { 229 | err = errObj.(error) 230 | if !isBidiDownstreamCloseError(err) { 231 | var we *websocket.CloseError 232 | if errors.As(err, &we) { 233 | closeMsg = websocket.FormatCloseMessage( 234 | we.Code, we.Text) 235 | 236 | c.Set("response_status", we.Code) 237 | } else { 238 | logger.Errorf("error processing bidirectional stream request: %v", err) 239 | 240 | if ue := errors.Unwrap(err); ue != nil { 241 | err = ue 242 | } 243 | closeMsg = websocket.FormatCloseMessage( 244 | websocket.CloseInternalServerErr, err.Error()) 245 | 246 | c.Set("response_status", websocket.CloseInternalServerErr) 247 | } 248 | } 249 | } 250 | 251 | _ = conn.WriteControl(websocket.CloseMessage, closeMsg, getDeadline(pingPeriod)) 252 | } 253 | 254 | // isBidiDownstreamCloseError returns true if the error is caused by the downstream closing the connection. 255 | func isBidiDownstreamCloseError(err error) bool { 256 | return errors.Is(err, context.Canceled) || 257 | errors.Is(err, context.DeadlineExceeded) || 258 | websocket.IsCloseError(err, 259 | websocket.CloseAbnormalClosure, 260 | websocket.CloseProtocolError, 261 | websocket.CloseGoingAway) 262 | } 263 | 264 | // getDeadline returns a deadline with the given duration. 265 | func getDeadline(duration time.Duration) time.Time { 266 | return time.Now().Add(duration) 267 | } 268 | -------------------------------------------------------------------------------- /pkg/apis/runtime/router_validation.go: -------------------------------------------------------------------------------- 1 | package runtime 2 | 3 | import ( 4 | "net/http" 5 | 6 | "github.com/gin-gonic/gin" 7 | ) 8 | 9 | // Validator holds the operation of validation. 10 | type Validator interface { 11 | // Validate returns error if the given request is invalid. 12 | Validate() error 13 | } 14 | 15 | type ( 16 | // RouteAuthorizer holds the operation of authorization. 17 | RouteAuthorizer interface { 18 | // Authorize returns the status code of authorization result, 19 | // 200 if success, 401 if unauthorized, 403 if forbidden. 20 | Authorize(*gin.Context, RouteProfile) int 21 | } 22 | 23 | // RouteAuthorizeFunc is the function type of RouteAuthorizer. 24 | RouteAuthorizeFunc func(*gin.Context, RouteProfile) int 25 | ) 26 | 27 | // Authorize implements the RouteAuthorizer interface. 28 | func (fn RouteAuthorizeFunc) Authorize(c *gin.Context, p RouteProfile) int { 29 | if fn == nil { 30 | return http.StatusOK 31 | } 32 | 33 | return fn(c, p) 34 | } 35 | 36 | // WithResourceAuthorizer if a RouterOption to configure the authorizer for the routes of IResourceHandler. 37 | func WithResourceAuthorizer(authorizer RouteAuthorizer) RouterOption { 38 | return routerOption(func(r *Router) { 39 | r.authorizer = authorizer 40 | }) 41 | } 42 | -------------------------------------------------------------------------------- /pkg/apis/runtime/scheme_route_extension.go: -------------------------------------------------------------------------------- 1 | package runtime 2 | 3 | import ( 4 | "github.com/getkin/kin-openapi/openapi3" 5 | ) 6 | 7 | func extendOperationSchema(r *Route, op *openapi3.Operation) { 8 | // TODO. 9 | } 10 | -------------------------------------------------------------------------------- /pkg/apis/runtime/scheme_route_openapi.go: -------------------------------------------------------------------------------- 1 | package runtime 2 | 3 | import ( 4 | "bytes" 5 | "embed" 6 | "fmt" 7 | "io" 8 | "net/http" 9 | "path" 10 | "sync" 11 | 12 | "github.com/gin-gonic/gin" 13 | "github.com/seal-io/walrus/utils/bytespool" 14 | "github.com/seal-io/walrus/utils/json" 15 | ) 16 | 17 | // ExposeOpenAPI is a RouterOption to add route to serve the OpenAPI schema spec, 18 | // and provide the SwaggerUI as well. 19 | func ExposeOpenAPI() RouterOption { 20 | return ginRouteOption(func(r gin.IRouter) { 21 | const openAPIPath = "/openapi" 22 | 23 | skipLoggingPath(openAPIPath) 24 | openAPIIndexer := indexOpenAPI() 25 | r.GET(openAPIPath, openAPIIndexer) 26 | 27 | const swaggerUIPath = "/swagger/*filepath" 28 | 29 | skipLoggingPath(swaggerUIPath) 30 | swaggerUIIndexer := indexSwaggerUI(openAPIPath) 31 | r.HEAD(swaggerUIPath, swaggerUIIndexer) 32 | r.GET(swaggerUIPath, swaggerUIIndexer) 33 | }) 34 | } 35 | 36 | func indexOpenAPI() Handle { 37 | var ( 38 | once sync.Once 39 | schemaBytes []byte 40 | ) 41 | 42 | return func(c *gin.Context) { 43 | once.Do(func() { 44 | var err error 45 | 46 | schemaBytes, err = json.Marshal(openAPISchemas) 47 | if err != nil { 48 | panic(fmt.Errorf("error marshaling openapi schema spec: %w", err)) 49 | } 50 | }) 51 | 52 | buff := bytespool.GetBytes(0) 53 | defer func() { bytespool.Put(buff) }() 54 | _, _ = io.CopyBuffer(c.Writer, bytes.NewBuffer(schemaBytes), buff) 55 | } 56 | } 57 | 58 | // downloaded form https://github.com/swagger-api/swagger-ui/releases. 59 | // 60 | //go:embed swagger-ui/* 61 | var swaggerUI embed.FS 62 | 63 | func indexSwaggerUI(schemaPath string) Handle { 64 | const dir = "swagger-ui" 65 | fs := StaticHttpFileSystem{ 66 | FileSystem: http.FS(swaggerUI), 67 | Embedded: true, 68 | } 69 | srv := http.FileServer(fs) 70 | index := fmt.Sprintf(swaggerUIIndexTemplate, schemaPath) 71 | 72 | return func(c *gin.Context) { 73 | if len(c.Params) == 0 { 74 | c.AbortWithStatus(http.StatusNotFound) 75 | return 76 | } 77 | 78 | p := path.Join(dir, c.Params[len(c.Params)-1].Value) 79 | if p == dir { 80 | // Index. 81 | _, _ = fmt.Fprint(c.Writer, index) 82 | return 83 | } 84 | // Assets. 85 | req := c.Request.Clone(c.Request.Context()) 86 | req.URL.Path = p 87 | req.URL.RawPath = p 88 | srv.ServeHTTP(c.Writer, req) 89 | c.Abort() 90 | } 91 | } 92 | 93 | const swaggerUIIndexTemplate = ` 94 | 95 | 96 | 97 | 98 | 99 | 100 | SwaggerUI 101 | 102 | 103 | 104 | 109 | 110 | 111 |
112 | 113 | 133 | 134 | 135 | ` 136 | -------------------------------------------------------------------------------- /pkg/apis/runtime/swagger-ui/favicon-16x16.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seal-io/hermitcrab/2986ef9ec52c6cf75dee51d2c4f97c2048264661/pkg/apis/runtime/swagger-ui/favicon-16x16.png -------------------------------------------------------------------------------- /pkg/apis/runtime/swagger-ui/favicon-32x32.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seal-io/hermitcrab/2986ef9ec52c6cf75dee51d2c4f97c2048264661/pkg/apis/runtime/swagger-ui/favicon-32x32.png -------------------------------------------------------------------------------- /pkg/apis/server.go: -------------------------------------------------------------------------------- 1 | package apis 2 | 3 | import ( 4 | "context" 5 | "crypto/tls" 6 | "errors" 7 | "fmt" 8 | stdlog "log" 9 | "net" 10 | "net/http" 11 | "strings" 12 | "time" 13 | 14 | "github.com/seal-io/walrus/utils/dynacert" 15 | "github.com/seal-io/walrus/utils/gopool" 16 | "github.com/seal-io/walrus/utils/log" 17 | "golang.org/x/crypto/acme" 18 | "golang.org/x/crypto/acme/autocert" 19 | 20 | "github.com/seal-io/hermitcrab/pkg/apis/config" 21 | ) 22 | 23 | func NewServer() (*Server, error) { 24 | logger := log.WithName("api") 25 | 26 | return &Server{ 27 | logger: logger, 28 | }, nil 29 | } 30 | 31 | type Server struct { 32 | logger log.Logger 33 | } 34 | 35 | type ServeOptions struct { 36 | SetupOptions 37 | 38 | BindAddress string 39 | BindWithDualStack bool 40 | TlsMode TlsMode 41 | TlsCertFile string 42 | TlsPrivateKeyFile string 43 | TlsCertDir string 44 | TlsAutoCertDomains []string 45 | } 46 | 47 | type TlsMode uint64 48 | 49 | const ( 50 | TlsModeDisabled TlsMode = iota 51 | TlsModeSelfGenerated 52 | TlsModeAutoGenerated 53 | TlsModeCustomized 54 | ) 55 | 56 | type TlsCertDirMode = string 57 | 58 | func (s *Server) Serve(c context.Context, opts ServeOptions) error { 59 | s.logger.Info("starting") 60 | 61 | config.TlsCertified.Set(opts.TlsCertified) 62 | 63 | handler, err := s.Setup(c, opts.SetupOptions) 64 | if err != nil { 65 | return fmt.Errorf("error setting up apis server: %w", err) 66 | } 67 | httpHandler := make(chan http.Handler) 68 | 69 | g := gopool.GroupWithContextIn(c) 70 | 71 | // Serve https. 72 | g.Go(func(ctx context.Context) error { 73 | if opts.TlsMode == TlsModeDisabled { 74 | s.logger.Info("serving in HTTP") 75 | 76 | httpHandler <- handler 77 | 78 | return nil 79 | } 80 | 81 | h := handler 82 | lg := newStdErrorLogger(s.logger.WithName("https")) 83 | 84 | nw, addr, err := parseBindAddress(opts.BindAddress, 443, opts.BindWithDualStack) 85 | if err != nil { 86 | return err 87 | } 88 | 89 | ls, err := newTcpListener(ctx, nw, addr) 90 | if err != nil { 91 | return err 92 | } 93 | 94 | defer func() { _ = ls.Close() }() 95 | 96 | tlsConfig := &tls.Config{ 97 | NextProtos: []string{"h2", "http/1.1"}, 98 | MinVersion: tls.VersionTLS12, 99 | } 100 | 101 | switch opts.TlsMode { 102 | default: // TlsModeSelfGenerated. 103 | cache := dynacert.DirCache(opts.TlsCertDir) 104 | 105 | s.logger.InfoS("serving in HTTPs with self-generated keypair", 106 | "cache", opts.TlsCertDir) 107 | 108 | mgr := &dynacert.Manager{ 109 | Cache: cache, 110 | } 111 | tlsConfig.GetCertificate = mgr.GetCertificate 112 | ls = tls.NewListener(ls, tlsConfig) 113 | httpHandler <- http.HandlerFunc(redirectHandler) 114 | case TlsModeAutoGenerated: 115 | cache := autocert.DirCache(opts.TlsCertDir) 116 | 117 | s.logger.InfoS("serving in HTTPs with auto-generated keypair", 118 | "domains", opts.TlsAutoCertDomains, 119 | "cache", opts.TlsCertDir) 120 | 121 | mgr := &autocert.Manager{ 122 | Prompt: autocert.AcceptTOS, 123 | Cache: cache, 124 | HostPolicy: autocert.HostWhitelist(opts.TlsAutoCertDomains...), 125 | } 126 | 127 | tlsConfig.NextProtos = append(tlsConfig.NextProtos, acme.ALPNProto) 128 | tlsConfig.GetCertificate = func(i *tls.ClientHelloInfo) (*tls.Certificate, error) { 129 | if i.ServerName == "localhost" || i.ServerName == "" { 130 | ni := *i 131 | ni.ServerName = opts.TlsAutoCertDomains[0] 132 | 133 | return mgr.GetCertificate(&ni) 134 | } 135 | 136 | return mgr.GetCertificate(i) 137 | } 138 | ls = tls.NewListener(ls, tlsConfig) 139 | httpHandler <- mgr.HTTPHandler(http.HandlerFunc(redirectHandler)) 140 | case TlsModeCustomized: 141 | s.logger.Info("serving in HTTPs with custom keypair") 142 | 143 | cert, err := tls.LoadX509KeyPair(opts.TlsCertFile, opts.TlsPrivateKeyFile) 144 | if err != nil { 145 | return err 146 | } 147 | tlsConfig.Certificates = []tls.Certificate{cert} 148 | ls = tls.NewListener(ls, tlsConfig) 149 | httpHandler <- http.HandlerFunc(redirectHandler) 150 | } 151 | 152 | s.logger.Infof("serving https on %q by %q", addr, nw) 153 | 154 | return serve(ctx, h, lg, ls) 155 | }) 156 | 157 | // Serve http. 158 | g.Go(func(ctx context.Context) error { 159 | h := <-httpHandler 160 | lg := newStdErrorLogger(s.logger.WithName("http")) 161 | 162 | nw, addr, err := parseBindAddress(opts.BindAddress, 80, opts.BindWithDualStack) 163 | if err != nil { 164 | return err 165 | } 166 | 167 | ls, err := newTcpListener(ctx, nw, addr) 168 | if err != nil { 169 | return err 170 | } 171 | 172 | defer func() { _ = ls.Close() }() 173 | 174 | s.logger.Infof("serving http on %q by %q", addr, nw) 175 | 176 | return serve(ctx, h, lg, ls) 177 | }) 178 | 179 | return g.Wait() 180 | } 181 | 182 | func serve(ctx context.Context, handler http.Handler, errorLog *stdlog.Logger, listener net.Listener) error { 183 | s := http.Server{ 184 | Handler: handler, 185 | ErrorLog: errorLog, 186 | BaseContext: func(_ net.Listener) context.Context { return ctx }, 187 | } 188 | defer func() { 189 | sCtx, sCancel := context.WithTimeout(context.Background(), 15*time.Second) 190 | defer sCancel() 191 | _ = s.Shutdown(sCtx) 192 | }() 193 | 194 | err := s.Serve(listener) 195 | if err != nil && !errors.Is(err, http.ErrServerClosed) { 196 | return err 197 | } 198 | 199 | return nil 200 | } 201 | 202 | func parseBindAddress(ip string, port int, dual bool) (network, address string, err error) { 203 | p := net.ParseIP(ip) 204 | if p == nil { 205 | return "", "", fmt.Errorf("invalid IP address: %s", ip) 206 | } 207 | 208 | nw := "tcp" 209 | 210 | p = p.To4() 211 | if p != nil { 212 | if !dual { 213 | nw = "tcp4" 214 | } 215 | 216 | return nw, fmt.Sprintf("%s:%d", p.String(), port), nil 217 | } 218 | 219 | if !dual { 220 | nw = "tcp6" 221 | } 222 | 223 | return nw, fmt.Sprintf("[%s]:%d", ip, port), nil 224 | } 225 | 226 | func newTcpListener(ctx context.Context, network, address string) (net.Listener, error) { 227 | lc := net.ListenConfig{ 228 | KeepAlive: 3 * time.Minute, 229 | } 230 | 231 | ls, err := lc.Listen(ctx, network, address) 232 | if err != nil { 233 | return nil, fmt.Errorf("error creating %s listener for %s: %w", 234 | network, address, err) 235 | } 236 | 237 | return ls, nil 238 | } 239 | 240 | func redirectHandler(w http.ResponseWriter, r *http.Request) { 241 | if r.Method != http.MethodGet && r.Method != http.MethodHead { 242 | http.Error(w, "Use HTTPS", http.StatusBadRequest) 243 | return 244 | } 245 | 246 | // From Kubernetes probes guide, 247 | // https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes, 248 | // the `User-Agent: kube-probe/` header identify the incoming request is for Kubelet health check, 249 | // in order to avoid stuck in the readiness check, we don't redirect the probes request. 250 | if ua := r.Header.Get("User-Agent"); strings.HasPrefix(ua, "kube-probe/") { 251 | w.WriteHeader(http.StatusOK) 252 | return 253 | } 254 | 255 | host := r.Host 256 | if rawHost, _, err := net.SplitHostPort(host); err == nil { 257 | host = net.JoinHostPort(rawHost, "443") 258 | } 259 | 260 | http.Redirect(w, r, "https://"+host+r.URL.RequestURI(), http.StatusFound) 261 | } 262 | -------------------------------------------------------------------------------- /pkg/apis/setup.go: -------------------------------------------------------------------------------- 1 | package apis 2 | 3 | import ( 4 | "context" 5 | "net/http" 6 | "time" 7 | 8 | "github.com/seal-io/hermitcrab/pkg/apis/debug" 9 | "github.com/seal-io/hermitcrab/pkg/apis/measure" 10 | providerapis "github.com/seal-io/hermitcrab/pkg/apis/provider" 11 | "github.com/seal-io/hermitcrab/pkg/apis/runtime" 12 | "github.com/seal-io/hermitcrab/pkg/provider" 13 | ) 14 | 15 | type SetupOptions struct { 16 | // Configure from launching. 17 | ConnQPS int 18 | ConnBurst int 19 | WebsocketConnMaxPerIP int 20 | // Derived from configuration. 21 | ProviderService *provider.Service 22 | TlsCertified bool 23 | } 24 | 25 | func (s *Server) Setup(ctx context.Context, opts SetupOptions) (http.Handler, error) { 26 | // Prepare middlewares. 27 | throttler := runtime.RequestThrottling(opts.ConnQPS, opts.ConnBurst) 28 | wsCounter := runtime.If( 29 | // Validate websocket connection. 30 | runtime.IsBidiStreamRequest, 31 | // Maximum 10 connection per ip. 32 | runtime.PerIP(func() runtime.Handle { 33 | return runtime.RequestCounting(opts.WebsocketConnMaxPerIP, 5*time.Second) 34 | }), 35 | ) 36 | 37 | // Initial router. 38 | apisOpts := []runtime.RouterOption{ 39 | runtime.WithDefaultWriter(s.logger), 40 | runtime.SkipLoggingPaths( 41 | "/", 42 | "/readyz", 43 | "/livez", 44 | "/metrics", 45 | "/debug/version"), 46 | runtime.ExposeOpenAPI(), 47 | } 48 | 49 | apis := runtime.NewRouter(apisOpts...) 50 | 51 | rootApis := apis.Group("/v1"). 52 | Use(throttler, wsCounter) 53 | { 54 | r := rootApis 55 | r.Group("/providers"). 56 | Routes(providerapis.Handle(opts.ProviderService)) 57 | } 58 | 59 | measureApis := apis.Group(""). 60 | Use(throttler) 61 | { 62 | r := measureApis 63 | r.Get("/readyz", measure.Readyz()) 64 | r.Get("/livez", measure.Livez()) 65 | r.Get("/metrics", measure.Metrics()) 66 | } 67 | 68 | debugApis := apis.Group("/debug"). 69 | Use(throttler) 70 | { 71 | r := debugApis 72 | r.Get("/version", debug.Version()) 73 | r.Get("/flags", debug.GetFlags()) 74 | r.Group(""). 75 | Use(runtime.OnlyLocalIP()). 76 | Get("/pprof/*any", debug.PProf()). 77 | Put("/flags", debug.SetFlags()) 78 | } 79 | 80 | return apis, nil 81 | } 82 | -------------------------------------------------------------------------------- /pkg/consts/consts.go: -------------------------------------------------------------------------------- 1 | package consts 2 | 3 | // DataDir is the path to expose the data consumed by HermitCrab. 4 | const DataDir = "/var/run/hermitcrab" 5 | -------------------------------------------------------------------------------- /pkg/database/bolt.go: -------------------------------------------------------------------------------- 1 | package database 2 | 3 | import ( 4 | "context" 5 | "path/filepath" 6 | "runtime" 7 | "sync" 8 | "time" 9 | 10 | "github.com/seal-io/walrus/utils/gopool" 11 | bolt "go.etcd.io/bbolt" 12 | "go.uber.org/multierr" 13 | ) 14 | 15 | // Bolt holds the BoltDB instance. 16 | type Bolt struct { 17 | m sync.Mutex 18 | db *bolt.DB 19 | } 20 | 21 | // Run starts the BoltDB instance. 22 | func (b *Bolt) Run(ctx context.Context, dir string, lockMemory bool) (err error) { 23 | b.m.Lock() 24 | 25 | opts := getBoltOpts() 26 | opts.Mlock = lockMemory 27 | 28 | b.db, err = bolt.Open(filepath.Join(dir, "metadata.db"), 0o600, opts) 29 | if err != nil { 30 | b.m.Unlock() 31 | return err 32 | } 33 | b.m.Unlock() 34 | 35 | var ( 36 | done = ctx.Done() 37 | down = make(chan error) 38 | ) 39 | 40 | gopool.Go(func() { 41 | <-done 42 | down <- multierr.Combine( 43 | b.db.Sync(), 44 | b.db.Close(), 45 | ) 46 | }) 47 | 48 | return <-down 49 | } 50 | 51 | // GetDriver returns the BoltDB driver. 52 | func (b *Bolt) GetDriver() BoltDriver { 53 | b.m.Lock() 54 | defer b.m.Unlock() 55 | 56 | const wait = 100 * time.Millisecond 57 | 58 | // Spinning until db is ready. 59 | for b.db == nil { 60 | b.m.Unlock() 61 | 62 | runtime.Gosched() 63 | time.Sleep(wait) 64 | 65 | b.m.Lock() 66 | } 67 | 68 | return b.db 69 | } 70 | -------------------------------------------------------------------------------- /pkg/database/bolt_linux.go: -------------------------------------------------------------------------------- 1 | package database 2 | 3 | import ( 4 | "syscall" 5 | "time" 6 | 7 | bolt "go.etcd.io/bbolt" 8 | ) 9 | 10 | func getBoltOpts() *bolt.Options { 11 | return &bolt.Options{ 12 | Timeout: 2 * time.Second, 13 | PreLoadFreelist: true, 14 | FreelistType: bolt.FreelistMapType, 15 | MmapFlags: syscall.MAP_POPULATE, 16 | Mlock: true, 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /pkg/database/bolt_nolinux.go: -------------------------------------------------------------------------------- 1 | //go:build !linux 2 | 3 | package database 4 | 5 | import ( 6 | "time" 7 | 8 | bolt "go.etcd.io/bbolt" 9 | ) 10 | 11 | func getBoltOpts() *bolt.Options { 12 | return &bolt.Options{ 13 | Timeout: 2 * time.Second, 14 | PreLoadFreelist: true, 15 | FreelistType: bolt.FreelistMapType, 16 | MmapFlags: 0, 17 | Mlock: true, 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /pkg/database/driver.go: -------------------------------------------------------------------------------- 1 | package database 2 | 3 | import bolt "go.etcd.io/bbolt" 4 | 5 | type BoltDriver interface { 6 | // Begin starts a new transaction. 7 | // Multiple read-only transactions can be used concurrently but only one 8 | // write transaction can be used at a time. Starting multiple write transactions 9 | // will cause the calls to block and be serialized until the current write 10 | // transaction finishes. 11 | // 12 | // Transactions should not be dependent on one another. Opening a read 13 | // transaction and a write transaction in the same goroutine can cause the 14 | // writer to deadlock because the database periodically needs to re-mmap itself 15 | // as it grows and it cannot do that while a read transaction is open. 16 | // 17 | // If a long running read transaction (for example, a snapshot transaction) is 18 | // needed, you might want to set DB.InitialMmapSize to a large enough value 19 | // to avoid potential blocking of write transaction. 20 | // 21 | // IMPORTANT: You must close read-only transactions after you are finished or 22 | // else the database will not reclaim old pages. 23 | Begin(writable bool) (*bolt.Tx, error) 24 | 25 | // Update executes a function within the context of a read-write managed transaction. 26 | // If no error is returned from the function then the transaction is committed. 27 | // If an error is returned then the entire transaction is rolled back. 28 | // Any error that is returned from the function or returned from the commit is 29 | // returned from the Update() method. 30 | // 31 | // Attempting to manually commit or rollback within the function will cause a panic. 32 | Update(fn func(*bolt.Tx) error) error 33 | 34 | // View executes a function within the context of a managed read-only transaction. 35 | // Any error that is returned from the function is returned from the View() method. 36 | // 37 | // Attempting to manually rollback within the function will cause a panic. 38 | View(fn func(*bolt.Tx) error) error 39 | 40 | // Batch calls fn as part of a batch. It behaves similar to Update, 41 | // except: 42 | // 43 | // 1. Concurrent Batch calls can be combined into a single Bolt 44 | // transaction. 45 | // 46 | // 2. The function passed to Batch may be called multiple times, 47 | // regardless of whether it returns error or not. 48 | // 49 | // This means that Batch function side effects must be idempotent and 50 | // take permanent effect only after a successful return is seen in 51 | // caller. 52 | // 53 | // The maximum batch size and delay can be adjusted with DB.MaxBatchSize 54 | // and DB.MaxBatchDelay, respectively. 55 | // 56 | // Batch is only useful when there are multiple goroutines calling it. 57 | Batch(fn func(*bolt.Tx) error) error 58 | 59 | // Sync executes fdatasync() against the database file handle. 60 | // 61 | // This is not necessary under normal operation, however, if you use NoSync 62 | // then it allows you to force the database file to sync against the disk. 63 | Sync() error 64 | 65 | // Stats retrieves ongoing performance stats for the database. 66 | // This is only updated when a transaction closes. 67 | Stats() bolt.Stats 68 | 69 | // Info is for internal access to the raw data bytes from the C cursor, use 70 | // carefully, or not at all. 71 | Info() *bolt.Info 72 | 73 | // IsReadOnly returns whether the database is opened in read-only mode. 74 | IsReadOnly() bool 75 | 76 | // Path returns the path to the file backing the database. 77 | Path() string 78 | } 79 | -------------------------------------------------------------------------------- /pkg/database/health.go: -------------------------------------------------------------------------------- 1 | package database 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "os" 7 | ) 8 | 9 | func IsConnected(ctx context.Context, db BoltDriver) error { 10 | _, err := os.Stat(db.Path()) 11 | if err != nil { 12 | return err 13 | } 14 | 15 | if db.IsReadOnly() { 16 | return errors.New("invalid database storage file: read-only") 17 | } 18 | 19 | return nil 20 | } 21 | -------------------------------------------------------------------------------- /pkg/download/client.go: -------------------------------------------------------------------------------- 1 | package download 2 | 3 | import ( 4 | "context" 5 | "crypto/sha256" 6 | "encoding/hex" 7 | "errors" 8 | "fmt" 9 | "io" 10 | "net/http" 11 | "os" 12 | "path/filepath" 13 | 14 | "github.com/seal-io/walrus/utils/bytespool" 15 | "github.com/seal-io/walrus/utils/gopool" 16 | "github.com/seal-io/walrus/utils/log" 17 | "github.com/seal-io/walrus/utils/runtimex" 18 | "github.com/seal-io/walrus/utils/version" 19 | ) 20 | 21 | var defaultHttpClient = NewHttpClient( 22 | WithUserAgent(version.GetUserAgentWith("hermitcrab")), 23 | WithInsecureSkipVerify(), 24 | ) 25 | 26 | type Client struct { 27 | httpCli *http.Client 28 | } 29 | 30 | func NewClient(httpCli *http.Client) *Client { 31 | if httpCli == nil { 32 | httpCli = defaultHttpClient 33 | } 34 | 35 | return &Client{ 36 | httpCli: httpCli, 37 | } 38 | } 39 | 40 | type GetOptions struct { 41 | DownloadURL string 42 | Directory string 43 | Filename string 44 | Shasum string 45 | } 46 | 47 | func (c *Client) Get(ctx context.Context, opts GetOptions) error { 48 | if opts.DownloadURL == "" || opts.Directory == "" || opts.Filename == "" { 49 | return errors.New("invalid options") 50 | } 51 | 52 | output := filepath.Join(opts.Directory, opts.Filename) 53 | 54 | // Validate the output, 55 | // if existed, return directly, 56 | // check corrupted if the shasum is provided. 57 | if info, err := os.Lstat(output); err != nil && !os.IsNotExist(err) { 58 | return fmt.Errorf("validate: failed to get output info: %w", err) 59 | } else if info != nil { 60 | // Validate if the output is a directory. 61 | if info.IsDir() { 62 | return errors.New("validate: output is a directory") 63 | } 64 | 65 | // Get real path if the output is a symlink. 66 | if info.Mode()&os.ModeSymlink != 0 { 67 | output, err = os.Readlink(output) 68 | if err != nil { 69 | return errors.New("validate: failed to get real output") 70 | } 71 | } 72 | 73 | // Validate the shasum. 74 | matched, err := validateShasum(output, opts.Shasum) 75 | if err != nil { 76 | return fmt.Errorf("validate: failed to validate existing output: %w", err) 77 | } 78 | 79 | // Return directly if the shasum is matched. 80 | if matched { 81 | return nil 82 | } 83 | 84 | // Remove the corrupted existing output. 85 | err = os.RemoveAll(output) 86 | if err != nil { 87 | return fmt.Errorf("validate: failed to remove corrupted existing output: %w", err) 88 | } 89 | } 90 | 91 | // Validate the temp output, 92 | // if existed, must check the shasum. 93 | var ( 94 | tempPath = filepath.Join(opts.Directory, "."+opts.Filename) 95 | receivedLength int64 96 | ) 97 | { 98 | if info, err := os.Lstat(tempPath); err != nil && !os.IsNotExist(err) { 99 | return fmt.Errorf("validate: failed to get temp output info: %w", err) 100 | } else if info != nil { 101 | receivedLength = info.Size() 102 | 103 | // Correct the temp output if it is not a regular file. 104 | if !info.Mode().IsRegular() { 105 | err = os.RemoveAll(tempPath) 106 | if err != nil { 107 | return fmt.Errorf("validate: failed to remove corrupted temp output: %w", err) 108 | } 109 | } 110 | } 111 | } 112 | 113 | // Check if the remote allowing range download. 114 | var ( 115 | partialDownload bool 116 | contentLength int64 117 | ) 118 | { 119 | req, err := http.NewRequestWithContext(ctx, http.MethodHead, opts.DownloadURL, nil) 120 | if err != nil { 121 | return fmt.Errorf("download: failed to create HEAD request: %w", err) 122 | } 123 | 124 | resp, err := c.httpCli.Do(req) 125 | if err == nil && resp.StatusCode == http.StatusOK { 126 | partialDownload = resp.Header.Get("Accept-Ranges") == "bytes" && 127 | resp.ContentLength > 0 && 128 | runtimex.NumCPU() > 1 129 | contentLength = resp.ContentLength 130 | } 131 | 132 | // If the remote allowing range download, 133 | // but the temp output is larger than the target size, 134 | // we should remove the temp output and download again. 135 | if partialDownload && receivedLength > contentLength { 136 | err = os.RemoveAll(tempPath) 137 | if err != nil { 138 | return fmt.Errorf("download: failed to remove corrupted temp output: %w", err) 139 | } 140 | 141 | receivedLength = 0 142 | } 143 | } 144 | 145 | // Prepare the output directory. 146 | err := os.MkdirAll(opts.Directory, 0o700) 147 | if err != nil && !os.IsExist(err) { 148 | return fmt.Errorf("download: failed to create output directory: %w", err) 149 | } 150 | 151 | // Download. 152 | req, err := http.NewRequestWithContext(ctx, http.MethodGet, opts.DownloadURL, nil) 153 | if err != nil { 154 | return fmt.Errorf("download: failed to create GET request: %w", err) 155 | } 156 | 157 | tempFile, err := os.OpenFile(tempPath, os.O_WRONLY|os.O_CREATE, 0o600) 158 | if err != nil { 159 | return fmt.Errorf("download: failed to open temp file: %w", err) 160 | } 161 | 162 | defer func() { 163 | _ = tempFile.Close() 164 | 165 | if err == nil || partialDownload { 166 | return 167 | } 168 | 169 | // Remove the temp file if failed to download. 170 | _ = os.Remove(tempPath) 171 | }() 172 | 173 | if partialDownload { 174 | err = c.downloadPartial(req, tempFile, receivedLength, contentLength) 175 | } else { 176 | err = c.download(req, tempFile) 177 | } 178 | 179 | if err != nil { 180 | return fmt.Errorf("download: %w", err) 181 | } 182 | 183 | // Validate whether the shasum is matched. 184 | matched, err := validateShasum(tempPath, opts.Shasum) 185 | if err != nil { 186 | return fmt.Errorf("validate: failed to validate downloaded temp output: %w", err) 187 | } 188 | 189 | if !matched { 190 | // Remove the corrupted download output. 191 | err = os.RemoveAll(tempPath) 192 | if err != nil { 193 | return fmt.Errorf("validate: failed to remove corrupted download output: %w", err) 194 | } 195 | 196 | return errors.New("validate: shasum mismatched") 197 | } 198 | 199 | err = os.Rename(tempPath, output) 200 | if err != nil { 201 | return fmt.Errorf("download: failed to rename output: %w", err) 202 | } 203 | 204 | return nil 205 | } 206 | 207 | func (c *Client) downloadPartial(req *http.Request, file *os.File, receivedLength, contentLength int64) error { 208 | if receivedLength == contentLength { 209 | return nil 210 | } 211 | 212 | logger := log.WithName("download").WithValues("url", req.URL) 213 | 214 | if receivedLength == 0 { 215 | err := file.Truncate(contentLength) 216 | if err != nil { 217 | return fmt.Errorf("failed to truncate file: %w", err) 218 | } 219 | } else { 220 | _, err := file.Seek(0, io.SeekEnd) 221 | if err != nil { 222 | return fmt.Errorf("failed to seek file to end: %w", err) 223 | } 224 | } 225 | 226 | const ( 227 | partialBuffer = 2 * 1024 * 1024 // 2mb. 228 | parallel = 5 229 | ) 230 | 231 | var bytesRanges [][2]int64 232 | { 233 | for start := receivedLength; start < contentLength; { 234 | end := start + partialBuffer 235 | if end >= contentLength { 236 | end = contentLength 237 | } 238 | 239 | bytesRanges = append(bytesRanges, [2]int64{start, end}) 240 | start = end 241 | } 242 | } 243 | 244 | logger.Debug("downloading") 245 | 246 | for i, t := 0, len(bytesRanges); i < t; { 247 | j := i + parallel 248 | if j >= t { 249 | j = t 250 | } 251 | 252 | err := func(bytesRanges [][2]int64) error { 253 | var ( 254 | partialStart = bytesRanges[0][0] 255 | partialEnd = bytesRanges[len(bytesRanges)-1][1] 256 | buf = make([]byte, partialEnd-partialStart) 257 | ) 258 | 259 | wg := gopool.GroupWithContextIn(req.Context()) 260 | 261 | for k := range bytesRanges { 262 | var ( 263 | rangeStart = bytesRanges[k][0] 264 | rangeEnd = bytesRanges[k][1] 265 | ) 266 | 267 | wg.Go(func(ctx context.Context) error { 268 | req := req.Clone(ctx) 269 | req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", rangeStart, rangeEnd)) 270 | 271 | resp, err := c.httpCli.Do(req) 272 | if err != nil { 273 | return fmt.Errorf("failed to send partital GET request: %w", err) 274 | } 275 | 276 | defer func() { _ = resp.Body.Close() }() 277 | 278 | if resp.StatusCode != http.StatusPartialContent { 279 | return fmt.Errorf("unexpected partital GET response status: %s", resp.Status) 280 | } 281 | 282 | var ( 283 | bufStart = rangeStart - partialStart 284 | bufEnd = rangeEnd - partialStart 285 | ) 286 | 287 | _, err = io.ReadFull(resp.Body, buf[bufStart:bufEnd]) 288 | if err != nil { 289 | return err 290 | } 291 | 292 | logger.V(6).Infof("received range %d-%d", rangeStart, rangeEnd) 293 | 294 | return nil 295 | }) 296 | } 297 | 298 | err := wg.Wait() 299 | if err != nil { 300 | return err 301 | } 302 | 303 | _, err = file.Write(buf) 304 | if err != nil { 305 | return fmt.Errorf("failed to output partital response body %d-%d: %w", 306 | partialStart, partialEnd, err) 307 | } 308 | 309 | return nil 310 | }(bytesRanges[i:j]) 311 | if err != nil { 312 | return err 313 | } 314 | 315 | i = j 316 | } 317 | 318 | logger.Debug("downloaded") 319 | 320 | return nil 321 | } 322 | 323 | const copyBuffer = 1024 * 1024 // 1mb. 324 | 325 | func (c *Client) download(req *http.Request, file *os.File) error { 326 | logger := log.WithName("download").WithValues("url", req.URL) 327 | 328 | // Seek to the beginning of the temp file. 329 | _, err := file.Seek(0, 0) 330 | if err != nil { 331 | return fmt.Errorf("failed to seek file beginning: %w", err) 332 | } 333 | 334 | logger.Debug("downloading") 335 | 336 | resp, err := c.httpCli.Do(req) 337 | if err != nil { 338 | return fmt.Errorf("failed to send GET request: %w", err) 339 | } 340 | 341 | defer func() { _ = resp.Body.Close() }() 342 | 343 | // Validate the response. 344 | if resp.StatusCode != http.StatusOK { 345 | return fmt.Errorf("unexpected GET response status: %s", resp.Status) 346 | } 347 | 348 | buf := bytespool.GetBytes(copyBuffer) 349 | defer bytespool.Put(buf) 350 | 351 | // Write the response body to the temp file. 352 | _, err = io.CopyBuffer(file, resp.Body, buf) 353 | if err != nil { 354 | return fmt.Errorf("failed to output response body: %w", err) 355 | } 356 | 357 | logger.Debug("downloaded") 358 | 359 | return nil 360 | } 361 | 362 | func validateShasum(path, shasum string) (bool, error) { 363 | if shasum == "" { 364 | return true, nil 365 | } 366 | 367 | f, err := os.Open(path) 368 | if err != nil { 369 | return false, err 370 | } 371 | 372 | defer func() { _ = f.Close() }() 373 | 374 | h := sha256.New() 375 | 376 | buf := bytespool.GetBytes(copyBuffer) 377 | defer bytespool.Put(buf) 378 | 379 | _, err = io.CopyBuffer(h, f, buf) 380 | if err != nil { 381 | return false, err 382 | } 383 | 384 | return hex.EncodeToString(h.Sum(nil)) == shasum, nil 385 | } 386 | -------------------------------------------------------------------------------- /pkg/download/http.go: -------------------------------------------------------------------------------- 1 | package download 2 | 3 | import ( 4 | "crypto/tls" 5 | "net" 6 | "net/http" 7 | "time" 8 | ) 9 | 10 | func NewHttpClient(opts ...HttpClientOption) *http.Client { 11 | hc := &http.Client{ 12 | Transport: &http.Transport{ 13 | Proxy: http.ProxyFromEnvironment, 14 | DialContext: (&net.Dialer{ 15 | Timeout: 30 * time.Second, 16 | KeepAlive: 30 * time.Second, 17 | }).DialContext, 18 | ForceAttemptHTTP2: true, 19 | MaxIdleConns: 100, 20 | IdleConnTimeout: 90 * time.Second, 21 | TLSHandshakeTimeout: 10 * time.Second, 22 | ExpectContinueTimeout: 1 * time.Second, 23 | }, 24 | } 25 | 26 | for i := range opts { 27 | if opts[i] == nil { 28 | continue 29 | } 30 | 31 | hc = opts[i](hc) 32 | } 33 | 34 | return hc 35 | } 36 | 37 | type HttpClientOption func(*http.Client) *http.Client 38 | 39 | func WithTimeout(timeout time.Duration) HttpClientOption { 40 | if timeout == 0 { 41 | return nil 42 | } 43 | 44 | return func(cli *http.Client) *http.Client { 45 | cli.Timeout = timeout 46 | return cli 47 | } 48 | } 49 | 50 | func WithUserAgent(userAgent string) HttpClientOption { 51 | if userAgent == "" { 52 | return nil 53 | } 54 | 55 | return func(cli *http.Client) *http.Client { 56 | cli.Transport = &_CustomTransport{ 57 | Base: cli.Transport, 58 | Custom: func(r *http.Request) { 59 | r.Header.Set("User-Agent", userAgent) 60 | }, 61 | } 62 | 63 | return cli 64 | } 65 | } 66 | 67 | func WithInsecureSkipVerify() HttpClientOption { 68 | return func(cli *http.Client) *http.Client { 69 | for tr := cli.Transport; tr != nil; { 70 | switch v := tr.(type) { 71 | case *_CustomTransport: 72 | tr = v.Base 73 | continue 74 | case *http.Transport: 75 | if v.TLSClientConfig == nil { 76 | v.TLSClientConfig = &tls.Config{ 77 | MinVersion: tls.VersionTLS12, 78 | } 79 | } 80 | v.TLSClientConfig.InsecureSkipVerify = true 81 | } 82 | 83 | break 84 | } 85 | 86 | return cli 87 | } 88 | } 89 | 90 | type _CustomTransport struct { 91 | Base http.RoundTripper 92 | Custom func(*http.Request) 93 | } 94 | 95 | func (t *_CustomTransport) RoundTrip(r *http.Request) (*http.Response, error) { 96 | r2 := r.Clone(r.Context()) 97 | t.Custom(r2) 98 | 99 | return t.Base.RoundTrip(r2) 100 | } 101 | -------------------------------------------------------------------------------- /pkg/health/registry.go: -------------------------------------------------------------------------------- 1 | package health 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "sync" 7 | ) 8 | 9 | type ( 10 | // Checker defines the operations of a health checker. 11 | Checker interface { 12 | Name() string 13 | Check(context.Context) error 14 | } 15 | 16 | // Checkers holds the list of Checker. 17 | Checkers []Checker 18 | ) 19 | 20 | var ( 21 | checkers Checkers 22 | o sync.Once 23 | ) 24 | 25 | // Register registers all health checkers. 26 | func Register(ctx context.Context, cs Checkers) (err error) { 27 | err = errors.New("not allowed duplicated registering") 28 | 29 | o.Do(func() { 30 | checkers = cs 31 | err = nil 32 | }) 33 | 34 | return err 35 | } 36 | 37 | // Check defines the stereotype for health checking. 38 | type Check func(context.Context) error 39 | 40 | // CheckerFunc wraps the given Check as a Checker. 41 | func CheckerFunc(name string, fn Check) Checker { 42 | return checker{n: name, f: fn} 43 | } 44 | 45 | type checker struct { 46 | n string 47 | f Check 48 | } 49 | 50 | func (c checker) Name() string { 51 | return c.n 52 | } 53 | 54 | func (c checker) Check(ctx context.Context) error { 55 | if c.f == nil { 56 | return nil 57 | } 58 | 59 | return c.f(ctx) 60 | } 61 | -------------------------------------------------------------------------------- /pkg/health/validate.go: -------------------------------------------------------------------------------- 1 | package health 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "strings" 7 | 8 | "k8s.io/apimachinery/pkg/util/sets" 9 | ) 10 | 11 | // MustValidate returns the validation results with the given including list. 12 | func MustValidate(ctx context.Context, includes []string) (string, bool) { 13 | if len(checkers) == 0 { 14 | return "no checkers", false 15 | } 16 | 17 | if len(includes) == 0 { 18 | return "no include list", false 19 | } 20 | 21 | var ( 22 | ok = true 23 | ns = sets.NewString(includes...) 24 | sb strings.Builder 25 | ) 26 | 27 | for i := range checkers { 28 | n := checkers[i].Name() 29 | 30 | if !ns.Has(n) { 31 | continue 32 | } 33 | 34 | if err := checkers[i].Check(ctx); err != nil { 35 | ok = false 36 | _, _ = fmt.Fprintf(&sb, "[-]%s: failed, %v\n", n, err) 37 | 38 | continue 39 | } 40 | 41 | _, _ = fmt.Fprintf(&sb, "[+]%s: ok\n", n) 42 | } 43 | 44 | return sb.String(), ok 45 | } 46 | 47 | // Validate returns the validation results, 48 | // skips the checker if its name exists in the excluding list. 49 | func Validate(ctx context.Context, excludes ...string) (string, bool) { 50 | if len(checkers) == 0 { 51 | return "no checkers", false 52 | } 53 | 54 | var ( 55 | ok = true 56 | ns = sets.NewString(excludes...) 57 | sb strings.Builder 58 | ) 59 | 60 | for i := range checkers { 61 | n := checkers[i].Name() 62 | 63 | if ns.Has(n) { 64 | _, _ = fmt.Fprintf(&sb, "[?]%s: excluded\n", n) 65 | continue 66 | } 67 | 68 | if err := checkers[i].Check(ctx); err != nil { 69 | ok = false 70 | _, _ = fmt.Fprintf(&sb, "[-]%s: failed, %v\n", n, err) 71 | 72 | continue 73 | } 74 | 75 | _, _ = fmt.Fprintf(&sb, "[+]%s: ok\n", n) 76 | } 77 | 78 | return sb.String(), ok 79 | } 80 | -------------------------------------------------------------------------------- /pkg/metric/index.go: -------------------------------------------------------------------------------- 1 | package metric 2 | 3 | import ( 4 | "net/http" 5 | "time" 6 | 7 | "github.com/prometheus/client_golang/prometheus/promhttp" 8 | "github.com/seal-io/walrus/utils/log" 9 | ) 10 | 11 | // Index returns a http.Handler to process the metrics exporting. 12 | func Index(maxInFlight int, timeout time.Duration) http.Handler { 13 | opts := promhttp.HandlerOpts{ 14 | ErrorLog: log.WithName("metrics"), 15 | ErrorHandling: promhttp.HTTPErrorOnError, 16 | Registry: reg, 17 | DisableCompression: false, 18 | MaxRequestsInFlight: maxInFlight, 19 | Timeout: timeout, 20 | EnableOpenMetrics: true, 21 | } 22 | 23 | return promhttp.HandlerFor(reg, opts) 24 | } 25 | -------------------------------------------------------------------------------- /pkg/metric/registry.go: -------------------------------------------------------------------------------- 1 | package metric 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "sync" 7 | 8 | "github.com/prometheus/client_golang/prometheus" 9 | "github.com/prometheus/client_golang/prometheus/collectors" 10 | ) 11 | 12 | // Collectors holds the list of prometheus.Collector. 13 | type Collectors = []prometheus.Collector 14 | 15 | var ( 16 | reg = prometheus.NewRegistry() 17 | o sync.Once 18 | ) 19 | 20 | // Register registers all metric collectors. 21 | func Register(ctx context.Context, cs Collectors) (err error) { 22 | err = errors.New("not allowed duplicated registering") 23 | 24 | o.Do(func() { 25 | err = reg.Register(collectors.NewBuildInfoCollector()) 26 | if err != nil { 27 | return 28 | } 29 | 30 | for i := range cs { 31 | err = reg.Register(cs[i]) 32 | if err != nil { 33 | break 34 | } 35 | } 36 | }) 37 | 38 | return err 39 | } 40 | -------------------------------------------------------------------------------- /pkg/provider/service.go: -------------------------------------------------------------------------------- 1 | package provider 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/seal-io/hermitcrab/pkg/database" 7 | "github.com/seal-io/hermitcrab/pkg/provider/metadata" 8 | "github.com/seal-io/hermitcrab/pkg/provider/storage" 9 | ) 10 | 11 | type Service struct { 12 | Metadata metadata.Service 13 | Storage storage.Service 14 | } 15 | 16 | func NewService(boltDriver database.BoltDriver, dataSourceDir string) (*Service, error) { 17 | ms, err := metadata.NewService(boltDriver) 18 | if err != nil { 19 | return nil, fmt.Errorf("error creating metadata service: %w", err) 20 | } 21 | 22 | ss, err := storage.NewService(dataSourceDir) 23 | if err != nil { 24 | return nil, fmt.Errorf("error creating storage service: %w", err) 25 | } 26 | 27 | return &Service{ 28 | Metadata: ms, 29 | Storage: ss, 30 | }, nil 31 | } 32 | -------------------------------------------------------------------------------- /pkg/provider/storage/service.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "os" 7 | "path/filepath" 8 | "sync" 9 | 10 | "github.com/seal-io/hermitcrab/pkg/apis/runtime" 11 | "github.com/seal-io/hermitcrab/pkg/download" 12 | ) 13 | 14 | type ( 15 | LoadArchiveOptions struct { 16 | Hostname string 17 | Namespace string 18 | Type string 19 | Filename string 20 | Shasum string 21 | DownloadURL string 22 | } 23 | 24 | Archive = runtime.ResponseFile 25 | 26 | // Service holds the operation of provider storage. 27 | // Takes a look of the filesystem layer structure: 28 | // {hostname} 29 | // └── {namespace} 30 | // └── {type} 31 | // └── terraform-provider-{type}_{version}_{os}_{arch}.zip 32 | Service interface { 33 | // LoadArchive loads the archive from the storage. 34 | LoadArchive(context.Context, LoadArchiveOptions) (Archive, error) 35 | } 36 | ) 37 | 38 | func NewService(dir string) (Service, error) { 39 | providerDir := filepath.Join(dir, "providers") 40 | 41 | err := os.Mkdir(providerDir, 0o700) 42 | if err != nil && !os.IsExist(err) { 43 | return nil, err 44 | } 45 | 46 | impliedDir := os.Getenv("TF_PLUGIN_MIRROR_DIR") 47 | if impliedDir != "" { 48 | impliedDir = os.ExpandEnv(impliedDir) 49 | } 50 | 51 | return &service{ 52 | impliedDir: impliedDir, 53 | explicitDir: providerDir, 54 | downloadCli: download.NewClient(nil), 55 | }, nil 56 | } 57 | 58 | type service struct { 59 | barriers sync.Map 60 | 61 | impliedDir string 62 | explicitDir string 63 | downloadCli *download.Client 64 | } 65 | 66 | func (s *service) LoadArchive(ctx context.Context, opts LoadArchiveOptions) (Archive, error) { 67 | // Check whether the archive is in the implied directory. 68 | if s.impliedDir != "" { 69 | p := filepath.Join( 70 | s.impliedDir, 71 | opts.Hostname, opts.Namespace, opts.Type, 72 | opts.Filename) 73 | 74 | fi, err := os.Stat(p) 75 | if err != nil { 76 | if !os.IsNotExist(err) { 77 | return Archive{}, fmt.Errorf("error stating archive: %w", err) 78 | } 79 | 80 | goto ExplicitDir 81 | } 82 | 83 | if fi.IsDir() { 84 | goto ExplicitDir 85 | } 86 | 87 | f, err := os.Open(p) 88 | if err != nil { 89 | goto ExplicitDir 90 | } 91 | 92 | return Archive{ 93 | ContentType: "application/zip", 94 | ContentLength: fi.Size(), 95 | Headers: map[string]string{ 96 | "Content-Disposition": fmt.Sprintf(`attachment; filename="%s"`, fi.Name()), 97 | }, 98 | Reader: f, 99 | }, nil 100 | } 101 | 102 | ExplicitDir: 103 | // Check whether the archive is in the explicit directory. 104 | 105 | d := filepath.Join(s.explicitDir, opts.Hostname, opts.Namespace, opts.Type) 106 | p := filepath.Join(d, opts.Filename) 107 | 108 | fi, err := os.Stat(p) 109 | if err != nil { 110 | if !os.IsNotExist(err) { 111 | return Archive{}, fmt.Errorf("error stating archive: %w", err) 112 | } 113 | 114 | err = os.MkdirAll(d, 0o700) 115 | if err != nil && !os.IsExist(err) { 116 | return Archive{}, fmt.Errorf("error creating archive directory: %w", err) 117 | } 118 | } 119 | 120 | if fi != nil && fi.IsDir() { 121 | err = os.RemoveAll(p) 122 | if err != nil { 123 | return Archive{}, fmt.Errorf("error correcting invalid archive: %w", err) 124 | } 125 | 126 | fi = nil 127 | } 128 | 129 | if fi != nil { 130 | var f *os.File 131 | 132 | f, err := os.Open(p) 133 | if err != nil { 134 | return Archive{}, fmt.Errorf("error opening file: %w", err) 135 | } 136 | 137 | return Archive{ 138 | ContentType: "application/zip", 139 | ContentLength: fi.Size(), 140 | Headers: map[string]string{ 141 | "Content-Disposition": fmt.Sprintf(`attachment; filename="%s"`, fi.Name()), 142 | }, 143 | Reader: f, 144 | }, nil 145 | } 146 | 147 | var ( 148 | br *barrier 149 | rd bool 150 | ) 151 | { 152 | var v any 153 | v, rd = s.barriers.LoadOrStore(d, newBarrier()) 154 | br = v.(*barrier) 155 | } 156 | 157 | br.Lock() 158 | 159 | if rd { 160 | // Wait for the download to complete. 161 | br.Wait() 162 | 163 | return s.LoadArchive(ctx, opts) 164 | } 165 | 166 | defer func() { 167 | s.barriers.Delete(d) 168 | br.Done() 169 | }() 170 | 171 | // Download the archive. 172 | err = s.downloadCli.Get(ctx, download.GetOptions{ 173 | DownloadURL: opts.DownloadURL, 174 | Directory: d, 175 | Filename: opts.Filename, 176 | Shasum: opts.Shasum, 177 | }) 178 | if err != nil { 179 | return Archive{}, fmt.Errorf("error downloading archive: %w", err) 180 | } 181 | 182 | return s.LoadArchive(ctx, opts) 183 | } 184 | 185 | type barrier struct { 186 | cond *sync.Cond 187 | done bool 188 | } 189 | 190 | func newBarrier() *barrier { 191 | return &barrier{ 192 | cond: sync.NewCond(&sync.Mutex{}), 193 | } 194 | } 195 | 196 | func (br *barrier) Lock() { 197 | br.cond.L.Lock() 198 | } 199 | 200 | func (br *barrier) Wait() { 201 | for !br.done { 202 | br.cond.Wait() 203 | } 204 | br.cond.L.Unlock() 205 | } 206 | 207 | func (br *barrier) Done() { 208 | br.done = true 209 | br.cond.L.Unlock() 210 | br.cond.Broadcast() 211 | } 212 | -------------------------------------------------------------------------------- /pkg/registry/service.go: -------------------------------------------------------------------------------- 1 | package registry 2 | 3 | import ( 4 | "context" 5 | "net/http" 6 | "net/url" 7 | "path" 8 | "time" 9 | 10 | "github.com/seal-io/walrus/utils/json" 11 | "github.com/seal-io/walrus/utils/req" 12 | "github.com/seal-io/walrus/utils/version" 13 | ) 14 | 15 | var httpCli = req.HTTP(). 16 | WithInsecureSkipVerifyEnabled(). 17 | WithUserAgent(version.GetUserAgentWith("hermitcrab")) 18 | 19 | type Host string 20 | 21 | // Discover discovers the given service endpoint by the given service type. 22 | // See https://developer.hashicorp.com/terraform/internals/remote-service-discovery. 23 | // 24 | // Response example: 25 | // 26 | // { 27 | // "modules.v1": "https://modules.example.com/v1/", 28 | // "providers.v1": "/terraform/providers/v1/" 29 | // } 30 | // 31 | 32 | func (h Host) Discover(ctx context.Context, service string) url.URL { 33 | var ( 34 | u = &url.URL{ 35 | Scheme: "https", 36 | Host: string(h), 37 | } 38 | b = map[string]string{} 39 | ) 40 | 41 | err := httpCli.Request(). 42 | GetWithContext(ctx, resolveURLString(u, "/.well-known/terraform.json")). 43 | BodyJSON(&b) 44 | if err == nil && b[service] != "" { 45 | return *resolveURL(u, b[service]) 46 | } 47 | 48 | return *u 49 | } 50 | 51 | type Provider url.URL 52 | 53 | // Provider switches the host to the provider endpoint. 54 | func (h Host) Provider(ctx context.Context) Provider { 55 | switch h { 56 | case "registry.terraform.io": 57 | return Provider(url.URL{ 58 | Scheme: "https", 59 | Host: "registry.terraform.io", 60 | Path: "/v1/providers/", 61 | }) 62 | case "registry.opentofu.org": 63 | return Provider(url.URL{ 64 | Scheme: "https", 65 | Host: "registry.opentofu.org", 66 | Path: "/v1/providers/", 67 | }) 68 | } 69 | 70 | return Provider(h.Discover(ctx, "providers.v1")) 71 | } 72 | 73 | // GetVersions fetches the provider version list by the given parameters. 74 | // See https://developer.hashicorp.com/terraform/internals/provider-registry-protocol#list-available-versions. 75 | // 76 | // Response example: 77 | // 78 | // { 79 | // "versions": [ 80 | // { 81 | // "version": "2.0.0", 82 | // "protocols": ["4.0", "5.1"], 83 | // "platforms": [ 84 | // {"os": "darwin", "arch": "amd64"}, 85 | // {"os": "linux", "arch": "amd64"}, 86 | // {"os": "linux", "arch": "arm"}, 87 | // {"os": "windows", "arch": "amd64"} 88 | // ] 89 | // }, 90 | // { 91 | // "version": "2.0.1", 92 | // "protocols": ["5.2"], 93 | // "platforms": [ 94 | // {"os": "darwin", "arch": "amd64"}, 95 | // {"os": "linux", "arch": "amd64"}, 96 | // {"os": "linux", "arch": "arm"}, 97 | // {"os": "windows", "arch": "amd64"} 98 | // ] 99 | // } 100 | // ] 101 | // } 102 | // 103 | // If the given since is not zero, and the remote has not modified, the function returns nil, nil. 104 | // 105 | 106 | func (p Provider) GetVersions(ctx context.Context, namespace, type_ string, since ...time.Time) ([]byte, error) { 107 | rq := httpCli.Request() 108 | if len(since) != 0 && !since[0].IsZero() { 109 | rq = rq.WithHeader("If-Modified-Since", since[0].Format(http.TimeFormat)) 110 | } 111 | 112 | r := rq.GetWithContext(ctx, 113 | resolveURLString((*url.URL)(&p), path.Join(namespace, type_, "versions"))) 114 | 115 | if len(since) != 0 && !since[0].IsZero() && r.StatusCode() == http.StatusNotModified { 116 | return nil, nil 117 | } 118 | 119 | bs, err := r.BodyBytes() 120 | if err != nil { 121 | return nil, err 122 | } 123 | 124 | if json.Get(bs, "versions").IsArray() { 125 | return bs, nil 126 | } 127 | 128 | return []byte(`{"versions":[]}`), nil 129 | } 130 | 131 | // GetPlatform fetches the provider versioned platform information by the given parameters. 132 | // See https://developer.hashicorp.com/terraform/internals/provider-registry-protocol#find-a-provider-package. 133 | // 134 | // Response example: 135 | // 136 | // { 137 | // "protocols": ["4.0", "5.1"], 138 | // "os": "linux", 139 | // "arch": "amd64", 140 | // "filename": "terraform-provider-random_2.0.0_linux_amd64.zip", 141 | // "download_url": "https://releases.hashicorp.com/terraform-provider-random/2.0.0/terraform-provider-random_2.0.0_linux_amd64.zip", 142 | // "shasums_url": "https://releases.hashicorp.com/terraform-provider-random/2.0.0/terraform-provider-random_2.0.0_SHA256SUMS", 143 | // "shasums_signature_url": "https://releases.hashicorp.com/terraform-provider-random/2.0.0/terraform-provider-random_2.0.0_SHA256SUMS.sig", 144 | // "shasum": "5f9c7aa76b7c34d722fc9123208e26b22d60440cb47150dd04733b9b94f4541a", 145 | // "signing_keys": { 146 | // "gpg_public_keys": [ 147 | // { 148 | // "key_id": "51852D87348FFC4C", 149 | // "ascii_armor": "-----BEGIN PGP PUBLIC KEY BLOCK-----\nVersion: GnuPG v1\n\nmQENBFMORM0BCADBRyKO1MhCirazOSVwcfTr1xUxjPvfxD3hjUwHtjsOy/bT6p9f\nW2mRPfwnq2JB5As+paL3UGDsSRDnK9KAxQb0NNF4+eVhr/EJ18s3wwXXDMjpIifq\nfIm2WyH3G+aRLTLPIpscUNKDyxFOUbsmgXAmJ46Re1fn8uKxKRHbfa39aeuEYWFA\n3drdL1WoUngvED7f+RnKBK2G6ZEpO+LDovQk19xGjiMTtPJrjMjZJ3QXqPvx5wca\nKSZLr4lMTuoTI/ZXyZy5bD4tShiZz6KcyX27cD70q2iRcEZ0poLKHyEIDAi3TM5k\nSwbbWBFd5RNPOR0qzrb/0p9ksKK48IIfH2FvABEBAAG0K0hhc2hpQ29ycCBTZWN1\ncml0eSA8c2VjdXJpdHlAaGFzaGljb3JwLmNvbT6JATgEEwECACIFAlMORM0CGwMG\nCwkIBwMCBhUIAgkKCwQWAgMBAh4BAheAAAoJEFGFLYc0j/xMyWIIAIPhcVqiQ59n\nJc07gjUX0SWBJAxEG1lKxfzS4Xp+57h2xxTpdotGQ1fZwsihaIqow337YHQI3q0i\nSqV534Ms+j/tU7X8sq11xFJIeEVG8PASRCwmryUwghFKPlHETQ8jJ+Y8+1asRydi\npsP3B/5Mjhqv/uOK+Vy3zAyIpyDOMtIpOVfjSpCplVRdtSTFWBu9Em7j5I2HMn1w\nsJZnJgXKpybpibGiiTtmnFLOwibmprSu04rsnP4ncdC2XRD4wIjoyA+4PKgX3sCO\nklEzKryWYBmLkJOMDdo52LttP3279s7XrkLEE7ia0fXa2c12EQ0f0DQ1tGUvyVEW\nWmJVccm5bq25AQ0EUw5EzQEIANaPUY04/g7AmYkOMjaCZ6iTp9hB5Rsj/4ee/ln9\nwArzRO9+3eejLWh53FoN1rO+su7tiXJA5YAzVy6tuolrqjM8DBztPxdLBbEi4V+j\n2tK0dATdBQBHEh3OJApO2UBtcjaZBT31zrG9K55D+CrcgIVEHAKY8Cb4kLBkb5wM\nskn+DrASKU0BNIV1qRsxfiUdQHZfSqtp004nrql1lbFMLFEuiY8FZrkkQ9qduixo\nmTT6f34/oiY+Jam3zCK7RDN/OjuWheIPGj/Qbx9JuNiwgX6yRj7OE1tjUx6d8g9y\n0H1fmLJbb3WZZbuuGFnK6qrE3bGeY8+AWaJAZ37wpWh1p0cAEQEAAYkBHwQYAQIA\nCQUCUw5EzQIbDAAKCRBRhS2HNI/8TJntCAClU7TOO/X053eKF1jqNW4A1qpxctVc\nz8eTcY8Om5O4f6a/rfxfNFKn9Qyja/OG1xWNobETy7MiMXYjaa8uUx5iFy6kMVaP\n0BXJ59NLZjMARGw6lVTYDTIvzqqqwLxgliSDfSnqUhubGwvykANPO+93BBx89MRG\nunNoYGXtPlhNFrAsB1VR8+EyKLv2HQtGCPSFBhrjuzH3gxGibNDDdFQLxxuJWepJ\nEK1UbTS4ms0NgZ2Uknqn1WRU1Ki7rE4sTy68iZtWpKQXZEJa0IGnuI2sSINGcXCJ\noEIgXTMyCILo34Fa/C6VCm2WBgz9zZO8/rHIiQm1J5zqz0DrDwKBUM9C\n=LYpS\n-----END PGP PUBLIC KEY BLOCK-----", 150 | // "trust_signature": "", 151 | // "source": "HashiCorp", 152 | // "source_url": "https://www.hashicorp.com/security.html" 153 | // } 154 | // ] 155 | // } 156 | // } 157 | // 158 | // If the given since is not zero, and the remote has not modified, the function returns nil, nil. 159 | // 160 | // nolint:lll 161 | func (p Provider) GetPlatform( 162 | ctx context.Context, 163 | namespace, type_, version, os, arch string, 164 | since ...time.Time, 165 | ) ([]byte, error) { 166 | rq := httpCli.Request() 167 | if len(since) != 0 && !since[0].IsZero() { 168 | rq = rq.WithHeader("If-Modified-Since", since[0].Format(http.TimeFormat)) 169 | } 170 | 171 | r := rq.GetWithContext(ctx, 172 | resolveURLString((*url.URL)(&p), path.Join(namespace, type_, version, "download", os, arch)), 173 | ) 174 | 175 | if len(since) != 0 && !since[0].IsZero() && r.StatusCode() == http.StatusNotModified { 176 | return nil, nil 177 | } 178 | 179 | bs, err := r.BodyBytes() 180 | if err != nil { 181 | return nil, err 182 | } 183 | 184 | if json.Get(bs, "@this").IsObject() { 185 | return bs, nil 186 | } 187 | 188 | return []byte(`{}`), nil 189 | } 190 | 191 | type Module url.URL 192 | 193 | // Module switches the host to the module endpoint. 194 | func (h Host) Module(ctx context.Context) Module { 195 | return Module(h.Discover(ctx, "modules.v1")) 196 | } 197 | 198 | // GetVersions fetches the module version list by the given parameters. 199 | // See https://developer.hashicorp.com/terraform/internals/module-registry-protocol#list-available-versions-for-a-specific-module. 200 | // Response example: 201 | // 202 | // { 203 | // "modules": [ 204 | // { 205 | // "versions": [ 206 | // {"version": "1.0.0"}, 207 | // {"version": "1.1.0"}, 208 | // {"version": "2.0.0"} 209 | // ] 210 | // } 211 | // ] 212 | // } 213 | // 214 | // If the given since is not zero, and the remote has not modified, the function returns nil, nil. 215 | func (m Module) GetVersions(ctx context.Context, namespace, name, system string, since ...time.Time) ([]byte, error) { 216 | rq := httpCli.Request() 217 | if len(since) != 0 && !since[0].IsZero() { 218 | rq = rq.WithHeader("If-Modified-Since", since[0].Format(http.TimeFormat)) 219 | } 220 | 221 | r := rq.GetWithContext(ctx, 222 | resolveURLString((*url.URL)(&m), path.Join(namespace, name, system, "versions"))) 223 | 224 | if len(since) != 0 && !since[0].IsZero() && r.StatusCode() == http.StatusNotModified { 225 | return nil, nil 226 | } 227 | 228 | bs, err := r.BodyBytes() 229 | if err != nil { 230 | return nil, err 231 | } 232 | 233 | if json.Get(bs, "modules").IsArray() { 234 | return bs, nil 235 | } 236 | 237 | return []byte(`{"modules":[]}`), nil 238 | } 239 | 240 | // GetVersion fetches the module versioned information by the given parameters. 241 | // See https://developer.hashicorp.com/terraform/internals/module-registry-protocol#download-source-code-for-a-specific-module-version. 242 | // Response example: 243 | // 244 | // { 245 | // "download_url": "https://api.github.com/repos/hashicorp/terraform-aws-consul/tarball/v0.0.1//*?archive=tar.gz" 246 | // } 247 | // 248 | // If the given since is not zero, and the remote has not modified, the function returns nil, nil. 249 | func (m Module) GetVersion( 250 | ctx context.Context, 251 | namespace, name, system, version string, 252 | since ...time.Time, 253 | ) ([]byte, error) { 254 | rq := httpCli.Request() 255 | if len(since) != 0 && !since[0].IsZero() { 256 | rq = rq.WithHeader("If-Modified-Since", since[0].Format(http.TimeFormat)) 257 | } 258 | 259 | r := rq.GetWithContext(ctx, 260 | resolveURLString((*url.URL)(&m), path.Join(namespace, name, system, version, "download")), 261 | ) 262 | 263 | if len(since) != 0 && !since[0].IsZero() && r.StatusCode() == http.StatusNotModified { 264 | return nil, nil 265 | } 266 | 267 | downloadURL := r.Header("X-Terraform-Get") 268 | if downloadURL != "" { 269 | return []byte(`{"download_url":"` + downloadURL + `"}`), nil 270 | } 271 | 272 | return []byte(`{}`), nil 273 | } 274 | 275 | func resolveURL(u *url.URL, p string) *url.URL { 276 | return u.ResolveReference(&url.URL{Path: p}) 277 | } 278 | 279 | func resolveURLString(u *url.URL, p string) string { 280 | return resolveURL(u, p).String() 281 | } 282 | -------------------------------------------------------------------------------- /pkg/server/cmd.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | cli "github.com/urfave/cli/v2" 5 | ) 6 | 7 | func Command() *cli.Command { 8 | var cmd cli.Command 9 | server := New() 10 | server.Flags(&cmd) 11 | server.Before(&cmd) 12 | server.Action(&cmd) 13 | cmd.Name = "server" 14 | 15 | return &cmd 16 | } 17 | -------------------------------------------------------------------------------- /pkg/server/init.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "path/filepath" 7 | "reflect" 8 | "runtime" 9 | "strings" 10 | 11 | "github.com/seal-io/walrus/utils/strs" 12 | 13 | "github.com/seal-io/hermitcrab/pkg/database" 14 | "github.com/seal-io/hermitcrab/pkg/provider" 15 | ) 16 | 17 | type initOptions struct { 18 | ProviderService *provider.Service 19 | SkipTLSVerify bool 20 | BoltDriver database.BoltDriver 21 | } 22 | 23 | func (r *Server) init(ctx context.Context, opts initOptions) error { 24 | // Initialize data for system. 25 | inits := []initiation{ 26 | r.registerHealthCheckers, 27 | r.registerMetricCollectors, 28 | r.startTasks, 29 | } 30 | 31 | for i := range inits { 32 | if err := inits[i](ctx, opts); err != nil { 33 | return fmt.Errorf("failed to %s: %w", 34 | loadInitiationName(inits[i]), err) 35 | } 36 | } 37 | 38 | return nil 39 | } 40 | 41 | type initiation func(context.Context, initOptions) error 42 | 43 | func loadInitiationName(i initiation) string { 44 | n := runtime.FuncForPC(reflect.ValueOf(i).Pointer()).Name() 45 | n = strings.TrimPrefix(strings.TrimSuffix(filepath.Ext(n), "-fm"), ".") 46 | 47 | return strs.Decamelize(n, true) 48 | } 49 | -------------------------------------------------------------------------------- /pkg/server/init_health_checkers.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/seal-io/walrus/utils/gopool" 7 | 8 | "github.com/seal-io/hermitcrab/pkg/database" 9 | "github.com/seal-io/hermitcrab/pkg/health" 10 | ) 11 | 12 | // registerHealthCheckers registers the health checkers into the global health registry. 13 | func (r *Server) registerHealthCheckers(ctx context.Context, opts initOptions) error { 14 | cs := health.Checkers{ 15 | health.CheckerFunc("database", getDatabaseHealthChecker(opts.BoltDriver)), 16 | health.CheckerFunc("gopool", getGoPoolHealthChecker()), 17 | } 18 | 19 | return health.Register(ctx, cs) 20 | } 21 | 22 | func getDatabaseHealthChecker(db database.BoltDriver) health.Check { 23 | return func(ctx context.Context) error { 24 | return database.IsConnected(ctx, db) 25 | } 26 | } 27 | 28 | func getGoPoolHealthChecker() health.Check { 29 | return func(_ context.Context) error { 30 | return gopool.IsHealthy() 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /pkg/server/init_metric_collectors.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/seal-io/walrus/utils/cron" 7 | "github.com/seal-io/walrus/utils/gopool" 8 | 9 | "github.com/seal-io/hermitcrab/pkg/apis/runtime" 10 | "github.com/seal-io/hermitcrab/pkg/database" 11 | "github.com/seal-io/hermitcrab/pkg/metric" 12 | ) 13 | 14 | // registerMetricCollectors registers the metric collectors into the global metric registry. 15 | func (r *Server) registerMetricCollectors(ctx context.Context, opts initOptions) error { 16 | cs := metric.Collectors{ 17 | database.NewStatsCollectorWith(opts.BoltDriver), 18 | gopool.NewStatsCollector(), 19 | cron.NewStatsCollector(), 20 | runtime.NewStatsCollector(), 21 | } 22 | 23 | return metric.Register(ctx, cs) 24 | } 25 | -------------------------------------------------------------------------------- /pkg/server/init_tasks.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/seal-io/walrus/utils/cron" 8 | 9 | "github.com/seal-io/hermitcrab/pkg/tasks/provider" 10 | ) 11 | 12 | // startTasks starts the tasks by Cron Expression to do something periodically in background. 13 | func (r *Server) startTasks(ctx context.Context, opts initOptions) (err error) { 14 | // Start cron scheduler. 15 | err = cron.Start(ctx, nil) 16 | if err != nil { 17 | return fmt.Errorf("error starting cron scheduler: %w", err) 18 | } 19 | 20 | // Register tasks. 21 | err = cron.Schedule(provider.SyncMetadata(ctx, opts.ProviderService)) 22 | 23 | return err 24 | } 25 | -------------------------------------------------------------------------------- /pkg/server/init_test.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func privateInitiationName(context.Context, initOptions) error { 11 | panic("test only") 12 | } 13 | 14 | func PublicInitiationName(context.Context, initOptions) error { 15 | panic("test only") 16 | } 17 | 18 | type _X struct{} 19 | 20 | func (_X) StructInitiationName(context.Context, initOptions) error { 21 | panic("test only") 22 | } 23 | 24 | func Test_loadInitiationName(t *testing.T) { 25 | anonymityInitiationName := func(context.Context, initOptions) error { 26 | panic("test only") 27 | } 28 | 29 | testCases := []struct { 30 | given initiation 31 | expected string 32 | }{ 33 | { 34 | given: privateInitiationName, 35 | expected: "private initiation name", 36 | }, 37 | { 38 | given: PublicInitiationName, 39 | expected: "public initiation name", 40 | }, 41 | { 42 | given: _X{}.StructInitiationName, 43 | expected: "struct initiation name", 44 | }, 45 | { 46 | given: anonymityInitiationName, 47 | expected: "func1", 48 | }, 49 | } 50 | 51 | for _, tc := range testCases { 52 | t.Run(tc.expected, func(t *testing.T) { 53 | actual := loadInitiationName(tc.given) 54 | assert.Equal(t, tc.expected, actual) 55 | }) 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /pkg/server/runner.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "flag" 7 | "fmt" 8 | stdlog "log" 9 | "net" 10 | "os" 11 | "path/filepath" 12 | "strconv" 13 | 14 | "github.com/seal-io/walrus/utils/clis" 15 | "github.com/seal-io/walrus/utils/files" 16 | "github.com/seal-io/walrus/utils/gopool" 17 | "github.com/seal-io/walrus/utils/log" 18 | "github.com/seal-io/walrus/utils/runtimex" 19 | "github.com/sirupsen/logrus" 20 | cli "github.com/urfave/cli/v2" 21 | "k8s.io/apimachinery/pkg/util/validation" 22 | "k8s.io/apimachinery/pkg/util/validation/field" 23 | klog "k8s.io/klog/v2" 24 | 25 | "github.com/seal-io/hermitcrab/pkg/consts" 26 | "github.com/seal-io/hermitcrab/pkg/database" 27 | "github.com/seal-io/hermitcrab/pkg/provider" 28 | ) 29 | 30 | type Server struct { 31 | Logger clis.Logger 32 | 33 | BindAddress string 34 | BindWithDualStack bool 35 | EnableTls bool 36 | TlsCertFile string 37 | TlsPrivateKeyFile string 38 | TlsCertDir string 39 | TlsAutoCertDomains []string 40 | ConnQPS int 41 | ConnBurst int 42 | WebsocketConnMaxPerIP int 43 | GopoolWorkerFactor int 44 | 45 | DataSourceDir string 46 | DataSourceLockMemory bool 47 | } 48 | 49 | func New() *Server { 50 | return &Server{ 51 | BindAddress: "0.0.0.0", 52 | BindWithDualStack: true, 53 | EnableTls: true, 54 | TlsCertDir: filepath.Join(consts.DataDir, "tls"), 55 | ConnQPS: 100, 56 | ConnBurst: 200, 57 | WebsocketConnMaxPerIP: 25, 58 | GopoolWorkerFactor: 100, 59 | 60 | DataSourceDir: filepath.Join(consts.DataDir, "data"), 61 | DataSourceLockMemory: false, 62 | } 63 | } 64 | 65 | func (r *Server) Flags(cmd *cli.Command) { 66 | flags := [...]cli.Flag{ 67 | &cli.StringFlag{ 68 | Name: "bind-address", 69 | Usage: "The IP address on which to listen.", 70 | Destination: &r.BindAddress, 71 | Value: r.BindAddress, 72 | Action: func(c *cli.Context, s string) error { 73 | if s != "" && net.ParseIP(s) == nil { 74 | return errors.New("--bind-address: invalid IP address") 75 | } 76 | return nil 77 | }, 78 | }, 79 | &cli.BoolFlag{ 80 | Name: "bind-with-dual-stack", 81 | Usage: "Enable dual stack socket listening.", 82 | Destination: &r.BindWithDualStack, 83 | Value: r.BindWithDualStack, 84 | }, 85 | &cli.BoolFlag{ 86 | Name: "enable-tls", 87 | Usage: "Enable HTTPs.", 88 | Destination: &r.EnableTls, 89 | Value: r.EnableTls, 90 | }, 91 | &cli.StringFlag{ 92 | Name: "tls-cert-file", 93 | Usage: "The file containing the default x509 certificate for HTTPS. " + 94 | "If any CA certs, concatenated after server cert file. ", 95 | Destination: &r.TlsCertFile, 96 | Value: r.TlsCertFile, 97 | Action: func(c *cli.Context, s string) error { 98 | if s != "" && 99 | !files.Exists(s) { 100 | return errors.New("--tls-cert-file: file is not existed") 101 | } 102 | return nil 103 | }, 104 | }, 105 | &cli.StringFlag{ 106 | Name: "tls-private-key-file", 107 | Usage: "The file containing the default x509 private key matching --tls-cert-file.", 108 | Destination: &r.TlsPrivateKeyFile, 109 | Value: r.TlsPrivateKeyFile, 110 | Action: func(c *cli.Context, s string) error { 111 | if s != "" && 112 | !files.Exists(s) { 113 | return errors.New("--tls-private-key-file: file is not existed") 114 | } 115 | return nil 116 | }, 117 | }, 118 | &cli.StringFlag{ 119 | Name: "tls-cert-dir", 120 | Usage: "The directory where the TLS certs are located. " + 121 | "If --tls-cert-file and --tls-private-key-file are provided, this flag will be ignored. " + 122 | "If --tls-cert-file and --tls-private-key-file are not provided, " + 123 | "the certificate and key of auto-signed or self-signed are saved to where this flag specified. ", 124 | Destination: &r.TlsCertDir, 125 | Value: r.TlsCertDir, 126 | Action: func(c *cli.Context, s string) error { 127 | if c.String("tls-cert-file") != "" && c.String("tls-private-key-file") != "" { 128 | return nil 129 | } 130 | 131 | if s == "" { 132 | return errors.New( 133 | "--tls-cert-dir: must be filled if --tls-cert-file and --tls-private-key-file are not provided") 134 | } 135 | 136 | if !filepath.IsAbs(s) { 137 | return errors.New("--tls-cert-dir: must be absolute path") 138 | } 139 | 140 | return nil 141 | }, 142 | }, 143 | &cli.StringSliceFlag{ 144 | Name: "tls-auto-cert-domains", 145 | Usage: "The domains to accept ACME HTTP-01 or TLS-ALPN-01 challenge to " + 146 | "generate HTTPS x509 certificate and private key, " + 147 | "and saved to the directory specified by --tls-cert-dir. " + 148 | "If --tls-cert-file and --tls-key-file are provided, this flag will be ignored.", 149 | Action: func(c *cli.Context, v []string) error { 150 | f := field.NewPath("--tls-auto-cert-domains") 151 | for i := range v { 152 | if err := validation.IsFullyQualifiedDomainName(f, v[i]).ToAggregate(); err != nil { 153 | return err 154 | } 155 | } 156 | if len(v) != 0 && 157 | (c.String("tls-cert-dir") == "" && 158 | (c.String("tls-cert-file") == "" || c.String("tls-private-key-file") == "")) { 159 | return errors.New("--tls-cert-dir: must be filled") 160 | } 161 | r.TlsAutoCertDomains = v 162 | return nil 163 | }, 164 | Value: cli.NewStringSlice(r.TlsAutoCertDomains...), 165 | }, 166 | &cli.IntFlag{ 167 | Name: "conn-qps", 168 | Usage: "The qps(maximum average number per second) when dialing the server.", 169 | Destination: &r.ConnQPS, 170 | Value: r.ConnQPS, 171 | }, 172 | &cli.IntFlag{ 173 | Name: "conn-burst", 174 | Usage: "The burst(maximum number at the same moment) when dialing the server.", 175 | Destination: &r.ConnBurst, 176 | Value: r.ConnBurst, 177 | }, 178 | &cli.IntFlag{ 179 | Name: "websocket-conn-max-per-ip", 180 | Usage: "The maximum number of websocket connections per IP.", 181 | Destination: &r.WebsocketConnMaxPerIP, 182 | Value: r.WebsocketConnMaxPerIP, 183 | }, 184 | &cli.IntFlag{ 185 | Name: "gopool-worker-factor", 186 | Usage: "The gopool worker factor determines the number of tasks of the goroutine worker pool," + 187 | "it is calculated by the number of CPU cores multiplied by this factor.", 188 | Action: func(c *cli.Context, i int) error { 189 | if i < 100 { 190 | return errors.New("too small --gopool-worker-factor: must be greater than 100") 191 | } 192 | return nil 193 | }, 194 | Destination: &r.GopoolWorkerFactor, 195 | Value: r.GopoolWorkerFactor, 196 | }, 197 | &cli.StringFlag{ 198 | Name: "data-source-dir", 199 | Usage: "The directory where the data are stored.", 200 | Action: func(c *cli.Context, s string) error { 201 | if s == "" { 202 | return errors.New("--data-source-dir: must be filled") 203 | } 204 | 205 | if !filepath.IsAbs(s) { 206 | return errors.New("--data-source-dir: must be absolute path") 207 | } 208 | 209 | return nil 210 | }, 211 | Destination: &r.DataSourceDir, 212 | Value: r.DataSourceDir, 213 | }, 214 | &cli.BoolFlag{ 215 | Name: "data-source-lock-memory", 216 | Usage: "Lock the data source files in memory, which can prevent potential page faults.", 217 | Destination: &r.DataSourceLockMemory, 218 | Value: r.DataSourceLockMemory, 219 | }, 220 | } 221 | for i := range flags { 222 | cmd.Flags = append(cmd.Flags, flags[i]) 223 | } 224 | 225 | r.Logger.Flags(cmd) 226 | } 227 | 228 | func (r *Server) Before(cmd *cli.Command) { 229 | pb := cmd.Before 230 | cmd.Before = func(c *cli.Context) error { 231 | l := log.GetLogger() 232 | 233 | // Sink the output of standard logger to util logger. 234 | stdlog.SetOutput(l) 235 | 236 | // Turn on the logrus logger 237 | // and sink the output to util logger. 238 | logrus.SetLevel(logrus.TraceLevel) 239 | logrus.SetFormatter(log.AsLogrusFormatter(l)) 240 | 241 | // Turn on klog logger according to the verbosity, 242 | // and sink the output to util logger. 243 | { 244 | var flags flag.FlagSet 245 | 246 | klog.InitFlags(&flags) 247 | _ = flags.Set("v", strconv.FormatUint(log.GetVerbosity(), 10)) 248 | _ = flags.Set("skip_headers", "true") 249 | } 250 | klog.SetLogger(log.AsLogr(l)) 251 | 252 | if pb != nil { 253 | return pb(c) 254 | } 255 | 256 | // Init set GOMAXPROCS. 257 | runtimex.Init() 258 | 259 | return nil 260 | } 261 | 262 | r.Logger.Before(cmd) 263 | } 264 | 265 | func (r *Server) Action(cmd *cli.Command) { 266 | cmd.Action = func(c *cli.Context) error { 267 | return r.Run(c.Context) 268 | } 269 | } 270 | 271 | func (r *Server) Run(c context.Context) error { 272 | if err := r.configure(); err != nil { 273 | return fmt.Errorf("error configuring: %w", err) 274 | } 275 | 276 | g, ctx := gopool.GroupWithContext(c) 277 | 278 | // Load database driver. 279 | var bolt database.Bolt 280 | 281 | g.Go(func() error { 282 | log.Info("running database") 283 | 284 | err := bolt.Run(ctx, r.DataSourceDir, r.DataSourceLockMemory) 285 | if err != nil { 286 | log.Errorf("error running database: %v", err) 287 | } 288 | 289 | return err 290 | }) 291 | 292 | // Create service clients. 293 | boltDriver := bolt.GetDriver() 294 | 295 | providerService, err := provider.NewService(boltDriver, r.DataSourceDir) 296 | if err != nil { 297 | return fmt.Errorf("error creating provider service: %w", err) 298 | } 299 | 300 | // Initialize some resources. 301 | log.Info("initializing") 302 | 303 | initOpts := initOptions{ 304 | ProviderService: providerService, 305 | SkipTLSVerify: len(r.TlsAutoCertDomains) != 0, 306 | BoltDriver: boltDriver, 307 | } 308 | 309 | if err := r.init(ctx, initOpts); err != nil { 310 | log.Errorf("error initializing: %v", err) 311 | return fmt.Errorf("error initializing: %w", err) 312 | } 313 | 314 | // Run apis. 315 | startApisOpts := startApisOptions{ 316 | ProviderService: providerService, 317 | } 318 | 319 | g.Go(func() error { 320 | log.Info("starting apis") 321 | 322 | err := r.startApis(ctx, startApisOpts) 323 | if err != nil { 324 | log.Errorf("error starting apis: %v", err) 325 | } 326 | 327 | return err 328 | }) 329 | 330 | return g.Wait() 331 | } 332 | 333 | func (r *Server) configure() error { 334 | // Configure gopool. 335 | gopool.Reset(r.GopoolWorkerFactor) 336 | 337 | // Configure data source dir. 338 | if err := os.MkdirAll(r.DataSourceDir, 0o700); err != nil { 339 | if !os.IsExist(err) { 340 | return fmt.Errorf("--data-source-dir: %w", err) 341 | } 342 | 343 | i, _ := os.Stat(r.DataSourceDir) 344 | if !i.IsDir() { 345 | return errors.New("--data-source-dir: not directory") 346 | } 347 | } 348 | 349 | return nil 350 | } 351 | -------------------------------------------------------------------------------- /pkg/server/start_apis.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | 7 | "github.com/seal-io/hermitcrab/pkg/apis" 8 | "github.com/seal-io/hermitcrab/pkg/provider" 9 | ) 10 | 11 | type startApisOptions struct { 12 | ProviderService *provider.Service 13 | } 14 | 15 | func (r *Server) startApis(ctx context.Context, opts startApisOptions) error { 16 | srv, err := apis.NewServer() 17 | if err != nil { 18 | return err 19 | } 20 | 21 | serveOpts := apis.ServeOptions{ 22 | SetupOptions: apis.SetupOptions{ 23 | ConnQPS: r.ConnQPS, 24 | ConnBurst: r.ConnBurst, 25 | WebsocketConnMaxPerIP: r.WebsocketConnMaxPerIP, 26 | ProviderService: opts.ProviderService, 27 | }, 28 | BindAddress: r.BindAddress, 29 | BindWithDualStack: r.BindWithDualStack, 30 | } 31 | 32 | switch { 33 | default: 34 | serveOpts.TlsMode = apis.TlsModeSelfGenerated 35 | serveOpts.TlsCertDir = r.TlsCertDir 36 | case !r.EnableTls: 37 | serveOpts.TlsMode = apis.TlsModeDisabled 38 | case r.TlsCertFile != "" && r.TlsPrivateKeyFile != "": 39 | serveOpts.TlsMode = apis.TlsModeCustomized 40 | serveOpts.TlsCertFile = r.TlsCertFile 41 | serveOpts.TlsPrivateKeyFile = r.TlsPrivateKeyFile 42 | case len(r.TlsAutoCertDomains) != 0: 43 | serveOpts.TlsMode = apis.TlsModeAutoGenerated 44 | serveOpts.TlsCertified = true 45 | serveOpts.TlsCertDir = r.TlsCertDir 46 | serveOpts.TlsAutoCertDomains = r.TlsAutoCertDomains 47 | } 48 | 49 | err = srv.Serve(ctx, serveOpts) 50 | if err != nil && !errors.Is(err, context.Canceled) { 51 | return err 52 | } 53 | 54 | return nil 55 | } 56 | -------------------------------------------------------------------------------- /pkg/tasks/provider/task.go: -------------------------------------------------------------------------------- 1 | package provider 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/seal-io/walrus/utils/cron" 7 | 8 | "github.com/seal-io/hermitcrab/pkg/provider" 9 | ) 10 | 11 | // SyncMetadata creates a Cron task to sync the metadata from remote to local 30 minutes. 12 | func SyncMetadata(_ context.Context, providerService *provider.Service) (name string, expr cron.Expr, task cron.Task) { 13 | name = "tasks.provider.sync_metadata" 14 | expr = cron.ImmediateExpr("0 */30 * ? * *") 15 | task = cron.TaskFunc(func(ctx context.Context, args ...any) error { 16 | return providerService.Metadata.Sync(ctx) 17 | }) 18 | 19 | return name, expr, task 20 | } 21 | --------------------------------------------------------------------------------