├── .dockerignore ├── .github ├── CODEOWNERS ├── dependabot.yml └── workflows │ ├── commit-linter.yaml │ ├── commit-staticcheck.yaml │ ├── docker-build.yaml │ ├── fuzz-test.yaml │ ├── release-please.yaml │ └── reuse.yaml ├── .gitignore ├── .pre-commit-config.yaml ├── .release-please-config.json ├── .release-please-config.json.license ├── .release-please-manifest.json ├── .release-please-manifest.json.license ├── CHANGELOG.md ├── Dockerfile ├── HACKING.md ├── LICENSE ├── LICENSES └── MIT.txt ├── Makefile ├── README.md ├── REUSE.toml ├── _examples └── terraform-project │ ├── README.md │ └── main.tf ├── _tools └── openssl │ ├── create_self_signed_cert.sh │ └── openssl.conf ├── cmd └── terraform-registry │ ├── main.go │ └── main_test.go ├── go.mod ├── go.sum ├── go.sum.license └── pkg ├── core └── core.go ├── registry ├── registry.go └── registry_test.go └── store ├── github ├── github.go └── github_test.go ├── memory ├── memory.go └── memory_test.go └── s3 ├── s3.go └── s3_test.go /.dockerignore: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2022 - 2025 NRK 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | Dockerfile 6 | terraform-registry 7 | -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2022 - 2024 NRK 2 | # 3 | # SPDX-License-Identifier: MIT 4 | * @nrkno/iac-admins 5 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2022 - 2024 NRK 2 | # 3 | # SPDX-License-Identifier: MIT 4 | version: 2 5 | updates: 6 | - package-ecosystem: gomod 7 | directory: "/" 8 | schedule: 9 | interval: weekly 10 | time: "13:00" 11 | timezone: "Europe/Oslo" 12 | commit-message: 13 | prefix: build(deps) 14 | open-pull-requests-limit: 5 15 | reviewers: 16 | - nrkno/iac-admins 17 | - package-ecosystem: "github-actions" 18 | directory: "/" 19 | schedule: 20 | interval: "daily" 21 | time: "13:00" 22 | timezone: "Europe/Oslo" 23 | commit-message: 24 | prefix: ci(workflow) 25 | open-pull-requests-limit: 5 26 | reviewers: 27 | - nrkno/iac-admins 28 | - package-ecosystem: "docker" 29 | directory: "/" 30 | schedule: 31 | interval: "daily" 32 | time: "13:00" 33 | timezone: "Europe/Oslo" 34 | commit-message: 35 | prefix: build(docker) 36 | open-pull-requests-limit: 5 37 | reviewers: 38 | - nrkno/iac-admins 39 | -------------------------------------------------------------------------------- /.github/workflows/commit-linter.yaml: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2022 - 2024 NRK 2 | # 3 | # SPDX-License-Identifier: MIT 4 | on: [push, pull_request] 5 | 6 | name: Commit lint and release 7 | jobs: 8 | lint_release: 9 | uses: nrkno/github-workflow-semantic-release/.github/workflows/workflow.yaml@v4.2.1 10 | with: 11 | release-enabled: false 12 | runs-on: ubuntu-latest 13 | -------------------------------------------------------------------------------- /.github/workflows/commit-staticcheck.yaml: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2022 - 2024 NRK 2 | # 3 | # SPDX-License-Identifier: MIT 4 | name: "Golang Static checker" 5 | on: 6 | - pull_request 7 | 8 | jobs: 9 | staticcheck: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v4.2.2 13 | - uses: actions/setup-go@v5.5.0 14 | with: 15 | go-version-file: "go.mod" 16 | id: setup-go 17 | - uses: dominikh/staticcheck-action@v1.3.1 18 | with: 19 | version: "2024.1.1" 20 | install-go: false 21 | min-go-version: "module" 22 | -------------------------------------------------------------------------------- /.github/workflows/docker-build.yaml: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2022 - 2024 NRK 2 | # 3 | # SPDX-License-Identifier: MIT 4 | name: "Build and test" 5 | 6 | on: 7 | push: 8 | branches: 9 | - main 10 | pull_request: 11 | paths-ignore: 12 | - '.github/' 13 | branches: 14 | - main 15 | 16 | jobs: 17 | build: 18 | runs-on: ubuntu-latest 19 | steps: 20 | - 21 | name: Checkout 22 | uses: actions/checkout@v4.2.2 23 | - 24 | name: Docker build 25 | env: 26 | DOCKER_TAG: terraform-registry:latest 27 | run: | 28 | make DOCKER_TAG="${DOCKER_TAG}" build-docker 29 | docker image save ${DOCKER_TAG} -o image.tar 30 | - 31 | name: Trivy vulnerability scan 32 | uses: aquasecurity/trivy-action@0.31.0 33 | with: 34 | input: image.tar 35 | format: 'table' 36 | exit-code: '1' 37 | ignore-unfixed: true 38 | vuln-type: 'os,library' 39 | severity: 'CRITICAL,HIGH' 40 | env: 41 | TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db 42 | TRIVY_JAVA_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-java-db 43 | -------------------------------------------------------------------------------- /.github/workflows/fuzz-test.yaml: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2022 - 2024 NRK 2 | # 3 | # SPDX-License-Identifier: MIT 4 | name: Fuzz Test 5 | 6 | on: 7 | pull_request: 8 | paths-ignore: 9 | - '.github/' 10 | branches: 11 | - main 12 | 13 | jobs: 14 | FuzzTests: 15 | strategy: 16 | matrix: 17 | tests: ["FuzzTokenAuth;./pkg/registry;60s", "FuzzRoutes;./pkg/registry;60s"] 18 | runs-on: ubuntu-latest 19 | steps: 20 | - uses: actions/checkout@v4.2.2 21 | - uses: actions/setup-go@v5.5.0 22 | with: 23 | go-version-file: "go.mod" 24 | - name: Fuzz test 25 | shell: bash 26 | env: 27 | TEST: "${{ matrix.tests }}" 28 | run: | 29 | IFS=";" read test pkg timeout <<<$TEST 30 | go test "${pkg}" -fuzztime="${timeout}" -fuzz="${test}" 31 | - name: Upload fuzz failure seed corpus as run artifact 32 | if: failure() 33 | uses: actions/upload-artifact@v4.6.2 34 | with: 35 | name: testdata 36 | path: pkg/registry/testdata 37 | - name: Report failure 38 | uses: actions/github-script@v7.0.1 39 | if: failure() && github.event_name == 'pull_request' 40 | with: 41 | script: | 42 | github.rest.issues.createComment({ 43 | issue_number: context.issue.number, 44 | owner: context.repo.owner, 45 | repo: context.repo.repo, 46 | body: 'Fuzz test failed on ${{ github.event.pull_request.head.sha }}. To troubleshoot locally, use the [GitHub CLI](https://cli.github.com) to download the seed corpus with\n```\ngh run download ${{ github.run_id }} -n testdata\n```' 47 | }) 48 | -------------------------------------------------------------------------------- /.github/workflows/release-please.yaml: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2022 - 2024 NRK 2 | # 3 | # SPDX-License-Identifier: MIT 4 | on: 5 | push: 6 | branches: 7 | - main 8 | 9 | name: Release 10 | jobs: 11 | release-please: 12 | runs-on: ubuntu-latest 13 | steps: 14 | - uses: googleapis/release-please-action@v4.2.0 15 | id: release-please 16 | with: 17 | config-file: .release-please-config.json 18 | manifest-file: .release-please-manifest.json 19 | outputs: 20 | release_created: ${{ steps.release-please.outputs.release_created }} 21 | tag_name: ${{ steps.release-please.outputs.tag_name }} 22 | 23 | build-binaries: 24 | name: Build and push binaries 25 | runs-on: ubuntu-latest 26 | needs: release-please 27 | if: needs.release-please.outputs.release_created 28 | strategy: 29 | matrix: 30 | arch: 31 | - amd64 32 | - arm64 33 | os: 34 | - linux 35 | steps: 36 | - 37 | name: Checkout 38 | uses: actions/checkout@v4.2.2 39 | with: 40 | ref: ${{ needs.release-please.outputs.tag_name }} 41 | - 42 | name: Setup Go 43 | uses: actions/setup-go@v5.5.0 44 | with: 45 | go-version-file: 'go.mod' 46 | - 47 | name: Build and upload 48 | env: 49 | GH_TOKEN: ${{ github.token }} 50 | GOOS: ${{ matrix.os }} 51 | GOARCH: ${{ matrix.arch }} 52 | VERSION: ${{ needs.release-please.outputs.tag_name }} 53 | run: | 54 | LD_FLAGS="" 55 | LD_FLAGS+=" -s" # no debug symbols 56 | LD_FLAGS+=" -w" # no DWARF debug info 57 | LD_FLAGS+=" -X 'main.buildDate=$(date --utc +%Y-%m-%dT%H:%M:%SZ)'" 58 | LD_FLAGS+=" -X 'main.version=${VERSION}'" 59 | ARCHIVE_NAME="terraform-registry_${VERSION}_${GOOS}_${GOARCH}.tar.gz" 60 | 61 | make LD_FLAGS="${LD_FLAGS}" build 62 | tar -cvzf "${ARCHIVE_NAME}" terraform-registry 63 | gh release upload ${VERSION} "${ARCHIVE_NAME}" 64 | -------------------------------------------------------------------------------- /.github/workflows/reuse.yaml: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2022 - 2024 NRK 2 | # 3 | # SPDX-License-Identifier: MIT 4 | on: 5 | push: 6 | branches: 7 | - main 8 | pull_request: 9 | branches: 10 | - main 11 | 12 | name: REUSE Compliance Check 13 | jobs: 14 | reuse-compliance-check: 15 | runs-on: ubuntu-latest 16 | steps: 17 | - 18 | name: Checkout 19 | uses: actions/checkout@v4.2.2 20 | - 21 | name: REUSE Compliance Check 22 | uses: fsfe/reuse-action@v5.0.0 23 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2022 - 2024 NRK 2 | # 3 | # SPDX-License-Identifier: MIT 4 | /terraform-registry 5 | /tmp 6 | .terraform 7 | /*.env 8 | _tools/openssl/cert.crt 9 | _tools/openssl/cert.key 10 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2022 - 2025 NRK 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | repos: 6 | - repo: https://github.com/compilerla/conventional-pre-commit 7 | rev: v1.3.0 8 | hooks: 9 | - id: conventional-pre-commit 10 | stages: [commit-msg] 11 | args: [build, chore, ci, docs, feat, fix, perf, refactor, test] # optional: list of Conventional Commits types to allow 12 | -------------------------------------------------------------------------------- /.release-please-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "packages": { 3 | ".": { 4 | "changelog-path": "CHANGELOG.md", 5 | "release-type": "go", 6 | "bump-minor-pre-major": true, 7 | "bump-patch-for-minor-pre-major": false, 8 | "draft": false, 9 | "draft-pull-request": true, 10 | "prerelease": false 11 | } 12 | }, 13 | "$schema": "https://raw.githubusercontent.com/googleapis/release-please/main/schemas/config.json" 14 | } 15 | -------------------------------------------------------------------------------- /.release-please-config.json.license: -------------------------------------------------------------------------------- 1 | SPDX-FileCopyrightText: 2024 - 2025 NRK 2 | 3 | SPDX-License-Identifier: MIT 4 | -------------------------------------------------------------------------------- /.release-please-manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | ".": "0.20.1" 3 | } 4 | -------------------------------------------------------------------------------- /.release-please-manifest.json.license: -------------------------------------------------------------------------------- 1 | SPDX-FileCopyrightText: 2024 - 2025 NRK 2 | 3 | SPDX-License-Identifier: MIT 4 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | 6 | 7 | # Changelog 8 | 9 | ## [0.20.1](https://github.com/nrkno/terraform-registry/compare/v0.20.0...v0.20.1) (2025-01-13) 10 | 11 | 12 | ### Bug Fixes 13 | 14 | * don't update cache if ratelimited ([63d87a0](https://github.com/nrkno/terraform-registry/commit/63d87a0621cd4711c34b6a16622c3b45c466ecf4)) 15 | * remove debug log for unchanged watchFile ([e70d874](https://github.com/nrkno/terraform-registry/commit/e70d8749c3b7004ad77f46dc7b38731ebb6fd683)) 16 | * **store/github:** improve details in empty provider result warning ([cf733fb](https://github.com/nrkno/terraform-registry/commit/cf733fb033b49798b690e551cd726cf3d3a1ff23)) 17 | * **store/github:** load module cache before providers ([a36dbbc](https://github.com/nrkno/terraform-registry/commit/a36dbbc1032a9fa1c8da81169989a83bc82f2844)) 18 | * **store/github:** log warning when no module repos were found ([b70f72f](https://github.com/nrkno/terraform-registry/commit/b70f72f5d505230fd50ecda039bdf0a6a6a53ecf)) 19 | 20 | ## [0.20.0](https://github.com/nrkno/terraform-registry/compare/v0.19.0...v0.20.0) (2024-04-17) 21 | 22 | 23 | ### Features 24 | 25 | * add v1/providers protocol ([2a7295c](https://github.com/nrkno/terraform-registry/commit/2a7295c4c741eb6b3a90d8d69cde35b7effb2368)) 26 | 27 | 28 | ### Bug Fixes 29 | 30 | * check if provider version exists in cache before trying to download ([9b1b0b7](https://github.com/nrkno/terraform-registry/commit/9b1b0b7a933b866cc0c83c1ae2ddb6871bb62229)) 31 | * ignore releases previously found not valid ([c9a3368](https://github.com/nrkno/terraform-registry/commit/c9a336837c3f606e66d2babe208526a7ce4705fd)) 32 | * proxy signature url ([c2aa0db](https://github.com/nrkno/terraform-registry/commit/c2aa0db0ce5c3e01a733c05e726ef6f8b7c02aa3)) 33 | 34 | ## [0.19.0](https://github.com/nrkno/terraform-registry/compare/v0.18.0...v0.19.0) (2024-01-17) 35 | 36 | 37 | ### ⚠ BREAKING CHANGES 38 | 39 | * remove redundant log on empty auth token file 40 | * require -auth-disabled when -auth-tokens-file is unset 41 | 42 | ### Bug Fixes 43 | 44 | * remove redundant log on empty auth token file ([03bc360](https://github.com/nrkno/terraform-registry/commit/03bc3605f7dc3c8e7c8504ded0a234a93a1a5fdc)) 45 | * require -auth-disabled when -auth-tokens-file is unset ([b632295](https://github.com/nrkno/terraform-registry/commit/b6322953ec46d6018cfb17ff31f0feae35cad3f7)) 46 | 47 | ## [0.18.0](https://github.com/nrkno/terraform-registry/compare/v0.17.0...v0.18.0) (2024-01-11) 48 | 49 | 50 | ### Features 51 | 52 | * add -version arg ([3d16771](https://github.com/nrkno/terraform-registry/commit/3d1677151209ddb565dbe71bb3626e950065290a)) 53 | 54 | 55 | ### Bug Fixes 56 | 57 | * avoid race when verifying auth tokens ([6fe703a](https://github.com/nrkno/terraform-registry/commit/6fe703afcf62da0e4751d8f9fc1062084da6a3af)) 58 | 59 | ## [0.17.0](https://github.com/nrkno/terraform-registry/compare/v0.16.0...v0.17.0) (2023-12-20) 60 | 61 | 62 | ### Features 63 | 64 | * Add support for s3 as a backend store ([3619ebe](https://github.com/nrkno/terraform-registry/commit/3619ebe23f61442d6695002748bacd46aa89f1c1)) 65 | 66 | ## [0.16.0](https://github.com/nrkno/terraform-registry/compare/v0.15.0...v0.16.0) (2023-12-19) 67 | 68 | 69 | ### Features 70 | 71 | * allow disabling HTTP access log ([dafa222](https://github.com/nrkno/terraform-registry/commit/dafa222aa39964ae676a3e0f4a89b13d2833c2fe)), closes [#63](https://github.com/nrkno/terraform-registry/issues/63) 72 | * allow ignoring certain paths from access log ([7c48d2f](https://github.com/nrkno/terraform-registry/commit/7c48d2f2f7587de41cbdbb6a43b992c2fbcd4c4b)) 73 | * use zap for http access logging ([df72911](https://github.com/nrkno/terraform-registry/commit/df7291144fd4972198ddb134afe1cdb8c5661598)), closes [#54](https://github.com/nrkno/terraform-registry/issues/54) 74 | 75 | ## [0.15.0](https://github.com/nrkno/terraform-registry/compare/v0.14.0...v0.15.0) (2023-03-17) 76 | 77 | 78 | ### ⚠ BREAKING CHANGES 79 | 80 | * do not crash if reading auth tokens fail 81 | * store registry auth tokens as a map 82 | * remove support for newline separated auth file 83 | 84 | ### Features 85 | 86 | * automatically update auth tokens on auth file changes ([e3d406e](https://github.com/nrkno/terraform-registry/commit/e3d406e632e2254001027afd5f57da240706ab91)) 87 | * remove support for newline separated auth file ([8fce7d7](https://github.com/nrkno/terraform-registry/commit/8fce7d7d659c5c812e983618f42285d4898d39c2)) 88 | * store registry auth tokens as a map ([6294753](https://github.com/nrkno/terraform-registry/commit/629475355c998e15a13b6bc722e3cd2ef166aa15)) 89 | 90 | 91 | ### Bug Fixes 92 | 93 | * do not crash if reading auth tokens fail ([40354e6](https://github.com/nrkno/terraform-registry/commit/40354e66bec988c4724d937a79dcdab348e76066)) 94 | * remove unimplemented login service definition ([b9e0ff0](https://github.com/nrkno/terraform-registry/commit/b9e0ff06207d39f76ff0bfe021d4c5de92fe9e9e)) 95 | 96 | ## 0.14.0 (2022-09-20) 97 | 98 | 99 | ### ⚠ BREAKING CHANGES 100 | 101 | * warn instead of panic for empty auth tokens file 102 | * only apply auth to /v1 API endpoints 103 | * set main binary as entrypoint in Dockerfile 104 | * rename github cmd args to better reflect usage 105 | 106 | ### Features 107 | 108 | * add build-docker target to makefile ([2e9bb34](https://github.com/nrkno/terraform-registry/commit/2e9bb34cf47899508f0d4fceff28c0698af1181e)) 109 | * add initial fuzz testing of auth ([5eb4c95](https://github.com/nrkno/terraform-registry/commit/5eb4c956486cf1b23c99c5d37ddff67852cff365)) 110 | * add log level and format arguments ([8cadf9a](https://github.com/nrkno/terraform-registry/commit/8cadf9afc931571eb3407792376d3d15d8ed3571)) 111 | * add routes fuzz test ([ef30a17](https://github.com/nrkno/terraform-registry/commit/ef30a179b5a9a19d8a4775f82c71df15c99c35ae)) 112 | * add support for auth token file to be a json file ([#26](https://github.com/nrkno/terraform-registry/issues/26)) ([ccbb94a](https://github.com/nrkno/terraform-registry/commit/ccbb94a04ef4500249fc90ae468437bb4af6d3cd)) 113 | * check github tags for valid semver version ([e3c1375](https://github.com/nrkno/terraform-registry/commit/e3c1375002aa33f6ee75306b995fb26e8b6773cf)) 114 | * more acurate handling of .well-known to avoid making the . in terraform.json regexp any ([858ba28](https://github.com/nrkno/terraform-registry/commit/858ba282edae7dcb8047665946c4a8a0dddc2b69)) 115 | * only apply auth to /v1 API endpoints ([b3e0521](https://github.com/nrkno/terraform-registry/commit/b3e0521c33dc0ea7844670b24e1f26f0718554fa)) 116 | * return JSON from /health endpoint ([66d160f](https://github.com/nrkno/terraform-registry/commit/66d160f4dd6d9e6b2c3b3d96ec9554cfa8f089e5)) 117 | * safeguard index handler, simplify test logic ([c3a4e86](https://github.com/nrkno/terraform-registry/commit/c3a4e865ba55523ac0152279a00882803ba76f12)) 118 | * show help text when started without args ([#23](https://github.com/nrkno/terraform-registry/issues/23)) ([e41da8a](https://github.com/nrkno/terraform-registry/commit/e41da8a940e0f926fbc75900809b6a40382c1485)) 119 | * **store/github:** require only at least one filter ([#24](https://github.com/nrkno/terraform-registry/issues/24)) ([771f335](https://github.com/nrkno/terraform-registry/commit/771f335a20041043d26ba937b623dcfb7a7dbfbf)) 120 | * support parsing and setting env from JSON env files ([686eddc](https://github.com/nrkno/terraform-registry/commit/686eddc426a73c100ed826baa32992224e3a992f)) 121 | * use a leveled structured log library ([59b6170](https://github.com/nrkno/terraform-registry/commit/59b61709cfba20b43f21c42a056b92b2234f1cc2)) 122 | 123 | 124 | ### Bug Fixes 125 | 126 | * avoid panic when no -env-json-files are specified ([3b972e2](https://github.com/nrkno/terraform-registry/commit/3b972e217578188f4684d1ff27b7a3a3b53f478f)) 127 | * handle odd cases for /v1 parsing ([7966819](https://github.com/nrkno/terraform-registry/commit/7966819c04ee157a33fa1fd775ea34e5eceb9c03)) 128 | * return known http error strings on NotFound and MethodNotAllowed ([b5509a9](https://github.com/nrkno/terraform-registry/commit/b5509a9dd1b59717020835ed258871632967b583)) 129 | * set main binary as entrypoint in Dockerfile ([70b5e60](https://github.com/nrkno/terraform-registry/commit/70b5e6011d91c827df5714bbc8b1ad5fc0d57c29)) 130 | * **store/github:** error instead of panic on initial cache load error ([f1887eb](https://github.com/nrkno/terraform-registry/commit/f1887eb0312baeb9dd825d8c8efc28f2004fe14d)) 131 | * use Makefile in Dockerfile ([f8feb73](https://github.com/nrkno/terraform-registry/commit/f8feb732da67f30a38fe48a336e0b31e92624789)), closes [#8](https://github.com/nrkno/terraform-registry/issues/8) 132 | * warn instead of panic for empty auth tokens file ([5eab4c2](https://github.com/nrkno/terraform-registry/commit/5eab4c2edcbfbc5c58a2f3b26208dfff55418268)) 133 | 134 | 135 | ### Miscellaneous Chores 136 | 137 | * rename github cmd args to better reflect usage ([1d7bdd3](https://github.com/nrkno/terraform-registry/commit/1d7bdd3563e4a800d944f0dddf8c76822b745041)) 138 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2022 - 2025 NRK 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | FROM golang:1.24-bookworm as build 6 | 7 | RUN wget -O /usr/local/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.5/dumb-init_1.2.5_x86_64 \ 8 | && chmod +x /usr/local/bin/dumb-init 9 | 10 | WORKDIR /go/src/app 11 | 12 | COPY go.mod go.sum ./ 13 | RUN go mod download -x 14 | 15 | COPY . /go/src/app 16 | RUN make GO_FLAGS="-buildvcs=false" test build 17 | 18 | FROM gcr.io/distroless/base-debian12 19 | COPY --from=build /go/src/app/terraform-registry /bin/ 20 | COPY --from=build /usr/local/bin/dumb-init /bin/ 21 | USER nonroot 22 | ENTRYPOINT ["/bin/dumb-init", "--", "/bin/terraform-registry"] 23 | CMD ["-help"] 24 | -------------------------------------------------------------------------------- /HACKING.md: -------------------------------------------------------------------------------- 1 | 6 | 7 | # Terraform Registry 8 | ## Development 9 | ### Commit hygiene 10 | 11 | This repository enforces conventional commit messages. Please install and make 12 | use of the pre-commit git hooks using [`pre-commit`](https://pre-commit.com/). 13 | 14 | ``` 15 | $ pip install pre-commit 16 | $ pre-commit install --install-hooks -t commit-msg 17 | ``` 18 | 19 | Moreover, commits that does more than what is described, does not do what is 20 | described, or that contains multiple unrelated changes will be rejected, and 21 | will require the committer to edit the commits and/or their messages. 22 | 23 | ### Testing the registry locally 24 | 25 | Terraform does not allow disabling TLS certificate verification when accessing 26 | a registry. Unless you have a valid certificate (signed by a valid CA) for your 27 | hostname, you will have to patch and build Terraform from source to disable the 28 | verification. 29 | 30 | #### Build Terraform 31 | 32 | Clone the official repository for the version you are using locally 33 | 34 | ``` 35 | $ terraform version 36 | Terraform v1.1.9 37 | on linux_amd6 38 | $ git clone https://github.com/hashicorp/terraform --ref=v1.1.9 --depth=1 39 | ``` 40 | 41 | Apply the patch to the cloned repository 42 | 43 | 44 | ```diff 45 | diff --git a/internal/httpclient/client.go b/internal/httpclient/client.go 46 | index bb06beb..5f9e424 100644 47 | --- a/internal/httpclient/client.go 48 | +++ b/internal/httpclient/client.go 49 | @@ -1,6 +1,7 @@ 50 | package httpclient 51 | 52 | import ( 53 | + "crypto/tls" 54 | "net/http" 55 | 56 | cleanhttp "github.com/hashicorp/go-cleanhttp" 57 | @@ -10,9 +11,14 @@ import ( 58 | // package that will also send a Terraform User-Agent string. 59 | func New() *http.Client { 60 | cli := cleanhttp.DefaultPooledClient() 61 | + transport := cli.Transport.(*http.Transport) 62 | + transport.TLSClientConfig = &tls.Config{ 63 | + InsecureSkipVerify: true, 64 | + } 65 | + 66 | cli.Transport = &userAgentRoundTripper{ 67 | userAgent: UserAgentString(), 68 | - inner: cli.Transport, 69 | + inner: transport, 70 | } 71 | return cli 72 | } 73 | ``` 74 | 75 | Then build Terraform 76 | 77 | ``` 78 | $ go build . 79 | ``` 80 | 81 | #### Create a self-signed TLS certificate 82 | 83 | A script and an OpenSSL config file to create a self-signed certificate is 84 | included in [./_tools/openssl/](./_tools/openssl/). 85 | 86 | Update *openssl.conf* with your desired `[alternate_names]` and run 87 | *create_self_signed_cert.sh*. 88 | 89 | (source: ) 90 | 91 | #### Build and Run the Private Registry 92 | 93 | ``` 94 | $ make build 95 | $ export GITHUB_TOKEN=mytoken 96 | $ ./terraform-registry -listen-addr=:8080 -auth-disabled=true \ 97 | -tls-enabled=true -tls-cert-file=cert.crt -tls-key-file=cert.key \ 98 | -store=github -github-owner-filter=myuserororg -github-topic-filter=terraform-module 99 | ``` 100 | 101 | Now use `localhost.localdomain:8080` as the registry URL for your module sources 102 | in Terraform 103 | 104 | ```terraform 105 | module "foo" { 106 | source = "localhost.localdomain:8080/myuserororg/my-module-repo/generic//my-module" 107 | version = "~> 1.2.3" 108 | } 109 | ``` 110 | 111 | #### Testing 112 | 113 | ``` 114 | $ make test 115 | ``` 116 | 117 | ### Adding license information 118 | 119 | This adds or updates licensing information of all relevant files in the 120 | respository using [reuse](https://git.fsfe.org/reuse/tool#install). 121 | It is available in some package managers and in The Python Package Index 122 | as `reuse` (`pip install reuse>=4.0.3`). 123 | 124 | ``` 125 | $ make reuse 126 | ``` 127 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | License information should be included in most files in the repository. 2 | See /LICENSES folder for complete license text. 3 | -------------------------------------------------------------------------------- /LICENSES/MIT.txt: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 6 | 7 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 8 | 9 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 10 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2022 - 2025 NRK 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | BINARY_NAME := ./terraform-registry 6 | CMD_SOURCE := ./cmd/terraform-registry 7 | DOCKER_TAG := terraform-registry 8 | 9 | .PHONY: all 10 | all : reuse build test 11 | 12 | .PHONY: build 13 | build : 14 | go build -ldflags "$(LD_FLAGS)" $(GO_FLAGS) -o $(BINARY_NAME) $(CMD_SOURCE) 15 | 16 | .PHONY: build-docker 17 | build-docker : 18 | docker build . -t $(DOCKER_TAG) 19 | 20 | .PHONY: test 21 | test : 22 | go test ./... -timeout 10s 23 | 24 | .PHONY: run 25 | run : 26 | go run $(CMD_SOURCE) 27 | 28 | .PHONY: reuse 29 | reuse : 30 | find . -type f \ 31 | | grep -vP '^(./.git|./.reuse|./LICENSES/|./terraform-registry)' \ 32 | | grep -vP '(/\.git/|/\.terraform/)' \ 33 | | grep -vP '(\\.tf)$$' \ 34 | | xargs reuse annotate --merge-copyrights --license MIT --copyright NRK --year `date +%Y` --skip-unrecognised 35 | find . -type f -name '*.tf' \ 36 | | grep -vP '(/\.git/|/\.terraform/)' \ 37 | | xargs reuse annotate --merge-copyrights --license MIT --copyright NRK --year `date +%Y` --style python 38 | 39 | .PHONY: clean 40 | clean : 41 | go clean 42 | rm $(BINARY_NAME) 43 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 6 | 7 | # Terraform Registry 8 | 9 | This is an implementation of the Terraform registry protocol used to host a 10 | private Terraform registry. Supports modular stores (backends) for discovering 11 | and exposing modules and providers. 12 | 13 | **NOTE:** the APIs of this program are not currently considered stable and 14 | might introduce breaking changes in minor versions before v1 major is reached. 15 | 16 | Please question and report any issues you encounter with this implementation. 17 | There is surely room for improvement. Raise an issue discussing your proposed 18 | changes before submitting a PR. There is however no guarantee we will merge 19 | incoming pull requests. 20 | 21 | Third-party provider registries (like this program) are supported only in 22 | Terraform CLI v0.13 and later. 23 | 24 | ## Features 25 | 26 | - [ ] login.v1 ([issue](https://github.com/nrkno/terraform-registry/issues/20)) 27 | - [x] modules.v1 28 | - [x] providers.v1 29 | 30 | ### Stores 31 | 32 | | Store | modules.v1 | providers.v1 | Description | 33 | |:---|:---:|:---:|:---| 34 | | GitHubStore | ✅ | ✅ | Uses the GitHub API to discover module and/or provider repositories using repository topics. | 35 | | MemoryStore | ✅ | ❌ | A dumb in-memory store used for internal unit testing. | 36 | | S3Store | ✅ | ❌ | Uses the S3 protocol to discover modules stored in a bucket. | 37 | 38 | ### Authentication 39 | 40 | You can configure the registry to require client authentication for the 41 | `/v1/*` and `/download/*` paths. Additionally, the different stores might 42 | implement other authentication schemes and details. 43 | 44 | ## Running 45 | ### Native 46 | 47 | ``` 48 | $ make build 49 | $ ./terraform-registry -h 50 | ``` 51 | 52 | ### Docker 53 | 54 | ``` 55 | $ docker build -t terraform-registry . 56 | $ docker run terraform-registry 57 | ``` 58 | 59 | ## Configuring 60 | 61 | These are the common configuration options. Stores might have specific options 62 | you can read more about in the stores section. 63 | 64 | #### Command line arguments 65 | 66 | - `-access-log-disabled`: Disable HTTP access log (default: `false`) 67 | - `-access-log-ignored-paths`: Ignore certain request paths from being logged (default: `""`) 68 | - `-listen-addr`: HTTP server bind address (default: `:8080`) 69 | - `-auth-disabled`: Disable HTTP bearer token authentication (default: `false`) 70 | - `-auth-tokens-file`: JSON encoded file containing a map of auth token descriptions and tokens. 71 | ```json 72 | { 73 | "description for some token": "some token", 74 | "description for some other token": "some other token" 75 | } 76 | ``` 77 | - `-env-json-files`: Comma-separated list of paths to JSON encoded files 78 | containing a map of environment variable names and values to set. 79 | Converts the keys to uppercase and replaces all occurences of `-` (dash) with 80 | `_` (underscore). 81 | E.g. prefix filepaths with 'myprefix_:' to prefix all keys in the file with 82 | 'MYPREFIX_' before they are set. 83 | - All variable names will be converted to uppercase, and `-` will become `_`. 84 | - If the filenames are prefixed with `myprefix_:`, the resulting environment 85 | variable names from the specific file will be prefixed with `MYPREFIX_` 86 | (e.g. `github_:/secret/github.json`). 87 | - If a variable name is unable to be converted to a valid format, a warning is 88 | logged, but the parsing continues without errors. 89 | - `-tls-enabled`: Whether to enable TLS termination (default: `false`) 90 | - `-tls-cert-file`: Path to TLS certificate file 91 | - `-tls-key-file`: Path to TLS certificate private key file 92 | - `-log-level`: Log level selection: `debug`, `info`, `warn`, `error` (default: `info`) 93 | - `-log-format`: Log output format selection: `json`, `console` (default: `console`) 94 | - `-version`: Print version info and exit 95 | 96 | #### Environment variables 97 | 98 | - `ASSET_DOWNLOAD_AUTH_SECRET`: secret used to sign JWTs protecting the `/download/provider/` routes. 99 | 100 | ### GitHub Store 101 | 102 | This store uses GitHub as a backend. Terraform modules and providers are discovered 103 | by [applying topics to your organisation's GitHub repositories][repository topics]. 104 | The store is configured by setting up search filters for the owner/org, and a topic 105 | you want to use to expose repository releases in the registry. 106 | 107 | The registry requires a GitHub token that has read access to all repositories in the 108 | organisation. 109 | 110 | [repository topics]: https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/classifying-your-repository-with-topics 111 | 112 | #### Modules 113 | 114 | A query for the module address `namespace/name/provider` will return the GitHub repository `namespace/name`. 115 | The `provider` part of the module URL must always be set to `generic` since 116 | this store implementation has no notion of the type of providers the modules 117 | are designed for. 118 | 119 | Upon loading the list of repositories, tags prefixed with `v` will have their prefix removed. 120 | I.e., a repository tag `v1.2.3` will be made available in the registry as version `1.2.3`. 121 | 122 | No verification is performed to check if the repo actually contains a Terraform module. 123 | This is left for Terraform to determine itself. 124 | 125 | The module source download URLs returned are using the [`git::ssh` prefix](https://developer.hashicorp.com/terraform/language/modules/sources#generic-git-repository), 126 | meaning that the client requesting the module must have a local SSH key linked with their 127 | GitHub user, and this user must have read access to the repository in question. In other words, 128 | repository source access is still maintained and handled by GitHub. 129 | 130 | #### Providers 131 | 132 | A query for the provider address `namespace/name` will return the GitHub repository `namespace/name`. 133 | 134 | Some simple verification steps are performed to help ensure that the repo contains a Terraform 135 | provider. A GitHub Release in the repository must follow the same 136 | [steps that HashiCorp requires when publishing a provider](https://developer.hashicorp.com/terraform/registry/providers/publishing) 137 | to their public registry. 138 | 139 | In addition, you must provide the public part of the GPG signing key as part the Github release. 140 | This is done by adding the GPG key in PEM format to your repository, and then 141 | extending the `extra_files` object of the `.goreleaser.yaml` from Hashicorp. 142 | 143 | Releases that do not follow this format will be ignored for the lifetime of the registry process and 144 | will not be attempted verified again. 145 | 146 | Example: 147 | ```yaml 148 | release: 149 | extra_files: 150 | - glob: 'terraform-registry-manifest.json' 151 | name_template: '{{ .ProjectName }}_{{ .Version }}_manifest.json' 152 | - glob: 'gpg-public-key.pem' 153 | name_template: '{{ .ProjectName }}_{{ .Version }}_gpg-public-key.pem' 154 | ``` 155 | 156 | #### Environment variables 157 | 158 | - `GITHUB_TOKEN`: auth token for the GitHub API 159 | 160 | #### Command line arguments 161 | 162 | - `-store github` 163 | - `-github-owner-filter`: Module discovery GitHub org/user repository filter 164 | - `-github-topic-filter`: Module discovery GitHub topic repository filter 165 | - `-github-providers-owner-filter`: Provider discovery GitHub org/user repository filter 166 | - `-github-providers-topic-filter`: Provider discovery GitHub topic repository filter 167 | 168 | ### S3 Store 169 | 170 | This store uses S3 as a backend. A query for the module address 171 | `namespace/name/provider` will be used directly as an S3 bucket key. 172 | Modules must therefore be stored under keys in the following format 173 | `namespace/name/provider/v1.2.3/v1.2.3.zip`. 174 | 175 | The module source download URLs returned are using the [`s3::https` prefix](https://developer.hashicorp.com/terraform/language/modules/sources#s3-bucket), 176 | meaning that the client requesting the module must have local access to the S3 bucket. 177 | 178 | The registry requires the `s3:ListBucket` permission to discover modules, and 179 | the clients will require the `s3:GetObject` permission. 180 | 181 | No verification is performed to check if the path actually contains a Terraform 182 | module. This is left for Terraform to determine. 183 | 184 | #### Command line arguments 185 | 186 | - `-store s3`: Switch store to S3 187 | - `-s3-region`: Region such as us-east-1 188 | - `-s3-bucket`: S3 bucket name 189 | 190 | ## Development 191 | 192 | See [HACKING.md](./HACKING.md). 193 | 194 | ## References 195 | 196 | - 197 | - 198 | - 199 | - 200 | 201 | ## License 202 | 203 | This project and all its files are licensed under MIT, unless stated 204 | otherwise with a different license header. See [./LICENSES](./LICENSES) for 205 | the full license text of all used licenses. 206 | -------------------------------------------------------------------------------- /REUSE.toml: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2024 - 2025 NRK 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | version = 1 6 | SPDX-PackageName = "terraform-registry" 7 | SPDX-PackageSupplier = "Stig Otnes Kolstad " 8 | SPDX-PackageDownloadLocation = "https://github.com/nrkno/terraform-registry" 9 | annotations = [] 10 | -------------------------------------------------------------------------------- /_examples/terraform-project/README.md: -------------------------------------------------------------------------------- 1 | 6 | 7 | An example terraform file that gets a module from a local private registry. 8 | -------------------------------------------------------------------------------- /_examples/terraform-project/main.tf: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2022 - 2025 NRK 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | terraform { 6 | required_version = ">= 1.0" 7 | } 8 | 9 | module "foo" { 10 | source = "localhost.localdomain:8080/terraform-aws-modules/terraform-aws-vpc/generic" 11 | version = "~> 3.13" 12 | } 13 | -------------------------------------------------------------------------------- /_tools/openssl/create_self_signed_cert.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # SPDX-FileCopyrightText: 2022 - 2025 NRK 4 | # 5 | # SPDX-License-Identifier: MIT 6 | 7 | openssl req -x509 -newkey rsa:4096 -sha256 -utf8 -days 365 -nodes -config openssl.conf -keyout cert.key -out cert.crt 8 | -------------------------------------------------------------------------------- /_tools/openssl/openssl.conf: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2022 - 2024 NRK 2 | # 3 | # SPDX-License-Identifier: MIT 4 | [CA_default] 5 | copy_extensions = copy 6 | 7 | [req] 8 | default_bits = 4096 9 | prompt = no 10 | default_md = sha256 11 | distinguished_name = req_distinguished_name 12 | x509_extensions = v3_ca 13 | 14 | [req_distinguished_name] 15 | C = NO 16 | ST = Oslo 17 | L = Oslo 18 | O = Internet Widgits Pty Ltd 19 | OU = Example 20 | emailAddress = someone@example.com 21 | CN = example.com 22 | 23 | [v3_ca] 24 | basicConstraints = CA:FALSE 25 | keyUsage = digitalSignature, keyEncipherment 26 | subjectAltName = @alternate_names 27 | 28 | [alternate_names] 29 | DNS.1 = localhost 30 | DNS.2 = localhost.localdomain 31 | -------------------------------------------------------------------------------- /cmd/terraform-registry/main.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2022 - 2025 NRK 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package main 6 | 7 | import ( 8 | "bytes" 9 | "context" 10 | "crypto/sha1" 11 | "encoding/json" 12 | "flag" 13 | "fmt" 14 | "net/http" 15 | "os" 16 | "regexp" 17 | "runtime" 18 | "runtime/debug" 19 | "strings" 20 | "time" 21 | 22 | "github.com/aws/aws-sdk-go/aws/session" 23 | awss3 "github.com/aws/aws-sdk-go/service/s3" 24 | "github.com/nrkno/terraform-registry/pkg/registry" 25 | "github.com/nrkno/terraform-registry/pkg/store/github" 26 | "github.com/nrkno/terraform-registry/pkg/store/s3" 27 | "go.uber.org/zap" 28 | ) 29 | 30 | var ( 31 | listenAddr string 32 | accessLogDisabled bool 33 | accessLogIgnoredPaths string 34 | authDisabled bool 35 | authTokensFile string 36 | envJSONFiles string 37 | tlsEnabled bool 38 | tlsCertFile string 39 | tlsKeyFile string 40 | storeType string 41 | providerStoreType string 42 | logLevelStr string 43 | logFormatStr string 44 | printVersionInfo bool 45 | 46 | assetDownloadAuthSecret string 47 | 48 | S3Region string 49 | S3Bucket string 50 | 51 | gitHubToken string 52 | gitHubOwnerFilter string 53 | gitHubTopicFilter string 54 | gitHubProvidersOwnerFilter string 55 | gitHubProvidersTopicFilter string 56 | 57 | // > Environment variable names used by the utilities in the Shell and Utilities 58 | // > volume of IEEE Std 1003.1-2001 consist solely of uppercase letters, digits, 59 | // > and the '_' (underscore) from the characters defined in Portable Character 60 | // > Set and do not begin with a digit. Other characters may be permitted by an 61 | // > implementation; applications shall tolerate the presence of such names. 62 | // https://pubs.opengroup.org/onlinepubs/000095399/basedefs/xbd_chap08.html 63 | patternEnvVarName = regexp.MustCompile(`^[A-Z_][A-Z0-9_]*`) 64 | 65 | // These variables are set at build time using ldflags. 66 | version = "(devel)" 67 | buildDate = "unknown" 68 | 69 | logger *zap.Logger = zap.NewNop() 70 | ) 71 | 72 | const ( 73 | developmentVersion = "(devel)" 74 | programName = "terraform-registry" 75 | ) 76 | 77 | func init() { 78 | flag.StringVar(&listenAddr, "listen-addr", ":8080", "") 79 | flag.BoolVar(&accessLogDisabled, "access-log-disabled", false, "") 80 | flag.StringVar(&accessLogIgnoredPaths, "access-log-ignored-paths", "", "Comma-separated list of request paths to ignore logging for") 81 | flag.BoolVar(&authDisabled, "auth-disabled", false, "") 82 | flag.StringVar(&authTokensFile, "auth-tokens-file", "", "JSON encoded file containing a map of auth token descriptions and tokens.") 83 | flag.StringVar(&envJSONFiles, "env-json-files", "", "Comma-separated list of paths to JSON encoded files containing a map of environment variable names and values to set. Converts the keys to uppercase and replaces all occurences of '-' with '_'. E.g. prefix filepaths with 'myprefix_:' to prefix all keys in the file with 'MYPREFIX_' before they are set.") 84 | flag.BoolVar(&tlsEnabled, "tls-enabled", false, "") 85 | flag.StringVar(&tlsCertFile, "tls-cert-file", "", "") 86 | flag.StringVar(&tlsKeyFile, "tls-key-file", "", "") 87 | flag.StringVar(&storeType, "store", "", "Store backend to use (choices: github, s3)") 88 | flag.StringVar(&providerStoreType, "provider-store", "", "Which backend to use for the provider store (choices: github)") 89 | flag.StringVar(&logLevelStr, "log-level", "info", "Levels: debug, info, warn, error") 90 | flag.StringVar(&logFormatStr, "log-format", "console", "Formats: json, console") 91 | flag.BoolVar(&printVersionInfo, "version", false, "Print version info and exit") 92 | 93 | flag.StringVar(&gitHubOwnerFilter, "github-owner-filter", "", "GitHub org/user repository filter") 94 | flag.StringVar(&gitHubTopicFilter, "github-topic-filter", "", "GitHub topic repository filter") 95 | flag.StringVar(&gitHubProvidersOwnerFilter, "github-providers-owner-filter", "", "GitHub providers topic repository filter") 96 | flag.StringVar(&gitHubProvidersTopicFilter, "github-providers-topic-filter", "", "GitHub providers topic repository filter") 97 | 98 | flag.StringVar(&S3Region, "s3-region", "", "S3 region such as us-east-1") 99 | flag.StringVar(&S3Bucket, "s3-bucket", "", "S3 bucket name") 100 | } 101 | 102 | func main() { 103 | flag.Parse() 104 | 105 | if len(os.Args[1:]) == 0 { 106 | flag.PrintDefaults() 107 | os.Exit(1) 108 | } 109 | 110 | if printVersionInfo { 111 | fmt.Printf("%s %s\n", programName, versionString()) 112 | os.Exit(0) 113 | } 114 | 115 | // Configure logging 116 | logConfig := zap.NewProductionConfig() 117 | logLevel, err := zap.ParseAtomicLevel(logLevelStr) 118 | if err != nil { 119 | panic(fmt.Sprintf("zap logger: %v", err)) 120 | } 121 | logConfig.Level = logLevel 122 | logConfig.Encoding = logFormatStr 123 | logger, err = logConfig.Build() 124 | if err != nil { 125 | panic(fmt.Sprintf("zap logger: %v", err)) 126 | } 127 | defer logger.Sync() 128 | 129 | // Load environment from files 130 | for _, item := range strings.Split(envJSONFiles, ",") { 131 | if len(item) == 0 { 132 | continue 133 | } 134 | 135 | prefix := "" 136 | filename := item 137 | // If the filename is prefixed, the prefix must be separated from the filename. 138 | if split := strings.SplitN(filename, ":", 2); len(split) == 2 { 139 | prefix = split[0] 140 | filename = split[1] 141 | } 142 | if err := setEnvironmentFromJSONFile(prefix, filename); err != nil { 143 | logger.Fatal("failed to load environment from file(s)", 144 | zap.Error(err), 145 | ) 146 | } 147 | } 148 | 149 | // Load environment variables here! 150 | gitHubToken = os.Getenv("GITHUB_TOKEN") 151 | assetDownloadAuthSecret = os.Getenv("ASSET_DOWNLOAD_AUTH_SECRET") 152 | 153 | reg := registry.NewRegistry(logger) 154 | reg.AccessLogIgnoredPaths = strings.Split(accessLogIgnoredPaths, ",") 155 | reg.IsAccessLogDisabled = accessLogDisabled 156 | reg.IsAuthDisabled = authDisabled 157 | reg.AssetDownloadAuthSecret = []byte(assetDownloadAuthSecret) 158 | 159 | if providerStoreType == "github" { 160 | reg.IsProviderEnabled = true 161 | logger.Info("enabling github provider store") 162 | } 163 | 164 | logger.Info("HTTP access log configuration", zap.Bool("disabled", reg.IsAccessLogDisabled), zap.Strings("ignoredPaths", reg.AccessLogIgnoredPaths)) 165 | 166 | // Configure authentication 167 | if !reg.IsAuthDisabled { 168 | if authTokensFile == "" { 169 | logger.Fatal("-auth-tokens-file is not set. Provide a valid path or set -auth-disabled.") 170 | } 171 | 172 | // Watch for changes of the auth file 173 | go watchFile(context.TODO(), authTokensFile, 10*time.Second, func(b []byte) { 174 | tokens, err := parseAuthTokens(b) 175 | if err != nil { 176 | logger.Error("failed to load auth tokens", 177 | zap.Error(err), 178 | ) 179 | } 180 | 181 | reg.SetAuthTokens(tokens) 182 | 183 | if len(tokens) == 0 { 184 | logger.Warn("reloaded auth token file", zap.Int("count", len(tokens))) 185 | } else { 186 | logger.Info("reloaded auth token file", zap.Int("count", len(tokens))) 187 | } 188 | }) 189 | logger.Info("authentication enabled") 190 | } else { 191 | logger.Warn("authentication disabled") 192 | } 193 | 194 | logger.Info("initialising stores") 195 | // Configure the chosen store type 196 | switch storeType { 197 | case "github": 198 | gitHubRegistry(reg) 199 | case "s3": 200 | s3Registry(reg) 201 | default: 202 | logger.Fatal("invalid store type", zap.String("selected", storeType)) 203 | } 204 | logger.Info("store initialisation complete") 205 | 206 | srv := http.Server{ 207 | Addr: listenAddr, 208 | Handler: reg, 209 | ReadTimeout: 3 * time.Second, 210 | ReadHeaderTimeout: 3 * time.Second, 211 | WriteTimeout: 3 * time.Second, 212 | IdleTimeout: 60 * time.Second, // keep-alive timeout 213 | } 214 | 215 | logger.Info("starting HTTP server", 216 | zap.Bool("tls", tlsEnabled), 217 | zap.String("listenAddr", listenAddr), 218 | ) 219 | if tlsEnabled { 220 | logger.Panic("ListenAndServe", 221 | zap.Errors("err", []error{srv.ListenAndServeTLS(tlsCertFile, tlsKeyFile)}), 222 | ) 223 | } else { 224 | logger.Panic("ListenAndServe", 225 | zap.Errors("err", []error{srv.ListenAndServe()}), 226 | ) 227 | } 228 | } 229 | 230 | // watchFile reads the contents of the file at `filename`, first immediately, then at every `interval`. 231 | // If and only if the file contents have changed since the last invocation of `callback` it is called again. 232 | // Note that the callback will always be called initially when this function is called. 233 | func watchFile(ctx context.Context, filename string, interval time.Duration, callback func(b []byte)) { 234 | var lastSum []byte 235 | h := sha1.New() 236 | 237 | fn := func() { 238 | b, err := os.ReadFile(filename) 239 | if err != nil { 240 | logger.Error("watchFile: failed to read file", 241 | zap.String("filename", filename), 242 | zap.Error(err), 243 | ) 244 | return 245 | } 246 | if sum := h.Sum(b); bytes.Equal(sum, lastSum) { 247 | return 248 | } else { 249 | logger.Debug("watchFile: file contents updated. triggering callback.", 250 | zap.String("filename", filename), 251 | ) 252 | callback(b) 253 | lastSum = sum 254 | } 255 | } 256 | 257 | // Don't wait for the first tick 258 | fn() 259 | 260 | t := time.NewTicker(interval) 261 | defer t.Stop() 262 | 263 | for { 264 | select { 265 | case <-ctx.Done(): 266 | logger.Debug("watchFile: goroutine stopped", 267 | zap.String("filename", filename), 268 | zap.Errors("err", []error{ctx.Err()}), 269 | ) 270 | return 271 | case <-t.C: 272 | fn() 273 | } 274 | } 275 | } 276 | 277 | // gitHubRegistry configures the registry to use GitHubStore. 278 | func gitHubRegistry(reg *registry.Registry) { 279 | if gitHubToken == "" { 280 | logger.Fatal("missing environment var GITHUB_TOKEN") 281 | } 282 | if gitHubOwnerFilter == "" && gitHubTopicFilter == "" { 283 | logger.Fatal("at least one of -github-owner-filter and -github-topic-filter must be set") 284 | } 285 | 286 | if reg.IsProviderEnabled && gitHubProvidersOwnerFilter == "" && gitHubProvidersTopicFilter == "" { 287 | logger.Fatal("at least one of -github-providers-owner-filter and -github-providers-topic-filter must be set when provider store is enabled") 288 | } 289 | 290 | store := github.NewGitHubStore(gitHubOwnerFilter, gitHubTopicFilter, gitHubProvidersOwnerFilter, gitHubProvidersTopicFilter, gitHubToken, logger.Named("github store")) 291 | reg.SetModuleStore(store) 292 | reg.SetProviderStore(store) 293 | 294 | // Fill module store cache initially 295 | logger.Debug("loading GitHub module store cache") 296 | if err := store.ReloadCache(context.Background()); err != nil { 297 | logger.Error("failed to load GitHub module store cache", 298 | zap.Error(err), 299 | ) 300 | } 301 | 302 | // Fill provider store cache initially 303 | if reg.IsProviderEnabled { 304 | logger.Debug("loading GitHub provider store cache") 305 | err := store.ReloadProviderCache(context.Background()) 306 | if err != nil { 307 | logger.Error("failed to load GitHub provider store cache", 308 | zap.Error(err), 309 | ) 310 | } 311 | } 312 | 313 | // Reload store caches on regular intervals 314 | go func() { 315 | t := time.NewTicker(5 * time.Minute) 316 | defer t.Stop() 317 | <-t.C // ignore the first tick 318 | 319 | for { 320 | logger.Debug("reloading GitHub module store cache") 321 | if err := store.ReloadCache(context.Background()); err != nil { 322 | logger.Error("failed to reload GitHub module store cache", 323 | zap.Error(err), 324 | ) 325 | } 326 | if reg.IsProviderEnabled { 327 | logger.Debug("reloading GitHub provider store cache") 328 | err := store.ReloadProviderCache(context.Background()) 329 | if err != nil { 330 | logger.Error("failed to reload GitHub provider store cache", 331 | zap.Error(err), 332 | ) 333 | } 334 | } 335 | <-t.C 336 | } 337 | }() 338 | } 339 | 340 | // s3Registry configures the registry to use S3Store. 341 | func s3Registry(reg *registry.Registry) { 342 | if S3Region == "" { 343 | logger.Fatal("Missing flag '-s3-region'") 344 | } 345 | if S3Bucket == "" { 346 | logger.Fatal("Missing flag '-s3-bucket'") 347 | } 348 | 349 | sess, err := session.NewSession() 350 | if err != nil { 351 | logger.Fatal("AWS session creation failed") 352 | } 353 | logger.Debug("AWS session created successfully") 354 | 355 | _, err = sess.Config.Credentials.Get() 356 | if err != nil { 357 | logger.Fatal("AWS session credentials not found") 358 | } 359 | s3Sess := awss3.New(sess) 360 | 361 | store := s3.NewS3Store(s3Sess, S3Region, S3Bucket, logger.Named("s3 store")) 362 | if err != nil { 363 | logger.Fatal("failed to create S3 store", 364 | zap.Error(err), 365 | ) 366 | } 367 | reg.SetModuleStore(store) 368 | } 369 | 370 | // parseAuthTokens returns a map of all elements in the JSON object contained in `b`. 371 | func parseAuthTokens(b []byte) (map[string]string, error) { 372 | tokens := make(map[string]string) 373 | if err := json.Unmarshal(b, &tokens); err != nil { 374 | return nil, err 375 | } 376 | return tokens, nil 377 | } 378 | 379 | // setEnvironmentFromJSONFile loads a JSON object from `filename` and updates the 380 | // runtime environment with keys and values from this object using `os.Setenv`. 381 | // Keys will be uppercased and `-` (dashes) will be replaced with `_` (underscores). 382 | func setEnvironmentFromJSONFile(prefix, filename string) error { 383 | vars := make(map[string]string) 384 | b, err := os.ReadFile(filename) 385 | if err != nil { 386 | return err 387 | } 388 | json.Unmarshal(b, &vars) 389 | if err != nil { 390 | return fmt.Errorf("while parsing file '%s': %w", filename, err) 391 | } 392 | for k, v := range vars { 393 | k = prefix + k 394 | k = strings.ToUpper(k) 395 | k = strings.ReplaceAll(k, "-", "_") 396 | if !patternEnvVarName.MatchString(k) { 397 | logger.Warn("unexpected environment variable name format", 398 | zap.String("expected pattern", patternEnvVarName.String()), 399 | zap.String("name", k), 400 | ) 401 | continue 402 | } 403 | os.Setenv(k, v) 404 | logger.Info("environment variable set from file", 405 | zap.String("name", k), 406 | zap.String("file", filename), 407 | ) 408 | } 409 | return nil 410 | } 411 | 412 | // versionString returns a string with version information for this program, 413 | // like `v5.0.4-0.20230601165947-6ce0bf390ce3 linux amd64` for release builds, 414 | // or `(devel).unknown-478ce46fb3ab76445001e614fec7ff1dd0c6cfe0 linux amd64` for local builds. 415 | func versionString() string { 416 | v := struct { 417 | Version string 418 | BuildDate string 419 | GitCommit string 420 | GoArch string 421 | GoOS string 422 | GoVersion string 423 | }{ 424 | Version: version, 425 | BuildDate: buildDate, 426 | GitCommit: "unknown", 427 | GoArch: runtime.GOARCH, 428 | GoOS: runtime.GOOS, 429 | GoVersion: runtime.Version(), 430 | } 431 | 432 | info, ok := debug.ReadBuildInfo() 433 | if ok { 434 | for _, setting := range info.Settings { 435 | if setting.Key == "vcs.revision" { 436 | v.GitCommit = setting.Value 437 | } 438 | } 439 | } 440 | 441 | return fmt.Sprintf("%s.%s-%s %s %s", v.Version, v.BuildDate, v.GitCommit, v.GoOS, v.GoArch) 442 | } 443 | -------------------------------------------------------------------------------- /cmd/terraform-registry/main_test.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2022 - 2025 NRK 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package main 6 | 7 | import ( 8 | "context" 9 | "fmt" 10 | "io" 11 | "os" 12 | "testing" 13 | "time" 14 | 15 | "github.com/matryer/is" 16 | ) 17 | 18 | func TestParseAuthTokenFile(t *testing.T) { 19 | is := is.New(t) 20 | 21 | tokens, err := parseAuthTokens([]byte(`{"token1": "foo", "token2": "bar", "token3": "baz"}`)) 22 | is.NoErr(err) 23 | 24 | is.Equal(len(tokens), 3) 25 | is.Equal(tokens["token1"], "foo") 26 | is.Equal(tokens["token2"], "bar") 27 | is.Equal(tokens["token3"], "baz") 28 | } 29 | 30 | func TestSetEnvironmentFromFileJSON(t *testing.T) { 31 | is := is.New(t) 32 | 33 | f, err := os.CreateTemp("", "*.json") 34 | is.NoErr(err) 35 | 36 | fmt.Fprintf(f, "{\"var-number-1\": \"value1\", \"var_number_TWO\": \"value2\"}") 37 | f.Seek(0, io.SeekStart) 38 | 39 | for _, prefix := range []string{"", "PREFIX_"} { 40 | t.Run("using prefix "+prefix, func(t *testing.T) { 41 | is := is.New(t) 42 | 43 | t.Setenv("VAR_NUMBER_1", "") 44 | t.Setenv("VAR_NUMBER_TWO", "") 45 | 46 | err = setEnvironmentFromJSONFile(prefix, f.Name()) 47 | is.NoErr(err) 48 | is.Equal(os.Getenv(prefix+"VAR_NUMBER_1"), "value1") 49 | is.Equal(os.Getenv(prefix+"VAR_NUMBER_TWO"), "value2") 50 | }) 51 | } 52 | } 53 | 54 | func TestWatchFile(t *testing.T) { 55 | is := is.New(t) 56 | 57 | f, err := os.CreateTemp(t.TempDir(), "*.json") 58 | is.NoErr(err) 59 | 60 | results := make(chan []byte, 1) 61 | ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) 62 | defer cancel() 63 | 64 | callback := func(b []byte) { results <- b } 65 | go watchFile(ctx, f.Name(), 50*time.Millisecond, callback) 66 | is.Equal(<-results, []byte{}) // initial callback before we've written anything 67 | 68 | _, err = f.WriteString("foo") 69 | is.NoErr(err) 70 | is.Equal(<-results, []byte("foo")) // after first write 71 | 72 | _, err = f.WriteString("bar") 73 | is.NoErr(err) 74 | is.Equal(<-results, []byte("foobar")) // after second write 75 | 76 | time.Sleep(100 * time.Millisecond) 77 | is.Equal(len(results), 0) // should not be any more events 78 | } 79 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2022 - 2025 NRK 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | module github.com/nrkno/terraform-registry 6 | 7 | go 1.23.0 8 | 9 | toolchain go1.23.3 10 | 11 | require ( 12 | github.com/ProtonMail/go-crypto v1.3.0 13 | github.com/aws/aws-sdk-go v1.55.7 14 | github.com/go-chi/chi/v5 v5.2.1 15 | github.com/golang-jwt/jwt/v5 v5.2.2 16 | github.com/google/go-github/v69 v69.2.0 17 | github.com/hashicorp/go-version v1.7.0 18 | github.com/matryer/is v1.4.1 19 | github.com/migueleliasweb/go-github-mock v1.1.0 20 | github.com/stretchr/testify v1.10.0 21 | go.uber.org/zap v1.27.0 22 | golang.org/x/oauth2 v0.30.0 23 | ) 24 | 25 | require ( 26 | github.com/cloudflare/circl v1.6.0 // indirect 27 | github.com/davecgh/go-spew v1.1.1 // indirect 28 | github.com/google/go-github/v64 v64.0.0 // indirect 29 | github.com/google/go-querystring v1.1.0 // indirect 30 | github.com/gorilla/mux v1.8.0 // indirect 31 | github.com/jmespath/go-jmespath v0.4.0 // indirect 32 | github.com/pmezard/go-difflib v1.0.0 // indirect 33 | github.com/stretchr/objx v0.5.2 // indirect 34 | go.uber.org/multierr v1.10.0 // indirect 35 | golang.org/x/crypto v0.35.0 // indirect 36 | golang.org/x/sys v0.30.0 // indirect 37 | golang.org/x/time v0.3.0 // indirect 38 | gopkg.in/yaml.v3 v3.0.1 // indirect 39 | ) 40 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/ProtonMail/go-crypto v1.3.0 h1:ILq8+Sf5If5DCpHQp4PbZdS1J7HDFRXz/+xKBiRGFrw= 2 | github.com/ProtonMail/go-crypto v1.3.0/go.mod h1:9whxjD8Rbs29b4XWbB8irEcE8KHMqaR2e7GWU1R+/PE= 3 | github.com/aws/aws-sdk-go v1.55.7 h1:UJrkFq7es5CShfBwlWAC8DA077vp8PyVbQd3lqLiztE= 4 | github.com/aws/aws-sdk-go v1.55.7/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= 5 | github.com/cloudflare/circl v1.6.0 h1:cr5JKic4HI+LkINy2lg3W2jF8sHCVTBncJr5gIIq7qk= 6 | github.com/cloudflare/circl v1.6.0/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs= 7 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 8 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 9 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 10 | github.com/go-chi/chi/v5 v5.2.1 h1:KOIHODQj58PmL80G2Eak4WdvUzjSJSm0vG72crDCqb8= 11 | github.com/go-chi/chi/v5 v5.2.1/go.mod h1:L2yAIGWB3H+phAw1NxKwWM+7eUH/lU8pOMm5hHcoops= 12 | github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8= 13 | github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= 14 | github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 15 | github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= 16 | github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 17 | github.com/google/go-github/v64 v64.0.0 h1:4G61sozmY3eiPAjjoOHponXDBONm+utovTKbyUb2Qdg= 18 | github.com/google/go-github/v64 v64.0.0/go.mod h1:xB3vqMQNdHzilXBiO2I+M7iEFtHf+DP/omBOv6tQzVo= 19 | github.com/google/go-github/v69 v69.2.0 h1:wR+Wi/fN2zdUx9YxSmYE0ktiX9IAR/BeePzeaUUbEHE= 20 | github.com/google/go-github/v69 v69.2.0/go.mod h1:xne4jymxLR6Uj9b7J7PyTpkMYstEMMwGZa0Aehh1azM= 21 | github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= 22 | github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= 23 | github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= 24 | github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= 25 | github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY= 26 | github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= 27 | github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= 28 | github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= 29 | github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= 30 | github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= 31 | github.com/matryer/is v1.4.1 h1:55ehd8zaGABKLXQUe2awZ99BD/PTc2ls+KV/dXphgEQ= 32 | github.com/matryer/is v1.4.1/go.mod h1:8I/i5uYgLzgsgEloJE1U6xx5HkBQpAZvepWuujKwMRU= 33 | github.com/migueleliasweb/go-github-mock v1.1.0 h1:GKaOBPsrPGkAKgtfuWY8MclS1xR6MInkx1SexJucMwE= 34 | github.com/migueleliasweb/go-github-mock v1.1.0/go.mod h1:pYe/XlGs4BGMfRY4vmeixVsODHnVDDhJ9zoi0qzSMHc= 35 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 36 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 37 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 38 | github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= 39 | github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= 40 | github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= 41 | github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 42 | go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= 43 | go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= 44 | go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= 45 | go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= 46 | go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= 47 | go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= 48 | golang.org/x/crypto v0.35.0 h1:b15kiHdrGCHrP6LvwaQ3c03kgNhhiMgvlhxHQhmg2Xs= 49 | golang.org/x/crypto v0.35.0/go.mod h1:dy7dXNW32cAb/6/PRuTNsix8T+vJAqvuIy5Bli/x0YQ= 50 | golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= 51 | golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= 52 | golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= 53 | golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 54 | golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= 55 | golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 56 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 57 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 58 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 59 | gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= 60 | gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 61 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 62 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 63 | -------------------------------------------------------------------------------- /go.sum.license: -------------------------------------------------------------------------------- 1 | SPDX-FileCopyrightText: 2022 - 2025 NRK 2 | 3 | SPDX-License-Identifier: MIT 4 | -------------------------------------------------------------------------------- /pkg/core/core.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2022 - 2025 NRK 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package core 6 | 7 | import ( 8 | "context" 9 | "io" 10 | ) 11 | 12 | type ModuleVersion struct { 13 | // Version is a SemVer version string that specifies the version for a module. 14 | Version string 15 | // SourceURL specifies the download URL where Terraform can get the module source. 16 | // https://www.terraform.io/language/modules/sources 17 | SourceURL string 18 | } 19 | 20 | type ProviderVersions struct { 21 | Versions []ProviderVersion `json:"versions"` 22 | } 23 | 24 | type ProviderVersion struct { 25 | Version string `json:"version"` 26 | Protocols []string `json:"protocols"` 27 | Platforms []Platform `json:"platforms"` 28 | } 29 | 30 | type Platform struct { 31 | OS string `json:"os,omitempty"` 32 | Arch string `json:"arch,omitempty"` 33 | } 34 | 35 | type SigningKeys struct { 36 | GPGPublicKeys []GpgPublicKeys `json:"gpg_public_keys"` 37 | } 38 | 39 | type GpgPublicKeys struct { 40 | KeyID string `json:"key_id"` 41 | ASCIIArmor string `json:"ascii_armor"` 42 | TrustSignature string `json:"trust_signature"` 43 | Source string `json:"source"` 44 | SourceURL string `json:"source_url"` 45 | } 46 | 47 | type Provider struct { 48 | Protocols []string `json:"protocols"` 49 | OS string `json:"os"` 50 | Arch string `json:"arch"` 51 | Filename string `json:"filename"` 52 | DownloadURL string `json:"download_url"` 53 | SHASumsURL string `json:"shasums_url"` 54 | SHASumsSignatureURL string `json:"shasums_signature_url"` 55 | SHASum string `json:"shasum"` 56 | SigningKeys SigningKeys `json:"signing_keys"` 57 | } 58 | 59 | func (p *Provider) Copy() *Provider { 60 | return &Provider{ 61 | Protocols: p.Protocols, 62 | OS: p.OS, 63 | Arch: p.Arch, 64 | Filename: p.Filename, 65 | DownloadURL: p.DownloadURL, 66 | SHASumsURL: p.SHASumsURL, 67 | SHASumsSignatureURL: p.SHASumsSignatureURL, 68 | SHASum: p.SHASum, 69 | SigningKeys: p.SigningKeys, 70 | } 71 | } 72 | 73 | type ProviderManifest struct { 74 | Version int `json:"version"` 75 | Metadata struct { 76 | ProtocolVersions []string `json:"protocol_versions"` 77 | } `json:"metadata"` 78 | } 79 | 80 | // ModuleStore is the store implementation interface for building custom module stores. 81 | type ModuleStore interface { 82 | ListModuleVersions(ctx context.Context, namespace, name, provider string) ([]*ModuleVersion, error) 83 | GetModuleVersion(ctx context.Context, namespace, name, provider, version string) (*ModuleVersion, error) 84 | } 85 | 86 | // ProviderStore is the store implementation interface for building custom provider stores 87 | type ProviderStore interface { 88 | ListProviderVersions(ctx context.Context, namespace string, name string) (*ProviderVersions, error) 89 | GetProviderVersion(ctx context.Context, namespace string, name string, version string, os string, arch string) (*Provider, error) 90 | GetProviderAsset(ctx context.Context, namespace string, name string, tag string, asset string) (io.ReadCloser, error) 91 | } 92 | -------------------------------------------------------------------------------- /pkg/registry/registry.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2022 - 2025 NRK 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package registry 6 | 7 | import ( 8 | "encoding/json" 9 | "errors" 10 | "fmt" 11 | "io" 12 | "net/http" 13 | "slices" 14 | "strings" 15 | "sync" 16 | "time" 17 | 18 | "github.com/go-chi/chi/v5" 19 | "github.com/go-chi/chi/v5/middleware" 20 | "github.com/golang-jwt/jwt/v5" 21 | "github.com/nrkno/terraform-registry/pkg/core" 22 | "go.uber.org/zap" 23 | ) 24 | 25 | var ( 26 | // WelcomeMessage is the message returned from the index route. 27 | WelcomeMessage = []byte("Terraform Registry\nhttps://github.com/nrkno/terraform-registry\n") 28 | ) 29 | 30 | // Registry implements the Terraform HTTP registry protocol. 31 | // Should not be instantiated directly. Use `NewRegistry` instead. 32 | type Registry struct { 33 | // Whether to disable auth 34 | IsAuthDisabled bool 35 | // Whether to disable HTTP access log 36 | IsAccessLogDisabled bool 37 | // Paths to ignore request logging for 38 | AccessLogIgnoredPaths []string 39 | 40 | // Whether to enable provider registry support 41 | IsProviderEnabled bool 42 | 43 | // Secret used to issue JTW for protecting the /download/provider/ route 44 | AssetDownloadAuthSecret []byte 45 | 46 | router *chi.Mux 47 | authTokens map[string]string 48 | moduleStore core.ModuleStore 49 | providerStore core.ProviderStore 50 | tokenMut sync.RWMutex 51 | 52 | logger *zap.Logger 53 | } 54 | 55 | func NewRegistry(logger *zap.Logger) *Registry { 56 | if logger == nil { 57 | logger = zap.NewNop() 58 | } 59 | 60 | reg := &Registry{ 61 | IsAuthDisabled: false, 62 | IsProviderEnabled: false, 63 | logger: logger, 64 | } 65 | reg.setupRoutes() 66 | return reg 67 | } 68 | 69 | // SetModuleStore sets the active module store for this instance. 70 | func (reg *Registry) SetModuleStore(s core.ModuleStore) { 71 | reg.moduleStore = s 72 | } 73 | 74 | // SetProviderStore sets the active provider store for this instance. 75 | func (reg *Registry) SetProviderStore(s core.ProviderStore) { 76 | reg.providerStore = s 77 | } 78 | 79 | // GetAuthTokens gets the valid auth tokens configured for this instance. 80 | func (reg *Registry) GetAuthTokens() map[string]string { 81 | reg.tokenMut.RLock() 82 | defer reg.tokenMut.RUnlock() 83 | 84 | // Make sure map can't be modified indirectly 85 | m := make(map[string]string, len(reg.authTokens)) 86 | for k, v := range reg.authTokens { 87 | m[k] = v 88 | } 89 | return m 90 | } 91 | 92 | // SetAuthTokens sets the valid auth tokens configured for this instance. 93 | func (reg *Registry) SetAuthTokens(authTokens map[string]string) { 94 | // Make sure map can't be modified indirectly 95 | m := make(map[string]string, len(authTokens)) 96 | for k, v := range authTokens { 97 | m[k] = v 98 | } 99 | 100 | reg.tokenMut.Lock() 101 | reg.authTokens = m 102 | reg.tokenMut.Unlock() 103 | } 104 | 105 | // setupRoutes initialises and configures the HTTP router. Must be called before starting the server (`ServeHTTP`). 106 | func (reg *Registry) setupRoutes() { 107 | reg.router = chi.NewRouter() 108 | reg.router.Use(reg.RequestLogger()) 109 | reg.router.NotFound(reg.NotFound()) 110 | reg.router.MethodNotAllowed(reg.MethodNotAllowed()) 111 | reg.router.Get("/", reg.Index()) 112 | reg.router.Get("/health", reg.Health()) 113 | reg.router.Get("/.well-known/{name}", reg.ServiceDiscovery()) 114 | 115 | // Only API routes are protected with authentication 116 | reg.router.Route("/v1", func(r chi.Router) { 117 | r.Use(reg.TokenAuth) 118 | r.Get("/modules/{namespace}/{name}/{provider}/versions", reg.ModuleVersions()) 119 | r.Get("/modules/{namespace}/{name}/{provider}/{version}/download", reg.ModuleDownload()) 120 | r.Get("/providers/{namespace}/{name}/versions", reg.ProviderVersions()) 121 | r.Get("/providers/{namespace}/{name}/{version}/download/{os}/{arch}", reg.ProviderDownload()) 122 | }) 123 | 124 | reg.router.Route("/download/provider", func(r chi.Router) { 125 | r.Use(reg.ProviderDownloadAuth) 126 | r.Get("/{namespace}/{name}/{version}/asset/{assetName}", reg.ProviderAssetDownload()) 127 | }) 128 | } 129 | 130 | // SPDX-SnippetBegin 131 | // SPDX-License-Identifier: MIT 132 | // SPDX-SnippetCopyrightText: Copyright (c) 2021 Manfred Touron (manfred.life) 133 | // SDPX—SnippetName: Function to configure Zap logger with Chi HTTP router 134 | // SPDX-SnippetComment: Original work at https://github.com/moul/chizap/blob/0ebf11a6a5535e3c6bb26f1236b2833ae7825675/chizap.go. All further changes are licensed under this file's main license. 135 | 136 | // Request logger for Chi using Zap as the logger. 137 | func (reg *Registry) RequestLogger() func(next http.Handler) http.Handler { 138 | return func(next http.Handler) http.Handler { 139 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 140 | if reg.IsAccessLogDisabled { 141 | next.ServeHTTP(w, r) 142 | return 143 | } 144 | 145 | if slices.Contains(reg.AccessLogIgnoredPaths, r.URL.Path) { 146 | next.ServeHTTP(w, r) 147 | return 148 | } 149 | 150 | wr := middleware.NewWrapResponseWriter(w, r.ProtoMajor) 151 | t1 := time.Now() 152 | defer func() { 153 | ua := wr.Header().Get("User-Agent") 154 | if ua == "" { 155 | ua = r.Header.Get("User-Agent") 156 | } 157 | 158 | reqLogger := reg.logger.With( 159 | zap.String("proto", r.Proto), 160 | zap.String("method", r.Method), 161 | zap.String("path", r.URL.Path), 162 | zap.Int("size", wr.BytesWritten()), 163 | zap.Int("status", wr.Status()), 164 | zap.String("reqId", middleware.GetReqID(r.Context())), 165 | zap.Duration("responseTimeNSec", time.Since(t1)), 166 | zap.String("userAgent", ua), 167 | ) 168 | 169 | reqLogger.Info("HTTP request") 170 | }() 171 | next.ServeHTTP(wr, r) 172 | }) 173 | } 174 | } 175 | 176 | // SPDX-SnippetEnd 177 | 178 | func (reg *Registry) ServeHTTP(w http.ResponseWriter, r *http.Request) { 179 | reg.router.ServeHTTP(w, r) 180 | } 181 | 182 | // TokenAuth is a middleware function for token header authentication. 183 | func (reg *Registry) TokenAuth(next http.Handler) http.Handler { 184 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 185 | if reg.IsAuthDisabled { 186 | next.ServeHTTP(w, r) 187 | return 188 | } 189 | 190 | header := r.Header.Get("Authorization") 191 | if header == "" { 192 | http.Error(w, http.StatusText(http.StatusForbidden), http.StatusForbidden) 193 | reg.logger.Debug("TokenAuth: Authorization header missing or empty") 194 | return 195 | } 196 | 197 | auth := strings.SplitN(header, " ", 2) 198 | if len(auth) != 2 { 199 | http.Error(w, http.StatusText(http.StatusForbidden), http.StatusForbidden) 200 | reg.logger.Debug("TokenAuth: Authorization header present, but invalid") 201 | return 202 | } 203 | 204 | tokenType := auth[0] 205 | token := auth[1] 206 | 207 | if tokenType != "Bearer" { 208 | http.Error(w, http.StatusText(http.StatusForbidden), http.StatusForbidden) 209 | reg.logger.Debug("TokenAuth: unexpected authorization header value prefix", 210 | zap.String("actual", tokenType), 211 | zap.String("expected", "Bearer"), 212 | ) 213 | return 214 | } 215 | 216 | for _, t := range reg.GetAuthTokens() { 217 | if t == token { 218 | next.ServeHTTP(w, r) 219 | return 220 | } 221 | } 222 | 223 | http.Error(w, http.StatusText(http.StatusForbidden), http.StatusForbidden) 224 | }) 225 | } 226 | 227 | func (reg *Registry) NotFound() http.HandlerFunc { 228 | return func(w http.ResponseWriter, r *http.Request) { 229 | http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) 230 | } 231 | } 232 | 233 | func (reg *Registry) MethodNotAllowed() http.HandlerFunc { 234 | return func(w http.ResponseWriter, r *http.Request) { 235 | http.Error(w, http.StatusText(http.StatusMethodNotAllowed), http.StatusMethodNotAllowed) 236 | } 237 | } 238 | 239 | func (reg *Registry) Index() http.HandlerFunc { 240 | return func(w http.ResponseWriter, r *http.Request) { 241 | if r.RequestURI != "/" { 242 | http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) 243 | return 244 | } 245 | if _, err := w.Write(WelcomeMessage); err != nil { 246 | reg.logger.Error("Index", zap.Error(err)) 247 | } 248 | } 249 | } 250 | 251 | type HealthResponse struct { 252 | Status string `json:"status"` 253 | } 254 | 255 | // Health is the endpoint to be checked to know the runtime health of the registry. 256 | // In its current implementation it will always report as healthy, i.e. it only 257 | // reports that the HTTP server still handles requests. 258 | func (reg *Registry) Health() http.HandlerFunc { 259 | return func(w http.ResponseWriter, r *http.Request) { 260 | resp := HealthResponse{ 261 | Status: "OK", 262 | } 263 | 264 | w.Header().Set("Content-Type", "application/json") 265 | enc := json.NewEncoder(w) 266 | if err := enc.Encode(resp); err != nil { 267 | reg.logger.Error("Health", zap.Error(err)) 268 | } 269 | } 270 | } 271 | 272 | type ServiceDiscoveryResponse struct { 273 | ModulesV1 string `json:"modules.v1"` 274 | ProvidersV1 string `json:"providers.v1"` 275 | } 276 | 277 | // ServiceDiscovery returns a handler that returns a JSON payload for Terraform service discovery. 278 | // https://www.terraform.io/internals/module-registry-protocol 279 | func (reg *Registry) ServiceDiscovery() http.HandlerFunc { 280 | spec := ServiceDiscoveryResponse{ 281 | ModulesV1: "/v1/modules/", 282 | ProvidersV1: "/v1/providers/", 283 | } 284 | 285 | resp, err := json.Marshal(spec) 286 | if err != nil { 287 | reg.logger.Panic("ServiceDiscovery", zap.Error(err)) 288 | } 289 | 290 | return func(w http.ResponseWriter, r *http.Request) { 291 | if chi.URLParam(r, "name") != "terraform.json" { 292 | http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) 293 | return 294 | } 295 | w.Header().Set("Content-Type", "application/json") 296 | if _, err := w.Write(resp); err != nil { 297 | reg.logger.Error("ServiceDiscovery", zap.Error(err)) 298 | } 299 | } 300 | } 301 | 302 | type ModuleVersionsResponse struct { 303 | Modules []ModuleVersionsResponseModule `json:"modules"` 304 | } 305 | 306 | type ModuleVersionsResponseModule struct { 307 | Versions []ModuleVersionsResponseModuleVersion `json:"versions"` 308 | } 309 | 310 | type ModuleVersionsResponseModuleVersion struct { 311 | Version string `json:"version"` 312 | } 313 | 314 | // ModuleVersions returns a handler that returns a list of available versions for a module. 315 | // https://www.terraform.io/internals/module-registry-protocol#list-available-versions-for-a-specific-module 316 | func (reg *Registry) ModuleVersions() http.HandlerFunc { 317 | return func(w http.ResponseWriter, r *http.Request) { 318 | var ( 319 | namespace = chi.URLParam(r, "namespace") 320 | name = chi.URLParam(r, "name") 321 | provider = chi.URLParam(r, "provider") 322 | ) 323 | 324 | versions, err := reg.moduleStore.ListModuleVersions(r.Context(), namespace, name, provider) 325 | if err != nil { 326 | http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) 327 | reg.logger.Debug("ListModuleVersions", zap.Error(err)) 328 | return 329 | } 330 | 331 | respObj := ModuleVersionsResponse{ 332 | Modules: []ModuleVersionsResponseModule{ 333 | ModuleVersionsResponseModule{}, 334 | }, 335 | } 336 | for _, v := range versions { 337 | respObj.Modules[0].Versions = append(respObj.Modules[0].Versions, ModuleVersionsResponseModuleVersion{Version: v.Version}) 338 | } 339 | 340 | b, err := json.Marshal(respObj) 341 | if err != nil { 342 | reg.logger.Error("ModuleVersions", zap.Error(err)) 343 | } 344 | 345 | w.Header().Set("Content-Type", "application/json") 346 | if _, err := w.Write(b); err != nil { 347 | reg.logger.Error("ModuleVersions", zap.Error(err)) 348 | } 349 | } 350 | } 351 | 352 | // ModuleDownload returns a handler that returns a download link for a specific version of a module. 353 | // https://www.terraform.io/internals/module-registry-protocol#download-source-code-for-a-specific-module-version 354 | func (reg *Registry) ModuleDownload() http.HandlerFunc { 355 | return func(w http.ResponseWriter, r *http.Request) { 356 | var ( 357 | namespace = chi.URLParam(r, "namespace") 358 | name = chi.URLParam(r, "name") 359 | provider = chi.URLParam(r, "provider") 360 | version = chi.URLParam(r, "version") 361 | ) 362 | 363 | ver, err := reg.moduleStore.GetModuleVersion(r.Context(), namespace, name, provider, version) 364 | if err != nil { 365 | http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) 366 | reg.logger.Error("GetModuleVersion", zap.Error(err)) 367 | return 368 | } 369 | 370 | w.Header().Set("X-Terraform-Get", ver.SourceURL) 371 | w.WriteHeader(http.StatusNoContent) 372 | } 373 | } 374 | 375 | // ProviderVersions returns a handler that returns a list of available versions for a provider. 376 | // https://developer.hashicorp.com/terraform/internals/provider-registry-protocol#list-available-versions 377 | func (reg *Registry) ProviderVersions() http.HandlerFunc { 378 | return func(w http.ResponseWriter, r *http.Request) { 379 | var ( 380 | namespace = chi.URLParam(r, "namespace") 381 | name = chi.URLParam(r, "name") 382 | ) 383 | 384 | ver, err := reg.providerStore.ListProviderVersions(r.Context(), namespace, name) 385 | if err != nil { 386 | http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) 387 | reg.logger.Error("ListProviderVersions", zap.Error(err)) 388 | return 389 | } 390 | 391 | err = json.NewEncoder(w).Encode(ver) 392 | if err != nil { 393 | http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) 394 | reg.logger.Error("ListProviderVersions", zap.Error(err)) 395 | return 396 | } 397 | 398 | w.WriteHeader(http.StatusOK) 399 | } 400 | } 401 | 402 | // ProviderDownload returns a handler that returns a download link for a specific version of a provider. 403 | // https://developer.hashicorp.com/terraform/internals/provider-registry-protocol#find-a-provider-package 404 | func (reg *Registry) ProviderDownload() http.HandlerFunc { 405 | return func(w http.ResponseWriter, r *http.Request) { 406 | var ( 407 | namespace = chi.URLParam(r, "namespace") 408 | name = chi.URLParam(r, "name") 409 | version = chi.URLParam(r, "version") 410 | os = chi.URLParam(r, "os") 411 | arch = chi.URLParam(r, "arch") 412 | ) 413 | 414 | provider, err := reg.providerStore.GetProviderVersion(r.Context(), namespace, name, version, os, arch) 415 | if err != nil { 416 | http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) 417 | reg.logger.Error("GetProviderVersion", zap.Error(err)) 418 | return 419 | } 420 | 421 | // Terraform does not send registry auth headers when downloading assets, we add a 422 | // token as query parameter to be able to protect the download routes which are 423 | // used when assets are not publicly available. 424 | if strings.HasPrefix(provider.DownloadURL, "/download") && !reg.IsAuthDisabled { 425 | // Create a copy of the provider before we modify URLs 426 | provider = provider.Copy() 427 | 428 | // create a token valid for 10 seconds. Should be more than enough. 429 | token := jwt.NewWithClaims(jwt.SigningMethodHS256, &jwt.RegisteredClaims{ 430 | ExpiresAt: jwt.NewNumericDate(time.Now().Add(time.Second * 10)), 431 | Issuer: "terraform-registry", 432 | }) 433 | 434 | tokenString, err := token.SignedString(reg.AssetDownloadAuthSecret) 435 | if err != nil { 436 | http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) 437 | reg.logger.Error("GetProviderVersion: unable to create token", zap.Error(err)) 438 | return 439 | } 440 | 441 | provider.DownloadURL = fmt.Sprintf("%s?token=%s", provider.DownloadURL, tokenString) 442 | provider.SHASumsURL = fmt.Sprintf("%s?token=%s", provider.SHASumsURL, tokenString) 443 | provider.SHASumsSignatureURL = fmt.Sprintf("%s?token=%s", provider.SHASumsSignatureURL, tokenString) 444 | } 445 | 446 | err = json.NewEncoder(w).Encode(provider) 447 | if err != nil { 448 | http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) 449 | reg.logger.Error("GetProviderVersion", zap.Error(err)) 450 | return 451 | } 452 | w.WriteHeader(http.StatusOK) 453 | } 454 | } 455 | 456 | // ProviderAssetDownload returns a handler that returns a provider asset. 457 | // When a provider binaries is not hosted on a public webserver, this handler can be used to fetch 458 | // assets directly from the store 459 | func (reg *Registry) ProviderAssetDownload() http.HandlerFunc { 460 | return func(w http.ResponseWriter, r *http.Request) { 461 | var ( 462 | owner = chi.URLParam(r, "namespace") 463 | repo = chi.URLParam(r, "name") 464 | tag = chi.URLParam(r, "version") 465 | assetName = chi.URLParam(r, "assetName") 466 | ) 467 | 468 | asset, err := reg.providerStore.GetProviderAsset(r.Context(), owner, repo, tag, assetName) 469 | if err != nil { 470 | http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) 471 | reg.logger.Error("ProviderAssetDownload", zap.Error(err)) 472 | return 473 | } 474 | defer asset.Close() 475 | 476 | written, err := io.Copy(w, asset) 477 | if err != nil { 478 | http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) 479 | reg.logger.Error("ProviderAssetDownload", zap.Error(err)) 480 | return 481 | } 482 | 483 | reg.logger.Debug(fmt.Sprintf("ProviderAssetDownload: wrote %d bytes to response", written)) 484 | w.WriteHeader(http.StatusOK) 485 | } 486 | } 487 | 488 | func (reg *Registry) ProviderDownloadAuth(next http.Handler) http.Handler { 489 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 490 | if reg.IsAuthDisabled { 491 | next.ServeHTTP(w, r) 492 | return 493 | } 494 | 495 | tokenString := r.URL.Query().Get("token") 496 | if tokenString == "" { 497 | http.Error(w, http.StatusText(http.StatusForbidden), http.StatusForbidden) 498 | reg.logger.Debug("ProviderDownloadAuth: Token query parameter missing or empty") 499 | return 500 | } 501 | 502 | token, err := jwt.Parse(tokenString, func(token *jwt.Token) (interface{}, error) { 503 | return reg.AssetDownloadAuthSecret, nil 504 | }) 505 | 506 | switch { 507 | case token.Valid: 508 | next.ServeHTTP(w, r) 509 | return 510 | case errors.Is(err, jwt.ErrTokenExpired) || errors.Is(err, jwt.ErrTokenNotValidYet): 511 | reg.logger.Error("ProviderDownloadAuth: Token is expired or not valid yet") 512 | default: 513 | reg.logger.Error("ProviderDownloadAuth: Token not valid") 514 | } 515 | 516 | http.Error(w, http.StatusText(http.StatusForbidden), http.StatusForbidden) 517 | }) 518 | } 519 | -------------------------------------------------------------------------------- /pkg/registry/registry_test.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2022 - 2025 NRK 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package registry 6 | 7 | import ( 8 | "bytes" 9 | "encoding/json" 10 | "io" 11 | "net/http" 12 | "net/http/httptest" 13 | "net/url" 14 | "regexp" 15 | "sort" 16 | "strings" 17 | "testing" 18 | 19 | "github.com/matryer/is" 20 | "github.com/nrkno/terraform-registry/pkg/core" 21 | memstore "github.com/nrkno/terraform-registry/pkg/store/memory" 22 | "go.uber.org/zap" 23 | ) 24 | 25 | func verifyServiceDiscovery(t *testing.T, resp *http.Response) { 26 | is := is.New(t) 27 | 28 | body, _ := io.ReadAll(resp.Body) 29 | 30 | is.Equal(resp.StatusCode, 200) 31 | is.Equal(resp.Header.Get("Content-Type"), "application/json") 32 | is.True(len(body) > 1) 33 | 34 | var compactJSON bytes.Buffer 35 | err := json.Compact(&compactJSON, body) 36 | is.NoErr(err) 37 | 38 | is.Equal( 39 | compactJSON.String(), 40 | `{"modules.v1":"/v1/modules/","providers.v1":"/v1/providers/"}`, 41 | ) 42 | } 43 | 44 | func TestServiceDiscovery(t *testing.T) { 45 | req := httptest.NewRequest("GET", "/.well-known/terraform.json", nil) 46 | w := httptest.NewRecorder() 47 | 48 | reg := Registry{ 49 | IsAuthDisabled: true, 50 | logger: zap.NewNop(), 51 | } 52 | reg.setupRoutes() 53 | reg.router.ServeHTTP(w, req) 54 | 55 | resp := w.Result() 56 | verifyServiceDiscovery(t, resp) 57 | } 58 | 59 | func FuzzTokenAuth(f *testing.F) { 60 | seeds := []struct { 61 | authToken string 62 | authorizationHeader string 63 | }{ 64 | { 65 | "valid", 66 | "Bearer valid", 67 | }, 68 | { 69 | "valid", 70 | "Bearer invalid", 71 | }, 72 | { 73 | "valid", 74 | "notvalid", 75 | }, 76 | } 77 | for _, seed := range seeds { 78 | f.Add(seed.authToken, seed.authorizationHeader) 79 | } 80 | f.Fuzz(func(t *testing.T, authToken string, authorizationHeader string) { 81 | reg := Registry{ 82 | authTokens: map[string]string{ 83 | "description": authToken, 84 | }, 85 | logger: zap.NewNop(), 86 | } 87 | reg.setupRoutes() 88 | 89 | is := is.New(t) 90 | 91 | req := httptest.NewRequest("GET", "/v1/", nil) 92 | req.Header.Set("Authorization", authorizationHeader) 93 | w := httptest.NewRecorder() 94 | 95 | reg.router.ServeHTTP(w, req) 96 | 97 | resp := w.Result() 98 | defer resp.Body.Close() 99 | if "Bearer "+authToken == authorizationHeader { 100 | is.Equal(resp.StatusCode, http.StatusNotFound) 101 | } else { 102 | is.Equal(resp.StatusCode, http.StatusForbidden) 103 | } 104 | }) 105 | } 106 | 107 | func verifyHealth(t *testing.T, resp *http.Response, expectedStatus int, expectedResponse HealthResponse) { 108 | is := is.New(t) 109 | 110 | body, err := io.ReadAll(resp.Body) 111 | is.NoErr(err) 112 | is.Equal(resp.StatusCode, expectedStatus) 113 | is.Equal(resp.Header.Get("Content-Type"), "application/json") 114 | 115 | var respObj HealthResponse 116 | err = json.Unmarshal(body, &respObj) 117 | is.NoErr(err) 118 | 119 | is.Equal(respObj, expectedResponse) 120 | } 121 | 122 | func TestHealth(t *testing.T) { 123 | mstore := memstore.NewMemoryStore() 124 | reg := Registry{ 125 | IsAuthDisabled: true, 126 | moduleStore: mstore, 127 | logger: zap.NewNop(), 128 | } 129 | reg.setupRoutes() 130 | 131 | testcases := []struct { 132 | name string 133 | statusCode int 134 | health HealthResponse 135 | }{ 136 | { 137 | "healthy", 138 | http.StatusOK, 139 | HealthResponse{ 140 | Status: "OK", 141 | }, 142 | }, 143 | } 144 | 145 | for _, tc := range testcases { 146 | t.Run(tc.name, func(t *testing.T) { 147 | req := httptest.NewRequest("GET", "/health", nil) 148 | w := httptest.NewRecorder() 149 | 150 | reg.router.ServeHTTP(w, req) 151 | 152 | resp := w.Result() 153 | verifyHealth(t, resp, tc.statusCode, tc.health) 154 | }) 155 | } 156 | } 157 | 158 | func verifyModuleVersions(t *testing.T, resp *http.Response, expectedStatus int, expectedVersion []string) { 159 | is := is.New(t) 160 | body, err := io.ReadAll(resp.Body) 161 | is.NoErr(err) 162 | is.Equal(resp.StatusCode, expectedStatus) 163 | 164 | if expectedStatus == http.StatusOK { 165 | is.Equal(resp.Header.Get("Content-Type"), "application/json") 166 | 167 | var respObj ModuleVersionsResponse 168 | err := json.Unmarshal(body, &respObj) 169 | is.NoErr(err) 170 | 171 | var actualVersions []string 172 | for _, v := range respObj.Modules[0].Versions { 173 | actualVersions = append(actualVersions, v.Version) 174 | } 175 | sort.Strings(actualVersions) 176 | 177 | is.Equal(actualVersions, expectedVersion) 178 | } 179 | } 180 | 181 | func TestListModuleVersions(t *testing.T) { 182 | mstore := memstore.NewMemoryStore() 183 | mstore.Set("hashicorp/consul/aws", []*core.ModuleVersion{ 184 | &core.ModuleVersion{ 185 | Version: "1.1.1", 186 | }, 187 | &core.ModuleVersion{ 188 | Version: "2.2.2", 189 | }, 190 | &core.ModuleVersion{ 191 | Version: "3.3.3", 192 | }, 193 | }) 194 | 195 | reg := Registry{ 196 | IsAuthDisabled: true, 197 | moduleStore: mstore, 198 | logger: zap.NewNop(), 199 | } 200 | reg.setupRoutes() 201 | 202 | testcases := []struct { 203 | name string 204 | module string 205 | status int 206 | versionsSeen []string 207 | }{ 208 | { 209 | "valid module", 210 | "hashicorp/consul/aws", 211 | http.StatusOK, 212 | []string{"1.1.1", "2.2.2", "3.3.3"}, 213 | }, 214 | { 215 | "unknown module", 216 | "some/random/name", 217 | http.StatusNotFound, 218 | []string{}, 219 | }, 220 | { 221 | "empty module name", 222 | "", 223 | http.StatusNotFound, 224 | []string{}, 225 | }, 226 | } 227 | 228 | for _, tc := range testcases { 229 | t.Run(tc.name, func(t *testing.T) { 230 | req := httptest.NewRequest("GET", "/v1/modules/"+tc.module+"/versions", nil) 231 | w := httptest.NewRecorder() 232 | 233 | reg.router.ServeHTTP(w, req) 234 | 235 | resp := w.Result() 236 | verifyModuleVersions(t, resp, tc.status, tc.versionsSeen) 237 | }) 238 | } 239 | } 240 | 241 | func verifyDownload(t *testing.T, resp *http.Response, expectedStatus int, expectedURL string) { 242 | is := is.New(t) 243 | is.Equal(resp.StatusCode, expectedStatus) 244 | is.Equal(resp.Header.Get("X-Terraform-Get"), expectedURL) // X-Terraform-Get header 245 | 246 | if expectedStatus == http.StatusOK { 247 | is.Equal(resp.Header.Get("Content-Type"), "application/json") 248 | } 249 | } 250 | 251 | func TestModuleDownload(t *testing.T) { 252 | mstore := memstore.NewMemoryStore() 253 | mstore.Set("hashicorp/consul/aws", []*core.ModuleVersion{ 254 | &core.ModuleVersion{ 255 | Version: "1.1.1", 256 | SourceURL: "git::ssh://git@github.com/hashicorp/consul.git?ref=v1.1.1", 257 | }, 258 | &core.ModuleVersion{ 259 | Version: "2.2.2", 260 | SourceURL: "git::ssh://git@github.com/hashicorp/consul.git?ref=v2.2.2", 261 | }, 262 | &core.ModuleVersion{ 263 | Version: "3.3.3", 264 | SourceURL: "git::ssh://git@github.com/hashicorp/consul.git?ref=v3.3.3", 265 | }, 266 | }) 267 | 268 | reg := Registry{ 269 | IsAuthDisabled: true, 270 | moduleStore: mstore, 271 | logger: zap.NewNop(), 272 | } 273 | reg.setupRoutes() 274 | 275 | testcases := []struct { 276 | name string 277 | moduleString string 278 | status int 279 | downloadURL string 280 | }{ 281 | { 282 | "valid module", 283 | "hashicorp/consul/aws/2.2.2", 284 | http.StatusNoContent, 285 | "git::ssh://git@github.com/hashicorp/consul.git?ref=v2.2.2", 286 | }, 287 | { 288 | "unknown module", 289 | "some/random/name/0.0.0", 290 | http.StatusNotFound, 291 | "", 292 | }, 293 | } 294 | 295 | for _, tc := range testcases { 296 | t.Run(tc.name, func(t *testing.T) { 297 | req := httptest.NewRequest("GET", "/v1/modules/"+tc.moduleString+"/download", nil) 298 | w := httptest.NewRecorder() 299 | 300 | reg.router.ServeHTTP(w, req) 301 | 302 | resp := w.Result() 303 | verifyDownload(t, resp, tc.status, tc.downloadURL) 304 | }) 305 | } 306 | } 307 | 308 | func setupTestRegistry() *Registry { 309 | mstore := memstore.NewMemoryStore() 310 | mstore.Set("hashicorp/consul/aws", []*core.ModuleVersion{ 311 | &core.ModuleVersion{ 312 | Version: "2.2.2", 313 | SourceURL: "git::ssh://git@github.com/hashicorp/consul.git?ref=v2.2.2", 314 | }, 315 | }) 316 | 317 | reg := &Registry{ 318 | IsAuthDisabled: false, 319 | moduleStore: mstore, 320 | logger: zap.NewNop(), 321 | } 322 | reg.setupRoutes() 323 | 324 | return reg 325 | } 326 | 327 | func verifyRoute(t *testing.T, resp *http.Response, path string, authenticated bool) { 328 | is := is.New(t) 329 | url, err := url.Parse(path) 330 | if err != nil { 331 | return 332 | } 333 | healthUrl := regexp.MustCompile("^/health($|[?].*)") 334 | wellknownUrl := regexp.MustCompile(`^/\.well-known/terraform\.json($|[?].*)`) 335 | moduleDownloadRoute := regexp.MustCompile("^/v1/modules/[^/]+/[^/]+/[^/]+/[^/]+/download($|[?].*)") 336 | moduleVersionRoute := regexp.MustCompile("^/v1/modules/[^/]+/[^/]+/[^/]+/versions($|[?].*)") 337 | providerDownloadRoute := regexp.MustCompile("^/v1/providers/[^/]+/[^/]+/versions($|[?].*)") 338 | providerVersionRoute := regexp.MustCompile("^/v1/providers/[^/]+/[^/]+/[^/]+/download/[^/]+/[^/]+($|[?].*)") 339 | providerDownloadAssetRoute := regexp.MustCompile("^/download/provider[^/]+/[^/]+/[^/]+/assets($|[?].*)") 340 | switch { 341 | case path == "/": 342 | t.Logf("Checking index path '%s', parsed path is '%s'", path, url.Path) 343 | is.Equal(resp.StatusCode, http.StatusOK) 344 | case healthUrl.MatchString(path): 345 | t.Logf("Checking health, path '%s'", path) 346 | verifyHealth(t, resp, http.StatusOK, HealthResponse{ 347 | Status: "OK", 348 | }) 349 | case wellknownUrl.MatchString(path): 350 | t.Logf("Checking well known, path '%s'", path) 351 | verifyServiceDiscovery(t, resp) 352 | case authenticated && moduleVersionRoute.MatchString(path): 353 | t.Logf("Checking module version, path '%s'", path) 354 | if strings.HasPrefix(path, "/v1/modules/hashicorp/consul/aws/versions") { 355 | verifyModuleVersions(t, resp, http.StatusOK, []string{"2.2.2"}) 356 | } else { 357 | is.Equal(resp.StatusCode, http.StatusNotFound) 358 | } 359 | case authenticated && moduleDownloadRoute.MatchString(path): 360 | t.Logf("Checking module download, path '%s'", path) 361 | if strings.HasPrefix(path, "/v1/modules/hashicorp/consul/aws/2.2.2/download") { 362 | verifyDownload(t, resp, http.StatusNoContent, "git::ssh://git@github.com/hashicorp/consul.git?ref=v2.2.2") 363 | } else { 364 | is.Equal(resp.StatusCode, http.StatusNotFound) 365 | } 366 | case authenticated && providerVersionRoute.MatchString(path): 367 | t.Logf("Checking provider version, path '%s'", path) 368 | if strings.HasPrefix(path, "/v1/providers/hashicorp/aws/versions") { 369 | verifyModuleVersions(t, resp, http.StatusOK, []string{"2.2.2"}) 370 | } else { 371 | is.Equal(resp.StatusCode, http.StatusNotFound) 372 | } 373 | case authenticated && providerDownloadRoute.MatchString(path): 374 | t.Logf("Checking provider download, path '%s'", path) 375 | is.Equal(resp.StatusCode, http.StatusNotFound) 376 | 377 | case authenticated && providerDownloadAssetRoute.MatchString(path): 378 | t.Logf("Checking provider asset download, path '%s'", path) 379 | is.Equal(resp.StatusCode, http.StatusNotFound) 380 | 381 | case authenticated && (url.Path == "/v1" || strings.HasPrefix(url.Path, "/v1/")): 382 | t.Logf("Checking authenticated v1, path '%s'", path) 383 | t.Logf("Response is '%v'", resp.StatusCode) 384 | t.Logf("authenticated %v", authenticated) 385 | is.Equal(resp.StatusCode, http.StatusNotFound) 386 | case url.Path == "/v1" || strings.HasPrefix(url.Path, "/v1/"): 387 | //case v1Url.MatchString(path): 388 | t.Logf("Checking unathenticated v1, path '%s', parsed path is '%s'", path, url.Path) 389 | t.Logf("Fragment is '%v'", url.Fragment) 390 | t.Logf("Response is '%v'", resp.StatusCode) 391 | t.Logf("authenticated %v", authenticated) 392 | if path == "/v1#" { 393 | is.Equal(resp.StatusCode, http.StatusNotFound) 394 | // for instance //0/v1 is parsed as /v1 395 | } else if !strings.HasPrefix(path, "/v1") { 396 | is.Equal(resp.StatusCode, http.StatusNotFound) 397 | } else if strings.HasPrefix(path, "/v1#") { 398 | is.Equal(resp.StatusCode, http.StatusNotFound) 399 | } else { 400 | is.Equal(resp.StatusCode, http.StatusForbidden) 401 | } 402 | default: 403 | body, _ := io.ReadAll(resp.Body) 404 | t.Logf("Is no match for url '%s'", path) 405 | t.Logf("Parsed path is '%s'", url.Path) 406 | t.Logf("Frament is '%s'", url.Fragment) 407 | t.Logf("authenticated %v", authenticated) 408 | t.Logf("Headers is %v", resp.Header) 409 | t.Logf("Body is '%v'", string(body)) 410 | is.Equal(resp.StatusCode, http.StatusNotFound) 411 | } 412 | } 413 | 414 | func FuzzRoutes(f *testing.F) { 415 | for _, seed := range []string{ 416 | "/", 417 | "/health", 418 | "/.well-known/terraform.json", 419 | "/v1/modules/hashicorp/consul/aws/versions", 420 | "/v1/modules/hashicorp/consul/aws/2.2.2/download", 421 | "/v1/modules/does/not/exist/versions", 422 | "/v1/providers/hashicorp/aws/versions", 423 | "/v1/providers/hashicorp/aws/2.2.2/download/darwin/arm64", 424 | "/v1/providers/does/not/exist/versions", 425 | } { 426 | f.Add(seed) 427 | } 428 | f.Fuzz(func(t *testing.T, url string) { 429 | defer func() { 430 | recover() 431 | }() 432 | 433 | reg := setupTestRegistry() 434 | reg.SetAuthTokens(map[string]string{ 435 | "foo": "testauth", 436 | }) 437 | 438 | req := httptest.NewRequest("GET", url, nil) 439 | w := httptest.NewRecorder() 440 | reg.router.ServeHTTP(w, req) 441 | 442 | resp := w.Result() 443 | defer resp.Body.Close() 444 | verifyRoute(t, resp, url, false) 445 | 446 | req = httptest.NewRequest("GET", url, nil) 447 | req.Header.Set("Authorization", "Bearer testauth") 448 | w = httptest.NewRecorder() 449 | reg.router.ServeHTTP(w, req) 450 | 451 | resp = w.Result() 452 | defer resp.Body.Close() 453 | verifyRoute(t, resp, url, true) 454 | }) 455 | } 456 | -------------------------------------------------------------------------------- /pkg/store/github/github.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2022 - 2025 NRK 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package github 6 | 7 | import ( 8 | "bufio" 9 | "bytes" 10 | "context" 11 | "encoding/json" 12 | "errors" 13 | "fmt" 14 | "io" 15 | "net/http" 16 | "regexp" 17 | "strings" 18 | "sync" 19 | "time" 20 | 21 | "github.com/ProtonMail/go-crypto/openpgp" 22 | "github.com/google/go-github/v69/github" 23 | goversion "github.com/hashicorp/go-version" 24 | "github.com/nrkno/terraform-registry/pkg/core" 25 | "go.uber.org/zap" 26 | "golang.org/x/oauth2" 27 | ) 28 | 29 | var ( 30 | releaseRegex = regexp.MustCompile(`_(freebsd|darwin|linux|windows)_([a-zA-Z0-9]+)\.+`) 31 | ) 32 | 33 | type SHASum struct { 34 | Hash string 35 | FileName string 36 | } 37 | 38 | func parseSHASumsFile(r io.Reader) map[string]string { 39 | sums := make(map[string]string) 40 | 41 | scanner := bufio.NewScanner(r) 42 | for scanner.Scan() { 43 | line := scanner.Text() 44 | parts := strings.Fields(line) 45 | 46 | hash := parts[0] 47 | fileName := parts[1] 48 | 49 | sums[fileName] = hash 50 | } 51 | return sums 52 | } 53 | 54 | // GitHubStore is a store implementation using GitHub as a backend. 55 | // Should not be instantiated directly. Use `NewGitHubStore` instead. 56 | type GitHubStore struct { 57 | // Org to filter repositories by. Leave empty for all. 58 | ownerFilter string 59 | // Topic to filter repositories by. Leave empty for all. 60 | topicFilter string 61 | // Topic to filter provider repositories by. Leave empty for all. 62 | providerOwnerFilter string 63 | // Topic to filter provider repositories by. Leave empty for all. 64 | providerTopicFilter string 65 | 66 | client *github.Client 67 | moduleCache map[string][]*core.ModuleVersion 68 | providerVersionsCache map[string]*core.ProviderVersions 69 | providerCache map[string]*core.Provider 70 | providerIgnoreCache sync.Map 71 | moduleMut sync.RWMutex 72 | providerMut sync.RWMutex 73 | 74 | logger *zap.Logger 75 | } 76 | 77 | func NewGitHubStore(ownerFilter, topicFilter, providerOwnerFilter, providerTopicFilter, accessToken string, logger *zap.Logger) *GitHubStore { 78 | ts := oauth2.StaticTokenSource( 79 | &oauth2.Token{AccessToken: accessToken}, 80 | ) 81 | c := oauth2.NewClient(context.TODO(), ts) 82 | 83 | if logger == nil { 84 | logger = zap.NewNop() 85 | } 86 | 87 | return &GitHubStore{ 88 | ownerFilter: ownerFilter, 89 | topicFilter: topicFilter, 90 | providerOwnerFilter: providerOwnerFilter, 91 | providerTopicFilter: providerTopicFilter, 92 | client: github.NewClient(c), 93 | moduleCache: make(map[string][]*core.ModuleVersion), 94 | providerVersionsCache: make(map[string]*core.ProviderVersions), 95 | providerCache: make(map[string]*core.Provider), 96 | logger: logger, 97 | } 98 | } 99 | 100 | // ListModuleVersions returns a list of module versions. 101 | func (s *GitHubStore) ListModuleVersions(ctx context.Context, namespace, name, provider string) ([]*core.ModuleVersion, error) { 102 | s.moduleMut.RLock() 103 | defer s.moduleMut.RUnlock() 104 | 105 | key := cacheKey(namespace, name, provider) 106 | versions, ok := s.moduleCache[key] 107 | if !ok { 108 | return nil, fmt.Errorf("module '%s' not found", key) 109 | } 110 | 111 | return versions, nil 112 | } 113 | 114 | // GetModuleVersion returns single module version. 115 | func (s *GitHubStore) GetModuleVersion(ctx context.Context, namespace, name, provider, version string) (*core.ModuleVersion, error) { 116 | s.moduleMut.RLock() 117 | defer s.moduleMut.RUnlock() 118 | 119 | key := cacheKey(namespace, name, provider) 120 | versions, ok := s.moduleCache[key] 121 | if !ok { 122 | return nil, fmt.Errorf("module '%s' not found", key) 123 | } 124 | 125 | for _, v := range versions { 126 | if v.Version == version { 127 | return v, nil 128 | } 129 | } 130 | 131 | return nil, fmt.Errorf("version '%s' not found for module '%s'", version, key) 132 | } 133 | 134 | func (s *GitHubStore) ListProviderVersions(ctx context.Context, namespace string, name string) (*core.ProviderVersions, error) { 135 | s.providerMut.RLock() 136 | defer s.providerMut.RUnlock() 137 | 138 | key := cacheKey(namespace, name) 139 | versions, ok := s.providerVersionsCache[key] 140 | if !ok { 141 | return nil, fmt.Errorf("provider '%s' not found", key) 142 | } 143 | 144 | return versions, nil 145 | } 146 | 147 | func (s *GitHubStore) GetProviderVersion(ctx context.Context, namespace string, name string, version string, os string, arch string) (*core.Provider, error) { 148 | s.providerMut.RLock() 149 | defer s.providerMut.RUnlock() 150 | 151 | key := cacheKey(namespace, name, version, os, arch) 152 | provider, ok := s.providerCache[key] 153 | if !ok { 154 | return nil, fmt.Errorf("provider '%s' not found", key) 155 | } 156 | 157 | return provider, nil 158 | } 159 | 160 | func (s *GitHubStore) GetProviderAsset(ctx context.Context, owner string, repo string, tag string, assetName string) (io.ReadCloser, error) { 161 | nameKey := strings.TrimPrefix(repo, "terraform-provider-") 162 | key := cacheKey(owner, nameKey) 163 | versions, ok := s.providerVersionsCache[key] 164 | if !ok { 165 | return nil, fmt.Errorf("provider '%s' not found", key) 166 | } 167 | 168 | found := false 169 | for _, version := range versions.Versions { 170 | if strings.TrimPrefix(tag, "v") == version.Version { 171 | found = true 172 | } 173 | } 174 | 175 | if !found { 176 | return nil, fmt.Errorf("provider version '%s' not found", tag) 177 | } 178 | 179 | releases, _, err := s.client.Repositories.GetReleaseByTag(ctx, owner, repo, tag) 180 | if err != nil { 181 | s.logger.Error(err.Error()) 182 | return nil, err 183 | } 184 | 185 | asset, err := s.findAsset(ctx, owner, repo, releases, assetName) 186 | if err != nil { 187 | return nil, err 188 | } 189 | 190 | return asset, nil 191 | } 192 | 193 | func (s *GitHubStore) findAsset(ctx context.Context, owner string, repo string, byTag *github.RepositoryRelease, assetName string) (io.ReadCloser, error) { 194 | var ( 195 | err error 196 | releaseAsset io.ReadCloser 197 | ) 198 | 199 | for _, asset := range byTag.Assets { 200 | if asset.GetName() == assetName { 201 | releaseAsset, _, err = s.client.Repositories.DownloadReleaseAsset(ctx, owner, repo, asset.GetID(), http.DefaultClient) 202 | if err != nil { 203 | s.logger.Error(err.Error()) 204 | return nil, fmt.Errorf("error getting asset: %s", err) 205 | } else { 206 | break 207 | } 208 | } 209 | } 210 | return releaseAsset, nil 211 | } 212 | 213 | // ReloadProviderCache queries the GitHub API and reloads the local providerCache of provider versions. 214 | // Should be called at least once after initialisation and probably on regular 215 | // intervals afterward to keep providerCache up-to-date. 216 | func (s *GitHubStore) ReloadProviderCache(ctx context.Context) error { 217 | var rateLimitErr *github.RateLimitError 218 | 219 | repos, err := s.searchProviderRepositories(ctx) 220 | if err != nil { 221 | return err 222 | } 223 | 224 | if len(repos) == 0 { 225 | s.logger.Warn("could not find any provider repos matching filter", 226 | zap.String("topic", s.providerTopicFilter), 227 | zap.String("owner", s.providerOwnerFilter)) 228 | } 229 | 230 | providerVersionsCache := make(map[string]*core.ProviderVersions) 231 | providerCache := make(map[string]*core.Provider) 232 | 233 | for _, repo := range repos { 234 | owner, name, err := getOwnerRepoName(repo) 235 | if err != nil { 236 | return err 237 | } 238 | 239 | // HashiCorp (and thus we) require that all provider repositories must match the pattern 240 | // terraform-provider-{NAME}. Only lowercase repository names are supported. 241 | if !strings.HasPrefix(name, "terraform-provider-") { 242 | continue 243 | } 244 | nameKey := strings.TrimPrefix(name, "terraform-provider-") 245 | 246 | start := time.Now() 247 | releases, err := s.listAllRepoReleases(ctx, owner, name) 248 | if err != nil { 249 | return err 250 | } 251 | 252 | var versions []core.ProviderVersion 253 | for _, release := range releases { 254 | var platforms []core.Platform 255 | version := strings.TrimPrefix(release.GetName(), "v") 256 | 257 | if _, ok := s.providerIgnoreCache.Load(cacheKey(nameKey, version)); ok { 258 | s.logger.Debug(fmt.Sprintf("ignoring release [%s/%s], previously found to be not valid", nameKey, version)) 259 | continue 260 | } 261 | 262 | SHASums, SHASumURL, SHASumFileName, err := s.getSHA256Sums(ctx, owner, name, release.Assets) 263 | if err != nil { 264 | if errors.As(err, &rateLimitErr) { 265 | return err 266 | } 267 | s.logger.Warn(fmt.Sprintf("not a valid release [%s/%s] - could not find SHA checksums: %s", nameKey, version, err)) 268 | s.providerIgnoreCache.Store(cacheKey(nameKey, version), true) 269 | continue 270 | } 271 | 272 | // not considered a valid release if a shasum file was not part of the release 273 | if SHASumURL == "" { 274 | if errors.As(err, &rateLimitErr) { 275 | return err 276 | } 277 | s.logger.Warn(fmt.Sprintf("not a valid release [%s/%s] - could not find SHA checksums", nameKey, version)) 278 | s.providerIgnoreCache.Store(cacheKey(nameKey, version), true) 279 | continue 280 | } 281 | 282 | providerProtocols, err := s.getProviderProtocols(ctx, owner, name, release.Assets) 283 | if err != nil { 284 | if errors.As(err, &rateLimitErr) { 285 | return err 286 | } 287 | s.logger.Warn(fmt.Sprintf("not a valid release [%s/%s] - unable to identify provider protocol", nameKey, version)) 288 | s.providerIgnoreCache.Store(cacheKey(nameKey, version), true) 289 | continue 290 | } 291 | 292 | keys, err := s.getGPGPublicKey(ctx, release, owner, name) 293 | if err != nil || len(keys) != 1 { 294 | if errors.As(err, &rateLimitErr) { 295 | return err 296 | } 297 | s.logger.Warn(fmt.Sprintf("not a valid release [%s/%s] - unable to get GPG Public Key", nameKey, version)) 298 | s.providerIgnoreCache.Store(cacheKey(nameKey, version), true) 299 | continue 300 | } 301 | 302 | for _, asset := range release.Assets { 303 | platform, ok := extractOsArch(asset.GetName()) 304 | 305 | // if asset does not contain os/arch info, it is not a provider binary 306 | if !ok { 307 | continue 308 | } 309 | 310 | platforms = append(platforms, platform) 311 | 312 | downloadUrl := asset.GetBrowserDownloadURL() 313 | SHASumSigURL := SHASumURL + ".sig" 314 | if repo.GetPrivate() { 315 | downloadUrl = fmt.Sprintf("/download/provider/%s/%s/v%s/asset/%s", owner, name, version, asset.GetName()) 316 | SHASumURL = fmt.Sprintf("/download/provider/%s/%s/v%s/asset/%s", owner, name, version, SHASumFileName) 317 | SHASumSigURL = fmt.Sprintf("/download/provider/%s/%s/v%s/asset/%s", owner, name, version, SHASumFileName+".sig") 318 | } 319 | 320 | p := &core.Provider{ 321 | Protocols: providerProtocols, 322 | OS: platform.OS, 323 | Arch: platform.Arch, 324 | Filename: asset.GetName(), 325 | DownloadURL: downloadUrl, 326 | SHASumsURL: SHASumURL, 327 | SHASumsSignatureURL: SHASumSigURL, 328 | SHASum: SHASums[asset.GetName()], 329 | SigningKeys: core.SigningKeys{GPGPublicKeys: keys}, 330 | } 331 | 332 | // update the fresh providerCache 333 | providerCache[cacheKey(owner, nameKey, version, platform.OS, platform.Arch)] = p 334 | } 335 | 336 | if len(platforms) > 0 { 337 | pv := core.ProviderVersion{ 338 | Version: version, 339 | Protocols: providerProtocols, 340 | Platforms: platforms, 341 | } 342 | versions = append(versions, pv) 343 | } 344 | } 345 | 346 | duration := time.Since(start) 347 | s.logger.Debug("found provider", 348 | zap.String("name", fmt.Sprintf("%s/%s", owner, nameKey)), 349 | zap.Int("versions", len(versions)), 350 | zap.Duration("duration", duration), 351 | ) 352 | 353 | // update the fresh providerVersionCache 354 | providerVersionsCache[cacheKey(owner, nameKey)] = &core.ProviderVersions{Versions: versions} 355 | } 356 | 357 | // This cleans up modules that are no longer available and 358 | // reduces write lock duration by not modifying the caches directly 359 | // on each iteration. 360 | s.providerMut.Lock() 361 | s.providerCache = providerCache 362 | s.providerVersionsCache = providerVersionsCache 363 | s.providerMut.Unlock() 364 | 365 | return nil 366 | } 367 | 368 | func (s *GitHubStore) getGPGPublicKey(ctx context.Context, release *github.RepositoryRelease, owner string, name string) ([]core.GpgPublicKeys, error) { 369 | var keys []core.GpgPublicKeys 370 | for _, asset := range release.Assets { 371 | if strings.Contains(asset.GetName(), "gpg-public-key.pem") { 372 | releaseAsset, _, err := s.client.Repositories.DownloadReleaseAsset(ctx, owner, name, asset.GetID(), http.DefaultClient) 373 | if err != nil { 374 | return nil, err 375 | } 376 | 377 | all, err := io.ReadAll(releaseAsset) 378 | releaseAsset.Close() 379 | if err != nil { 380 | return nil, err 381 | } 382 | 383 | els, err := openpgp.ReadArmoredKeyRing(bytes.NewReader(all)) 384 | if err != nil { 385 | return nil, err 386 | } 387 | 388 | if len(els) != 1 { 389 | return nil, fmt.Errorf("GPG Key contains %d entities, wanted 1", len(els)) 390 | } 391 | 392 | key := els[0] 393 | keys = []core.GpgPublicKeys{ 394 | {KeyID: key.PrimaryKey.KeyIdString(), ASCIIArmor: string(all), TrustSignature: "", Source: "", SourceURL: ""}, 395 | } 396 | } 397 | } 398 | return keys, nil 399 | } 400 | 401 | // ReloadCache queries the GitHub API and reloads the local moduleCache of module versions. 402 | // Should be called at least once after initialisation and probably on regular 403 | // intervals afterward to keep moduleCache up-to-date. 404 | func (s *GitHubStore) ReloadCache(ctx context.Context) error { 405 | repos, err := s.searchModuleRepositories(ctx) 406 | if err != nil { 407 | return err 408 | } 409 | 410 | if len(repos) == 0 { 411 | s.logger.Warn("could not find any module repos matching filter", 412 | zap.String("topic", s.topicFilter), 413 | zap.String("owner", s.ownerFilter)) 414 | } 415 | 416 | fresh := make(map[string][]*core.ModuleVersion) 417 | 418 | for _, repo := range repos { 419 | owner, name, err := getOwnerRepoName(repo) 420 | if err != nil { 421 | return err 422 | } 423 | 424 | key := fmt.Sprintf("%s/%s/generic", owner, name) 425 | 426 | tags, err := s.listAllRepoTags(ctx, owner, name) 427 | if err != nil { 428 | return err 429 | } 430 | 431 | versions := make([]*core.ModuleVersion, 0) 432 | for _, tag := range tags { 433 | version := strings.TrimPrefix(tag.GetName(), "v") // Terraform uses SemVer names without 'v' prefix 434 | if _, err := goversion.NewSemver(version); err == nil { 435 | versions = append(versions, &core.ModuleVersion{ 436 | Version: version, 437 | SourceURL: fmt.Sprintf("git::ssh://git@github.com/%s/%s.git?ref=%s", owner, name, tag.GetName()), 438 | }) 439 | } 440 | } 441 | 442 | s.logger.Debug("found module", 443 | zap.String("name", key), 444 | zap.Int("version_count", len(versions)), 445 | ) 446 | 447 | fresh[key] = versions 448 | } 449 | 450 | // This cleans up modules that are no longer available and 451 | // reduces write lock duration by not modifying the moduleCache directly 452 | // on each iteration. 453 | s.moduleMut.Lock() 454 | s.moduleCache = fresh 455 | s.moduleMut.Unlock() 456 | 457 | return nil 458 | } 459 | 460 | // listAllRepoTags lists all tags for the specified repository. 461 | // When an error is returned, the tags fetched up until the point of error 462 | // is also returned. 463 | func (s *GitHubStore) listAllRepoTags(ctx context.Context, owner, repo string) ([]*github.RepositoryTag, error) { 464 | var allTags []*github.RepositoryTag 465 | 466 | opts := &github.ListOptions{PerPage: 100} 467 | 468 | for { 469 | tags, resp, err := s.client.Repositories.ListTags(ctx, owner, repo, opts) 470 | if err != nil { 471 | return allTags, err 472 | } 473 | 474 | allTags = append(allTags, tags...) 475 | if resp.NextPage == 0 { 476 | break 477 | } 478 | opts.Page = resp.NextPage 479 | } 480 | 481 | return allTags, nil 482 | } 483 | 484 | func (s *GitHubStore) listAllRepoReleases(ctx context.Context, owner, repo string) ([]*github.RepositoryRelease, error) { 485 | var allReleases []*github.RepositoryRelease 486 | 487 | opts := &github.ListOptions{ 488 | PerPage: 100, 489 | } 490 | for { 491 | releases, resp, err := s.client.Repositories.ListReleases(ctx, owner, repo, opts) 492 | if err != nil { 493 | return allReleases, err 494 | } 495 | allReleases = append(allReleases, releases...) 496 | if resp.NextPage == 0 { 497 | break 498 | } 499 | opts.Page = resp.NextPage 500 | } 501 | return allReleases, nil 502 | } 503 | func (s *GitHubStore) searchModuleRepositories(ctx context.Context) ([]*github.Repository, error) { 504 | var filters []string 505 | 506 | if s.ownerFilter != "" { 507 | filters = append(filters, fmt.Sprintf(`org:"%s"`, s.ownerFilter)) 508 | } 509 | if s.topicFilter != "" { 510 | filters = append(filters, fmt.Sprintf(`topic:"%s"`, s.topicFilter)) 511 | } 512 | return s.searchRepositories(ctx, filters) 513 | } 514 | 515 | func (s *GitHubStore) searchProviderRepositories(ctx context.Context) ([]*github.Repository, error) { 516 | var filters []string 517 | 518 | if s.ownerFilter != "" { 519 | filters = append(filters, fmt.Sprintf(`org:"%s"`, s.providerOwnerFilter)) 520 | } 521 | if s.topicFilter != "" { 522 | filters = append(filters, fmt.Sprintf(`topic:"%s"`, s.providerTopicFilter)) 523 | } 524 | return s.searchRepositories(ctx, filters) 525 | } 526 | 527 | // searchRepositories fetches all repositories matching the configured filters. 528 | // When an error is returned, the repositories fetched up until the point of error 529 | // is also returned. 530 | func (s *GitHubStore) searchRepositories(ctx context.Context, filters []string) ([]*github.Repository, error) { 531 | var ( 532 | allRepos []*github.Repository 533 | ) 534 | 535 | opts := &github.SearchOptions{} 536 | opts.ListOptions.PerPage = 100 537 | 538 | for { 539 | result, resp, err := s.client.Search.Repositories(ctx, strings.Join(filters, " "), opts) 540 | if err != nil { 541 | return allRepos, err 542 | } 543 | 544 | allRepos = append(allRepos, result.Repositories...) 545 | 546 | if resp.NextPage == 0 { 547 | break 548 | } 549 | opts.Page = resp.NextPage 550 | } 551 | 552 | return allRepos, nil 553 | } 554 | 555 | func cacheKey(s ...string) string { 556 | return strings.Join(s, "/") 557 | } 558 | 559 | // extractOsArch inputs the filename of the Github release asset. 560 | // Function uses regex to extract operating system and architecture about the binary inside the release 561 | // Example input: terraform-provider-test_1.0.3_darwin_arm64.zip, output would be darwin as OS and arm64 as arch 562 | func extractOsArch(name string) (core.Platform, bool) { 563 | matches := releaseRegex.FindStringSubmatch(name) 564 | 565 | if len(matches) >= 3 { 566 | osType := matches[1] 567 | arch := matches[2] 568 | return core.Platform{ 569 | OS: osType, 570 | Arch: arch, 571 | }, true 572 | } 573 | 574 | return core.Platform{}, false 575 | } 576 | 577 | // Provider Protocol version should be set in the terraform-registry-manifest.json file in the root of the repo. 578 | // This file should be included in the release. If not present, default is 5.0 according to Terraform docs. 579 | // https://developer.hashicorp.com/terraform/registry/providers/publishing 580 | func (s *GitHubStore) getProviderProtocols(ctx context.Context, owner string, repo string, assets []*github.ReleaseAsset) ([]string, error) { 581 | providerProtocols := []string{"5.0"} 582 | 583 | for _, asset := range assets { 584 | if strings.Contains(asset.GetName(), "manifest.json") { 585 | 586 | responseBody, _, err := s.client.Repositories.DownloadReleaseAsset(ctx, owner, repo, asset.GetID(), http.DefaultClient) 587 | if err != nil { 588 | return nil, fmt.Errorf("unable to get manifest: %s", err) 589 | } 590 | 591 | manifest := &core.ProviderManifest{} 592 | err = json.NewDecoder(responseBody).Decode(manifest) 593 | responseBody.Close() 594 | if err != nil { 595 | return nil, fmt.Errorf("unable to decode manifest: %s", err) 596 | } 597 | 598 | providerProtocols = manifest.Metadata.ProtocolVersions 599 | break 600 | } 601 | } 602 | return providerProtocols, nil 603 | } 604 | 605 | // A file named SHA256SUMS containing sums must be included in the release. Look for the file, and download it 606 | func (s *GitHubStore) getSHA256Sums(ctx context.Context, owner string, repo string, assets []*github.ReleaseAsset) (map[string]string, string, string, error) { 607 | var ( 608 | SHASums map[string]string 609 | SHASumURL string 610 | SHASumFileName string 611 | ) 612 | 613 | for _, asset := range assets { 614 | if strings.Contains(asset.GetName(), "SHA256SUMS") && !strings.HasSuffix(asset.GetName(), ".sig") { 615 | responseBody, _, err := s.client.Repositories.DownloadReleaseAsset(ctx, owner, repo, asset.GetID(), http.DefaultClient) 616 | if err != nil { 617 | return nil, "", "", fmt.Errorf("unable to get SHA checksums: %s", err) 618 | } 619 | 620 | SHASums = parseSHASumsFile(responseBody) 621 | SHASumURL = asset.GetBrowserDownloadURL() 622 | SHASumFileName = asset.GetName() 623 | responseBody.Close() 624 | break 625 | } 626 | } 627 | 628 | return SHASums, SHASumURL, SHASumFileName, nil 629 | } 630 | 631 | // Splitting owner from FullName to avoid getting it from GetOwner().GetName(), 632 | // as it seems to be empty, maybe due to missing OAuth permission scopes. 633 | func getOwnerRepoName(repo *github.Repository) (string, string, error) { 634 | parts := strings.Split(repo.GetFullName(), "/") 635 | if len(parts) != 2 { 636 | return "", "", fmt.Errorf("repo.FullName is not in expected format 'owner/repo', is '%s'", repo.GetFullName()) 637 | } 638 | 639 | owner := parts[0] 640 | name := parts[1] 641 | return owner, name, nil 642 | } 643 | -------------------------------------------------------------------------------- /pkg/store/github/github_test.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2022 - 2025 NRK 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package github 6 | 7 | import ( 8 | "context" 9 | "net/http" 10 | "reflect" 11 | "testing" 12 | 13 | "github.com/google/go-github/v69/github" 14 | "github.com/matryer/is" 15 | "github.com/migueleliasweb/go-github-mock/src/mock" 16 | "github.com/nrkno/terraform-registry/pkg/core" 17 | "go.uber.org/zap" 18 | ) 19 | 20 | func TestGithubStore(t *testing.T) { 21 | t.Run("create GitHubStore", func(t *testing.T) { 22 | is := is.New(t) 23 | emptyResult := new(github.RepositoriesSearchResult) 24 | total := 0 25 | emptyResult.Total = &total 26 | mockedHTTPClient := mock.NewMockedHTTPClient( 27 | mock.WithRequestMatch( 28 | mock.GetSearchRepositories, 29 | emptyResult, 30 | ), 31 | ) 32 | 33 | c := github.NewClient(mockedHTTPClient) 34 | store := &GitHubStore{ 35 | ownerFilter: "test-owner", 36 | topicFilter: "test-topic", 37 | client: c, 38 | moduleCache: make(map[string][]*core.ModuleVersion), 39 | logger: zap.NewNop(), 40 | } 41 | 42 | err := store.ReloadCache(context.Background()) 43 | is.NoErr(err) 44 | }) 45 | 46 | t.Run("create GitHubStore with github error", func(t *testing.T) { 47 | is := is.New(t) 48 | mockedHTTPClient := mock.NewMockedHTTPClient( 49 | mock.WithRequestMatchHandler( 50 | mock.GetSearchRepositories, 51 | http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 52 | mock.WriteError( 53 | w, 54 | http.StatusInternalServerError, 55 | "github went belly up or something", 56 | ) 57 | }), 58 | ), 59 | ) 60 | c := github.NewClient(mockedHTTPClient) 61 | store := &GitHubStore{ 62 | ownerFilter: "test-owner", 63 | topicFilter: "test-topic", 64 | client: c, 65 | moduleCache: make(map[string][]*core.ModuleVersion), 66 | logger: zap.NewNop(), 67 | } 68 | store.client = c 69 | err := store.ReloadCache(context.Background()) 70 | is.True(err != nil) 71 | ghErr, ok := err.(*github.ErrorResponse) 72 | if !ok { 73 | t.Fatal("couldn't cast userErr to *github.ErrorResponse") 74 | } 75 | 76 | if ghErr.Message != "github went belly up or something" { 77 | t.Errorf("user err is %s, want 'github went belly up or something'", err.Error()) 78 | } 79 | }) 80 | } 81 | 82 | func TestGetModuleVersion(t *testing.T) { 83 | result := new(github.RepositoriesSearchResult) 84 | total := 1 85 | result.Total = &total 86 | result.Repositories = []*github.Repository{ 87 | { 88 | Name: github.Ptr("testrepo"), 89 | FullName: github.Ptr("test-owner/test-repo"), 90 | }, 91 | } 92 | mockedHTTPClient := mock.NewMockedHTTPClient( 93 | mock.WithRequestMatch( 94 | mock.GetSearchRepositories, 95 | result, 96 | ), 97 | mock.WithRequestMatch( 98 | mock.GetReposTagsByOwnerByRepo, 99 | []github.RepositoryTag{ 100 | { 101 | Name: github.Ptr("v1.0.0"), 102 | }, 103 | }, 104 | ), 105 | ) 106 | 107 | c := github.NewClient(mockedHTTPClient) 108 | store := &GitHubStore{ 109 | ownerFilter: "test-owner", 110 | topicFilter: "test-topic", 111 | client: c, 112 | moduleCache: make(map[string][]*core.ModuleVersion), 113 | logger: zap.NewNop(), 114 | } 115 | 116 | err := store.ReloadCache(context.Background()) 117 | if err != nil { 118 | t.Fatal("Could not ReloadCache") 119 | } 120 | 121 | t.Run("returns matching version", func(t *testing.T) { 122 | is := is.New(t) 123 | ver, err := store.GetModuleVersion(context.Background(), "test-owner", "test-repo", "generic", "1.0.0") 124 | is.True(err == nil) 125 | is.Equal(ver.Version, "1.0.0") 126 | is.Equal(ver.SourceURL, "git::ssh://git@github.com/test-owner/test-repo.git?ref=v1.0.0") 127 | }) 128 | 129 | t.Run("errs when missing", func(t *testing.T) { 130 | is := is.New(t) 131 | ver, err := store.GetModuleVersion(context.Background(), "test-owner", "test-repo", "generic", "1.0.1") 132 | is.True(err != nil) 133 | is.True(ver == nil) 134 | is.Equal(err.Error(), "version '1.0.1' not found for module 'test-owner/test-repo/generic'") 135 | }) 136 | 137 | } 138 | 139 | func TestListModuleVersions(t *testing.T) { 140 | result := new(github.RepositoriesSearchResult) 141 | total := 1 142 | result.Total = &total 143 | result.Repositories = []*github.Repository{ 144 | { 145 | Name: github.Ptr("testrepo"), 146 | FullName: github.Ptr("test-owner/test-repo"), 147 | }, 148 | } 149 | mockedHTTPClient := mock.NewMockedHTTPClient( 150 | mock.WithRequestMatch( 151 | mock.GetSearchRepositories, 152 | result, 153 | ), 154 | mock.WithRequestMatch( 155 | mock.GetReposTagsByOwnerByRepo, 156 | []github.RepositoryTag{ 157 | { 158 | Name: github.Ptr("v1.0.0"), 159 | }, 160 | { 161 | Name: github.Ptr("v1.0.1"), 162 | }, 163 | { 164 | Name: github.Ptr("v2.0.0"), 165 | }, 166 | { 167 | Name: github.Ptr("non-semver"), 168 | }, 169 | }, 170 | ), 171 | ) 172 | 173 | c := github.NewClient(mockedHTTPClient) 174 | store := &GitHubStore{ 175 | ownerFilter: "test-owner", 176 | topicFilter: "test-topic", 177 | client: c, 178 | moduleCache: make(map[string][]*core.ModuleVersion), 179 | logger: zap.NewNop(), 180 | } 181 | 182 | err := store.ReloadCache(context.Background()) 183 | if err != nil { 184 | t.Fatal("Could not ReloadCache") 185 | } 186 | 187 | t.Run("returns list of versions", func(t *testing.T) { 188 | is := is.New(t) 189 | versions, err := store.ListModuleVersions(context.Background(), "test-owner", "test-repo", "generic") 190 | is.True(err == nil) 191 | is.Equal(len(versions), 3) 192 | is.Equal(versions[0].Version, "1.0.0") 193 | is.Equal(versions[1].Version, "1.0.1") 194 | is.Equal(versions[2].Version, "2.0.0") 195 | }) 196 | 197 | t.Run("errs when missing", func(t *testing.T) { 198 | is := is.New(t) 199 | versions, err := store.ListModuleVersions(context.Background(), "wrong", "wrong", "wrong") 200 | is.True(err != nil) 201 | is.Equal(versions, nil) 202 | }) 203 | 204 | } 205 | 206 | func Test_extractOsArch(t *testing.T) { 207 | tests := []struct { 208 | name string 209 | args string 210 | result core.Platform 211 | found bool 212 | }{ 213 | {"name", "terraform-provider-test_1.0.3_darwin_amd64.zip", core.Platform{OS: "darwin", Arch: "amd64"}, true}, 214 | {"name", "terraform-provider-test_1.0.3_darwin_arm64.zip", core.Platform{OS: "darwin", Arch: "arm64"}, true}, 215 | {"name", "terraform-provider-test_1.0.3_linux_amd64.zip", core.Platform{OS: "linux", Arch: "amd64"}, true}, 216 | {"name", "terraform-provider-test_1.0.3_linux_arm64.zip", core.Platform{OS: "linux", Arch: "arm64"}, true}, 217 | {"name", "terraform-provider-test_1.0.3_ugga_arm644.zip", core.Platform{}, false}, 218 | } 219 | for _, tt := range tests { 220 | t.Run(tt.name, func(t *testing.T) { 221 | result, found := extractOsArch(tt.args) 222 | if !reflect.DeepEqual(result, tt.result) { 223 | t.Errorf("extractOsArch() result = %v, want %v", result, tt.result) 224 | } 225 | if found != tt.found { 226 | t.Errorf("extractOsArch() found = %v, want %v", found, tt.found) 227 | } 228 | }) 229 | } 230 | } 231 | -------------------------------------------------------------------------------- /pkg/store/memory/memory.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2022 - 2025 NRK 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package memory 6 | 7 | import ( 8 | "context" 9 | "fmt" 10 | "sync" 11 | 12 | "github.com/nrkno/terraform-registry/pkg/core" 13 | ) 14 | 15 | // MemoryStore is an in-memory store implementation without a backend. 16 | // Should not be instantiated directly. Use `NewMemoryStore` instead. 17 | type MemoryStore struct { 18 | store map[string][]*core.ModuleVersion 19 | mut sync.RWMutex 20 | } 21 | 22 | func NewMemoryStore() *MemoryStore { 23 | return &MemoryStore{ 24 | store: make(map[string][]*core.ModuleVersion), 25 | } 26 | } 27 | 28 | // Get returns a pointer to an item by key, or `nil` if it's not found. 29 | func (s *MemoryStore) Get(key string) []*core.ModuleVersion { 30 | s.mut.RLock() 31 | defer s.mut.RUnlock() 32 | 33 | m, ok := s.store[key] 34 | if !ok { 35 | return nil 36 | } 37 | return m 38 | } 39 | 40 | // Set stores an item under the specified `key`. 41 | func (s *MemoryStore) Set(key string, m []*core.ModuleVersion) { 42 | s.mut.Lock() 43 | defer s.mut.Unlock() 44 | s.store[key] = m 45 | } 46 | 47 | // ListModuleVersions returns a list of module versions. 48 | func (s *MemoryStore) ListModuleVersions(ctx context.Context, namespace, name, provider string) ([]*core.ModuleVersion, error) { 49 | key := fmt.Sprintf("%s/%s/%s", namespace, name, provider) 50 | versions := s.Get(key) 51 | if versions == nil { 52 | return nil, fmt.Errorf("module '%s' not found", key) 53 | } 54 | 55 | return versions, nil 56 | } 57 | 58 | // GetModuleVersion returns single module version. 59 | func (s *MemoryStore) GetModuleVersion(ctx context.Context, namespace, name, provider, version string) (*core.ModuleVersion, error) { 60 | key := fmt.Sprintf("%s/%s/%s", namespace, name, provider) 61 | versions := s.Get(key) 62 | if versions == nil { 63 | return nil, fmt.Errorf("module '%s' not found", key) 64 | } 65 | 66 | for _, v := range versions { 67 | if v.Version == version { 68 | return v, nil 69 | } 70 | } 71 | 72 | return nil, fmt.Errorf("version '%s' not found for module '%s'", version, key) 73 | } 74 | -------------------------------------------------------------------------------- /pkg/store/memory/memory_test.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2022 - 2025 NRK 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package memory 6 | 7 | import ( 8 | "context" 9 | "testing" 10 | 11 | "github.com/matryer/is" 12 | "github.com/nrkno/terraform-registry/pkg/core" 13 | ) 14 | 15 | func TestGet(t *testing.T) { 16 | is := is.New(t) 17 | 18 | v := &core.ModuleVersion{ 19 | Version: "1", 20 | } 21 | s := NewMemoryStore() 22 | s.store["foo"] = []*core.ModuleVersion{v} 23 | 24 | res := s.Get("foo") 25 | is.Equal(len(res), 1) 26 | is.Equal(res[0], v) 27 | } 28 | 29 | func TestSet(t *testing.T) { 30 | is := is.New(t) 31 | 32 | v := &core.ModuleVersion{ 33 | Version: "1", 34 | } 35 | s := NewMemoryStore() 36 | s.Set("foo", []*core.ModuleVersion{v}) 37 | 38 | res := s.Get("foo") 39 | is.Equal(len(res), 1) 40 | is.Equal(res[0], v) 41 | } 42 | 43 | func TestListModuleVersions(t *testing.T) { 44 | is := is.New(t) 45 | 46 | s := NewMemoryStore() 47 | s.Set("foo/bar/baz", []*core.ModuleVersion{ 48 | {Version: "1"}, 49 | {Version: "2"}, 50 | {Version: "3"}, 51 | }) 52 | 53 | t.Run("returns list of versions", func(t *testing.T) { 54 | is := is.New(t) 55 | versions, err := s.ListModuleVersions(context.TODO(), "foo", "bar", "baz") 56 | is.NoErr(err) 57 | is.Equal(len(versions), 3) 58 | is.Equal(versions[1].Version, "2") 59 | }) 60 | 61 | t.Run("errs when missing", func(t *testing.T) { 62 | is := is.New(t) 63 | versions, err := s.ListModuleVersions(context.TODO(), "wrong", "wrong", "wrong") 64 | is.True(err != nil) 65 | is.Equal(versions, nil) 66 | }) 67 | } 68 | 69 | func TestGetModuleVersion(t *testing.T) { 70 | is := is.New(t) 71 | 72 | s := NewMemoryStore() 73 | s.Set("foo/bar/baz", []*core.ModuleVersion{ 74 | {Version: "1", SourceURL: "https://example.com/foo/bar/baz/v1.tar.gz"}, 75 | {Version: "2", SourceURL: "https://example.com/foo/bar/baz/v2.tar.gz"}, 76 | {Version: "3", SourceURL: "https://example.com/foo/bar/baz/v3.tar.gz"}, 77 | }) 78 | 79 | t.Run("returns matching version", func(t *testing.T) { 80 | is := is.New(t) 81 | ver, err := s.GetModuleVersion(context.TODO(), "foo", "bar", "baz", "2") 82 | is.NoErr(err) 83 | is.Equal(ver.Version, "2") 84 | is.Equal(ver.SourceURL, "https://example.com/foo/bar/baz/v2.tar.gz") 85 | }) 86 | 87 | t.Run("errs when missing", func(t *testing.T) { 88 | is := is.New(t) 89 | ver, err := s.GetModuleVersion(context.TODO(), "foo", "bar", "baz", "13") 90 | is.True(err != nil) 91 | is.Equal(ver, nil) 92 | }) 93 | } 94 | -------------------------------------------------------------------------------- /pkg/store/s3/s3.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2024 - 2025 NRK 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package s3 6 | 7 | import ( 8 | "context" 9 | "fmt" 10 | "path" 11 | "path/filepath" 12 | "regexp" 13 | "strings" 14 | "sync" 15 | 16 | "github.com/aws/aws-sdk-go/aws" 17 | "github.com/aws/aws-sdk-go/service/s3" 18 | "github.com/aws/aws-sdk-go/service/s3/s3iface" 19 | "github.com/nrkno/terraform-registry/pkg/core" 20 | "go.uber.org/zap" 21 | ) 22 | 23 | // S3API defines the subset of S3 client methods used by S3Store 24 | type S3API interface { 25 | ListObjectsV2WithContext(ctx aws.Context, input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) 26 | HeadObjectWithContext(ctx aws.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) 27 | } 28 | 29 | // S3StoreInterface defines the interface for S3Store 30 | type S3StoreInterface interface { 31 | ListModuleVersions(ctx context.Context, namespace, name, system string) ([]*core.ModuleVersion, error) 32 | GetModuleVersion(ctx context.Context, namespace, name, system, version string) (*core.ModuleVersion, error) 33 | } 34 | 35 | // S3Store implements S3StoreInterface 36 | type S3Store struct { 37 | client s3iface.S3API 38 | cache map[string][]*core.ModuleVersion 39 | region string 40 | bucket string 41 | logger *zap.Logger 42 | mut sync.Mutex 43 | } 44 | 45 | func NewS3Store(client s3iface.S3API, region string, bucket string, logger *zap.Logger) *S3Store { 46 | if logger == nil { 47 | logger = zap.NewNop() 48 | } 49 | 50 | return &S3Store{ 51 | client: client, 52 | cache: make(map[string][]*core.ModuleVersion), 53 | region: region, 54 | bucket: bucket, 55 | logger: logger, 56 | } 57 | } 58 | 59 | func (s *S3Store) ListModuleVersions(ctx context.Context, namespace, name, system string) ([]*core.ModuleVersion, error) { 60 | addr := filepath.Join(namespace, name, system) 61 | vers, err := s.fetchModuleVersions(ctx, addr) 62 | if err != nil { 63 | return nil, err 64 | } 65 | 66 | return vers, nil 67 | } 68 | 69 | func (s *S3Store) fetchModuleVersions(ctx context.Context, address string) ([]*core.ModuleVersion, error) { 70 | s.mut.Lock() 71 | defer s.mut.Unlock() 72 | 73 | p := address + "/" 74 | in := &s3.ListObjectsV2Input{ 75 | Bucket: aws.String(s.bucket), 76 | Prefix: aws.String(p), 77 | } 78 | out, err := s.client.ListObjectsV2WithContext(ctx, in) 79 | if err != nil { 80 | return nil, err 81 | } 82 | 83 | vers := make([]*core.ModuleVersion, 0) 84 | for _, o := range out.Contents { 85 | path := o.Key 86 | if isValidModuleSourcePath(*path) { 87 | vers = append(vers, &core.ModuleVersion{ 88 | Version: strings.Split(*path, "/")[3], 89 | SourceURL: fmt.Sprintf("s3::https://%s.s3.%s.amazonaws.com/%s", s.bucket, s.region, *path), 90 | }) 91 | } 92 | } 93 | 94 | s.cache[address] = vers 95 | 96 | return vers, nil 97 | } 98 | 99 | func (s *S3Store) GetModuleVersion(ctx context.Context, namespace, name, system, version string) (*core.ModuleVersion, error) { 100 | addr := filepath.Join(namespace, name, system) 101 | ver, err := s.fetchModuleVersion(ctx, addr, version) 102 | if err != nil { 103 | return nil, err 104 | } 105 | 106 | return ver, nil 107 | } 108 | 109 | func (s *S3Store) fetchModuleVersion(ctx context.Context, address, version string) (*core.ModuleVersion, error) { 110 | s.mut.Lock() 111 | defer s.mut.Unlock() 112 | 113 | vers := s.cache[address] 114 | for _, o := range vers { 115 | if o.Version == version { 116 | return o, nil 117 | } 118 | } 119 | 120 | path := path.Join(address, version) 121 | keySuffix := version + ".zip" 122 | if !isValidModuleSourcePath(path) { 123 | s.logger.Warn("invalid module path requested: " + path) 124 | return nil, fmt.Errorf("module version path '%s' is not valid", path) 125 | } 126 | _, err := s.client.HeadObjectWithContext(ctx, &s3.HeadObjectInput{ 127 | Bucket: aws.String(s.bucket), 128 | Key: aws.String(path + "/" + keySuffix), 129 | }) 130 | if err != nil { 131 | return nil, err 132 | } 133 | 134 | ver := &core.ModuleVersion{ 135 | Version: version, 136 | SourceURL: fmt.Sprintf("s3::https://%s.s3.%s.amazonaws.com/%s/%s", s.bucket, s.region, path, keySuffix), 137 | } 138 | 139 | s.cache[address] = append(vers, ver) 140 | 141 | return ver, nil 142 | } 143 | 144 | func isValidModuleSourcePath(path string) bool { 145 | // https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string 146 | verRegExp := `(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?` 147 | addrRegExp := `\w+/\w+/\w+` 148 | r := regexp.MustCompile("^" + addrRegExp + "/" + verRegExp) 149 | return r.MatchString(path) 150 | } 151 | -------------------------------------------------------------------------------- /pkg/store/s3/s3_test.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2024 - 2025 NRK 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package s3 6 | 7 | import ( 8 | "context" 9 | "testing" 10 | 11 | "github.com/aws/aws-sdk-go/aws" 12 | "github.com/aws/aws-sdk-go/aws/request" 13 | "github.com/aws/aws-sdk-go/service/s3" 14 | "github.com/aws/aws-sdk-go/service/s3/s3iface" 15 | "github.com/matryer/is" 16 | "github.com/stretchr/testify/mock" 17 | "go.uber.org/zap" 18 | ) 19 | 20 | // MockS3API is a mock implementation for the S3API interface 21 | type MockS3API struct { 22 | mock.Mock 23 | s3iface.S3API 24 | } 25 | 26 | func (m *MockS3API) ListObjectsV2WithContext(ctx aws.Context, input *s3.ListObjectsV2Input, opts ...request.Option) (*s3.ListObjectsV2Output, error) { 27 | args := m.Called(ctx, input) 28 | return args.Get(0).(*s3.ListObjectsV2Output), args.Error(1) 29 | } 30 | 31 | func (m *MockS3API) HeadObjectWithContext(ctx aws.Context, input *s3.HeadObjectInput, opts ...request.Option) (*s3.HeadObjectOutput, error) { 32 | args := m.Called(ctx, input) 33 | return args.Get(0).(*s3.HeadObjectOutput), args.Error(1) 34 | } 35 | 36 | func TestListModuleVersions(t *testing.T) { 37 | is := is.New(t) 38 | mockS3 := new(MockS3API) 39 | 40 | store := NewS3Store(mockS3, "us-east-1", "mytestbucket", zap.NewNop()) 41 | 42 | mockS3.On("ListObjectsV2WithContext", mock.Anything, mock.Anything).Return(&s3.ListObjectsV2Output{ 43 | Contents: []*s3.Object{ 44 | {Key: aws.String("testnamespace/testname/testprovider/1.0.0/1.0.0.zip")}, 45 | {Key: aws.String("testnamespace/testname/testprovider/1.1.1/1.1.1.zip")}, 46 | }, 47 | }, nil) 48 | 49 | vers, err := store.ListModuleVersions(context.Background(), "testnamespace", "testname", "testprovider") 50 | is.True(err == nil) 51 | is.Equal(len(vers), 2) 52 | is.Equal(vers[0].SourceURL, "s3::https://mytestbucket.s3.us-east-1.amazonaws.com/testnamespace/testname/testprovider/1.0.0/1.0.0.zip") 53 | is.Equal(vers[1].SourceURL, "s3::https://mytestbucket.s3.us-east-1.amazonaws.com/testnamespace/testname/testprovider/1.1.1/1.1.1.zip") 54 | 55 | mockS3.AssertExpectations(t) 56 | } 57 | 58 | func TestGetModuleVersion(t *testing.T) { 59 | mockS3 := new(MockS3API) 60 | 61 | store := NewS3Store(mockS3, "us-east-1", "mytestbucket", zap.NewNop()) 62 | 63 | t.Run("returns matching version", func(t *testing.T) { 64 | is := is.New(t) 65 | mockS3.On("HeadObjectWithContext", mock.Anything, mock.Anything).Return(&s3.HeadObjectOutput{}, nil) 66 | ver, err := store.GetModuleVersion(context.Background(), "testnamespace", "testname", "testprovider", "1.0.0") 67 | is.True(err == nil) 68 | is.Equal(ver.Version, "1.0.0") 69 | is.Equal(ver.SourceURL, "s3::https://mytestbucket.s3.us-east-1.amazonaws.com/testnamespace/testname/testprovider/1.0.0/1.0.0.zip") 70 | }) 71 | 72 | t.Run("invalid version", func(t *testing.T) { 73 | is := is.New(t) 74 | ver, err := store.GetModuleVersion(context.Background(), "test-owner", "test-repo", "generic", "1.0.0") 75 | is.True(err != nil) 76 | is.True(ver == nil) 77 | is.Equal(err.Error(), "module version path 'test-owner/test-repo/generic/1.0.0' is not valid") 78 | }) 79 | } 80 | --------------------------------------------------------------------------------