├── .github ├── ISSUE_TEMPLATE │ ├── bug-report.yaml │ ├── config.yml │ └── feature-request.yaml ├── dependabot.yaml └── workflows │ ├── artifacthub.yaml │ ├── e2e.yaml │ ├── go.yaml │ ├── helm.yaml │ └── release.yaml ├── .gitignore ├── .golangci.yaml ├── CHANGELOG.md ├── CONTRIBUTING.md ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── charts └── spegel │ ├── .helmignore │ ├── Chart.yaml │ ├── README.md │ ├── README.md.gotmpl │ ├── artifacthub-repo.yml │ ├── monitoring │ └── grafana-dashboard.json │ ├── templates │ ├── _helpers.tpl │ ├── daemonset.yaml │ ├── grafana-dashboard.yaml │ ├── post-delete-hook.yaml │ ├── rbac.yaml │ ├── service.yaml │ ├── servicemonitor.yaml │ └── verticalpodautoscaler.yaml │ └── values.yaml ├── go.mod ├── go.sum ├── internal ├── channel │ └── channel.go ├── mux │ ├── mux.go │ ├── mux_test.go │ ├── response.go │ └── response_test.go └── web │ ├── templates │ ├── index.html │ ├── measure.html │ └── stats.html │ ├── web.go │ └── web_test.go ├── main.go ├── pkg ├── metrics │ ├── metrics.go │ └── metrics_test.go ├── oci │ ├── containerd.go │ ├── containerd_test.go │ ├── distribution.go │ ├── distribution_test.go │ ├── image.go │ ├── image_test.go │ ├── memory.go │ ├── oci.go │ ├── oci_test.go │ └── testdata │ │ ├── blobs │ │ └── sha256 │ │ │ ├── 0ad7c556c55464fa44d4c41e5236715e015b0266daced62140fb5c6b983c946b │ │ │ ├── 3caa2469de2a23cbcc209dd0b9d01cd78ff9a0f88741655991d36baede5b0996 │ │ │ ├── 44cb2cf712c060f69df7310e99339c1eb51a085446f1bb6d44469acff35b4355 │ │ │ ├── 68b8a989a3e08ddbdb3a0077d35c0d0e59c9ecf23d0634584def8bdbb7d6824f │ │ │ ├── 9430beb291fa7b96997711fc486bc46133c719631aefdbeebe58dd3489217bfe │ │ │ ├── 9506c8e7a2d0a098d43cadfd7ecdc3c91697e8188d3a1245943b669f717747b4 │ │ │ ├── addc990c58744bdf96364fe89bd4aab38b1e824d51c688edb36c75247cd45fa9 │ │ │ ├── aec8273a5e5aca369fcaa8cecef7bf6c7959d482f5c8cfa2236a6a16e46bbdcf │ │ │ ├── b6d6089ca6c395fd563c2084f5dd7bc56a2f5e6a81413558c5be0083287a77e9 │ │ │ ├── d8df04365d06181f037251de953aca85cc16457581a8fc168f4957c978e1008b │ │ │ └── dce623533c59af554b85f859e91fc1cbb7f574e873c82f36b9ea05a09feb0b53 │ │ └── images.json ├── registry │ ├── registry.go │ └── registry_test.go ├── routing │ ├── bootstrap.go │ ├── bootstrap_test.go │ ├── memory.go │ ├── memory_test.go │ ├── p2p.go │ ├── p2p_test.go │ └── routing.go └── state │ ├── state.go │ └── state_test.go └── test └── e2e ├── e2e_test.go └── testdata ├── conformance-job.yaml └── test-nginx.yaml /.github/ISSUE_TEMPLATE/bug-report.yaml: -------------------------------------------------------------------------------- 1 | name: Bug Report 2 | description: Create a report to help improve Spegel 3 | labels: ["bug"] 4 | body: 5 | - type: markdown 6 | attributes: 7 | value: | 8 | Thank you for taking the time to fill ot this bug report! Please read the [FAQ](https://spegel.dev/docs/faq/) and check existing issues before submitting a new issue. 9 | - type: input 10 | attributes: 11 | label: Spegel version 12 | placeholder: eg. v0.0.16 13 | validations: 14 | required: true 15 | - type: input 16 | attributes: 17 | label: Kubernetes distribution 18 | placeholder: eg. AKS, EKS, K3S, Kubeadm... 19 | validations: 20 | required: true 21 | - type: input 22 | attributes: 23 | label: Kubernetes version 24 | placeholder: eg. v1.29.0 25 | validations: 26 | required: true 27 | - type: input 28 | attributes: 29 | label: CNI 30 | placeholder: eg. Calico, Cilium, Azure CNI... 31 | validations: 32 | required: true 33 | - type: textarea 34 | attributes: 35 | label: Describe the bug 36 | description: A clear description of what the bug is. 37 | validations: 38 | required: true 39 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | blank_issues_enabled: true 2 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature-request.yaml: -------------------------------------------------------------------------------- 1 | name: Feature Request 2 | description: Suggest a new feature for Spegel 3 | labels: ["enhancement"] 4 | body: 5 | - type: markdown 6 | attributes: 7 | value: | 8 | Thank you for creating a feature request! Please check existing issues before submitting. 9 | - type: textarea 10 | attributes: 11 | label: Describe the problem to be solved 12 | description: A clear description of the problem that needs to be addressed by this feature request. 13 | validations: 14 | required: true 15 | - type: textarea 16 | attributes: 17 | label: Proposed solution to the problem 18 | description: A clear description of the solution or multiple possible solutions to implement this feature request. 19 | validations: 20 | required: false 21 | -------------------------------------------------------------------------------- /.github/dependabot.yaml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "github-actions" 4 | directory: "/" 5 | schedule: 6 | interval: "daily" 7 | open-pull-requests-limit: 15 8 | - package-ecosystem: "gomod" 9 | directory: "/" 10 | schedule: 11 | interval: "daily" 12 | open-pull-requests-limit: 15 13 | groups: 14 | k8s: 15 | patterns: 16 | - "k8s.io/*" 17 | -------------------------------------------------------------------------------- /.github/workflows/artifacthub.yaml: -------------------------------------------------------------------------------- 1 | name: artifacthub 2 | on: 3 | push: 4 | branches: ["main"] 5 | paths: 6 | - "charts/spegel/artifacthub-repo.yml" 7 | permissions: 8 | contents: read 9 | packages: write 10 | defaults: 11 | run: 12 | shell: bash 13 | jobs: 14 | release: 15 | runs-on: ubuntu-latest 16 | steps: 17 | - name: Clone repo 18 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #v4.2.2 19 | with: 20 | submodules: true 21 | - name: Login to GitHub Container Registry 22 | uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 #v3.4.0 23 | with: 24 | registry: ghcr.io 25 | username: ${{ github.repository_owner }} 26 | password: ${{ secrets.GITHUB_TOKEN }} 27 | - name: Setup ORAS 28 | uses: oras-project/setup-oras@5c0b487ce3fe0ce3ab0d034e63669e426e294e4d #v1.2.2 29 | - name: Push Artifact Hub metadata 30 | run: oras push ghcr.io/spegel-org/helm-charts/spegel:artifacthub.io --config /dev/null:application/vnd.cncf.artifacthub.config.v1+yaml charts/spegel/artifacthub-repo.yml:application/vnd.cncf.artifacthub.repository-metadata.layer.v1.yaml 31 | -------------------------------------------------------------------------------- /.github/workflows/e2e.yaml: -------------------------------------------------------------------------------- 1 | name: e2e 2 | on: 3 | pull_request: 4 | defaults: 5 | run: 6 | shell: bash 7 | jobs: 8 | test: 9 | runs-on: ubuntu-latest 10 | strategy: 11 | matrix: 12 | include: 13 | - proxy-mode: iptables 14 | ip-family: ipv4 15 | - proxy-mode: iptables 16 | ip-family: ipv6 17 | - proxy-mode: ipvs 18 | ip-family: ipv4 19 | steps: 20 | - name: Checkout 21 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #v4.2.2 22 | - name: Setup Go 23 | uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b #v5.4.0 24 | with: 25 | go-version-file: go.mod 26 | - name: Setup Kind 27 | uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 #v1.12.0 28 | with: 29 | version: v0.27.0 30 | install_only: true 31 | - name: Run e2e 32 | run: make test-e2e E2E_PROXY_MODE=${{ matrix.proxy-mode }} E2E_IP_FAMILY=${{ matrix.ip-family }} 33 | -------------------------------------------------------------------------------- /.github/workflows/go.yaml: -------------------------------------------------------------------------------- 1 | name: go 2 | on: 3 | pull_request: 4 | defaults: 5 | run: 6 | shell: bash 7 | jobs: 8 | lint: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - name: Checkout 12 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #v4.2.2 13 | - name: Setup Go 14 | uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b #v5.4.0 15 | with: 16 | go-version-file: go.mod 17 | - name: Setup golangci-lint 18 | uses: golangci/golangci-lint-action@1481404843c368bc19ca9406f87d6e0fc97bdcfd #v7.0.0 19 | unit: 20 | runs-on: ubuntu-latest 21 | steps: 22 | - name: Checkout 23 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #v4.2.2 24 | - name: Setup Go 25 | uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b #v5.4.0 26 | with: 27 | go-version-file: go.mod 28 | - name: Run tests 29 | run: go test -race -coverprofile=coverage.txt -covermode=atomic ./... 30 | - name: Upload coverage reports to Codecov 31 | uses: codecov/codecov-action@ad3126e916f78f00edff4ed0317cf185271ccc2d #v5.4.2 32 | with: 33 | token: ${{ secrets.CODECOV_TOKEN }} 34 | -------------------------------------------------------------------------------- /.github/workflows/helm.yaml: -------------------------------------------------------------------------------- 1 | name: helm 2 | on: 3 | pull_request: 4 | defaults: 5 | run: 6 | shell: bash 7 | jobs: 8 | docs: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - name: Checkout 12 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #v4.2.2 13 | - name: Setup Go 14 | uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b #v5.4.0 15 | with: 16 | go-version-file: go.mod 17 | - name: Run helm-docs 18 | run: make helm-docs 19 | - name: Check if working tree is dirty 20 | run: | 21 | if [[ $(git diff --stat) != '' ]]; then 22 | git diff 23 | echo 'run make helm-docs and commit changes' 24 | exit 1 25 | fi 26 | -------------------------------------------------------------------------------- /.github/workflows/release.yaml: -------------------------------------------------------------------------------- 1 | name: release 2 | on: 3 | release: 4 | types: [published] 5 | permissions: 6 | contents: read 7 | packages: write 8 | id-token: write 9 | defaults: 10 | run: 11 | shell: bash 12 | jobs: 13 | release: 14 | runs-on: ubuntu-latest 15 | steps: 16 | - name: Clone repo 17 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #v4.2.2 18 | - name: Setup Cosign 19 | uses: sigstore/cosign-installer@3454372f43399081ed03b604cb2d021dabca52bb #v3.8.2 20 | - name: Setup Helm 21 | uses: azure/setup-helm@b9e51907a09c216f16ebe8536097933489208112 #v4.3.0 22 | with: 23 | version: v3.12.1 24 | - name: Setup QEMU 25 | uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 #v3.6.0 26 | - name: Setup Docker Buildx 27 | id: buildx 28 | uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 #v3.10.0 29 | - name: Setup yq 30 | uses: frenck/action-setup-yq@c4b5be8b4a215c536a41d436757d9feb92836d4f #v1.0.2 31 | - name: Login to GitHub Container Registry 32 | uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 #v3.4.0 33 | with: 34 | registry: ghcr.io 35 | username: ${{ github.repository_owner }} 36 | password: ${{ secrets.GITHUB_TOKEN }} 37 | - name: Prepare 38 | id: prep 39 | run: | 40 | VERSION=sha-${GITHUB_SHA::8} 41 | if [[ $GITHUB_REF == refs/tags/* ]]; then 42 | VERSION=${GITHUB_REF/refs\/tags\//} 43 | fi 44 | echo "VERSION=${VERSION}" >> $GITHUB_OUTPUT 45 | - name: Generate images meta 46 | id: meta 47 | uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 #v5.7.0 48 | with: 49 | images: ghcr.io/spegel-org/spegel 50 | tags: type=raw,value=${{ steps.prep.outputs.VERSION }} 51 | - name: Publish multi-arch image 52 | uses: docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4 #v6.15.0 53 | id: build 54 | with: 55 | push: true 56 | builder: ${{ steps.buildx.outputs.name }} 57 | context: . 58 | file: ./Dockerfile 59 | platforms: linux/amd64,linux/arm/v7,linux/arm64 60 | tags: ghcr.io/spegel-org/spegel:${{ steps.prep.outputs.VERSION }} 61 | labels: ${{ steps.meta.outputs.labels }} 62 | - name: Sign the image with Cosign 63 | run: | 64 | cosign sign --yes ghcr.io/spegel-org/spegel@${{ steps.build.outputs.DIGEST }} 65 | - name: Publish Helm chart to GHCR 66 | id: helm 67 | run: | 68 | HELM_VERSION=${{ steps.prep.outputs.VERSION }} 69 | HELM_VERSION=${HELM_VERSION#v} 70 | rm charts/spegel/artifacthub-repo.yml 71 | yq -i '.image.digest = "${{ steps.build.outputs.DIGEST }}"' charts/spegel/values.yaml 72 | helm package --app-version ${{ steps.prep.outputs.VERSION }} --version ${HELM_VERSION} charts/spegel 73 | helm push spegel-${HELM_VERSION}.tgz oci://ghcr.io/spegel-org/helm-charts 2> .digest 74 | DIGEST=$(cat .digest | awk -F "[, ]+" '/Digest/{print $NF}') 75 | echo "DIGEST=${DIGEST}" >> $GITHUB_OUTPUT 76 | - name: Sign the Helm chart with Cosign 77 | run: | 78 | cosign sign --yes ghcr.io/spegel-org/helm-charts/spegel@${{ steps.helm.outputs.DIGEST }} 79 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # If you prefer the allow list template instead of the deny list, see community template: 2 | # https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore 3 | # 4 | # Binaries for programs and plugins 5 | *.exe 6 | *.exe~ 7 | *.dll 8 | *.so 9 | *.dylib 10 | 11 | # Test binary, built with `go test -c` 12 | *.test 13 | 14 | # Output of the go coverage tool, specifically when used with LiteIDE 15 | *.out 16 | 17 | # Dependency directories (remove the comment below to include it) 18 | # vendor/ 19 | 20 | # Go workspace file 21 | go.work 22 | -------------------------------------------------------------------------------- /.golangci.yaml: -------------------------------------------------------------------------------- 1 | version: "2" 2 | linters: 3 | default: none 4 | enable: 5 | - bodyclose 6 | - errcheck 7 | - gocritic 8 | - govet 9 | - importas 10 | - ineffassign 11 | - ireturn 12 | - misspell 13 | - nolintlint 14 | - paralleltest 15 | - perfsprint 16 | - staticcheck 17 | - testifylint 18 | - unused 19 | settings: 20 | errcheck: 21 | disable-default-exclusions: true 22 | check-type-assertions: true 23 | check-blank: true 24 | gocritic: 25 | enable-all: true 26 | disabled-checks: 27 | - importShadow 28 | - hugeParam 29 | - rangeValCopy 30 | - whyNoLint 31 | - unnamedResult 32 | - httpNoBody 33 | govet: 34 | disable: 35 | - shadow 36 | enable-all: true 37 | importas: 38 | alias: 39 | - pkg: io/fs 40 | alias: iofs 41 | - pkg: github.com/go-logr/logr/testing 42 | alias: tlog 43 | - pkg: github.com/pelletier/go-toml/v2/unstable 44 | alias: tomlu 45 | - pkg: github.com/multiformats/go-multiaddr/net 46 | alias: manet 47 | - pkg: github.com/multiformats/go-multiaddr 48 | alias: ma 49 | - pkg: github.com/multiformats/go-multicodec 50 | alias: mc 51 | - pkg: github.com/multiformats/go-multihash 52 | alias: mh 53 | - pkg: github.com/ipfs/go-cid 54 | alias: cid 55 | - pkg: github.com/libp2p/go-libp2p-kad-dht 56 | alias: dht 57 | - pkg: github.com/libp2p/go-libp2p/p2p/net/mock 58 | alias: mocknet 59 | - pkg: go.etcd.io/bbolt 60 | alias: bolt 61 | - pkg: k8s.io/cri-api/pkg/apis/runtime/v1 62 | alias: runtimeapi 63 | - pkg: github.com/containerd/containerd/api/events 64 | alias: eventtypes 65 | - pkg: github.com/opencontainers/go-digest 66 | alias: digest 67 | - pkg: github.com/opencontainers/image-spec/specs-go/v1 68 | alias: ocispec 69 | - pkg: k8s.io/apimachinery/pkg/util/version 70 | alias: utilversion 71 | no-extra-aliases: true 72 | nolintlint: 73 | require-explanation: true 74 | require-specific: true 75 | perfsprint: 76 | strconcat: false 77 | testifylint: 78 | enable-all: true 79 | exclusions: 80 | generated: lax 81 | presets: 82 | - comments 83 | - common-false-positives 84 | - legacy 85 | - std-error-handling 86 | paths: 87 | - third_party$ 88 | - builtin$ 89 | - examples$ 90 | formatters: 91 | enable: 92 | - goimports 93 | exclusions: 94 | generated: lax 95 | paths: 96 | - third_party$ 97 | - builtin$ 98 | - examples$ 99 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | Thank you for considering contributing to Spegel, hopefully this document will make this process easier. 4 | 5 | ## Running tests 6 | 7 | The following tools are required to run the tests properly. 8 | 9 | * go 10 | * golangci-lint 11 | * kind 12 | 13 | Run the linter and the unit tests to quickly validate changes. 14 | 15 | ```shell 16 | make lint test 17 | ``` 18 | 19 | Run the e2e tests which take a bit more time. 20 | 21 | ```shell 22 | make test-e2e 23 | ``` 24 | 25 | There are e2e tests for the different CNIs iptables, iptables-v6, and ipvs. 26 | 27 | ```shell 28 | make test-e2e E2E_CNI=ipvs 29 | ``` 30 | 31 | ## Building 32 | 33 | Build the Docker image locally. 34 | 35 | ```shell 36 | make docker-build 37 | ``` 38 | 39 | It is possible to specify a different image name and tag. 40 | 41 | ```shell 42 | make docker-build IMG=example.com/spegel TAG=feature 43 | ``` 44 | 45 | ### Local testing 46 | 47 | In order to manually test or debug Spegel, you will need the following tools. 48 | 49 | * kind 50 | * docker 51 | * helm 52 | * kubectl 53 | 54 | First run dev deploy which will create a Kind cluster with the proper configuration and deploy Spegel into it. If you run this command a second time the cluster will be kept but Spegel will be updated. 55 | 56 | ```shell 57 | make dev-deploy 58 | ``` 59 | 60 | After the command has run a Kind cluster named `spegel-dev` should be created. 61 | 62 | ## Generating documentation 63 | 64 | Changes to the Helm chart values will require the documentation to be regenerated. 65 | 66 | ```shell 67 | make helm-docs 68 | ``` 69 | 70 | ## Acceptance policy 71 | 72 | Pull requests need to fulfill the following requirements to be accepted. 73 | 74 | * New code has tests where applicable. 75 | * The change has been added to the [changelog](./CHANGELOG.md). 76 | * Documentation has been generated if applicable. 77 | * The unit tests pass. 78 | * Linter does not report any errors. 79 | * All end to end tests pass. 80 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.24.1@sha256:52ff1b35ff8de185bf9fd26c70077190cd0bed1e9f16a2d498ce907e5c421268 AS builder 2 | RUN mkdir /build 3 | WORKDIR /build 4 | COPY go.mod go.mod 5 | COPY go.sum go.sum 6 | RUN go mod download 7 | COPY main.go main.go 8 | COPY internal/ internal/ 9 | COPY pkg/ pkg/ 10 | RUN CGO_ENABLED=0 go build -installsuffix 'static' -o spegel . 11 | 12 | FROM gcr.io/distroless/static:nonroot 13 | COPY --from=builder /build/spegel /app/ 14 | WORKDIR /app 15 | USER root:root 16 | ENTRYPOINT ["./spegel"] 17 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 The Spegel Authors 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | TAG = $$(git rev-parse --short HEAD) 2 | IMG_NAME ?= ghcr.io/spegel-org/spegel 3 | IMG_REF = $(IMG_NAME):$(TAG) 4 | E2E_PROXY_MODE ?= iptables 5 | E2E_IP_FAMILY ?= ipv4 6 | 7 | lint: 8 | golangci-lint run ./... 9 | 10 | docker-build: 11 | docker build -t ${IMG_REF} . 12 | 13 | test-unit: 14 | go test ./... 15 | 16 | test-e2e: docker-build 17 | IMG_REF=${IMG_REF} \ 18 | E2E_PROXY_MODE=${E2E_PROXY_MODE} \ 19 | E2E_IP_FAMILY=${E2E_IP_FAMILY} \ 20 | go test ./test/e2e -v -timeout 200s -tags e2e -count 1 -run TestE2E 21 | 22 | dev-deploy: docker-build 23 | IMG_REF=${IMG_REF} go test ./test/e2e -v -timeout 200s -tags e2e -count 1 -run TestDevDeploy 24 | 25 | tools: 26 | GO111MODULE=on go install github.com/norwoodj/helm-docs/cmd/helm-docs 27 | 28 | helm-docs: tools 29 | cd ./charts/spegel && helm-docs 30 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | > [!NOTE] 2 | > We’ve started hosting community meetings every Tuesday at 17:00 CET. Find out how to participate at https://spegel.dev/project/community/#meeting. 3 | 4 | # Spegel 5 | 6 | Spegel, mirror in Swedish, is a stateless cluster local OCI registry mirror. 7 | 8 |

9 | 10 |

11 | 12 | ## Features 13 | 14 | Spegel is for you if you are looking to do any of the following. 15 | 16 | * Locally cache images from external registries with no explicit configuration. 17 | * Avoid cluster failure during external registry downtime. 18 | * Improve image pull speed and pod startup time by pulling images from the local cache first. 19 | * Avoid rate-limiting when pulling images from external registries (e.g. Docker Hub). 20 | * Decrease egressing traffic outside of the clusters network. 21 | * Increase image pull efficiency in edge node deployments. 22 | 23 | ## Getting Started 24 | 25 | Read the [getting started](https://spegel.dev/docs/getting-started/) guide to deploy Spegel. 26 | 27 | ## Contributing 28 | 29 | Read [contribution guidelines](./CONTRIBUTING.md) for instructions on how to build and test Spegel. 30 | 31 | ## Acknowledgements 32 | 33 | Spegel was initially developed at [Xenit AB](https://xenit.se/). 34 | 35 | ## License 36 | 37 | This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details. 38 | -------------------------------------------------------------------------------- /charts/spegel/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *.orig 18 | *~ 19 | # Various IDEs 20 | .project 21 | .idea/ 22 | *.tmproj 23 | .vscode/ 24 | -------------------------------------------------------------------------------- /charts/spegel/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: spegel 3 | description: Stateless cluster local OCI registry mirror. 4 | type: application 5 | version: v0.0.1 6 | appVersion: v0.0.1 7 | annotations: 8 | artifacthub.io/category: "integration-delivery" 9 | artifacthub.io/license: "MIT" 10 | artifacthub.io/operator: "false" 11 | artifacthub.io/prerelease: "false" 12 | -------------------------------------------------------------------------------- /charts/spegel/README.md: -------------------------------------------------------------------------------- 1 | # Spegel 2 | 3 | Stateless cluster local OCI registry mirror. 4 | 5 | Read the [getting started](https://spegel.dev/docs/getting-started/) guide to deploy Spegel. 6 | 7 | ## Values 8 | 9 | | Key | Type | Default | Description | 10 | |-----|------|---------|-------------| 11 | | affinity | object | `{}` | Affinity settings for pod assignment. | 12 | | basicAuthSecretName | string | `""` | Name of secret containing basic authentication credentials for registry. | 13 | | clusterDomain | string | `"cluster.local."` | Domain configured for service domain names. | 14 | | commonLabels | object | `{}` | Common labels to apply to all rendered resources. | 15 | | fullnameOverride | string | `""` | Overrides the full name of the chart. | 16 | | grafanaDashboard.annotations | object | `{}` | Annotations that ConfigMaps can have to get configured in Grafana, See: sidecar.dashboards.folderAnnotation for specifying the dashboard folder. https://github.com/grafana/helm-charts/tree/main/charts/grafana | 17 | | grafanaDashboard.enabled | bool | `false` | If true creates a Grafana dashboard. | 18 | | grafanaDashboard.sidecarLabel | string | `"grafana_dashboard"` | Label that ConfigMaps should have to be loaded as dashboards. | 19 | | grafanaDashboard.sidecarLabelValue | string | `"1"` | Label value that ConfigMaps should have to be loaded as dashboards. | 20 | | image.digest | string | `""` | Image digest. | 21 | | image.pullPolicy | string | `"IfNotPresent"` | Image Pull Policy. | 22 | | image.repository | string | `"ghcr.io/spegel-org/spegel"` | Image repository. | 23 | | image.tag | string | `""` | Overrides the image tag whose default is the chart appVersion. | 24 | | imagePullSecrets | list | `[]` | Image Pull Secrets | 25 | | nameOverride | string | `""` | Overrides the name of the chart. | 26 | | namespaceOverride | string | `""` | Overrides the namespace where spegel resources are installed. | 27 | | nodeSelector | object | `{"kubernetes.io/os":"linux"}` | Node selector for pod assignment. | 28 | | podAnnotations | object | `{}` | Annotations to add to the pod. | 29 | | podSecurityContext | object | `{}` | Security context for the pod. | 30 | | priorityClassName | string | `"system-node-critical"` | Priority class name to use for the pod. | 31 | | resources | object | `{"limits":{"memory":"128Mi"},"requests":{"memory":"128Mi"}}` | Resource requests and limits for the Spegel container. | 32 | | revisionHistoryLimit | int | `10` | The number of old history to retain to allow rollback. | 33 | | securityContext | object | `{}` | Security context for the Spegel container. | 34 | | service.cleanup.port | int | `8080` | Port to expose cleanup probe on. | 35 | | service.metrics.port | int | `9090` | Port to expose the metrics via the service. | 36 | | service.registry.hostPort | int | `30020` | Local host port to expose the registry. | 37 | | service.registry.nodeIp | string | `""` | Override the NODE_ID environment variable. It defaults to the field status.hostIP | 38 | | service.registry.nodePort | int | `30021` | Node port to expose the registry via the service. | 39 | | service.registry.port | int | `5000` | Port to expose the registry via the service. | 40 | | service.registry.topologyAwareHintsEnabled | bool | `true` | If true adds topology aware hints annotation to node port service. | 41 | | service.router.port | int | `5001` | Port to expose the router via the service. | 42 | | serviceAccount.annotations | object | `{}` | Annotations to add to the service account | 43 | | serviceAccount.name | string | `""` | The name of the service account to use. If not set and create is true, a name is generated using the fullname template. | 44 | | serviceMonitor.enabled | bool | `false` | If true creates a Prometheus Service Monitor. | 45 | | serviceMonitor.interval | string | `"60s"` | Prometheus scrape interval. | 46 | | serviceMonitor.labels | object | `{}` | Service monitor specific labels for prometheus to discover servicemonitor. | 47 | | serviceMonitor.metricRelabelings | list | `[]` | List of relabeling rules to apply to the samples before ingestion. | 48 | | serviceMonitor.relabelings | list | `[]` | List of relabeling rules to apply the target’s metadata labels. | 49 | | serviceMonitor.scrapeTimeout | string | `"30s"` | Prometheus scrape interval timeout. | 50 | | spegel.additionalMirrorTargets | list | `[]` | Additional target mirror registries other than Spegel. | 51 | | spegel.containerdContentPath | string | `"/var/lib/containerd/io.containerd.content.v1.content"` | Path to Containerd content store.. | 52 | | spegel.containerdMirrorAdd | bool | `true` | If true Spegel will add mirror configuration to the node. | 53 | | spegel.containerdNamespace | string | `"k8s.io"` | Containerd namespace where images are stored. | 54 | | spegel.containerdRegistryConfigPath | string | `"/etc/containerd/certs.d"` | Path to Containerd mirror configuration. | 55 | | spegel.containerdSock | string | `"/run/containerd/containerd.sock"` | Path to Containerd socket. | 56 | | spegel.debugWebEnabled | bool | `false` | When true enables debug web page. | 57 | | spegel.logLevel | string | `"INFO"` | Minimum log level to output. Value should be DEBUG, INFO, WARN, or ERROR. | 58 | | spegel.mirrorResolveRetries | int | `3` | Max amount of mirrors to attempt. | 59 | | spegel.mirrorResolveTimeout | string | `"20ms"` | Max duration spent finding a mirror. | 60 | | spegel.mirroredRegistries | list | `[]` | Registries for which mirror configuration will be created. Empty means all registires will be mirrored. | 61 | | spegel.prependExisting | bool | `false` | When true existing mirror configuration will be kept and Spegel will prepend it's configuration. | 62 | | spegel.resolveLatestTag | bool | `true` | When true latest tags will be resolved to digests. | 63 | | spegel.resolveTags | bool | `true` | When true Spegel will resolve tags to digests. | 64 | | tolerations | list | `[{"key":"CriticalAddonsOnly","operator":"Exists"},{"effect":"NoExecute","operator":"Exists"},{"effect":"NoSchedule","operator":"Exists"}]` | Tolerations for pod assignment. | 65 | | updateStrategy | object | `{}` | An update strategy to replace existing pods with new pods. | 66 | | verticalPodAutoscaler.controlledResources | list | `[]` | List of resources that the vertical pod autoscaler can control. Defaults to cpu and memory | 67 | | verticalPodAutoscaler.controlledValues | string | `"RequestsAndLimits"` | Specifies which resource values should be controlled: RequestsOnly or RequestsAndLimits. | 68 | | verticalPodAutoscaler.enabled | bool | `false` | If true creates a Vertical Pod Autoscaler. | 69 | | verticalPodAutoscaler.maxAllowed | object | `{}` | Define the max allowed resources for the pod | 70 | | verticalPodAutoscaler.minAllowed | object | `{}` | Define the min allowed resources for the pod | 71 | | verticalPodAutoscaler.recommenders | list | `[]` | Recommender responsible for generating recommendation for the object. List should be empty (then the default recommender will generate the recommendation) or contain exactly one recommender. | 72 | | verticalPodAutoscaler.updatePolicy.minReplicas | int | `2` | Specifies minimal number of replicas which need to be alive for VPA Updater to attempt pod eviction | 73 | | verticalPodAutoscaler.updatePolicy.updateMode | string | `"Auto"` | Specifies whether recommended updates are applied when a Pod is started and whether recommended updates are applied during the life of a Pod. Possible values are "Off", "Initial", "Recreate", and "Auto". | 74 | -------------------------------------------------------------------------------- /charts/spegel/README.md.gotmpl: -------------------------------------------------------------------------------- 1 | # Spegel 2 | 3 | {{ template "chart.description" . }} 4 | 5 | Read the [getting started](https://spegel.dev/docs/getting-started/) guide to deploy Spegel. 6 | 7 | {{ template "chart.valuesSection" . }} 8 | -------------------------------------------------------------------------------- /charts/spegel/artifacthub-repo.yml: -------------------------------------------------------------------------------- 1 | repositoryID: 8122016b-c465-4eaf-be87-f51423aa76f1 2 | owners: 3 | - name: Philip Laine 4 | email: philip.laine@gmail.com 5 | -------------------------------------------------------------------------------- /charts/spegel/templates/_helpers.tpl: -------------------------------------------------------------------------------- 1 | {{/* 2 | Expand the name of the chart. 3 | */}} 4 | {{- define "spegel.name" -}} 5 | {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} 6 | {{- end }} 7 | 8 | {{/* 9 | Create a default fully qualified app name. 10 | We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). 11 | If release name contains chart name it will be used as a full name. 12 | */}} 13 | {{- define "spegel.fullname" -}} 14 | {{- if .Values.fullnameOverride }} 15 | {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} 16 | {{- else }} 17 | {{- $name := default .Chart.Name .Values.nameOverride }} 18 | {{- if contains $name .Release.Name }} 19 | {{- .Release.Name | trunc 63 | trimSuffix "-" }} 20 | {{- else }} 21 | {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} 22 | {{- end }} 23 | {{- end }} 24 | {{- end }} 25 | 26 | {{/* 27 | Creates the namespace for the chart. 28 | Defaults to the Release namespace unless the namespaceOverride is defined. 29 | */}} 30 | {{- define "spegel.namespace" -}} 31 | {{- if .Values.namespaceOverride }} 32 | {{- printf "%s" .Values.namespaceOverride -}} 33 | {{- else }} 34 | {{- printf "%s" .Release.Namespace -}} 35 | {{- end }} 36 | {{- end }} 37 | 38 | 39 | {{/* 40 | Create chart name and version as used by the chart label. 41 | */}} 42 | {{- define "spegel.chart" -}} 43 | {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} 44 | {{- end }} 45 | 46 | {{/* 47 | Common labels 48 | */}} 49 | {{- define "spegel.labels" -}} 50 | helm.sh/chart: {{ include "spegel.chart" . }} 51 | {{ include "spegel.selectorLabels" . }} 52 | {{- if .Chart.AppVersion }} 53 | app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} 54 | {{- end }} 55 | app.kubernetes.io/managed-by: {{ .Release.Service }} 56 | {{- with .Values.commonLabels }} 57 | {{ toYaml . }} 58 | {{- end }} 59 | {{- end }} 60 | 61 | {{/* 62 | {{- end }} 63 | {{- end }} 64 | 65 | {{/* 66 | Selector labels 67 | */}} 68 | {{- define "spegel.selectorLabels" -}} 69 | app.kubernetes.io/name: {{ include "spegel.name" . }} 70 | app.kubernetes.io/instance: {{ .Release.Name }} 71 | {{- end }} 72 | 73 | {{/* 74 | Create the name of the service account to use 75 | */}} 76 | {{- define "spegel.serviceAccountName" -}} 77 | {{- default (include "spegel.fullname" .) .Values.serviceAccount.name }} 78 | {{- end }} 79 | 80 | {{/* 81 | Image reference 82 | */}} 83 | {{- define "spegel.image" -}} 84 | {{- if .Values.image.digest }} 85 | {{- .Values.image.repository }}@{{ .Values.image.digest }} 86 | {{- else }} 87 | {{- .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }} 88 | {{- end }} 89 | {{- end }} 90 | 91 | {{/* 92 | Host networking 93 | */}} 94 | {{- define "networking.nodeIp" -}} 95 | {{- if .Values.service.registry.nodeIp -}} 96 | value: {{ .Values.service.registry.nodeIp }} 97 | {{- else -}} 98 | valueFrom: 99 | fieldRef: 100 | fieldPath: status.hostIP 101 | {{- end -}} 102 | {{- end -}} 103 | -------------------------------------------------------------------------------- /charts/spegel/templates/daemonset.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: DaemonSet 3 | metadata: 4 | name: {{ include "spegel.fullname" . }} 5 | namespace: {{ include "spegel.namespace" . }} 6 | labels: 7 | {{- include "spegel.labels" . | nindent 4 }} 8 | spec: 9 | revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} 10 | updateStrategy: 11 | {{- toYaml .Values.updateStrategy | nindent 4 }} 12 | selector: 13 | matchLabels: 14 | {{- include "spegel.selectorLabels" . | nindent 6 }} 15 | template: 16 | metadata: 17 | {{- with .Values.podAnnotations }} 18 | annotations: 19 | {{- toYaml . | nindent 8 }} 20 | {{- end }} 21 | labels: 22 | {{- include "spegel.selectorLabels" . | nindent 8 }} 23 | {{- with .Values.commonLabels }} 24 | {{- toYaml . | nindent 8 }} 25 | {{- end }} 26 | spec: 27 | {{- with .Values.imagePullSecrets }} 28 | imagePullSecrets: 29 | {{- toYaml . | nindent 8 }} 30 | {{- end }} 31 | serviceAccountName: {{ include "spegel.serviceAccountName" . }} 32 | securityContext: 33 | {{- toYaml .Values.podSecurityContext | nindent 8 }} 34 | priorityClassName: {{ .Values.priorityClassName }} 35 | {{- if .Values.spegel.containerdMirrorAdd }} 36 | initContainers: 37 | - name: configuration 38 | image: "{{ include "spegel.image" . }}" 39 | imagePullPolicy: {{ .Values.image.pullPolicy }} 40 | securityContext: 41 | {{- toYaml .Values.securityContext | nindent 12 }} 42 | args: 43 | - configuration 44 | - --log-level={{ .Values.spegel.logLevel }} 45 | - --containerd-registry-config-path={{ .Values.spegel.containerdRegistryConfigPath }} 46 | {{- with .Values.spegel.mirroredRegistries }} 47 | - --mirrored-registries 48 | {{- range . }} 49 | - {{ . | quote }} 50 | {{- end }} 51 | {{- end }} 52 | - --mirror-targets 53 | - http://$(NODE_IP):{{ .Values.service.registry.hostPort }} 54 | - http://$(NODE_IP):{{ .Values.service.registry.nodePort }} 55 | {{- with .Values.spegel.additionalMirrorTargets }} 56 | {{- range . }} 57 | - {{ . | quote }} 58 | {{- end }} 59 | {{- end }} 60 | - --resolve-tags={{ .Values.spegel.resolveTags }} 61 | - --prepend-existing={{ .Values.spegel.prependExisting }} 62 | env: 63 | - name: NODE_IP 64 | {{- include "networking.nodeIp" . | nindent 10 }} 65 | resources: 66 | {{- toYaml .Values.resources | nindent 10 }} 67 | volumeMounts: 68 | - name: containerd-config 69 | mountPath: {{ .Values.spegel.containerdRegistryConfigPath }} 70 | {{- if .Values.basicAuthSecretName }} 71 | - name: basic-auth 72 | mountPath: "/etc/secrets/basic-auth" 73 | readOnly: true 74 | {{- end }} 75 | {{- end }} 76 | containers: 77 | - name: registry 78 | image: "{{ include "spegel.image" . }}" 79 | imagePullPolicy: {{ .Values.image.pullPolicy }} 80 | securityContext: 81 | {{- toYaml .Values.securityContext | nindent 12 }} 82 | args: 83 | - registry 84 | - --log-level={{ .Values.spegel.logLevel }} 85 | - --mirror-resolve-retries={{ .Values.spegel.mirrorResolveRetries }} 86 | - --mirror-resolve-timeout={{ .Values.spegel.mirrorResolveTimeout }} 87 | - --registry-addr=:{{ .Values.service.registry.port }} 88 | - --router-addr=:{{ .Values.service.router.port }} 89 | - --metrics-addr=:{{ .Values.service.metrics.port }} 90 | {{- with .Values.spegel.mirroredRegistries }} 91 | - --mirrored-registries 92 | {{- range . }} 93 | - {{ . | quote }} 94 | {{- end }} 95 | {{- end }} 96 | - --containerd-sock={{ .Values.spegel.containerdSock }} 97 | - --containerd-namespace={{ .Values.spegel.containerdNamespace }} 98 | - --containerd-registry-config-path={{ .Values.spegel.containerdRegistryConfigPath }} 99 | - --bootstrap-kind=dns 100 | - --dns-bootstrap-domain={{ include "spegel.fullname" . }}-bootstrap.{{ include "spegel.namespace" . }}.svc.{{ .Values.clusterDomain }} 101 | - --resolve-latest-tag={{ .Values.spegel.resolveLatestTag }} 102 | {{- with .Values.spegel.containerdContentPath }} 103 | - --containerd-content-path={{ . }} 104 | {{- end }} 105 | - --debug-web-enabled={{ .Values.spegel.debugWebEnabled }} 106 | env: 107 | {{- if ((.Values.resources).limits).cpu }} 108 | - name: GOMAXPROCS 109 | valueFrom: 110 | resourceFieldRef: 111 | resource: limits.cpu 112 | divisor: 1 113 | {{- end }} 114 | {{- if ((.Values.resources).limits).memory }} 115 | - name: GOMEMLIMIT 116 | valueFrom: 117 | resourceFieldRef: 118 | resource: limits.memory 119 | divisor: 1 120 | {{- end }} 121 | - name: NODE_IP 122 | {{- include "networking.nodeIp" . | nindent 10 }} 123 | ports: 124 | - name: registry 125 | containerPort: {{ .Values.service.registry.port }} 126 | hostPort: {{ .Values.service.registry.hostPort }} 127 | protocol: TCP 128 | - name: router 129 | containerPort: {{ .Values.service.router.port }} 130 | protocol: TCP 131 | - name: metrics 132 | containerPort: {{ .Values.service.metrics.port }} 133 | protocol: TCP 134 | # Startup may take a bit longer on bootsrap as Pods need to find each other. 135 | # This is why the startup proben is a bit more forgiving, while hitting the endpoint more often. 136 | startupProbe: 137 | periodSeconds: 3 138 | failureThreshold: 60 139 | httpGet: 140 | path: /healthz 141 | port: registry 142 | readinessProbe: 143 | httpGet: 144 | path: /healthz 145 | port: registry 146 | volumeMounts: 147 | {{- if .Values.basicAuthSecretName }} 148 | - name: basic-auth 149 | mountPath: "/etc/secrets/basic-auth" 150 | readOnly: true 151 | {{- end }} 152 | - name: containerd-sock 153 | mountPath: {{ .Values.spegel.containerdSock }} 154 | {{- with .Values.spegel.containerdContentPath }} 155 | - name: containerd-content 156 | mountPath: {{ . }} 157 | readOnly: true 158 | {{- end }} 159 | resources: 160 | {{- toYaml .Values.resources | nindent 10 }} 161 | volumes: 162 | {{- with .Values.basicAuthSecretName }} 163 | - name: basic-auth 164 | secret: 165 | secretName: {{ . }} 166 | {{- end }} 167 | - name: containerd-sock 168 | hostPath: 169 | path: {{ .Values.spegel.containerdSock }} 170 | type: Socket 171 | {{- with .Values.spegel.containerdContentPath }} 172 | - name: containerd-content 173 | hostPath: 174 | path: {{ . }} 175 | type: Directory 176 | {{- end }} 177 | {{- if .Values.spegel.containerdMirrorAdd }} 178 | - name: containerd-config 179 | hostPath: 180 | path: {{ .Values.spegel.containerdRegistryConfigPath }} 181 | type: DirectoryOrCreate 182 | {{- end }} 183 | {{- with .Values.nodeSelector }} 184 | nodeSelector: 185 | {{- toYaml . | nindent 8 }} 186 | {{- end }} 187 | {{- with .Values.affinity }} 188 | affinity: 189 | {{- toYaml . | nindent 8 }} 190 | {{- end }} 191 | {{- with .Values.tolerations }} 192 | tolerations: 193 | {{- toYaml . | nindent 8 }} 194 | {{- end }} 195 | -------------------------------------------------------------------------------- /charts/spegel/templates/grafana-dashboard.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.grafanaDashboard.enabled }} 2 | apiVersion: v1 3 | kind: ConfigMap 4 | metadata: 5 | name: {{ include "spegel.fullname" . }}-dashboard 6 | namespace: {{ include "spegel.namespace" . }} 7 | labels: 8 | {{ .Values.grafanaDashboard.sidecarLabel }}: {{ .Values.grafanaDashboard.sidecarLabelValue | quote }} 9 | {{- include "spegel.labels" . | nindent 4 }} 10 | {{- with .Values.grafanaDashboard.annotations }} 11 | annotations: 12 | {{- toYaml . | nindent 4 }} 13 | {{- end }} 14 | data: 15 | spegel.json: |- 16 | {{ .Files.Get "monitoring/grafana-dashboard.json" | indent 6 }} 17 | {{- end }} 18 | -------------------------------------------------------------------------------- /charts/spegel/templates/post-delete-hook.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.spegel.containerdMirrorAdd }} 2 | apiVersion: apps/v1 3 | kind: DaemonSet 4 | metadata: 5 | name: {{ include "spegel.fullname" . }}-cleanup 6 | namespace: {{ include "spegel.namespace" . }} 7 | labels: 8 | app.kubernetes.io/component: cleanup 9 | {{- include "spegel.labels" . | nindent 4 }} 10 | annotations: 11 | helm.sh/hook: "post-delete" 12 | helm.sh/hook-delete-policy: "before-hook-creation, hook-succeeded" 13 | helm.sh/hook-weight: "0" 14 | spec: 15 | selector: 16 | matchLabels: 17 | app.kubernetes.io/component: cleanup 18 | {{- include "spegel.selectorLabels" . | nindent 6 }} 19 | template: 20 | metadata: 21 | labels: 22 | app.kubernetes.io/component: cleanup 23 | {{- include "spegel.selectorLabels" . | nindent 8 }} 24 | spec: 25 | {{- with .Values.imagePullSecrets }} 26 | imagePullSecrets: 27 | {{- toYaml . | nindent 8 }} 28 | {{- end }} 29 | securityContext: 30 | {{- toYaml .Values.podSecurityContext | nindent 8 }} 31 | priorityClassName: {{ .Values.priorityClassName }} 32 | containers: 33 | - name: cleanup 34 | image: "{{ include "spegel.image" . }}" 35 | imagePullPolicy: {{ .Values.image.pullPolicy }} 36 | args: 37 | - cleanup 38 | - --containerd-registry-config-path={{ .Values.spegel.containerdRegistryConfigPath }} 39 | - --addr=:{{ .Values.service.cleanup.port }} 40 | readinessProbe: 41 | httpGet: 42 | path: /healthz 43 | port: readiness 44 | ports: 45 | - name: readiness 46 | containerPort: {{ .Values.service.cleanup.port }} 47 | protocol: TCP 48 | volumeMounts: 49 | - name: containerd-config 50 | mountPath: {{ .Values.spegel.containerdRegistryConfigPath }} 51 | volumes: 52 | - name: containerd-config 53 | hostPath: 54 | path: {{ .Values.spegel.containerdRegistryConfigPath }} 55 | type: DirectoryOrCreate 56 | {{- with .Values.tolerations }} 57 | tolerations: 58 | {{- toYaml . | nindent 8 }} 59 | {{- end }} 60 | --- 61 | apiVersion: v1 62 | kind: Service 63 | metadata: 64 | name: {{ include "spegel.fullname" . }}-cleanup 65 | namespace: {{ include "spegel.namespace" . }} 66 | labels: 67 | app.kubernetes.io/component: cleanup 68 | {{- include "spegel.labels" . | nindent 4 }} 69 | annotations: 70 | helm.sh/hook: "post-delete" 71 | helm.sh/hook-delete-policy: "before-hook-creation, hook-succeeded" 72 | helm.sh/hook-weight: "0" 73 | spec: 74 | selector: 75 | app.kubernetes.io/component: cleanup 76 | {{- include "spegel.selectorLabels" . | nindent 4 }} 77 | clusterIP: None 78 | publishNotReadyAddresses: true 79 | ports: 80 | - name: readiness 81 | port: {{ .Values.service.cleanup.port }} 82 | protocol: TCP 83 | --- 84 | apiVersion: v1 85 | kind: Pod 86 | metadata: 87 | name: {{ include "spegel.fullname" . }}-cleanup-wait 88 | namespace: {{ include "spegel.namespace" . }} 89 | labels: 90 | app.kubernetes.io/component: cleanup-wait 91 | {{- include "spegel.labels" . | nindent 4 }} 92 | annotations: 93 | helm.sh/hook: "post-delete" 94 | helm.sh/hook-delete-policy: "before-hook-creation, hook-succeeded" 95 | helm.sh/hook-weight: "1" 96 | spec: 97 | containers: 98 | - name: cleanup-wait 99 | image: "{{ include "spegel.image" . }}" 100 | imagePullPolicy: {{ .Values.image.pullPolicy }} 101 | args: 102 | - cleanup-wait 103 | - --probe-endpoint={{ include "spegel.fullname" . }}-cleanup.{{ include "spegel.namespace" . }}.svc.{{ .Values.clusterDomain }}:{{ .Values.service.cleanup.port }} 104 | restartPolicy: Never 105 | terminationGracePeriodSeconds: 0 106 | {{- end }} 107 | -------------------------------------------------------------------------------- /charts/spegel/templates/rbac.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: {{ include "spegel.serviceAccountName" . }} 5 | namespace: {{ include "spegel.namespace" . }} 6 | labels: 7 | {{- include "spegel.labels" . | nindent 4 }} 8 | {{- with .Values.serviceAccount.annotations }} 9 | annotations: 10 | {{- toYaml . | nindent 4 }} 11 | {{- end }} 12 | -------------------------------------------------------------------------------- /charts/spegel/templates/service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: {{ include "spegel.fullname" . }} 5 | namespace: {{ include "spegel.namespace" . }} 6 | labels: 7 | app.kubernetes.io/component: metrics 8 | {{- include "spegel.labels" . | nindent 4 }} 9 | spec: 10 | selector: 11 | {{- include "spegel.selectorLabels" . | nindent 4 }} 12 | ports: 13 | - name: metrics 14 | port: {{ .Values.service.metrics.port }} 15 | targetPort: metrics 16 | protocol: TCP 17 | --- 18 | apiVersion: v1 19 | kind: Service 20 | metadata: 21 | name: {{ include "spegel.fullname" . }}-registry 22 | namespace: {{ include "spegel.namespace" . }} 23 | labels: 24 | {{- include "spegel.labels" . | nindent 4 }} 25 | {{- if .Values.service.registry.topologyAwareHintsEnabled }} 26 | annotations: 27 | service.kubernetes.io/topology-mode: "auto" 28 | {{- end }} 29 | spec: 30 | type: NodePort 31 | selector: 32 | {{- include "spegel.selectorLabels" . | nindent 4 }} 33 | ports: 34 | - name: registry 35 | port: {{ .Values.service.registry.port }} 36 | targetPort: registry 37 | nodePort: {{ .Values.service.registry.nodePort }} 38 | protocol: TCP 39 | --- 40 | apiVersion: v1 41 | kind: Service 42 | metadata: 43 | name: {{ include "spegel.fullname" . }}-bootstrap 44 | namespace: {{ include "spegel.namespace" . }} 45 | labels: 46 | {{- include "spegel.labels" . | nindent 4 }} 47 | spec: 48 | selector: 49 | {{- include "spegel.selectorLabels" . | nindent 4 }} 50 | clusterIP: None 51 | publishNotReadyAddresses: true 52 | ports: 53 | - name: router 54 | port: {{ .Values.service.router.port }} 55 | protocol: TCP 56 | -------------------------------------------------------------------------------- /charts/spegel/templates/servicemonitor.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.serviceMonitor.enabled }} 2 | apiVersion: monitoring.coreos.com/v1 3 | kind: ServiceMonitor 4 | metadata: 5 | name: {{ include "spegel.fullname" . }} 6 | namespace: {{ include "spegel.namespace" . }} 7 | labels: 8 | {{- include "spegel.labels" . | nindent 4 }} 9 | {{- if .Values.serviceMonitor.labels -}} 10 | {{ toYaml .Values.serviceMonitor.labels | nindent 4}} 11 | {{- end }} 12 | spec: 13 | selector: 14 | matchLabels: 15 | app.kubernetes.io/component: metrics 16 | {{- include "spegel.selectorLabels" . | nindent 6 }} 17 | endpoints: 18 | - port: metrics 19 | interval: {{ .Values.serviceMonitor.interval }} 20 | scrapeTimeout: {{ .Values.serviceMonitor.scrapeTimeout }} 21 | {{- with .Values.serviceMonitor.relabelings }} 22 | relabelings: 23 | {{- toYaml . | nindent 8 }} 24 | {{- end }} 25 | {{- with .Values.serviceMonitor.metricRelabelings }} 26 | metricRelabelings: 27 | {{- toYaml . | nindent 8 }} 28 | {{- end }} 29 | {{- end }} 30 | -------------------------------------------------------------------------------- /charts/spegel/templates/verticalpodautoscaler.yaml: -------------------------------------------------------------------------------- 1 | {{- if and (.Capabilities.APIVersions.Has "autoscaling.k8s.io/v1") (.Values.verticalPodAutoscaler.enabled) }} 2 | apiVersion: autoscaling.k8s.io/v1 3 | kind: VerticalPodAutoscaler 4 | metadata: 5 | name: {{ include "spegel.fullname" . }} 6 | namespace: {{ include "spegel.namespace" . }} 7 | labels: 8 | {{- include "spegel.labels" . | nindent 4 }} 9 | spec: 10 | {{- with .Values.verticalPodAutoscaler.recommenders }} 11 | recommenders: 12 | {{- toYaml . | nindent 4 }} 13 | {{- end }} 14 | resourcePolicy: 15 | containerPolicies: 16 | - containerName: registry 17 | {{- with .Values.verticalPodAutoscaler.controlledResources }} 18 | controlledResources: 19 | {{- toYaml . | nindent 8 }} 20 | {{- end }} 21 | {{- if .Values.verticalPodAutoscaler.controlledValues }} 22 | controlledValues: {{ .Values.verticalPodAutoscaler.controlledValues }} 23 | {{- end }} 24 | {{- if .Values.verticalPodAutoscaler.maxAllowed }} 25 | maxAllowed: 26 | {{- toYaml .Values.verticalPodAutoscaler.maxAllowed | nindent 8 }} 27 | {{- end }} 28 | {{- if .Values.verticalPodAutoscaler.minAllowed }} 29 | minAllowed: 30 | {{- toYaml .Values.verticalPodAutoscaler.minAllowed | nindent 8 }} 31 | {{- end }} 32 | targetRef: 33 | apiVersion: apps/v1 34 | kind: DaemonSet 35 | name: {{ include "spegel.fullname" . }} 36 | {{- with .Values.verticalPodAutoscaler.updatePolicy }} 37 | updatePolicy: 38 | {{- toYaml . | nindent 4 }} 39 | {{- end }} 40 | {{- end }} 41 | -------------------------------------------------------------------------------- /charts/spegel/values.yaml: -------------------------------------------------------------------------------- 1 | image: 2 | # -- Image repository. 3 | repository: ghcr.io/spegel-org/spegel 4 | # -- Image Pull Policy. 5 | pullPolicy: IfNotPresent 6 | # -- Overrides the image tag whose default is the chart appVersion. 7 | tag: "" 8 | # -- Image digest. 9 | digest: "" 10 | 11 | # -- Image Pull Secrets 12 | imagePullSecrets: [] 13 | # -- Overrides the name of the chart. 14 | nameOverride: "" 15 | # -- Overrides the full name of the chart. 16 | fullnameOverride: "" 17 | # -- Overrides the namespace where spegel resources are installed. 18 | namespaceOverride: "" 19 | 20 | serviceAccount: 21 | # -- Annotations to add to the service account 22 | annotations: {} 23 | # -- The name of the service account to use. 24 | # If not set and create is true, a name is generated using the fullname template. 25 | name: "" 26 | 27 | # -- Annotations to add to the pod. 28 | podAnnotations: {} 29 | 30 | # -- Security context for the pod. 31 | podSecurityContext: {} 32 | # fsGroup: 2000 33 | 34 | # -- The number of old history to retain to allow rollback. 35 | revisionHistoryLimit: 10 36 | 37 | # -- Security context for the Spegel container. 38 | securityContext: {} 39 | # capabilities: 40 | # drop: 41 | # - ALL 42 | # readOnlyRootFilesystem: true 43 | # runAsNonRoot: true 44 | # runAsUser: 1000 45 | 46 | service: 47 | registry: 48 | # -- Override the NODE_ID environment variable. It defaults to the field status.hostIP 49 | nodeIp: "" 50 | # -- Port to expose the registry via the service. 51 | port: 5000 52 | # -- Node port to expose the registry via the service. 53 | nodePort: 30021 54 | # -- Local host port to expose the registry. 55 | hostPort: 30020 56 | # -- If true adds topology aware hints annotation to node port service. 57 | topologyAwareHintsEnabled: true 58 | router: 59 | # -- Port to expose the router via the service. 60 | port: 5001 61 | metrics: 62 | # -- Port to expose the metrics via the service. 63 | port: 9090 64 | cleanup: 65 | # -- Port to expose cleanup probe on. 66 | port: 8080 67 | 68 | # -- Resource requests and limits for the Spegel container. 69 | resources: 70 | requests: 71 | memory: 128Mi 72 | limits: 73 | memory: 128Mi 74 | 75 | # -- Node selector for pod assignment. 76 | nodeSelector: 77 | kubernetes.io/os: linux 78 | 79 | # -- An update strategy to replace existing pods with new pods. 80 | updateStrategy: {} 81 | # type: RollingUpdate 82 | # rollingUpdate: 83 | # maxSurge: 0 84 | # maxUnavailable: 1 85 | 86 | # -- Tolerations for pod assignment. 87 | tolerations: 88 | - key: CriticalAddonsOnly 89 | operator: Exists 90 | - effect: NoExecute 91 | operator: Exists 92 | - effect: NoSchedule 93 | operator: Exists 94 | 95 | # -- Affinity settings for pod assignment. 96 | affinity: {} 97 | 98 | # -- Common labels to apply to all rendered resources. 99 | commonLabels: {} 100 | 101 | # -- Domain configured for service domain names. 102 | clusterDomain: cluster.local. 103 | 104 | serviceMonitor: 105 | # -- If true creates a Prometheus Service Monitor. 106 | enabled: false 107 | # -- Prometheus scrape interval. 108 | interval: 60s 109 | # -- Prometheus scrape interval timeout. 110 | scrapeTimeout: 30s 111 | # -- Service monitor specific labels for prometheus to discover servicemonitor. 112 | labels: {} 113 | # -- List of relabeling rules to apply the target’s metadata labels. 114 | relabelings: [] 115 | # -- List of relabeling rules to apply to the samples before ingestion. 116 | metricRelabelings: [] 117 | 118 | grafanaDashboard: 119 | # -- If true creates a Grafana dashboard. 120 | enabled: false 121 | # -- Label that ConfigMaps should have to be loaded as dashboards. 122 | sidecarLabel: "grafana_dashboard" 123 | # -- Label value that ConfigMaps should have to be loaded as dashboards. 124 | sidecarLabelValue: "1" 125 | # -- Annotations that ConfigMaps can have to get configured in Grafana, 126 | # See: sidecar.dashboards.folderAnnotation for specifying the dashboard folder. 127 | # https://github.com/grafana/helm-charts/tree/main/charts/grafana 128 | annotations: {} 129 | 130 | # -- Priority class name to use for the pod. 131 | priorityClassName: system-node-critical 132 | 133 | # -- Name of secret containing basic authentication credentials for registry. 134 | basicAuthSecretName: "" 135 | 136 | spegel: 137 | # -- Minimum log level to output. Value should be DEBUG, INFO, WARN, or ERROR. 138 | logLevel: "INFO" 139 | # -- Registries for which mirror configuration will be created. Empty means all registires will be mirrored. 140 | mirroredRegistries: [] 141 | # - https://docker.io 142 | # - https://ghcr.io 143 | # -- Additional target mirror registries other than Spegel. 144 | additionalMirrorTargets: [] 145 | # -- Max amount of mirrors to attempt. 146 | mirrorResolveRetries: 3 147 | # -- Max duration spent finding a mirror. 148 | mirrorResolveTimeout: "20ms" 149 | # -- Path to Containerd socket. 150 | containerdSock: "/run/containerd/containerd.sock" 151 | # -- Containerd namespace where images are stored. 152 | containerdNamespace: "k8s.io" 153 | # -- Path to Containerd mirror configuration. 154 | containerdRegistryConfigPath: "/etc/containerd/certs.d" 155 | # -- Path to Containerd content store.. 156 | containerdContentPath: "/var/lib/containerd/io.containerd.content.v1.content" 157 | # -- If true Spegel will add mirror configuration to the node. 158 | containerdMirrorAdd: true 159 | # -- When true Spegel will resolve tags to digests. 160 | resolveTags: true 161 | # -- When true latest tags will be resolved to digests. 162 | resolveLatestTag: true 163 | # -- When true existing mirror configuration will be kept and Spegel will prepend it's configuration. 164 | prependExisting: false 165 | # -- When true enables debug web page. 166 | debugWebEnabled: false 167 | 168 | verticalPodAutoscaler: 169 | # -- If true creates a Vertical Pod Autoscaler. 170 | enabled: false 171 | 172 | # -- Recommender responsible for generating recommendation for the object. 173 | # List should be empty (then the default recommender will generate the recommendation) 174 | # or contain exactly one recommender. 175 | recommenders: [] 176 | # - name: custom-recommender-performance 177 | 178 | # -- List of resources that the vertical pod autoscaler can control. Defaults to cpu and memory 179 | controlledResources: [] 180 | # -- Specifies which resource values should be controlled: RequestsOnly or RequestsAndLimits. 181 | controlledValues: RequestsAndLimits 182 | 183 | # -- Define the max allowed resources for the pod 184 | maxAllowed: {} 185 | # cpu: 100m 186 | # memory: 128Mi 187 | # -- Define the min allowed resources for the pod 188 | minAllowed: {} 189 | # cpu: 100m 190 | # memory: 128Mi 191 | 192 | updatePolicy: 193 | # -- Specifies minimal number of replicas which need to be alive for VPA Updater to attempt pod eviction 194 | minReplicas: 2 195 | 196 | # -- Specifies whether recommended updates are applied when a Pod is started and whether recommended updates 197 | # are applied during the life of a Pod. Possible values are "Off", "Initial", "Recreate", and "Auto". 198 | updateMode: Auto 199 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/spegel-org/spegel 2 | 3 | go 1.24.1 4 | 5 | require ( 6 | github.com/alexflint/go-arg v1.5.1 7 | github.com/containerd/containerd/api v1.8.0 8 | github.com/containerd/containerd/v2 v2.0.5 9 | github.com/containerd/errdefs v1.0.0 10 | github.com/containerd/typeurl/v2 v2.2.3 11 | github.com/go-logr/logr v1.4.2 12 | github.com/ipfs/go-cid v0.5.0 13 | github.com/libp2p/go-libp2p v0.41.1 14 | github.com/libp2p/go-libp2p-kad-dht v0.31.0 15 | github.com/multiformats/go-multiaddr v0.15.0 16 | github.com/multiformats/go-multicodec v0.9.0 17 | github.com/multiformats/go-multihash v0.2.3 18 | github.com/opencontainers/go-digest v1.0.0 19 | github.com/opencontainers/image-spec v1.1.1 20 | github.com/pelletier/go-toml/v2 v2.2.4 21 | github.com/prometheus/client_golang v1.22.0 22 | github.com/prometheus/common v0.63.0 23 | github.com/spf13/afero v1.14.0 24 | github.com/stretchr/testify v1.10.0 25 | go.etcd.io/bbolt v1.4.0 26 | golang.org/x/sync v0.13.0 27 | google.golang.org/grpc v1.72.0 28 | k8s.io/apimachinery v0.32.4 29 | k8s.io/cri-api v0.32.4 30 | k8s.io/klog/v2 v2.130.1 31 | ) 32 | 33 | require ( 34 | dario.cat/mergo v1.0.1 // indirect 35 | github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 // indirect 36 | github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20231105174938-2b5cbb29f3e2 // indirect 37 | github.com/Masterminds/goutils v1.1.1 // indirect 38 | github.com/Masterminds/semver/v3 v3.3.1 // indirect 39 | github.com/Masterminds/sprig/v3 v3.3.0 // indirect 40 | github.com/Microsoft/go-winio v0.6.2 // indirect 41 | github.com/Microsoft/hcsshim v0.12.9 // indirect 42 | github.com/alexflint/go-scalar v1.2.0 // indirect 43 | github.com/benbjohnson/clock v1.3.5 // indirect 44 | github.com/beorn7/perks v1.0.1 // indirect 45 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 46 | github.com/containerd/cgroups v1.1.0 // indirect 47 | github.com/containerd/cgroups/v3 v3.0.3 // indirect 48 | github.com/containerd/continuity v0.4.4 // indirect 49 | github.com/containerd/errdefs/pkg v0.3.0 // indirect 50 | github.com/containerd/fifo v1.1.0 // indirect 51 | github.com/containerd/log v0.1.0 // indirect 52 | github.com/containerd/platforms v1.0.0-rc.1 // indirect 53 | github.com/containerd/plugin v1.0.0 // indirect 54 | github.com/containerd/ttrpc v1.2.7 // indirect 55 | github.com/coreos/go-systemd/v22 v22.5.0 // indirect 56 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect 57 | github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect 58 | github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 // indirect 59 | github.com/distribution/reference v0.6.0 // indirect 60 | github.com/docker/go-units v0.5.0 // indirect 61 | github.com/elastic/gosigar v0.14.3 // indirect 62 | github.com/felixge/httpsnoop v1.0.4 // indirect 63 | github.com/flynn/noise v1.1.0 // indirect 64 | github.com/francoispqt/gojay v1.2.13 // indirect 65 | github.com/fsnotify/fsnotify v1.7.0 // indirect 66 | github.com/go-logr/stdr v1.2.2 // indirect 67 | github.com/go-task/slim-sprig/v3 v3.0.0 // indirect 68 | github.com/gobwas/glob v0.2.3 // indirect 69 | github.com/godbus/dbus/v5 v5.1.0 // indirect 70 | github.com/gogo/protobuf v1.3.2 // indirect 71 | github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect 72 | github.com/google/go-cmp v0.7.0 // indirect 73 | github.com/google/gopacket v1.1.19 // indirect 74 | github.com/google/pprof v0.0.0-20250208200701-d0013a598941 // indirect 75 | github.com/google/uuid v1.6.0 // indirect 76 | github.com/gorilla/websocket v1.5.3 // indirect 77 | github.com/hashicorp/golang-lru v1.0.2 // indirect 78 | github.com/hashicorp/hcl v1.0.0 // indirect 79 | github.com/huandu/xstrings v1.5.0 // indirect 80 | github.com/huin/goupnp v1.3.0 // indirect 81 | github.com/inconshreveable/mousetrap v1.1.0 // indirect 82 | github.com/ipfs/boxo v0.29.1 // indirect 83 | github.com/ipfs/go-datastore v0.8.2 // indirect 84 | github.com/ipfs/go-log v1.0.5 // indirect 85 | github.com/ipfs/go-log/v2 v2.5.1 // indirect 86 | github.com/ipld/go-ipld-prime v0.21.0 // indirect 87 | github.com/jackpal/go-nat-pmp v1.0.2 // indirect 88 | github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect 89 | github.com/klauspost/compress v1.18.0 // indirect 90 | github.com/klauspost/cpuid/v2 v2.2.10 // indirect 91 | github.com/koron/go-ssdp v0.0.5 // indirect 92 | github.com/libp2p/go-buffer-pool v0.1.0 // indirect 93 | github.com/libp2p/go-cidranger v1.1.0 // indirect 94 | github.com/libp2p/go-flow-metrics v0.2.0 // indirect 95 | github.com/libp2p/go-libp2p-asn-util v0.4.1 // indirect 96 | github.com/libp2p/go-libp2p-kbucket v0.7.0 // indirect 97 | github.com/libp2p/go-libp2p-record v0.3.1 // indirect 98 | github.com/libp2p/go-libp2p-routing-helpers v0.7.5 // indirect 99 | github.com/libp2p/go-msgio v0.3.0 // indirect 100 | github.com/libp2p/go-netroute v0.2.2 // indirect 101 | github.com/libp2p/go-reuseport v0.4.0 // indirect 102 | github.com/libp2p/go-yamux/v5 v5.0.0 // indirect 103 | github.com/magiconair/properties v1.8.7 // indirect 104 | github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect 105 | github.com/mattn/go-isatty v0.0.20 // indirect 106 | github.com/miekg/dns v1.1.63 // indirect 107 | github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect 108 | github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect 109 | github.com/minio/sha256-simd v1.0.1 // indirect 110 | github.com/mitchellh/copystructure v1.2.0 // indirect 111 | github.com/mitchellh/mapstructure v1.5.0 // indirect 112 | github.com/mitchellh/reflectwalk v1.0.2 // indirect 113 | github.com/moby/locker v1.0.1 // indirect 114 | github.com/moby/sys/mountinfo v0.7.2 // indirect 115 | github.com/moby/sys/sequential v0.6.0 // indirect 116 | github.com/moby/sys/signal v0.7.1 // indirect 117 | github.com/moby/sys/user v0.3.0 // indirect 118 | github.com/moby/sys/userns v0.1.0 // indirect 119 | github.com/mr-tron/base58 v1.2.0 // indirect 120 | github.com/multiformats/go-base32 v0.1.0 // indirect 121 | github.com/multiformats/go-base36 v0.2.0 // indirect 122 | github.com/multiformats/go-multiaddr-dns v0.4.1 // indirect 123 | github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect 124 | github.com/multiformats/go-multibase v0.2.0 // indirect 125 | github.com/multiformats/go-multistream v0.6.0 // indirect 126 | github.com/multiformats/go-varint v0.0.7 // indirect 127 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 128 | github.com/norwoodj/helm-docs v1.14.2 // indirect 129 | github.com/onsi/ginkgo/v2 v2.22.2 // indirect 130 | github.com/opencontainers/runtime-spec v1.2.0 // indirect 131 | github.com/opencontainers/selinux v1.11.1 // indirect 132 | github.com/opentracing/opentracing-go v1.2.0 // indirect 133 | github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect 134 | github.com/pion/datachannel v1.5.10 // indirect 135 | github.com/pion/dtls/v2 v2.2.12 // indirect 136 | github.com/pion/dtls/v3 v3.0.4 // indirect 137 | github.com/pion/ice/v4 v4.0.8 // indirect 138 | github.com/pion/interceptor v0.1.37 // indirect 139 | github.com/pion/logging v0.2.3 // indirect 140 | github.com/pion/mdns/v2 v2.0.7 // indirect 141 | github.com/pion/randutil v0.1.0 // indirect 142 | github.com/pion/rtcp v1.2.15 // indirect 143 | github.com/pion/rtp v1.8.11 // indirect 144 | github.com/pion/sctp v1.8.37 // indirect 145 | github.com/pion/sdp/v3 v3.0.10 // indirect 146 | github.com/pion/srtp/v3 v3.0.4 // indirect 147 | github.com/pion/stun v0.6.1 // indirect 148 | github.com/pion/stun/v3 v3.0.0 // indirect 149 | github.com/pion/transport/v2 v2.2.10 // indirect 150 | github.com/pion/transport/v3 v3.0.7 // indirect 151 | github.com/pion/turn/v4 v4.0.0 // indirect 152 | github.com/pion/webrtc/v4 v4.0.10 // indirect 153 | github.com/pkg/errors v0.9.1 // indirect 154 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect 155 | github.com/polydawn/refmt v0.89.0 // indirect 156 | github.com/prometheus/client_model v0.6.1 // indirect 157 | github.com/prometheus/procfs v0.15.1 // indirect 158 | github.com/quic-go/qpack v0.5.1 // indirect 159 | github.com/quic-go/quic-go v0.50.1 // indirect 160 | github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66 // indirect 161 | github.com/raulk/go-watchdog v1.3.0 // indirect 162 | github.com/shopspring/decimal v1.4.0 // indirect 163 | github.com/sirupsen/logrus v1.9.3 // indirect 164 | github.com/spaolacci/murmur3 v1.1.0 // indirect 165 | github.com/spf13/cast v1.7.0 // indirect 166 | github.com/spf13/cobra v1.8.1 // indirect 167 | github.com/spf13/jwalterweatherman v1.1.0 // indirect 168 | github.com/spf13/pflag v1.0.6 // indirect 169 | github.com/spf13/viper v1.16.0 // indirect 170 | github.com/subosito/gotenv v1.4.2 // indirect 171 | github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect 172 | github.com/wlynxg/anet v0.0.5 // indirect 173 | go.opencensus.io v0.24.0 // indirect 174 | go.opentelemetry.io/auto/sdk v1.1.0 // indirect 175 | go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.56.0 // indirect 176 | go.opentelemetry.io/otel v1.34.0 // indirect 177 | go.opentelemetry.io/otel/metric v1.34.0 // indirect 178 | go.opentelemetry.io/otel/trace v1.34.0 // indirect 179 | go.uber.org/dig v1.18.0 // indirect 180 | go.uber.org/fx v1.23.0 // indirect 181 | go.uber.org/mock v0.5.0 // indirect 182 | go.uber.org/multierr v1.11.0 // indirect 183 | go.uber.org/zap v1.27.0 // indirect 184 | golang.org/x/crypto v0.36.0 // indirect 185 | golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 // indirect 186 | golang.org/x/mod v0.24.0 // indirect 187 | golang.org/x/net v0.38.0 // indirect 188 | golang.org/x/sys v0.31.0 // indirect 189 | golang.org/x/text v0.23.0 // indirect 190 | golang.org/x/time v0.8.0 // indirect 191 | golang.org/x/tools v0.31.0 // indirect 192 | gonum.org/v1/gonum v0.15.1 // indirect 193 | google.golang.org/genproto/googleapis/rpc v0.0.0-20250218202821-56aae31c358a // indirect 194 | google.golang.org/protobuf v1.36.6 // indirect 195 | gopkg.in/ini.v1 v1.67.0 // indirect 196 | gopkg.in/yaml.v3 v3.0.1 // indirect 197 | helm.sh/helm/v3 v3.17.3 // indirect 198 | lukechampine.com/blake3 v1.4.0 // indirect 199 | ) 200 | 201 | tool github.com/norwoodj/helm-docs/cmd/helm-docs 202 | -------------------------------------------------------------------------------- /internal/channel/channel.go: -------------------------------------------------------------------------------- 1 | package channel 2 | 3 | import ( 4 | "sync" 5 | ) 6 | 7 | func Merge[T any](cs ...<-chan T) <-chan T { 8 | var wg sync.WaitGroup 9 | out := make(chan T) 10 | 11 | output := func(c <-chan T) { 12 | for n := range c { 13 | out <- n 14 | } 15 | wg.Done() 16 | } 17 | wg.Add(len(cs)) 18 | for _, c := range cs { 19 | go output(c) 20 | } 21 | 22 | go func() { 23 | wg.Wait() 24 | close(out) 25 | }() 26 | return out 27 | } 28 | -------------------------------------------------------------------------------- /internal/mux/mux.go: -------------------------------------------------------------------------------- 1 | package mux 2 | 3 | import ( 4 | "errors" 5 | "net/http" 6 | ) 7 | 8 | type Handler func(rw ResponseWriter, req *http.Request) 9 | 10 | type ServeMux struct { 11 | h Handler 12 | } 13 | 14 | func NewServeMux(h Handler) (*ServeMux, error) { 15 | if h == nil { 16 | return nil, errors.New("handler cannot be nil") 17 | } 18 | return &ServeMux{h: h}, nil 19 | } 20 | 21 | func (s *ServeMux) ServeHTTP(rw http.ResponseWriter, req *http.Request) { 22 | s.h(&response{ResponseWriter: rw}, req) 23 | } 24 | -------------------------------------------------------------------------------- /internal/mux/mux_test.go: -------------------------------------------------------------------------------- 1 | package mux 2 | 3 | import ( 4 | "net/http" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/require" 8 | ) 9 | 10 | func TestServeMux(t *testing.T) { 11 | t.Parallel() 12 | 13 | m, err := NewServeMux(nil) 14 | require.Nil(t, m) 15 | require.EqualError(t, err, "handler cannot be nil") 16 | 17 | handlerCalled := false 18 | h := func(rw ResponseWriter, req *http.Request) { 19 | handlerCalled = true 20 | } 21 | m, err = NewServeMux(h) 22 | require.NoError(t, err) 23 | m.ServeHTTP(nil, nil) 24 | require.True(t, handlerCalled) 25 | } 26 | -------------------------------------------------------------------------------- /internal/mux/response.go: -------------------------------------------------------------------------------- 1 | package mux 2 | 3 | import ( 4 | "bufio" 5 | "io" 6 | "net" 7 | "net/http" 8 | ) 9 | 10 | type ResponseWriter interface { 11 | http.ResponseWriter 12 | WriteError(statusCode int, err error) 13 | Error() error 14 | Status() int 15 | Size() int64 16 | } 17 | 18 | var ( 19 | _ http.ResponseWriter = &response{} 20 | _ http.Flusher = &response{} 21 | _ http.Hijacker = &response{} 22 | _ io.ReaderFrom = &response{} 23 | ) 24 | 25 | type response struct { 26 | http.ResponseWriter 27 | error error 28 | status int 29 | size int64 30 | writtenHeader bool 31 | } 32 | 33 | func (r *response) WriteHeader(statusCode int) { 34 | if !r.writtenHeader { 35 | r.writtenHeader = true 36 | r.status = statusCode 37 | } 38 | r.ResponseWriter.WriteHeader(statusCode) 39 | } 40 | 41 | func (r *response) Write(b []byte) (int, error) { 42 | r.writtenHeader = true 43 | n, err := r.ResponseWriter.Write(b) 44 | r.size += int64(n) 45 | return n, err 46 | } 47 | 48 | func (r *response) WriteError(statusCode int, err error) { 49 | r.error = err 50 | r.WriteHeader(statusCode) 51 | } 52 | 53 | func (r *response) Flush() { 54 | r.writtenHeader = true 55 | //nolint: errcheck // No method to throw the error. 56 | flusher := r.ResponseWriter.(http.Flusher) 57 | flusher.Flush() 58 | } 59 | 60 | func (r *response) Hijack() (net.Conn, *bufio.ReadWriter, error) { 61 | //nolint: errcheck // No method to throw the error. 62 | hijacker := r.ResponseWriter.(http.Hijacker) 63 | return hijacker.Hijack() 64 | } 65 | 66 | func (r *response) ReadFrom(rd io.Reader) (int64, error) { 67 | n, err := io.Copy(r.ResponseWriter, rd) 68 | r.size += n 69 | return n, err 70 | } 71 | 72 | func (r *response) Unwrap() http.ResponseWriter { 73 | return r.ResponseWriter 74 | } 75 | 76 | func (r *response) Status() int { 77 | if r.status == 0 { 78 | return http.StatusOK 79 | } 80 | return r.status 81 | } 82 | 83 | func (r *response) Error() error { 84 | return r.error 85 | } 86 | 87 | func (r *response) Size() int64 { 88 | return r.size 89 | } 90 | -------------------------------------------------------------------------------- /internal/mux/response_test.go: -------------------------------------------------------------------------------- 1 | package mux 2 | 3 | import ( 4 | "errors" 5 | "io" 6 | "net/http" 7 | "net/http/httptest" 8 | "strings" 9 | "testing" 10 | 11 | "github.com/stretchr/testify/require" 12 | ) 13 | 14 | func TestResponseWriter(t *testing.T) { 15 | t.Parallel() 16 | 17 | var httpRw http.ResponseWriter = &response{} 18 | _, ok := httpRw.(io.ReaderFrom) 19 | require.True(t, ok) 20 | 21 | httpRw = httptest.NewRecorder() 22 | rw := &response{ 23 | ResponseWriter: httpRw, 24 | } 25 | require.Equal(t, httpRw, rw.Unwrap()) 26 | require.NoError(t, rw.Error()) 27 | require.Equal(t, int64(0), rw.Size()) 28 | require.Equal(t, http.StatusOK, rw.Status()) 29 | 30 | rw = &response{ 31 | ResponseWriter: httptest.NewRecorder(), 32 | } 33 | rw.WriteHeader(http.StatusNotFound) 34 | require.True(t, rw.writtenHeader) 35 | require.Equal(t, http.StatusNotFound, rw.Status()) 36 | rw.WriteHeader(http.StatusBadGateway) 37 | require.Equal(t, http.StatusNotFound, rw.Status()) 38 | _, err := rw.Write([]byte("foo")) 39 | require.NoError(t, err) 40 | require.Equal(t, http.StatusNotFound, rw.Status()) 41 | 42 | rw = &response{ 43 | ResponseWriter: httptest.NewRecorder(), 44 | } 45 | err = errors.New("some server error") 46 | rw.WriteError(http.StatusInternalServerError, err) 47 | require.Equal(t, err, rw.Error()) 48 | require.Equal(t, http.StatusInternalServerError, rw.Status()) 49 | 50 | rw = &response{ 51 | ResponseWriter: httptest.NewRecorder(), 52 | } 53 | first := "hello world" 54 | n, err := rw.Write([]byte(first)) 55 | require.Equal(t, http.StatusOK, rw.Status()) 56 | require.NoError(t, err) 57 | require.Equal(t, len(first), n) 58 | require.Equal(t, int64(len(first)), rw.Size()) 59 | second := "foo bar" 60 | n, err = rw.Write([]byte(second)) 61 | require.NoError(t, err) 62 | require.Equal(t, len(second), n) 63 | require.Equal(t, int64(len(first)+len(second)), rw.Size()) 64 | 65 | rw = &response{ 66 | ResponseWriter: httptest.NewRecorder(), 67 | } 68 | r := strings.NewReader("reader") 69 | readFromN, err := rw.ReadFrom(r) 70 | require.NoError(t, err) 71 | require.Equal(t, r.Size(), readFromN) 72 | require.Equal(t, r.Size(), rw.Size()) 73 | } 74 | -------------------------------------------------------------------------------- /internal/web/templates/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | Spegel Debug 8 | 9 | 10 | 11 | 105 | 106 | 107 | 108 |
109 |

Spegel

110 | 111 |
112 | 113 |
114 |

Measure Image Pull

115 |
116 | 117 | 118 |
119 |
120 |
121 |
122 | 123 | 124 | -------------------------------------------------------------------------------- /internal/web/templates/measure.html: -------------------------------------------------------------------------------- 1 | {{ if .PeerResults }} 2 |

Resolved Peers

3 |
4 | 5 | 6 | 7 | 8 | 9 | 10 | {{ range .PeerResults }} 11 | 12 | 13 | 14 | 15 | {{ end }} 16 |
PeerDuration
{{ .Peer.Addr }}{{ .Duration }}
17 |
18 | 19 |

Result

20 |
21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | {{ range .PullResults }} 29 | 30 | 31 | 32 | 33 | 34 | 35 | {{ end }} 36 |
IdentifierTypeSizeDuration
{{ .Identifier }}{{ .ContentType }}{{ .ContentLength }}{{ .Duration }}
37 |
38 | {{ else }} 39 |

No peers found for image

40 | {{ end }} -------------------------------------------------------------------------------- /internal/web/templates/stats.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |
4 |
Images
5 |
{{ .ImageCount }}
6 |
7 |
8 |
Layers
9 |
{{ .LayerCount }}
10 |
11 |
12 |
-------------------------------------------------------------------------------- /internal/web/web.go: -------------------------------------------------------------------------------- 1 | package web 2 | 3 | import ( 4 | "embed" 5 | "encoding/json" 6 | "errors" 7 | "fmt" 8 | "html/template" 9 | "io" 10 | "net" 11 | "net/http" 12 | "net/netip" 13 | "net/url" 14 | "runtime" 15 | "strconv" 16 | "time" 17 | 18 | "github.com/containerd/containerd/v2/core/images" 19 | "github.com/go-logr/logr" 20 | ocispec "github.com/opencontainers/image-spec/specs-go/v1" 21 | "github.com/prometheus/common/expfmt" 22 | 23 | "github.com/spegel-org/spegel/pkg/oci" 24 | "github.com/spegel-org/spegel/pkg/routing" 25 | ) 26 | 27 | //go:embed templates/* 28 | var templatesFS embed.FS 29 | 30 | type Web struct { 31 | router routing.Router 32 | client *http.Client 33 | tmpls *template.Template 34 | } 35 | 36 | func NewWeb(router routing.Router) (*Web, error) { 37 | tmpls, err := template.New("").ParseFS(templatesFS, "templates/*") 38 | if err != nil { 39 | return nil, err 40 | } 41 | return &Web{ 42 | router: router, 43 | client: &http.Client{}, 44 | tmpls: tmpls, 45 | }, nil 46 | } 47 | 48 | func (w *Web) Handler(log logr.Logger) http.Handler { 49 | log = log.WithName("web") 50 | handlers := map[string]func(*http.Request) (string, any, error){ 51 | "/debug/web/": func(r *http.Request) (string, any, error) { 52 | return "index", nil, nil 53 | }, 54 | "/debug/web/stats": w.stats, 55 | "/debug/web/measure": w.measure, 56 | } 57 | return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { 58 | h, ok := handlers[req.URL.Path] 59 | if !ok { 60 | rw.WriteHeader(http.StatusNotFound) 61 | return 62 | } 63 | t, data, err := h(req) 64 | if err != nil { 65 | log.Error(err, "error when running handler", "path", req.URL.Path) 66 | rw.WriteHeader(http.StatusInternalServerError) 67 | return 68 | } 69 | err = w.tmpls.ExecuteTemplate(rw, t+".html", data) 70 | if err != nil { 71 | log.Error(err, "error rendering page", "path", req.URL.Path) 72 | rw.WriteHeader(http.StatusInternalServerError) 73 | return 74 | } 75 | }) 76 | } 77 | 78 | func (w *Web) stats(req *http.Request) (string, any, error) { 79 | //nolint: errcheck // Ignore error. 80 | srvAddr := req.Context().Value(http.LocalAddrContextKey).(net.Addr) 81 | resp, err := http.Get(fmt.Sprintf("http://%s/metrics", srvAddr.String())) 82 | if err != nil { 83 | return "", nil, err 84 | } 85 | defer resp.Body.Close() 86 | parser := expfmt.TextParser{} 87 | metricFamilies, err := parser.TextToMetricFamilies(resp.Body) 88 | if err != nil { 89 | return "", nil, err 90 | } 91 | 92 | data := struct { 93 | ImageCount int64 94 | LayerCount int64 95 | }{} 96 | for _, metric := range metricFamilies["spegel_advertised_images"].Metric { 97 | data.ImageCount += int64(*metric.Gauge.Value) 98 | } 99 | for _, metric := range metricFamilies["spegel_advertised_keys"].Metric { 100 | data.LayerCount += int64(*metric.Gauge.Value) 101 | } 102 | return "stats", data, nil 103 | } 104 | 105 | type measureResult struct { 106 | PeerResults []peerResult 107 | PullResults []pullResult 108 | } 109 | 110 | type peerResult struct { 111 | Peer netip.AddrPort 112 | Duration time.Duration 113 | } 114 | 115 | type pullResult struct { 116 | Identifier string 117 | ContentType string 118 | ContentLength string 119 | Duration time.Duration 120 | } 121 | 122 | func (w *Web) measure(req *http.Request) (string, any, error) { 123 | // Parse image name. 124 | imgName := req.URL.Query().Get("image") 125 | if imgName == "" { 126 | return "", nil, errors.New("image name cannot be empty") 127 | } 128 | img, err := oci.ParseImage(imgName) 129 | if err != nil { 130 | return "", nil, err 131 | } 132 | 133 | res := measureResult{} 134 | 135 | // Resolve peers for the given image. 136 | resolveStart := time.Now() 137 | peerCh, err := w.router.Resolve(req.Context(), imgName, 0) 138 | if err != nil { 139 | return "", nil, err 140 | } 141 | for peer := range peerCh { 142 | res.PeerResults = append(res.PeerResults, peerResult{ 143 | Peer: peer, 144 | Duration: time.Since(resolveStart), 145 | }) 146 | } 147 | if len(res.PeerResults) == 0 { 148 | return "measure", res, nil 149 | } 150 | 151 | // Pull the image and measure performance. 152 | pullResults, err := measureImagePull(w.client, "http://localhost:5000", img) 153 | if err != nil { 154 | return "", nil, err 155 | } 156 | res.PullResults = pullResults 157 | 158 | return "measure", res, nil 159 | } 160 | 161 | func measureImagePull(client *http.Client, regURL string, img oci.Image) ([]pullResult, error) { 162 | pullResults := []pullResult{} 163 | queue := []oci.DistributionPath{ 164 | { 165 | Kind: oci.DistributionKindManifest, 166 | Name: img.Repository, 167 | Digest: img.Digest, 168 | Tag: img.Tag, 169 | Registry: img.Registry, 170 | }, 171 | } 172 | for { 173 | //nolint: staticcheck // Ignore until we have proper tests. 174 | if len(queue) == 0 { 175 | break 176 | } 177 | pr, dists, err := fetchDistributionPath(client, regURL, queue[0]) 178 | if err != nil { 179 | return nil, err 180 | } 181 | queue = queue[1:] 182 | queue = append(queue, dists...) 183 | pullResults = append(pullResults, pr) 184 | } 185 | return pullResults, nil 186 | } 187 | 188 | func fetchDistributionPath(client *http.Client, regURL string, dist oci.DistributionPath) (pullResult, []oci.DistributionPath, error) { 189 | regU, err := url.Parse(regURL) 190 | if err != nil { 191 | return pullResult{}, nil, err 192 | } 193 | u := dist.URL() 194 | u.Scheme = regU.Scheme 195 | u.Host = regU.Host 196 | 197 | pullStart := time.Now() 198 | pullReq, err := http.NewRequest(http.MethodGet, u.String(), nil) 199 | if err != nil { 200 | return pullResult{}, nil, err 201 | } 202 | pullResp, err := client.Do(pullReq) 203 | if err != nil { 204 | return pullResult{}, nil, err 205 | } 206 | defer pullResp.Body.Close() 207 | if pullResp.StatusCode != http.StatusOK { 208 | _, err = io.Copy(io.Discard, pullResp.Body) 209 | if err != nil { 210 | return pullResult{}, nil, err 211 | } 212 | return pullResult{}, nil, fmt.Errorf("request returned unexpected status code %s", pullResp.Status) 213 | } 214 | 215 | queue := []oci.DistributionPath{} 216 | ct := pullResp.Header.Get("Content-Type") 217 | switch dist.Kind { 218 | case oci.DistributionKindBlob: 219 | _, err = io.Copy(io.Discard, pullResp.Body) 220 | if err != nil { 221 | return pullResult{}, nil, err 222 | } 223 | ct = "Layer" 224 | case oci.DistributionKindManifest: 225 | b, err := io.ReadAll(pullResp.Body) 226 | if err != nil { 227 | return pullResult{}, nil, err 228 | } 229 | switch ct { 230 | case images.MediaTypeDockerSchema2ManifestList, ocispec.MediaTypeImageIndex: 231 | var idx ocispec.Index 232 | if err := json.Unmarshal(b, &idx); err != nil { 233 | return pullResult{}, nil, err 234 | } 235 | for _, m := range idx.Manifests { 236 | //nolint: staticcheck // Simplify in the future. 237 | if !(m.Platform.OS == runtime.GOOS && m.Platform.Architecture == runtime.GOARCH) { 238 | continue 239 | } 240 | queue = append(queue, oci.DistributionPath{ 241 | Kind: oci.DistributionKindManifest, 242 | Name: dist.Name, 243 | Digest: m.Digest, 244 | Registry: dist.Registry, 245 | }) 246 | } 247 | ct = "Index" 248 | case images.MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest: 249 | var manifest ocispec.Manifest 250 | err := json.Unmarshal(b, &manifest) 251 | if err != nil { 252 | return pullResult{}, nil, err 253 | } 254 | queue = append(queue, oci.DistributionPath{ 255 | Kind: oci.DistributionKindManifest, 256 | Name: dist.Name, 257 | Digest: manifest.Config.Digest, 258 | Registry: dist.Registry, 259 | }) 260 | for _, layer := range manifest.Layers { 261 | queue = append(queue, oci.DistributionPath{ 262 | Kind: oci.DistributionKindBlob, 263 | Name: dist.Name, 264 | Digest: layer.Digest, 265 | Registry: dist.Registry, 266 | }) 267 | } 268 | ct = "Manifest" 269 | case ocispec.MediaTypeImageConfig: 270 | ct = "Config" 271 | } 272 | } 273 | pullResp.Body.Close() 274 | 275 | i, err := strconv.ParseInt(pullResp.Header.Get("Content-Length"), 10, 0) 276 | if err != nil { 277 | return pullResult{}, nil, err 278 | } 279 | return pullResult{ 280 | Identifier: dist.Reference(), 281 | ContentType: ct, 282 | ContentLength: formatByteSize(i), 283 | Duration: time.Since(pullStart), 284 | }, queue, nil 285 | } 286 | 287 | func formatByteSize(size int64) string { 288 | const unit = 1000 289 | if size < unit { 290 | return fmt.Sprintf("%d B", size) 291 | } 292 | div, exp := int64(unit), 0 293 | for n := size / unit; n >= unit; n /= unit { 294 | div *= unit 295 | exp++ 296 | } 297 | return fmt.Sprintf("%.1f %cB", float64(size)/float64(div), "kMGTPE"[exp]) 298 | } 299 | -------------------------------------------------------------------------------- /internal/web/web_test.go: -------------------------------------------------------------------------------- 1 | package web 2 | 3 | import ( 4 | "encoding/json" 5 | "net/http" 6 | "net/http/httptest" 7 | "runtime" 8 | "strconv" 9 | "testing" 10 | 11 | "github.com/opencontainers/go-digest" 12 | ocispec "github.com/opencontainers/image-spec/specs-go/v1" 13 | "github.com/stretchr/testify/require" 14 | 15 | "github.com/spegel-org/spegel/pkg/oci" 16 | ) 17 | 18 | func TestMeasureImagePull(t *testing.T) { 19 | t.Parallel() 20 | 21 | srv := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { 22 | switch req.URL.Path { 23 | case "/v2/test/image/manifests/index": 24 | rw.Header().Set("Content-Type", ocispec.MediaTypeImageIndex) 25 | idx := ocispec.Index{ 26 | Manifests: []ocispec.Descriptor{ 27 | { 28 | Digest: digest.Digest("manifest"), 29 | Platform: &ocispec.Platform{ 30 | OS: runtime.GOOS, 31 | Architecture: runtime.GOARCH, 32 | }, 33 | }, 34 | }, 35 | } 36 | b, err := json.Marshal(&idx) 37 | if err != nil { 38 | rw.WriteHeader(http.StatusInternalServerError) 39 | return 40 | } 41 | //nolint: errcheck // Ignore error. 42 | rw.Write(b) 43 | case "/v2/test/image/manifests/manifest": 44 | rw.Header().Set("Content-Type", ocispec.MediaTypeImageManifest) 45 | manifest := ocispec.Manifest{ 46 | Config: ocispec.Descriptor{ 47 | Digest: digest.Digest("config"), 48 | }, 49 | Layers: []ocispec.Descriptor{ 50 | { 51 | Digest: digest.Digest("layer"), 52 | }, 53 | }, 54 | } 55 | b, err := json.Marshal(&manifest) 56 | if err != nil { 57 | rw.WriteHeader(http.StatusInternalServerError) 58 | return 59 | } 60 | //nolint: errcheck // Ignore error. 61 | rw.Write(b) 62 | case "/v2/test/image/manifests/config": 63 | rw.Header().Set("Content-Type", ocispec.MediaTypeImageConfig) 64 | config := ocispec.ImageConfig{ 65 | User: "root", 66 | } 67 | b, err := json.Marshal(&config) 68 | if err != nil { 69 | rw.WriteHeader(http.StatusInternalServerError) 70 | return 71 | } 72 | //nolint: errcheck // Ignore error. 73 | rw.Write(b) 74 | case "/v2/test/image/blobs/layer": 75 | //nolint: errcheck // Ignore error. 76 | rw.Write([]byte("Hello World")) 77 | default: 78 | rw.WriteHeader(http.StatusNotFound) 79 | } 80 | })) 81 | t.Cleanup(func() { 82 | srv.Close() 83 | }) 84 | 85 | img := oci.Image{ 86 | Repository: "test/image", 87 | Digest: digest.Digest("index"), 88 | Registry: "example.com", 89 | } 90 | pullResults, err := measureImagePull(srv.Client(), srv.URL, img) 91 | require.NoError(t, err) 92 | 93 | require.NotEmpty(t, pullResults) 94 | } 95 | 96 | func TestFormatByteSize(t *testing.T) { 97 | t.Parallel() 98 | 99 | tests := []struct { 100 | expected string 101 | size int64 102 | }{ 103 | { 104 | size: 1, 105 | expected: "1 B", 106 | }, 107 | { 108 | size: 18954, 109 | expected: "19.0 kB", 110 | }, 111 | { 112 | size: 1000000000, 113 | expected: "1.0 GB", 114 | }, 115 | } 116 | for _, tt := range tests { 117 | t.Run(strconv.FormatInt(tt.size, 10), func(t *testing.T) { 118 | t.Parallel() 119 | 120 | result := formatByteSize(tt.size) 121 | require.Equal(t, tt.expected, result) 122 | }) 123 | } 124 | } 125 | -------------------------------------------------------------------------------- /pkg/metrics/metrics.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "github.com/prometheus/client_golang/prometheus" 5 | ) 6 | 7 | var ( 8 | // DefaultRegisterer and DefaultGatherer are the implementations of the 9 | // prometheus Registerer and Gatherer interfaces that all metrics operations 10 | // will use. They are variables so that packages that embed this library can 11 | // replace them at runtime, instead of having to pass around specific 12 | // registries. 13 | DefaultRegisterer = prometheus.DefaultRegisterer 14 | DefaultGatherer = prometheus.DefaultGatherer 15 | ) 16 | 17 | var ( 18 | MirrorRequestsTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ 19 | Name: "spegel_mirror_requests_total", 20 | Help: "Total number of mirror requests.", 21 | }, []string{"registry", "cache"}) 22 | ResolveDurHistogram = prometheus.NewHistogramVec(prometheus.HistogramOpts{ 23 | Name: "spegel_resolve_duration_seconds", 24 | Help: "The duration for router to resolve a peer.", 25 | }, []string{"router"}) 26 | AdvertisedImages = prometheus.NewGaugeVec(prometheus.GaugeOpts{ 27 | Name: "spegel_advertised_images", 28 | Help: "Number of images advertised to be available.", 29 | }, []string{"registry"}) 30 | AdvertisedImageTags = prometheus.NewGaugeVec(prometheus.GaugeOpts{ 31 | Name: "spegel_advertised_image_tags", 32 | Help: "Number of image tags advertised to be available.", 33 | }, []string{"registry"}) 34 | AdvertisedImageDigests = prometheus.NewGaugeVec(prometheus.GaugeOpts{ 35 | Name: "spegel_advertised_image_digests", 36 | Help: "Number of image digests advertised to be available.", 37 | }, []string{"registry"}) 38 | AdvertisedKeys = prometheus.NewGaugeVec(prometheus.GaugeOpts{ 39 | Name: "spegel_advertised_keys", 40 | Help: "Number of keys advertised to be available.", 41 | }, []string{"registry"}) 42 | HttpRequestDurHistogram = prometheus.NewHistogramVec(prometheus.HistogramOpts{ 43 | Subsystem: "http", 44 | Name: "request_duration_seconds", 45 | Help: "The latency of the HTTP requests.", 46 | }, []string{"handler", "method", "code"}) 47 | HttpResponseSizeHistogram = prometheus.NewHistogramVec(prometheus.HistogramOpts{ 48 | Subsystem: "http", 49 | Name: "response_size_bytes", 50 | Help: "The size of the HTTP responses.", 51 | // 1kB up to 2GB 52 | Buckets: prometheus.ExponentialBuckets(1024, 5, 10), 53 | }, []string{"handler", "method", "code"}) 54 | HttpRequestsInflight = prometheus.NewGaugeVec(prometheus.GaugeOpts{ 55 | Subsystem: "http", 56 | Name: "requests_inflight", 57 | Help: "The number of inflight requests being handled at the same time.", 58 | }, []string{"handler"}) 59 | ) 60 | 61 | func Register() { 62 | DefaultRegisterer.MustRegister(MirrorRequestsTotal) 63 | DefaultRegisterer.MustRegister(ResolveDurHistogram) 64 | DefaultRegisterer.MustRegister(AdvertisedImages) 65 | DefaultRegisterer.MustRegister(AdvertisedImageTags) 66 | DefaultRegisterer.MustRegister(AdvertisedImageDigests) 67 | DefaultRegisterer.MustRegister(AdvertisedKeys) 68 | DefaultRegisterer.MustRegister(HttpRequestDurHistogram) 69 | DefaultRegisterer.MustRegister(HttpResponseSizeHistogram) 70 | DefaultRegisterer.MustRegister(HttpRequestsInflight) 71 | } 72 | -------------------------------------------------------------------------------- /pkg/metrics/metrics_test.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import "testing" 4 | 5 | func TestRegister(t *testing.T) { 6 | t.Parallel() 7 | 8 | Register() 9 | } 10 | -------------------------------------------------------------------------------- /pkg/oci/distribution.go: -------------------------------------------------------------------------------- 1 | package oci 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "net/url" 7 | "regexp" 8 | 9 | "github.com/opencontainers/go-digest" 10 | ) 11 | 12 | var ( 13 | nameRegex = regexp.MustCompile(`([a-z0-9]+([._-][a-z0-9]+)*(/[a-z0-9]+([._-][a-z0-9]+)*)*)`) 14 | tagRegex = regexp.MustCompile(`([a-zA-Z0-9_][a-zA-Z0-9._-]{0,127})`) 15 | manifestRegexTag = regexp.MustCompile(`/v2/` + nameRegex.String() + `/manifests/` + tagRegex.String() + `$`) 16 | manifestRegexDigest = regexp.MustCompile(`/v2/` + nameRegex.String() + `/manifests/(.*)`) 17 | blobsRegexDigest = regexp.MustCompile(`/v2/` + nameRegex.String() + `/blobs/(.*)`) 18 | ) 19 | 20 | // DistributionKind represents the kind of content. 21 | type DistributionKind string 22 | 23 | const ( 24 | DistributionKindManifest = "manifests" 25 | DistributionKindBlob = "blobs" 26 | ) 27 | 28 | // DistributionPath contains the individual parameters from a OCI distribution spec request. 29 | type DistributionPath struct { 30 | Kind DistributionKind 31 | Name string 32 | Digest digest.Digest 33 | Tag string 34 | Registry string 35 | } 36 | 37 | // Reference returns the digest if set or alternatively if not the full image reference with the tag. 38 | func (d DistributionPath) Reference() string { 39 | if d.Digest != "" { 40 | return d.Digest.String() 41 | } 42 | return fmt.Sprintf("%s/%s:%s", d.Registry, d.Name, d.Tag) 43 | } 44 | 45 | // IsLatestTag returns true if the tag has the value latest. 46 | func (d DistributionPath) IsLatestTag() bool { 47 | return d.Tag == "latest" 48 | } 49 | 50 | // URL returns the reconstructed URL containing the path and query parameters. 51 | func (d DistributionPath) URL() *url.URL { 52 | ref := d.Digest.String() 53 | if ref == "" { 54 | ref = d.Tag 55 | } 56 | return &url.URL{ 57 | Path: fmt.Sprintf("/v2/%s/%s/%s", d.Name, d.Kind, ref), 58 | RawQuery: fmt.Sprintf("ns=%s", d.Registry), 59 | } 60 | } 61 | 62 | // ParseDistributionPath gets the parameters from a URL which conforms with the OCI distribution spec. 63 | // It returns a distribution path which contains all the individual parameters. 64 | // https://github.com/opencontainers/distribution-spec/blob/main/spec.md 65 | func ParseDistributionPath(u *url.URL) (DistributionPath, error) { 66 | registry := u.Query().Get("ns") 67 | comps := manifestRegexTag.FindStringSubmatch(u.Path) 68 | if len(comps) == 6 { 69 | if registry == "" { 70 | return DistributionPath{}, errors.New("registry parameter needs to be set for tag references") 71 | } 72 | dist := DistributionPath{ 73 | Kind: DistributionKindManifest, 74 | Name: comps[1], 75 | Tag: comps[5], 76 | Registry: registry, 77 | } 78 | return dist, nil 79 | } 80 | comps = manifestRegexDigest.FindStringSubmatch(u.Path) 81 | if len(comps) == 6 { 82 | dgst, err := digest.Parse(comps[5]) 83 | if err != nil { 84 | return DistributionPath{}, err 85 | } 86 | dist := DistributionPath{ 87 | Kind: DistributionKindManifest, 88 | Name: comps[1], 89 | Digest: dgst, 90 | Registry: registry, 91 | } 92 | return dist, nil 93 | } 94 | comps = blobsRegexDigest.FindStringSubmatch(u.Path) 95 | if len(comps) == 6 { 96 | dgst, err := digest.Parse(comps[5]) 97 | if err != nil { 98 | return DistributionPath{}, err 99 | } 100 | dist := DistributionPath{ 101 | Kind: DistributionKindBlob, 102 | Name: comps[1], 103 | Digest: dgst, 104 | Registry: registry, 105 | } 106 | return dist, nil 107 | } 108 | return DistributionPath{}, errors.New("distribution path could not be parsed") 109 | } 110 | -------------------------------------------------------------------------------- /pkg/oci/distribution_test.go: -------------------------------------------------------------------------------- 1 | package oci 2 | 3 | import ( 4 | "fmt" 5 | "net/url" 6 | "testing" 7 | 8 | "github.com/opencontainers/go-digest" 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func TestParseDistributionPath(t *testing.T) { 13 | t.Parallel() 14 | 15 | tests := []struct { 16 | name string 17 | registry string 18 | path string 19 | expectedName string 20 | expectedDgst digest.Digest 21 | expectedTag string 22 | expectedRef string 23 | expectedKind DistributionKind 24 | execptedIsLatestTag bool 25 | }{ 26 | { 27 | name: "manifest tag", 28 | registry: "example.com", 29 | path: "/v2/foo/bar/manifests/hello-world", 30 | expectedName: "foo/bar", 31 | expectedDgst: "", 32 | expectedTag: "hello-world", 33 | expectedRef: "example.com/foo/bar:hello-world", 34 | expectedKind: DistributionKindManifest, 35 | execptedIsLatestTag: false, 36 | }, 37 | { 38 | name: "manifest with latest tag", 39 | registry: "example.com", 40 | path: "/v2/test/manifests/latest", 41 | expectedName: "test", 42 | expectedDgst: "", 43 | expectedTag: "latest", 44 | expectedRef: "example.com/test:latest", 45 | expectedKind: DistributionKindManifest, 46 | execptedIsLatestTag: true, 47 | }, 48 | { 49 | name: "manifest digest", 50 | registry: "docker.io", 51 | path: "/v2/library/nginx/manifests/sha256:0a404ca8e119d061cdb2dceee824c914cdc69b31bc7b5956ef5a520436a80d39", 52 | expectedName: "library/nginx", 53 | expectedDgst: digest.Digest("sha256:0a404ca8e119d061cdb2dceee824c914cdc69b31bc7b5956ef5a520436a80d39"), 54 | expectedTag: "", 55 | expectedRef: "sha256:0a404ca8e119d061cdb2dceee824c914cdc69b31bc7b5956ef5a520436a80d39", 56 | expectedKind: DistributionKindManifest, 57 | execptedIsLatestTag: false, 58 | }, 59 | { 60 | name: "blob digest", 61 | registry: "docker.io", 62 | path: "/v2/library/nginx/blobs/sha256:295c7be079025306c4f1d65997fcf7adb411c88f139ad1d34b537164aa060369", 63 | expectedName: "library/nginx", 64 | expectedDgst: digest.Digest("sha256:295c7be079025306c4f1d65997fcf7adb411c88f139ad1d34b537164aa060369"), 65 | expectedTag: "", 66 | expectedRef: "sha256:295c7be079025306c4f1d65997fcf7adb411c88f139ad1d34b537164aa060369", 67 | expectedKind: DistributionKindBlob, 68 | execptedIsLatestTag: false, 69 | }, 70 | } 71 | for _, tt := range tests { 72 | t.Run(tt.name, func(t *testing.T) { 73 | t.Parallel() 74 | 75 | u := &url.URL{ 76 | Path: tt.path, 77 | RawQuery: fmt.Sprintf("ns=%s", tt.registry), 78 | } 79 | dist, err := ParseDistributionPath(u) 80 | require.NoError(t, err) 81 | require.Equal(t, tt.expectedName, dist.Name) 82 | require.Equal(t, tt.expectedDgst, dist.Digest) 83 | require.Equal(t, tt.expectedTag, dist.Tag) 84 | require.Equal(t, tt.expectedRef, dist.Reference()) 85 | require.Equal(t, tt.expectedKind, dist.Kind) 86 | require.Equal(t, tt.registry, dist.Registry) 87 | require.Equal(t, tt.path, dist.URL().Path) 88 | require.Equal(t, tt.registry, dist.URL().Query().Get("ns")) 89 | require.Equal(t, tt.execptedIsLatestTag, dist.IsLatestTag()) 90 | }) 91 | } 92 | } 93 | 94 | func TestParseDistributionPathErrors(t *testing.T) { 95 | t.Parallel() 96 | 97 | tests := []struct { 98 | name string 99 | url *url.URL 100 | expectedError string 101 | }{ 102 | { 103 | name: "invalid path", 104 | url: &url.URL{ 105 | Path: "/v2/spegel-org/spegel/v0.0.1", 106 | RawQuery: "ns=example.com", 107 | }, 108 | expectedError: "distribution path could not be parsed", 109 | }, 110 | { 111 | name: "blob with tag reference", 112 | url: &url.URL{ 113 | Path: "/v2/spegel-org/spegel/blobs/v0.0.1", 114 | RawQuery: "ns=example.com", 115 | }, 116 | expectedError: "invalid checksum digest format", 117 | }, 118 | { 119 | name: "blob with invalid digest", 120 | url: &url.URL{ 121 | Path: "/v2/spegel-org/spegel/blobs/sha256:123", 122 | RawQuery: "ns=example.com", 123 | }, 124 | expectedError: "invalid checksum digest length", 125 | }, 126 | { 127 | name: "manifest tag with missing registry", 128 | url: &url.URL{ 129 | Path: "/v2/spegel-org/spegel/manifests/v0.0.1", 130 | }, 131 | expectedError: "registry parameter needs to be set for tag references", 132 | }, 133 | { 134 | name: "manifest with invalid digest", 135 | url: &url.URL{ 136 | Path: "/v2/spegel-org/spegel/manifests/sha253:foobar", 137 | }, 138 | expectedError: "unsupported digest algorithm", 139 | }, 140 | } 141 | for _, tt := range tests { 142 | t.Run(tt.name, func(t *testing.T) { 143 | t.Parallel() 144 | 145 | _, err := ParseDistributionPath(tt.url) 146 | require.EqualError(t, err, tt.expectedError) 147 | }) 148 | } 149 | } 150 | -------------------------------------------------------------------------------- /pkg/oci/image.go: -------------------------------------------------------------------------------- 1 | package oci 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "net/url" 7 | "regexp" 8 | "strings" 9 | 10 | digest "github.com/opencontainers/go-digest" 11 | ) 12 | 13 | type Image struct { 14 | Registry string 15 | Repository string 16 | Tag string 17 | Digest digest.Digest 18 | } 19 | 20 | func NewImage(registry, repository, tag string, dgst digest.Digest) (Image, error) { 21 | if registry == "" { 22 | return Image{}, errors.New("image needs to contain a registry") 23 | } 24 | if repository == "" { 25 | return Image{}, errors.New("image needs to contain a repository") 26 | } 27 | if dgst != "" { 28 | if err := dgst.Validate(); err != nil { 29 | return Image{}, err 30 | } 31 | } 32 | return Image{ 33 | Registry: registry, 34 | Repository: repository, 35 | Tag: tag, 36 | Digest: dgst, 37 | }, nil 38 | } 39 | 40 | func (i Image) IsLatestTag() bool { 41 | return i.Tag == "latest" 42 | } 43 | 44 | func (i Image) String() string { 45 | tag := "" 46 | if i.Tag != "" { 47 | tag = ":" + i.Tag 48 | } 49 | digest := "" 50 | if i.Digest != "" { 51 | digest = "@" + i.Digest.String() 52 | } 53 | return fmt.Sprintf("%s/%s%s%s", i.Registry, i.Repository, tag, digest) 54 | } 55 | 56 | func (i Image) TagName() (string, bool) { 57 | if i.Tag == "" { 58 | return "", false 59 | } 60 | return fmt.Sprintf("%s/%s:%s", i.Registry, i.Repository, i.Tag), true 61 | } 62 | 63 | var splitRe = regexp.MustCompile(`[:@]`) 64 | 65 | func ParseImage(s string) (Image, error) { 66 | if strings.Contains(s, "://") { 67 | return Image{}, errors.New("invalid reference") 68 | } 69 | u, err := url.Parse("dummy://" + s) 70 | if err != nil { 71 | return Image{}, err 72 | } 73 | if u.Scheme != "dummy" { 74 | return Image{}, errors.New("invalid reference") 75 | } 76 | if u.Host == "" { 77 | return Image{}, errors.New("hostname required") 78 | } 79 | var object string 80 | if idx := splitRe.FindStringIndex(u.Path); idx != nil { 81 | // This allows us to retain the @ to signify digests or shortened digests in 82 | // the object. 83 | object = u.Path[idx[0]:] 84 | if object[:1] == ":" { 85 | object = object[1:] 86 | } 87 | u.Path = u.Path[:idx[0]] 88 | } 89 | tag, dgst := splitObject(object) 90 | tag, _, _ = strings.Cut(tag, "@") 91 | repository := strings.TrimPrefix(u.Path, "/") 92 | 93 | img, err := NewImage(u.Host, repository, tag, dgst) 94 | if err != nil { 95 | return Image{}, err 96 | } 97 | return img, nil 98 | } 99 | 100 | func ParseImageRequireDigest(s string, dgst digest.Digest) (Image, error) { 101 | img, err := ParseImage(s) 102 | if err != nil { 103 | return Image{}, err 104 | } 105 | if img.Digest != "" && dgst == "" { 106 | return img, nil 107 | } 108 | if img.Digest == "" && dgst == "" { 109 | return Image{}, errors.New("image needs to contain a digest") 110 | } 111 | if img.Digest == "" && dgst != "" { 112 | return NewImage(img.Registry, img.Repository, img.Tag, dgst) 113 | } 114 | if img.Digest != dgst { 115 | return Image{}, fmt.Errorf("invalid digest set does not match parsed digest: %v %v", s, img.Digest) 116 | } 117 | return img, nil 118 | } 119 | 120 | func splitObject(obj string) (tag string, dgst digest.Digest) { 121 | parts := strings.SplitAfterN(obj, "@", 2) 122 | if len(parts) < 2 { 123 | return parts[0], "" 124 | } 125 | return parts[0], digest.Digest(parts[1]) 126 | } 127 | -------------------------------------------------------------------------------- /pkg/oci/image_test.go: -------------------------------------------------------------------------------- 1 | package oci 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | digest "github.com/opencontainers/go-digest" 8 | "github.com/stretchr/testify/require" 9 | ) 10 | 11 | func TestParseImageRequireDigest(t *testing.T) { 12 | t.Parallel() 13 | 14 | tests := []struct { 15 | name string 16 | image string 17 | expectedRepository string 18 | expectedTag string 19 | expectedString string 20 | expectedDigest digest.Digest 21 | expectedIsLatest bool 22 | digestInImage bool 23 | }{ 24 | { 25 | name: "Latest tag", 26 | image: "library/ubuntu:latest", 27 | digestInImage: false, 28 | expectedRepository: "library/ubuntu", 29 | expectedTag: "latest", 30 | expectedDigest: digest.Digest("sha256:c0669ef34cdc14332c0f1ab0c2c01acb91d96014b172f1a76f3a39e63d1f0bda"), 31 | expectedIsLatest: true, 32 | expectedString: "library/ubuntu:latest@sha256:c0669ef34cdc14332c0f1ab0c2c01acb91d96014b172f1a76f3a39e63d1f0bda", 33 | }, 34 | { 35 | name: "Only tag", 36 | image: "library/alpine:3.18.0", 37 | digestInImage: false, 38 | expectedRepository: "library/alpine", 39 | expectedTag: "3.18.0", 40 | expectedDigest: digest.Digest("sha256:c0669ef34cdc14332c0f1ab0c2c01acb91d96014b172f1a76f3a39e63d1f0bda"), 41 | expectedIsLatest: false, 42 | expectedString: "library/alpine:3.18.0@sha256:c0669ef34cdc14332c0f1ab0c2c01acb91d96014b172f1a76f3a39e63d1f0bda", 43 | }, 44 | { 45 | name: "Tag and digest", 46 | image: "jetstack/cert-manager-controller:3.18.0@sha256:c0669ef34cdc14332c0f1ab0c2c01acb91d96014b172f1a76f3a39e63d1f0bda", 47 | digestInImage: true, 48 | expectedRepository: "jetstack/cert-manager-controller", 49 | expectedTag: "3.18.0", 50 | expectedDigest: digest.Digest("sha256:c0669ef34cdc14332c0f1ab0c2c01acb91d96014b172f1a76f3a39e63d1f0bda"), 51 | expectedIsLatest: false, 52 | expectedString: "jetstack/cert-manager-controller:3.18.0@sha256:c0669ef34cdc14332c0f1ab0c2c01acb91d96014b172f1a76f3a39e63d1f0bda", 53 | }, 54 | { 55 | name: "Only digest", 56 | image: "fluxcd/helm-controller@sha256:c0669ef34cdc14332c0f1ab0c2c01acb91d96014b172f1a76f3a39e63d1f0bda", 57 | digestInImage: true, 58 | expectedRepository: "fluxcd/helm-controller", 59 | expectedTag: "", 60 | expectedDigest: digest.Digest("sha256:c0669ef34cdc14332c0f1ab0c2c01acb91d96014b172f1a76f3a39e63d1f0bda"), 61 | expectedIsLatest: false, 62 | expectedString: "fluxcd/helm-controller@sha256:c0669ef34cdc14332c0f1ab0c2c01acb91d96014b172f1a76f3a39e63d1f0bda", 63 | }, 64 | { 65 | name: "Digest only in extra digest", 66 | image: "foo/bar", 67 | digestInImage: false, 68 | expectedRepository: "foo/bar", 69 | expectedDigest: digest.Digest("sha256:c0669ef34cdc14332c0f1ab0c2c01acb91d96014b172f1a76f3a39e63d1f0bda"), 70 | expectedIsLatest: false, 71 | expectedString: "foo/bar@sha256:c0669ef34cdc14332c0f1ab0c2c01acb91d96014b172f1a76f3a39e63d1f0bda", 72 | }, 73 | } 74 | registries := []string{"docker.io", "quay.io", "ghcr.com", "127.0.0.1"} 75 | for _, registry := range registries { 76 | for _, tt := range tests { 77 | t.Run(fmt.Sprintf("%s_%s", tt.name, registry), func(t *testing.T) { 78 | t.Parallel() 79 | 80 | for _, extraDgst := range []string{tt.expectedDigest.String(), ""} { 81 | img, err := ParseImageRequireDigest(fmt.Sprintf("%s/%s", registry, tt.image), digest.Digest(extraDgst)) 82 | if !tt.digestInImage && extraDgst == "" { 83 | require.EqualError(t, err, "image needs to contain a digest") 84 | continue 85 | } 86 | require.NoError(t, err) 87 | require.Equal(t, registry, img.Registry) 88 | require.Equal(t, tt.expectedRepository, img.Repository) 89 | require.Equal(t, tt.expectedTag, img.Tag) 90 | require.Equal(t, tt.expectedDigest, img.Digest) 91 | require.Equal(t, tt.expectedIsLatest, img.IsLatestTag()) 92 | tagName, ok := img.TagName() 93 | if tt.expectedTag == "" { 94 | require.False(t, ok) 95 | require.Empty(t, tagName) 96 | } else { 97 | require.True(t, ok) 98 | require.Equal(t, registry+"/"+tt.expectedRepository+":"+tt.expectedTag, tagName) 99 | } 100 | require.Equal(t, fmt.Sprintf("%s/%s", registry, tt.expectedString), img.String()) 101 | } 102 | }) 103 | } 104 | } 105 | } 106 | 107 | func TestParseImageRequireDigestErrors(t *testing.T) { 108 | t.Parallel() 109 | 110 | tests := []struct { 111 | name string 112 | s string 113 | dgst digest.Digest 114 | expectedError string 115 | }{ 116 | { 117 | name: "digests do not match", 118 | s: "quay.io/jetstack/cert-manager-webhook@sha256:13fd9eaadb4e491ef0e1d82de60cb199f5ad2ea5a3f8e0c19fdf31d91175b9cb", 119 | dgst: digest.Digest("sha256:ec4306b243d98cce7c3b1f994f2dae660059ef521b2b24588cfdc950bd816d4c"), 120 | expectedError: "invalid digest set does not match parsed digest: quay.io/jetstack/cert-manager-webhook@sha256:13fd9eaadb4e491ef0e1d82de60cb199f5ad2ea5a3f8e0c19fdf31d91175b9cb sha256:13fd9eaadb4e491ef0e1d82de60cb199f5ad2ea5a3f8e0c19fdf31d91175b9cb", 121 | }, 122 | { 123 | name: "no tag or digest", 124 | s: "ghcr.io/spegel-org/spegel", 125 | dgst: "", 126 | expectedError: "image needs to contain a digest", 127 | }, 128 | { 129 | name: "reference contains protocol", 130 | s: "https://example.com/test:latest", 131 | dgst: "", 132 | expectedError: "invalid reference", 133 | }, 134 | { 135 | name: "unparsable url", 136 | s: "example%#$.com/foo", 137 | dgst: "", 138 | expectedError: "parse \"dummy://example%\": invalid URL escape \"%\"", 139 | }, 140 | } 141 | for _, tt := range tests { 142 | t.Run(tt.name, func(t *testing.T) { 143 | t.Parallel() 144 | 145 | _, err := ParseImageRequireDigest(tt.s, tt.dgst) 146 | require.EqualError(t, err, tt.expectedError) 147 | }) 148 | } 149 | } 150 | 151 | func TestNewImageErrors(t *testing.T) { 152 | t.Parallel() 153 | 154 | // TODO (phillebaba): Add test case for no digest or tag. One needs to be set. 155 | tests := []struct { 156 | name string 157 | registry string 158 | repository string 159 | tag string 160 | dgst digest.Digest 161 | expectedError string 162 | }{ 163 | { 164 | name: "missing registry", 165 | registry: "", 166 | repository: "foo/bar", 167 | tag: "latest", 168 | dgst: digest.Digest("sha256:ec4306b243d98cce7c3b1f994f2dae660059ef521b2b24588cfdc950bd816d4c"), 169 | expectedError: "image needs to contain a registry", 170 | }, 171 | { 172 | name: "missing repository", 173 | registry: "example.com", 174 | repository: "", 175 | tag: "latest", 176 | dgst: digest.Digest("sha256:ec4306b243d98cce7c3b1f994f2dae660059ef521b2b24588cfdc950bd816d4c"), 177 | expectedError: "image needs to contain a repository", 178 | }, 179 | { 180 | name: "invalid digest", 181 | registry: "example.com", 182 | repository: "foo/bar", 183 | tag: "latest", 184 | dgst: digest.Digest("test"), 185 | expectedError: "invalid checksum digest format", 186 | }, 187 | } 188 | for _, tt := range tests { 189 | t.Run(tt.name, func(t *testing.T) { 190 | t.Parallel() 191 | 192 | _, err := NewImage(tt.registry, tt.repository, tt.tag, tt.dgst) 193 | require.EqualError(t, err, tt.expectedError) 194 | }) 195 | } 196 | } 197 | -------------------------------------------------------------------------------- /pkg/oci/memory.go: -------------------------------------------------------------------------------- 1 | package oci 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "errors" 7 | "fmt" 8 | "io" 9 | "sync" 10 | 11 | "github.com/opencontainers/go-digest" 12 | ) 13 | 14 | var _ Client = &Memory{} 15 | 16 | type Memory struct { 17 | blobs map[digest.Digest][]byte 18 | tags map[string]digest.Digest 19 | images []Image 20 | mx sync.RWMutex 21 | } 22 | 23 | func NewMemory() *Memory { 24 | return &Memory{ 25 | images: []Image{}, 26 | tags: map[string]digest.Digest{}, 27 | blobs: map[digest.Digest][]byte{}, 28 | } 29 | } 30 | 31 | func (m *Memory) Name() string { 32 | return "memory" 33 | } 34 | 35 | func (m *Memory) Verify(ctx context.Context) error { 36 | return nil 37 | } 38 | 39 | func (m *Memory) Subscribe(ctx context.Context) (<-chan ImageEvent, <-chan error, error) { 40 | return nil, nil, nil 41 | } 42 | 43 | func (m *Memory) ListImages(ctx context.Context) ([]Image, error) { 44 | m.mx.RLock() 45 | defer m.mx.RUnlock() 46 | 47 | return m.images, nil 48 | } 49 | 50 | func (m *Memory) Resolve(ctx context.Context, ref string) (digest.Digest, error) { 51 | m.mx.RLock() 52 | defer m.mx.RUnlock() 53 | 54 | dgst, ok := m.tags[ref] 55 | if !ok { 56 | return "", fmt.Errorf("could not resolve tag %s to a digest", ref) 57 | } 58 | return dgst, nil 59 | } 60 | 61 | func (m *Memory) Size(ctx context.Context, dgst digest.Digest) (int64, error) { 62 | m.mx.RLock() 63 | defer m.mx.RUnlock() 64 | 65 | b, ok := m.blobs[dgst] 66 | if !ok { 67 | return 0, errors.Join(ErrNotFound, fmt.Errorf("size information for digest %s not found", dgst)) 68 | } 69 | return int64(len(b)), nil 70 | } 71 | 72 | func (m *Memory) GetManifest(ctx context.Context, dgst digest.Digest) ([]byte, string, error) { 73 | m.mx.RLock() 74 | defer m.mx.RUnlock() 75 | 76 | b, ok := m.blobs[dgst] 77 | if !ok { 78 | return nil, "", errors.Join(ErrNotFound, fmt.Errorf("manifest with digest %s not found", dgst)) 79 | } 80 | mt, err := DetermineMediaType(b) 81 | if err != nil { 82 | return nil, "", err 83 | } 84 | return b, mt, nil 85 | } 86 | 87 | func (m *Memory) GetBlob(ctx context.Context, dgst digest.Digest) (io.ReadSeekCloser, error) { 88 | m.mx.RLock() 89 | defer m.mx.RUnlock() 90 | 91 | b, ok := m.blobs[dgst] 92 | if !ok { 93 | return nil, errors.Join(ErrNotFound, fmt.Errorf("blob with digest %s not found", dgst)) 94 | } 95 | rc := io.NewSectionReader(bytes.NewReader(b), 0, int64(len(b))) 96 | return struct { 97 | io.ReadSeeker 98 | io.Closer 99 | }{ 100 | ReadSeeker: rc, 101 | Closer: io.NopCloser(nil), 102 | }, nil 103 | } 104 | 105 | func (m *Memory) AddImage(img Image) { 106 | m.mx.Lock() 107 | defer m.mx.Unlock() 108 | 109 | m.images = append(m.images, img) 110 | tagName, ok := img.TagName() 111 | if !ok { 112 | return 113 | } 114 | m.tags[tagName] = img.Digest 115 | } 116 | 117 | func (m *Memory) AddBlob(b []byte, dgst digest.Digest) { 118 | m.mx.Lock() 119 | defer m.mx.Unlock() 120 | 121 | m.blobs[dgst] = b 122 | } 123 | -------------------------------------------------------------------------------- /pkg/oci/oci.go: -------------------------------------------------------------------------------- 1 | package oci 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "errors" 7 | "fmt" 8 | "io" 9 | 10 | "github.com/containerd/containerd/v2/core/images" 11 | "github.com/opencontainers/go-digest" 12 | "github.com/opencontainers/image-spec/specs-go" 13 | ocispec "github.com/opencontainers/image-spec/specs-go/v1" 14 | ) 15 | 16 | var ( 17 | ErrNotFound = errors.New("content not found") 18 | ) 19 | 20 | type EventType string 21 | 22 | const ( 23 | CreateEvent EventType = "CREATE" 24 | UpdateEvent EventType = "UPDATE" 25 | DeleteEvent EventType = "DELETE" 26 | ) 27 | 28 | type ImageEvent struct { 29 | Image Image 30 | Type EventType 31 | } 32 | 33 | type Client interface { 34 | // Name returns the name of the Client implementation. 35 | Name() string 36 | 37 | // Verify checks that all expected configuration is set. 38 | Verify(ctx context.Context) error 39 | 40 | // Subscribe will notify for any image events ocuring in the store backend. 41 | Subscribe(ctx context.Context) (<-chan ImageEvent, <-chan error, error) 42 | 43 | // ListImages returns a list of all local images. 44 | ListImages(ctx context.Context) ([]Image, error) 45 | 46 | // Resolve returns the digest for the tagged image name reference. 47 | // The ref is expected to be in the format `registry/name:tag`. 48 | Resolve(ctx context.Context, ref string) (digest.Digest, error) 49 | 50 | // Size returns the content byte size for the given digest. 51 | // Will return ErrNotFound if the digest cannot be found. 52 | Size(ctx context.Context, dgst digest.Digest) (int64, error) 53 | 54 | // GetManifest returns the manifest content for the given digest. 55 | // Will return ErrNotFound if the digest cannot be found. 56 | GetManifest(ctx context.Context, dgst digest.Digest) ([]byte, string, error) 57 | 58 | // GetBlob returns a stream of the blob content for the given digest. 59 | // Will return ErrNotFound if the digest cannot be found. 60 | GetBlob(ctx context.Context, dgst digest.Digest) (io.ReadSeekCloser, error) 61 | } 62 | 63 | type UnknownDocument struct { 64 | MediaType string `json:"mediaType"` 65 | specs.Versioned 66 | } 67 | 68 | func DetermineMediaType(b []byte) (string, error) { 69 | var ud UnknownDocument 70 | if err := json.Unmarshal(b, &ud); err != nil { 71 | return "", err 72 | } 73 | if ud.SchemaVersion == 2 && ud.MediaType != "" { 74 | return ud.MediaType, nil 75 | } 76 | data := map[string]json.RawMessage{} 77 | if err := json.Unmarshal(b, &data); err != nil { 78 | return "", err 79 | } 80 | _, architectureOk := data["architecture"] 81 | _, osOk := data["os"] 82 | _, rootfsOk := data["rootfs"] 83 | if architectureOk && osOk && rootfsOk { 84 | return ocispec.MediaTypeImageConfig, nil 85 | } 86 | _, manifestsOk := data["manifests"] 87 | if ud.SchemaVersion == 2 && manifestsOk { 88 | return ocispec.MediaTypeImageIndex, nil 89 | } 90 | _, configOk := data["config"] 91 | if ud.SchemaVersion == 2 && configOk { 92 | return ocispec.MediaTypeImageManifest, nil 93 | } 94 | return "", errors.New("not able to determine media type") 95 | } 96 | 97 | func WalkImage(ctx context.Context, client Client, img Image) ([]string, error) { 98 | keys := []string{} 99 | err := walk(ctx, []digest.Digest{img.Digest}, func(dgst digest.Digest) ([]digest.Digest, error) { 100 | b, mt, err := client.GetManifest(ctx, dgst) 101 | if err != nil { 102 | return nil, err 103 | } 104 | keys = append(keys, dgst.String()) 105 | switch mt { 106 | case images.MediaTypeDockerSchema2ManifestList, ocispec.MediaTypeImageIndex: 107 | var idx ocispec.Index 108 | if err := json.Unmarshal(b, &idx); err != nil { 109 | return nil, err 110 | } 111 | manifestDgsts := []digest.Digest{} 112 | for _, m := range idx.Manifests { 113 | _, err := client.Size(ctx, m.Digest) 114 | if errors.Is(err, ErrNotFound) { 115 | continue 116 | } 117 | if err != nil { 118 | return nil, err 119 | } 120 | manifestDgsts = append(manifestDgsts, m.Digest) 121 | } 122 | if len(manifestDgsts) == 0 { 123 | return nil, fmt.Errorf("could not find any platforms with local content in manifest %s", dgst) 124 | } 125 | return manifestDgsts, nil 126 | case images.MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest: 127 | var manifest ocispec.Manifest 128 | err := json.Unmarshal(b, &manifest) 129 | if err != nil { 130 | return nil, err 131 | } 132 | keys = append(keys, manifest.Config.Digest.String()) 133 | for _, layer := range manifest.Layers { 134 | keys = append(keys, layer.Digest.String()) 135 | } 136 | return nil, nil 137 | default: 138 | return nil, fmt.Errorf("unexpected media type %s for digest %s", mt, dgst) 139 | } 140 | }) 141 | if err != nil { 142 | return nil, fmt.Errorf("failed to walk image manifests: %w", err) 143 | } 144 | if len(keys) == 0 { 145 | return nil, errors.New("no image digests found") 146 | } 147 | return keys, nil 148 | } 149 | 150 | func walk(ctx context.Context, dgsts []digest.Digest, handler func(dgst digest.Digest) ([]digest.Digest, error)) error { 151 | for _, dgst := range dgsts { 152 | children, err := handler(dgst) 153 | if err != nil { 154 | return err 155 | } 156 | if len(children) == 0 { 157 | continue 158 | } 159 | err = walk(ctx, children, handler) 160 | if err != nil { 161 | return err 162 | } 163 | } 164 | return nil 165 | } 166 | -------------------------------------------------------------------------------- /pkg/oci/testdata/blobs/sha256/0ad7c556c55464fa44d4c41e5236715e015b0266daced62140fb5c6b983c946b: -------------------------------------------------------------------------------- 1 | { 2 | "mediaType": "application/vnd.oci.image.manifest.v1+json", 3 | "schemaVersion": 2, 4 | "config": { 5 | "mediaType": "application/vnd.oci.image.config.v1+json", 6 | "digest": "sha256:1079836371d57a148a0afa5abfe00bd91825c869fcc6574a418f4371d53cab4c", 7 | "size": 2855 8 | }, 9 | "layers": [ 10 | { 11 | "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", 12 | "digest": "sha256:b437b30b8b4cc4e02865517b5ca9b66501752012a028e605da1c98beb0ed9f50", 13 | "size": 103732 14 | }, 15 | { 16 | "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", 17 | "digest": "sha256:fe5ca62666f04366c8e7f605aa82997d71320183e99962fa76b3209fdfbb8b58", 18 | "size": 21202 19 | }, 20 | { 21 | "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", 22 | "digest": "sha256:b02a7525f878e61fc1ef8a7405a2cc17f866e8de222c1c98fd6681aff6e509db", 23 | "size": 716491 24 | }, 25 | { 26 | "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", 27 | "digest": "sha256:fcb6f6d2c9986d9cd6a2ea3cc2936e5fc613e09f1af9042329011e43057f3265", 28 | "size": 317 29 | }, 30 | { 31 | "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", 32 | "digest": "sha256:e8c73c638ae9ec5ad70c49df7e484040d889cca6b4a9af056579c3d058ea93f0", 33 | "size": 198 34 | }, 35 | { 36 | "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", 37 | "digest": "sha256:1e3d9b7d145208fa8fa3ee1c9612d0adaac7255f1bbc9ddea7e461e0b317805c", 38 | "size": 113 39 | }, 40 | { 41 | "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", 42 | "digest": "sha256:4aa0ea1413d37a58615488592a0b827ea4b2e48fa5a77cf707d0e35f025e613f", 43 | "size": 385 44 | }, 45 | { 46 | "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", 47 | "digest": "sha256:7c881f9ab25e0d86562a123b5fb56aebf8aa0ddd7d48ef602faf8d1e7cf43d8c", 48 | "size": 355 49 | }, 50 | { 51 | "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", 52 | "digest": "sha256:5627a970d25e752d971a501ec7e35d0d6fdcd4a3ce9e958715a686853024794a", 53 | "size": 130562 54 | }, 55 | { 56 | "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", 57 | "digest": "sha256:01d28554416aa05390e2827a653a1289a2a549e46cc78d65915a75377c6008ba", 58 | "size": 34318536 59 | }, 60 | { 61 | "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", 62 | "digest": "sha256:4f4fb700ef54461cfa02571ae0db9a0dc1e0cdb5577484a6d75e68dc38e8acc1", 63 | "size": 32 64 | } 65 | ] 66 | } -------------------------------------------------------------------------------- /pkg/oci/testdata/blobs/sha256/3caa2469de2a23cbcc209dd0b9d01cd78ff9a0f88741655991d36baede5b0996: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spegel-org/spegel/ba39b19f99cd95cf014d66e562d655a0db3b7b24/pkg/oci/testdata/blobs/sha256/3caa2469de2a23cbcc209dd0b9d01cd78ff9a0f88741655991d36baede5b0996 -------------------------------------------------------------------------------- /pkg/oci/testdata/blobs/sha256/44cb2cf712c060f69df7310e99339c1eb51a085446f1bb6d44469acff35b4355: -------------------------------------------------------------------------------- 1 | { 2 | "mediaType": "application/vnd.oci.image.manifest.v1+json", 3 | "schemaVersion": 2, 4 | "config": { 5 | "mediaType": "application/vnd.oci.image.config.v1+json", 6 | "digest": "sha256:d715ba0d85ee7d37da627d0679652680ed2cb23dde6120f25143a0b8079ee47e", 7 | "size": 2842 8 | }, 9 | "layers": [ 10 | { 11 | "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", 12 | "digest": "sha256:a7ca0d9ba68fdce7e15bc0952d3e898e970548ca24d57698725836c039086639", 13 | "size": 103732 14 | }, 15 | { 16 | "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", 17 | "digest": "sha256:fe5ca62666f04366c8e7f605aa82997d71320183e99962fa76b3209fdfbb8b58", 18 | "size": 21202 19 | }, 20 | { 21 | "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", 22 | "digest": "sha256:b02a7525f878e61fc1ef8a7405a2cc17f866e8de222c1c98fd6681aff6e509db", 23 | "size": 716491 24 | }, 25 | { 26 | "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", 27 | "digest": "sha256:fcb6f6d2c9986d9cd6a2ea3cc2936e5fc613e09f1af9042329011e43057f3265", 28 | "size": 317 29 | }, 30 | { 31 | "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", 32 | "digest": "sha256:e8c73c638ae9ec5ad70c49df7e484040d889cca6b4a9af056579c3d058ea93f0", 33 | "size": 198 34 | }, 35 | { 36 | "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", 37 | "digest": "sha256:1e3d9b7d145208fa8fa3ee1c9612d0adaac7255f1bbc9ddea7e461e0b317805c", 38 | "size": 113 39 | }, 40 | { 41 | "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", 42 | "digest": "sha256:4aa0ea1413d37a58615488592a0b827ea4b2e48fa5a77cf707d0e35f025e613f", 43 | "size": 385 44 | }, 45 | { 46 | "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", 47 | "digest": "sha256:7c881f9ab25e0d86562a123b5fb56aebf8aa0ddd7d48ef602faf8d1e7cf43d8c", 48 | "size": 355 49 | }, 50 | { 51 | "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", 52 | "digest": "sha256:5627a970d25e752d971a501ec7e35d0d6fdcd4a3ce9e958715a686853024794a", 53 | "size": 130562 54 | }, 55 | { 56 | "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", 57 | "digest": "sha256:76f3a495ffdc00c612747ba0c59fc56d0a2610d2785e80e9edddbf214c2709ef", 58 | "size": 36529876 59 | }, 60 | { 61 | "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", 62 | "digest": "sha256:4f4fb700ef54461cfa02571ae0db9a0dc1e0cdb5577484a6d75e68dc38e8acc1", 63 | "size": 32 64 | } 65 | ] 66 | } -------------------------------------------------------------------------------- /pkg/oci/testdata/blobs/sha256/68b8a989a3e08ddbdb3a0077d35c0d0e59c9ecf23d0634584def8bdbb7d6824f: -------------------------------------------------------------------------------- 1 | {"architecture":"amd64","config":{"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"WorkingDir":"/","OnBuild":null},"created":"2024-02-13T20:57:33.241342971+01:00","history":[{"created":"2024-02-13T20:57:33.241342971+01:00","created_by":"COPY test.txt . # buildkit","comment":"buildkit.dockerfile.v0"}],"moby.buildkit.buildinfo.v1":"eyJmcm9udGVuZCI6ImRvY2tlcmZpbGUudjAifQ==","os":"linux","rootfs":{"type":"layers","diff_ids":["sha256:788bbd044fb43c00246a6351dae95d8a3d2510ba680dd404d5bae5e80479484d"]}} -------------------------------------------------------------------------------- /pkg/oci/testdata/blobs/sha256/9430beb291fa7b96997711fc486bc46133c719631aefdbeebe58dd3489217bfe: -------------------------------------------------------------------------------- 1 | { 2 | "mediaType": "application/vnd.oci.image.index.v1+json", 3 | "schemaVersion": 2, 4 | "manifests": [ 5 | { 6 | "mediaType": "application/vnd.oci.image.manifest.v1+json", 7 | "digest": "sha256:aec8273a5e5aca369fcaa8cecef7bf6c7959d482f5c8cfa2236a6a16e46bbdcf", 8 | "size": 476, 9 | "platform": { 10 | "architecture": "amd64", 11 | "os": "linux" 12 | } 13 | } 14 | ] 15 | } -------------------------------------------------------------------------------- /pkg/oci/testdata/blobs/sha256/9506c8e7a2d0a098d43cadfd7ecdc3c91697e8188d3a1245943b669f717747b4: -------------------------------------------------------------------------------- 1 | { 2 | "mediaType": "application/vnd.oci.image.index.v1+json", 3 | "schemaVersion": 2, 4 | "manifests": [ 5 | { 6 | "mediaType": "application/vnd.oci.image.manifest.v1+json", 7 | "digest": "sha256:32cb2cf712c060f69df7310e99339c1eb51a085446f1bb6d44469acff35b4355", 8 | "size": 2372, 9 | "platform": { 10 | "architecture": "fake", 11 | "os": "linux" 12 | } 13 | }, 14 | { 15 | "mediaType": "application/vnd.oci.image.manifest.v1+json", 16 | "digest": "sha256:44cb2cf712c060f69df7310e99339c1eb51a085446f1bb6d44469acff35b4355", 17 | "size": 2372, 18 | "platform": { 19 | "architecture": "amd64", 20 | "os": "linux" 21 | } 22 | }, 23 | { 24 | "mediaType": "application/vnd.oci.image.manifest.v1+json", 25 | "digest": "sha256:0ad7c556c55464fa44d4c41e5236715e015b0266daced62140fb5c6b983c946b", 26 | "size": 2372, 27 | "platform": { 28 | "architecture": "arm", 29 | "os": "linux", 30 | "variant": "v7" 31 | } 32 | }, 33 | { 34 | "mediaType": "application/vnd.oci.image.manifest.v1+json", 35 | "digest": "sha256:dce623533c59af554b85f859e91fc1cbb7f574e873c82f36b9ea05a09feb0b53", 36 | "size": 2372, 37 | "platform": { 38 | "architecture": "arm64", 39 | "os": "linux" 40 | } 41 | }, 42 | { 43 | "mediaType": "application/vnd.oci.image.manifest.v1+json", 44 | "digest": "sha256:73af5483f4d2d636275dcef14d5443ff96d7347a0720ca5a73a32c73855c4aac", 45 | "size": 566, 46 | "annotations": { 47 | "vnd.docker.reference.digest": "sha256:44cb2cf712c060f69df7310e99339c1eb51a085446f1bb6d44469acff35b4355", 48 | "vnd.docker.reference.type": "attestation-manifest" 49 | }, 50 | "platform": { 51 | "architecture": "unknown", 52 | "os": "unknown" 53 | } 54 | }, 55 | { 56 | "mediaType": "application/vnd.oci.image.manifest.v1+json", 57 | "digest": "sha256:36e11bf470af256febbdfad9d803e60b7290b0268218952991b392be9e8153bd", 58 | "size": 566, 59 | "annotations": { 60 | "vnd.docker.reference.digest": "sha256:0ad7c556c55464fa44d4c41e5236715e015b0266daced62140fb5c6b983c946b", 61 | "vnd.docker.reference.type": "attestation-manifest" 62 | }, 63 | "platform": { 64 | "architecture": "unknown", 65 | "os": "unknown" 66 | } 67 | }, 68 | { 69 | "mediaType": "application/vnd.oci.image.manifest.v1+json", 70 | "digest": "sha256:42d1c43f2285e8e3d39f80b8eed8e4c5c28b8011c942b5413ecc6a0050600609", 71 | "size": 566, 72 | "annotations": { 73 | "vnd.docker.reference.digest": "sha256:dce623533c59af554b85f859e91fc1cbb7f574e873c82f36b9ea05a09feb0b53", 74 | "vnd.docker.reference.type": "attestation-manifest" 75 | }, 76 | "platform": { 77 | "architecture": "unknown", 78 | "os": "unknown" 79 | } 80 | } 81 | ] 82 | } -------------------------------------------------------------------------------- /pkg/oci/testdata/blobs/sha256/addc990c58744bdf96364fe89bd4aab38b1e824d51c688edb36c75247cd45fa9: -------------------------------------------------------------------------------- 1 | { 2 | "mediaType": "application/vnd.oci.image.index.v1+json", 3 | "schemaVersion": 2, 4 | "manifests": [ 5 | { 6 | "mediaType": "application/vnd.oci.image.manifest.v1+json", 7 | "digest": "sha256:2fc401df92a31e32189eb27d9bd1e2fa649ff42a665557ec4aa3eaf5de2685df", 8 | "size": 2372, 9 | "platform": { 10 | "architecture": "amd64", 11 | "os": "linux" 12 | } 13 | }, 14 | { 15 | "mediaType": "application/vnd.oci.image.manifest.v1+json", 16 | "digest": "sha256:f9fc991c2afd0fa22cb4999b1134a937f6e15820a58426ba288a0aec8207cf07", 17 | "size": 2372, 18 | "platform": { 19 | "architecture": "arm", 20 | "os": "linux", 21 | "variant": "v7" 22 | } 23 | }, 24 | { 25 | "mediaType": "application/vnd.oci.image.manifest.v1+json", 26 | "digest": "sha256:dbd95b715407e29b42cd0366e593b545a81c901dec93427860c87d7a87044441", 27 | "size": 2372, 28 | "platform": { 29 | "architecture": "arm64", 30 | "os": "linux" 31 | } 32 | } 33 | ] 34 | } -------------------------------------------------------------------------------- /pkg/oci/testdata/blobs/sha256/aec8273a5e5aca369fcaa8cecef7bf6c7959d482f5c8cfa2236a6a16e46bbdcf: -------------------------------------------------------------------------------- 1 | { 2 | "mediaType": "application/vnd.oci.image.manifest.v1+json", 3 | "schemaVersion": 2, 4 | "config": { 5 | "mediaType": "application/vnd.oci.image.config.v1+json", 6 | "digest": "sha256:68b8a989a3e08ddbdb3a0077d35c0d0e59c9ecf23d0634584def8bdbb7d6824f", 7 | "size": 529 8 | }, 9 | "layers": [ 10 | { 11 | "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", 12 | "digest": "sha256:3caa2469de2a23cbcc209dd0b9d01cd78ff9a0f88741655991d36baede5b0996", 13 | "size": 118 14 | } 15 | ] 16 | } -------------------------------------------------------------------------------- /pkg/oci/testdata/blobs/sha256/b6d6089ca6c395fd563c2084f5dd7bc56a2f5e6a81413558c5be0083287a77e9: -------------------------------------------------------------------------------- 1 | { 2 | "schemaVersion": 2, 3 | "config": { 4 | "mediaType": "application/vnd.oci.image.config.v1+json", 5 | "digest": "sha256:68b8a989a3e08ddbdb3a0077d35c0d0e59c9ecf23d0634584def8bdbb7d6824f", 6 | "size": 529 7 | }, 8 | "layers": [ 9 | { 10 | "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", 11 | "digest": "sha256:3caa2469de2a23cbcc209dd0b9d01cd78ff9a0f88741655991d36baede5b0996", 12 | "size": 118 13 | } 14 | ] 15 | } 16 | -------------------------------------------------------------------------------- /pkg/oci/testdata/blobs/sha256/d8df04365d06181f037251de953aca85cc16457581a8fc168f4957c978e1008b: -------------------------------------------------------------------------------- 1 | { 2 | "schemaVersion": 2, 3 | "manifests": [ 4 | { 5 | "mediaType": "application/vnd.oci.image.manifest.v1+json", 6 | "digest": "sha256:32cb2cf712c060f69df7310e99339c1eb51a085446f1bb6d44469acff35b4355", 7 | "size": 2372, 8 | "platform": { 9 | "architecture": "fake", 10 | "os": "linux" 11 | } 12 | }, 13 | { 14 | "mediaType": "application/vnd.oci.image.manifest.v1+json", 15 | "digest": "sha256:44cb2cf712c060f69df7310e99339c1eb51a085446f1bb6d44469acff35b4355", 16 | "size": 2372, 17 | "platform": { 18 | "architecture": "amd64", 19 | "os": "linux" 20 | } 21 | }, 22 | { 23 | "mediaType": "application/vnd.oci.image.manifest.v1+json", 24 | "digest": "sha256:0ad7c556c55464fa44d4c41e5236715e015b0266daced62140fb5c6b983c946b", 25 | "size": 2372, 26 | "platform": { 27 | "architecture": "arm", 28 | "os": "linux", 29 | "variant": "v7" 30 | } 31 | }, 32 | { 33 | "mediaType": "application/vnd.oci.image.manifest.v1+json", 34 | "digest": "sha256:dce623533c59af554b85f859e91fc1cbb7f574e873c82f36b9ea05a09feb0b53", 35 | "size": 2372, 36 | "platform": { 37 | "architecture": "arm64", 38 | "os": "linux" 39 | } 40 | }, 41 | { 42 | "mediaType": "application/vnd.oci.image.manifest.v1+json", 43 | "digest": "sha256:73af5483f4d2d636275dcef14d5443ff96d7347a0720ca5a73a32c73855c4aac", 44 | "size": 566, 45 | "annotations": { 46 | "vnd.docker.reference.digest": "sha256:44cb2cf712c060f69df7310e99339c1eb51a085446f1bb6d44469acff35b4355", 47 | "vnd.docker.reference.type": "attestation-manifest" 48 | }, 49 | "platform": { 50 | "architecture": "unknown", 51 | "os": "unknown" 52 | } 53 | }, 54 | { 55 | "mediaType": "application/vnd.oci.image.manifest.v1+json", 56 | "digest": "sha256:36e11bf470af256febbdfad9d803e60b7290b0268218952991b392be9e8153bd", 57 | "size": 566, 58 | "annotations": { 59 | "vnd.docker.reference.digest": "sha256:0ad7c556c55464fa44d4c41e5236715e015b0266daced62140fb5c6b983c946b", 60 | "vnd.docker.reference.type": "attestation-manifest" 61 | }, 62 | "platform": { 63 | "architecture": "unknown", 64 | "os": "unknown" 65 | } 66 | }, 67 | { 68 | "mediaType": "application/vnd.oci.image.manifest.v1+json", 69 | "digest": "sha256:42d1c43f2285e8e3d39f80b8eed8e4c5c28b8011c942b5413ecc6a0050600609", 70 | "size": 566, 71 | "annotations": { 72 | "vnd.docker.reference.digest": "sha256:dce623533c59af554b85f859e91fc1cbb7f574e873c82f36b9ea05a09feb0b53", 73 | "vnd.docker.reference.type": "attestation-manifest" 74 | }, 75 | "platform": { 76 | "architecture": "unknown", 77 | "os": "unknown" 78 | } 79 | } 80 | ] 81 | } -------------------------------------------------------------------------------- /pkg/oci/testdata/blobs/sha256/dce623533c59af554b85f859e91fc1cbb7f574e873c82f36b9ea05a09feb0b53: -------------------------------------------------------------------------------- 1 | { 2 | "mediaType": "application/vnd.oci.image.manifest.v1+json", 3 | "schemaVersion": 2, 4 | "config": { 5 | "mediaType": "application/vnd.oci.image.config.v1+json", 6 | "digest": "sha256:c73129c9fb699b620aac2df472196ed41797fd0f5a90e1942bfbf19849c4a1c9", 7 | "size": 2842 8 | }, 9 | "layers": [ 10 | { 11 | "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", 12 | "digest": "sha256:0b41f743fd4d78cb50ba86dd3b951b51458744109e1f5063a76bc5a792c3d8e7", 13 | "size": 103732 14 | }, 15 | { 16 | "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", 17 | "digest": "sha256:fe5ca62666f04366c8e7f605aa82997d71320183e99962fa76b3209fdfbb8b58", 18 | "size": 21202 19 | }, 20 | { 21 | "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", 22 | "digest": "sha256:b02a7525f878e61fc1ef8a7405a2cc17f866e8de222c1c98fd6681aff6e509db", 23 | "size": 716491 24 | }, 25 | { 26 | "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", 27 | "digest": "sha256:fcb6f6d2c9986d9cd6a2ea3cc2936e5fc613e09f1af9042329011e43057f3265", 28 | "size": 317 29 | }, 30 | { 31 | "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", 32 | "digest": "sha256:e8c73c638ae9ec5ad70c49df7e484040d889cca6b4a9af056579c3d058ea93f0", 33 | "size": 198 34 | }, 35 | { 36 | "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", 37 | "digest": "sha256:1e3d9b7d145208fa8fa3ee1c9612d0adaac7255f1bbc9ddea7e461e0b317805c", 38 | "size": 113 39 | }, 40 | { 41 | "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", 42 | "digest": "sha256:4aa0ea1413d37a58615488592a0b827ea4b2e48fa5a77cf707d0e35f025e613f", 43 | "size": 385 44 | }, 45 | { 46 | "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", 47 | "digest": "sha256:7c881f9ab25e0d86562a123b5fb56aebf8aa0ddd7d48ef602faf8d1e7cf43d8c", 48 | "size": 355 49 | }, 50 | { 51 | "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", 52 | "digest": "sha256:5627a970d25e752d971a501ec7e35d0d6fdcd4a3ce9e958715a686853024794a", 53 | "size": 130562 54 | }, 55 | { 56 | "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", 57 | "digest": "sha256:0dc769edeab7d9f622b9703579f6c89298a4cf45a84af1908e26fffca55341e1", 58 | "size": 34168923 59 | }, 60 | { 61 | "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", 62 | "digest": "sha256:4f4fb700ef54461cfa02571ae0db9a0dc1e0cdb5577484a6d75e68dc38e8acc1", 63 | "size": 32 64 | } 65 | ] 66 | } -------------------------------------------------------------------------------- /pkg/oci/testdata/images.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "example.com/org/scratch:latest", 4 | "mediaType": "application/vnd.oci.image.index.v1+json", 5 | "digest": "sha256:9430beb291fa7b96997711fc486bc46133c719631aefdbeebe58dd3489217bfe" 6 | }, 7 | { 8 | "name": "ghcr.io/spegel-org/spegel:v0.0.8", 9 | "mediaType": "application/vnd.oci.image.index.v1+json", 10 | "digest": "sha256:9506c8e7a2d0a098d43cadfd7ecdc3c91697e8188d3a1245943b669f717747b4" 11 | }, 12 | { 13 | "name": "ghcr.io/spegel-org/spegel:v0.0.8-with-media-type", 14 | "mediaType": "application/vnd.oci.image.index.v1+json", 15 | "digest": "sha256:9506c8e7a2d0a098d43cadfd7ecdc3c91697e8188d3a1245943b669f717747b4" 16 | }, 17 | { 18 | "name": "ghcr.io/spegel-org/spegel:v0.0.8-without-media-type", 19 | "mediaType": "application/vnd.oci.image.index.v1+json", 20 | "digest": "sha256:d8df04365d06181f037251de953aca85cc16457581a8fc168f4957c978e1008b" 21 | }, 22 | { 23 | "name": "example.com/org/no-platform:test", 24 | "mediaType": "application/vnd.oci.image.index.v1+json", 25 | "digest": "sha256:addc990c58744bdf96364fe89bd4aab38b1e824d51c688edb36c75247cd45fa9" 26 | } 27 | ] -------------------------------------------------------------------------------- /pkg/registry/registry.go: -------------------------------------------------------------------------------- 1 | package registry 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "io" 8 | "net" 9 | "net/http" 10 | "net/netip" 11 | "net/url" 12 | "path" 13 | "strconv" 14 | "strings" 15 | "sync" 16 | "time" 17 | 18 | "github.com/go-logr/logr" 19 | 20 | "github.com/spegel-org/spegel/internal/mux" 21 | "github.com/spegel-org/spegel/pkg/metrics" 22 | "github.com/spegel-org/spegel/pkg/oci" 23 | "github.com/spegel-org/spegel/pkg/routing" 24 | ) 25 | 26 | const ( 27 | MirroredHeaderKey = "X-Spegel-Mirrored" 28 | ) 29 | 30 | type RegistryConfig struct { 31 | Client *http.Client 32 | Log logr.Logger 33 | Username string 34 | Password string 35 | ResolveRetries int 36 | ResolveLatestTag bool 37 | ResolveTimeout time.Duration 38 | } 39 | 40 | func (cfg *RegistryConfig) Apply(opts ...RegistryOption) error { 41 | for _, opt := range opts { 42 | if opt == nil { 43 | continue 44 | } 45 | if err := opt(cfg); err != nil { 46 | return err 47 | } 48 | } 49 | return nil 50 | } 51 | 52 | type RegistryOption func(cfg *RegistryConfig) error 53 | 54 | func WithResolveRetries(resolveRetries int) RegistryOption { 55 | return func(cfg *RegistryConfig) error { 56 | cfg.ResolveRetries = resolveRetries 57 | return nil 58 | } 59 | } 60 | 61 | func WithResolveLatestTag(resolveLatestTag bool) RegistryOption { 62 | return func(cfg *RegistryConfig) error { 63 | cfg.ResolveLatestTag = resolveLatestTag 64 | return nil 65 | } 66 | } 67 | 68 | func WithResolveTimeout(resolveTimeout time.Duration) RegistryOption { 69 | return func(cfg *RegistryConfig) error { 70 | cfg.ResolveTimeout = resolveTimeout 71 | return nil 72 | } 73 | } 74 | 75 | func WithTransport(transport http.RoundTripper) RegistryOption { 76 | return func(cfg *RegistryConfig) error { 77 | if cfg.Client == nil { 78 | cfg.Client = &http.Client{} 79 | } 80 | cfg.Client.Transport = transport 81 | return nil 82 | } 83 | } 84 | 85 | func WithLogger(log logr.Logger) RegistryOption { 86 | return func(cfg *RegistryConfig) error { 87 | cfg.Log = log 88 | return nil 89 | } 90 | } 91 | 92 | func WithBasicAuth(username, password string) RegistryOption { 93 | return func(cfg *RegistryConfig) error { 94 | cfg.Username = username 95 | cfg.Password = password 96 | return nil 97 | } 98 | } 99 | 100 | type Registry struct { 101 | client *http.Client 102 | bufferPool *sync.Pool 103 | log logr.Logger 104 | ociClient oci.Client 105 | router routing.Router 106 | username string 107 | password string 108 | resolveRetries int 109 | resolveTimeout time.Duration 110 | resolveLatestTag bool 111 | } 112 | 113 | func NewRegistry(ociClient oci.Client, router routing.Router, opts ...RegistryOption) (*Registry, error) { 114 | transport, ok := http.DefaultTransport.(*http.Transport) 115 | if !ok { 116 | return nil, errors.New("default transporn is not of type http.Transport") 117 | } 118 | cfg := RegistryConfig{ 119 | Client: &http.Client{ 120 | Transport: transport.Clone(), 121 | }, 122 | Log: logr.Discard(), 123 | ResolveRetries: 3, 124 | ResolveLatestTag: true, 125 | ResolveTimeout: 20 * time.Millisecond, 126 | } 127 | err := cfg.Apply(opts...) 128 | if err != nil { 129 | return nil, err 130 | } 131 | 132 | bufferPool := &sync.Pool{ 133 | New: func() any { 134 | buf := make([]byte, 32*1024) 135 | return &buf 136 | }, 137 | } 138 | r := &Registry{ 139 | ociClient: ociClient, 140 | router: router, 141 | client: cfg.Client, 142 | log: cfg.Log, 143 | resolveRetries: cfg.ResolveRetries, 144 | resolveLatestTag: cfg.ResolveLatestTag, 145 | resolveTimeout: cfg.ResolveTimeout, 146 | username: cfg.Username, 147 | password: cfg.Password, 148 | bufferPool: bufferPool, 149 | } 150 | return r, nil 151 | } 152 | 153 | func (r *Registry) Server(addr string) (*http.Server, error) { 154 | m, err := mux.NewServeMux(r.handle) 155 | if err != nil { 156 | return nil, err 157 | } 158 | srv := &http.Server{ 159 | Addr: addr, 160 | Handler: m, 161 | } 162 | return srv, nil 163 | } 164 | 165 | func (r *Registry) handle(rw mux.ResponseWriter, req *http.Request) { 166 | start := time.Now() 167 | handler := "" 168 | path := req.URL.Path 169 | if strings.HasPrefix(path, "/v2") { 170 | path = "/v2/*" 171 | } 172 | defer func() { 173 | latency := time.Since(start) 174 | statusCode := strconv.FormatInt(int64(rw.Status()), 10) 175 | 176 | metrics.HttpRequestsInflight.WithLabelValues(path).Add(-1) 177 | metrics.HttpRequestDurHistogram.WithLabelValues(path, req.Method, statusCode).Observe(latency.Seconds()) 178 | metrics.HttpResponseSizeHistogram.WithLabelValues(path, req.Method, statusCode).Observe(float64(rw.Size())) 179 | 180 | // Ignore logging requests to healthz to reduce log noise 181 | if req.URL.Path == "/healthz" { 182 | return 183 | } 184 | 185 | kvs := []any{ 186 | "path", req.URL.Path, 187 | "status", rw.Status(), 188 | "method", req.Method, 189 | "latency", latency.String(), 190 | "ip", getClientIP(req), 191 | "handler", handler, 192 | } 193 | if rw.Status() >= 200 && rw.Status() < 300 { 194 | r.log.Info("", kvs...) 195 | return 196 | } 197 | r.log.Error(rw.Error(), "", kvs...) 198 | }() 199 | metrics.HttpRequestsInflight.WithLabelValues(path).Add(1) 200 | 201 | if req.URL.Path == "/healthz" && req.Method == http.MethodGet { 202 | r.readyHandler(rw, req) 203 | handler = "ready" 204 | return 205 | } 206 | if strings.HasPrefix(req.URL.Path, "/v2") && (req.Method == http.MethodGet || req.Method == http.MethodHead) { 207 | handler = r.registryHandler(rw, req) 208 | return 209 | } 210 | rw.WriteHeader(http.StatusNotFound) 211 | } 212 | 213 | func (r *Registry) readyHandler(rw mux.ResponseWriter, req *http.Request) { 214 | ok, err := r.router.Ready(req.Context()) 215 | if err != nil { 216 | rw.WriteError(http.StatusInternalServerError, fmt.Errorf("could not determine router readiness: %w", err)) 217 | return 218 | } 219 | if !ok { 220 | rw.WriteHeader(http.StatusInternalServerError) 221 | return 222 | } 223 | } 224 | 225 | func (r *Registry) registryHandler(rw mux.ResponseWriter, req *http.Request) string { 226 | // Check basic authentication 227 | if r.username != "" || r.password != "" { 228 | username, password, _ := req.BasicAuth() 229 | if r.username != username || r.password != password { 230 | rw.WriteError(http.StatusUnauthorized, errors.New("invalid basic authentication")) 231 | return "registry" 232 | } 233 | } 234 | 235 | // Quickly return 200 for /v2 to indicate that registry supports v2. 236 | if path.Clean(req.URL.Path) == "/v2" { 237 | rw.WriteHeader(http.StatusOK) 238 | return "v2" 239 | } 240 | 241 | // Parse out path components from request. 242 | dist, err := oci.ParseDistributionPath(req.URL) 243 | if err != nil { 244 | rw.WriteError(http.StatusNotFound, fmt.Errorf("could not parse path according to OCI distribution spec: %w", err)) 245 | return "registry" 246 | } 247 | 248 | // Request with mirror header are proxied. 249 | if req.Header.Get(MirroredHeaderKey) != "true" { 250 | // Set mirrored header in request to stop infinite loops 251 | req.Header.Set(MirroredHeaderKey, "true") 252 | 253 | // If content is present locally we should skip the mirroring and just serve it. 254 | var ociErr error 255 | if dist.Digest == "" { 256 | _, ociErr = r.ociClient.Resolve(req.Context(), dist.Reference()) 257 | } else { 258 | _, ociErr = r.ociClient.Size(req.Context(), dist.Digest) 259 | } 260 | if ociErr != nil { 261 | r.handleMirror(rw, req, dist) 262 | return "mirror" 263 | } 264 | } 265 | 266 | // Serve registry endpoints. 267 | switch dist.Kind { 268 | case oci.DistributionKindManifest: 269 | r.handleManifest(rw, req, dist) 270 | return "manifest" 271 | case oci.DistributionKindBlob: 272 | r.handleBlob(rw, req, dist) 273 | return "blob" 274 | default: 275 | rw.WriteError(http.StatusNotFound, fmt.Errorf("unknown distribution path kind %s", dist.Kind)) 276 | return "registry" 277 | } 278 | } 279 | 280 | func (r *Registry) handleMirror(rw mux.ResponseWriter, req *http.Request, dist oci.DistributionPath) { 281 | log := r.log.WithValues("ref", dist.Reference(), "path", req.URL.Path, "ip", getClientIP(req)) 282 | 283 | defer func() { 284 | cacheType := "hit" 285 | if rw.Status() != http.StatusOK { 286 | cacheType = "miss" 287 | } 288 | metrics.MirrorRequestsTotal.WithLabelValues(dist.Registry, cacheType).Inc() 289 | }() 290 | 291 | if !r.resolveLatestTag && dist.IsLatestTag() { 292 | r.log.V(4).Info("skipping mirror request for image with latest tag", "image", dist.Reference()) 293 | rw.WriteHeader(http.StatusNotFound) 294 | return 295 | } 296 | 297 | // Resolve mirror with the requested reference 298 | resolveCtx, cancel := context.WithTimeout(req.Context(), r.resolveTimeout) 299 | defer cancel() 300 | resolveCtx = logr.NewContext(resolveCtx, log) 301 | peerCh, err := r.router.Resolve(resolveCtx, dist.Reference(), r.resolveRetries) 302 | if err != nil { 303 | rw.WriteError(http.StatusInternalServerError, fmt.Errorf("error occurred when attempting to resolve mirrors: %w", err)) 304 | return 305 | } 306 | 307 | mirrorAttempts := 0 308 | for { 309 | select { 310 | case <-req.Context().Done(): 311 | // Request has been closed by server or client. No use continuing. 312 | rw.WriteError(http.StatusNotFound, fmt.Errorf("mirroring for image component %s has been cancelled: %w", dist.Reference(), resolveCtx.Err())) 313 | return 314 | case peer, ok := <-peerCh: 315 | // Channel closed means no more mirrors will be received and max retries has been reached. 316 | if !ok { 317 | err = fmt.Errorf("mirror with image component %s could not be found", dist.Reference()) 318 | if mirrorAttempts > 0 { 319 | err = errors.Join(err, fmt.Errorf("requests to %d mirrors failed, all attempts have been exhausted or timeout has been reached", mirrorAttempts)) 320 | } 321 | rw.WriteError(http.StatusNotFound, err) 322 | return 323 | } 324 | 325 | mirrorAttempts++ 326 | 327 | err := forwardRequest(r.client, r.bufferPool, req, rw, peer) 328 | if err != nil { 329 | log.Error(err, "request to mirror failed", "attempt", mirrorAttempts, "path", req.URL.Path, "mirror", peer) 330 | continue 331 | } 332 | log.V(4).Info("mirrored request", "path", req.URL.Path, "mirror", peer) 333 | return 334 | } 335 | } 336 | } 337 | 338 | func (r *Registry) handleManifest(rw mux.ResponseWriter, req *http.Request, dist oci.DistributionPath) { 339 | if dist.Digest == "" { 340 | dgst, err := r.ociClient.Resolve(req.Context(), dist.Reference()) 341 | if err != nil { 342 | rw.WriteError(http.StatusNotFound, fmt.Errorf("could not get digest for image %s: %w", dist.Reference(), err)) 343 | return 344 | } 345 | dist.Digest = dgst 346 | } 347 | b, mediaType, err := r.ociClient.GetManifest(req.Context(), dist.Digest) 348 | if err != nil { 349 | rw.WriteError(http.StatusNotFound, fmt.Errorf("could not get manifest content for digest %s: %w", dist.Digest.String(), err)) 350 | return 351 | } 352 | rw.Header().Set("Content-Type", mediaType) 353 | rw.Header().Set("Content-Length", strconv.FormatInt(int64(len(b)), 10)) 354 | rw.Header().Set("Docker-Content-Digest", dist.Digest.String()) 355 | if req.Method == http.MethodHead { 356 | return 357 | } 358 | _, err = rw.Write(b) 359 | if err != nil { 360 | r.log.Error(err, "error occurred when writing manifest") 361 | return 362 | } 363 | } 364 | 365 | func (r *Registry) handleBlob(rw mux.ResponseWriter, req *http.Request, dist oci.DistributionPath) { 366 | size, err := r.ociClient.Size(req.Context(), dist.Digest) 367 | if err != nil { 368 | rw.WriteError(http.StatusInternalServerError, fmt.Errorf("could not determine size of blob with digest %s: %w", dist.Digest.String(), err)) 369 | return 370 | } 371 | rw.Header().Set("Accept-Ranges", "bytes") 372 | rw.Header().Set("Content-Type", "application/octet-stream") 373 | rw.Header().Set("Content-Length", strconv.FormatInt(size, 10)) 374 | rw.Header().Set("Docker-Content-Digest", dist.Digest.String()) 375 | if req.Method == http.MethodHead { 376 | return 377 | } 378 | 379 | rc, err := r.ociClient.GetBlob(req.Context(), dist.Digest) 380 | if err != nil { 381 | rw.WriteError(http.StatusInternalServerError, fmt.Errorf("could not get reader for blob with digest %s: %w", dist.Digest.String(), err)) 382 | return 383 | } 384 | defer rc.Close() 385 | 386 | http.ServeContent(rw, req, "", time.Time{}, rc) 387 | } 388 | 389 | func forwardRequest(client *http.Client, bufferPool *sync.Pool, req *http.Request, rw http.ResponseWriter, addrPort netip.AddrPort) error { 390 | // Do request to mirror. 391 | forwardScheme := "http" 392 | if req.TLS != nil { 393 | forwardScheme = "https" 394 | } 395 | u := &url.URL{ 396 | Scheme: forwardScheme, 397 | Host: addrPort.String(), 398 | Path: req.URL.Path, 399 | RawQuery: req.URL.RawQuery, 400 | } 401 | forwardReq, err := http.NewRequestWithContext(req.Context(), req.Method, u.String(), nil) 402 | if err != nil { 403 | return err 404 | } 405 | copyHeader(forwardReq.Header, req.Header) 406 | forwardResp, err := client.Do(forwardReq) 407 | if err != nil { 408 | return err 409 | } 410 | defer forwardResp.Body.Close() 411 | 412 | // Clear body and try next if non 200 response. 413 | if forwardResp.StatusCode != http.StatusOK { 414 | _, err = io.Copy(io.Discard, forwardResp.Body) 415 | if err != nil { 416 | return err 417 | } 418 | return fmt.Errorf("expected mirror to respond with 200 OK but received: %s", forwardResp.Status) 419 | } 420 | 421 | // TODO (phillebaba): Is it possible to retry if copy fails half way through? 422 | // Copy forward response to response writer. 423 | copyHeader(rw.Header(), forwardResp.Header) 424 | rw.WriteHeader(http.StatusOK) 425 | //nolint: errcheck // Ignore 426 | buf := bufferPool.Get().(*[]byte) 427 | defer bufferPool.Put(buf) 428 | _, err = io.CopyBuffer(rw, forwardResp.Body, *buf) 429 | if err != nil { 430 | return err 431 | } 432 | return nil 433 | } 434 | 435 | func copyHeader(dst, src http.Header) { 436 | for k, vv := range src { 437 | for _, v := range vv { 438 | dst.Add(k, v) 439 | } 440 | } 441 | } 442 | 443 | func getClientIP(req *http.Request) string { 444 | forwardedFor := req.Header.Get("X-Forwarded-For") 445 | if forwardedFor != "" { 446 | comps := strings.Split(forwardedFor, ",") 447 | if len(comps) > 1 { 448 | return comps[0] 449 | } 450 | return forwardedFor 451 | } 452 | h, _, err := net.SplitHostPort(req.RemoteAddr) 453 | if err != nil { 454 | return "" 455 | } 456 | return h 457 | } 458 | -------------------------------------------------------------------------------- /pkg/registry/registry_test.go: -------------------------------------------------------------------------------- 1 | package registry 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "net/http" 7 | "net/http/httptest" 8 | "net/netip" 9 | "testing" 10 | "time" 11 | 12 | "github.com/go-logr/logr" 13 | "github.com/stretchr/testify/require" 14 | 15 | "github.com/spegel-org/spegel/internal/mux" 16 | "github.com/spegel-org/spegel/pkg/oci" 17 | "github.com/spegel-org/spegel/pkg/routing" 18 | ) 19 | 20 | func TestRegistryOptions(t *testing.T) { 21 | t.Parallel() 22 | 23 | transport := &http.Transport{} 24 | log := logr.Discard() 25 | opts := []RegistryOption{ 26 | WithResolveRetries(5), 27 | WithResolveLatestTag(true), 28 | WithResolveTimeout(10 * time.Minute), 29 | WithTransport(transport), 30 | WithLogger(log), 31 | WithBasicAuth("foo", "bar"), 32 | } 33 | cfg := RegistryConfig{} 34 | err := cfg.Apply(opts...) 35 | require.NoError(t, err) 36 | require.Equal(t, 5, cfg.ResolveRetries) 37 | require.True(t, cfg.ResolveLatestTag) 38 | require.Equal(t, 10*time.Minute, cfg.ResolveTimeout) 39 | require.Equal(t, transport, cfg.Client.Transport) 40 | require.Equal(t, log, cfg.Log) 41 | require.Equal(t, "foo", cfg.Username) 42 | require.Equal(t, "bar", cfg.Password) 43 | } 44 | 45 | func TestBasicAuth(t *testing.T) { 46 | t.Parallel() 47 | 48 | tests := []struct { 49 | name string 50 | username string 51 | password string 52 | reqUsername string 53 | reqPassword string 54 | expected int 55 | }{ 56 | { 57 | name: "no registry authentication", 58 | expected: http.StatusOK, 59 | }, 60 | { 61 | name: "unnecessary authentication", 62 | reqUsername: "foo", 63 | reqPassword: "bar", 64 | expected: http.StatusOK, 65 | }, 66 | { 67 | name: "correct authentication", 68 | username: "foo", 69 | password: "bar", 70 | reqUsername: "foo", 71 | reqPassword: "bar", 72 | expected: http.StatusOK, 73 | }, 74 | { 75 | name: "invalid username", 76 | username: "foo", 77 | password: "bar", 78 | reqUsername: "wrong", 79 | reqPassword: "bar", 80 | expected: http.StatusUnauthorized, 81 | }, 82 | { 83 | name: "invalid password", 84 | username: "foo", 85 | password: "bar", 86 | reqUsername: "foo", 87 | reqPassword: "wrong", 88 | expected: http.StatusUnauthorized, 89 | }, 90 | { 91 | name: "missing authentication", 92 | username: "foo", 93 | password: "bar", 94 | expected: http.StatusUnauthorized, 95 | }, 96 | { 97 | name: "missing authentication", 98 | username: "foo", 99 | password: "bar", 100 | expected: http.StatusUnauthorized, 101 | }, 102 | } 103 | for _, tt := range tests { 104 | t.Run(tt.name, func(t *testing.T) { 105 | t.Parallel() 106 | 107 | reg, err := NewRegistry(nil, nil, WithBasicAuth(tt.username, tt.password)) 108 | require.NoError(t, err) 109 | rw := httptest.NewRecorder() 110 | req := httptest.NewRequest(http.MethodGet, "http://localhost/v2", nil) 111 | req.SetBasicAuth(tt.reqUsername, tt.reqPassword) 112 | m, err := mux.NewServeMux(reg.handle) 113 | require.NoError(t, err) 114 | m.ServeHTTP(rw, req) 115 | 116 | require.Equal(t, tt.expected, rw.Result().StatusCode) 117 | }) 118 | } 119 | } 120 | 121 | func TestMirrorHandler(t *testing.T) { 122 | t.Parallel() 123 | 124 | badSvr := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 125 | w.WriteHeader(http.StatusInternalServerError) 126 | w.Header().Set("foo", "bar") 127 | if r.Method == http.MethodGet { 128 | //nolint:errcheck // ignore 129 | w.Write([]byte("hello world")) 130 | } 131 | })) 132 | t.Cleanup(func() { 133 | badSvr.Close() 134 | }) 135 | badAddrPort := netip.MustParseAddrPort(badSvr.Listener.Addr().String()) 136 | goodSvr := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 137 | w.Header().Set("foo", "bar") 138 | if r.Method == http.MethodGet { 139 | //nolint:errcheck // ignore 140 | w.Write([]byte("hello world")) 141 | } 142 | })) 143 | t.Cleanup(func() { 144 | goodSvr.Close() 145 | }) 146 | goodAddrPort := netip.MustParseAddrPort(goodSvr.Listener.Addr().String()) 147 | unreachableAddrPort := netip.MustParseAddrPort("127.0.0.1:0") 148 | 149 | resolver := map[string][]netip.AddrPort{ 150 | // No working peers 151 | "sha256:c3e30fbcf3b231356a1efbd30a8ccec75134a7a8b45217ede97f4ff483540b04": {badAddrPort, unreachableAddrPort, badAddrPort}, 152 | // First Peer 153 | "sha256:3b8a55c543ccc7ae01c47b1d35af5826a6439a9b91ab0ca96de9967759279896": {goodAddrPort, badAddrPort, badAddrPort}, 154 | // First peer error 155 | "sha256:a0daab85ec30e2809a38c32fa676515aba22f481c56fda28637ae964ff398e3d": {unreachableAddrPort, goodAddrPort}, 156 | // Last peer working 157 | "sha256:11242d2a347bf8ab30b9f92d5ca219bbbedf95df5a8b74631194561497c1fae8": {badAddrPort, badAddrPort, goodAddrPort}, 158 | } 159 | router := routing.NewMemoryRouter(resolver, netip.AddrPort{}) 160 | reg, err := NewRegistry(oci.NewMemory(), router) 161 | require.NoError(t, err) 162 | 163 | tests := []struct { 164 | expectedHeaders map[string][]string 165 | name string 166 | key string 167 | expectedBody string 168 | expectedStatus int 169 | }{ 170 | { 171 | name: "request should timeout when no peers exists", 172 | key: "no-peers", 173 | expectedStatus: http.StatusNotFound, 174 | expectedBody: "", 175 | expectedHeaders: nil, 176 | }, 177 | { 178 | name: "request should not timeout and give 404 if all peers fail", 179 | key: "sha256:c3e30fbcf3b231356a1efbd30a8ccec75134a7a8b45217ede97f4ff483540b04", 180 | expectedStatus: http.StatusNotFound, 181 | expectedBody: "", 182 | expectedHeaders: nil, 183 | }, 184 | { 185 | name: "request should work when first peer responds", 186 | key: "sha256:3b8a55c543ccc7ae01c47b1d35af5826a6439a9b91ab0ca96de9967759279896", 187 | expectedStatus: http.StatusOK, 188 | expectedBody: "hello world", 189 | expectedHeaders: map[string][]string{"foo": {"bar"}}, 190 | }, 191 | { 192 | name: "second peer should respond when first gives error", 193 | key: "sha256:a0daab85ec30e2809a38c32fa676515aba22f481c56fda28637ae964ff398e3d", 194 | expectedStatus: http.StatusOK, 195 | expectedBody: "hello world", 196 | expectedHeaders: map[string][]string{"foo": {"bar"}}, 197 | }, 198 | { 199 | name: "last peer should respond when two first fail", 200 | key: "sha256:11242d2a347bf8ab30b9f92d5ca219bbbedf95df5a8b74631194561497c1fae8", 201 | expectedStatus: http.StatusOK, 202 | expectedBody: "hello world", 203 | expectedHeaders: map[string][]string{"foo": {"bar"}}, 204 | }, 205 | } 206 | for _, tt := range tests { 207 | for _, method := range []string{http.MethodGet, http.MethodHead} { 208 | t.Run(fmt.Sprintf("%s-%s", method, tt.name), func(t *testing.T) { 209 | t.Parallel() 210 | 211 | target := fmt.Sprintf("http://example.com/v2/foo/bar/blobs/%s", tt.key) 212 | rw := httptest.NewRecorder() 213 | req := httptest.NewRequest(method, target, nil) 214 | m, err := mux.NewServeMux(reg.handle) 215 | require.NoError(t, err) 216 | m.ServeHTTP(rw, req) 217 | 218 | resp := rw.Result() 219 | defer resp.Body.Close() 220 | b, err := io.ReadAll(resp.Body) 221 | require.NoError(t, err) 222 | require.Equal(t, tt.expectedStatus, resp.StatusCode) 223 | 224 | if method == http.MethodGet { 225 | require.Equal(t, tt.expectedBody, string(b)) 226 | } 227 | if method == http.MethodHead { 228 | require.Empty(t, b) 229 | } 230 | 231 | if tt.expectedHeaders == nil { 232 | require.Empty(t, resp.Header) 233 | } 234 | for k, v := range tt.expectedHeaders { 235 | require.Equal(t, v, resp.Header.Values(k)) 236 | } 237 | }) 238 | } 239 | } 240 | } 241 | 242 | func TestCopyHeader(t *testing.T) { 243 | t.Parallel() 244 | 245 | src := http.Header{ 246 | "foo": []string{"2", "1"}, 247 | } 248 | dst := http.Header{} 249 | copyHeader(dst, src) 250 | 251 | require.Equal(t, []string{"2", "1"}, dst.Values("foo")) 252 | } 253 | 254 | func TestGetClientIP(t *testing.T) { 255 | t.Parallel() 256 | 257 | tests := []struct { 258 | name string 259 | request *http.Request 260 | expected string 261 | }{ 262 | { 263 | name: "x forwarded for single", 264 | request: &http.Request{ 265 | Header: http.Header{ 266 | "X-Forwarded-For": []string{"localhost"}, 267 | }, 268 | }, 269 | expected: "localhost", 270 | }, 271 | { 272 | name: "x forwarded for multiple", 273 | request: &http.Request{ 274 | Header: http.Header{ 275 | "X-Forwarded-For": []string{"localhost,127.0.0.1"}, 276 | }, 277 | }, 278 | expected: "localhost", 279 | }, 280 | { 281 | name: "remote address", 282 | request: &http.Request{ 283 | RemoteAddr: "127.0.0.1:9090", 284 | }, 285 | expected: "127.0.0.1", 286 | }, 287 | } 288 | for _, tt := range tests { 289 | t.Run(tt.name, func(t *testing.T) { 290 | t.Parallel() 291 | 292 | ip := getClientIP(tt.request) 293 | require.Equal(t, tt.expected, ip) 294 | }) 295 | } 296 | } 297 | -------------------------------------------------------------------------------- /pkg/routing/bootstrap.go: -------------------------------------------------------------------------------- 1 | package routing 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "io" 7 | "net" 8 | "net/http" 9 | "slices" 10 | "strings" 11 | "sync" 12 | "time" 13 | 14 | "golang.org/x/sync/errgroup" 15 | 16 | "github.com/libp2p/go-libp2p/core/peer" 17 | ma "github.com/multiformats/go-multiaddr" 18 | manet "github.com/multiformats/go-multiaddr/net" 19 | ) 20 | 21 | // Bootstrapper resolves peers to bootstrap with for the P2P router. 22 | type Bootstrapper interface { 23 | // Run starts the bootstrap process. Should be blocking even if not needed. 24 | Run(ctx context.Context, id string) error 25 | // Get returns a list of peers that should be used as bootstrap nodes. 26 | // If the peer ID is empty it will be resolved. 27 | // If the address is missing a port the P2P router port will be used. 28 | Get(ctx context.Context) ([]peer.AddrInfo, error) 29 | } 30 | 31 | var _ Bootstrapper = &StaticBootstrapper{} 32 | 33 | type StaticBootstrapper struct { 34 | peers []peer.AddrInfo 35 | mx sync.RWMutex 36 | } 37 | 38 | func NewStaticBootstrapper() *StaticBootstrapper { 39 | return &StaticBootstrapper{} 40 | } 41 | 42 | func (b *StaticBootstrapper) Run(ctx context.Context, id string) error { 43 | <-ctx.Done() 44 | return nil 45 | } 46 | 47 | func (b *StaticBootstrapper) Get(ctx context.Context) ([]peer.AddrInfo, error) { 48 | b.mx.RLock() 49 | defer b.mx.RUnlock() 50 | return b.peers, nil 51 | } 52 | 53 | func (b *StaticBootstrapper) SetPeers(peers []peer.AddrInfo) { 54 | b.mx.Lock() 55 | defer b.mx.Unlock() 56 | b.peers = peers 57 | } 58 | 59 | var _ Bootstrapper = &DNSBootstrapper{} 60 | 61 | type DNSBootstrapper struct { 62 | resolver *net.Resolver 63 | host string 64 | limit int 65 | } 66 | 67 | func NewDNSBootstrapper(host string, limit int) *DNSBootstrapper { 68 | return &DNSBootstrapper{ 69 | resolver: &net.Resolver{}, 70 | host: host, 71 | limit: limit, 72 | } 73 | } 74 | 75 | func (b *DNSBootstrapper) Run(ctx context.Context, id string) error { 76 | <-ctx.Done() 77 | return nil 78 | } 79 | 80 | func (b *DNSBootstrapper) Get(ctx context.Context) ([]peer.AddrInfo, error) { 81 | ips, err := b.resolver.LookupIPAddr(ctx, b.host) 82 | if err != nil { 83 | return nil, err 84 | } 85 | if len(ips) == 0 { 86 | return nil, err 87 | } 88 | slices.SortFunc(ips, func(a, b net.IPAddr) int { 89 | return strings.Compare(a.String(), b.String()) 90 | }) 91 | addrInfos := []peer.AddrInfo{} 92 | for _, ip := range ips { 93 | addr, err := manet.FromIPAndZone(ip.IP, ip.Zone) 94 | if err != nil { 95 | return nil, err 96 | } 97 | addrInfos = append(addrInfos, peer.AddrInfo{ 98 | ID: "", 99 | Addrs: []ma.Multiaddr{addr}, 100 | }) 101 | } 102 | limit := min(len(addrInfos), b.limit) 103 | return addrInfos[:limit], nil 104 | } 105 | 106 | var _ Bootstrapper = &HTTPBootstrapper{} 107 | 108 | type HTTPBootstrapper struct { 109 | addr string 110 | peer string 111 | } 112 | 113 | func NewHTTPBootstrapper(addr, peer string) *HTTPBootstrapper { 114 | return &HTTPBootstrapper{ 115 | addr: addr, 116 | peer: peer, 117 | } 118 | } 119 | 120 | func (bs *HTTPBootstrapper) Run(ctx context.Context, id string) error { 121 | g, ctx := errgroup.WithContext(ctx) 122 | mux := http.NewServeMux() 123 | mux.HandleFunc("/id", func(w http.ResponseWriter, r *http.Request) { 124 | w.WriteHeader(http.StatusOK) 125 | //nolint:errcheck // ignore 126 | w.Write([]byte(id)) 127 | }) 128 | srv := http.Server{ 129 | Addr: bs.addr, 130 | Handler: mux, 131 | } 132 | g.Go(func() error { 133 | if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { 134 | return err 135 | } 136 | return nil 137 | }) 138 | g.Go(func() error { 139 | <-ctx.Done() 140 | shutdownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) 141 | defer cancel() 142 | return srv.Shutdown(shutdownCtx) 143 | }) 144 | return g.Wait() 145 | } 146 | 147 | func (bs *HTTPBootstrapper) Get(ctx context.Context) ([]peer.AddrInfo, error) { 148 | resp, err := http.DefaultClient.Get(bs.peer) 149 | if err != nil { 150 | return nil, err 151 | } 152 | defer resp.Body.Close() 153 | b, err := io.ReadAll(resp.Body) 154 | if err != nil { 155 | return nil, err 156 | } 157 | addr, err := ma.NewMultiaddr(string(b)) 158 | if err != nil { 159 | return nil, err 160 | } 161 | addrInfo, err := peer.AddrInfoFromP2pAddr(addr) 162 | if err != nil { 163 | return nil, err 164 | } 165 | return []peer.AddrInfo{*addrInfo}, nil 166 | } 167 | -------------------------------------------------------------------------------- /pkg/routing/bootstrap_test.go: -------------------------------------------------------------------------------- 1 | package routing 2 | 3 | import ( 4 | "context" 5 | "net/http" 6 | "net/http/httptest" 7 | "testing" 8 | 9 | "golang.org/x/sync/errgroup" 10 | 11 | "github.com/libp2p/go-libp2p/core/peer" 12 | ma "github.com/multiformats/go-multiaddr" 13 | manet "github.com/multiformats/go-multiaddr/net" 14 | "github.com/stretchr/testify/require" 15 | ) 16 | 17 | func TestStaticBootstrap(t *testing.T) { 18 | t.Parallel() 19 | 20 | peers := []peer.AddrInfo{ 21 | { 22 | ID: "foo", 23 | Addrs: []ma.Multiaddr{ma.StringCast("/ip4/192.168.1.1")}, 24 | }, 25 | { 26 | ID: "bar", 27 | Addrs: []ma.Multiaddr{manet.IP6Loopback}, 28 | }, 29 | } 30 | bs := NewStaticBootstrapper() 31 | bs.SetPeers(peers) 32 | 33 | ctx, cancel := context.WithCancel(t.Context()) 34 | g, gCtx := errgroup.WithContext(ctx) 35 | g.Go(func() error { 36 | return bs.Run(gCtx, "") 37 | }) 38 | 39 | bsPeers, err := bs.Get(t.Context()) 40 | require.NoError(t, err) 41 | require.ElementsMatch(t, peers, bsPeers) 42 | 43 | cancel() 44 | err = g.Wait() 45 | require.NoError(t, err) 46 | } 47 | 48 | func TestHTTPBootstrap(t *testing.T) { 49 | t.Parallel() 50 | 51 | id := "/ip4/104.131.131.82/tcp/4001/ipfs/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ" 52 | svr := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 53 | //nolint:errcheck // ignore 54 | w.Write([]byte(id)) 55 | })) 56 | defer svr.Close() 57 | 58 | bs := NewHTTPBootstrapper(":", svr.URL) 59 | 60 | ctx, cancel := context.WithCancel(t.Context()) 61 | g, gCtx := errgroup.WithContext(ctx) 62 | g.Go(func() error { 63 | return bs.Run(gCtx, "") 64 | }) 65 | 66 | addrInfos, err := bs.Get(t.Context()) 67 | require.NoError(t, err) 68 | require.Len(t, addrInfos, 1) 69 | addrInfo := addrInfos[0] 70 | require.Len(t, addrInfo.Addrs, 1) 71 | require.Equal(t, "/ip4/104.131.131.82/tcp/4001", addrInfo.Addrs[0].String()) 72 | require.Equal(t, "QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ", addrInfo.ID.String()) 73 | 74 | cancel() 75 | err = g.Wait() 76 | require.NoError(t, err) 77 | } 78 | -------------------------------------------------------------------------------- /pkg/routing/memory.go: -------------------------------------------------------------------------------- 1 | package routing 2 | 3 | import ( 4 | "context" 5 | "net/netip" 6 | "slices" 7 | "sync" 8 | ) 9 | 10 | var _ Router = &MemoryRouter{} 11 | 12 | type MemoryRouter struct { 13 | resolver map[string][]netip.AddrPort 14 | self netip.AddrPort 15 | mx sync.RWMutex 16 | } 17 | 18 | func NewMemoryRouter(resolver map[string][]netip.AddrPort, self netip.AddrPort) *MemoryRouter { 19 | return &MemoryRouter{ 20 | resolver: resolver, 21 | self: self, 22 | } 23 | } 24 | 25 | func (m *MemoryRouter) Ready(ctx context.Context) (bool, error) { 26 | m.mx.RLock() 27 | defer m.mx.RUnlock() 28 | 29 | return len(m.resolver) > 0, nil 30 | } 31 | 32 | func (m *MemoryRouter) Resolve(ctx context.Context, key string, count int) (<-chan netip.AddrPort, error) { 33 | m.mx.RLock() 34 | peers, ok := m.resolver[key] 35 | m.mx.RUnlock() 36 | 37 | peerCh := make(chan netip.AddrPort, count) 38 | // If no peers exist close the channel to stop any consumer. 39 | if !ok { 40 | close(peerCh) 41 | return peerCh, nil 42 | } 43 | go func() { 44 | for _, peer := range peers { 45 | peerCh <- peer 46 | } 47 | close(peerCh) 48 | }() 49 | return peerCh, nil 50 | } 51 | 52 | func (m *MemoryRouter) Advertise(ctx context.Context, keys []string) error { 53 | for _, key := range keys { 54 | m.Add(key, m.self) 55 | } 56 | return nil 57 | } 58 | 59 | func (m *MemoryRouter) Add(key string, ap netip.AddrPort) { 60 | m.mx.Lock() 61 | defer m.mx.Unlock() 62 | 63 | v, ok := m.resolver[key] 64 | if !ok { 65 | m.resolver[key] = []netip.AddrPort{ap} 66 | return 67 | } 68 | if slices.Contains(v, ap) { 69 | return 70 | } 71 | m.resolver[key] = append(v, ap) 72 | } 73 | 74 | func (m *MemoryRouter) Lookup(key string) ([]netip.AddrPort, bool) { 75 | m.mx.RLock() 76 | defer m.mx.RUnlock() 77 | 78 | v, ok := m.resolver[key] 79 | return v, ok 80 | } 81 | -------------------------------------------------------------------------------- /pkg/routing/memory_test.go: -------------------------------------------------------------------------------- 1 | package routing 2 | 3 | import ( 4 | "net/netip" 5 | "testing" 6 | "time" 7 | 8 | "github.com/stretchr/testify/require" 9 | ) 10 | 11 | func TestMemoryRouter(t *testing.T) { 12 | t.Parallel() 13 | 14 | r := NewMemoryRouter(map[string][]netip.AddrPort{}, netip.AddrPort{}) 15 | 16 | isReady, err := r.Ready(t.Context()) 17 | require.NoError(t, err) 18 | require.False(t, isReady) 19 | err = r.Advertise(t.Context(), []string{"foo"}) 20 | require.NoError(t, err) 21 | isReady, err = r.Ready(t.Context()) 22 | require.NoError(t, err) 23 | require.True(t, isReady) 24 | 25 | r.Add("foo", netip.MustParseAddrPort("127.0.0.1:9090")) 26 | peerCh, err := r.Resolve(t.Context(), "foo", 2) 27 | require.NoError(t, err) 28 | peers := []netip.AddrPort{} 29 | for peer := range peerCh { 30 | peers = append(peers, peer) 31 | } 32 | require.Len(t, peers, 2) 33 | peers, ok := r.Lookup("foo") 34 | require.True(t, ok) 35 | require.Len(t, peers, 2) 36 | 37 | peerCh, err = r.Resolve(t.Context(), "bar", 1) 38 | require.NoError(t, err) 39 | time.Sleep(1 * time.Second) 40 | select { 41 | case <-peerCh: 42 | default: 43 | t.Error("expected peer channel to be closed") 44 | } 45 | _, ok = r.Lookup("bar") 46 | require.False(t, ok) 47 | } 48 | -------------------------------------------------------------------------------- /pkg/routing/p2p.go: -------------------------------------------------------------------------------- 1 | package routing 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "net" 8 | "net/netip" 9 | "strconv" 10 | "strings" 11 | "time" 12 | 13 | "github.com/go-logr/logr" 14 | cid "github.com/ipfs/go-cid" 15 | "github.com/libp2p/go-libp2p" 16 | dht "github.com/libp2p/go-libp2p-kad-dht" 17 | "github.com/libp2p/go-libp2p/core/host" 18 | "github.com/libp2p/go-libp2p/core/peer" 19 | "github.com/libp2p/go-libp2p/core/sec" 20 | "github.com/libp2p/go-libp2p/p2p/discovery/routing" 21 | ma "github.com/multiformats/go-multiaddr" 22 | manet "github.com/multiformats/go-multiaddr/net" 23 | mc "github.com/multiformats/go-multicodec" 24 | mh "github.com/multiformats/go-multihash" 25 | "github.com/prometheus/client_golang/prometheus" 26 | 27 | "github.com/spegel-org/spegel/pkg/metrics" 28 | ) 29 | 30 | const KeyTTL = 10 * time.Minute 31 | 32 | type P2PRouterConfig struct { 33 | libp2pOpts []libp2p.Option 34 | } 35 | 36 | func (cfg *P2PRouterConfig) Apply(opts ...P2PRouterOption) error { 37 | for _, opt := range opts { 38 | if opt == nil { 39 | continue 40 | } 41 | if err := opt(cfg); err != nil { 42 | return err 43 | } 44 | } 45 | return nil 46 | } 47 | 48 | type P2PRouterOption func(cfg *P2PRouterConfig) error 49 | 50 | func LibP2POptions(opts ...libp2p.Option) P2PRouterOption { 51 | return func(cfg *P2PRouterConfig) error { 52 | cfg.libp2pOpts = opts 53 | return nil 54 | } 55 | } 56 | 57 | var _ Router = &P2PRouter{} 58 | 59 | type P2PRouter struct { 60 | bootstrapper Bootstrapper 61 | host host.Host 62 | kdht *dht.IpfsDHT 63 | rd *routing.RoutingDiscovery 64 | registryPort uint16 65 | } 66 | 67 | func NewP2PRouter(ctx context.Context, addr string, bs Bootstrapper, registryPortStr string, opts ...P2PRouterOption) (*P2PRouter, error) { 68 | cfg := P2PRouterConfig{} 69 | err := cfg.Apply(opts...) 70 | if err != nil { 71 | return nil, err 72 | } 73 | 74 | registryPort, err := strconv.ParseUint(registryPortStr, 10, 16) 75 | if err != nil { 76 | return nil, err 77 | } 78 | 79 | multiAddrs, err := listenMultiaddrs(addr) 80 | if err != nil { 81 | return nil, err 82 | } 83 | addrFactoryOpt := libp2p.AddrsFactory(func(addrs []ma.Multiaddr) []ma.Multiaddr { 84 | var ip4Ma, ip6Ma ma.Multiaddr 85 | for _, addr := range addrs { 86 | if manet.IsIPLoopback(addr) { 87 | continue 88 | } 89 | if isIp6(addr) { 90 | ip6Ma = addr 91 | continue 92 | } 93 | ip4Ma = addr 94 | } 95 | if ip6Ma != nil { 96 | return []ma.Multiaddr{ip6Ma} 97 | } 98 | if ip4Ma != nil { 99 | return []ma.Multiaddr{ip4Ma} 100 | } 101 | return nil 102 | }) 103 | libp2pOpts := []libp2p.Option{ 104 | libp2p.ListenAddrs(multiAddrs...), 105 | libp2p.PrometheusRegisterer(metrics.DefaultRegisterer), 106 | addrFactoryOpt, 107 | } 108 | libp2pOpts = append(libp2pOpts, cfg.libp2pOpts...) 109 | host, err := libp2p.New(libp2pOpts...) 110 | if err != nil { 111 | return nil, fmt.Errorf("could not create host: %w", err) 112 | } 113 | if len(host.Addrs()) != 1 { 114 | addrs := []string{} 115 | for _, addr := range host.Addrs() { 116 | addrs = append(addrs, addr.String()) 117 | } 118 | return nil, fmt.Errorf("expected single host address but got %d %s", len(addrs), strings.Join(addrs, ", ")) 119 | } 120 | 121 | dhtOpts := []dht.Option{ 122 | dht.Mode(dht.ModeServer), 123 | dht.ProtocolPrefix("/spegel"), 124 | dht.DisableValues(), 125 | dht.MaxRecordAge(KeyTTL), 126 | dht.BootstrapPeersFunc(bootstrapFunc(ctx, bs, host)), 127 | } 128 | kdht, err := dht.New(ctx, host, dhtOpts...) 129 | if err != nil { 130 | return nil, fmt.Errorf("could not create distributed hash table: %w", err) 131 | } 132 | rd := routing.NewRoutingDiscovery(kdht) 133 | 134 | return &P2PRouter{ 135 | bootstrapper: bs, 136 | host: host, 137 | kdht: kdht, 138 | rd: rd, 139 | registryPort: uint16(registryPort), 140 | }, nil 141 | } 142 | 143 | func (r *P2PRouter) Run(ctx context.Context) (err error) { 144 | self := fmt.Sprintf("%s/p2p/%s", r.host.Addrs()[0].String(), r.host.ID().String()) 145 | logr.FromContextOrDiscard(ctx).WithName("p2p").Info("starting p2p router", "id", self) 146 | if err := r.kdht.Bootstrap(ctx); err != nil { 147 | return fmt.Errorf("could not bootstrap distributed hash table: %w", err) 148 | } 149 | defer func() { 150 | cerr := r.host.Close() 151 | if cerr != nil { 152 | err = errors.Join(err, cerr) 153 | } 154 | }() 155 | err = r.bootstrapper.Run(ctx, self) 156 | if err != nil { 157 | return err 158 | } 159 | return nil 160 | } 161 | 162 | func (r *P2PRouter) Ready(ctx context.Context) (bool, error) { 163 | addrInfos, err := r.bootstrapper.Get(ctx) 164 | if err != nil { 165 | return false, err 166 | } 167 | if len(addrInfos) == 0 { 168 | return false, nil 169 | } 170 | if len(addrInfos) == 1 { 171 | matches, err := hostMatches(*host.InfoFromHost(r.host), addrInfos[0]) 172 | if err != nil { 173 | return false, err 174 | } 175 | if matches { 176 | return true, nil 177 | } 178 | } 179 | if r.kdht.RoutingTable().Size() > 0 { 180 | return true, nil 181 | } 182 | err = r.kdht.Bootstrap(ctx) 183 | if err != nil { 184 | return false, err 185 | } 186 | return false, nil 187 | } 188 | 189 | func (r *P2PRouter) Resolve(ctx context.Context, key string, count int) (<-chan netip.AddrPort, error) { 190 | log := logr.FromContextOrDiscard(ctx).WithValues("host", r.host.ID().String(), "key", key) 191 | c, err := createCid(key) 192 | if err != nil { 193 | return nil, err 194 | } 195 | // If using unlimited retries (count=0), ensure that the peer address channel 196 | // does not become blocking by using a reasonable non-zero buffer size. 197 | peerBufferSize := count 198 | if peerBufferSize == 0 { 199 | peerBufferSize = 20 200 | } 201 | addrInfoCh := r.rd.FindProvidersAsync(ctx, c, count) 202 | peerCh := make(chan netip.AddrPort, peerBufferSize) 203 | go func() { 204 | resolveTimer := prometheus.NewTimer(metrics.ResolveDurHistogram.WithLabelValues("libp2p")) 205 | for addrInfo := range addrInfoCh { 206 | resolveTimer.ObserveDuration() 207 | if len(addrInfo.Addrs) != 1 { 208 | addrs := []string{} 209 | for _, addr := range addrInfo.Addrs { 210 | addrs = append(addrs, addr.String()) 211 | } 212 | log.Info("expected address list to only contain a single item", "addresses", strings.Join(addrs, ", ")) 213 | continue 214 | } 215 | ip, err := manet.ToIP(addrInfo.Addrs[0]) 216 | if err != nil { 217 | log.Error(err, "could not get IP address") 218 | continue 219 | } 220 | ipAddr, ok := netip.AddrFromSlice(ip) 221 | if !ok { 222 | log.Error(errors.New("IP is not IPV4 or IPV6"), "could not convert IP") 223 | continue 224 | } 225 | peer := netip.AddrPortFrom(ipAddr, r.registryPort) 226 | // Don't block if the client has disconnected before reading all values from the channel 227 | select { 228 | case peerCh <- peer: 229 | default: 230 | log.V(4).Info("mirror endpoint dropped: peer channel is full") 231 | } 232 | } 233 | close(peerCh) 234 | }() 235 | return peerCh, nil 236 | } 237 | 238 | func (r *P2PRouter) Advertise(ctx context.Context, keys []string) error { 239 | logr.FromContextOrDiscard(ctx).V(4).Info("advertising keys", "host", r.host.ID().String(), "keys", keys) 240 | for _, key := range keys { 241 | c, err := createCid(key) 242 | if err != nil { 243 | return err 244 | } 245 | err = r.rd.Provide(ctx, c, false) 246 | if err != nil { 247 | return err 248 | } 249 | } 250 | return nil 251 | } 252 | 253 | func bootstrapFunc(ctx context.Context, bootstrapper Bootstrapper, h host.Host) func() []peer.AddrInfo { 254 | log := logr.FromContextOrDiscard(ctx).WithName("p2p") 255 | return func() []peer.AddrInfo { 256 | bootstrapCtx, bootstrapCancel := context.WithTimeout(context.Background(), 10*time.Second) 257 | defer bootstrapCancel() 258 | 259 | // TODO (phillebaba): Consider if we should do a best effort bootstrap without host address. 260 | hostAddrs := h.Addrs() 261 | if len(hostAddrs) == 0 { 262 | return nil 263 | } 264 | var hostPort ma.Component 265 | ma.ForEach(hostAddrs[0], func(c ma.Component) bool { 266 | if c.Protocol().Code == ma.P_TCP { 267 | hostPort = c 268 | return false 269 | } 270 | return true 271 | }) 272 | 273 | addrInfos, err := bootstrapper.Get(bootstrapCtx) 274 | if err != nil { 275 | log.Error(err, "could not get bootstrap addresses") 276 | return nil 277 | } 278 | filteredAddrInfos := []peer.AddrInfo{} 279 | for _, addrInfo := range addrInfos { 280 | // Skip addresses that match host. 281 | matches, err := hostMatches(*host.InfoFromHost(h), addrInfo) 282 | if err != nil { 283 | log.Error(err, "could not compare host with address") 284 | continue 285 | } 286 | if matches { 287 | log.Info("skipping bootstrap peer that is same as host") 288 | continue 289 | } 290 | 291 | // Add port to address if it is missing. 292 | modifiedAddrs := []ma.Multiaddr{} 293 | for _, addr := range addrInfo.Addrs { 294 | hasPort := false 295 | ma.ForEach(addr, func(c ma.Component) bool { 296 | if c.Protocol().Code == ma.P_TCP { 297 | hasPort = true 298 | return false 299 | } 300 | return true 301 | }) 302 | if hasPort { 303 | modifiedAddrs = append(modifiedAddrs, addr) 304 | continue 305 | } 306 | modifiedAddrs = append(modifiedAddrs, ma.Join(addr, &hostPort)) 307 | } 308 | addrInfo.Addrs = modifiedAddrs 309 | 310 | // Resolve ID if it is missing. 311 | if addrInfo.ID != "" { 312 | filteredAddrInfos = append(filteredAddrInfos, addrInfo) 313 | continue 314 | } 315 | addrInfo.ID = "id" 316 | err = h.Connect(bootstrapCtx, addrInfo) 317 | var mismatchErr sec.ErrPeerIDMismatch 318 | if !errors.As(err, &mismatchErr) { 319 | log.Error(err, "could not get peer id") 320 | continue 321 | } 322 | addrInfo.ID = mismatchErr.Actual 323 | filteredAddrInfos = append(filteredAddrInfos, addrInfo) 324 | } 325 | if len(filteredAddrInfos) == 0 { 326 | log.Info("no bootstrap nodes found") 327 | return nil 328 | } 329 | return filteredAddrInfos 330 | } 331 | } 332 | 333 | func listenMultiaddrs(addr string) ([]ma.Multiaddr, error) { 334 | h, p, err := net.SplitHostPort(addr) 335 | if err != nil { 336 | return nil, err 337 | } 338 | tcpComp, err := ma.NewMultiaddr(fmt.Sprintf("/tcp/%s", p)) 339 | if err != nil { 340 | return nil, err 341 | } 342 | ipComps := []ma.Multiaddr{} 343 | ip := net.ParseIP(h) 344 | if ip.To4() != nil { 345 | ipComp, err := ma.NewMultiaddr(fmt.Sprintf("/ip4/%s", h)) 346 | if err != nil { 347 | return nil, fmt.Errorf("could not create host multi address: %w", err) 348 | } 349 | ipComps = append(ipComps, ipComp) 350 | } else if ip.To16() != nil { 351 | ipComp, err := ma.NewMultiaddr(fmt.Sprintf("/ip6/%s", h)) 352 | if err != nil { 353 | return nil, fmt.Errorf("could not create host multi address: %w", err) 354 | } 355 | ipComps = append(ipComps, ipComp) 356 | } 357 | if len(ipComps) == 0 { 358 | ipComps = []ma.Multiaddr{manet.IP6Unspecified, manet.IP4Unspecified} 359 | } 360 | multiAddrs := []ma.Multiaddr{} 361 | for _, ipComp := range ipComps { 362 | multiAddrs = append(multiAddrs, ipComp.Encapsulate(tcpComp)) 363 | } 364 | return multiAddrs, nil 365 | } 366 | 367 | func isIp6(m ma.Multiaddr) bool { 368 | c, _ := ma.SplitFirst(m) 369 | if c == nil || c.Protocol().Code != ma.P_IP6 { 370 | return false 371 | } 372 | return true 373 | } 374 | 375 | func createCid(key string) (cid.Cid, error) { 376 | pref := cid.Prefix{ 377 | Version: 1, 378 | Codec: uint64(mc.Raw), 379 | MhType: mh.SHA2_256, 380 | MhLength: -1, 381 | } 382 | c, err := pref.Sum([]byte(key)) 383 | if err != nil { 384 | return cid.Cid{}, err 385 | } 386 | return c, nil 387 | } 388 | 389 | func hostMatches(host, addrInfo peer.AddrInfo) (bool, error) { 390 | // Skip self when address ID matches host ID. 391 | if host.ID != "" && addrInfo.ID != "" { 392 | return host.ID == addrInfo.ID, nil 393 | } 394 | 395 | // Skip self when IP matches 396 | hostIP, err := manet.ToIP(host.Addrs[0]) 397 | if err != nil { 398 | return false, err 399 | } 400 | for _, addr := range addrInfo.Addrs { 401 | addrIP, err := manet.ToIP(addr) 402 | if err != nil { 403 | return false, err 404 | } 405 | if hostIP.Equal(addrIP) { 406 | return true, nil 407 | } 408 | } 409 | 410 | return false, nil 411 | } 412 | -------------------------------------------------------------------------------- /pkg/routing/p2p_test.go: -------------------------------------------------------------------------------- 1 | package routing 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "testing" 7 | "time" 8 | 9 | "github.com/go-logr/logr" 10 | tlog "github.com/go-logr/logr/testing" 11 | "github.com/libp2p/go-libp2p" 12 | "github.com/libp2p/go-libp2p/core/host" 13 | "github.com/libp2p/go-libp2p/core/peer" 14 | mocknet "github.com/libp2p/go-libp2p/p2p/net/mock" 15 | ma "github.com/multiformats/go-multiaddr" 16 | "github.com/stretchr/testify/require" 17 | "golang.org/x/sync/errgroup" 18 | ) 19 | 20 | func TestP2PRouterOptions(t *testing.T) { 21 | t.Parallel() 22 | 23 | libp2pOpts := []libp2p.Option{ 24 | libp2p.ListenAddrStrings("foo"), 25 | } 26 | opts := []P2PRouterOption{ 27 | LibP2POptions(libp2pOpts...), 28 | } 29 | cfg := P2PRouterConfig{} 30 | err := cfg.Apply(opts...) 31 | require.NoError(t, err) 32 | require.Equal(t, libp2pOpts, cfg.libp2pOpts) 33 | } 34 | 35 | func TestP2PRouter(t *testing.T) { 36 | t.Parallel() 37 | 38 | ctx, cancel := context.WithCancel(t.Context()) 39 | 40 | bs := NewStaticBootstrapper() 41 | router, err := NewP2PRouter(ctx, "localhost:0", bs, "9090") 42 | require.NoError(t, err) 43 | 44 | g, gCtx := errgroup.WithContext(ctx) 45 | g.Go(func() error { 46 | return router.Run(gCtx) 47 | }) 48 | 49 | // TODO (phillebaba): There is a test flake that sometime occurs sometimes if code runs too fast. 50 | // Flake results in a peer being returned without an address. Revisit in Go 1.24 to see if this can be solved better. 51 | time.Sleep(1 * time.Second) 52 | 53 | err = router.Advertise(ctx, nil) 54 | require.NoError(t, err) 55 | peerCh, err := router.Resolve(ctx, "foo", 1) 56 | require.NoError(t, err) 57 | peer := <-peerCh 58 | require.False(t, peer.IsValid()) 59 | 60 | err = router.Advertise(ctx, []string{"foo"}) 61 | require.NoError(t, err) 62 | peerCh, err = router.Resolve(ctx, "foo", 1) 63 | require.NoError(t, err) 64 | peer = <-peerCh 65 | require.True(t, peer.IsValid()) 66 | 67 | cancel() 68 | err = g.Wait() 69 | require.NoError(t, err) 70 | } 71 | 72 | func TestReady(t *testing.T) { 73 | t.Parallel() 74 | 75 | bs := NewStaticBootstrapper() 76 | router, err := NewP2PRouter(t.Context(), "localhost:0", bs, "9090") 77 | require.NoError(t, err) 78 | 79 | // Should not be ready if no peers are found. 80 | isReady, err := router.Ready(t.Context()) 81 | require.NoError(t, err) 82 | require.False(t, isReady) 83 | 84 | // Should be ready if only peer is host. 85 | bs.SetPeers([]peer.AddrInfo{*host.InfoFromHost(router.host)}) 86 | isReady, err = router.Ready(t.Context()) 87 | require.NoError(t, err) 88 | require.True(t, isReady) 89 | 90 | // Shouldd be not ready with multiple peers but empty routing table. 91 | bs.SetPeers([]peer.AddrInfo{{}, {}}) 92 | isReady, err = router.Ready(t.Context()) 93 | require.NoError(t, err) 94 | require.False(t, isReady) 95 | 96 | // Should be ready with multiple peers and populated routing table. 97 | newPeer, err := router.kdht.RoutingTable().GenRandPeerID(0) 98 | require.NoError(t, err) 99 | ok, err := router.kdht.RoutingTable().TryAddPeer(newPeer, false, false) 100 | require.NoError(t, err) 101 | require.True(t, ok) 102 | bs.SetPeers([]peer.AddrInfo{{}, {}}) 103 | isReady, err = router.Ready(t.Context()) 104 | require.NoError(t, err) 105 | require.True(t, isReady) 106 | } 107 | 108 | func TestBootstrapFunc(t *testing.T) { 109 | t.Parallel() 110 | 111 | log := tlog.NewTestLogger(t) 112 | ctx := logr.NewContext(t.Context(), log) 113 | 114 | mn, err := mocknet.WithNPeers(2) 115 | require.NoError(t, err) 116 | 117 | tests := []struct { 118 | name string 119 | peers []peer.AddrInfo 120 | expected []string 121 | }{ 122 | { 123 | name: "no peers", 124 | peers: []peer.AddrInfo{}, 125 | expected: []string{}, 126 | }, 127 | { 128 | name: "nothing missing", 129 | peers: []peer.AddrInfo{ 130 | { 131 | ID: "foo", 132 | Addrs: []ma.Multiaddr{ma.StringCast("/ip4/192.168.1.1/tcp/8080")}, 133 | }, 134 | }, 135 | expected: []string{"/ip4/192.168.1.1/tcp/8080/p2p/foo"}, 136 | }, 137 | { 138 | name: "only self", 139 | peers: []peer.AddrInfo{ 140 | { 141 | ID: mn.Hosts()[0].ID(), 142 | Addrs: []ma.Multiaddr{ma.StringCast("/ip4/192.168.1.1/tcp/8080")}, 143 | }, 144 | }, 145 | expected: []string{}, 146 | }, 147 | { 148 | name: "missing port", 149 | peers: []peer.AddrInfo{ 150 | { 151 | ID: "foo", 152 | Addrs: []ma.Multiaddr{ma.StringCast("/ip4/192.168.1.1")}, 153 | }, 154 | }, 155 | expected: []string{"/ip4/192.168.1.1/tcp/4242/p2p/foo"}, 156 | }, 157 | } 158 | for _, tt := range tests { 159 | t.Run(tt.name, func(t *testing.T) { 160 | t.Parallel() 161 | 162 | bs := NewStaticBootstrapper() 163 | bs.SetPeers(tt.peers) 164 | f := bootstrapFunc(ctx, bs, mn.Hosts()[0]) 165 | peers := f() 166 | 167 | peerStrs := []string{} 168 | for _, p := range peers { 169 | id, err := p.ID.Marshal() 170 | require.NoError(t, err) 171 | peerStrs = append(peerStrs, fmt.Sprintf("%s/p2p/%s", p.Addrs[0].String(), string(id))) 172 | } 173 | require.ElementsMatch(t, tt.expected, peerStrs) 174 | }) 175 | } 176 | } 177 | 178 | func TestListenMultiaddrs(t *testing.T) { 179 | t.Parallel() 180 | 181 | tests := []struct { 182 | name string 183 | addr string 184 | expected []string 185 | }{ 186 | { 187 | name: "listen address type not specified", 188 | addr: ":9090", 189 | expected: []string{"/ip6/::/tcp/9090", "/ip4/0.0.0.0/tcp/9090"}, 190 | }, 191 | { 192 | name: "ipv4 only", 193 | addr: "0.0.0.0:9090", 194 | expected: []string{"/ip4/0.0.0.0/tcp/9090"}, 195 | }, 196 | { 197 | name: "ipv6 only", 198 | addr: "[::]:9090", 199 | expected: []string{"/ip6/::/tcp/9090"}, 200 | }, 201 | } 202 | for _, tt := range tests { 203 | t.Run(tt.name, func(t *testing.T) { 204 | t.Parallel() 205 | 206 | multiAddrs, err := listenMultiaddrs(tt.addr) 207 | require.NoError(t, err) 208 | //nolint: testifylint // This is easier to read and understand. 209 | require.Equal(t, len(tt.expected), len(multiAddrs)) 210 | for i, e := range tt.expected { 211 | require.Equal(t, e, multiAddrs[i].String()) 212 | } 213 | }) 214 | } 215 | } 216 | 217 | func TestIsIp6(t *testing.T) { 218 | t.Parallel() 219 | 220 | m, err := ma.NewMultiaddr("/ip6/::") 221 | require.NoError(t, err) 222 | require.True(t, isIp6(m)) 223 | m, err = ma.NewMultiaddr("/ip4/0.0.0.0") 224 | require.NoError(t, err) 225 | require.False(t, isIp6(m)) 226 | } 227 | 228 | func TestCreateCid(t *testing.T) { 229 | t.Parallel() 230 | 231 | c, err := createCid("foobar") 232 | require.NoError(t, err) 233 | require.Equal(t, "bafkreigdvoh7cnza5cwzar65hfdgwpejotszfqx2ha6uuolaofgk54ge6i", c.String()) 234 | } 235 | 236 | func TestHostMatches(t *testing.T) { 237 | t.Parallel() 238 | 239 | tests := []struct { 240 | name string 241 | host peer.AddrInfo 242 | addrInfo peer.AddrInfo 243 | expected bool 244 | }{ 245 | { 246 | name: "ID match", 247 | host: peer.AddrInfo{ 248 | ID: "foo", 249 | Addrs: []ma.Multiaddr{}, 250 | }, 251 | addrInfo: peer.AddrInfo{ 252 | ID: "foo", 253 | Addrs: []ma.Multiaddr{}, 254 | }, 255 | expected: true, 256 | }, 257 | { 258 | name: "ID do not match", 259 | host: peer.AddrInfo{ 260 | ID: "foo", 261 | Addrs: []ma.Multiaddr{}, 262 | }, 263 | addrInfo: peer.AddrInfo{ 264 | ID: "bar", 265 | Addrs: []ma.Multiaddr{}, 266 | }, 267 | expected: false, 268 | }, 269 | { 270 | name: "IP4 match", 271 | host: peer.AddrInfo{ 272 | ID: "", 273 | Addrs: []ma.Multiaddr{ma.StringCast("/ip4/192.168.1.1")}, 274 | }, 275 | addrInfo: peer.AddrInfo{ 276 | ID: "", 277 | Addrs: []ma.Multiaddr{ma.StringCast("/ip4/192.168.1.1")}, 278 | }, 279 | expected: true, 280 | }, 281 | { 282 | name: "IP4 do not match", 283 | host: peer.AddrInfo{ 284 | ID: "", 285 | Addrs: []ma.Multiaddr{ma.StringCast("/ip4/192.168.1.1")}, 286 | }, 287 | addrInfo: peer.AddrInfo{ 288 | ID: "", 289 | Addrs: []ma.Multiaddr{ma.StringCast("/ip4/192.168.1.2")}, 290 | }, 291 | expected: false, 292 | }, 293 | { 294 | name: "IP6 match", 295 | host: peer.AddrInfo{ 296 | ID: "", 297 | Addrs: []ma.Multiaddr{ma.StringCast("/ip6/c3c9:152b:73d1:dad0:e2f9:a521:6356:88ba")}, 298 | }, 299 | addrInfo: peer.AddrInfo{ 300 | ID: "", 301 | Addrs: []ma.Multiaddr{ma.StringCast("/ip6/c3c9:152b:73d1:dad0:e2f9:a521:6356:88ba")}, 302 | }, 303 | expected: true, 304 | }, 305 | } 306 | for _, tt := range tests { 307 | t.Run(tt.name, func(t *testing.T) { 308 | t.Parallel() 309 | 310 | matches, err := hostMatches(tt.host, tt.addrInfo) 311 | require.NoError(t, err) 312 | require.Equal(t, tt.expected, matches) 313 | }) 314 | } 315 | } 316 | -------------------------------------------------------------------------------- /pkg/routing/routing.go: -------------------------------------------------------------------------------- 1 | package routing 2 | 3 | import ( 4 | "context" 5 | "net/netip" 6 | ) 7 | 8 | // Router implements the discovery of content. 9 | type Router interface { 10 | // Ready returns true when the router is ready. 11 | Ready(ctx context.Context) (bool, error) 12 | // Resolve asynchronously discovers addresses that can serve the content defined by the give key. 13 | Resolve(ctx context.Context, key string, count int) (<-chan netip.AddrPort, error) 14 | // Advertise broadcasts that the current router can serve the content. 15 | Advertise(ctx context.Context, keys []string) error 16 | } 17 | -------------------------------------------------------------------------------- /pkg/state/state.go: -------------------------------------------------------------------------------- 1 | package state 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "time" 8 | 9 | "github.com/go-logr/logr" 10 | 11 | "github.com/spegel-org/spegel/internal/channel" 12 | "github.com/spegel-org/spegel/pkg/metrics" 13 | "github.com/spegel-org/spegel/pkg/oci" 14 | "github.com/spegel-org/spegel/pkg/routing" 15 | ) 16 | 17 | func Track(ctx context.Context, ociClient oci.Client, router routing.Router, resolveLatestTag bool) error { 18 | log := logr.FromContextOrDiscard(ctx) 19 | eventCh, errCh, err := ociClient.Subscribe(ctx) 20 | if err != nil { 21 | return err 22 | } 23 | immediateCh := make(chan time.Time, 1) 24 | immediateCh <- time.Now() 25 | close(immediateCh) 26 | expirationTicker := time.NewTicker(routing.KeyTTL - time.Minute) 27 | defer expirationTicker.Stop() 28 | tickerCh := channel.Merge(immediateCh, expirationTicker.C) 29 | for { 30 | select { 31 | case <-ctx.Done(): 32 | return nil 33 | case <-tickerCh: 34 | log.Info("running scheduled image state update") 35 | if err := all(ctx, ociClient, router, resolveLatestTag); err != nil { 36 | log.Error(err, "received errors when updating all images") 37 | continue 38 | } 39 | case event, ok := <-eventCh: 40 | if !ok { 41 | return errors.New("image event channel closed") 42 | } 43 | log.Info("received image event", "image", event.Image.String(), "type", event.Type) 44 | if _, err := update(ctx, ociClient, router, event, false, resolveLatestTag); err != nil { 45 | log.Error(err, "received error when updating image") 46 | continue 47 | } 48 | case err, ok := <-errCh: 49 | if !ok { 50 | return errors.New("image error channel closed") 51 | } 52 | log.Error(err, "event channel error") 53 | } 54 | } 55 | } 56 | 57 | func all(ctx context.Context, ociClient oci.Client, router routing.Router, resolveLatestTag bool) error { 58 | log := logr.FromContextOrDiscard(ctx).V(4) 59 | imgs, err := ociClient.ListImages(ctx) 60 | if err != nil { 61 | return err 62 | } 63 | 64 | // TODO: Update metrics on subscribed events. This will require keeping state in memory to know about key count changes. 65 | metrics.AdvertisedKeys.Reset() 66 | metrics.AdvertisedImages.Reset() 67 | metrics.AdvertisedImageTags.Reset() 68 | metrics.AdvertisedImageDigests.Reset() 69 | errs := []error{} 70 | targets := map[string]any{} 71 | for _, img := range imgs { 72 | _, skipDigests := targets[img.Digest.String()] 73 | // Handle the list re-sync as update events; this will also prevent the 74 | // update function from setting metrics values. 75 | event := oci.ImageEvent{Image: img, Type: oci.UpdateEvent} 76 | log.Info("sync image event", "image", event.Image.String(), "type", event.Type) 77 | keyTotal, err := update(ctx, ociClient, router, event, skipDigests, resolveLatestTag) 78 | if err != nil { 79 | errs = append(errs, err) 80 | continue 81 | } 82 | targets[img.Digest.String()] = nil 83 | metrics.AdvertisedKeys.WithLabelValues(img.Registry).Add(float64(keyTotal)) 84 | metrics.AdvertisedImages.WithLabelValues(img.Registry).Add(1) 85 | if img.Tag == "" { 86 | metrics.AdvertisedImageDigests.WithLabelValues(event.Image.Registry).Add(1) 87 | } else { 88 | metrics.AdvertisedImageTags.WithLabelValues(event.Image.Registry).Add(1) 89 | } 90 | } 91 | return errors.Join(errs...) 92 | } 93 | 94 | func update(ctx context.Context, ociClient oci.Client, router routing.Router, event oci.ImageEvent, skipDigests, resolveLatestTag bool) (int, error) { 95 | keys := []string{} 96 | //nolint: staticcheck // Simplify in future. 97 | if !(!resolveLatestTag && event.Image.IsLatestTag()) { 98 | if tagName, ok := event.Image.TagName(); ok { 99 | keys = append(keys, tagName) 100 | } 101 | } 102 | if event.Type == oci.DeleteEvent { 103 | // We don't know how many digest keys were associated with the deleted image; 104 | // that can only be updated by the full image list sync in all(). 105 | metrics.AdvertisedImages.WithLabelValues(event.Image.Registry).Sub(1) 106 | // DHT doesn't actually have any way to stop providing a key, you just have to wait for the record to expire 107 | // from the datastore. Record TTL is a datastore-level value, so we can't even re-provide with a shorter TTL. 108 | return 0, nil 109 | } 110 | if !skipDigests { 111 | dgsts, err := oci.WalkImage(ctx, ociClient, event.Image) 112 | if err != nil { 113 | return 0, fmt.Errorf("could not get digests for image %s: %w", event.Image.String(), err) 114 | } 115 | keys = append(keys, dgsts...) 116 | } 117 | err := router.Advertise(ctx, keys) 118 | if err != nil { 119 | return 0, fmt.Errorf("could not advertise image %s: %w", event.Image.String(), err) 120 | } 121 | if event.Type == oci.CreateEvent { 122 | // We don't know how many unique digest keys will be associated with the new image; 123 | // that can only be updated by the full image list sync in all(). 124 | metrics.AdvertisedImages.WithLabelValues(event.Image.Registry).Add(1) 125 | if event.Image.Tag == "" { 126 | metrics.AdvertisedImageDigests.WithLabelValues(event.Image.Registry).Add(1) 127 | } else { 128 | metrics.AdvertisedImageTags.WithLabelValues(event.Image.Registry).Add(1) 129 | } 130 | } 131 | return len(keys), nil 132 | } 133 | -------------------------------------------------------------------------------- /pkg/state/state_test.go: -------------------------------------------------------------------------------- 1 | package state 2 | 3 | import ( 4 | "context" 5 | "crypto/sha256" 6 | "encoding/json" 7 | "math/rand/v2" 8 | "net/netip" 9 | "strconv" 10 | "testing" 11 | "time" 12 | 13 | "golang.org/x/sync/errgroup" 14 | 15 | "github.com/go-logr/logr" 16 | tlog "github.com/go-logr/logr/testing" 17 | "github.com/opencontainers/go-digest" 18 | "github.com/opencontainers/image-spec/specs-go" 19 | ocispec "github.com/opencontainers/image-spec/specs-go/v1" 20 | "github.com/stretchr/testify/require" 21 | 22 | "github.com/spegel-org/spegel/pkg/oci" 23 | "github.com/spegel-org/spegel/pkg/routing" 24 | ) 25 | 26 | func TestTrack(t *testing.T) { 27 | t.Parallel() 28 | ociClient := oci.NewMemory() 29 | 30 | imgRefs := []string{ 31 | "docker.io/library/ubuntu:latest", 32 | "ghcr.io/spegel-org/spegel:v0.0.9", 33 | } 34 | imgs := []oci.Image{} 35 | for _, imageStr := range imgRefs { 36 | manifest := ocispec.Manifest{ 37 | Versioned: specs.Versioned{ 38 | SchemaVersion: 2, 39 | }, 40 | MediaType: ocispec.MediaTypeImageManifest, 41 | Annotations: map[string]string{ 42 | "random": strconv.Itoa(rand.Int()), 43 | }, 44 | } 45 | b, err := json.Marshal(&manifest) 46 | require.NoError(t, err) 47 | hash := sha256.New() 48 | _, err = hash.Write(b) 49 | require.NoError(t, err) 50 | dgst := digest.NewDigest(digest.SHA256, hash) 51 | ociClient.AddBlob(b, dgst) 52 | img, err := oci.ParseImageRequireDigest(imageStr, dgst) 53 | require.NoError(t, err) 54 | ociClient.AddImage(img) 55 | 56 | imgs = append(imgs, img) 57 | } 58 | 59 | tests := []struct { 60 | name string 61 | resolveLatestTag bool 62 | }{ 63 | { 64 | name: "resolve latest", 65 | resolveLatestTag: true, 66 | }, 67 | { 68 | name: "do not resolve latest", 69 | resolveLatestTag: false, 70 | }, 71 | } 72 | for _, tt := range tests { 73 | t.Run(tt.name, func(t *testing.T) { 74 | t.Parallel() 75 | 76 | log := tlog.NewTestLogger(t) 77 | ctx := logr.NewContext(t.Context(), log) 78 | ctx, cancel := context.WithCancel(ctx) 79 | 80 | router := routing.NewMemoryRouter(map[string][]netip.AddrPort{}, netip.MustParseAddrPort("127.0.0.1:5000")) 81 | g, gCtx := errgroup.WithContext(ctx) 82 | g.Go(func() error { 83 | return Track(gCtx, ociClient, router, tt.resolveLatestTag) 84 | }) 85 | time.Sleep(100 * time.Millisecond) 86 | 87 | for _, img := range imgs { 88 | peers, ok := router.Lookup(img.Digest.String()) 89 | require.True(t, ok) 90 | require.Len(t, peers, 1) 91 | tagName, ok := img.TagName() 92 | if !ok { 93 | continue 94 | } 95 | peers, ok = router.Lookup(tagName) 96 | if img.IsLatestTag() && !tt.resolveLatestTag { 97 | require.False(t, ok) 98 | continue 99 | } 100 | require.True(t, ok) 101 | require.Len(t, peers, 1) 102 | } 103 | 104 | cancel() 105 | err := g.Wait() 106 | require.NoError(t, err) 107 | }) 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /test/e2e/e2e_test.go: -------------------------------------------------------------------------------- 1 | //go:build e2e 2 | 3 | package e2e 4 | 5 | import ( 6 | "bytes" 7 | "context" 8 | "fmt" 9 | "net/netip" 10 | "os" 11 | "os/exec" 12 | "path/filepath" 13 | "strings" 14 | "testing" 15 | "time" 16 | 17 | "golang.org/x/sync/errgroup" 18 | 19 | "github.com/stretchr/testify/require" 20 | ) 21 | 22 | func TestE2E(t *testing.T) { 23 | t.Log("Running E2E tests") 24 | 25 | imageRef := os.Getenv("IMG_REF") 26 | require.NotEmpty(t, imageRef) 27 | proxyMode := os.Getenv("E2E_PROXY_MODE") 28 | require.NotEmpty(t, proxyMode) 29 | ipFamily := os.Getenv("E2E_IP_FAMILY") 30 | require.NotEmpty(t, ipFamily) 31 | kindName := "spegel-e2e" 32 | 33 | // Create kind cluster. 34 | kcPath := createKindCluster(t.Context(), t, kindName, proxyMode, ipFamily) 35 | t.Cleanup(func() { 36 | t.Log("Deleting Kind cluster") 37 | ctx, cancel := context.WithTimeout(context.Background(), time.Minute) 38 | defer cancel() 39 | command(ctx, t, fmt.Sprintf("kind delete cluster --name %s", kindName)) 40 | }) 41 | 42 | // Pull test images. 43 | g, gCtx := errgroup.WithContext(t.Context()) 44 | images := []string{ 45 | "ghcr.io/spegel-org/conformance:583e014", 46 | "docker.io/library/nginx:1.23.0", 47 | "docker.io/library/nginx@sha256:b3a676a9145dc005062d5e79b92d90574fb3bf2396f4913dc1732f9065f55c4b", 48 | "mcr.microsoft.com/containernetworking/azure-cns@sha256:7944413c630746a35d5596f56093706e8d6a3db0569bec0c8e58323f965f7416", 49 | } 50 | for _, image := range images { 51 | g.Go(func() error { 52 | t.Logf("Pulling image %s", image) 53 | _, err := commandWithError(gCtx, t, fmt.Sprintf("docker exec %s-worker ctr -n k8s.io image pull %s", kindName, image)) 54 | if err != nil { 55 | return err 56 | } 57 | return nil 58 | }) 59 | } 60 | err := g.Wait() 61 | require.NoError(t, err) 62 | 63 | // Write existing configuration to test backup. 64 | hostsToml := `server = https://docker.io 65 | 66 | [host.https://registry-1.docker.io] 67 | capabilities = [push]` 68 | command(t.Context(), t, fmt.Sprintf("docker exec %s-worker2 bash -c \"mkdir -p /etc/containerd/certs.d/docker.io; echo -e '%s' > /etc/containerd/certs.d/docker.io/hosts.toml\"", kindName, hostsToml)) 69 | 70 | // Deploy Spegel. 71 | deploySpegel(t.Context(), t, kindName, imageRef, kcPath) 72 | podOutput := command(t.Context(), t, fmt.Sprintf("kubectl --kubeconfig %s --namespace spegel get pods --no-headers", kcPath)) 73 | require.Len(t, strings.Split(podOutput, "\n"), 5) 74 | 75 | // Verify that configuration has been backed up. 76 | backupHostToml := command(t.Context(), t, fmt.Sprintf("docker exec %s-worker2 cat /etc/containerd/certs.d/_backup/docker.io/hosts.toml", kindName)) 77 | require.Equal(t, hostsToml, backupHostToml) 78 | 79 | // Cleanup backup for uninstall tests 80 | command(t.Context(), t, fmt.Sprintf("docker exec %s-worker2 rm -rf /etc/containerd/certs.d/_backup", kindName)) 81 | command(t.Context(), t, fmt.Sprintf("docker exec %s-worker2 mkdir /etc/containerd/certs.d/_backup", kindName)) 82 | 83 | // Run conformance tests. 84 | t.Log("Running conformance tests") 85 | command(t.Context(), t, fmt.Sprintf("kubectl --kubeconfig %s create namespace conformance --dry-run=client -o yaml | kubectl --kubeconfig %s apply -f -", kcPath, kcPath)) 86 | command(t.Context(), t, fmt.Sprintf("kubectl --kubeconfig %s apply --namespace conformance -f ./testdata/conformance-job.yaml", kcPath)) 87 | command(t.Context(), t, fmt.Sprintf("kubectl --kubeconfig %s --namespace conformance wait --timeout 90s --for=condition=complete job/conformance", kcPath)) 88 | 89 | // Remove Spegel from the last node to test that the mirror fallback is working. 90 | workerPod := command(t.Context(), t, fmt.Sprintf("kubectl --kubeconfig %s --namespace spegel get pods --no-headers -o name --field-selector spec.nodeName=%s-worker4", kcPath, kindName)) 91 | command(t.Context(), t, fmt.Sprintf("kubectl --kubeconfig %s label nodes %s-worker4 spegel-", kcPath, kindName)) 92 | command(t.Context(), t, fmt.Sprintf("kubectl --kubeconfig %s --namespace spegel wait --for=delete %s --timeout=60s", kcPath, workerPod)) 93 | 94 | // Pull image from registry after Spegel has started. 95 | command(t.Context(), t, fmt.Sprintf("docker exec %s-worker ctr -n k8s.io image pull docker.io/library/nginx:1.21.0@sha256:2f1cd90e00fe2c991e18272bb35d6a8258eeb27785d121aa4cc1ae4235167cfd", kindName)) 96 | 97 | // Verify that both local and external ports are working. 98 | tests := []struct { 99 | node string 100 | port string 101 | expected string 102 | }{ 103 | { 104 | node: "worker", 105 | port: "30020", 106 | expected: "200", 107 | }, 108 | { 109 | node: "worker", 110 | port: "30021", 111 | expected: "200", 112 | }, 113 | { 114 | node: "worker4", 115 | port: "30020", 116 | expected: "000", 117 | }, 118 | { 119 | node: "worker4", 120 | port: "30021", 121 | expected: "200", 122 | }, 123 | } 124 | for _, tt := range tests { 125 | hostIP := command(t.Context(), t, fmt.Sprintf("kubectl --kubeconfig %s --namespace spegel get nodes %s-%s -o jsonpath='{.status.addresses[?(@.type==\"InternalIP\")].address}'", kcPath, kindName, tt.node)) 126 | addr, err := netip.ParseAddr(hostIP) 127 | require.NoError(t, err) 128 | if addr.Is6() { 129 | hostIP = fmt.Sprintf("[%s]", hostIP) 130 | } 131 | httpCode := command(t.Context(), t, fmt.Sprintf("docker exec %s-worker curl -s -o /dev/null -w \"%%{http_code}\" http://%s:%s/healthz || true", kindName, hostIP, tt.port)) 132 | require.Equal(t, tt.expected, httpCode) 133 | } 134 | 135 | // Block internet access by only allowing RFC1918 CIDR. 136 | nodes := getNodes(t.Context(), t, kindName) 137 | for _, node := range nodes { 138 | command(t.Context(), t, fmt.Sprintf("docker exec %s-%s iptables -A OUTPUT -o eth0 -d 10.0.0.0/8 -j ACCEPT", kindName, node)) 139 | command(t.Context(), t, fmt.Sprintf("docker exec %s-%s iptables -A OUTPUT -o eth0 -d 172.16.0.0/12 -j ACCEPT", kindName, node)) 140 | command(t.Context(), t, fmt.Sprintf("docker exec %s-%s iptables -A OUTPUT -o eth0 -d 192.168.0.0/16 -j ACCEPT", kindName, node)) 141 | command(t.Context(), t, fmt.Sprintf("docker exec %s-%s iptables -A OUTPUT -o eth0 -j REJECT", kindName, node)) 142 | } 143 | 144 | // Pull test image that does not contain any media types. 145 | command(t.Context(), t, fmt.Sprintf("docker exec %s-worker3 crictl pull mcr.microsoft.com/containernetworking/azure-cns@sha256:7944413c630746a35d5596f56093706e8d6a3db0569bec0c8e58323f965f7416", kindName)) 146 | 147 | // Deploy test Nginx pods and verify deployment status. 148 | t.Log("Deploy test Nginx pods") 149 | command(t.Context(), t, fmt.Sprintf("kubectl --kubeconfig %s apply -f ./testdata/test-nginx.yaml", kcPath)) 150 | command(t.Context(), t, fmt.Sprintf("kubectl --kubeconfig %s --namespace nginx wait --timeout=30s deployment/nginx-tag --for condition=available", kcPath)) 151 | command(t.Context(), t, fmt.Sprintf("kubectl --kubeconfig %s --namespace nginx wait --timeout=30s deployment/nginx-digest --for condition=available", kcPath)) 152 | command(t.Context(), t, fmt.Sprintf("kubectl --kubeconfig %s --namespace nginx wait --timeout=30s deployment/nginx-tag-and-digest --for condition=available", kcPath)) 153 | command(t.Context(), t, fmt.Sprintf("kubectl --kubeconfig %s --namespace nginx wait --timeout=30s -l app=nginx-not-present --for jsonpath='{.status.containerStatuses[*].state.waiting.reason}'=ImagePullBackOff pod", kcPath)) 154 | 155 | // Verify that Spegel has never restarted. 156 | restartOutput := command(t.Context(), t, fmt.Sprintf("kubectl --kubeconfig %s --namespace spegel get pods -o=jsonpath='{.items[*].status.containerStatuses[0].restartCount}'", kcPath)) 157 | require.Equal(t, "0 0 0 0", restartOutput) 158 | 159 | // Remove all Spegel Pods and only restart one to verify that running a single instance works. 160 | t.Log("Scale down Spegel to single instance") 161 | command(t.Context(), t, fmt.Sprintf("kubectl --kubeconfig %s label nodes %s-control-plane %s-worker %s-worker2 spegel-", kcPath, kindName, kindName, kindName)) 162 | command(t.Context(), t, fmt.Sprintf("kubectl --kubeconfig %s --namespace spegel delete pods --all", kcPath)) 163 | command(t.Context(), t, fmt.Sprintf("kubectl --kubeconfig %s --namespace spegel rollout status daemonset spegel --timeout 60s", kcPath)) 164 | podOutput = command(t.Context(), t, fmt.Sprintf("kubectl --kubeconfig %s --namespace spegel get pods --no-headers", kcPath)) 165 | require.Len(t, strings.Split(podOutput, "\n"), 1) 166 | 167 | // Verify that Spegel has never restarted 168 | restartOutput = command(t.Context(), t, fmt.Sprintf("kubectl --kubeconfig %s --namespace spegel get pods -o=jsonpath='{.items[*].status.containerStatuses[0].restartCount}'", kcPath)) 169 | require.Equal(t, "0", restartOutput) 170 | 171 | // Restart Containerd and verify that Spegel restarts 172 | t.Log("Restarting Containerd") 173 | command(t.Context(), t, fmt.Sprintf("docker exec %s-worker3 systemctl restart containerd", kindName)) 174 | require.Eventually(t, func() bool { 175 | restartOutput = command(t.Context(), t, fmt.Sprintf("kubectl --kubeconfig %s --namespace spegel get pods -o=jsonpath='{.items[*].status.containerStatuses[0].restartCount}'", kcPath)) 176 | return restartOutput == "1" 177 | }, 5*time.Second, 1*time.Second) 178 | 179 | // Uninstall Spegel and make sure cleanup is run 180 | t.Log("Uninstalling Spegel") 181 | command(t.Context(), t, fmt.Sprintf("helm --kubeconfig %s uninstall --timeout 60s --namespace spegel spegel", kcPath)) 182 | require.Eventually(t, func() bool { 183 | allOutput := command(t.Context(), t, fmt.Sprintf("kubectl --kubeconfig %s get all --namespace spegel", kcPath)) 184 | return allOutput == "" 185 | }, 10*time.Second, 1*time.Second) 186 | nodes = getNodes(t.Context(), t, kindName) 187 | for _, node := range nodes { 188 | lsOutput := command(t.Context(), t, fmt.Sprintf("docker exec %s-%s ls /etc/containerd/certs.d", kindName, node)) 189 | require.Empty(t, lsOutput) 190 | } 191 | } 192 | 193 | func TestDevDeploy(t *testing.T) { 194 | t.Log("Running Dev Deploy") 195 | 196 | imageRef := os.Getenv("IMG_REF") 197 | require.NotEmpty(t, imageRef) 198 | kindName := "spegel-dev" 199 | 200 | clusterOutput := command(t.Context(), t, "kind get clusters") 201 | exists := false 202 | for _, cluster := range strings.Split(clusterOutput, "\n") { 203 | if cluster != kindName { 204 | continue 205 | } 206 | exists = true 207 | break 208 | } 209 | kcPath := "" 210 | if exists { 211 | kcPath = filepath.Join(t.TempDir(), "kind.kubeconfig") 212 | kcOutput := command(t.Context(), t, fmt.Sprintf("kind get kubeconfig --name %s", kindName)) 213 | err := os.WriteFile(kcPath, []byte(kcOutput), 0o644) 214 | require.NoError(t, err) 215 | } else { 216 | kcPath = createKindCluster(t.Context(), t, kindName, "iptables", "ipv4") 217 | } 218 | deploySpegel(t.Context(), t, kindName, imageRef, kcPath) 219 | } 220 | 221 | func createKindCluster(ctx context.Context, t *testing.T, kindName, proxyMode, ipFamily string) string { 222 | t.Helper() 223 | 224 | kindConfig := fmt.Sprintf(`apiVersion: kind.x-k8s.io/v1alpha4 225 | kind: Cluster 226 | networking: 227 | kubeProxyMode: %s 228 | ipFamily: %s 229 | containerdConfigPatches: 230 | - |- 231 | [plugins."io.containerd.grpc.v1.cri".registry] 232 | config_path = "/etc/containerd/certs.d" 233 | # Discarding unpacked layers causes them to be removed, which defeats the purpose of a local cache. 234 | # Aditioanlly nodes will report having layers which no long exist. 235 | # This is by default false in containerd. 236 | [plugins."io.containerd.grpc.v1.cri".containerd] 237 | discard_unpacked_layers = false 238 | # This is just to make sure that images are not shared between namespaces. 239 | [plugins."io.containerd.metadata.v1.bolt"] 240 | content_sharing_policy = "isolated" 241 | nodes: 242 | - role: control-plane 243 | labels: 244 | spegel: schedule 245 | - role: worker 246 | labels: 247 | spegel: schedule 248 | - role: worker 249 | labels: 250 | spegel: schedule 251 | test: true 252 | - role: worker 253 | labels: 254 | spegel: schedule 255 | test: true 256 | - role: worker 257 | labels: 258 | spegel: schedule 259 | test: true`, proxyMode, ipFamily) 260 | path := filepath.Join(t.TempDir(), "kind-config.yaml") 261 | err := os.WriteFile(path, []byte(kindConfig), 0o644) 262 | require.NoError(t, err) 263 | 264 | t.Log("Creating Kind cluster", "proxy mode", proxyMode, "ip family", ipFamily) 265 | kcPath := filepath.Join(t.TempDir(), "kind.kubeconfig") 266 | command(ctx, t, fmt.Sprintf("kind create cluster --kubeconfig %s --config %s --name %s", kcPath, path, kindName)) 267 | return kcPath 268 | } 269 | 270 | func deploySpegel(ctx context.Context, t *testing.T, kindName, imageRef, kcPath string) { 271 | t.Helper() 272 | 273 | t.Log("Deploying Spegel") 274 | command(ctx, t, fmt.Sprintf("kind load docker-image --name %s %s", kindName, imageRef)) 275 | imagesOutput := command(ctx, t, fmt.Sprintf("docker exec %s-worker ctr -n k8s.io images ls name==%s", kindName, imageRef)) 276 | _, imagesOutput, ok := strings.Cut(imagesOutput, "\n") 277 | require.True(t, ok) 278 | imageDigest := strings.Split(imagesOutput, " ")[2] 279 | nodes := getNodes(ctx, t, kindName) 280 | for _, node := range nodes { 281 | command(ctx, t, fmt.Sprintf("docker exec %s-%s ctr -n k8s.io image tag %s ghcr.io/spegel-org/spegel@%s", kindName, node, imageRef, imageDigest)) 282 | } 283 | command(ctx, t, fmt.Sprintf("helm --kubeconfig %s upgrade --timeout 60s --create-namespace --wait --install --namespace=\"spegel\" spegel ../../charts/spegel --set \"image.pullPolicy=Never\" --set \"image.digest=%s\" --set \"nodeSelector.spegel=schedule\"", kcPath, imageDigest)) 284 | } 285 | 286 | func getNodes(ctx context.Context, t *testing.T, kindName string) []string { 287 | t.Helper() 288 | 289 | nodes := []string{} 290 | nodeOutput := command(ctx, t, fmt.Sprintf("kind get nodes --name %s", kindName)) 291 | for _, node := range strings.Split(nodeOutput, "\n") { 292 | nodes = append(nodes, strings.TrimPrefix(node, kindName+"-")) 293 | } 294 | return nodes 295 | } 296 | 297 | func commandWithError(ctx context.Context, t *testing.T, e string) (string, error) { 298 | t.Helper() 299 | 300 | cmd := exec.CommandContext(ctx, "bash", "-c", e) 301 | stdout := bytes.NewBuffer(nil) 302 | cmd.Stdout = stdout 303 | stderr := bytes.NewBuffer(nil) 304 | cmd.Stderr = stderr 305 | err := cmd.Run() 306 | if err != nil { 307 | return "", err 308 | } 309 | return strings.TrimSuffix(stdout.String(), "\n"), nil 310 | } 311 | 312 | func command(ctx context.Context, t *testing.T, e string) string { 313 | t.Helper() 314 | 315 | cmd := exec.CommandContext(ctx, "bash", "-c", e) 316 | stdout := bytes.NewBuffer(nil) 317 | cmd.Stdout = stdout 318 | stderr := bytes.NewBuffer(nil) 319 | cmd.Stderr = stderr 320 | err := cmd.Run() 321 | require.NoError(t, err, "command: %s\nstderr: %s", e, stderr.String()) 322 | return strings.TrimSuffix(stdout.String(), "\n") 323 | } 324 | -------------------------------------------------------------------------------- /test/e2e/testdata/conformance-job.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: batch/v1 2 | kind: Job 3 | metadata: 4 | name: conformance 5 | spec: 6 | backoffLimit: 0 7 | template: 8 | spec: 9 | restartPolicy: Never 10 | containers: 11 | - name: conformance 12 | image: ghcr.io/spegel-org/conformance:583e014 13 | env: 14 | - name: OCI_TEST_PULL 15 | value: "1" 16 | - name: "OCI_ROOT_URL" 17 | value: "http://spegel-registry.spegel.svc.cluster.local.:5000" 18 | - name: "OCI_MIRROR_URL" 19 | value: "docker.io" 20 | - name: "OCI_NAMESPACE" 21 | value: "library/nginx" 22 | - name: "OCI_TAG_NAME" 23 | value: "1.23.0" 24 | - name: "OCI_MANIFEST_DIGEST" 25 | value: "sha256:db345982a2f2a4257c6f699a499feb1d79451a1305e8022f16456ddc3ad6b94c" 26 | - name: "OCI_BLOB_DIGEST" 27 | value: "sha256:461246efe0a75316d99afdbf348f7063b57b0caeee8daab775f1f08152ea36f4" 28 | -------------------------------------------------------------------------------- /test/e2e/testdata/test-nginx.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: nginx 5 | --- 6 | apiVersion: apps/v1 7 | kind: Deployment 8 | metadata: 9 | name: nginx-tag 10 | namespace: nginx 11 | labels: 12 | app: nginx-tag 13 | spec: 14 | replicas: 3 15 | selector: 16 | matchLabels: 17 | app: nginx-tag 18 | template: 19 | metadata: 20 | labels: 21 | app: nginx-tag 22 | spec: 23 | containers: 24 | - name: nginx 25 | image: docker.io/library/nginx:1.23.0 26 | imagePullPolicy: Always 27 | ports: 28 | - containerPort: 80 29 | nodeSelector: 30 | test: "true" 31 | --- 32 | apiVersion: apps/v1 33 | kind: Deployment 34 | metadata: 35 | name: nginx-digest 36 | namespace: nginx 37 | labels: 38 | app: nginx-digest 39 | spec: 40 | replicas: 3 41 | selector: 42 | matchLabels: 43 | app: nginx-digest 44 | template: 45 | metadata: 46 | labels: 47 | app: nginx-digest 48 | spec: 49 | containers: 50 | - name: nginx 51 | image: docker.io/library/nginx@sha256:b3a676a9145dc005062d5e79b92d90574fb3bf2396f4913dc1732f9065f55c4b # 1.22.0 52 | imagePullPolicy: Always 53 | ports: 54 | - containerPort: 80 55 | nodeSelector: 56 | test: "true" 57 | --- 58 | apiVersion: apps/v1 59 | kind: Deployment 60 | metadata: 61 | name: nginx-tag-and-digest 62 | namespace: nginx 63 | labels: 64 | app: nginx-tag-and-digest 65 | spec: 66 | replicas: 3 67 | selector: 68 | matchLabels: 69 | app: nginx-tag-and-digest 70 | template: 71 | metadata: 72 | labels: 73 | app: nginx-tag-and-digest 74 | spec: 75 | containers: 76 | - name: nginx 77 | image: docker.io/library/nginx:1.21.0@sha256:2f1cd90e00fe2c991e18272bb35d6a8258eeb27785d121aa4cc1ae4235167cfd 78 | imagePullPolicy: Always 79 | ports: 80 | - containerPort: 80 81 | nodeSelector: 82 | test: "true" 83 | --- 84 | apiVersion: apps/v1 85 | kind: Deployment 86 | metadata: 87 | name: nginx-not-present 88 | namespace: nginx 89 | labels: 90 | app: nginx-not-present 91 | spec: 92 | replicas: 3 93 | selector: 94 | matchLabels: 95 | app: nginx-not-present 96 | template: 97 | metadata: 98 | labels: 99 | app: nginx-not-present 100 | spec: 101 | containers: 102 | - name: nginx 103 | image: docker.io/library/nginx:1.1.0-bullseye-perl 104 | imagePullPolicy: Always 105 | ports: 106 | - containerPort: 80 107 | nodeSelector: 108 | test: "true" 109 | --------------------------------------------------------------------------------