├── .dockerignore ├── .errcheck_excludes.txt ├── .github ├── actions │ ├── metadata │ │ └── action.yaml │ └── setup-go │ │ └── action.yaml └── workflows │ ├── build.yml │ └── release.yml ├── .gitignore ├── .gitpod.yml ├── .golangci.yaml ├── .vscode └── settings.json ├── LICENSE ├── README.md ├── docs ├── deploy_with_networks.md └── load_balancers.md ├── go.mod ├── go.sum ├── hack └── version.sh ├── hcloud ├── cloud.go ├── cloud_test.go ├── instances.go ├── instances_test.go ├── load_balancers.go ├── load_balancers_test.go ├── routes.go ├── routes_test.go ├── testing.go ├── util.go └── util_test.go ├── images └── hetzner-cloud-controller-manager │ └── Dockerfile ├── internal ├── annotation │ ├── load_balancer.go │ ├── load_balancer_test.go │ ├── name.go │ ├── name_test.go │ └── testing.go ├── credentials │ └── hotreload.go ├── hcops │ ├── action.go │ ├── certificates.go │ ├── certificates_test.go │ ├── errors.go │ ├── load_balancer.go │ ├── load_balancer_internal_test.go │ ├── load_balancer_test.go │ ├── mocks.go │ ├── network.go │ ├── ratelimit.go │ ├── ratelimit_test.go │ ├── server.go │ ├── server_test.go │ └── testing.go ├── metrics │ └── metrics.go ├── mocks │ ├── action.go │ ├── casts.go │ ├── certificates.go │ ├── loadbalancer.go │ ├── network.go │ ├── robot.go │ └── server.go ├── robot │ └── client │ │ ├── cache │ │ ├── client.go │ │ └── client_test.go │ │ └── interface.go ├── testsupport │ └── tls.go └── util │ └── util.go └── main.go /.dockerignore: -------------------------------------------------------------------------------- 1 | * 2 | !hcloud/ 3 | !internal/ 4 | !go.mod 5 | !go.sum 6 | !main.go 7 | -------------------------------------------------------------------------------- /.errcheck_excludes.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/syself/hetzner-cloud-controller-manager/f0a2074d061a8162cd1151bbfc4b8fcb2cd93021/.errcheck_excludes.txt -------------------------------------------------------------------------------- /.github/actions/metadata/action.yaml: -------------------------------------------------------------------------------- 1 | name: "Metadata" 2 | description: "Generate Image Metadata" 3 | inputs: 4 | metadata_flavor: 5 | description: "metadata_flavor" 6 | required: true 7 | metadata_tags: 8 | description: "metadata_tags" 9 | required: true 10 | outputs: 11 | tags: 12 | description: "generated image tags" 13 | value: ${{ steps.meta.outputs.tags }} 14 | labels: 15 | description: "generated image labels" 16 | value: ${{ steps.meta.outputs.labels }} 17 | version: 18 | description: "generated image version" 19 | value: ${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.version'] }} 20 | runs: 21 | using: "composite" 22 | steps: 23 | - name: Docker manager metadata 24 | id: meta 25 | uses: docker/metadata-action@369eb591f429131d6889c46b94e711f089e6ca96 # v5.6.1 26 | with: 27 | images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} 28 | flavor: ${{ inputs.metadata_flavor }} 29 | tags: ${{ inputs.metadata_tags }} 30 | -------------------------------------------------------------------------------- /.github/actions/setup-go/action.yaml: -------------------------------------------------------------------------------- 1 | name: "Setup Go" 2 | description: "Setup Go" 3 | runs: 4 | using: "composite" 5 | steps: 6 | - name: Install go 7 | uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe # v4.1.0 8 | with: 9 | go-version-file: "go.mod" 10 | cache: true 11 | cache-dependency-path: go.sum 12 | - id: go-cache-paths 13 | shell: bash 14 | run: | 15 | echo "::set-output name=go-build::$(go env GOCACHE)" 16 | echo "::set-output name=go-mod::$(go env GOMODCACHE)" 17 | - name: Go Mod Cache 18 | uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3 19 | with: 20 | path: ${{ steps.go-cache-paths.outputs.go-mod }} 21 | key: ${{ runner.os }}-go-mod-${{ hashFiles('**/go.sum') }} 22 | restore-keys: | 23 | ${{ runner.os }}-go-mod- 24 | - name: Go Build Cache 25 | uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3 26 | with: 27 | path: ${{ steps.go-cache-paths.outputs.go-build }} 28 | key: ${{ runner.os }}-go-build-${{ hashFiles('**/go.sum') }} 29 | restore-keys: | 30 | ${{ runner.os }}-go-build- 31 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: Build Syself Hetzner CCM Image 2 | # yamllint disable rule:line-length 3 | on: # yamllint disable-line rule:truthy 4 | push: 5 | branches: 6 | - main 7 | env: 8 | IMAGE_NAME: hetzner-cloud-controller-manager-staging 9 | REGISTRY: ghcr.io/syself 10 | metadata_flavor: latest=true 11 | metadata_tags: type=sha,prefix=sha-,format=short 12 | permissions: 13 | contents: read 14 | packages: write 15 | # Required to generate OIDC tokens for `sigstore/cosign-installer` authentication 16 | id-token: write 17 | jobs: 18 | manager-image: 19 | name: Build and push manager image 20 | runs-on: ubuntu-24.04 21 | steps: 22 | - name: Checkout code 23 | uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 24 | with: 25 | fetch-depth: 0 26 | - name: Install go 27 | uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 28 | with: 29 | go-version-file: "go.mod" 30 | - name: Set up QEMU 31 | uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf # v3.2.0 32 | - name: Set up Docker Buildx 33 | uses: docker/setup-buildx-action@c47758b77c9736f4b2ef4073d4d51994fabfe349 # v3.7.1 34 | 35 | - name: Generate metadata 36 | id: meta 37 | uses: ./.github/actions/metadata 38 | with: 39 | metadata_flavor: ${{ env.metadata_flavor }} 40 | metadata_tags: ${{ env.metadata_tags }} 41 | 42 | - name: Login to ghcr.io for CI 43 | uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 44 | with: 45 | registry: ghcr.io 46 | username: ${{ github.actor }} 47 | password: ${{ secrets.GITHUB_TOKEN }} 48 | 49 | - name: Install Cosign 50 | uses: sigstore/cosign-installer@dc72c7d5c4d10cd6bcb8cf6e3fd625a9e5e537da # v3.7.0 51 | 52 | - name: Setup Env 53 | run: | 54 | DOCKER_BUILD_LDFLAGS="$(hack/version.sh)" 55 | echo 'DOCKER_BUILD_LDFLAGS<> $GITHUB_ENV 56 | echo $DOCKER_BUILD_LDFLAGS >> $GITHUB_ENV 57 | echo 'EOF' >> $GITHUB_ENV 58 | 59 | - name: Build and push manager image 60 | uses: docker/build-push-action@4f58ea79222b3b9dc2c8bbdd6debcef730109a75 # v6.9.0 61 | id: docker_build_release 62 | with: 63 | provenance: false 64 | context: . 65 | file: ./images/hetzner-cloud-controller-manager/Dockerfile 66 | push: true 67 | build-args: | 68 | LDFLAGS=${{ env.DOCKER_BUILD_LDFLAGS }} 69 | tags: ${{ steps.meta.outputs.tags }} 70 | labels: ${{ steps.meta.outputs.labels }} 71 | platforms: linux/amd64,linux/arm64 72 | 73 | - name: Sign Container Images 74 | env: 75 | COSIGN_EXPERIMENTAL: "true" 76 | run: | 77 | cosign sign --yes ghcr.io/syself/hetzner-cloud-controller-manager-staging@${{ steps.docker_build_release.outputs.digest }} 78 | 79 | - name: Image Releases digests 80 | shell: bash 81 | run: | 82 | mkdir -p image-digest/ 83 | echo "ghcr.io/syself/hetzner-cloud-controller-manager-staging:{{ steps.meta.outputs.tags }}@${{ steps.docker_build_release.outputs.digest }}" >> image-digest/hetzner-cloud-controller-manager.txt 84 | 85 | # Upload artifact digests 86 | - name: Upload artifact digests 87 | uses: actions/upload-artifact@604373da6381bf24206979c74d06a550515601b9 # v4.4.1 88 | with: 89 | name: image-digest hetzner-cloud-controller-manager 90 | path: image-digest 91 | retention-days: 90 92 | 93 | - name: Image Digests Output 94 | shell: bash 95 | run: | 96 | cd image-digest/ 97 | find -type f | sort | xargs -d '\n' cat 98 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release 2 | on: # yamllint disable-line rule:truthy 3 | push: 4 | tags: 5 | - v[0-9]+.[0-9]+.[0-9]+ 6 | - v[0-9]+.[0-9]+.[0-9]+-alpha.[0-9]+ 7 | - v[0-9]+.[0-9]+.[0-9]+-beta.[0-9]+ 8 | - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ 9 | 10 | env: 11 | IMAGE_NAME: hetzner-cloud-controller-manager 12 | REGISTRY: ghcr.io/syself 13 | metadata_flavor: latest=true 14 | metadata_tags: type=ref,event=tag 15 | permissions: 16 | contents: write 17 | packages: write 18 | # Required to generate OIDC tokens for `sigstore/cosign-installer` authentication 19 | id-token: write 20 | # yamllint disable rule:line-length 21 | jobs: 22 | manager-image: 23 | name: Build and push manager image 24 | runs-on: ubuntu-24.04 25 | steps: 26 | - name: Checkout code 27 | uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 28 | with: 29 | fetch-depth: 0 30 | - uses: ./.github/actions/setup-go 31 | - name: Set up QEMU 32 | uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf # v3.2.0 33 | - name: Set up Docker Buildx 34 | uses: docker/setup-buildx-action@c47758b77c9736f4b2ef4073d4d51994fabfe349 # v3.7.1 35 | 36 | - name: Generate metadata 37 | id: meta 38 | uses: ./.github/actions/metadata 39 | with: 40 | metadata_flavor: ${{ env.metadata_flavor }} 41 | metadata_tags: ${{ env.metadata_tags }} 42 | 43 | - name: Login to ghcr.io for CI 44 | uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 45 | with: 46 | registry: ghcr.io 47 | username: ${{ github.actor }} 48 | password: ${{ secrets.GITHUB_TOKEN }} 49 | 50 | - name: Install Cosign 51 | uses: sigstore/cosign-installer@dc72c7d5c4d10cd6bcb8cf6e3fd625a9e5e537da # v3.7.0 52 | 53 | - name: Install Bom 54 | shell: bash 55 | run: | 56 | curl -L https://github.com/kubernetes-sigs/bom/releases/download/v0.6.0/bom-amd64-linux -o bom 57 | sudo mv ./bom /usr/local/bin/bom 58 | sudo chmod +x /usr/local/bin/bom 59 | 60 | - name: Setup Env 61 | run: | 62 | DOCKER_BUILD_LDFLAGS="$(hack/version.sh)" 63 | echo 'DOCKER_BUILD_LDFLAGS<> $GITHUB_ENV 64 | echo $DOCKER_BUILD_LDFLAGS >> $GITHUB_ENV 65 | echo 'EOF' >> $GITHUB_ENV 66 | 67 | - name: Build and push manager image 68 | uses: docker/build-push-action@4f58ea79222b3b9dc2c8bbdd6debcef730109a75 # v6.9.0 69 | id: docker_build_release 70 | with: 71 | provenance: false 72 | context: . 73 | file: ./images/hetzner-cloud-controller-manager/Dockerfile 74 | push: true 75 | build-args: | 76 | LDFLAGS=${{ env.DOCKER_BUILD_LDFLAGS }} 77 | tags: ${{ steps.meta.outputs.tags }} 78 | labels: ${{ steps.meta.outputs.labels }} 79 | platforms: linux/amd64,linux/arm64 80 | cache-from: type=gha, scope=${{ github.workflow }} 81 | cache-to: type=gha, mode=max, scope=${{ github.workflow }} 82 | 83 | - name: Sign Container Images 84 | run: | 85 | cosign sign --yes ghcr.io/syself/hetzner-cloud-controller-manager@${{ steps.docker_build_release.outputs.digest }} 86 | 87 | - name: Generate SBOM 88 | shell: bash 89 | # To-Do: generate SBOM from source after https://github.com/kubernetes-sigs/bom/issues/202 is fixed 90 | run: | 91 | bom generate --format=json -o sbom_ci_main_hetzner-cloud-controller-manager_${{ steps.meta.outputs.version }}-spdx.json \ 92 | --image=ghcr.io/syself/hetzner-cloud-controller-manager:${{ steps.meta.outputs.version }} 93 | 94 | - name: Attach SBOM to Container Images 95 | run: | 96 | cosign attest --yes --type=spdxjson --predicate sbom_ci_main_hetzner-cloud-controller-manager_${{ steps.meta.outputs.version }}-spdx.json ghcr.io/syself/hetzner-cloud-controller-manager@${{ steps.docker_build_release.outputs.digest }} 97 | 98 | - name: Sign SBOM Images 99 | env: 100 | COSIGN_EXPERIMENTAL: "true" 101 | run: | 102 | docker_build_release_digest="${{ steps.docker_build_release.outputs.digest }}" 103 | image_name="ghcr.io/syself/hetzner-cloud-controller-manager:${docker_build_release_digest/:/-}.sbom" 104 | docker_build_release_sbom_digest="sha256:$(docker buildx imagetools inspect --raw ${image_name} | sha256sum | head -c 64)" 105 | cosign sign --yes "ghcr.io/syself/hetzner-cloud-controller-manager@${docker_build_release_sbom_digest}" 106 | 107 | - name: Image Releases digests 108 | shell: bash 109 | run: | 110 | mkdir -p image-digest/ 111 | echo "ghcr.io/syself/hetzner-cloud-controller-manager:{{ steps.meta.outputs.version }}@${{ steps.docker_build_release.outputs.digest }}" >> image-digest/hetzner-cloud-controller-manager.txt 112 | 113 | # Upload artifact digests 114 | - name: Upload artifact digests 115 | uses: actions/upload-artifact@604373da6381bf24206979c74d06a550515601b9 # v4.4.1 116 | with: 117 | name: image-digest hetzner-cloud-controller-manager 118 | path: image-digest 119 | retention-days: 90 120 | 121 | - name: Image Digests Output 122 | shell: bash 123 | run: | 124 | cd image-digest/ 125 | find -type f | sort | xargs -d '\n' cat 126 | 127 | release: 128 | name: Create draft release 129 | runs-on: ubuntu-24.04 130 | needs: 131 | - manager-image 132 | steps: 133 | - name: Set env 134 | run: echo "RELEASE_TAG=${GITHUB_REF:10}" >> $GITHUB_ENV 135 | - name: checkout code 136 | uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 137 | with: 138 | fetch-depth: 0 139 | - name: Install go 140 | uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 141 | with: 142 | go-version-file: "go.mod" 143 | cache: true 144 | cache-dependency-path: go.sum 145 | - name: Release 146 | uses: softprops/action-gh-release@e7a8f85e1c67a31e6ed99a94b41bd0b71bbee6b8 # v2 147 | with: 148 | draft: true 149 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /bin 2 | /vendor 3 | dist/ 4 | deploy/gen/ 5 | .coverage.out 6 | .envrc 7 | ./hetzner-cloud-controller-manager 8 | *.tgz 9 | hack/.* 10 | /*.kubeconfig 11 | /etc 12 | /*.yaml 13 | -------------------------------------------------------------------------------- /.gitpod.yml: -------------------------------------------------------------------------------- 1 | tasks: 2 | - init: |- 3 | curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" && 4 | sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl && rm kubectl && 5 | curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash 6 | - init: go get && go build ./... 7 | command: go run . 8 | -------------------------------------------------------------------------------- /.golangci.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | linters-settings: 3 | errcheck: 4 | exclude: ./.errcheck_excludes.txt 5 | exhaustive: 6 | default-signifies-exhaustive: true 7 | gci: 8 | sections: 9 | - standard 10 | - default 11 | 12 | importas: 13 | no-unaliased: true 14 | alias: 15 | # Kubernetes 16 | - pkg: k8s.io/api/core/v1 17 | alias: corev1 18 | - pkg: k8s.io/apimachinery/pkg/apis/meta/v1 19 | alias: metav1 20 | 21 | misspell: 22 | locale: "US" 23 | 24 | linters: 25 | disable-all: true 26 | enable: 27 | - bodyclose 28 | - errcheck 29 | - errname 30 | - exhaustive 31 | - exportloopref 32 | - gci 33 | - gocritic 34 | - godot 35 | - goimports 36 | - gomodguard 37 | - gosec 38 | - gosimple 39 | - govet 40 | - importas 41 | - ineffassign 42 | - misspell 43 | - prealloc 44 | - revive 45 | - staticcheck 46 | - typecheck 47 | - unparam 48 | - unused 49 | - whitespace 50 | 51 | issues: 52 | exclude-rules: 53 | - path: (_test\.go|testing\.go|testsupport|e2etests) 54 | linters: 55 | - gosec 56 | - errcheck 57 | - path: internal/mocks 58 | linters: 59 | - unparam 60 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "workbench.colorCustomizations": { 3 | "activityBar.activeBackground": "#93e6fc", 4 | "activityBar.background": "#93e6fc", 5 | "activityBar.foreground": "#15202b", 6 | "activityBar.inactiveForeground": "#15202b99", 7 | "activityBarBadge.background": "#fa45d4", 8 | "activityBarBadge.foreground": "#15202b", 9 | "commandCenter.border": "#15202b99", 10 | "sash.hoverBorder": "#93e6fc", 11 | "statusBar.background": "#61dafb", 12 | "statusBar.foreground": "#15202b", 13 | "statusBarItem.hoverBackground": "#2fcefa", 14 | "statusBarItem.remoteBackground": "#61dafb", 15 | "statusBarItem.remoteForeground": "#15202b", 16 | "titleBar.activeBackground": "#61dafb", 17 | "titleBar.activeForeground": "#15202b", 18 | "titleBar.inactiveBackground": "#61dafb99", 19 | "titleBar.inactiveForeground": "#15202b99" 20 | }, 21 | "peacock.color": "#61dafb" 22 | } -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2018 Hetzner Cloud GmbH 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /docs/deploy_with_networks.md: -------------------------------------------------------------------------------- 1 | # Deployment with Networks support 2 | 3 | The deployment of the Hetzner Cloud Cloud Controller Manager with Networks support is quite different to the normal one. 4 | If you would like to use the version without using our Networks feature, you can follow the Steps at "[Basic deployment](../README.md#deployment)". 5 | 6 | We assume, that you have knowledge about Kubernetes and the Hetzner Cloud. 7 | 8 | ## How to deploy 9 | 1. Create a new Network via `hcloud-cli` (`hcloud network create --name my-network --ip-range=10.0.0.0/8`)or the [Hetzner Cloud Console](https://console.hetzner.cloud) 10 | 2. Add each Node of the Cluster to the Hetzner Cloud Network 11 | 3. Download the latest deployment file with networks support from [Github](https://github.com/syself/hetzner-cloud-controller-manager/releases/latest) to your local machine 12 | 4. Change the `--cluster-cidr=` flag in the deployment file to fit your pod range. Default is `10.244.0.0/16`. 13 | 5. Create a new secret containing a Hetzner Cloud API Token and the name or the ID of the Network you want to use `kubectl -n kube-system create secret generic hcloud --from-literal=token= --from-literal=network=` 14 | 6. Deploy the deployment file `kubectl -n kube-system apply -f path/to/your/deployment.yaml` 15 | 7. (Recommended) Deploy a CNI (like Cilium `kubectl create -f https://raw.githubusercontent.com/cilium/cilium/v1.5/examples/kubernetes//cilium.yaml` - please replace `` with your version like `1.15`) 16 | 17 | 18 | When deploying Cilium, make sure that you have set `tunnel: disabled` and `nativeRoutingCIDR` to your clusters subnet CIDR. If you are using Cilium < 1.9.0 you also have to set `blacklist-conflicting-routes: false`. 19 | 20 | After this, you should be able to see the correct routes in the [Hetzner Cloud Console](https://console.hetzner.cloud) or via `hcloud-cli` (`hcloud network describe `). 21 | 22 | ## Common Issues 23 | #### FailedToCreateRoute 24 | Error Message: 25 | ``` 26 | Could not create route xy-xy-xy-xy-xy 10.244.0.0/24 for node xy.example.com after 1s: hcloud/CreateRoute: network route destination overlaps with another subnetwork or network route (invalid_input) 27 | ``` 28 | Solution: 29 | Make sure the cluster-cidr does not overlap with the Hetzner Cloud Network. 30 | For example your Subnetwork could be `10.10.10.0/24` when the *cluster-cidr* is set to `10.244.0.0/16`. 31 | -------------------------------------------------------------------------------- /docs/load_balancers.md: -------------------------------------------------------------------------------- 1 | # Load Balancers 2 | 3 | Load Balancer support is implemented in the Cloud Controller as of 4 | version v1.6.0. For using the Hetzner Cloud Load Balancers you need to 5 | deploy a `Service` of type `LoadBalancer`. 6 | 7 | ## Sample Service: 8 | 9 | ``` 10 | apiVersion: v1 11 | kind: Service 12 | metadata: 13 | name: example-service 14 | annotations: 15 | load-balancer.hetzner.cloud/location: hel1 16 | spec: 17 | selector: 18 | app: example 19 | ports: 20 | - port: 80 21 | targetPort: 8080 22 | type: LoadBalancer 23 | ``` 24 | 25 | The sample service will create a Load Balancer in the location `hel1` 26 | with a service with `listen_port = 80` and `destination_port = 8080`. So 27 | every traffic that arrives at the Load Balancer on Port 80 will be 28 | routed to the public interface of the targets on port 8080. You can 29 | change the behavior of the Load Balancer by specifying more annotations. 30 | A list of all available annotations can be found on 31 | [pkg.go.dev](https://pkg.go.dev/github.com/syself/hetzner-cloud-controller-manager/internal/annotation#Name). 32 | If you have the cloud controller deployed with Private Network Support, 33 | we attach the Load Balancer to the specific network automatically. You 34 | can specify with an annotation that the Load Balancer should use the 35 | private network instead of the public network. 36 | 37 | ## Sample Service with Networks: 38 | 39 | ``` 40 | apiVersion: v1 41 | kind: Service 42 | metadata: 43 | name: example-service 44 | annotations: 45 | load-balancer.hetzner.cloud/location: hel1 46 | load-balancer.hetzner.cloud/use-private-ip: "true" 47 | spec: 48 | selector: 49 | app: example 50 | ports: 51 | - port: 80 52 | targetPort: 8080 53 | type: LoadBalancer 54 | ``` 55 | 56 | For IPVS based plugins (kube-router, kube-proxy in ipvs mode, etc...) make sure you 57 | supply '**load-balancer.hetzner.cloud/disable-private-ingress: "true"**' annotation 58 | to your service or set "**HCLOUD_LOAD_BALANCERS_DISABLE_PRIVATE_INGRESS**" environment variable 59 | to true on hcloud-cloud-controller-manager deployment as mentioned in a paragraph below. Otherwise, network 60 | plugin installs load balancer's IP address on system's dummy interface effectively 61 | looping IPVS system in a cycle. In such scenario cluster nodes won't ever pass load balancer's health probes 62 | 63 | ## Cluster-wide Defaults 64 | 65 | For convenience, you can set the following environment variables as cluster-wide defaults, so you don't have to set them on each load balancer service. If a load balancer service has the corresponding annotation set, it overrides the default. 66 | 67 | * `HCLOUD_LOAD_BALANCERS_LOCATION` (mutually exclusive with `HCLOUD_LOAD_BALANCERS_NETWORK_ZONE`) 68 | * `HCLOUD_LOAD_BALANCERS_NETWORK_ZONE` (mutually exclusive with `HCLOUD_LOAD_BALANCERS_LOCATION`) 69 | * `HCLOUD_LOAD_BALANCERS_DISABLE_PRIVATE_INGRESS` 70 | * `HCLOUD_LOAD_BALANCERS_USE_PRIVATE_IP` 71 | * `HCLOUD_LOAD_BALANCERS_ENABLED` 72 | 73 | ## Reference existing Load Balancers 74 | 75 | If you already have a Load Balancer that you want to use in Kubernetes, for 76 | example if you provisioned it in Terraform, you can specify these two 77 | annotations: 78 | 79 | ```yaml 80 | apiVersion: v1 81 | kind: Service 82 | metadata: 83 | name: example-service 84 | annotations: 85 | load-balancer.hetzner.cloud/location: hel1 86 | load-balancer.hetzner.cloud/name: your-existing-lb-name 87 | ``` 88 | 89 | The Load Balancer will then be adopted by the hcloud-cloud-controller-manager, 90 | and the services and targets are set up for your cluster. 91 | 92 | If you delete this `Service` in Kubernetes, the hcloud-cloud-controller-manager 93 | will delete the associated Load Balancer. If the Load Balancer is managed 94 | through Terraform, this causes problems. To disable this, you can enable 95 | deletion protection on the Load Balancer, this way hcloud-cloud-controller-manager 96 | will just skip deleting it when the associated `Service` is deleted. 97 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/syself/hetzner-cloud-controller-manager 2 | 3 | go 1.23.0 4 | 5 | require ( 6 | github.com/fsnotify/fsnotify v1.8.0 7 | github.com/hetznercloud/hcloud-go/v2 v2.17.0 8 | github.com/prometheus/client_golang v1.20.5 9 | github.com/spf13/pflag v1.0.5 10 | github.com/stretchr/testify v1.10.0 11 | github.com/syself/hrobot-go v0.2.6-beta.1 12 | k8s.io/api v0.30.3 13 | k8s.io/apimachinery v0.30.3 14 | k8s.io/client-go v0.30.3 15 | k8s.io/cloud-provider v0.30.3 16 | k8s.io/component-base v0.30.3 17 | k8s.io/klog/v2 v2.130.1 18 | k8s.io/kubectl v0.30.3 19 | ) 20 | 21 | require ( 22 | github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect 23 | github.com/NYTimes/gziphandler v1.1.1 // indirect 24 | github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df // indirect 25 | github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect 26 | github.com/beorn7/perks v1.0.1 // indirect 27 | github.com/blang/semver/v4 v4.0.0 // indirect 28 | github.com/cenkalti/backoff/v4 v4.3.0 // indirect 29 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 30 | github.com/coreos/go-semver v0.3.1 // indirect 31 | github.com/coreos/go-systemd/v22 v22.5.0 // indirect 32 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect 33 | github.com/emicklei/go-restful/v3 v3.12.1 // indirect 34 | github.com/evanphx/json-patch v4.12.0+incompatible // indirect 35 | github.com/felixge/httpsnoop v1.0.4 // indirect 36 | github.com/go-logr/logr v1.4.2 // indirect 37 | github.com/go-logr/stdr v1.2.2 // indirect 38 | github.com/go-openapi/jsonpointer v0.21.0 // indirect 39 | github.com/go-openapi/jsonreference v0.21.0 // indirect 40 | github.com/go-openapi/swag v0.23.0 // indirect 41 | github.com/gogo/protobuf v1.3.2 // indirect 42 | github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect 43 | github.com/golang/protobuf v1.5.4 // indirect 44 | github.com/google/btree v1.1.3 // indirect 45 | github.com/google/cel-go v0.17.8 // indirect 46 | github.com/google/gnostic-models v0.6.9 // indirect 47 | github.com/google/go-cmp v0.6.0 // indirect 48 | github.com/google/gofuzz v1.2.0 // indirect 49 | github.com/google/uuid v1.6.0 // indirect 50 | github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect 51 | github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 // indirect 52 | github.com/imdario/mergo v0.3.6 // indirect 53 | github.com/inconshreveable/mousetrap v1.1.0 // indirect 54 | github.com/josharian/intern v1.0.0 // indirect 55 | github.com/json-iterator/go v1.1.12 // indirect 56 | github.com/klauspost/compress v1.17.11 // indirect 57 | github.com/kylelemons/godebug v1.1.0 // indirect 58 | github.com/mailru/easyjson v0.7.7 // indirect 59 | github.com/moby/term v0.5.0 // indirect 60 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 61 | github.com/modern-go/reflect2 v1.0.2 // indirect 62 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 63 | github.com/pkg/errors v0.9.1 // indirect 64 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect 65 | github.com/prometheus/client_model v0.6.1 // indirect 66 | github.com/prometheus/common v0.61.0 // indirect 67 | github.com/prometheus/procfs v0.15.1 // indirect 68 | github.com/spf13/cobra v1.8.1 // indirect 69 | github.com/stoewer/go-strcase v1.3.0 // indirect 70 | github.com/stretchr/objx v0.5.2 // indirect 71 | go.etcd.io/etcd/api/v3 v3.5.17 // indirect 72 | go.etcd.io/etcd/client/pkg/v3 v3.5.17 // indirect 73 | go.etcd.io/etcd/client/v3 v3.5.17 // indirect 74 | go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.57.0 // indirect 75 | go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.57.0 // indirect 76 | go.opentelemetry.io/otel v1.32.0 // indirect 77 | go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.32.0 // indirect 78 | go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.32.0 // indirect 79 | go.opentelemetry.io/otel/metric v1.32.0 // indirect 80 | go.opentelemetry.io/otel/sdk v1.32.0 // indirect 81 | go.opentelemetry.io/otel/trace v1.32.0 // indirect 82 | go.opentelemetry.io/proto/otlp v1.4.0 // indirect 83 | go.uber.org/multierr v1.11.0 // indirect 84 | go.uber.org/zap v1.27.0 // indirect 85 | golang.org/x/crypto v0.31.0 // indirect 86 | golang.org/x/exp v0.0.0-20241210194714-1829a127f884 // indirect 87 | golang.org/x/net v0.32.0 // indirect 88 | golang.org/x/oauth2 v0.24.0 // indirect 89 | golang.org/x/sync v0.10.0 // indirect 90 | golang.org/x/sys v0.28.0 // indirect 91 | golang.org/x/term v0.27.0 // indirect 92 | golang.org/x/text v0.21.0 // indirect 93 | golang.org/x/time v0.8.0 // indirect 94 | google.golang.org/genproto v0.0.0-20241209162323-e6fa225c2576 // indirect 95 | google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 // indirect 96 | google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576 // indirect 97 | google.golang.org/grpc v1.68.1 // indirect 98 | google.golang.org/protobuf v1.35.2 // indirect 99 | gopkg.in/inf.v0 v0.9.1 // indirect 100 | gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect 101 | gopkg.in/yaml.v2 v2.4.0 // indirect 102 | gopkg.in/yaml.v3 v3.0.1 // indirect 103 | k8s.io/apiserver v0.30.3 // indirect 104 | k8s.io/component-helpers v0.30.3 // indirect 105 | k8s.io/controller-manager v0.30.3 // indirect 106 | k8s.io/kms v0.30.3 // indirect 107 | k8s.io/kube-openapi v0.0.0-20241212045625-5ad02ce6640f // indirect 108 | k8s.io/utils v0.0.0-20241210054802-24370beab758 // indirect 109 | sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.1 // indirect 110 | sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect 111 | sigs.k8s.io/structured-merge-diff/v4 v4.5.0 // indirect 112 | sigs.k8s.io/yaml v1.4.0 // indirect 113 | ) 114 | -------------------------------------------------------------------------------- /hack/version.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2020 The Kubernetes Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -o errexit 17 | set -o nounset 18 | set -o pipefail 19 | 20 | version::get_version_vars() { 21 | GIT_COMMIT="$(git rev-parse HEAD^{commit})" 22 | 23 | if git_status=$(git status --porcelain 2>/dev/null) && [[ -z ${git_status} ]]; then 24 | GIT_TREE_STATE="clean" 25 | else 26 | GIT_TREE_STATE="dirty" 27 | fi 28 | 29 | # borrowed from k8s.io/hack/lib/version.sh 30 | # Use git describe to find the version based on tags. 31 | if GIT_VERSION=$(git describe --tags --abbrev=14 2>/dev/null); then 32 | # This translates the "git describe" to an actual semver.org 33 | # compatible semantic version that looks something like this: 34 | # v1.1.0-alpha.0.6+84c76d1142ea4d 35 | DASHES_IN_VERSION=$(echo "${GIT_VERSION}" | sed "s/[^-]//g") 36 | if [[ "${DASHES_IN_VERSION}" == "---" ]] ; then 37 | # We have distance to subversion (v1.1.0-subversion-1-gCommitHash) 38 | GIT_VERSION=$(echo "${GIT_VERSION}" | sed "s/-\([0-9]\{1,\}\)-g\([0-9a-f]\{14\}\)$/.\1\-\2/") 39 | elif [[ "${DASHES_IN_VERSION}" == "--" ]] ; then 40 | # We have distance to base tag (v1.1.0-1-gCommitHash) 41 | GIT_VERSION=$(echo "${GIT_VERSION}" | sed "s/-g\([0-9a-f]\{14\}\)$/-\1/") 42 | fi 43 | if [[ "${GIT_TREE_STATE}" == "dirty" ]]; then 44 | # git describe --dirty only considers changes to existing files, but 45 | # that is problematic since new untracked .go files affect the build, 46 | # so use our idea of "dirty" from git status instead. 47 | GIT_VERSION+="-dirty" 48 | fi 49 | 50 | 51 | # Try to match the "git describe" output to a regex to try to extract 52 | # the "major" and "minor" versions and whether this is the exact tagged 53 | # version or whether the tree is between two tagged versions. 54 | if [[ "${GIT_VERSION}" =~ ^v([0-9]+)\.([0-9]+)(\.[0-9]+)?([-].*)?([+].*)?$ ]]; then 55 | GIT_MAJOR=${BASH_REMATCH[1]} 56 | GIT_MINOR=${BASH_REMATCH[2]} 57 | fi 58 | 59 | # If GIT_VERSION is not a valid Semantic Version, then refuse to build. 60 | if ! [[ "${GIT_VERSION}" =~ ^v([0-9]+)\.([0-9]+)(\.[0-9]+)?(-[0-9A-Za-z.-]+)?(\+[0-9A-Za-z.-]+)?$ ]]; then 61 | echo "GIT_VERSION should be a valid Semantic Version. Current value: ${GIT_VERSION}" 62 | echo "Please see more details here: https://semver.org" 63 | exit 1 64 | fi 65 | fi 66 | 67 | GIT_RELEASE_TAG=$(git describe --abbrev=0 --tags) 68 | } 69 | 70 | # borrowed from k8s.io/hack/lib/version.sh and modified 71 | # Prints the value that needs to be passed to the -ldflags parameter of go build 72 | version::ldflags() { 73 | version::get_version_vars 74 | 75 | local -a ldflags 76 | function add_ldflag() { 77 | local key=${1} 78 | local val=${2} 79 | ldflags+=( 80 | "-X 'github.com/syself/hetzner-cloud-controller-manager/hcloud.${key}=${val}'" 81 | ) 82 | } 83 | 84 | add_ldflag "providerVersion" "${GIT_VERSION}" 85 | 86 | # The -ldflags parameter takes a single string, so join the output. 87 | echo "${ldflags[*]-}" 88 | } 89 | 90 | version::ldflags 91 | -------------------------------------------------------------------------------- /hcloud/instances.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Hetzner Cloud GmbH. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package hcloud 18 | 19 | import ( 20 | "context" 21 | "fmt" 22 | 23 | "github.com/hetznercloud/hcloud-go/v2/hcloud" 24 | "github.com/syself/hetzner-cloud-controller-manager/internal/metrics" 25 | robotclient "github.com/syself/hetzner-cloud-controller-manager/internal/robot/client" 26 | "github.com/syself/hrobot-go/models" 27 | corev1 "k8s.io/api/core/v1" 28 | cloudprovider "k8s.io/cloud-provider" 29 | ) 30 | 31 | type addressFamily int 32 | 33 | const ( 34 | AddressFamilyDualStack addressFamily = iota 35 | AddressFamilyIPv6 36 | AddressFamilyIPv4 37 | ) 38 | 39 | type instances struct { 40 | client *hcloud.Client 41 | robotClient robotclient.Client 42 | addressFamily addressFamily 43 | networkID int64 44 | } 45 | 46 | var errServerNotFound = fmt.Errorf("server not found") 47 | 48 | func newInstances(client *hcloud.Client, robotClient robotclient.Client, addressFamily addressFamily, networkID int64) *instances { 49 | return &instances{client, robotClient, addressFamily, networkID} 50 | } 51 | 52 | // lookupServer attempts to locate the corresponding hcloud.Server or models.Server (robot server) for a given v1.Node. 53 | // It returns an error if the Node has an invalid provider ID or if API requests failed. 54 | // It can return a nil [*hcloud.Server] if neither the ProviderID nor the Name matches an existing server. 55 | func (i *instances) lookupServer( 56 | ctx context.Context, 57 | node *corev1.Node, 58 | ) (hcloudServer *hcloud.Server, bmServer *models.Server, isHCloudServer bool, err error) { 59 | if node.Spec.ProviderID != "" { 60 | var serverID int64 61 | serverID, isHCloudServer, err = providerIDToServerID(node.Spec.ProviderID) 62 | if err != nil { 63 | return nil, nil, false, fmt.Errorf("failed to convert provider id to server id: %w", err) 64 | } 65 | 66 | if isHCloudServer { 67 | hcloudServer, err = getHCloudServerByID(ctx, i.client, serverID) 68 | if err != nil { 69 | return nil, nil, false, fmt.Errorf("failed to get hcloud server \"%d\": %w", serverID, err) 70 | } 71 | } else { 72 | if i.robotClient == nil { 73 | return nil, nil, false, errMissingRobotCredentials 74 | } 75 | bmServer, err = getRobotServerByID(i.robotClient, int(serverID), node) 76 | if err != nil { 77 | return nil, nil, false, fmt.Errorf("failed to get robot server \"%d\": %w", serverID, err) 78 | } 79 | } 80 | } else { 81 | if isHCloudServerByName(string(node.Name)) { 82 | isHCloudServer = true 83 | hcloudServer, err = getHCloudServerByName(ctx, i.client, string(node.Name)) 84 | if err != nil { 85 | return nil, nil, false, fmt.Errorf("failed to get hcloud server %q: %w", string(node.Name), err) 86 | } 87 | } else { 88 | if i.robotClient == nil { 89 | return nil, nil, false, errMissingRobotCredentials 90 | } 91 | bmServer, err = getRobotServerByName(i.robotClient, node) 92 | if err != nil { 93 | return nil, nil, false, fmt.Errorf("failed to get robot server %q: %w", string(node.Name), err) 94 | } 95 | } 96 | } 97 | return hcloudServer, bmServer, isHCloudServer, nil 98 | } 99 | 100 | func (i *instances) InstanceExists(ctx context.Context, node *corev1.Node) (bool, error) { 101 | const op = "hcloud/instancesv2.InstanceExists" 102 | metrics.OperationCalled.WithLabelValues(op).Inc() 103 | 104 | hcloudServer, bmServer, _, err := i.lookupServer(ctx, node) 105 | if err != nil { 106 | return false, fmt.Errorf("%s: %w", op, err) 107 | } 108 | 109 | return hcloudServer != nil || bmServer != nil, nil 110 | } 111 | 112 | func (i *instances) InstanceShutdown(ctx context.Context, node *corev1.Node) (bool, error) { 113 | const op = "hcloud/instancesv2.InstanceShutdown" 114 | metrics.OperationCalled.WithLabelValues(op).Inc() 115 | 116 | hcloudServer, _, isHCloudServer, err := i.lookupServer(ctx, node) 117 | if err != nil { 118 | return false, fmt.Errorf("%s: %w", op, err) 119 | } 120 | 121 | if isHCloudServer { 122 | if hcloudServer == nil { 123 | return false, fmt.Errorf("failed to find server status: no matching hcloud server found for node '%s': %w", node.Name, errServerNotFound) 124 | } 125 | return hcloudServer.Status == hcloud.ServerStatusOff, nil 126 | } 127 | 128 | // Robot does not support shutdowns 129 | return false, nil 130 | } 131 | 132 | func (i *instances) InstanceMetadata(ctx context.Context, node *corev1.Node) (*cloudprovider.InstanceMetadata, error) { 133 | const op = "hcloud/instancesv2.InstanceMetadata" 134 | metrics.OperationCalled.WithLabelValues(op).Inc() 135 | 136 | hcloudServer, bmServer, isHCloudServer, err := i.lookupServer(ctx, node) 137 | if err != nil { 138 | return nil, err 139 | } 140 | 141 | if isHCloudServer { 142 | if hcloudServer == nil { 143 | return nil, fmt.Errorf("failed to get instance metadata: no matching hcloud server found for node '%s': %w", 144 | node.Name, errServerNotFound) 145 | } 146 | return &cloudprovider.InstanceMetadata{ 147 | ProviderID: serverIDToProviderIDHCloud(hcloudServer.ID), 148 | InstanceType: hcloudServer.ServerType.Name, 149 | NodeAddresses: hcloudNodeAddresses(i.addressFamily, i.networkID, hcloudServer), 150 | Zone: hcloudServer.Datacenter.Name, 151 | Region: hcloudServer.Datacenter.Location.Name, 152 | }, nil 153 | } 154 | if bmServer == nil { 155 | return nil, fmt.Errorf("failed to get instance metadata: no matching bare metal server found for node '%s': %w", 156 | node.Name, errServerNotFound) 157 | } 158 | return &cloudprovider.InstanceMetadata{ 159 | ProviderID: serverIDToProviderIDRobot(bmServer.ServerNumber), 160 | InstanceType: getInstanceTypeOfRobotServer(bmServer), 161 | NodeAddresses: robotNodeAddresses(i.addressFamily, bmServer), 162 | Zone: getZoneOfRobotServer(bmServer), 163 | Region: getRegionOfRobotServer(bmServer), 164 | }, nil 165 | } 166 | 167 | func hcloudNodeAddresses(addressFamily addressFamily, networkID int64, server *hcloud.Server) []corev1.NodeAddress { 168 | var addresses []corev1.NodeAddress 169 | addresses = append( 170 | addresses, 171 | corev1.NodeAddress{Type: corev1.NodeHostName, Address: server.Name}, 172 | ) 173 | 174 | if addressFamily == AddressFamilyIPv4 || addressFamily == AddressFamilyDualStack { 175 | if !server.PublicNet.IPv4.IsUnspecified() { 176 | addresses = append( 177 | addresses, 178 | corev1.NodeAddress{Type: corev1.NodeExternalIP, Address: server.PublicNet.IPv4.IP.String()}, 179 | ) 180 | } 181 | } 182 | 183 | if addressFamily == AddressFamilyIPv6 || addressFamily == AddressFamilyDualStack { 184 | if !server.PublicNet.IPv6.IsUnspecified() { 185 | // For a given IPv6 network of 2001:db8:1234::/64, the instance address is 2001:db8:1234::1 186 | hostAddress := server.PublicNet.IPv6.IP 187 | hostAddress[len(hostAddress)-1] |= 0x01 188 | 189 | addresses = append( 190 | addresses, 191 | corev1.NodeAddress{Type: corev1.NodeExternalIP, Address: hostAddress.String()}, 192 | ) 193 | } 194 | } 195 | 196 | // Add private IP from network if network is specified 197 | if networkID > 0 { 198 | for _, privateNet := range server.PrivateNet { 199 | if privateNet.Network.ID == networkID { 200 | addresses = append( 201 | addresses, 202 | corev1.NodeAddress{Type: corev1.NodeInternalIP, Address: privateNet.IP.String()}, 203 | ) 204 | } 205 | } 206 | } 207 | return addresses 208 | } 209 | 210 | func robotNodeAddresses(addressFamily addressFamily, server *models.Server) []corev1.NodeAddress { 211 | var addresses []corev1.NodeAddress 212 | addresses = append( 213 | addresses, 214 | corev1.NodeAddress{Type: corev1.NodeHostName, Address: server.Name}, 215 | ) 216 | 217 | if addressFamily == AddressFamilyIPv6 || addressFamily == AddressFamilyDualStack { 218 | // For a given IPv6 network of 2a01:f48:111:4221::, the instance address is 2a01:f48:111:4221::1 219 | hostAddress := server.ServerIPv6Net 220 | hostAddress += "1" 221 | 222 | addresses = append( 223 | addresses, 224 | corev1.NodeAddress{Type: corev1.NodeExternalIP, Address: hostAddress}, 225 | ) 226 | } 227 | 228 | if addressFamily == AddressFamilyIPv4 || addressFamily == AddressFamilyDualStack { 229 | addresses = append( 230 | addresses, 231 | corev1.NodeAddress{Type: corev1.NodeExternalIP, Address: server.ServerIP}, 232 | ) 233 | } 234 | 235 | return addresses 236 | } 237 | -------------------------------------------------------------------------------- /hcloud/load_balancers.go: -------------------------------------------------------------------------------- 1 | package hcloud 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | 8 | "github.com/hetznercloud/hcloud-go/v2/hcloud" 9 | "github.com/syself/hetzner-cloud-controller-manager/internal/annotation" 10 | "github.com/syself/hetzner-cloud-controller-manager/internal/hcops" 11 | "github.com/syself/hetzner-cloud-controller-manager/internal/metrics" 12 | corev1 "k8s.io/api/core/v1" 13 | "k8s.io/apimachinery/pkg/labels" 14 | cloudprovider "k8s.io/cloud-provider" 15 | "k8s.io/klog/v2" 16 | ) 17 | 18 | // LoadBalancerOps defines the Load Balancer related operations required by 19 | // the hcloud-cloud-controller-manager. 20 | type LoadBalancerOps interface { 21 | GetByName(ctx context.Context, name string) (*hcloud.LoadBalancer, error) 22 | GetByID(ctx context.Context, id int64) (*hcloud.LoadBalancer, error) 23 | GetByK8SServiceUID(ctx context.Context, svc *corev1.Service) (*hcloud.LoadBalancer, error) 24 | Create(ctx context.Context, lbName string, service *corev1.Service) (*hcloud.LoadBalancer, error) 25 | Delete(ctx context.Context, lb *hcloud.LoadBalancer) error 26 | ReconcileHCLB(ctx context.Context, lb *hcloud.LoadBalancer, svc *corev1.Service) (bool, error) 27 | ReconcileHCLBTargets(ctx context.Context, lb *hcloud.LoadBalancer, svc *corev1.Service, nodes []*corev1.Node) (bool, error) 28 | ReconcileHCLBServices(ctx context.Context, lb *hcloud.LoadBalancer, svc *corev1.Service) (bool, error) 29 | } 30 | 31 | type loadBalancers struct { 32 | lbOps LoadBalancerOps 33 | ac hcops.HCloudActionClient // Deprecated: should only be referenced by hcops types 34 | disablePrivateIngressDefault bool 35 | disableIPv6Default bool 36 | } 37 | 38 | func newLoadBalancers(lbOps LoadBalancerOps, ac hcops.HCloudActionClient, disablePrivateIngressDefault, disableIPv6Default bool) *loadBalancers { 39 | return &loadBalancers{ 40 | lbOps: lbOps, 41 | ac: ac, 42 | disablePrivateIngressDefault: disablePrivateIngressDefault, 43 | disableIPv6Default: disableIPv6Default, 44 | } 45 | } 46 | 47 | func matchNodeSelector(svc *corev1.Service, nodes []*corev1.Node) ([]*corev1.Node, error) { 48 | var ( 49 | err error 50 | selectedNodes []*corev1.Node 51 | ) 52 | 53 | selector := labels.Everything() 54 | if v, ok := annotation.LBNodeSelector.StringFromService(svc); ok { 55 | selector, err = labels.Parse(v) 56 | if err != nil { 57 | return nil, fmt.Errorf("unable to parse the node-selector annotation: %w", err) 58 | } 59 | } 60 | 61 | for _, n := range nodes { 62 | if selector.Matches(labels.Set(n.GetLabels())) { 63 | selectedNodes = append(selectedNodes, n) 64 | } 65 | } 66 | 67 | return selectedNodes, nil 68 | } 69 | 70 | func (l *loadBalancers) GetLoadBalancer( 71 | ctx context.Context, _ string, service *corev1.Service, 72 | ) (status *corev1.LoadBalancerStatus, exists bool, err error) { 73 | const op = "hcloud/loadBalancers.GetLoadBalancer" 74 | metrics.OperationCalled.WithLabelValues(op).Inc() 75 | 76 | lb, err := l.lbOps.GetByK8SServiceUID(ctx, service) 77 | if err != nil { 78 | if errors.Is(err, hcops.ErrNotFound) { 79 | return nil, false, nil 80 | } 81 | return nil, false, fmt.Errorf("%s: %v", op, err) 82 | } 83 | 84 | if v, ok := annotation.LBHostname.StringFromService(service); ok { 85 | return &corev1.LoadBalancerStatus{ 86 | Ingress: []corev1.LoadBalancerIngress{{Hostname: v}}, 87 | }, true, nil 88 | } 89 | 90 | ingresses := []corev1.LoadBalancerIngress{ 91 | { 92 | IP: lb.PublicNet.IPv4.IP.String(), 93 | }, 94 | } 95 | 96 | disableIPV6, err := l.getDisableIPv6(service) 97 | if err != nil { 98 | return nil, false, fmt.Errorf("%s: %v", op, err) 99 | } 100 | if !disableIPV6 { 101 | ingresses = append(ingresses, corev1.LoadBalancerIngress{ 102 | IP: lb.PublicNet.IPv6.IP.String(), 103 | }) 104 | } 105 | 106 | return &corev1.LoadBalancerStatus{Ingress: ingresses}, true, nil 107 | } 108 | 109 | func (l *loadBalancers) GetLoadBalancerName(_ context.Context, _ string, service *corev1.Service) string { 110 | if v, ok := annotation.LBName.StringFromService(service); ok { 111 | return v 112 | } 113 | return cloudprovider.DefaultLoadBalancerName(service) 114 | } 115 | 116 | func (l *loadBalancers) EnsureLoadBalancer( 117 | ctx context.Context, clusterName string, svc *corev1.Service, nodes []*corev1.Node, 118 | ) (*corev1.LoadBalancerStatus, error) { 119 | const op = "hcloud/loadBalancers.EnsureLoadBalancer" 120 | metrics.OperationCalled.WithLabelValues(op).Inc() 121 | 122 | var ( 123 | reload bool 124 | lb *hcloud.LoadBalancer 125 | err error 126 | selectedNodes []*corev1.Node 127 | ) 128 | 129 | selectedNodes, err = matchNodeSelector(svc, nodes) 130 | if err != nil { 131 | return nil, fmt.Errorf("%s: %w", op, err) 132 | } 133 | 134 | nodeNames := make([]string, len(selectedNodes)) 135 | for i, n := range selectedNodes { 136 | nodeNames[i] = n.Name 137 | } 138 | klog.InfoS("ensure Load Balancer", "op", op, "service", svc.Name, "nodes", nodeNames) 139 | 140 | lb, err = l.lbOps.GetByK8SServiceUID(ctx, svc) 141 | if err != nil && !errors.Is(err, hcops.ErrNotFound) { 142 | return nil, fmt.Errorf("%s: %v", op, err) 143 | } 144 | 145 | // Try the load balancer's name if we were not able to find it using the 146 | // service UID. This is required for two reasons: 147 | // 148 | // 1. Migration of load balancers which where created before identification 149 | // via the service UID was introduced. 150 | // 151 | // 2. Import of load balancers which were created by other means but 152 | // should be re-used by the cloud controller manager. 153 | lbName := l.GetLoadBalancerName(ctx, clusterName, svc) 154 | if errors.Is(err, hcops.ErrNotFound) { 155 | lb, err = l.lbOps.GetByName(ctx, lbName) 156 | if err != nil && !errors.Is(err, hcops.ErrNotFound) { 157 | return nil, fmt.Errorf("%s: %v", op, err) 158 | } 159 | } 160 | 161 | // If we were still not able to find the load balancer we create it. 162 | if errors.Is(err, hcops.ErrNotFound) { 163 | lb, err = l.lbOps.Create(ctx, lbName, svc) 164 | if err != nil { 165 | return nil, fmt.Errorf("%s: %w", op, err) 166 | } 167 | } 168 | 169 | lbChanged, err := l.lbOps.ReconcileHCLB(ctx, lb, svc) 170 | if err != nil { 171 | return nil, fmt.Errorf("%s: %w", op, err) 172 | } 173 | reload = reload || lbChanged 174 | 175 | servicesChanged, err := l.lbOps.ReconcileHCLBServices(ctx, lb, svc) 176 | if err != nil { 177 | return nil, fmt.Errorf("%s: %w", op, err) 178 | } 179 | reload = reload || servicesChanged 180 | 181 | targetsChanged, err := l.lbOps.ReconcileHCLBTargets(ctx, lb, svc, selectedNodes) 182 | if err != nil { 183 | return nil, fmt.Errorf("%s: %w", op, err) 184 | } 185 | reload = reload || targetsChanged 186 | 187 | if reload { 188 | klog.InfoS("reload HC Load Balancer", "op", op, "loadBalancerID", lb.ID) 189 | lb, err = l.lbOps.GetByID(ctx, lb.ID) 190 | if err != nil { 191 | return nil, fmt.Errorf("%s: %w", op, err) 192 | } 193 | } 194 | 195 | if err := annotation.LBToService(svc, lb); err != nil { 196 | return nil, fmt.Errorf("%s: %w", op, err) 197 | } 198 | 199 | // Either set the Hostname or the IPs (below). 200 | // See: https://github.com/kubernetes/kubernetes/issues/66607 201 | if v, ok := annotation.LBHostname.StringFromService(svc); ok { 202 | return &corev1.LoadBalancerStatus{ 203 | Ingress: []corev1.LoadBalancerIngress{{Hostname: v}}, 204 | }, nil 205 | } 206 | 207 | var ingress []corev1.LoadBalancerIngress 208 | 209 | disablePubNet, err := annotation.LBDisablePublicNetwork.BoolFromService(svc) 210 | if err != nil && !errors.Is(err, annotation.ErrNotSet) { 211 | return nil, fmt.Errorf("%s: %w", op, err) 212 | } 213 | 214 | if !disablePubNet { 215 | ingress = append(ingress, corev1.LoadBalancerIngress{IP: lb.PublicNet.IPv4.IP.String()}) 216 | 217 | disableIPV6, err := l.getDisableIPv6(svc) 218 | if err != nil { 219 | return nil, fmt.Errorf("%s: %w", op, err) 220 | } 221 | if !disableIPV6 { 222 | ingress = append(ingress, corev1.LoadBalancerIngress{IP: lb.PublicNet.IPv6.IP.String()}) 223 | } 224 | } 225 | 226 | disablePrivIngress, err := l.getDisablePrivateIngress(svc) 227 | if err != nil { 228 | return nil, fmt.Errorf("%s: %w", op, err) 229 | } 230 | if !disablePrivIngress { 231 | for _, nw := range lb.PrivateNet { 232 | ingress = append(ingress, corev1.LoadBalancerIngress{IP: nw.IP.String()}) 233 | } 234 | } 235 | 236 | return &corev1.LoadBalancerStatus{Ingress: ingress}, nil 237 | } 238 | 239 | func (l *loadBalancers) getDisablePrivateIngress(svc *corev1.Service) (bool, error) { 240 | disable, err := annotation.LBDisablePrivateIngress.BoolFromService(svc) 241 | if err == nil { 242 | return disable, nil 243 | } 244 | if errors.Is(err, annotation.ErrNotSet) { 245 | return l.disablePrivateIngressDefault, nil 246 | } 247 | return false, err 248 | } 249 | 250 | func (l *loadBalancers) getDisableIPv6(svc *corev1.Service) (bool, error) { 251 | disable, err := annotation.LBIPv6Disabled.BoolFromService(svc) 252 | if err == nil { 253 | return disable, nil 254 | } 255 | if errors.Is(err, annotation.ErrNotSet) { 256 | return l.disableIPv6Default, nil 257 | } 258 | return false, err 259 | } 260 | 261 | func (l *loadBalancers) UpdateLoadBalancer( 262 | ctx context.Context, clusterName string, svc *corev1.Service, nodes []*corev1.Node, 263 | ) error { 264 | const op = "hcloud/loadBalancers.UpdateLoadBalancer" 265 | metrics.OperationCalled.WithLabelValues(op).Inc() 266 | 267 | var ( 268 | lb *hcloud.LoadBalancer 269 | err error 270 | selectedNodes []*corev1.Node 271 | ) 272 | 273 | selectedNodes, err = matchNodeSelector(svc, nodes) 274 | if err != nil { 275 | return fmt.Errorf("%s: %w", op, err) 276 | } 277 | 278 | nodeNames := make([]string, len(selectedNodes)) 279 | for i, n := range selectedNodes { 280 | nodeNames[i] = n.Name 281 | } 282 | klog.InfoS("update Load Balancer", "op", op, "service", svc.Name, "nodes", nodeNames) 283 | 284 | lb, err = l.lbOps.GetByK8SServiceUID(ctx, svc) 285 | if errors.Is(err, hcops.ErrNotFound) { 286 | lbName := l.GetLoadBalancerName(ctx, clusterName, svc) 287 | 288 | lb, err = l.lbOps.GetByName(ctx, lbName) 289 | if errors.Is(err, hcops.ErrNotFound) { 290 | return nil 291 | } 292 | // further error types handled below 293 | } 294 | if err != nil { 295 | return fmt.Errorf("%s: %w", op, err) 296 | } 297 | 298 | if _, err = l.lbOps.ReconcileHCLB(ctx, lb, svc); err != nil { 299 | return fmt.Errorf("%s: %w", op, err) 300 | } 301 | 302 | if _, err = l.lbOps.ReconcileHCLBTargets(ctx, lb, svc, selectedNodes); err != nil { 303 | return fmt.Errorf("%s: %w", op, err) 304 | } 305 | if _, err = l.lbOps.ReconcileHCLBServices(ctx, lb, svc); err != nil { 306 | return fmt.Errorf("%s: %w", op, err) 307 | } 308 | return nil 309 | } 310 | 311 | func (l *loadBalancers) EnsureLoadBalancerDeleted(ctx context.Context, _ string, service *corev1.Service) error { 312 | const op = "hcloud/loadBalancers.EnsureLoadBalancerDeleted" 313 | metrics.OperationCalled.WithLabelValues(op).Inc() 314 | 315 | loadBalancer, err := l.lbOps.GetByK8SServiceUID(ctx, service) 316 | if errors.Is(err, hcops.ErrNotFound) { 317 | return nil 318 | } 319 | if err != nil { 320 | return fmt.Errorf("%s: %w", op, err) 321 | } 322 | 323 | if loadBalancer.Protection.Delete { 324 | klog.InfoS("ignored: load balancer deletion protected", "op", op, "loadBalancerID", loadBalancer.ID) 325 | return nil 326 | } 327 | 328 | klog.InfoS("delete Load Balancer", "op", op, "loadBalancerID", loadBalancer.ID) 329 | err = l.lbOps.Delete(ctx, loadBalancer) 330 | if errors.Is(err, hcops.ErrNotFound) { 331 | return nil 332 | } 333 | if err != nil { 334 | return fmt.Errorf("%s: %w", op, err) 335 | } 336 | 337 | return nil 338 | } 339 | -------------------------------------------------------------------------------- /hcloud/routes.go: -------------------------------------------------------------------------------- 1 | package hcloud 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "net" 8 | "time" 9 | 10 | "github.com/hetznercloud/hcloud-go/v2/hcloud" 11 | "github.com/syself/hetzner-cloud-controller-manager/internal/hcops" 12 | "github.com/syself/hetzner-cloud-controller-manager/internal/metrics" 13 | "k8s.io/apimachinery/pkg/types" 14 | cloudprovider "k8s.io/cloud-provider" 15 | "k8s.io/klog/v2" 16 | ) 17 | 18 | type routes struct { 19 | client *hcloud.Client 20 | network *hcloud.Network 21 | serverCache *hcops.AllServersCache 22 | } 23 | 24 | func newRoutes(client *hcloud.Client, networkID int64) (*routes, error) { 25 | const op = "hcloud/newRoutes" 26 | metrics.OperationCalled.WithLabelValues(op).Inc() 27 | 28 | networkObj, _, err := client.Network.GetByID(context.Background(), networkID) 29 | if err != nil { 30 | return nil, fmt.Errorf("%s: %w", op, err) 31 | } 32 | if networkObj == nil { 33 | return nil, fmt.Errorf("network not found: %d", networkID) 34 | } 35 | 36 | return &routes{ 37 | client: client, 38 | network: networkObj, 39 | serverCache: &hcops.AllServersCache{ 40 | // client.Server.All will load ALL the servers in the project, even those 41 | // that are not part of the Kubernetes cluster. 42 | LoadFunc: client.Server.All, 43 | Network: networkObj, 44 | }, 45 | }, nil 46 | } 47 | 48 | func (r *routes) reloadNetwork(ctx context.Context) error { 49 | const op = "hcloud/reloadNetwork" 50 | metrics.OperationCalled.WithLabelValues(op).Inc() 51 | 52 | networkObj, _, err := r.client.Network.GetByID(ctx, r.network.ID) 53 | if err != nil { 54 | return fmt.Errorf("%s: %w", op, err) 55 | } 56 | if networkObj == nil { 57 | return fmt.Errorf("network not found: %s", r.network.Name) 58 | } 59 | r.network = networkObj 60 | return nil 61 | } 62 | 63 | // ListRoutes lists all managed routes that belong to the specified clusterName. 64 | func (r *routes) ListRoutes(ctx context.Context, _ string) ([]*cloudprovider.Route, error) { 65 | const op = "hcloud/ListRoutes" 66 | metrics.OperationCalled.WithLabelValues(op).Inc() 67 | 68 | if err := r.reloadNetwork(ctx); err != nil { 69 | return nil, fmt.Errorf("%s: %w", op, err) 70 | } 71 | 72 | routes := make([]*cloudprovider.Route, 0, len(r.network.Routes)) 73 | for _, route := range r.network.Routes { 74 | ro, err := r.hcloudRouteToRoute(route) 75 | if err != nil { 76 | return routes, fmt.Errorf("%s: %w", op, err) 77 | } 78 | routes = append(routes, ro) 79 | } 80 | return routes, nil 81 | } 82 | 83 | // CreateRoute creates the described managed route 84 | // route.Name will be ignored, although the cloud-provider may use nameHint 85 | // to create a more user-meaningful name. 86 | func (r *routes) CreateRoute(ctx context.Context, clusterName, nameHint string, route *cloudprovider.Route) error { 87 | const op = "hcloud/CreateRoute" 88 | metrics.OperationCalled.WithLabelValues(op).Inc() 89 | 90 | srv, err := r.serverCache.ByName(string(route.TargetNode)) 91 | if err != nil { 92 | return fmt.Errorf("%s: %v", op, err) 93 | } 94 | 95 | privNet, ok := findServerPrivateNetByID(srv, r.network.ID) 96 | if !ok { 97 | r.serverCache.InvalidateCache() 98 | srv, err = r.serverCache.ByName(string(route.TargetNode)) 99 | if err != nil { 100 | return fmt.Errorf("%s: %v", op, err) 101 | } 102 | 103 | privNet, ok = findServerPrivateNetByID(srv, r.network.ID) 104 | if !ok { 105 | return fmt.Errorf("%s: server %v: network with id %d not attached to this server ", op, route.TargetNode, r.network.ID) 106 | } 107 | } 108 | ip := privNet.IP 109 | 110 | _, cidr, err := net.ParseCIDR(route.DestinationCIDR) 111 | if err != nil { 112 | return fmt.Errorf("%s: %w", op, err) 113 | } 114 | 115 | doesRouteAlreadyExist, err := r.checkIfRouteAlreadyExists(ctx, route) 116 | if err != nil { 117 | return fmt.Errorf("%s: %w", op, err) 118 | } 119 | 120 | if !doesRouteAlreadyExist { 121 | opts := hcloud.NetworkAddRouteOpts{ 122 | Route: hcloud.NetworkRoute{ 123 | Destination: cidr, 124 | Gateway: ip, 125 | }, 126 | } 127 | action, _, err := r.client.Network.AddRoute(ctx, r.network, opts) 128 | if err != nil { 129 | if hcloud.IsError(err, hcloud.ErrorCodeLocked) || hcloud.IsError(err, hcloud.ErrorCodeConflict) { 130 | retryDelay := time.Second * 5 131 | klog.InfoS("retry due to conflict or lock", 132 | "op", op, "delay", fmt.Sprintf("%v", retryDelay), "err", fmt.Sprintf("%v", err)) 133 | time.Sleep(retryDelay) 134 | 135 | return r.CreateRoute(ctx, clusterName, nameHint, route) 136 | } 137 | return fmt.Errorf("%s: %w", op, err) 138 | } 139 | 140 | if err := hcops.WatchAction(ctx, &r.client.Action, action); err != nil { 141 | return fmt.Errorf("%s: %w", op, err) 142 | } 143 | } 144 | return nil 145 | } 146 | 147 | // DeleteRoute deletes the specified managed route 148 | // Route should be as returned by ListRoutes. 149 | func (r *routes) DeleteRoute(ctx context.Context, _ string, route *cloudprovider.Route) error { 150 | const op = "hcloud/DeleteRoute" 151 | metrics.OperationCalled.WithLabelValues(op).Inc() 152 | 153 | // Get target IP from current list of routes, routes can be uniquely identified by their destination cidr. 154 | var ip net.IP 155 | for _, cloudRoute := range r.network.Routes { 156 | if cloudRoute.Destination.String() == route.DestinationCIDR { 157 | ip = cloudRoute.Gateway 158 | break 159 | } 160 | } 161 | if ip.IsUnspecified() { 162 | return fmt.Errorf("%s: route %s not found in cloud network routes", op, route.Name) 163 | } 164 | 165 | _, cidr, err := net.ParseCIDR(route.DestinationCIDR) 166 | if err != nil { 167 | return fmt.Errorf("%s: %w", op, err) 168 | } 169 | 170 | err = r.deleteRouteFromHcloud(ctx, cidr, ip) 171 | if err != nil { 172 | return fmt.Errorf("%s: %w", op, err) 173 | } 174 | return nil 175 | } 176 | 177 | func (r *routes) deleteRouteFromHcloud(ctx context.Context, cidr *net.IPNet, ip net.IP) error { 178 | const op = "hcloud/deleteRouteFromHcloud" 179 | metrics.OperationCalled.WithLabelValues(op).Inc() 180 | 181 | opts := hcloud.NetworkDeleteRouteOpts{ 182 | Route: hcloud.NetworkRoute{ 183 | Destination: cidr, 184 | Gateway: ip, 185 | }, 186 | } 187 | 188 | action, _, err := r.client.Network.DeleteRoute(ctx, r.network, opts) 189 | if err != nil { 190 | if hcloud.IsError(err, hcloud.ErrorCodeLocked) || hcloud.IsError(err, hcloud.ErrorCodeConflict) { 191 | retryDelay := time.Second * 5 192 | klog.InfoS("retry due to conflict or lock", 193 | "op", op, "delay", fmt.Sprintf("%v", retryDelay), "err", fmt.Sprintf("%v", err)) 194 | time.Sleep(retryDelay) 195 | 196 | return r.deleteRouteFromHcloud(ctx, cidr, ip) 197 | } 198 | return fmt.Errorf("%s: %w", op, err) 199 | } 200 | if err := hcops.WatchAction(ctx, &r.client.Action, action); err != nil { 201 | return fmt.Errorf("%s: %w", op, err) 202 | } 203 | return nil 204 | } 205 | 206 | func (r *routes) hcloudRouteToRoute(route hcloud.NetworkRoute) (*cloudprovider.Route, error) { 207 | const op = "hcloud/hcloudRouteToRoute" 208 | metrics.OperationCalled.WithLabelValues(op).Inc() 209 | 210 | cpRoute := &cloudprovider.Route{ 211 | DestinationCIDR: route.Destination.String(), 212 | Name: fmt.Sprintf("%s-%s", route.Gateway.String(), route.Destination.String()), 213 | } 214 | 215 | srv, err := r.serverCache.ByPrivateIP(route.Gateway) 216 | if err != nil { 217 | if errors.Is(err, hcops.ErrNotFound) { 218 | // Route belongs to non-existing target 219 | cpRoute.Blackhole = true 220 | return cpRoute, nil 221 | } 222 | 223 | return nil, fmt.Errorf("%s: %w", op, err) 224 | } 225 | 226 | cpRoute.TargetNode = types.NodeName(srv.Name) 227 | return cpRoute, nil 228 | } 229 | 230 | func (r *routes) checkIfRouteAlreadyExists(ctx context.Context, route *cloudprovider.Route) (bool, error) { 231 | const op = "hcloud/checkIfRouteAlreadyExists" 232 | metrics.OperationCalled.WithLabelValues(op).Inc() 233 | 234 | if err := r.reloadNetwork(ctx); err != nil { 235 | return false, fmt.Errorf("%s: %w", op, err) 236 | } 237 | 238 | for _, _route := range r.network.Routes { 239 | if _route.Destination.String() == route.DestinationCIDR { 240 | srv, err := r.serverCache.ByName(string(route.TargetNode)) 241 | if err != nil { 242 | return false, fmt.Errorf("%s: %v", op, err) 243 | } 244 | privNet, ok := findServerPrivateNetByID(srv, r.network.ID) 245 | if !ok { 246 | return false, fmt.Errorf("%s: server %v: no network with id: %d", op, route.TargetNode, r.network.ID) 247 | } 248 | ip := privNet.IP 249 | 250 | if !_route.Gateway.Equal(ip) { 251 | action, _, err := r.client.Network.DeleteRoute(context.Background(), r.network, hcloud.NetworkDeleteRouteOpts{ 252 | Route: _route, 253 | }) 254 | if err != nil { 255 | return false, fmt.Errorf("%s: %w", op, err) 256 | } 257 | 258 | if err := hcops.WatchAction(ctx, &r.client.Action, action); err != nil { 259 | return false, fmt.Errorf("%s: %w", op, err) 260 | } 261 | } 262 | 263 | return true, nil 264 | } 265 | } 266 | return false, nil 267 | } 268 | 269 | func findServerPrivateNetByID(srv *hcloud.Server, id int64) (hcloud.ServerPrivateNet, bool) { 270 | for _, n := range srv.PrivateNet { 271 | if n.Network.ID == id { 272 | return n, true 273 | } 274 | } 275 | return hcloud.ServerPrivateNet{}, false 276 | } 277 | -------------------------------------------------------------------------------- /hcloud/routes_test.go: -------------------------------------------------------------------------------- 1 | package hcloud 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "net/http" 7 | "testing" 8 | 9 | "github.com/hetznercloud/hcloud-go/v2/hcloud" 10 | "github.com/hetznercloud/hcloud-go/v2/hcloud/schema" 11 | cloudprovider "k8s.io/cloud-provider" 12 | ) 13 | 14 | func TestRoutes_CreateRoute(t *testing.T) { 15 | env := newTestEnv() 16 | defer env.Teardown() 17 | env.Mux.HandleFunc("/servers", func(w http.ResponseWriter, r *http.Request) { 18 | json.NewEncoder(w).Encode(schema.ServerListResponse{ 19 | Servers: []schema.Server{ 20 | { 21 | ID: 1, 22 | Name: "node15", 23 | PrivateNet: []schema.ServerPrivateNet{ 24 | { 25 | Network: 1, 26 | IP: "10.0.0.2", 27 | }, 28 | }, 29 | }, 30 | }, 31 | }) 32 | }) 33 | env.Mux.HandleFunc("/networks/1", func(w http.ResponseWriter, r *http.Request) { 34 | json.NewEncoder(w).Encode(schema.NetworkGetResponse{ 35 | Network: schema.Network{ 36 | ID: 1, 37 | Name: "network-1", 38 | IPRange: "10.0.0.0/8", 39 | }, 40 | }) 41 | }) 42 | env.Mux.HandleFunc("/actions", func(w http.ResponseWriter, _ *http.Request) { 43 | json.NewEncoder(w).Encode(schema.ActionListResponse{ 44 | Actions: []schema.Action{ 45 | { 46 | ID: 1, 47 | Status: string(hcloud.ActionStatusSuccess), 48 | Progress: 100, 49 | }, 50 | }, 51 | }) 52 | }) 53 | env.Mux.HandleFunc("/networks/1/actions/add_route", func(w http.ResponseWriter, r *http.Request) { 54 | var reqBody schema.NetworkActionAddRouteRequest 55 | if err := json.NewDecoder(r.Body).Decode(&reqBody); err != nil { 56 | t.Fatal(err) 57 | } 58 | if reqBody.Destination != "10.5.0.0/24" { 59 | t.Errorf("unexpected Destination: %v", reqBody.Destination) 60 | } 61 | if reqBody.Gateway != "10.0.0.2" { 62 | t.Errorf("unexpected Gateway: %v", reqBody.Gateway) 63 | } 64 | json.NewEncoder(w).Encode(schema.NetworkActionAddRouteResponse{ 65 | Action: schema.Action{ 66 | ID: 1, 67 | Progress: 0, 68 | Status: string(hcloud.ActionStatusRunning), 69 | }, 70 | }) 71 | }) 72 | routes, err := newRoutes(env.Client, 1) 73 | if err != nil { 74 | t.Fatalf("Unexpected error: %v", err) 75 | } 76 | 77 | err = routes.CreateRoute(context.TODO(), "my-cluster", "route", &cloudprovider.Route{ 78 | Name: "route", 79 | TargetNode: "node15", 80 | DestinationCIDR: "10.5.0.0/24", 81 | }) 82 | if err != nil { 83 | t.Fatalf("Unexpected error: %v", err) 84 | } 85 | } 86 | 87 | func TestRoutes_ListRoutes(t *testing.T) { 88 | env := newTestEnv() 89 | defer env.Teardown() 90 | env.Mux.HandleFunc("/servers", func(w http.ResponseWriter, r *http.Request) { 91 | json.NewEncoder(w).Encode(schema.ServerListResponse{ 92 | Servers: []schema.Server{ 93 | { 94 | ID: 1, 95 | Name: "node15", 96 | PrivateNet: []schema.ServerPrivateNet{ 97 | { 98 | Network: 1, 99 | IP: "10.0.0.2", 100 | }, 101 | }, 102 | }, 103 | }, 104 | }) 105 | }) 106 | env.Mux.HandleFunc("/networks/1", func(w http.ResponseWriter, r *http.Request) { 107 | json.NewEncoder(w).Encode(schema.NetworkGetResponse{ 108 | Network: schema.Network{ 109 | ID: 1, 110 | Name: "network-1", 111 | IPRange: "10.0.0.0/8", 112 | Routes: []schema.NetworkRoute{ 113 | { 114 | Destination: "10.5.0.0/24", 115 | Gateway: "10.0.0.2", 116 | }, 117 | }, 118 | }, 119 | }) 120 | }) 121 | routes, err := newRoutes(env.Client, 1) 122 | if err != nil { 123 | t.Fatalf("Unexpected error: %v", err) 124 | } 125 | 126 | r, err := routes.ListRoutes(context.TODO(), "my-cluster") 127 | if err != nil { 128 | t.Fatalf("Unexpected error: %v", err) 129 | } 130 | if len(r) != 1 { 131 | t.Errorf("Unexpected routes %v", len(r)) 132 | } 133 | if r[0].DestinationCIDR != "10.5.0.0/24" { 134 | t.Errorf("Unexpected DestinationCIDR %v", r[0].DestinationCIDR) 135 | } 136 | if r[0].TargetNode != "node15" { 137 | t.Errorf("Unexpected TargetNode %v", r[0].TargetNode) 138 | } 139 | } 140 | 141 | func TestRoutes_DeleteRoute(t *testing.T) { 142 | env := newTestEnv() 143 | defer env.Teardown() 144 | env.Mux.HandleFunc("/networks/1", func(w http.ResponseWriter, r *http.Request) { 145 | json.NewEncoder(w).Encode(schema.NetworkGetResponse{ 146 | Network: schema.Network{ 147 | ID: 1, 148 | Name: "network-1", 149 | IPRange: "10.0.0.0/8", 150 | Routes: []schema.NetworkRoute{ 151 | { 152 | Destination: "10.5.0.0/24", 153 | Gateway: "10.0.0.2", 154 | }, 155 | }, 156 | }, 157 | }) 158 | }) 159 | env.Mux.HandleFunc("/actions", func(w http.ResponseWriter, _ *http.Request) { 160 | json.NewEncoder(w).Encode(schema.ActionListResponse{ 161 | Actions: []schema.Action{ 162 | { 163 | ID: 1, 164 | Status: string(hcloud.ActionStatusSuccess), 165 | Progress: 100, 166 | }, 167 | }, 168 | }) 169 | }) 170 | env.Mux.HandleFunc("/networks/1/actions/delete_route", func(w http.ResponseWriter, r *http.Request) { 171 | var reqBody schema.NetworkActionDeleteRouteRequest 172 | if err := json.NewDecoder(r.Body).Decode(&reqBody); err != nil { 173 | t.Fatal(err) 174 | } 175 | if reqBody.Destination != "10.5.0.0/24" { 176 | t.Errorf("unexpected Destination: %v", reqBody.Destination) 177 | } 178 | if reqBody.Gateway != "10.0.0.2" { 179 | t.Errorf("unexpected Gateway: %v", reqBody.Gateway) 180 | } 181 | json.NewEncoder(w).Encode(schema.NetworkActionDeleteRouteResponse{ 182 | Action: schema.Action{ 183 | ID: 1, 184 | Progress: 0, 185 | Status: string(hcloud.ActionStatusRunning), 186 | }, 187 | }) 188 | }) 189 | routes, err := newRoutes(env.Client, 1) 190 | if err != nil { 191 | t.Fatalf("Unexpected error: %v", err) 192 | } 193 | 194 | err = routes.DeleteRoute(context.TODO(), "my-cluster", &cloudprovider.Route{ 195 | Name: "route", 196 | TargetNode: "node15", 197 | DestinationCIDR: "10.5.0.0/24", 198 | }) 199 | if err != nil { 200 | t.Fatalf("Unexpected error: %v", err) 201 | } 202 | } 203 | -------------------------------------------------------------------------------- /hcloud/testing.go: -------------------------------------------------------------------------------- 1 | package hcloud 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "testing" 7 | 8 | "github.com/hetznercloud/hcloud-go/v2/hcloud" 9 | "github.com/syself/hetzner-cloud-controller-manager/internal/annotation" 10 | "github.com/syself/hetzner-cloud-controller-manager/internal/hcops" 11 | "github.com/syself/hetzner-cloud-controller-manager/internal/mocks" 12 | corev1 "k8s.io/api/core/v1" 13 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 14 | "k8s.io/apimachinery/pkg/types" 15 | ) 16 | 17 | // Setenv prepares the environment for testing the 18 | // hcloud-cloud-controller-manager. 19 | func Setenv(t *testing.T, args ...string) func() { 20 | if len(args)%2 != 0 { 21 | t.Fatal("Sentenv: uneven number of args") 22 | } 23 | 24 | newVars := make([]string, 0, len(args)/2) 25 | oldEnv := make(map[string]string, len(newVars)) 26 | 27 | for i := 0; i < len(args); i += 2 { 28 | k, v := args[i], args[i+1] 29 | newVars = append(newVars, k) 30 | 31 | if old, ok := os.LookupEnv(k); ok { 32 | oldEnv[k] = old 33 | } 34 | if err := os.Setenv(k, v); err != nil { 35 | t.Fatalf("Setenv failed: %v", err) 36 | } 37 | } 38 | 39 | return func() { 40 | for _, k := range newVars { 41 | v, ok := oldEnv[k] 42 | if !ok { 43 | if err := os.Unsetenv(k); err != nil { 44 | t.Errorf("Unsetenv failed: %v", err) 45 | } 46 | continue 47 | } 48 | if err := os.Setenv(k, v); err != nil { 49 | t.Errorf("Setenv failed: %v", err) 50 | } 51 | } 52 | } 53 | } 54 | 55 | // SkipEnv skips t if any of the passed vars is not set as an environment 56 | // variable. 57 | // 58 | // SkipEnv uses os.LookupEnv. The empty string is therefore a valid value. 59 | func SkipEnv(t *testing.T, vars ...string) { 60 | for _, v := range vars { 61 | if _, ok := os.LookupEnv(v); !ok { 62 | t.Skipf("Environment variable not set: %s", v) 63 | return 64 | } 65 | } 66 | } 67 | 68 | type LoadBalancerTestCase struct { 69 | Name string 70 | 71 | // Defined in test case as needed 72 | ClusterName string 73 | NetworkID int 74 | ServiceUID string 75 | ServiceAnnotations map[annotation.Name]interface{} 76 | DisablePrivateIngressDefault bool 77 | DisableIPv6Default bool 78 | Nodes []*corev1.Node 79 | LB *hcloud.LoadBalancer 80 | LBCreateResult *hcloud.LoadBalancerCreateResult 81 | Mock func(t *testing.T, tt *LoadBalancerTestCase) 82 | 83 | // Defines the actual test 84 | Perform func(t *testing.T, tt *LoadBalancerTestCase) 85 | 86 | Ctx context.Context // Set to context.Background by run if not defined in test 87 | 88 | // Set by run 89 | LBOps *hcops.MockLoadBalancerOps 90 | LBClient *mocks.LoadBalancerClient 91 | ActionClient *mocks.ActionClient 92 | LoadBalancers *loadBalancers 93 | Service *corev1.Service 94 | } 95 | 96 | func (tt *LoadBalancerTestCase) run(t *testing.T) { 97 | t.Helper() 98 | 99 | tt.LBOps = &hcops.MockLoadBalancerOps{} 100 | tt.LBOps.Test(t) 101 | 102 | tt.ActionClient = &mocks.ActionClient{} 103 | tt.ActionClient.Test(t) 104 | 105 | tt.LBClient = &mocks.LoadBalancerClient{} 106 | tt.LBClient.Test(t) 107 | 108 | if tt.ClusterName == "" { 109 | tt.ClusterName = "test-cluster" 110 | } 111 | tt.Service = &corev1.Service{ 112 | ObjectMeta: metav1.ObjectMeta{UID: types.UID(tt.ServiceUID)}, 113 | } 114 | for k, v := range tt.ServiceAnnotations { 115 | if err := k.AnnotateService(tt.Service, v); err != nil { 116 | t.Fatal(err) 117 | } 118 | } 119 | if tt.Ctx == nil { 120 | tt.Ctx = context.Background() 121 | } 122 | 123 | if tt.Mock != nil { 124 | tt.Mock(t, tt) 125 | } 126 | 127 | tt.LoadBalancers = newLoadBalancers(tt.LBOps, tt.ActionClient, tt.DisablePrivateIngressDefault, tt.DisableIPv6Default) 128 | tt.Perform(t, tt) 129 | 130 | tt.LBOps.AssertExpectations(t) 131 | tt.LBClient.AssertExpectations(t) 132 | tt.ActionClient.AssertExpectations(t) 133 | } 134 | 135 | func RunLoadBalancerTests(t *testing.T, tests []LoadBalancerTestCase) { 136 | for _, tt := range tests { 137 | tt.run(t) 138 | } 139 | } 140 | -------------------------------------------------------------------------------- /hcloud/util.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Hetzner Cloud GmbH. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package hcloud 18 | 19 | import ( 20 | "context" 21 | "fmt" 22 | "regexp" 23 | "strconv" 24 | "strings" 25 | 26 | "github.com/hetznercloud/hcloud-go/v2/hcloud" 27 | "github.com/syself/hetzner-cloud-controller-manager/internal/hcops" 28 | "github.com/syself/hetzner-cloud-controller-manager/internal/metrics" 29 | robotclient "github.com/syself/hetzner-cloud-controller-manager/internal/robot/client" 30 | "github.com/syself/hrobot-go/models" 31 | corev1 "k8s.io/api/core/v1" 32 | "k8s.io/klog/v2" 33 | ) 34 | 35 | func getHCloudServerByName(ctx context.Context, c *hcloud.Client, name string) (*hcloud.Server, error) { 36 | const op = "hcloud/getServerByName" 37 | metrics.OperationCalled.WithLabelValues(op).Inc() 38 | 39 | server, _, err := c.Server.GetByName(ctx, name) 40 | if err != nil { 41 | return nil, fmt.Errorf("%s: %w", op, err) 42 | } 43 | 44 | return server, nil 45 | } 46 | 47 | func getHCloudServerByID(ctx context.Context, c *hcloud.Client, id int64) (*hcloud.Server, error) { 48 | const op = "hcloud/getServerByID" 49 | metrics.OperationCalled.WithLabelValues(op).Inc() 50 | 51 | server, _, err := c.Server.GetByID(ctx, id) 52 | if err != nil { 53 | return nil, fmt.Errorf("%s: %w", op, err) 54 | } 55 | return server, nil 56 | } 57 | 58 | func getRobotServerByName(c robotclient.Client, node *corev1.Node) (server *models.Server, err error) { 59 | const op = "robot/getServerByName" 60 | 61 | if c == nil { 62 | return nil, errMissingRobotCredentials 63 | } 64 | 65 | // check for rate limit 66 | if hcops.IsRateLimitExceeded(node) { 67 | return nil, fmt.Errorf("%s: rate limit exceeded - next try at %q", op, hcops.TimeOfNextPossibleAPICall().String()) 68 | } 69 | 70 | serverList, err := c.ServerGetList() 71 | if err != nil { 72 | hcops.HandleRateLimitExceededError(err, node) 73 | return nil, fmt.Errorf("%s: %w", op, err) 74 | } 75 | 76 | for i, s := range serverList { 77 | if s.Name == node.Name { 78 | server = &serverList[i] 79 | } 80 | } 81 | 82 | return server, nil 83 | } 84 | 85 | func getRobotServerByID(c robotclient.Client, id int, node *corev1.Node) (s *models.Server, e error) { 86 | const op = "robot/getServerByID" 87 | if node.Name == "" { 88 | return nil, fmt.Errorf("%s: node name is empty", op) 89 | } 90 | 91 | if c == nil { 92 | return nil, errMissingRobotCredentials 93 | } 94 | 95 | // check for rate limit 96 | if hcops.IsRateLimitExceeded(node) { 97 | return nil, fmt.Errorf("%s: rate limit exceeded - next try at %q", op, hcops.TimeOfNextPossibleAPICall().String()) 98 | } 99 | 100 | server, err := c.ServerGet(id) 101 | if models.IsError(err, models.ErrorCodeServerNotFound) { 102 | return nil, nil 103 | } 104 | if err != nil { 105 | hcops.HandleRateLimitExceededError(err, node) 106 | return nil, fmt.Errorf("%s: %w", op, err) 107 | } 108 | 109 | // check whether name matches - otherwise this server does not belong to the respective node anymore 110 | if server == nil { 111 | return nil, nil 112 | } 113 | if server.Name != node.Name { 114 | return nil, nil 115 | } 116 | 117 | // return nil, nil if server could not be found 118 | return server, nil 119 | } 120 | 121 | func providerIDToServerID(providerID string) (id int64, isHCloudServer bool, err error) { 122 | const op = "hcloud/providerIDToServerID" 123 | metrics.OperationCalled.WithLabelValues(op).Inc() 124 | 125 | providerPrefixHCloud := providerName + "://" 126 | providerPrefixRobot := providerName + "://" + hostNamePrefixRobot 127 | 128 | if !strings.HasPrefix(providerID, providerPrefixHCloud) && !strings.HasPrefix(providerID, providerPrefixRobot) { 129 | klog.Infof("%s: make sure your cluster configured for an external cloud provider", op) 130 | return 0, false, fmt.Errorf("%s: missing prefix %s or %s. %s", providerPrefixHCloud, providerPrefixRobot, op, providerID) 131 | } 132 | 133 | isHCloudServer = true 134 | idString := providerID 135 | if strings.HasPrefix(providerID, providerPrefixRobot) { 136 | isHCloudServer = false 137 | idString = strings.ReplaceAll(idString, providerPrefixRobot, "") 138 | } else { 139 | idString = strings.ReplaceAll(providerID, providerPrefixHCloud, "") 140 | } 141 | 142 | if idString == "" { 143 | return 0, false, fmt.Errorf("%s: missing serverID: %s", op, providerID) 144 | } 145 | 146 | id, err = strconv.ParseInt(idString, 10, 64) 147 | if err != nil { 148 | return 0, false, fmt.Errorf("%s: invalid serverID: %s", op, providerID) 149 | } 150 | return id, isHCloudServer, nil 151 | } 152 | 153 | func isHCloudServerByName(name string) bool { 154 | return !strings.HasPrefix(name, hostNamePrefixRobot) 155 | } 156 | 157 | func serverIDToProviderIDRobot(serverID int) string { 158 | return fmt.Sprintf("%s://%s%d", providerName, hostNamePrefixRobot, serverID) 159 | } 160 | 161 | func serverIDToProviderIDHCloud(serverID int64) string { 162 | return fmt.Sprintf("%s://%d", providerName, serverID) 163 | } 164 | 165 | func getInstanceTypeOfRobotServer(bmServer *models.Server) string { 166 | if bmServer == nil { 167 | panic("getInstanceTypeOfRobotServer called with nil server") 168 | } 169 | return stringToLabelValue(bmServer.Product) 170 | } 171 | 172 | var stringToLabelValueRegex = regexp.MustCompile(`[^a-zA-Z0-9_.]+`) 173 | 174 | func stringToLabelValue(s string) string { 175 | s = stringToLabelValueRegex.ReplaceAllString(s, "-") 176 | trimChars := "_.-" 177 | s = strings.Trim(s, trimChars) 178 | return s 179 | } 180 | 181 | func getZoneOfRobotServer(bmServer *models.Server) string { 182 | if bmServer == nil { 183 | panic("getZoneOfRobotServer called with nil server") 184 | } 185 | return strings.ToLower(bmServer.Dc[:4]) 186 | } 187 | 188 | func getRegionOfRobotServer(bmServer *models.Server) string { 189 | if bmServer == nil { 190 | panic("getZoneOfRobotServer called with nil server") 191 | } 192 | zoneToRegionMap := map[string]string{ 193 | "nbg1": "eu-central", 194 | "fsn1": "eu-central", 195 | "hel1": "eu-central", 196 | "ash": "us-east", 197 | } 198 | zone := getZoneOfRobotServer(bmServer) 199 | region, found := zoneToRegionMap[zone] 200 | if !found { 201 | panic("zoneToRegionMap: unknown zone") 202 | } 203 | return region 204 | } 205 | -------------------------------------------------------------------------------- /hcloud/util_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Hetzner Cloud GmbH. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package hcloud 18 | 19 | import ( 20 | "testing" 21 | ) 22 | 23 | func Test_stringToLabelValue(t *testing.T) { 24 | tests := []struct { 25 | in string 26 | want string 27 | }{ 28 | {"üaü", "a"}, 29 | {" Foo - - Power™ ", "Foo-Power"}, 30 | {"üa--./_", "a"}, 31 | } 32 | for _, tt := range tests { 33 | actual := stringToLabelValue(tt.in) 34 | if actual != tt.want { 35 | t.Errorf("stringToLabelValue(%q) = %q, want %q", tt.in, actual, tt.want) 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /images/hetzner-cloud-controller-manager/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2019 The Kubernetes Authors. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # Build the manager binary 16 | FROM --platform=${BUILDPLATFORM} docker.io/library/golang:1.23.4 AS build 17 | ARG TARGETOS TARGETARCH 18 | 19 | COPY . /src/hetzner-cloud-controller-manager 20 | WORKDIR /src/hetzner-cloud-controller-manager 21 | RUN --mount=type=cache,target=/root/.cache --mount=type=cache,target=/go/pkg \ 22 | GOOS=${TARGETOS} GOARCH=${TARGETARCH} CGO_ENABLED=0 \ 23 | go build -mod=readonly -ldflags "${LDFLAGS} -extldflags '-static'" \ 24 | -o manager main.go 25 | 26 | FROM --platform=${BUILDPLATFORM} gcr.io/distroless/static:nonroot 27 | WORKDIR / 28 | COPY --from=build /src/hetzner-cloud-controller-manager/manager /bin/hetzner-cloud-controller-manager 29 | # Use uid of nonroot user (65532) because kubernetes expects numeric user when applying pod security policies 30 | USER 65532 31 | ENTRYPOINT ["/bin/hetzner-cloud-controller-manager"] 32 | -------------------------------------------------------------------------------- /internal/annotation/load_balancer.go: -------------------------------------------------------------------------------- 1 | package annotation 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/hetznercloud/hcloud-go/v2/hcloud" 7 | "github.com/syself/hetzner-cloud-controller-manager/internal/metrics" 8 | corev1 "k8s.io/api/core/v1" 9 | ) 10 | 11 | const ( 12 | // LBID is the ID assigned to the Hetzner Cloud Load Balancer by the 13 | // backend. Read-only. 14 | LBID Name = "load-balancer.hetzner.cloud/id" 15 | 16 | // LBPublicIPv4 is the public IPv4 address assigned to the Load Balancer by 17 | // the backend. Read-only. 18 | LBPublicIPv4 Name = "load-balancer.hetzner.cloud/ipv4" 19 | 20 | // LBPublicIPv4RDNS is the reverse DNS record assigned to the IPv4 address of 21 | // the Load Balancer. 22 | LBPublicIPv4RDNS Name = "load-balancer.hetzner.cloud/ipv4-rdns" 23 | 24 | // LBPublicIPv6 is the public IPv6 address assigned to the Load Balancer by 25 | // the backend. Read-only. 26 | LBPublicIPv6 Name = "load-balancer.hetzner.cloud/ipv6" 27 | 28 | // LBPublicIPv6RDNS is the reverse DNS record assigned to the IPv6 address of 29 | // the Load Balancer. 30 | LBPublicIPv6RDNS Name = "load-balancer.hetzner.cloud/ipv6-rdns" 31 | 32 | // LBIPv6Disabled disables the use of IPv6 for the Load Balancer. 33 | // 34 | // Set this annotation if you use external-dns. 35 | // 36 | // Default: false. 37 | LBIPv6Disabled Name = "load-balancer.hetzner.cloud/ipv6-disabled" 38 | 39 | // LBName is the name of the Load Balancer. The name will be visible in 40 | // the Hetzner Cloud API console. 41 | LBName Name = "load-balancer.hetzner.cloud/name" 42 | 43 | // LBDisablePublicNetwork disables the public network of the Hetzner Cloud 44 | // Load Balancer. It will still have a public network assigned, but all 45 | // traffic is routed over the private network. 46 | LBDisablePublicNetwork Name = "load-balancer.hetzner.cloud/disable-public-network" 47 | 48 | // LBDisablePrivateIngress disables the use of the private network for 49 | // ingress. 50 | LBDisablePrivateIngress Name = "load-balancer.hetzner.cloud/disable-private-ingress" 51 | 52 | // LBUsePrivateIP configures the Load Balancer to use the private IP for 53 | // Load Balancer server targets. 54 | LBUsePrivateIP Name = "load-balancer.hetzner.cloud/use-private-ip" 55 | 56 | // LBHostname specifies the hostname of the Load Balancer. This will be 57 | // used as ingress address instead of the Load Balancer IP addresses if 58 | // specified. 59 | LBHostname Name = "load-balancer.hetzner.cloud/hostname" 60 | 61 | // LBSvcProtocol specifies the protocol of the service. Default: tcp, Possible 62 | // values: tcp, http, https 63 | LBSvcProtocol Name = "load-balancer.hetzner.cloud/protocol" 64 | 65 | // LBAlgorithmType specifies the algorithm type of the Load Balancer. 66 | // 67 | // Possible values: round_robin, least_connections 68 | // 69 | // Default: round_robin. 70 | LBAlgorithmType Name = "load-balancer.hetzner.cloud/algorithm-type" 71 | 72 | // LBType specifies the type of the Load Balancer. 73 | // 74 | // Default: lb11. 75 | LBType Name = "load-balancer.hetzner.cloud/type" 76 | 77 | // LBLocation specifies the location where the Load Balancer will be 78 | // created in. 79 | // 80 | // Changing the location to a different value after the load balancer was 81 | // created has no effect. In order to move a load balancer to a different 82 | // location it is necessary to delete and re-create it. Note, that this 83 | // will lead to the load balancer getting new public IPs assigned. 84 | // 85 | // Mutually exclusive with LBNetworkZone. 86 | LBLocation Name = "load-balancer.hetzner.cloud/location" 87 | 88 | // LBNetworkZone specifies the network zone where the Load Balancer will be 89 | // created in. 90 | // 91 | // Changing the network zone to a different value after the load balancer 92 | // was created has no effect. In order to move a load balancer to a 93 | // different network zone it is necessary to delete and re-create it. Note, 94 | // that this will lead to the load balancer getting new public IPs 95 | // assigned. 96 | // 97 | // Mutually exclusive with LBLocation. 98 | LBNetworkZone Name = "load-balancer.hetzner.cloud/network-zone" 99 | 100 | // LBNodeSelector can be set to restrict which Nodes are added as targets to the 101 | // Load Balancer. It accepts a Kubernetes label selector string, using either the 102 | // set-based or equality-based formats. 103 | // 104 | // If the selector can not be parsed, the targets in the Load Balancer are not 105 | // updated and an Event is created with the error message. 106 | // 107 | // Format: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors 108 | LBNodeSelector Name = "load-balancer.hetzner.cloud/node-selector" 109 | 110 | // LBSvcProxyProtocol specifies if the Load Balancer services should 111 | // use the proxy protocol. 112 | // 113 | // Default: false. 114 | LBSvcProxyProtocol Name = "load-balancer.hetzner.cloud/uses-proxyprotocol" 115 | 116 | // LBSvcHTTPCookieName specifies the cookie name when using HTTP or HTTPS 117 | // as protocol. 118 | LBSvcHTTPCookieName Name = "load-balancer.hetzner.cloud/http-cookie-name" 119 | 120 | // LBSvcHTTPCookieLifetime specifies the lifetime of the HTTP cookie. 121 | LBSvcHTTPCookieLifetime Name = "load-balancer.hetzner.cloud/http-cookie-lifetime" 122 | 123 | // LBSvcHTTPCertificateType defines the type of certificate the Load 124 | // Balancer should use. 125 | // 126 | // Possible values are "uploaded" and "managed". 127 | // 128 | // If not set LBSvcHTTPCertificateType defaults to "uploaded". 129 | // LBSvcHTTPManagedCertificateDomains is ignored in this case. 130 | // 131 | // HTTPS only. 132 | LBSvcHTTPCertificateType Name = "load-balancer.hetzner.cloud/certificate-type" 133 | 134 | // LBSvcHTTPCertificates a comma separated list of IDs or Names of 135 | // Certificates assigned to the service. 136 | // 137 | // HTTPS only. 138 | LBSvcHTTPCertificates Name = "load-balancer.hetzner.cloud/http-certificates" 139 | 140 | // LBSvcHTTPManagedCertificateName contains the names of the managed 141 | // certificate to create by the Cloud Controller manager. Ignored if 142 | // LBSvcHTTPCertificateType is missing or set to "uploaded". Optional. 143 | LBSvcHTTPManagedCertificateName Name = "load-balancer.hetzner.cloud/http-managed-certificate-name" 144 | 145 | // LBSvcHTTPManagedCertificateUseACMEStaging tells the cloud controller manager to create 146 | // the certificate using Let's Encrypt staging. 147 | // 148 | // This annotation is exclusively for Hetzner internal testing purposes. 149 | // Users should not use this annotation. There is no guarantee that it 150 | // remains or continues to function as it currently functions. 151 | LBSvcHTTPManagedCertificateUseACMEStaging Name = "load-balancer.hetzner.cloud/http-managed-certificate-acme-staging" 152 | 153 | // LBSvcHTTPManagedCertificateDomains contains a coma separated list of the 154 | // domain names of the managed certificate. 155 | // 156 | // All domains are used to create a single managed certificate. 157 | LBSvcHTTPManagedCertificateDomains Name = "load-balancer.hetzner.cloud/http-managed-certificate-domains" 158 | 159 | // LBSvcRedirectHTTP create a redirect from HTTP to HTTPS. HTTPS only. 160 | LBSvcRedirectHTTP Name = "load-balancer.hetzner.cloud/http-redirect-http" 161 | 162 | // LBSvcHTTPStickySessions enables the sticky sessions feature of Hetzner 163 | // Cloud HTTP Load Balancers. 164 | // 165 | // Default: false. 166 | LBSvcHTTPStickySessions Name = "load-balancer.hetzner.cloud/http-sticky-sessions" 167 | 168 | // LBSvcHealthCheckProtocol sets the protocol the health check should be 169 | // performed over. 170 | // 171 | // Possible values: tcp, http, https 172 | // 173 | // Default: tcp. 174 | LBSvcHealthCheckProtocol Name = "load-balancer.hetzner.cloud/health-check-protocol" 175 | 176 | // LBSvcHealthCheckPort specifies the port the health check is be performed 177 | // on. 178 | LBSvcHealthCheckPort Name = "load-balancer.hetzner.cloud/health-check-port" 179 | 180 | // LBSvcHealthCheckInterval specifies the interval in which time we perform 181 | // a health check in seconds. 182 | LBSvcHealthCheckInterval Name = "load-balancer.hetzner.cloud/health-check-interval" 183 | 184 | // LBSvcHealthCheckTimeout specifies the timeout of a single health check. 185 | LBSvcHealthCheckTimeout Name = "load-balancer.hetzner.cloud/health-check-timeout" 186 | 187 | // LBSvcHealthCheckRetries specifies the number of time a health check is 188 | // retried until a target is marked as unhealthy. 189 | LBSvcHealthCheckRetries Name = "load-balancer.hetzner.cloud/health-check-retries" 190 | 191 | // LBSvcHealthCheckHTTPDomain specifies the domain we try to access when 192 | // performing the health check. 193 | LBSvcHealthCheckHTTPDomain Name = "load-balancer.hetzner.cloud/health-check-http-domain" 194 | 195 | // LBSvcHealthCheckHTTPPath specifies the path we try to access when 196 | // performing the health check. 197 | LBSvcHealthCheckHTTPPath Name = "load-balancer.hetzner.cloud/health-check-http-path" 198 | 199 | // LBSvcHealthCheckHTTPValidateCertificate specifies whether the health 200 | // check should validate the SSL certificate that comes from the target 201 | // nodes. 202 | LBSvcHealthCheckHTTPValidateCertificate Name = "load-balancer.hetzner.cloud/health-check-http-validate-certificate" 203 | 204 | // LBSvcHealthCheckHTTPStatusCodes is a comma separated list of HTTP status 205 | // codes which we expect. 206 | LBSvcHealthCheckHTTPStatusCodes Name = "load-balancer.hetzner.cloud/http-status-codes" 207 | ) 208 | 209 | // LBToService sets the relevant annotations on svc to their respective values 210 | // from lb. 211 | func LBToService(svc *corev1.Service, lb *hcloud.LoadBalancer) error { 212 | const op = "annotation/LBToService" 213 | metrics.OperationCalled.WithLabelValues(op).Inc() 214 | 215 | sa := &serviceAnnotator{Svc: svc} 216 | 217 | sa.Annotate(LBID, lb.ID) 218 | sa.Annotate(LBName, lb.Name) 219 | sa.Annotate(LBType, lb.LoadBalancerType.Name) 220 | sa.Annotate(LBAlgorithmType, lb.Algorithm.Type) 221 | sa.Annotate(LBLocation, lb.Location.Name) 222 | sa.Annotate(LBNetworkZone, lb.Location.NetworkZone) 223 | sa.Annotate(LBPublicIPv4, lb.PublicNet.IPv4.IP) 224 | sa.Annotate(LBPublicIPv6, lb.PublicNet.IPv6.IP) 225 | 226 | for _, hclbService := range lb.Services { 227 | var found bool 228 | 229 | // Find the HC Load Balancer service that matches our K8S service by 230 | // comparing the port numbers. 231 | for _, p := range svc.Spec.Ports { 232 | if hclbService.ListenPort == int(p.Port) { 233 | found = true 234 | break 235 | } 236 | } 237 | 238 | // This hclbService does not match our K8S service. Continue with the 239 | // next one. 240 | if !found { 241 | continue 242 | } 243 | 244 | // Once we found a matching service we copy its annotations. 245 | sa.Annotate(LBSvcProtocol, hclbService.Protocol) 246 | sa.Annotate(LBSvcProxyProtocol, hclbService.Proxyprotocol) 247 | 248 | if isHTTP(hclbService) || isHTTPS(hclbService) { 249 | sa.Annotate(LBSvcHTTPCookieName, hclbService.HTTP.CookieName) 250 | sa.Annotate(LBSvcHTTPCookieLifetime, hclbService.HTTP.CookieLifetime) 251 | } 252 | 253 | if isHTTPS(hclbService) { 254 | sa.Annotate(LBSvcRedirectHTTP, hclbService.HTTP.RedirectHTTP) 255 | sa.Annotate(LBSvcHTTPCertificates, hclbService.HTTP.Certificates) 256 | } 257 | 258 | sa.Annotate(LBSvcHealthCheckProtocol, hclbService.HealthCheck.Protocol) 259 | sa.Annotate(LBSvcHealthCheckPort, hclbService.HealthCheck.Port) 260 | sa.Annotate(LBSvcHealthCheckInterval, hclbService.HealthCheck.Interval) 261 | sa.Annotate(LBSvcHealthCheckTimeout, hclbService.HealthCheck.Timeout) 262 | sa.Annotate(LBSvcHealthCheckRetries, hclbService.HealthCheck.Retries) 263 | 264 | if isHTTPHealthCheck(hclbService) || isHTTPSHealthCheck(hclbService) { 265 | sa.Annotate(LBSvcHealthCheckHTTPDomain, hclbService.HealthCheck.HTTP.Domain) 266 | sa.Annotate(LBSvcHealthCheckHTTPPath, hclbService.HealthCheck.HTTP.Path) 267 | sa.Annotate(LBSvcHealthCheckHTTPStatusCodes, hclbService.HealthCheck.HTTP.StatusCodes) 268 | } 269 | if isHTTPSHealthCheck(hclbService) { 270 | sa.Annotate(LBSvcHealthCheckHTTPValidateCertificate, hclbService.HealthCheck.HTTP.TLS) 271 | } 272 | 273 | // At most one service matches and we've already found it. There 274 | // is no need to bother with the remaining services. 275 | break 276 | } 277 | 278 | if sa.Err != nil { 279 | return fmt.Errorf("%s: %w", op, sa.Err) 280 | } 281 | return nil 282 | } 283 | 284 | func isHTTP(s hcloud.LoadBalancerService) bool { 285 | return s.Protocol == hcloud.LoadBalancerServiceProtocolHTTP 286 | } 287 | 288 | func isHTTPHealthCheck(s hcloud.LoadBalancerService) bool { 289 | return s.HealthCheck.HTTP != nil && s.HealthCheck.Protocol == hcloud.LoadBalancerServiceProtocolHTTP 290 | } 291 | 292 | func isHTTPS(s hcloud.LoadBalancerService) bool { 293 | return s.Protocol == hcloud.LoadBalancerServiceProtocolHTTPS 294 | } 295 | 296 | func isHTTPSHealthCheck(s hcloud.LoadBalancerService) bool { 297 | return s.HealthCheck.HTTP != nil && s.HealthCheck.Protocol == hcloud.LoadBalancerServiceProtocolHTTPS 298 | } 299 | -------------------------------------------------------------------------------- /internal/annotation/load_balancer_test.go: -------------------------------------------------------------------------------- 1 | package annotation_test 2 | 3 | import ( 4 | "net" 5 | "testing" 6 | "time" 7 | 8 | "github.com/hetznercloud/hcloud-go/v2/hcloud" 9 | "github.com/stretchr/testify/assert" 10 | "github.com/syself/hetzner-cloud-controller-manager/internal/annotation" 11 | corev1 "k8s.io/api/core/v1" 12 | ) 13 | 14 | func TestLBToService_AddAnnotations(t *testing.T) { 15 | tests := []struct { 16 | name string 17 | svc corev1.Service 18 | lb hcloud.LoadBalancer 19 | expected map[annotation.Name]interface{} 20 | }{ 21 | { 22 | name: "tcp load balancer", 23 | svc: corev1.Service{ 24 | Spec: corev1.ServiceSpec{ 25 | Ports: []corev1.ServicePort{{Port: 1234}}, 26 | }, 27 | }, 28 | lb: hcloud.LoadBalancer{ 29 | ID: 4711, 30 | Name: "common annotations lb", 31 | LoadBalancerType: &hcloud.LoadBalancerType{Name: "lb11"}, 32 | Algorithm: hcloud.LoadBalancerAlgorithm{Type: hcloud.LoadBalancerAlgorithmTypeRoundRobin}, 33 | Location: &hcloud.Location{ 34 | Name: "fsn1", 35 | NetworkZone: hcloud.NetworkZoneEUCentral, 36 | }, 37 | PublicNet: hcloud.LoadBalancerPublicNet{ 38 | IPv4: hcloud.LoadBalancerPublicNetIPv4{ 39 | IP: net.ParseIP("1.2.3.4"), 40 | }, 41 | IPv6: hcloud.LoadBalancerPublicNetIPv6{ 42 | IP: net.ParseIP("b196:ead5:f1e5:8c66:864c:6716:5450:891c"), 43 | }, 44 | }, 45 | Services: []hcloud.LoadBalancerService{ 46 | { 47 | ListenPort: 1234, 48 | Protocol: hcloud.LoadBalancerServiceProtocolTCP, 49 | Proxyprotocol: true, 50 | HealthCheck: hcloud.LoadBalancerServiceHealthCheck{ 51 | Protocol: hcloud.LoadBalancerServiceProtocolTCP, 52 | Port: 2525, 53 | Interval: time.Hour, 54 | Timeout: 5 * time.Minute, 55 | Retries: 3, 56 | }, 57 | }, 58 | }, 59 | }, 60 | expected: map[annotation.Name]interface{}{ 61 | annotation.LBID: 4711, 62 | annotation.LBName: "common annotations lb", 63 | annotation.LBType: "lb11", 64 | annotation.LBAlgorithmType: hcloud.LoadBalancerAlgorithmTypeRoundRobin, 65 | annotation.LBLocation: "fsn1", 66 | annotation.LBNetworkZone: hcloud.NetworkZoneEUCentral, 67 | annotation.LBPublicIPv4: net.ParseIP("1.2.3.4"), 68 | annotation.LBPublicIPv6: net.ParseIP("b196:ead5:f1e5:8c66:864c:6716:5450:891c"), 69 | 70 | annotation.LBSvcProtocol: hcloud.LoadBalancerServiceProtocolTCP, 71 | annotation.LBSvcProxyProtocol: true, 72 | annotation.LBSvcHealthCheckProtocol: hcloud.LoadBalancerServiceProtocolTCP, 73 | annotation.LBSvcHealthCheckPort: 2525, 74 | annotation.LBSvcHealthCheckInterval: time.Hour, 75 | annotation.LBSvcHealthCheckTimeout: 5 * time.Minute, 76 | annotation.LBSvcHealthCheckRetries: 3, 77 | }, 78 | }, 79 | { 80 | name: "http load balancer", 81 | svc: corev1.Service{ 82 | Spec: corev1.ServiceSpec{ 83 | Ports: []corev1.ServicePort{{Port: 1235}}, 84 | }, 85 | }, 86 | lb: hcloud.LoadBalancer{ 87 | ID: 4712, 88 | Name: "https load balancer", 89 | LoadBalancerType: &hcloud.LoadBalancerType{Name: "lb11"}, 90 | Algorithm: hcloud.LoadBalancerAlgorithm{Type: hcloud.LoadBalancerAlgorithmTypeRoundRobin}, 91 | Location: &hcloud.Location{ 92 | Name: "fsn1", 93 | NetworkZone: hcloud.NetworkZoneEUCentral, 94 | }, 95 | PublicNet: hcloud.LoadBalancerPublicNet{ 96 | IPv4: hcloud.LoadBalancerPublicNetIPv4{ 97 | IP: net.ParseIP("1.2.3.4"), 98 | }, 99 | IPv6: hcloud.LoadBalancerPublicNetIPv6{ 100 | IP: net.ParseIP("b196:ead5:f1e5:8c66:864c:6716:5450:891c"), 101 | }, 102 | }, 103 | Services: []hcloud.LoadBalancerService{ 104 | { 105 | ListenPort: 1235, 106 | Protocol: hcloud.LoadBalancerServiceProtocolHTTPS, 107 | Proxyprotocol: true, 108 | HTTP: hcloud.LoadBalancerServiceHTTP{ 109 | Certificates: []*hcloud.Certificate{{ID: 3}, {ID: 5}}, 110 | CookieName: "TESTCOOKIE", 111 | CookieLifetime: time.Hour, 112 | RedirectHTTP: true, 113 | }, 114 | HealthCheck: hcloud.LoadBalancerServiceHealthCheck{ 115 | Protocol: hcloud.LoadBalancerServiceProtocolHTTPS, 116 | Port: 2525, 117 | Interval: time.Hour, 118 | Timeout: 5 * time.Minute, 119 | Retries: 3, 120 | HTTP: &hcloud.LoadBalancerServiceHealthCheckHTTP{ 121 | Domain: "example.com", 122 | Path: "/internal/health", 123 | TLS: true, 124 | StatusCodes: []string{"200", "202"}, 125 | }, 126 | }, 127 | }, 128 | }, 129 | }, 130 | expected: map[annotation.Name]interface{}{ 131 | annotation.LBID: 4712, 132 | annotation.LBName: "https load balancer", 133 | annotation.LBType: "lb11", 134 | annotation.LBAlgorithmType: hcloud.LoadBalancerAlgorithmTypeRoundRobin, 135 | annotation.LBLocation: "fsn1", 136 | annotation.LBNetworkZone: hcloud.NetworkZoneEUCentral, 137 | annotation.LBPublicIPv4: net.ParseIP("1.2.3.4"), 138 | annotation.LBPublicIPv6: net.ParseIP("b196:ead5:f1e5:8c66:864c:6716:5450:891c"), 139 | 140 | annotation.LBSvcProtocol: hcloud.LoadBalancerServiceProtocolHTTPS, 141 | annotation.LBSvcProxyProtocol: true, 142 | annotation.LBSvcHTTPCookieName: "TESTCOOKIE", 143 | annotation.LBSvcHTTPCookieLifetime: time.Hour, 144 | annotation.LBSvcRedirectHTTP: true, 145 | annotation.LBSvcHTTPCertificates: []*hcloud.Certificate{{ID: 3}, {ID: 5}}, 146 | 147 | annotation.LBSvcHealthCheckProtocol: hcloud.LoadBalancerServiceProtocolHTTPS, 148 | annotation.LBSvcHealthCheckPort: 2525, 149 | annotation.LBSvcHealthCheckInterval: time.Hour, 150 | annotation.LBSvcHealthCheckTimeout: 5 * time.Minute, 151 | annotation.LBSvcHealthCheckRetries: 3, 152 | annotation.LBSvcHealthCheckHTTPDomain: "example.com", 153 | annotation.LBSvcHealthCheckHTTPPath: "/internal/health", 154 | annotation.LBSvcHealthCheckHTTPValidateCertificate: true, 155 | annotation.LBSvcHealthCheckHTTPStatusCodes: []string{"200", "202"}, 156 | }, 157 | }, 158 | } 159 | 160 | for _, tt := range tests { 161 | tt := tt 162 | t.Run(tt.name, func(t *testing.T) { 163 | err := annotation.LBToService(&tt.svc, &tt.lb) 164 | assert.NoError(t, err) 165 | annotation.AssertServiceAnnotated(t, &tt.svc, tt.expected) 166 | }) 167 | } 168 | } 169 | -------------------------------------------------------------------------------- /internal/annotation/name.go: -------------------------------------------------------------------------------- 1 | package annotation 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "net" 7 | "strconv" 8 | "strings" 9 | "time" 10 | 11 | "github.com/hetznercloud/hcloud-go/v2/hcloud" 12 | "github.com/syself/hetzner-cloud-controller-manager/internal/metrics" 13 | corev1 "k8s.io/api/core/v1" 14 | ) 15 | 16 | // ErrNotSet signals that an annotation was not set. 17 | var ErrNotSet = errors.New("not set") 18 | 19 | // Name defines the name of a K8S annotation. 20 | type Name string 21 | 22 | // AnnotateService adds the value v as an annotation with s.Name to svc. 23 | // 24 | // AnnotateService returns an error if converting v to a string fails. 25 | func (s Name) AnnotateService(svc *corev1.Service, v interface{}) error { 26 | const op = "annotation/Name.AnnotateService" 27 | metrics.OperationCalled.WithLabelValues(op).Inc() 28 | 29 | if svc.ObjectMeta.Annotations == nil { 30 | svc.ObjectMeta.Annotations = make(map[string]string) 31 | } 32 | k := string(s) 33 | switch vt := v.(type) { 34 | case bool: 35 | svc.ObjectMeta.Annotations[k] = strconv.FormatBool(vt) 36 | case int: 37 | svc.ObjectMeta.Annotations[k] = strconv.Itoa(vt) 38 | case int64: 39 | svc.ObjectMeta.Annotations[k] = strconv.FormatInt(vt, 10) 40 | case string: 41 | svc.ObjectMeta.Annotations[k] = vt 42 | case []string: 43 | svc.ObjectMeta.Annotations[k] = strings.Join(vt, ",") 44 | case hcloud.CertificateType: 45 | svc.ObjectMeta.Annotations[k] = string(vt) 46 | case []*hcloud.Certificate: 47 | idsOrNames := make([]string, len(vt)) 48 | for i, c := range vt { 49 | if c.ID == 0 && c.Name != "" { 50 | idsOrNames[i] = c.Name 51 | continue 52 | } 53 | idsOrNames[i] = strconv.FormatInt(c.ID, 10) 54 | } 55 | svc.ObjectMeta.Annotations[k] = strings.Join(idsOrNames, ",") 56 | case hcloud.NetworkZone: 57 | svc.ObjectMeta.Annotations[k] = string(vt) 58 | case hcloud.LoadBalancerAlgorithmType: 59 | svc.ObjectMeta.Annotations[k] = string(vt) 60 | case hcloud.LoadBalancerServiceProtocol: 61 | svc.ObjectMeta.Annotations[k] = string(vt) 62 | case fmt.Stringer: 63 | svc.ObjectMeta.Annotations[k] = vt.String() 64 | default: 65 | return fmt.Errorf("%s: %v: unsupported type: %T", op, s, v) 66 | } 67 | return nil 68 | } 69 | 70 | // StringFromService retrieves the value belonging to the annotation from svc. 71 | // 72 | // If svc has no value for the annotation the second return value is false. 73 | func (s Name) StringFromService(svc *corev1.Service) (string, bool) { 74 | if svc.Annotations == nil { 75 | return "", false 76 | } 77 | v, ok := svc.Annotations[string(s)] 78 | return v, ok 79 | } 80 | 81 | // StringsFromService retrieves the []string value belonging to the annotation 82 | // from svc. 83 | // 84 | // StringsFromService returns ErrNotSet annotation was not set. 85 | func (s Name) StringsFromService(svc *corev1.Service) ([]string, error) { 86 | const op = "annotation/Name.StringsFromService" 87 | metrics.OperationCalled.WithLabelValues(op).Inc() 88 | 89 | var ss []string 90 | 91 | err := s.applyToValue(op, svc, func(v string) error { 92 | ss = strings.Split(v, ",") 93 | return nil 94 | }) 95 | 96 | return ss, err 97 | } 98 | 99 | // BoolFromService retrieves the boolean value belonging to the annotation from 100 | // svc. 101 | // 102 | // BoolFromService returns an error if the value could not be converted to a 103 | // boolean, or the annotation was not set. In the case of a missing value, the 104 | // error wraps ErrNotSet. 105 | func (s Name) BoolFromService(svc *corev1.Service) (bool, error) { 106 | const op = "annotation/Name.BoolFromService" 107 | metrics.OperationCalled.WithLabelValues(op).Inc() 108 | 109 | v, ok := s.StringFromService(svc) 110 | if !ok { 111 | return false, fmt.Errorf("%s: %v: %w", op, s, ErrNotSet) 112 | } 113 | b, err := strconv.ParseBool(v) 114 | if err != nil { 115 | return false, fmt.Errorf("%s: %v: %w", op, s, err) 116 | } 117 | return b, nil 118 | } 119 | 120 | // IntFromService retrieves the int value belonging to the annotation from svc. 121 | // 122 | // IntFromService returns an error if the value could not be converted to an 123 | // int, or the annotation was not set. In the case of a missing value, the 124 | // error wraps ErrNotSet. 125 | func (s Name) IntFromService(svc *corev1.Service) (int, error) { 126 | const op = "annotation/Name.IntFromService" 127 | metrics.OperationCalled.WithLabelValues(op).Inc() 128 | 129 | v, ok := s.StringFromService(svc) 130 | if !ok { 131 | return 0, fmt.Errorf("%s: %v: %w", op, s, ErrNotSet) 132 | } 133 | i, err := strconv.Atoi(v) 134 | if err != nil { 135 | return 0, fmt.Errorf("%s: %v: %w", op, s, err) 136 | } 137 | return i, nil 138 | } 139 | 140 | // IntsFromService retrieves the []int value belonging to the annotation from 141 | // svc. 142 | // 143 | // IntsFromService returns an error if the value could not be converted to a 144 | // []int, or the annotation was not set. In the case of a missing value, the 145 | // error wraps ErrNotSet. 146 | func (s Name) IntsFromService(svc *corev1.Service) ([]int, error) { 147 | const op = "annotation/Name.IntsFromService" 148 | metrics.OperationCalled.WithLabelValues(op).Inc() 149 | 150 | var is []int 151 | 152 | err := s.applyToValue(op, svc, func(v string) error { 153 | ss := strings.Split(v, ",") 154 | is = make([]int, len(ss)) 155 | 156 | for i, s := range ss { 157 | iv, err := strconv.Atoi(s) 158 | if err != nil { 159 | return err 160 | } 161 | is[i] = iv 162 | } 163 | return nil 164 | }) 165 | 166 | return is, err 167 | } 168 | 169 | // IPFromService retrieves the net.IP value belonging to the annotation from 170 | // svc. 171 | // 172 | // IPFromService returns an error if the value could not be converted to a 173 | // net.IP, or the annotation was not set. In the case of a missing value, the 174 | // error wraps ErrNotSet. 175 | func (s Name) IPFromService(svc *corev1.Service) (net.IP, error) { 176 | const op = "annotation/Name.IPFromService" 177 | metrics.OperationCalled.WithLabelValues(op).Inc() 178 | 179 | var ip net.IP 180 | 181 | err := s.applyToValue(op, svc, func(v string) error { 182 | ip = net.ParseIP(v) 183 | if ip == nil { 184 | return fmt.Errorf("invalid ip address: %s", v) 185 | } 186 | return nil 187 | }) 188 | 189 | return ip, err 190 | } 191 | 192 | // DurationFromService retrieves the time.Duration value belonging to the 193 | // annotation from svc. 194 | // 195 | // DurationFromService returns an error if the value could not be converted to 196 | // a time.Duration, or the annotation was not set. In the case of a missing 197 | // value, the error wraps ErrNotSet. 198 | func (s Name) DurationFromService(svc *corev1.Service) (time.Duration, error) { 199 | const op = "annotation/Name.DurationFromService" 200 | metrics.OperationCalled.WithLabelValues(op).Inc() 201 | 202 | var d time.Duration 203 | 204 | err := s.applyToValue(op, svc, func(v string) error { 205 | var err error 206 | 207 | d, err = time.ParseDuration(v) 208 | return err 209 | }) 210 | 211 | return d, err 212 | } 213 | 214 | // LBSvcProtocolFromService retrieves the hcloud.LoadBalancerServiceProtocol 215 | // value belonging to the annotation from svc. 216 | // 217 | // LBSvcProtocolFromService returns an error if the value could not be 218 | // converted to a hcloud.LoadBalancerServiceProtocol, or the annotation was not 219 | // set. In the case of a missing value, the error wraps ErrNotSet. 220 | func (s Name) LBSvcProtocolFromService(svc *corev1.Service) (hcloud.LoadBalancerServiceProtocol, error) { 221 | const op = "annotation/Name.LBSvcProtocolFromService" 222 | metrics.OperationCalled.WithLabelValues(op).Inc() 223 | 224 | var p hcloud.LoadBalancerServiceProtocol 225 | 226 | err := s.applyToValue(op, svc, func(v string) error { 227 | var err error 228 | 229 | p, err = validateServiceProtocol(v) 230 | return err 231 | }) 232 | 233 | return p, err 234 | } 235 | 236 | // LBAlgorithmTypeFromService retrieves the hcloud.LoadBalancerAlgorithmType 237 | // value belonging to the annotation from svc. 238 | // 239 | // LBAlgorithmTypeFromService returns an error if the value could not be 240 | // converted to a hcloud.LoadBalancerAlgorithmType, or the annotation was not 241 | // set. In the case of a missing value, the error wraps ErrNotSet. 242 | func (s Name) LBAlgorithmTypeFromService(svc *corev1.Service) (hcloud.LoadBalancerAlgorithmType, error) { 243 | const op = "annotation/Name.LBAlgorithmTypeFromService" 244 | metrics.OperationCalled.WithLabelValues(op).Inc() 245 | 246 | var alg hcloud.LoadBalancerAlgorithmType 247 | 248 | err := s.applyToValue(op, svc, func(v string) error { 249 | var err error 250 | 251 | alg, err = validateAlgorithmType(v) 252 | return err 253 | }) 254 | 255 | return alg, err 256 | } 257 | 258 | // NetworkZoneFromService retrieves the hcloud.NetworkZone value belonging to 259 | // the annotation from svc. 260 | // 261 | // NetworkZoneFromService returns ErrNotSet if the annotation was not set. 262 | func (s Name) NetworkZoneFromService(svc *corev1.Service) (hcloud.NetworkZone, error) { 263 | const op = "annotation/Name.NetworkZoneFromService" 264 | metrics.OperationCalled.WithLabelValues(op).Inc() 265 | 266 | var nz hcloud.NetworkZone 267 | 268 | err := s.applyToValue(op, svc, func(v string) error { 269 | nz = hcloud.NetworkZone(v) 270 | return nil 271 | }) 272 | 273 | return nz, err 274 | } 275 | 276 | // CertificatesFromService retrieves the []*hcloud.Certificate value belonging 277 | // to the annotation from svc. 278 | // 279 | // CertificatesFromService returns an error if the value could not be converted 280 | // to a []*hcloud.Certificate, or the annotation was not set. In the case of a 281 | // missing value, the error wraps ErrNotSet. 282 | func (s Name) CertificatesFromService(svc *corev1.Service) ([]*hcloud.Certificate, error) { 283 | const op = "annotation/Name.CertificatesFromService" 284 | metrics.OperationCalled.WithLabelValues(op).Inc() 285 | 286 | var cs []*hcloud.Certificate 287 | 288 | err := s.applyToValue(op, svc, func(v string) error { 289 | ss := strings.Split(v, ",") 290 | cs = make([]*hcloud.Certificate, len(ss)) 291 | 292 | for i, s := range ss { 293 | id, err := strconv.ParseInt(s, 10, 64) 294 | if err != nil { 295 | // If we could not parse the string as an integer we assume it 296 | // is a name not an id. 297 | cs[i] = &hcloud.Certificate{Name: s} 298 | continue 299 | } 300 | cs[i] = &hcloud.Certificate{ID: id} 301 | } 302 | 303 | return nil 304 | }) 305 | 306 | return cs, err 307 | } 308 | 309 | // CertificateTypeFromService retrieves the hcloud.CertificateType value 310 | // belonging to the annotation from svc. 311 | // 312 | // CertificateTypeFromService returns an error if the value could not be 313 | // converted to a hcloud.CertificateType. In the case of a missing value, the 314 | // error wraps ErrNotSet. 315 | func (s Name) CertificateTypeFromService(svc *corev1.Service) (hcloud.CertificateType, error) { 316 | const op = "annotation/Name.CertificateTypeFromService" 317 | metrics.OperationCalled.WithLabelValues(op).Inc() 318 | 319 | var ct hcloud.CertificateType 320 | 321 | err := s.applyToValue(op, svc, func(v string) error { 322 | switch strings.ToLower(v) { 323 | case string(hcloud.CertificateTypeUploaded): 324 | ct = hcloud.CertificateTypeUploaded 325 | case string(hcloud.CertificateTypeManaged): 326 | ct = hcloud.CertificateTypeManaged 327 | default: 328 | return fmt.Errorf("%s: unsupported certificate type: %s", op, v) 329 | } 330 | return nil 331 | }) 332 | 333 | return ct, err 334 | } 335 | 336 | func (s Name) applyToValue(op string, svc *corev1.Service, f func(string) error) error { 337 | v, ok := s.StringFromService(svc) 338 | if !ok { 339 | return fmt.Errorf("%s: %v: %w", op, s, ErrNotSet) 340 | } 341 | if err := f(v); err != nil { 342 | return fmt.Errorf("%s: %w", op, err) 343 | } 344 | return nil 345 | } 346 | 347 | func validateAlgorithmType(algorithmType string) (hcloud.LoadBalancerAlgorithmType, error) { 348 | const op = "annotation/validateAlgorithmType" 349 | metrics.OperationCalled.WithLabelValues(op).Inc() 350 | 351 | algorithmType = strings.ToLower(algorithmType) // Lowercase because all our protocols are lowercase 352 | hcloudAlgorithmType := hcloud.LoadBalancerAlgorithmType(algorithmType) 353 | 354 | switch hcloudAlgorithmType { 355 | case hcloud.LoadBalancerAlgorithmTypeLeastConnections: 356 | case hcloud.LoadBalancerAlgorithmTypeRoundRobin: 357 | default: 358 | return "", fmt.Errorf("%s: invalid: %s", op, algorithmType) 359 | } 360 | 361 | return hcloudAlgorithmType, nil 362 | } 363 | 364 | func validateServiceProtocol(protocol string) (hcloud.LoadBalancerServiceProtocol, error) { 365 | const op = "annotation/validateServiceProtocol" 366 | metrics.OperationCalled.WithLabelValues(op).Inc() 367 | 368 | protocol = strings.ToLower(protocol) // Lowercase because all our protocols are lowercase 369 | hcloudProtocol := hcloud.LoadBalancerServiceProtocol(protocol) 370 | switch hcloudProtocol { 371 | case hcloud.LoadBalancerServiceProtocolTCP: 372 | case hcloud.LoadBalancerServiceProtocolHTTPS: 373 | case hcloud.LoadBalancerServiceProtocolHTTP: 374 | // Valid 375 | break 376 | default: 377 | return "", fmt.Errorf("%s: invalid: %s", op, protocol) 378 | } 379 | return hcloudProtocol, nil 380 | } 381 | 382 | type serviceAnnotator struct { 383 | Svc *corev1.Service 384 | Err error 385 | } 386 | 387 | func (sa *serviceAnnotator) Annotate(n Name, v interface{}) { 388 | if sa.Err != nil { 389 | return 390 | } 391 | sa.Err = n.AnnotateService(sa.Svc, v) 392 | } 393 | -------------------------------------------------------------------------------- /internal/annotation/name_test.go: -------------------------------------------------------------------------------- 1 | package annotation_test 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "net" 7 | "strconv" 8 | "testing" 9 | "time" 10 | 11 | "github.com/hetznercloud/hcloud-go/v2/hcloud" 12 | "github.com/stretchr/testify/assert" 13 | "github.com/syself/hetzner-cloud-controller-manager/internal/annotation" 14 | corev1 "k8s.io/api/core/v1" 15 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 16 | ) 17 | 18 | const ann annotation.Name = "some/annotation" 19 | 20 | func TestName_AddToService(t *testing.T) { 21 | tests := []struct { 22 | name string 23 | value interface{} 24 | svc corev1.Service 25 | err error 26 | expected map[string]string 27 | }{ 28 | { 29 | name: "set string", 30 | value: "some value", 31 | expected: map[string]string{string(ann): "some value"}, 32 | }, 33 | { 34 | name: "set stringer", 35 | value: stringer{"some value"}, 36 | expected: map[string]string{string(ann): "some value"}, 37 | }, 38 | 39 | { 40 | name: "set bool", 41 | value: true, 42 | expected: map[string]string{string(ann): "true"}, 43 | }, 44 | { 45 | name: "set int", 46 | value: 10, 47 | expected: map[string]string{string(ann): "10"}, 48 | }, 49 | { 50 | name: "set []string", 51 | value: []string{"a", "b"}, 52 | expected: map[string]string{string(ann): "a,b"}, 53 | }, 54 | { 55 | name: "set hcloud.LoadBalancerServiceProtocol", 56 | value: hcloud.LoadBalancerServiceProtocolTCP, 57 | expected: map[string]string{string(ann): string(hcloud.LoadBalancerServiceProtocolTCP)}, 58 | }, 59 | { 60 | name: "set []*hcloud.Certificate", 61 | value: []*hcloud.Certificate{{ID: 1}, {ID: 2}}, 62 | expected: map[string]string{string(ann): "1,2"}, 63 | }, 64 | { 65 | name: "set []*hcloud.Certificate by name", 66 | value: []*hcloud.Certificate{{Name: "cert-1"}, {Name: "cert-2"}}, 67 | expected: map[string]string{string(ann): "cert-1,cert-2"}, 68 | }, 69 | { 70 | name: "set unsupported value", 71 | value: struct{}{}, 72 | err: fmt.Errorf("annotation/Name.AnnotateService: %v: unsupported type: %T", ann, struct{}{}), 73 | }, 74 | { 75 | name: "does not overwrite unrelated annotations", 76 | value: "some value", 77 | svc: corev1.Service{ 78 | ObjectMeta: metav1.ObjectMeta{ 79 | Annotations: map[string]string{"other/annotation": "other value"}, 80 | }, 81 | }, 82 | expected: map[string]string{ 83 | string(ann): "some value", 84 | "other/annotation": "other value", 85 | }, 86 | }, 87 | } 88 | 89 | for _, tt := range tests { 90 | tt := tt 91 | t.Run(tt.name, func(t *testing.T) { 92 | err := ann.AnnotateService(&tt.svc, tt.value) 93 | if tt.err != nil { 94 | assert.EqualError(t, err, tt.err.Error()) 95 | return 96 | } 97 | assert.NoError(t, err) 98 | assert.Equal(t, tt.expected, tt.svc.ObjectMeta.Annotations) 99 | }) 100 | } 101 | } 102 | 103 | func TestName_StringFromService(t *testing.T) { 104 | tests := []struct { 105 | name string 106 | svcAnnotations map[annotation.Name]interface{} 107 | ok bool 108 | expected string 109 | }{ 110 | { 111 | name: "value as string", 112 | svcAnnotations: map[annotation.Name]interface{}{ann: "some value"}, 113 | ok: true, 114 | expected: "some value", 115 | }, 116 | { 117 | name: "Service has no annotations", 118 | }, 119 | { 120 | name: "value not set", 121 | }, 122 | } 123 | 124 | for _, tt := range tests { 125 | tt := tt 126 | t.Run(tt.name, func(t *testing.T) { 127 | var svc corev1.Service 128 | 129 | for k, v := range tt.svcAnnotations { 130 | if err := k.AnnotateService(&svc, v); err != nil { 131 | t.Error(err) 132 | } 133 | } 134 | actual, ok := ann.StringFromService(&svc) 135 | assert.Equal(t, tt.ok, ok) 136 | assert.Equal(t, tt.expected, actual) 137 | }) 138 | } 139 | } 140 | 141 | func TestName_StringsFromService(t *testing.T) { 142 | tests := []typedAccessorTest{ 143 | { 144 | name: "value set", 145 | svcAnnotations: map[annotation.Name]interface{}{ 146 | ann: "a,b,c", 147 | }, 148 | expected: []string{"a", "b", "c"}, 149 | }, 150 | { 151 | name: "value missing", 152 | err: annotation.ErrNotSet, 153 | }, 154 | } 155 | 156 | runAllTypedAccessorTests(t, tests, func(svc *corev1.Service) (interface{}, error) { 157 | return ann.StringsFromService(svc) 158 | }) 159 | } 160 | 161 | func TestName_BoolFromService(t *testing.T) { 162 | tests := []typedAccessorTest{ 163 | { 164 | name: "value set to true", 165 | svcAnnotations: map[annotation.Name]interface{}{ann: "true"}, 166 | expected: true, 167 | }, 168 | { 169 | name: "value set to false", 170 | svcAnnotations: map[annotation.Name]interface{}{ann: "false"}, 171 | expected: false, 172 | }, 173 | { 174 | name: "value missing", 175 | expected: false, 176 | err: annotation.ErrNotSet, 177 | }, 178 | { 179 | name: "value invalid", 180 | svcAnnotations: map[annotation.Name]interface{}{ann: "invalid"}, 181 | expected: false, 182 | err: strconv.ErrSyntax, 183 | }, 184 | } 185 | 186 | runAllTypedAccessorTests(t, tests, func(svc *corev1.Service) (interface{}, error) { 187 | return ann.BoolFromService(svc) 188 | }) 189 | } 190 | 191 | func TestName_IntFromService(t *testing.T) { 192 | tests := []typedAccessorTest{ 193 | { 194 | name: "value set to 10", 195 | svcAnnotations: map[annotation.Name]interface{}{ann: 10}, 196 | expected: 10, 197 | }, 198 | { 199 | name: "value missing", 200 | expected: 0, 201 | err: fmt.Errorf("annotation/Name.IntFromService: %s: %w", ann, annotation.ErrNotSet), 202 | }, 203 | { 204 | name: "value invalid", 205 | svcAnnotations: map[annotation.Name]interface{}{ann: "invalid"}, 206 | expected: 0, 207 | err: strconv.ErrSyntax, 208 | }, 209 | } 210 | 211 | runAllTypedAccessorTests(t, tests, func(svc *corev1.Service) (interface{}, error) { 212 | return ann.IntFromService(svc) 213 | }) 214 | } 215 | 216 | func TestName_IntsFromService(t *testing.T) { 217 | tests := []typedAccessorTest{ 218 | { 219 | name: "value set", 220 | svcAnnotations: map[annotation.Name]interface{}{ 221 | ann: "5,8", 222 | }, 223 | expected: []int{5, 8}, 224 | }, 225 | { 226 | name: "value missing", 227 | err: annotation.ErrNotSet, 228 | }, 229 | { 230 | name: "value invalid", 231 | svcAnnotations: map[annotation.Name]interface{}{ann: "invalid"}, 232 | err: strconv.ErrSyntax, 233 | }, 234 | } 235 | 236 | runAllTypedAccessorTests(t, tests, func(svc *corev1.Service) (interface{}, error) { 237 | return ann.IntsFromService(svc) 238 | }) 239 | } 240 | 241 | func TestName_IPFromService(t *testing.T) { 242 | tests := []typedAccessorTest{ 243 | { 244 | name: "value set to valid IPv4", 245 | svcAnnotations: map[annotation.Name]interface{}{ 246 | ann: net.ParseIP("1.2.3.4"), 247 | }, 248 | expected: net.ParseIP("1.2.3.4"), 249 | }, 250 | { 251 | name: "value set to valid IPv6", 252 | svcAnnotations: map[annotation.Name]interface{}{ 253 | ann: net.ParseIP("3c2e:2ef9:a7e9:1a5b:30ba:4912:e3fe:91b2"), 254 | }, 255 | expected: net.ParseIP("3c2e:2ef9:a7e9:1a5b:30ba:4912:e3fe:91b2"), 256 | }, 257 | { 258 | name: "value invalid", 259 | svcAnnotations: map[annotation.Name]interface{}{ 260 | ann: "invalid", 261 | }, 262 | err: errors.New("annotation/Name.IPFromService: invalid ip address: invalid"), 263 | }, 264 | { 265 | name: "value not set", 266 | err: annotation.ErrNotSet, 267 | }, 268 | } 269 | 270 | runAllTypedAccessorTests(t, tests, func(svc *corev1.Service) (interface{}, error) { 271 | return ann.IPFromService(svc) 272 | }) 273 | } 274 | 275 | func TestName_DurationFromService(t *testing.T) { 276 | tests := []typedAccessorTest{ 277 | { 278 | name: "value set", 279 | svcAnnotations: map[annotation.Name]interface{}{ 280 | ann: time.Hour, 281 | }, 282 | expected: time.Hour, 283 | }, 284 | { 285 | name: "value not set", 286 | err: annotation.ErrNotSet, 287 | }, 288 | { 289 | name: "value invalid", 290 | svcAnnotations: map[annotation.Name]interface{}{ 291 | ann: "invalid", 292 | }, 293 | err: errors.New("annotation/Name.DurationFromService: time: invalid duration \"invalid\""), 294 | }, 295 | } 296 | 297 | runAllTypedAccessorTests(t, tests, func(svc *corev1.Service) (interface{}, error) { 298 | return ann.DurationFromService(svc) 299 | }) 300 | } 301 | 302 | func TestName_LBSvcProtocolFromService(t *testing.T) { 303 | tests := []typedAccessorTest{ 304 | { 305 | name: "value set", 306 | svcAnnotations: map[annotation.Name]interface{}{ 307 | ann: hcloud.LoadBalancerServiceProtocolHTTP, 308 | }, 309 | expected: hcloud.LoadBalancerServiceProtocolHTTP, 310 | }, 311 | { 312 | name: "value not set", 313 | err: annotation.ErrNotSet, 314 | }, 315 | { 316 | name: "value invalid", 317 | svcAnnotations: map[annotation.Name]interface{}{ 318 | ann: "invalid", 319 | }, 320 | err: errors.New("annotation/Name.LBSvcProtocolFromService: annotation/validateServiceProtocol: invalid: invalid"), 321 | }, 322 | } 323 | 324 | runAllTypedAccessorTests(t, tests, func(svc *corev1.Service) (interface{}, error) { 325 | return ann.LBSvcProtocolFromService(svc) 326 | }) 327 | } 328 | 329 | func TestName_LBAlgorithmTypeFromService(t *testing.T) { 330 | tests := []typedAccessorTest{ 331 | { 332 | name: "value set", 333 | svcAnnotations: map[annotation.Name]interface{}{ 334 | ann: hcloud.LoadBalancerAlgorithmTypeLeastConnections, 335 | }, 336 | expected: hcloud.LoadBalancerAlgorithmTypeLeastConnections, 337 | }, 338 | { 339 | name: "value not set", 340 | err: annotation.ErrNotSet, 341 | }, 342 | { 343 | name: "value invalid", 344 | svcAnnotations: map[annotation.Name]interface{}{ 345 | ann: "invalid", 346 | }, 347 | err: errors.New("annotation/Name.LBAlgorithmTypeFromService: annotation/validateAlgorithmType: invalid: invalid"), 348 | }, 349 | } 350 | 351 | runAllTypedAccessorTests(t, tests, func(svc *corev1.Service) (interface{}, error) { 352 | return ann.LBAlgorithmTypeFromService(svc) 353 | }) 354 | } 355 | 356 | func TestName_NetworkZoneFromService(t *testing.T) { 357 | tests := []typedAccessorTest{ 358 | { 359 | name: "value set", 360 | svcAnnotations: map[annotation.Name]interface{}{ 361 | ann: hcloud.NetworkZoneEUCentral, 362 | }, 363 | expected: hcloud.NetworkZoneEUCentral, 364 | }, 365 | { 366 | name: "value not set", 367 | err: annotation.ErrNotSet, 368 | }, 369 | } 370 | 371 | runAllTypedAccessorTests(t, tests, func(svc *corev1.Service) (interface{}, error) { 372 | return ann.NetworkZoneFromService(svc) 373 | }) 374 | } 375 | 376 | func TestName_CertificatesFromService(t *testing.T) { 377 | tests := []typedAccessorTest{ 378 | { 379 | name: "ids set", 380 | svcAnnotations: map[annotation.Name]interface{}{ 381 | ann: []*hcloud.Certificate{{ID: 3}, {ID: 5}}, 382 | }, 383 | expected: []*hcloud.Certificate{{ID: 3}, {ID: 5}}, 384 | }, 385 | { 386 | name: "names set", 387 | svcAnnotations: map[annotation.Name]interface{}{ 388 | ann: []*hcloud.Certificate{{Name: "cert-1"}, {Name: "cert-2"}}, 389 | }, 390 | expected: []*hcloud.Certificate{{Name: "cert-1"}, {Name: "cert-2"}}, 391 | }, 392 | { 393 | name: "value not set", 394 | err: annotation.ErrNotSet, 395 | }, 396 | } 397 | 398 | runAllTypedAccessorTests(t, tests, func(svc *corev1.Service) (interface{}, error) { 399 | return ann.CertificatesFromService(svc) 400 | }) 401 | } 402 | 403 | func TestName_CertificateTypeFromService(t *testing.T) { 404 | tests := []typedAccessorTest{ 405 | { 406 | name: "uploaded certificate", 407 | svcAnnotations: map[annotation.Name]interface{}{ 408 | ann: hcloud.CertificateTypeUploaded, 409 | }, 410 | expected: hcloud.CertificateTypeUploaded, 411 | }, 412 | { 413 | name: "managed certificate", 414 | svcAnnotations: map[annotation.Name]interface{}{ 415 | ann: hcloud.CertificateTypeManaged, 416 | }, 417 | expected: hcloud.CertificateTypeManaged, 418 | }, 419 | { 420 | name: "unsupported certificate type", 421 | svcAnnotations: map[annotation.Name]interface{}{ 422 | ann: "unsupported type", 423 | }, 424 | err: fmt.Errorf("annotation/Name.CertificateTypeFromService: annotation/Name.CertificateTypeFromService: unsupported certificate type: unsupported type"), 425 | }, 426 | } 427 | 428 | runAllTypedAccessorTests(t, tests, func(svc *corev1.Service) (interface{}, error) { 429 | return ann.CertificateTypeFromService(svc) 430 | }) 431 | } 432 | 433 | type stringer struct{ Value string } 434 | 435 | func (s stringer) String() string { 436 | return s.Value 437 | } 438 | 439 | type typedAccessorTest struct { 440 | name string 441 | svcAnnotations map[annotation.Name]interface{} 442 | err error 443 | expected interface{} 444 | } 445 | 446 | func (tt *typedAccessorTest) run(t *testing.T, call func(svc *corev1.Service) (interface{}, error)) { 447 | var svc corev1.Service 448 | 449 | t.Helper() 450 | 451 | for k, v := range tt.svcAnnotations { 452 | if err := k.AnnotateService(&svc, v); !assert.NoError(t, err) { 453 | return 454 | } 455 | } 456 | 457 | actual, err := call(&svc) 458 | if tt.err != nil { 459 | if errors.Is(err, tt.err) { 460 | return 461 | } 462 | assert.EqualError(t, err, tt.err.Error()) 463 | return 464 | } 465 | assert.NoError(t, err) 466 | // Don't use assert.Equal to compare nil values, as it requires the nil 467 | // values to be casted to the correct type. 468 | if tt.expected == nil && actual == nil { 469 | return 470 | } 471 | assert.Equal(t, tt.expected, actual) 472 | } 473 | 474 | func runAllTypedAccessorTests( 475 | t *testing.T, tests []typedAccessorTest, call func(svc *corev1.Service) (interface{}, error), 476 | ) { 477 | t.Helper() 478 | 479 | for _, tt := range tests { 480 | tt := tt 481 | t.Run(tt.name, func(t *testing.T) { 482 | tt.run(t, call) 483 | }) 484 | } 485 | } 486 | -------------------------------------------------------------------------------- /internal/annotation/testing.go: -------------------------------------------------------------------------------- 1 | package annotation 2 | 3 | import ( 4 | "net" 5 | "testing" 6 | "time" 7 | 8 | "github.com/hetznercloud/hcloud-go/v2/hcloud" 9 | "github.com/stretchr/testify/assert" 10 | corev1 "k8s.io/api/core/v1" 11 | ) 12 | 13 | // AssertServiceAnnotated asserts that svc has been annotated with all 14 | // annotations in expected. 15 | func AssertServiceAnnotated(t *testing.T, svc *corev1.Service, expected map[Name]interface{}) { 16 | t.Helper() 17 | for ek, ev := range expected { 18 | var ( 19 | actual interface{} 20 | ok bool 21 | err error 22 | ) 23 | actual, ok = ek.StringFromService(svc) 24 | if !ok { 25 | t.Errorf("not annotated with: %v", ek) 26 | continue 27 | } 28 | 29 | switch ev.(type) { 30 | case bool: 31 | actual, err = ek.BoolFromService(svc) 32 | case int: 33 | actual, err = ek.IntFromService(svc) 34 | case []string: 35 | actual, err = ek.StringsFromService(svc) 36 | case net.IP: 37 | actual, err = ek.IPFromService(svc) 38 | case time.Duration: 39 | actual, err = ek.DurationFromService(svc) 40 | case []*hcloud.Certificate: 41 | actual, err = ek.CertificatesFromService(svc) 42 | case hcloud.LoadBalancerAlgorithmType: 43 | actual, err = ek.LBAlgorithmTypeFromService(svc) 44 | case hcloud.LoadBalancerServiceProtocol: 45 | actual, err = ek.LBSvcProtocolFromService(svc) 46 | case hcloud.NetworkZone: 47 | actual, err = ek.NetworkZoneFromService(svc) 48 | } 49 | assert.NoError(t, err) 50 | assert.Equal(t, ev, actual) 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /internal/credentials/hotreload.go: -------------------------------------------------------------------------------- 1 | package credentials 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | "strings" 8 | "sync" 9 | 10 | fsnotify "github.com/fsnotify/fsnotify" 11 | "github.com/hetznercloud/hcloud-go/v2/hcloud" 12 | robotclient "github.com/syself/hetzner-cloud-controller-manager/internal/robot/client" 13 | "k8s.io/klog/v2" 14 | ) 15 | 16 | var ( 17 | // fsnotify creates several events for a single update of a mounted secret. 18 | // To avoid multiple reloads, we store the old values and only reload when 19 | // the values have changed. 20 | oldRobotUser string 21 | oldRobotPassword string 22 | oldHcloudToken string 23 | 24 | // robotReloadCounter gets incremented when the credentials get reloaded. 25 | // Mosty used for testing. 26 | robotReloadCounter uint64 27 | 28 | // hcloudTokenReloadCounter gets incremented when the credentials get reloaded. 29 | // Mosty used for testing. 30 | hcloudTokenReloadCounter uint64 31 | 32 | hcloudMutex sync.Mutex 33 | robotMutex sync.Mutex 34 | ) 35 | 36 | // GetRobotReloadCounter returns the number of times the robot credentials have been reloaded. 37 | // Mostly used for testing. 38 | func GetRobotReloadCounter() uint64 { 39 | robotMutex.Lock() 40 | defer robotMutex.Unlock() 41 | return robotReloadCounter 42 | } 43 | 44 | // GetHcloudReloadCounter returns the number of times the hcloud credentials have been reloaded. 45 | // Mostly used for testing. 46 | func GetHcloudReloadCounter() uint64 { 47 | hcloudMutex.Lock() 48 | defer hcloudMutex.Unlock() 49 | return hcloudTokenReloadCounter 50 | } 51 | 52 | // Watch the mounted secrets. Reload the credentials, when the files get updated. The robotClient can be nil. 53 | func Watch(credentialsDir string, hcloudClient *hcloud.Client, robotClient robotclient.Client) error { 54 | watcher, err := fsnotify.NewWatcher() 55 | if err != nil { 56 | klog.Fatal(err) 57 | } 58 | 59 | go func() { 60 | for { 61 | select { 62 | case event := <-watcher.Events: 63 | if !isValidEvent(event) { 64 | continue 65 | } 66 | 67 | // get last element of path. Example: /etc/hetzner-secret/robot-user -> robot-user 68 | baseName := filepath.Base(event.Name) 69 | 70 | if err := handleEvent(credentialsDir, baseName, hcloudClient, robotClient, event); err != nil { 71 | klog.Errorf("error processing fsnotify event: %s", err.Error()) 72 | } 73 | 74 | case err := <-watcher.Errors: 75 | klog.Infof("error from fsnotify file watcher of %q: %s", credentialsDir, err) 76 | } 77 | } 78 | }() 79 | 80 | err = watcher.Add(credentialsDir) 81 | if err != nil { 82 | return fmt.Errorf("watcher.Add: %w", err) 83 | } 84 | return nil 85 | } 86 | 87 | func handleEvent(credentialsDir, baseName string, hcloudClient *hcloud.Client, robotClient robotclient.Client, event fsnotify.Event) error { 88 | var err error 89 | switch baseName { 90 | case "robot-user", "robot-password": 91 | // This case is executed, when the process is running on a local machine. 92 | return loadRobotCredentials(credentialsDir, robotClient) 93 | 94 | case "hcloud": 95 | // This case is executed, when the process is running on a local machine. 96 | return loadHcloudCredentials(credentialsDir, hcloudClient) 97 | 98 | case "..data": 99 | // This case is executed, when the secrets are mounted in a Kubernetes pod. 100 | // The files (for example hcloud) are symlinks to ..data/. 101 | // For example the file "hcloud" is a symlink to ../data/hcloud 102 | // This means the files/symlinks don't change. When the secrets get changed, then 103 | // a new ..data directory gets created. This is done by Kubernetes to make the 104 | // update of all files atomic. 105 | err = loadHcloudCredentials(credentialsDir, hcloudClient) 106 | if err != nil { 107 | return err 108 | } 109 | if robotClient == nil { 110 | return nil 111 | } 112 | return loadRobotCredentials(credentialsDir, robotClient) 113 | 114 | default: 115 | klog.Infof("Ignoring fsnotify event for file %q: %s", baseName, event.String()) 116 | return nil 117 | } 118 | } 119 | 120 | func isValidEvent(event fsnotify.Event) bool { 121 | baseName := filepath.Base(event.Name) 122 | if strings.HasPrefix(baseName, "..") && baseName != "..data" { 123 | // Skip ..data_tmp and ..YYYY_MM_DD... 124 | return false 125 | } 126 | if event.Has(fsnotify.Write) || event.Has(fsnotify.Create) { 127 | return true 128 | } 129 | return false 130 | } 131 | 132 | func loadRobotCredentials(credentialsDir string, robotClient robotclient.Client) error { 133 | robotMutex.Lock() 134 | defer robotMutex.Unlock() 135 | 136 | username, password, err := readRobotCredentials(credentialsDir) 137 | if err != nil { 138 | return fmt.Errorf("reading robot credentials from secret failed: %w", err) 139 | } 140 | 141 | if username == oldRobotUser && password == oldRobotPassword { 142 | return nil 143 | } 144 | 145 | // Update global variables 146 | oldRobotUser = username 147 | oldRobotPassword = password 148 | robotReloadCounter++ 149 | 150 | err = robotClient.SetCredentials(username, password) 151 | if err != nil { 152 | return fmt.Errorf("SetCredentials: %w", err) 153 | } 154 | 155 | klog.Infof("Hetzner Robot credentials updated to new value: %q %s...", username, password[:3]) 156 | return nil 157 | } 158 | 159 | func GetInitialRobotCredentials(credentialsDir string) (username, password string, err error) { 160 | u, p, err := readRobotCredentials(credentialsDir) 161 | if err != nil { 162 | return "", "", fmt.Errorf("readRobotCredentials: %w", err) 163 | } 164 | 165 | // Update global variables 166 | oldRobotUser = u 167 | oldRobotPassword = p 168 | 169 | return u, p, nil 170 | } 171 | 172 | func readRobotCredentials(credentialsDir string) (username, password string, err error) { 173 | robotUserNameFile := filepath.Join(credentialsDir, "robot-user") 174 | robotPasswordFile := filepath.Join(credentialsDir, "robot-password") 175 | 176 | u, err := os.ReadFile(robotUserNameFile) 177 | if err != nil { 178 | return "", "", fmt.Errorf("reading robot user name from %q failed: %w", robotUserNameFile, err) 179 | } 180 | 181 | p, err := os.ReadFile(robotPasswordFile) 182 | if err != nil { 183 | return "", "", fmt.Errorf("reading robot password from %q failed: %w", robotPasswordFile, err) 184 | } 185 | 186 | return strings.TrimSpace(string(u)), strings.TrimSpace(string(p)), nil 187 | } 188 | 189 | func loadHcloudCredentials(credentialsDir string, hcloudClient *hcloud.Client) error { 190 | hcloudMutex.Lock() 191 | defer hcloudMutex.Unlock() 192 | 193 | token, err := readHcloudCredentials(credentialsDir) 194 | if err != nil { 195 | return err 196 | } 197 | 198 | if len(token) != 64 { 199 | return fmt.Errorf("loadHcloudCredentials: entered token (%s...) is invalid (must be exactly 64 characters long)", 200 | token[:5]) 201 | } 202 | 203 | if token == oldHcloudToken { 204 | return nil 205 | } 206 | 207 | // Update global variables 208 | oldHcloudToken = token 209 | hcloudTokenReloadCounter++ 210 | 211 | // Update credentials of hcloudClient 212 | hcloud.WithToken(token)(hcloudClient) 213 | 214 | klog.Infof("Hetzner Cloud token updated to new value: %s...", token[:5]) 215 | return nil 216 | } 217 | 218 | func GetInitialHcloudCredentialsFromDirectory(credentialsDir string) (string, error) { 219 | token, err := readHcloudCredentials(credentialsDir) 220 | if err != nil { 221 | return "", fmt.Errorf("readHcloudCredentials: %w", err) 222 | } 223 | 224 | // Update global variable 225 | oldHcloudToken = token 226 | 227 | return token, nil 228 | } 229 | 230 | func readHcloudCredentials(credentialsDir string) (string, error) { 231 | hcloudTokenFile := filepath.Join(credentialsDir, "hcloud") 232 | data, err := os.ReadFile(hcloudTokenFile) 233 | if err != nil { 234 | return "", fmt.Errorf("reading hcloud token from %q failed: %w", hcloudTokenFile, err) 235 | } 236 | return strings.TrimSpace(string(data)), nil 237 | } 238 | 239 | // GetDirectory returns the directory where the credentials are stored. 240 | // The credentials are stored in the directory etc/hetzner-secret. 241 | func GetDirectory(rootDir string) string { 242 | return filepath.Join(rootDir, "etc", "hetzner-secret") 243 | } 244 | -------------------------------------------------------------------------------- /internal/hcops/action.go: -------------------------------------------------------------------------------- 1 | package hcops 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/hetznercloud/hcloud-go/v2/hcloud" 7 | ) 8 | 9 | type HCloudActionClient interface { 10 | WatchProgress(ctx context.Context, a *hcloud.Action) (<-chan int, <-chan error) 11 | } 12 | 13 | func WatchAction(ctx context.Context, ac HCloudActionClient, a *hcloud.Action) error { 14 | _, errCh := ac.WatchProgress(ctx, a) 15 | err := <-errCh 16 | return err 17 | } 18 | -------------------------------------------------------------------------------- /internal/hcops/certificates.go: -------------------------------------------------------------------------------- 1 | package hcops 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/hetznercloud/hcloud-go/v2/hcloud" 8 | "github.com/syself/hetzner-cloud-controller-manager/internal/metrics" 9 | ) 10 | 11 | // HCloudCertificateClient defines the hcloud-go function related to 12 | // certificate management. 13 | type HCloudCertificateClient interface { 14 | AllWithOpts(context.Context, hcloud.CertificateListOpts) ([]*hcloud.Certificate, error) 15 | Get(ctx context.Context, idOrName string) (*hcloud.Certificate, *hcloud.Response, error) 16 | CreateCertificate( 17 | ctx context.Context, opts hcloud.CertificateCreateOpts, 18 | ) (hcloud.CertificateCreateResult, *hcloud.Response, error) 19 | } 20 | 21 | // CertificateOps implements all operations regarding Hetzner Cloud Certificates. 22 | type CertificateOps struct { 23 | CertClient HCloudCertificateClient 24 | } 25 | 26 | // GetCertificateByNameOrID obtains a certificate from the Hetzner Cloud 27 | // backend using its ID or Name. 28 | // 29 | // If a certificate could not be found the returned error wraps ErrNotFound. 30 | func (co *CertificateOps) GetCertificateByNameOrID(ctx context.Context, idOrName string) (*hcloud.Certificate, error) { 31 | const op = "hcops/CertificateOps.GetCertificateByNameOrID" 32 | metrics.OperationCalled.WithLabelValues(op).Inc() 33 | 34 | cert, _, err := co.CertClient.Get(ctx, idOrName) 35 | if err != nil { 36 | return nil, fmt.Errorf("%s: %w", op, err) 37 | } 38 | if cert == nil { 39 | return nil, fmt.Errorf("%s: %w", op, ErrNotFound) 40 | } 41 | return cert, nil 42 | } 43 | 44 | // GetCertificateByLabel obtains a single certificate by the passed label. 45 | // 46 | // If the label matches more than one certificate a wrapped ErrNonUniqueResult 47 | // is returned. If no certificate could be found a wrapped ErrNotFound is 48 | // returned. 49 | func (co *CertificateOps) GetCertificateByLabel(ctx context.Context, label string) (*hcloud.Certificate, error) { 50 | const op = "hcops/CertificateOps.GetCertificateByLabel" 51 | metrics.OperationCalled.WithLabelValues(op).Inc() 52 | 53 | opts := hcloud.CertificateListOpts{ListOpts: hcloud.ListOpts{LabelSelector: label}} 54 | certs, err := co.CertClient.AllWithOpts(ctx, opts) 55 | if err != nil { 56 | return nil, fmt.Errorf("%s: %v", op, err) 57 | } 58 | if len(certs) == 0 { 59 | return nil, fmt.Errorf("%s: %w", op, ErrNotFound) 60 | } 61 | if len(certs) > 1 { 62 | return nil, fmt.Errorf("%s: %w", op, ErrNonUniqueResult) 63 | } 64 | return certs[0], nil 65 | } 66 | 67 | // CreateManagedCertificate creates a managed certificate for domains labeled 68 | // with label. 69 | // 70 | // CreateManagedCertificate returns a wrapped ErrAlreadyExists if the 71 | // certificate already exists. 72 | func (co *CertificateOps) CreateManagedCertificate( 73 | ctx context.Context, name string, domains []string, labels map[string]string, 74 | ) error { 75 | const op = "hcops/CertificateOps.CreateManagedCertificate" 76 | metrics.OperationCalled.WithLabelValues(op).Inc() 77 | 78 | opts := hcloud.CertificateCreateOpts{ 79 | Name: name, 80 | Type: hcloud.CertificateTypeManaged, 81 | DomainNames: domains, 82 | Labels: labels, 83 | } 84 | _, _, err := co.CertClient.CreateCertificate(ctx, opts) 85 | if hcloud.IsError(err, hcloud.ErrorCodeUniquenessError) { 86 | return fmt.Errorf("%s: %w", op, ErrAlreadyExists) 87 | } 88 | if err != nil { 89 | return fmt.Errorf("%s: %v", op, err) 90 | } 91 | return nil 92 | } 93 | -------------------------------------------------------------------------------- /internal/hcops/certificates_test.go: -------------------------------------------------------------------------------- 1 | package hcops_test 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "strings" 7 | "testing" 8 | 9 | "github.com/hetznercloud/hcloud-go/v2/hcloud" 10 | "github.com/stretchr/testify/assert" 11 | "github.com/stretchr/testify/mock" 12 | "github.com/syself/hetzner-cloud-controller-manager/internal/hcops" 13 | "github.com/syself/hetzner-cloud-controller-manager/internal/mocks" 14 | ) 15 | 16 | func TestCertificateOps_GetCertificateByNameOrID(t *testing.T) { 17 | tests := []certificateOpsTestCase{ 18 | { 19 | Name: "certificate not found", 20 | Mock: func(t *testing.T, tt *certificateOpsTestCase) { 21 | tt.CertClient. 22 | On("Get", tt.Ctx, "15"). 23 | Return(nil, nil, nil) 24 | }, 25 | Perform: func(t *testing.T, tt *certificateOpsTestCase) { 26 | cert, err := tt.CertOps.GetCertificateByNameOrID(tt.Ctx, "15") 27 | assert.ErrorIs(t, err, hcops.ErrNotFound) 28 | assert.Nil(t, cert) 29 | }, 30 | }, 31 | { 32 | Name: "error calling API", 33 | ClientErr: errors.New("some error"), 34 | Mock: func(t *testing.T, tt *certificateOpsTestCase) { 35 | tt.CertClient. 36 | On("Get", tt.Ctx, "some-cert"). 37 | Return(nil, nil, tt.ClientErr) 38 | }, 39 | Perform: func(t *testing.T, tt *certificateOpsTestCase) { 40 | cert, err := tt.CertOps.GetCertificateByNameOrID(tt.Ctx, "some-cert") 41 | assert.ErrorIs(t, err, tt.ClientErr) 42 | assert.Nil(t, cert) 43 | }, 44 | }, 45 | { 46 | Name: "certificate found", 47 | Certificate: &hcloud.Certificate{ID: 16}, 48 | Mock: func(t *testing.T, tt *certificateOpsTestCase) { 49 | tt.CertClient. 50 | On("Get", tt.Ctx, "16"). 51 | Return(tt.Certificate, nil, nil) 52 | }, 53 | Perform: func(t *testing.T, tt *certificateOpsTestCase) { 54 | cert, err := tt.CertOps.GetCertificateByNameOrID(tt.Ctx, "16") 55 | assert.NoError(t, err) 56 | assert.Equal(t, tt.Certificate, cert) 57 | }, 58 | }, 59 | } 60 | 61 | runCertificateOpsTestCases(t, tests) 62 | } 63 | 64 | func TestCertificateOps_GetCertificateByLabel(t *testing.T) { 65 | tests := []certificateOpsTestCase{ 66 | { 67 | Name: "call to backend fails", 68 | Mock: func(t *testing.T, tt *certificateOpsTestCase) { 69 | tt.CertClient. 70 | On("AllWithOpts", tt.Ctx, hcloud.CertificateListOpts{ 71 | ListOpts: hcloud.ListOpts{LabelSelector: "key=value"}, 72 | }). 73 | Return(nil, errors.New("test error")) 74 | }, 75 | Perform: func(t *testing.T, tt *certificateOpsTestCase) { 76 | cert, err := tt.CertOps.GetCertificateByLabel(tt.Ctx, "key=value") 77 | assert.Nil(t, cert) 78 | assert.Error(t, err) 79 | assert.True(t, strings.HasSuffix(err.Error(), "test error")) 80 | }, 81 | }, 82 | { 83 | Name: "no matching certificate found", 84 | Mock: func(t *testing.T, tt *certificateOpsTestCase) { 85 | tt.CertClient. 86 | On("AllWithOpts", tt.Ctx, hcloud.CertificateListOpts{ 87 | ListOpts: hcloud.ListOpts{LabelSelector: "key=value"}, 88 | }). 89 | Return(nil, nil) 90 | }, 91 | Perform: func(t *testing.T, tt *certificateOpsTestCase) { 92 | cert, err := tt.CertOps.GetCertificateByLabel(tt.Ctx, "key=value") 93 | assert.ErrorIs(t, err, hcops.ErrNotFound) 94 | assert.Nil(t, cert) 95 | }, 96 | }, 97 | { 98 | Name: "more than one matching certificate found", 99 | Mock: func(t *testing.T, tt *certificateOpsTestCase) { 100 | tt.CertClient. 101 | On("AllWithOpts", tt.Ctx, hcloud.CertificateListOpts{ 102 | ListOpts: hcloud.ListOpts{LabelSelector: "key=value"}, 103 | }). 104 | Return([]*hcloud.Certificate{{ID: 1}, {ID: 2}}, nil) 105 | }, 106 | Perform: func(t *testing.T, tt *certificateOpsTestCase) { 107 | cert, err := tt.CertOps.GetCertificateByLabel(tt.Ctx, "key=value") 108 | assert.ErrorIs(t, err, hcops.ErrNonUniqueResult) 109 | assert.Nil(t, cert) 110 | }, 111 | }, 112 | { 113 | Name: "exactly one certificate found", 114 | Mock: func(t *testing.T, tt *certificateOpsTestCase) { 115 | tt.CertClient. 116 | On("AllWithOpts", tt.Ctx, hcloud.CertificateListOpts{ 117 | ListOpts: hcloud.ListOpts{LabelSelector: "key=value"}, 118 | }). 119 | Return([]*hcloud.Certificate{{ID: 1}}, nil) 120 | }, 121 | Perform: func(t *testing.T, tt *certificateOpsTestCase) { 122 | cert, err := tt.CertOps.GetCertificateByLabel(tt.Ctx, "key=value") 123 | assert.NoError(t, err) 124 | assert.Equal(t, &hcloud.Certificate{ID: 1}, cert) 125 | }, 126 | }, 127 | } 128 | 129 | runCertificateOpsTestCases(t, tests) 130 | } 131 | 132 | func TestCertificateOps_CreateManagedCertificate(t *testing.T) { 133 | tests := []certificateOpsTestCase{ 134 | { 135 | Name: "certificate creation fails", 136 | Mock: func(t *testing.T, tt *certificateOpsTestCase) { 137 | tt.CertClient. 138 | On("CreateCertificate", tt.Ctx, mock.AnythingOfType("hcloud.CertificateCreateOpts")). 139 | Return(nil, nil, errors.New("test error")) 140 | }, 141 | Perform: func(t *testing.T, tt *certificateOpsTestCase) { 142 | err := tt.CertOps.CreateManagedCertificate( 143 | tt.Ctx, 144 | "test-cert", 145 | []string{"example.com", "*.example.com"}, 146 | map[string]string{"key": "value"}, 147 | ) 148 | assert.Error(t, err) 149 | assert.True(t, strings.HasSuffix(err.Error(), "test error")) 150 | }, 151 | }, 152 | { 153 | Name: "certificate already exists", 154 | Mock: func(t *testing.T, tt *certificateOpsTestCase) { 155 | err := hcloud.Error{Code: hcloud.ErrorCodeUniquenessError} 156 | tt.CertClient. 157 | On("CreateCertificate", tt.Ctx, mock.AnythingOfType("hcloud.CertificateCreateOpts")). 158 | Return(nil, nil, err) 159 | }, 160 | Perform: func(t *testing.T, tt *certificateOpsTestCase) { 161 | err := tt.CertOps.CreateManagedCertificate( 162 | tt.Ctx, 163 | "test-cert", 164 | []string{"example.com", "*.example.com"}, 165 | map[string]string{"key": "value"}, 166 | ) 167 | assert.ErrorIs(t, err, hcops.ErrAlreadyExists) 168 | }, 169 | }, 170 | { 171 | Name: "certificate creation successful", 172 | Mock: func(t *testing.T, tt *certificateOpsTestCase) { 173 | res := hcloud.CertificateCreateResult{Certificate: &hcloud.Certificate{ID: 1}} 174 | tt.CertClient. 175 | On("CreateCertificate", tt.Ctx, hcloud.CertificateCreateOpts{ 176 | Name: "test-cert", 177 | Type: hcloud.CertificateTypeManaged, 178 | DomainNames: []string{"example.com", "*.example.com"}, 179 | Labels: map[string]string{"key": "value"}, 180 | }). 181 | Return(res, nil, nil) 182 | }, 183 | Perform: func(t *testing.T, tt *certificateOpsTestCase) { 184 | err := tt.CertOps.CreateManagedCertificate( 185 | tt.Ctx, 186 | "test-cert", 187 | []string{"example.com", "*.example.com"}, 188 | map[string]string{"key": "value"}, 189 | ) 190 | assert.NoError(t, err) 191 | }, 192 | }, 193 | } 194 | 195 | runCertificateOpsTestCases(t, tests) 196 | } 197 | 198 | type certificateOpsTestCase struct { 199 | Name string 200 | Mock func(t *testing.T, tt *certificateOpsTestCase) 201 | Perform func(t *testing.T, tt *certificateOpsTestCase) 202 | Certificate *hcloud.Certificate 203 | ClientErr error 204 | 205 | // Set in run before actual test execution 206 | Ctx context.Context 207 | CertOps *hcops.CertificateOps 208 | CertClient *mocks.CertificateClient 209 | } 210 | 211 | func (tt *certificateOpsTestCase) run(t *testing.T) { 212 | t.Helper() 213 | 214 | tt.Ctx = context.Background() 215 | tt.CertClient = &mocks.CertificateClient{} 216 | tt.CertClient.Test(t) 217 | tt.CertOps = &hcops.CertificateOps{CertClient: tt.CertClient} 218 | 219 | if tt.Mock != nil { 220 | tt.Mock(t, tt) 221 | } 222 | tt.Perform(t, tt) 223 | 224 | tt.CertClient.AssertExpectations(t) 225 | } 226 | 227 | func runCertificateOpsTestCases(t *testing.T, tests []certificateOpsTestCase) { 228 | for _, tt := range tests { 229 | tt := tt 230 | t.Run(tt.Name, tt.run) 231 | } 232 | } 233 | -------------------------------------------------------------------------------- /internal/hcops/errors.go: -------------------------------------------------------------------------------- 1 | package hcops 2 | 3 | import "errors" 4 | 5 | var ( 6 | // ErrNotFound signals that an item was not found by the Hetzner Cloud 7 | // backend. 8 | ErrNotFound = errors.New("not found") 9 | 10 | // ErrNonUniqueResult signals that more than one matching item was returned 11 | // by the Hetzner Cloud backend was returned where only one item was 12 | // expected. 13 | ErrNonUniqueResult = errors.New("non-unique result") 14 | 15 | // ErrAlreadyExists signals that the resource creation failed, because the 16 | // resource already exists. 17 | ErrAlreadyExists = errors.New("already exists") 18 | ) 19 | -------------------------------------------------------------------------------- /internal/hcops/mocks.go: -------------------------------------------------------------------------------- 1 | package hcops 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/hetznercloud/hcloud-go/v2/hcloud" 7 | "github.com/stretchr/testify/mock" 8 | "github.com/syself/hetzner-cloud-controller-manager/internal/mocks" 9 | corev1 "k8s.io/api/core/v1" 10 | ) 11 | 12 | type MockLoadBalancerOps struct { 13 | mock.Mock 14 | } 15 | 16 | func (m *MockLoadBalancerOps) GetByName(ctx context.Context, name string) (*hcloud.LoadBalancer, error) { 17 | args := m.Called(ctx, name) 18 | return mocks.GetLoadBalancerPtr(args, 0), args.Error(1) 19 | } 20 | 21 | func (m *MockLoadBalancerOps) GetByID(ctx context.Context, id int64) (*hcloud.LoadBalancer, error) { 22 | args := m.Called(ctx, id) 23 | return mocks.GetLoadBalancerPtr(args, 0), args.Error(1) 24 | } 25 | 26 | func (m *MockLoadBalancerOps) Create( 27 | ctx context.Context, lbName string, service *corev1.Service, 28 | ) (*hcloud.LoadBalancer, error) { 29 | args := m.Called(ctx, lbName, service) 30 | return mocks.GetLoadBalancerPtr(args, 0), args.Error(1) 31 | } 32 | 33 | func (m *MockLoadBalancerOps) Delete(ctx context.Context, lb *hcloud.LoadBalancer) error { 34 | args := m.Called(ctx, lb) 35 | return args.Error(0) 36 | } 37 | 38 | func (m *MockLoadBalancerOps) ReconcileHCLB( 39 | ctx context.Context, lb *hcloud.LoadBalancer, svc *corev1.Service, 40 | ) (bool, error) { 41 | args := m.Called(ctx, lb, svc) 42 | return args.Bool(0), args.Error(1) 43 | } 44 | 45 | func (m *MockLoadBalancerOps) ReconcileHCLBTargets( 46 | ctx context.Context, lb *hcloud.LoadBalancer, svc *corev1.Service, nodes []*corev1.Node, 47 | ) (bool, error) { 48 | args := m.Called(ctx, lb, svc, nodes) 49 | return args.Bool(0), args.Error(1) 50 | } 51 | 52 | func (m *MockLoadBalancerOps) ReconcileHCLBServices( 53 | ctx context.Context, lb *hcloud.LoadBalancer, svc *corev1.Service, 54 | ) (bool, error) { 55 | args := m.Called(ctx, lb, svc) 56 | return args.Bool(0), args.Error(1) 57 | } 58 | 59 | func (m *MockLoadBalancerOps) GetByK8SServiceUID(ctx context.Context, svc *corev1.Service) (*hcloud.LoadBalancer, error) { 60 | args := m.Called(ctx, svc) 61 | return mocks.GetLoadBalancerPtr(args, 0), args.Error(1) 62 | } 63 | -------------------------------------------------------------------------------- /internal/hcops/network.go: -------------------------------------------------------------------------------- 1 | package hcops 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/hetznercloud/hcloud-go/v2/hcloud" 7 | ) 8 | 9 | type HCloudNetworkClient interface { 10 | GetByID(ctx context.Context, id int64) (*hcloud.Network, *hcloud.Response, error) 11 | } 12 | -------------------------------------------------------------------------------- /internal/hcops/ratelimit.go: -------------------------------------------------------------------------------- 1 | package hcops 2 | 3 | import ( 4 | "strings" 5 | "time" 6 | 7 | "github.com/syself/hetzner-cloud-controller-manager/internal/util" 8 | "github.com/syself/hrobot-go/models" 9 | corev1 "k8s.io/api/core/v1" 10 | "k8s.io/apimachinery/pkg/runtime" 11 | "k8s.io/client-go/tools/record" 12 | "k8s.io/kubectl/pkg/scheme" 13 | ) 14 | 15 | func init() { 16 | rateLimitWaitTimeRobot, err := util.GetEnvDuration("RATE_LIMIT_WAIT_TIME_ROBOT") 17 | if err != nil { 18 | panic(err) 19 | } 20 | 21 | if rateLimitWaitTimeRobot == 0 { 22 | rateLimitWaitTimeRobot = 5 * time.Minute 23 | } 24 | 25 | handler = rateLimitHandler{ 26 | waitTime: rateLimitWaitTimeRobot, 27 | } 28 | 29 | eventBroadcaster := record.NewBroadcaster() 30 | recorder = eventBroadcaster.NewRecorder(scheme.Scheme, corev1.EventSource{Component: "hetzner-ccm-ratelimit"}) 31 | } 32 | 33 | var recorder record.EventRecorder 34 | 35 | var handler rateLimitHandler 36 | 37 | type rateLimitHandler struct { 38 | waitTime time.Duration 39 | exceeded bool 40 | lastChecked time.Time 41 | } 42 | 43 | func (rl *rateLimitHandler) set() { 44 | rl.exceeded = true 45 | rl.lastChecked = time.Now() 46 | } 47 | 48 | func (rl *rateLimitHandler) isExceeded() bool { 49 | if !rl.exceeded { 50 | return false 51 | } 52 | 53 | if time.Now().Before(rl.lastChecked.Add(rl.waitTime)) { 54 | return true 55 | } 56 | // Waiting time is over. Should try again 57 | rl.exceeded = false 58 | rl.lastChecked = time.Time{} 59 | return false 60 | } 61 | 62 | func (rl *rateLimitHandler) timeOfNextPossibleAPICall() time.Time { 63 | emptyTime := time.Time{} 64 | if rl.lastChecked == emptyTime { 65 | return emptyTime 66 | } 67 | return rl.lastChecked.Add(rl.waitTime) 68 | } 69 | 70 | // implement rate limit that is stored in memory 71 | 72 | func IsRateLimitExceeded(obj runtime.Object) bool { 73 | if handler.isExceeded() { 74 | recorder.Event(obj, "Warning", "RobotRateLimitExceeded", "exceeded Hetzner Robot API rate limit") 75 | return true 76 | } 77 | return false 78 | } 79 | 80 | func SetRateLimit() { 81 | handler.set() 82 | } 83 | 84 | func TimeOfNextPossibleAPICall() time.Time { 85 | return handler.timeOfNextPossibleAPICall() 86 | } 87 | 88 | func HandleRateLimitExceededError(err error, obj runtime.Object) { 89 | if models.IsError(err, models.ErrorCodeRateLimitExceeded) || strings.Contains(err.Error(), "server responded with status code 403") { 90 | recorder.Event(obj, "Warning", "RobotRateLimitExceeded", "exceeded Hetzner Robot API rate limit") 91 | SetRateLimit() 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /internal/hcops/ratelimit_test.go: -------------------------------------------------------------------------------- 1 | package hcops 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/stretchr/testify/require" 8 | ) 9 | 10 | func TestRateLimitSet(t *testing.T) { 11 | rl := rateLimitHandler{} 12 | now := time.Now() 13 | rl.set() 14 | require.Equal(t, true, rl.exceeded) 15 | require.NotEqual(t, time.Time{}, rl.lastChecked) 16 | require.Equal(t, true, now.Before(rl.lastChecked)) 17 | } 18 | 19 | func TestRateLimitIsExceeded(t *testing.T) { 20 | now := time.Now() 21 | 22 | rateLimitNotExceeded := rateLimitHandler{} 23 | 24 | require.Equal(t, false, rateLimitNotExceeded.isExceeded()) 25 | 26 | rateLimitExceeded := rateLimitHandler{ 27 | exceeded: true, 28 | lastChecked: now.Add(-3 * time.Minute), 29 | waitTime: 5 * time.Minute, 30 | } 31 | 32 | require.Equal(t, true, rateLimitExceeded.isExceeded()) 33 | 34 | rateLimitWaitingTimeOver := rateLimitHandler{ 35 | exceeded: true, 36 | lastChecked: now.Add(-10 * time.Minute), 37 | } 38 | 39 | require.Equal(t, false, rateLimitWaitingTimeOver.isExceeded()) 40 | require.Equal(t, time.Time{}, rateLimitWaitingTimeOver.lastChecked) 41 | require.Equal(t, false, rateLimitWaitingTimeOver.exceeded) 42 | } 43 | 44 | func TestRateLimitTimeOfNextPossibleAPICall(t *testing.T) { 45 | now := time.Now() 46 | lastChecked := now.Add(-3 * time.Minute) 47 | rateLimitExceeded := rateLimitHandler{ 48 | waitTime: 5 * time.Minute, 49 | exceeded: true, 50 | lastChecked: lastChecked, 51 | } 52 | 53 | require.Equal(t, lastChecked.Add(rateLimitExceeded.waitTime), rateLimitExceeded.timeOfNextPossibleAPICall()) 54 | 55 | rateLimitNotExceeded := rateLimitHandler{} 56 | 57 | require.Equal(t, time.Time{}, rateLimitNotExceeded.timeOfNextPossibleAPICall()) 58 | } 59 | -------------------------------------------------------------------------------- /internal/hcops/server.go: -------------------------------------------------------------------------------- 1 | package hcops 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net" 7 | "sync" 8 | "time" 9 | 10 | "github.com/hetznercloud/hcloud-go/v2/hcloud" 11 | "github.com/syself/hetzner-cloud-controller-manager/internal/metrics" 12 | "k8s.io/klog/v2" 13 | ) 14 | 15 | // AllServersCache caches the result of the LoadFunc and provides random access 16 | // to servers using select hcloud.Server attributes. 17 | // 18 | // To simplify things the allServersCache reloads all servers on every cache 19 | // miss, or whenever a timeout expired. 20 | type AllServersCache struct { 21 | LoadFunc func(context.Context) ([]*hcloud.Server, error) 22 | LoadTimeout time.Duration 23 | MaxAge time.Duration 24 | 25 | // If set, only IPs in this network will be considered for [ByPrivateIP] 26 | Network *hcloud.Network 27 | 28 | lastRefresh time.Time 29 | byPrivIP map[string]*hcloud.Server 30 | byName map[string]*hcloud.Server 31 | 32 | mu sync.Mutex // protects by* maps 33 | } 34 | 35 | // ByPrivateIP obtains a server from the cache using the IP of one of its 36 | // private networks. 37 | // 38 | // Note that a pointer to the object stored in the cache is returned. Modifying 39 | // this object affects the cache and all other code parts holding a reference. 40 | // Furthermore modifying the returned server is not concurrency safe. 41 | func (c *AllServersCache) ByPrivateIP(ip net.IP) (*hcloud.Server, error) { 42 | const op = "hcops/AllServersCache.ByPrivateIP" 43 | metrics.OperationCalled.WithLabelValues(op).Inc() 44 | 45 | srv, err := c.getCache(func() (*hcloud.Server, bool) { 46 | srv, ok := c.byPrivIP[ip.String()] 47 | return srv, ok 48 | }) 49 | if err != nil { 50 | return nil, fmt.Errorf("%s: %v %w", op, ip, err) 51 | } 52 | 53 | return srv, nil 54 | } 55 | 56 | // ByName obtains a server from the cache using the servers name. 57 | // 58 | // Note that a pointer to the object stored in the cache is returned. Modifying 59 | // this object affects the cache and all other code parts holding a reference. 60 | // Furthermore modifying the returned server is not concurrency safe. 61 | func (c *AllServersCache) ByName(name string) (*hcloud.Server, error) { 62 | const op = "hcops/AllServersCache.ByName" 63 | metrics.OperationCalled.WithLabelValues(op).Inc() 64 | 65 | srv, err := c.getCache(func() (*hcloud.Server, bool) { 66 | srv, ok := c.byName[name] 67 | return srv, ok 68 | }) 69 | if err != nil { 70 | return nil, fmt.Errorf("%s: %s %w", op, name, err) 71 | } 72 | 73 | return srv, nil 74 | } 75 | 76 | func (c *AllServersCache) getCache(getSrv func() (*hcloud.Server, bool)) (*hcloud.Server, error) { 77 | const op = "hcops/AllServersCache.getCache" 78 | metrics.OperationCalled.WithLabelValues(op).Inc() 79 | 80 | c.mu.Lock() 81 | defer c.mu.Unlock() 82 | 83 | // First try to get the value from the cache if the cache is not yet 84 | // expired. 85 | if srv, ok := getSrv(); ok && !c.isExpired() { 86 | return srv, nil 87 | } 88 | 89 | // Reload from the backend API if we didn't find srv. 90 | to := c.LoadTimeout 91 | if to == 0 { 92 | to = 20 * time.Second 93 | } 94 | ctx, cancel := context.WithTimeout(context.Background(), to) 95 | defer cancel() 96 | 97 | srvs, err := c.LoadFunc(ctx) 98 | if err != nil { 99 | return nil, fmt.Errorf("%s: %w", op, err) 100 | } 101 | 102 | // Re-initialize all maps. This effectively clears the current cache. 103 | c.byPrivIP = make(map[string]*hcloud.Server) 104 | c.byName = make(map[string]*hcloud.Server) 105 | 106 | for _, srv := range srvs { 107 | // Index servers by the IPs of their private networks 108 | for _, n := range srv.PrivateNet { 109 | if c.Network != nil { 110 | if n.Network.ID != c.Network.ID { 111 | // Only consider private IPs in the right network 112 | continue 113 | } 114 | } 115 | if n.IP == nil { 116 | continue 117 | } 118 | if _, ok := c.byPrivIP[n.IP.String()]; ok { 119 | klog.Warningf("Overriding AllServersCache entry for private ip %s with server %s\n", n.IP.String(), srv.Name) 120 | } 121 | c.byPrivIP[n.IP.String()] = srv 122 | } 123 | 124 | // Index servers by their names. 125 | c.byName[srv.Name] = srv 126 | } 127 | 128 | c.lastRefresh = time.Now() 129 | 130 | // Re-try to find the server after the reload. 131 | if srv, ok := getSrv(); ok { 132 | return srv, nil 133 | } 134 | return nil, fmt.Errorf("%s: %w", op, ErrNotFound) 135 | } 136 | 137 | // InvalidateCache invalidates the cache so that on the next cache call the cache gets refreshed. 138 | func (c *AllServersCache) InvalidateCache() { 139 | c.mu.Lock() 140 | defer c.mu.Unlock() 141 | 142 | c.lastRefresh = time.Time{} 143 | } 144 | 145 | func (c *AllServersCache) isExpired() bool { 146 | maxAge := c.MaxAge 147 | if maxAge == 0 { 148 | maxAge = 10 * time.Minute 149 | } 150 | return time.Now().After(c.lastRefresh.Add(maxAge)) 151 | } 152 | -------------------------------------------------------------------------------- /internal/hcops/server_test.go: -------------------------------------------------------------------------------- 1 | package hcops_test 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "net" 7 | "testing" 8 | "time" 9 | 10 | "github.com/hetznercloud/hcloud-go/v2/hcloud" 11 | "github.com/stretchr/testify/assert" 12 | "github.com/stretchr/testify/mock" 13 | "github.com/syself/hetzner-cloud-controller-manager/internal/hcops" 14 | "github.com/syself/hetzner-cloud-controller-manager/internal/mocks" 15 | ) 16 | 17 | func TestAllServersCache_CacheMiss(t *testing.T) { 18 | srv := &hcloud.Server{ 19 | ID: 12345, 20 | Name: "cache-miss", 21 | PrivateNet: []hcloud.ServerPrivateNet{ 22 | { 23 | IP: net.ParseIP("10.0.0.2"), 24 | }, 25 | }, 26 | } 27 | cacheOps := newAllServersCacheOps(t, srv) 28 | tmpl := allServersCacheTestCase{ 29 | SetUp: func(t *testing.T, tt *allServersCacheTestCase) { 30 | tt.ServerClient. 31 | On("All", mock.Anything). 32 | Return([]*hcloud.Server{srv}, nil) 33 | }, 34 | Expected: srv, 35 | } 36 | 37 | runAllServersCacheTests(t, "Cache miss", tmpl, cacheOps) 38 | } 39 | 40 | func TestAllServersCache_CacheHit(t *testing.T) { 41 | srv := &hcloud.Server{ 42 | ID: 54321, 43 | Name: "cache-hit", 44 | PrivateNet: []hcloud.ServerPrivateNet{ 45 | { 46 | IP: net.ParseIP("10.0.0.3"), 47 | }, 48 | }, 49 | } 50 | cacheOps := newAllServersCacheOps(t, srv) 51 | tmpl := allServersCacheTestCase{ 52 | SetUp: func(t *testing.T, tt *allServersCacheTestCase) { 53 | tt.ServerClient. 54 | On("All", mock.Anything). 55 | Return([]*hcloud.Server{srv}, nil). 56 | Once() 57 | 58 | // Perform any cache op to initialize caches 59 | if _, err := tt.Cache.ByName(srv.Name); err != nil { 60 | t.Fatalf("SetUp: %v", err) 61 | } 62 | }, 63 | Assert: func(t *testing.T, tt *allServersCacheTestCase) { 64 | // All must be called only once. This call has happened during the 65 | // test SetUp method. All additional calls indicate an error. 66 | tt.ServerClient.AssertNumberOfCalls(t, "All", 1) 67 | }, 68 | Expected: srv, 69 | } 70 | 71 | runAllServersCacheTests(t, "Cache hit", tmpl, cacheOps) 72 | } 73 | 74 | func TestAllServersCache_InvalidateCache(t *testing.T) { 75 | srv := &hcloud.Server{ 76 | ID: 54321, 77 | Name: "cache-hit", 78 | PrivateNet: []hcloud.ServerPrivateNet{ 79 | { 80 | IP: net.ParseIP("10.0.0.3"), 81 | }, 82 | }, 83 | } 84 | cacheOps := newAllServersCacheOps(t, srv) 85 | tmpl := allServersCacheTestCase{ 86 | SetUp: func(t *testing.T, tt *allServersCacheTestCase) { 87 | tt.ServerClient. 88 | On("All", mock.Anything). 89 | Return([]*hcloud.Server{srv}, nil). 90 | Times(2) 91 | 92 | // Perform any cache op to initialize caches 93 | if _, err := tt.Cache.ByName(srv.Name); err != nil { 94 | t.Fatalf("SetUp: %v", err) 95 | } 96 | 97 | // Invalidate Cache 98 | tt.Cache.InvalidateCache() 99 | 100 | // Perform a second cache lookup 101 | if _, err := tt.Cache.ByName(srv.Name); err != nil { 102 | t.Fatalf("SetUp: %v", err) 103 | } 104 | }, 105 | Assert: func(t *testing.T, tt *allServersCacheTestCase) { 106 | // All must be called twice. This call has happened during the 107 | // test SetUp method. 108 | tt.ServerClient.AssertNumberOfCalls(t, "All", 2) 109 | }, 110 | Expected: srv, 111 | } 112 | 113 | runAllServersCacheTests(t, "Cache hit", tmpl, cacheOps) 114 | } 115 | 116 | func TestAllServersCache_CacheRefresh(t *testing.T) { 117 | srv := &hcloud.Server{ 118 | ID: 56789, 119 | Name: "cache-refresh", 120 | PrivateNet: []hcloud.ServerPrivateNet{ 121 | { 122 | IP: net.ParseIP("10.0.0.9"), 123 | }, 124 | }, 125 | } 126 | cacheOps := newAllServersCacheOps(t, srv) 127 | tmpl := allServersCacheTestCase{ 128 | SetUp: func(t *testing.T, tt *allServersCacheTestCase) { 129 | tt.ServerClient. 130 | On("All", mock.Anything). 131 | Return([]*hcloud.Server{srv}, nil) 132 | 133 | if _, err := tt.Cache.ByName(srv.Name); err != nil { 134 | t.Fatalf("SetUp: %v", err) 135 | } 136 | // Set the maximum cache age to a ridiculously low time to 137 | // speed the test up. 138 | tt.Cache.MaxAge = time.Nanosecond 139 | time.Sleep(2 * tt.Cache.MaxAge) 140 | }, 141 | Assert: func(t *testing.T, tt *allServersCacheTestCase) { 142 | // All must be called only twice. Once during set-up and once 143 | // during the refresh because of the cache age expired. 144 | tt.ServerClient.AssertNumberOfCalls(t, "All", 2) 145 | }, 146 | Expected: srv, 147 | } 148 | 149 | runAllServersCacheTests(t, "Cache refresh", tmpl, cacheOps) 150 | } 151 | 152 | func TestAllServersCache_NotFound(t *testing.T) { 153 | srv := &hcloud.Server{ 154 | ID: 101010, 155 | Name: "not-found", 156 | PrivateNet: []hcloud.ServerPrivateNet{ 157 | { 158 | IP: net.ParseIP("10.0.0.4"), 159 | }, 160 | }, 161 | } 162 | cacheOps := newAllServersCacheOps(t, srv) 163 | tmpl := allServersCacheTestCase{ 164 | SetUp: func(t *testing.T, tt *allServersCacheTestCase) { 165 | tt.ServerClient. 166 | On("All", mock.Anything). 167 | Return(nil, nil) 168 | }, 169 | ExpectedErr: hcops.ErrNotFound, 170 | } 171 | 172 | runAllServersCacheTests(t, "Not found", tmpl, cacheOps) 173 | } 174 | 175 | func TestAllServersCache_ClientError(t *testing.T) { 176 | err := errors.New("client-error") 177 | srv := &hcloud.Server{ 178 | ID: 202020, 179 | Name: "client-error", 180 | PrivateNet: []hcloud.ServerPrivateNet{ 181 | { 182 | IP: net.ParseIP("10.0.0.5"), 183 | }, 184 | }, 185 | } 186 | cacheOps := newAllServersCacheOps(t, srv) 187 | tmpl := allServersCacheTestCase{ 188 | SetUp: func(t *testing.T, tt *allServersCacheTestCase) { 189 | tt.ServerClient. 190 | On("All", mock.Anything). 191 | Return(nil, err) 192 | }, 193 | ExpectedErr: err, 194 | } 195 | 196 | runAllServersCacheTests(t, "Not found", tmpl, cacheOps) 197 | } 198 | 199 | func TestAllServersCache_DuplicatePrivateIP(t *testing.T) { 200 | // Regression test for https://github.com/hetznercloud/hcloud-cloud-controller-manager/issues/470 201 | 202 | network := &hcloud.Network{ 203 | ID: 12345, 204 | Name: "cluster-network", 205 | } 206 | srv := &hcloud.Server{ 207 | ID: 101010, 208 | Name: "cluster-node", 209 | PrivateNet: []hcloud.ServerPrivateNet{ 210 | { 211 | IP: net.ParseIP("10.0.0.4"), 212 | Network: network, 213 | }, 214 | }, 215 | } 216 | srvInvalid := &hcloud.Server{ 217 | ID: 101012, 218 | Name: "invalid-node", 219 | PrivateNet: []hcloud.ServerPrivateNet{ 220 | { 221 | IP: net.ParseIP("10.0.0.4"), 222 | Network: &hcloud.Network{ 223 | ID: 54321, 224 | Name: "invalid-network", 225 | }, 226 | }, 227 | }, 228 | } 229 | 230 | cacheOps := newAllServersCacheOps(t, srv) 231 | tmpl := allServersCacheTestCase{ 232 | SetUp: func(t *testing.T, tt *allServersCacheTestCase) { 233 | tt.Cache.Network = network 234 | 235 | tt.ServerClient. 236 | On("All", mock.Anything). 237 | Return([]*hcloud.Server{srv, srvInvalid}, nil) 238 | }, 239 | Expected: srv, 240 | } 241 | 242 | runAllServersCacheTests(t, "DuplicatePrivateIP", tmpl, cacheOps) 243 | } 244 | 245 | type allServersCacheOp func(c *hcops.AllServersCache) (*hcloud.Server, error) 246 | 247 | func newAllServersCacheOps(t *testing.T, srv *hcloud.Server) map[string]allServersCacheOp { 248 | return map[string]allServersCacheOp{ 249 | "ByPrivateIP": func(c *hcops.AllServersCache) (*hcloud.Server, error) { 250 | if len(srv.PrivateNet) == 0 { 251 | t.Fatal("ByPrivateIP: server has no private net") 252 | } 253 | if len(srv.PrivateNet) > 1 { 254 | t.Fatal("ByPrivateIP: server more than one private net") 255 | } 256 | ip := srv.PrivateNet[0].IP 257 | if ip == nil { 258 | t.Fatal("ByPrivateIP: server has no private ip") 259 | } 260 | return c.ByPrivateIP(ip) 261 | }, 262 | "ByName": func(c *hcops.AllServersCache) (*hcloud.Server, error) { 263 | if srv.Name == "" { 264 | t.Fatal("ByName: server has no name") 265 | } 266 | return c.ByName(srv.Name) 267 | }, 268 | } 269 | } 270 | 271 | type allServersCacheTestCase struct { 272 | SetUp func(t *testing.T, tt *allServersCacheTestCase) 273 | CacheOp allServersCacheOp 274 | Expected *hcloud.Server 275 | ExpectedErr error 276 | Assert func(t *testing.T, tt *allServersCacheTestCase) 277 | 278 | // set in run method. 279 | ServerClient *mocks.ServerClient 280 | Cache *hcops.AllServersCache 281 | } 282 | 283 | func (tt *allServersCacheTestCase) run(t *testing.T) { 284 | tt.ServerClient = mocks.NewServerClient(t) 285 | tt.Cache = &hcops.AllServersCache{ 286 | LoadFunc: tt.ServerClient.All, 287 | } 288 | 289 | if tt.SetUp != nil { 290 | tt.SetUp(t, tt) 291 | } 292 | actual, err := tt.CacheOp(tt.Cache) 293 | if tt.ExpectedErr == nil && !assert.NoError(t, err) { 294 | return 295 | } 296 | if tt.ExpectedErr != nil { 297 | assert.Truef(t, errors.Is(err, tt.ExpectedErr), "expected error: %v; got %v", tt.ExpectedErr, err) 298 | return 299 | } 300 | assert.Equal(t, tt.Expected, actual) 301 | 302 | if tt.Assert != nil { 303 | tt.Assert(t, tt) 304 | } 305 | tt.ServerClient.AssertExpectations(t) 306 | } 307 | 308 | func runAllServersCacheTests( 309 | t *testing.T, name string, tmpl allServersCacheTestCase, cacheOps map[string]allServersCacheOp, 310 | ) { 311 | for opName, op := range cacheOps { 312 | assert.Nil(t, tmpl.CacheOp) // just make sure tmpl was not modified 313 | 314 | // Copy tmpl as we do not want to modify the original 315 | tt := tmpl 316 | tt.CacheOp = op 317 | t.Run(fmt.Sprintf("%s: %s", name, opName), tt.run) 318 | } 319 | } 320 | -------------------------------------------------------------------------------- /internal/hcops/testing.go: -------------------------------------------------------------------------------- 1 | package hcops 2 | 3 | import ( 4 | "context" 5 | "math/rand" 6 | "net" 7 | "testing" 8 | 9 | "github.com/hetznercloud/hcloud-go/v2/hcloud" 10 | "github.com/syself/hetzner-cloud-controller-manager/internal/mocks" 11 | "github.com/syself/hrobot-go/models" 12 | ) 13 | 14 | type LoadBalancerOpsFixture struct { 15 | Name string 16 | Ctx context.Context 17 | LBClient *mocks.LoadBalancerClient 18 | CertClient *mocks.CertificateClient 19 | ActionClient *mocks.ActionClient 20 | NetworkClient *mocks.NetworkClient 21 | RobotClient *mocks.RobotClient 22 | 23 | LBOps *LoadBalancerOps 24 | 25 | T *testing.T 26 | } 27 | 28 | func NewLoadBalancerOpsFixture(t *testing.T) *LoadBalancerOpsFixture { 29 | fx := &LoadBalancerOpsFixture{ 30 | Ctx: context.Background(), 31 | ActionClient: &mocks.ActionClient{}, 32 | LBClient: &mocks.LoadBalancerClient{}, 33 | CertClient: &mocks.CertificateClient{}, 34 | NetworkClient: &mocks.NetworkClient{}, 35 | RobotClient: &mocks.RobotClient{}, 36 | T: t, 37 | } 38 | 39 | fx.ActionClient.Test(t) 40 | fx.LBClient.Test(t) 41 | fx.CertClient.Test(t) 42 | fx.NetworkClient.Test(t) 43 | fx.RobotClient.Test(t) 44 | 45 | fx.LBOps = &LoadBalancerOps{ 46 | LBClient: fx.LBClient, 47 | CertOps: &CertificateOps{CertClient: fx.CertClient}, 48 | ActionClient: fx.ActionClient, 49 | NetworkClient: fx.NetworkClient, 50 | RobotClient: fx.RobotClient, 51 | } 52 | 53 | return fx 54 | } 55 | 56 | func (fx *LoadBalancerOpsFixture) MockGetByID(lb *hcloud.LoadBalancer, err error) { 57 | fx.LBClient.On("GetByID", fx.Ctx, lb.ID).Return(lb, nil, err) 58 | } 59 | 60 | func (fx *LoadBalancerOpsFixture) MockCreate( 61 | opts hcloud.LoadBalancerCreateOpts, lb *hcloud.LoadBalancer, err error, 62 | ) *hcloud.Action { 63 | action := &hcloud.Action{ID: rand.Int63()} 64 | createResult := hcloud.LoadBalancerCreateResult{Action: action, LoadBalancer: lb} 65 | fx.LBClient.On("Create", fx.Ctx, opts).Return(createResult, nil, err) 66 | return action 67 | } 68 | 69 | func (fx *LoadBalancerOpsFixture) MockAddService( 70 | opts hcloud.LoadBalancerAddServiceOpts, lb *hcloud.LoadBalancer, err error, 71 | ) *hcloud.Action { 72 | action := &hcloud.Action{ID: rand.Int63()} 73 | fx.LBClient.On("AddService", fx.Ctx, lb, opts).Return(action, nil, err) 74 | return action 75 | } 76 | 77 | func (fx *LoadBalancerOpsFixture) MockUpdateService( 78 | opts hcloud.LoadBalancerUpdateServiceOpts, lb *hcloud.LoadBalancer, listenPort int, err error, 79 | ) *hcloud.Action { 80 | action := &hcloud.Action{ID: rand.Int63()} 81 | fx.LBClient.On("UpdateService", fx.Ctx, lb, listenPort, opts).Return(action, nil, err) 82 | return action 83 | } 84 | 85 | func (fx *LoadBalancerOpsFixture) MockDeleteService(lb *hcloud.LoadBalancer, port int, err error) *hcloud.Action { 86 | action := &hcloud.Action{ID: rand.Int63()} 87 | fx.LBClient.On("DeleteService", fx.Ctx, lb, port).Return(action, nil, err) 88 | return action 89 | } 90 | 91 | func (fx *LoadBalancerOpsFixture) MockAddServerTarget( 92 | lb *hcloud.LoadBalancer, opts hcloud.LoadBalancerAddServerTargetOpts, err error, 93 | ) *hcloud.Action { 94 | action := &hcloud.Action{ID: rand.Int63()} 95 | fx.LBClient.On("AddServerTarget", fx.Ctx, lb, opts).Return(action, nil, err) 96 | return action 97 | } 98 | 99 | func (fx *LoadBalancerOpsFixture) MockRemoveServerTarget( 100 | lb *hcloud.LoadBalancer, s *hcloud.Server, err error, 101 | ) *hcloud.Action { 102 | action := &hcloud.Action{ID: rand.Int63()} 103 | fx.LBClient.On("RemoveServerTarget", fx.Ctx, lb, s).Return(action, nil, err) 104 | return action 105 | } 106 | 107 | func (fx *LoadBalancerOpsFixture) MockAddIPTarget( 108 | lb *hcloud.LoadBalancer, opts hcloud.LoadBalancerAddIPTargetOpts, err error, 109 | ) *hcloud.Action { 110 | action := &hcloud.Action{ID: rand.Int63()} 111 | fx.LBClient.On("AddIPTarget", fx.Ctx, lb, opts).Return(action, nil, err) 112 | return action 113 | } 114 | 115 | func (fx *LoadBalancerOpsFixture) MockRemoveIPTarget( 116 | lb *hcloud.LoadBalancer, ip net.IP, err error, 117 | ) *hcloud.Action { 118 | action := &hcloud.Action{ID: rand.Int63()} 119 | fx.LBClient.On("RemoveIPTarget", fx.Ctx, lb, ip).Return(action, nil, err) 120 | return action 121 | } 122 | 123 | func (fx *LoadBalancerOpsFixture) MockListRobotServers( 124 | serverList []models.Server, err error, 125 | ) { 126 | fx.RobotClient.On("ServerGetList").Return(serverList, err) 127 | } 128 | 129 | func (fx *LoadBalancerOpsFixture) MockWatchProgress(a *hcloud.Action, err error) { 130 | fx.ActionClient.MockWatchProgress(fx.Ctx, a, err) 131 | } 132 | 133 | func (fx *LoadBalancerOpsFixture) AssertExpectations() { 134 | fx.ActionClient.AssertExpectations(fx.T) 135 | fx.LBClient.AssertExpectations(fx.T) 136 | fx.CertClient.AssertExpectations(fx.T) 137 | fx.NetworkClient.AssertExpectations(fx.T) 138 | } 139 | -------------------------------------------------------------------------------- /internal/metrics/metrics.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Hetzner Cloud GmbH. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package metrics 18 | 19 | import ( 20 | "net/http" 21 | 22 | "github.com/prometheus/client_golang/prometheus" 23 | "github.com/prometheus/client_golang/prometheus/promhttp" 24 | "k8s.io/klog/v2" 25 | ) 26 | 27 | var OperationCalled = prometheus.NewCounterVec(prometheus.CounterOpts{ 28 | Name: "cloud_controller_manager_operations_total", 29 | Help: "The total number of operation was called", 30 | }, []string{"op"}) 31 | 32 | var registry = prometheus.NewRegistry() 33 | 34 | func GetRegistry() *prometheus.Registry { 35 | return registry 36 | } 37 | 38 | func Serve(address string) { 39 | klog.Info("Starting metrics server at ", address) 40 | 41 | registry.MustRegister(OperationCalled) 42 | 43 | gatherers := prometheus.Gatherers{ 44 | prometheus.DefaultGatherer, 45 | registry, 46 | } 47 | 48 | http.Handle("/metrics", promhttp.HandlerFor(gatherers, promhttp.HandlerOpts{})) 49 | // TODO: Setup proper timeouts for metrics server and remove nolint:gosec 50 | if err := http.ListenAndServe(address, nil); err != nil { //nolint:gosec 51 | klog.ErrorS(err, "create metrics service") 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /internal/mocks/action.go: -------------------------------------------------------------------------------- 1 | package mocks 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/hetznercloud/hcloud-go/v2/hcloud" 7 | "github.com/stretchr/testify/mock" 8 | ) 9 | 10 | type ActionClient struct { 11 | mock.Mock 12 | } 13 | 14 | func (m *ActionClient) WatchProgress(ctx context.Context, a *hcloud.Action) (<-chan int, <-chan error) { 15 | args := m.Called(ctx, a) 16 | return getIntChan(args, 0), getErrChan(args, 1) 17 | } 18 | 19 | func (m *ActionClient) MockWatchProgress(ctx context.Context, a *hcloud.Action, err error) { 20 | resC := make(chan int) 21 | errC := make(chan error, 1) 22 | if err != nil { 23 | errC <- err 24 | } 25 | close(resC) 26 | close(errC) 27 | m.On("WatchProgress", ctx, a).Return(resC, errC) 28 | } 29 | -------------------------------------------------------------------------------- /internal/mocks/casts.go: -------------------------------------------------------------------------------- 1 | package mocks 2 | 3 | import ( 4 | "github.com/hetznercloud/hcloud-go/v2/hcloud" 5 | "github.com/stretchr/testify/mock" 6 | "github.com/syself/hrobot-go/models" 7 | ) 8 | 9 | func getResponsePtr(args mock.Arguments, i int) *hcloud.Response { 10 | v := args.Get(i) 11 | if v == nil { 12 | return nil 13 | } 14 | return v.(*hcloud.Response) 15 | } 16 | 17 | func getActionPtr(args mock.Arguments, i int) *hcloud.Action { 18 | v := args.Get(i) 19 | if v == nil { 20 | return nil 21 | } 22 | return v.(*hcloud.Action) 23 | } 24 | 25 | func GetLoadBalancerPtr(args mock.Arguments, i int) *hcloud.LoadBalancer { 26 | v := args.Get(i) 27 | if v == nil { 28 | return nil 29 | } 30 | return v.(*hcloud.LoadBalancer) 31 | } 32 | 33 | func getLoadBalancerPtrS(args mock.Arguments, i int) []*hcloud.LoadBalancer { 34 | v := args.Get(i) 35 | if v == nil { 36 | return nil 37 | } 38 | return v.([]*hcloud.LoadBalancer) 39 | } 40 | 41 | func getRobotServers(args mock.Arguments, i int) []models.Server { 42 | v := args.Get(i) 43 | if v == nil { 44 | return nil 45 | } 46 | return v.([]models.Server) 47 | } 48 | 49 | func getNetworkPtr(args mock.Arguments, i int) *hcloud.Network { 50 | v := args.Get(i) 51 | if v == nil { 52 | return nil 53 | } 54 | return v.(*hcloud.Network) 55 | } 56 | 57 | func getIntChan(args mock.Arguments, i int) chan int { 58 | v := args.Get(i) 59 | if v == nil { 60 | return nil 61 | } 62 | return v.(chan int) 63 | } 64 | 65 | func getErrChan(args mock.Arguments, i int) chan error { 66 | v := args.Get(i) 67 | if v == nil { 68 | return nil 69 | } 70 | return v.(chan error) 71 | } 72 | 73 | func getCertificatePtr(args mock.Arguments, i int) *hcloud.Certificate { 74 | v := args.Get(i) 75 | if v == nil { 76 | return nil 77 | } 78 | return v.(*hcloud.Certificate) 79 | } 80 | 81 | func getCertificatePtrS(args mock.Arguments, i int) []*hcloud.Certificate { 82 | v := args.Get(i) 83 | if v == nil { 84 | return nil 85 | } 86 | return v.([]*hcloud.Certificate) 87 | } 88 | 89 | func getCertificateCreateResult(args mock.Arguments, i int) hcloud.CertificateCreateResult { 90 | v := args.Get(i) 91 | if v == nil { 92 | return hcloud.CertificateCreateResult{} 93 | } 94 | return v.(hcloud.CertificateCreateResult) 95 | } 96 | -------------------------------------------------------------------------------- /internal/mocks/certificates.go: -------------------------------------------------------------------------------- 1 | package mocks 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/hetznercloud/hcloud-go/v2/hcloud" 7 | "github.com/stretchr/testify/mock" 8 | ) 9 | 10 | type CertificateClient struct { 11 | mock.Mock 12 | } 13 | 14 | func (m *CertificateClient) AllWithOpts( 15 | ctx context.Context, opts hcloud.CertificateListOpts, 16 | ) ([]*hcloud.Certificate, error) { 17 | args := m.Called(ctx, opts) 18 | return getCertificatePtrS(args, 0), args.Error(1) 19 | } 20 | 21 | func (m *CertificateClient) Get(ctx context.Context, idOrName string) (*hcloud.Certificate, *hcloud.Response, error) { 22 | args := m.Called(ctx, idOrName) 23 | return getCertificatePtr(args, 0), getResponsePtr(args, 1), args.Error(2) 24 | } 25 | 26 | func (m *CertificateClient) CreateCertificate( 27 | ctx context.Context, opts hcloud.CertificateCreateOpts, 28 | ) (hcloud.CertificateCreateResult, *hcloud.Response, error) { 29 | args := m.Called(ctx, opts) 30 | return getCertificateCreateResult(args, 0), getResponsePtr(args, 1), args.Error(2) 31 | } 32 | -------------------------------------------------------------------------------- /internal/mocks/loadbalancer.go: -------------------------------------------------------------------------------- 1 | package mocks 2 | 3 | import ( 4 | "context" 5 | "net" 6 | 7 | "github.com/hetznercloud/hcloud-go/v2/hcloud" 8 | "github.com/stretchr/testify/mock" 9 | ) 10 | 11 | type LoadBalancerClient struct { 12 | mock.Mock 13 | } 14 | 15 | func (m *LoadBalancerClient) GetByID(ctx context.Context, id int64) (*hcloud.LoadBalancer, *hcloud.Response, error) { 16 | args := m.Called(ctx, id) 17 | return GetLoadBalancerPtr(args, 0), getResponsePtr(args, 1), args.Error(2) 18 | } 19 | 20 | func (m *LoadBalancerClient) GetByName( 21 | ctx context.Context, name string, 22 | ) (*hcloud.LoadBalancer, *hcloud.Response, error) { 23 | args := m.Called(ctx, name) 24 | return GetLoadBalancerPtr(args, 0), getResponsePtr(args, 1), args.Error(2) 25 | } 26 | 27 | func (m *LoadBalancerClient) Create( 28 | ctx context.Context, opts hcloud.LoadBalancerCreateOpts, 29 | ) (hcloud.LoadBalancerCreateResult, *hcloud.Response, error) { 30 | args := m.Called(ctx, opts) 31 | return args.Get(0).(hcloud.LoadBalancerCreateResult), getResponsePtr(args, 1), args.Error(2) 32 | } 33 | 34 | func (m *LoadBalancerClient) Update( 35 | ctx context.Context, lb *hcloud.LoadBalancer, opts hcloud.LoadBalancerUpdateOpts, 36 | ) (*hcloud.LoadBalancer, *hcloud.Response, error) { 37 | args := m.Called(ctx, lb, opts) 38 | return GetLoadBalancerPtr(args, 0), getResponsePtr(args, 1), args.Error(2) 39 | } 40 | 41 | func (m *LoadBalancerClient) Delete(ctx context.Context, lb *hcloud.LoadBalancer) (*hcloud.Response, error) { 42 | args := m.Called(ctx, lb) 43 | return getResponsePtr(args, 0), args.Error(1) 44 | } 45 | 46 | func (m *LoadBalancerClient) AddService( 47 | ctx context.Context, lb *hcloud.LoadBalancer, opts hcloud.LoadBalancerAddServiceOpts, 48 | ) (*hcloud.Action, *hcloud.Response, error) { 49 | args := m.Called(ctx, lb, opts) 50 | return getActionPtr(args, 0), getResponsePtr(args, 1), args.Error(2) 51 | } 52 | 53 | func (m *LoadBalancerClient) DeleteService( 54 | ctx context.Context, lb *hcloud.LoadBalancer, listenPort int, 55 | ) (*hcloud.Action, *hcloud.Response, error) { 56 | args := m.Called(ctx, lb, listenPort) 57 | return getActionPtr(args, 0), getResponsePtr(args, 1), args.Error(2) 58 | } 59 | 60 | func (m *LoadBalancerClient) ChangeAlgorithm( 61 | ctx context.Context, lb *hcloud.LoadBalancer, opts hcloud.LoadBalancerChangeAlgorithmOpts, 62 | ) (*hcloud.Action, *hcloud.Response, error) { 63 | args := m.Called(ctx, lb, opts) 64 | return getActionPtr(args, 0), getResponsePtr(args, 1), args.Error(2) 65 | } 66 | 67 | func (m *LoadBalancerClient) ChangeDNSPtr( 68 | ctx context.Context, lb *hcloud.LoadBalancer, ip string, ptr *string, 69 | ) (*hcloud.Action, *hcloud.Response, error) { 70 | args := m.Called(ctx, lb, ip, ptr) 71 | return getActionPtr(args, 0), getResponsePtr(args, 1), args.Error(2) 72 | } 73 | 74 | func (m *LoadBalancerClient) ChangeType( 75 | ctx context.Context, lb *hcloud.LoadBalancer, opts hcloud.LoadBalancerChangeTypeOpts, 76 | ) (*hcloud.Action, *hcloud.Response, error) { 77 | args := m.Called(ctx, lb, opts) 78 | return getActionPtr(args, 0), getResponsePtr(args, 1), args.Error(2) 79 | } 80 | 81 | func (m *LoadBalancerClient) AddServerTarget( 82 | ctx context.Context, lb *hcloud.LoadBalancer, opts hcloud.LoadBalancerAddServerTargetOpts, 83 | ) (*hcloud.Action, *hcloud.Response, error) { 84 | args := m.Called(ctx, lb, opts) 85 | return getActionPtr(args, 0), getResponsePtr(args, 1), args.Error(2) 86 | } 87 | 88 | func (m *LoadBalancerClient) RemoveServerTarget(ctx context.Context, lb *hcloud.LoadBalancer, server *hcloud.Server) (*hcloud.Action, *hcloud.Response, error) { 89 | args := m.Called(ctx, lb, server) 90 | return getActionPtr(args, 0), getResponsePtr(args, 1), args.Error(2) 91 | } 92 | 93 | func (m *LoadBalancerClient) AddIPTarget( 94 | ctx context.Context, lb *hcloud.LoadBalancer, opts hcloud.LoadBalancerAddIPTargetOpts, 95 | ) (*hcloud.Action, *hcloud.Response, error) { 96 | args := m.Called(ctx, lb, opts) 97 | return getActionPtr(args, 0), getResponsePtr(args, 1), args.Error(2) 98 | } 99 | 100 | func (m *LoadBalancerClient) RemoveIPTarget(ctx context.Context, lb *hcloud.LoadBalancer, ip net.IP) (*hcloud.Action, *hcloud.Response, error) { 101 | args := m.Called(ctx, lb, ip) 102 | return getActionPtr(args, 0), getResponsePtr(args, 1), args.Error(2) 103 | } 104 | 105 | func (m *LoadBalancerClient) UpdateService( 106 | ctx context.Context, lb *hcloud.LoadBalancer, listenPort int, opts hcloud.LoadBalancerUpdateServiceOpts, 107 | ) (*hcloud.Action, *hcloud.Response, error) { 108 | args := m.Called(ctx, lb, listenPort, opts) 109 | return getActionPtr(args, 0), getResponsePtr(args, 1), args.Error(2) 110 | } 111 | 112 | func (m *LoadBalancerClient) AttachToNetwork( 113 | ctx context.Context, lb *hcloud.LoadBalancer, opts hcloud.LoadBalancerAttachToNetworkOpts, 114 | ) (*hcloud.Action, *hcloud.Response, error) { 115 | args := m.Called(ctx, lb, opts) 116 | return getActionPtr(args, 0), getResponsePtr(args, 1), args.Error(2) 117 | } 118 | 119 | func (m *LoadBalancerClient) DetachFromNetwork( 120 | ctx context.Context, lb *hcloud.LoadBalancer, opts hcloud.LoadBalancerDetachFromNetworkOpts, 121 | ) (*hcloud.Action, *hcloud.Response, error) { 122 | args := m.Called(ctx, lb, opts) 123 | return getActionPtr(args, 0), getResponsePtr(args, 1), args.Error(2) 124 | } 125 | 126 | func (m *LoadBalancerClient) EnablePublicInterface( 127 | ctx context.Context, lb *hcloud.LoadBalancer, 128 | ) (*hcloud.Action, *hcloud.Response, error) { 129 | args := m.Called(ctx, lb) 130 | return getActionPtr(args, 0), getResponsePtr(args, 1), args.Error(2) 131 | } 132 | 133 | func (m *LoadBalancerClient) DisablePublicInterface( 134 | ctx context.Context, lb *hcloud.LoadBalancer, 135 | ) (*hcloud.Action, *hcloud.Response, error) { 136 | args := m.Called(ctx, lb) 137 | return getActionPtr(args, 0), getResponsePtr(args, 1), args.Error(2) 138 | } 139 | 140 | func (m *LoadBalancerClient) AllWithOpts( 141 | ctx context.Context, opts hcloud.LoadBalancerListOpts, 142 | ) ([]*hcloud.LoadBalancer, error) { 143 | args := m.Called(ctx, opts) 144 | return getLoadBalancerPtrS(args, 0), args.Error(1) 145 | } 146 | -------------------------------------------------------------------------------- /internal/mocks/network.go: -------------------------------------------------------------------------------- 1 | package mocks 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/hetznercloud/hcloud-go/v2/hcloud" 7 | "github.com/stretchr/testify/mock" 8 | ) 9 | 10 | type NetworkClient struct { 11 | mock.Mock 12 | } 13 | 14 | func (m *NetworkClient) GetByID(ctx context.Context, id int64) (*hcloud.Network, *hcloud.Response, error) { 15 | args := m.Called(ctx, id) 16 | return getNetworkPtr(args, 0), getResponsePtr(args, 1), args.Error(2) 17 | } 18 | -------------------------------------------------------------------------------- /internal/mocks/robot.go: -------------------------------------------------------------------------------- 1 | package mocks 2 | 3 | import ( 4 | "github.com/stretchr/testify/mock" 5 | "github.com/syself/hrobot-go/models" 6 | ) 7 | 8 | type RobotClient struct { 9 | mock.Mock 10 | } 11 | 12 | func (m *RobotClient) ServerGetList() ([]models.Server, error) { 13 | args := m.Called() 14 | return getRobotServers(args, 0), args.Error(1) 15 | } 16 | 17 | func (m *RobotClient) SetCredentials(_, _ string) error { 18 | args := m.Called() 19 | return args.Error(3) 20 | } 21 | 22 | func (m *RobotClient) BootLinuxDelete(id int) (*models.Linux, error) { 23 | panic("this method should not be called") 24 | } 25 | 26 | func (m *RobotClient) BootLinuxGet(id int) (*models.Linux, error) { 27 | panic("this method should not be called") 28 | } 29 | 30 | func (m *RobotClient) BootLinuxSet(id int, input *models.LinuxSetInput) (*models.Linux, error) { 31 | panic("this method should not be called") 32 | } 33 | 34 | func (m *RobotClient) BootRescueDelete(id int) (*models.Rescue, error) { 35 | panic("this method should not be called") 36 | } 37 | 38 | func (m *RobotClient) BootRescueGet(id int) (*models.Rescue, error) { 39 | panic("this method should not be called") 40 | } 41 | 42 | func (m *RobotClient) BootRescueSet(id int, input *models.RescueSetInput) (*models.Rescue, error) { 43 | panic("this method should not be called") 44 | } 45 | 46 | func (m *RobotClient) FailoverGet(ip string) (*models.Failover, error) { 47 | panic("this method should not be called") 48 | } 49 | 50 | func (m *RobotClient) FailoverGetList() ([]models.Failover, error) { 51 | panic("this method should not be called") 52 | } 53 | 54 | func (m *RobotClient) GetVersion() string { 55 | panic("this method should not be called") 56 | } 57 | 58 | func (m *RobotClient) IPGetList() ([]models.IP, error) { 59 | panic("this method should not be called") 60 | } 61 | 62 | func (m *RobotClient) KeyGetList() ([]models.Key, error) { 63 | panic("this method should not be called") 64 | } 65 | 66 | func (m *RobotClient) KeySet(input *models.KeySetInput) (*models.Key, error) { 67 | panic("this method should not be called") 68 | } 69 | 70 | func (m *RobotClient) RDnsGet(ip string) (*models.Rdns, error) { 71 | panic("this method should not be called") 72 | } 73 | 74 | func (m *RobotClient) RDnsGetList() ([]models.Rdns, error) { 75 | panic("this method should not be called") 76 | } 77 | 78 | func (m *RobotClient) ResetGet(id int) (*models.Reset, error) { 79 | panic("this method should not be called") 80 | } 81 | 82 | func (m *RobotClient) ResetSet(id int, input *models.ResetSetInput) (*models.ResetPost, error) { 83 | panic("this method should not be called") 84 | } 85 | 86 | func (m *RobotClient) ServerGet(id int) (*models.Server, error) { 87 | panic("this method should not be called") 88 | } 89 | 90 | func (m *RobotClient) ServerReverse(id int) (*models.Cancellation, error) { 91 | panic("this method should not be called") 92 | } 93 | 94 | func (m *RobotClient) ServerSetName(id int, input *models.ServerSetNameInput) (*models.Server, error) { 95 | panic("this method should not be called") 96 | } 97 | 98 | func (m *RobotClient) SetBaseURL(baseURL string) { 99 | panic("this method should not be called") 100 | } 101 | 102 | func (m *RobotClient) SetUserAgent(userAgent string) { 103 | panic("this method should not be called") 104 | } 105 | 106 | func (m *RobotClient) ValidateCredentials() error { 107 | panic("this method should not be called") 108 | } 109 | -------------------------------------------------------------------------------- /internal/mocks/server.go: -------------------------------------------------------------------------------- 1 | package mocks 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/hetznercloud/hcloud-go/v2/hcloud" 8 | "github.com/stretchr/testify/mock" 9 | ) 10 | 11 | // ServerClient is a mock implementation of the hcloud.ServerClient. 12 | type ServerClient struct { 13 | mock.Mock 14 | T *testing.T 15 | } 16 | 17 | // NewServerClient creates a new mock server client ready for use. 18 | func NewServerClient(t *testing.T) *ServerClient { 19 | m := &ServerClient{T: t} 20 | m.Test(t) 21 | return m 22 | } 23 | 24 | // All registers a call to obtain all servers from the Hetzner Cloud API. 25 | func (m *ServerClient) All(ctx context.Context) ([]*hcloud.Server, error) { 26 | args := m.Called(ctx) 27 | return serverPtrSlice(m.T, args.Get(0)), args.Error(1) 28 | } 29 | 30 | func serverPtrSlice(t *testing.T, v interface{}) []*hcloud.Server { 31 | const op = "mocks/serverPtrSlice" 32 | 33 | t.Helper() 34 | 35 | if v == nil { 36 | return nil 37 | } 38 | ss, ok := v.([]*hcloud.Server) 39 | if !ok { 40 | t.Fatalf("%s: not a []*Server: %t", op, v) 41 | } 42 | return ss 43 | } 44 | -------------------------------------------------------------------------------- /internal/robot/client/cache/client.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "os" 7 | "time" 8 | 9 | "github.com/syself/hetzner-cloud-controller-manager/internal/credentials" 10 | robotclient "github.com/syself/hetzner-cloud-controller-manager/internal/robot/client" 11 | "github.com/syself/hetzner-cloud-controller-manager/internal/util" 12 | hrobot "github.com/syself/hrobot-go" 13 | "github.com/syself/hrobot-go/models" 14 | "k8s.io/klog/v2" 15 | ) 16 | 17 | const ( 18 | robotUserNameENVVar = "ROBOT_USER_NAME" 19 | robotPasswordENVVar = "ROBOT_PASSWORD" 20 | cacheTimeoutENVVar = "CACHE_TIMEOUT" 21 | ) 22 | 23 | var _ robotclient.Client = &cacheRobotClient{} 24 | 25 | type cacheRobotClient struct { 26 | robotClient hrobot.RobotClient 27 | timeout time.Duration 28 | 29 | lastUpdate time.Time 30 | 31 | // cache 32 | l []models.Server 33 | m map[int]*models.Server 34 | } 35 | 36 | // NewCachedRobotClient creates a new robot client with caching enabled. 37 | // rootDir: root directory for reading credentials from file. 38 | // httpClient: http client to use for the robot client. 39 | // baseURL: base URL for the robot client. Optional, leave empty for default. 40 | // Returns nil and no error if the robot client could not be created, because 41 | // the credentials are optional. 42 | func NewCachedRobotClient(rootDir string, httpClient *http.Client, baseURL string) (robotclient.Client, error) { 43 | const op = "hcloud/newRobotClient" 44 | 45 | if httpClient == nil { 46 | return nil, fmt.Errorf("%s: httpClient is nil", op) 47 | } 48 | cacheTimeout, err := util.GetEnvDuration(cacheTimeoutENVVar) 49 | if err != nil { 50 | return nil, fmt.Errorf("%s: %w", op, err) 51 | } 52 | 53 | if cacheTimeout == 0 { 54 | cacheTimeout = 5 * time.Minute 55 | } 56 | 57 | credentialsDir := credentials.GetDirectory(rootDir) 58 | _, err = os.Stat(credentialsDir) 59 | var robotUser, robotPassword string 60 | if err != nil { 61 | klog.V(1).Infof("reading Hetzner Robot credentials from file failed. %q does not exist", credentialsDir) 62 | robotUser = os.Getenv(robotUserNameENVVar) 63 | robotPassword = os.Getenv(robotPasswordENVVar) 64 | if robotUser == "" || robotPassword == "" { 65 | klog.Infof("Hetzner robot is not support because of insufficient credentials: Env vars (%q, %q) not set, and from file failed: %s", 66 | robotUserNameENVVar, robotPasswordENVVar, 67 | err.Error()) 68 | return nil, nil 69 | } 70 | } else { 71 | robotUser, robotPassword, err = credentials.GetInitialRobotCredentials(credentialsDir) 72 | if err != nil { 73 | return nil, fmt.Errorf("%s: %w", op, err) 74 | } 75 | } 76 | c := hrobot.NewBasicAuthClientWithCustomHttpClient(robotUser, robotPassword, httpClient) 77 | if baseURL != "" { 78 | c.SetBaseURL(baseURL) 79 | } 80 | 81 | handler := &cacheRobotClient{} 82 | handler.timeout = cacheTimeout 83 | handler.robotClient = c 84 | return handler, nil 85 | } 86 | 87 | func (c *cacheRobotClient) ServerGet(id int) (*models.Server, error) { 88 | if c.shouldSync() { 89 | list, err := c.robotClient.ServerGetList() 90 | if err != nil { 91 | return nil, err 92 | } 93 | 94 | // populate list 95 | c.l = list 96 | 97 | // remove all entries from map and populate it freshly 98 | c.m = make(map[int]*models.Server) 99 | for i, server := range list { 100 | c.m[server.ServerNumber] = &list[i] 101 | } 102 | 103 | // set time of last update 104 | c.lastUpdate = time.Now() 105 | } 106 | 107 | server, found := c.m[id] 108 | if !found { 109 | // return not found error 110 | return nil, models.Error{Code: models.ErrorCodeServerNotFound, Message: "server not found"} 111 | } 112 | 113 | return server, nil 114 | } 115 | 116 | func (c *cacheRobotClient) ServerGetList() ([]models.Server, error) { 117 | if c.shouldSync() { 118 | list, err := c.robotClient.ServerGetList() 119 | if err != nil { 120 | return list, err 121 | } 122 | 123 | // populate list 124 | c.l = list 125 | 126 | // remove all entries from map and populate it freshly 127 | c.m = make(map[int]*models.Server) 128 | for i, server := range list { 129 | c.m[server.ServerNumber] = &list[i] 130 | } 131 | 132 | // set time of last update 133 | c.lastUpdate = time.Now() 134 | } 135 | 136 | return c.l, nil 137 | } 138 | 139 | func (c *cacheRobotClient) shouldSync() bool { 140 | // map is nil means we have no cached value yet 141 | if c.m == nil { 142 | c.m = make(map[int]*models.Server) 143 | return true 144 | } 145 | if time.Now().After(c.lastUpdate.Add(c.timeout)) { 146 | return true 147 | } 148 | return false 149 | } 150 | 151 | func (c *cacheRobotClient) SetCredentials(username, password string) error { 152 | err := c.robotClient.SetCredentials(username, password) 153 | if err != nil { 154 | return err 155 | } 156 | // The credentials have been updated, so we need to invalidate the cache. 157 | c.m = nil 158 | return nil 159 | } 160 | -------------------------------------------------------------------------------- /internal/robot/client/cache/client_test.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "encoding/base64" 5 | "encoding/json" 6 | "fmt" 7 | "net/http" 8 | "net/http/httptest" 9 | "os" 10 | "path/filepath" 11 | "testing" 12 | "time" 13 | 14 | "github.com/stretchr/testify/require" 15 | "github.com/syself/hetzner-cloud-controller-manager/internal/credentials" 16 | "github.com/syself/hrobot-go/models" 17 | ) 18 | 19 | func Test_updateRobotCredentials(t *testing.T) { 20 | mux := http.NewServeMux() 21 | server := httptest.NewServer(mux) 22 | defer server.Close() 23 | 24 | os.Unsetenv(robotUserNameENVVar) 25 | os.Unsetenv(robotPasswordENVVar) 26 | 27 | rootDir, err := os.MkdirTemp("", "Test_newHcloudClient-*") 28 | require.NoError(t, err) 29 | 30 | credentialsDir := credentials.GetDirectory(rootDir) 31 | err = os.MkdirAll(credentialsDir, 0o755) 32 | require.NoError(t, err) 33 | 34 | err = os.Symlink("..data/robot-user", filepath.Join(credentialsDir, "robot-user")) 35 | require.NoError(t, err) 36 | 37 | err = os.Symlink("..data/robot-password", filepath.Join(credentialsDir, "robot-password")) 38 | require.NoError(t, err) 39 | 40 | err = writeCredentials(rootDir, "my-robot-user", "my-robot-password") 41 | require.NoError(t, err) 42 | 43 | wantAuth := base64.StdEncoding.EncodeToString([]byte("my-robot-user:my-robot-password")) 44 | 45 | mux.HandleFunc("/robot/server", func(w http.ResponseWriter, r *http.Request) { 46 | header := r.Header.Get("Authorization") 47 | require.Equal(t, "Basic "+wantAuth, header) 48 | fmt.Println(header) 49 | json.NewEncoder(w).Encode([]models.ServerResponse{ 50 | { 51 | Server: models.Server{ 52 | ServerIP: "123.123.123.12", 53 | ServerIPv6Net: "2a01:f48:111:4221::", 54 | ServerNumber: 321, 55 | Name: "bm-server1", 56 | }, 57 | }, 58 | }) 59 | }) 60 | 61 | httpClient := server.Client() 62 | robotClient, err := NewCachedRobotClient(rootDir, httpClient, server.URL+"/robot") 63 | require.NoError(t, err) 64 | require.NotNil(t, robotClient) 65 | err = credentials.Watch(credentials.GetDirectory(rootDir), nil, robotClient) 66 | require.NoError(t, err) 67 | servers, err := robotClient.ServerGetList() 68 | require.NoError(t, err) 69 | require.Len(t, servers, 1) 70 | 71 | oldCount := credentials.GetRobotReloadCounter() 72 | err = writeCredentials(rootDir, "user2", "password2") 73 | require.NoError(t, err) 74 | start := time.Now() 75 | for { 76 | // if credentials.robotReloadCounter > oldCount { 77 | if credentials.GetRobotReloadCounter() > oldCount { 78 | break 79 | } 80 | if time.Since(start) > time.Second*3 { 81 | t.Fatal("timeout waiting for reload") 82 | } 83 | time.Sleep(time.Millisecond * 100) 84 | } 85 | 86 | wantAuth = base64.StdEncoding.EncodeToString([]byte("user2:password2")) 87 | servers, err = robotClient.ServerGetList() 88 | require.NoError(t, err) 89 | require.Len(t, servers, 1) 90 | } 91 | 92 | func writeCredentials(rootDir, user, password string) error { 93 | credentialsDir := credentials.GetDirectory(rootDir) 94 | newDir := filepath.Join(credentialsDir, "..dataNew") 95 | if err := os.MkdirAll(newDir, 0o700); err != nil { 96 | return err 97 | } 98 | err := os.WriteFile(filepath.Join(newDir, "robot-user"), 99 | []byte(user), 0o600) 100 | if err != nil { 101 | return err 102 | } 103 | 104 | err = os.WriteFile(filepath.Join(newDir, "robot-password"), 105 | []byte(password), 0o600) 106 | if err != nil { 107 | return err 108 | } 109 | targetDir := filepath.Join(credentialsDir, "..data") 110 | if err := os.RemoveAll(targetDir); err != nil { 111 | return err 112 | } 113 | if err := os.Rename(newDir, targetDir); err != nil { 114 | return err 115 | } 116 | return nil 117 | } 118 | -------------------------------------------------------------------------------- /internal/robot/client/interface.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import "github.com/syself/hrobot-go/models" 4 | 5 | type Client interface { 6 | ServerGet(id int) (*models.Server, error) 7 | ServerGetList() ([]models.Server, error) 8 | SetCredentials(username, password string) error 9 | } 10 | -------------------------------------------------------------------------------- /internal/testsupport/tls.go: -------------------------------------------------------------------------------- 1 | package testsupport 2 | 3 | // Copied and adapted from https://github.com/jsha/minica rev 4 | // e81e95a9e94be80e9da3c1e2b55accdd4884128b which cannot be used as a library. 5 | 6 | import ( 7 | "bytes" 8 | "crypto" 9 | "crypto/rand" 10 | "crypto/rsa" 11 | "crypto/sha1" 12 | "crypto/x509" 13 | "crypto/x509/pkix" 14 | "encoding/asn1" 15 | "encoding/hex" 16 | "encoding/pem" 17 | "math" 18 | "math/big" 19 | "testing" 20 | "time" 21 | ) 22 | 23 | const rsaBits = 2048 24 | 25 | type issuer struct { 26 | key crypto.Signer 27 | cert *x509.Certificate 28 | } 29 | 30 | func makeIssuer(t *testing.T) *issuer { 31 | const op = "testsupport/makeIssuer" 32 | t.Helper() 33 | 34 | key, err := rsa.GenerateKey(rand.Reader, rsaBits) 35 | if err != nil { 36 | t.Fatalf("%s: %v", op, err) 37 | } 38 | cert, err := makeRootCert(key) 39 | if err != nil { 40 | t.Fatalf("%s: %v", op, err) 41 | } 42 | return &issuer{key: key, cert: cert} 43 | } 44 | 45 | func makeKey(t *testing.T) *rsa.PrivateKey { 46 | const op = "testsupport/makeKey" 47 | t.Helper() 48 | 49 | key, err := rsa.GenerateKey(rand.Reader, 2048) 50 | if err != nil { 51 | t.Fatalf("%s: %v", op, err) 52 | } 53 | return key 54 | } 55 | 56 | func makeRootCert(key crypto.Signer) (*x509.Certificate, error) { 57 | serial, err := rand.Int(rand.Reader, big.NewInt(math.MaxInt64)) 58 | if err != nil { 59 | return nil, err 60 | } 61 | skid, err := calculateSKID(key.Public()) 62 | if err != nil { 63 | return nil, err 64 | } 65 | template := &x509.Certificate{ 66 | Subject: pkix.Name{ 67 | CommonName: "minica root ca " + hex.EncodeToString(serial.Bytes()[:3]), 68 | }, 69 | SerialNumber: serial, 70 | NotBefore: time.Now(), 71 | NotAfter: time.Now().AddDate(100, 0, 0), 72 | 73 | SubjectKeyId: skid, 74 | AuthorityKeyId: skid, 75 | KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, 76 | ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, 77 | BasicConstraintsValid: true, 78 | IsCA: true, 79 | MaxPathLenZero: true, 80 | } 81 | 82 | der, err := x509.CreateCertificate(rand.Reader, template, template, key.Public(), key) 83 | if err != nil { 84 | return nil, err 85 | } 86 | return x509.ParseCertificate(der) 87 | } 88 | 89 | func calculateSKID(pubKey crypto.PublicKey) ([]byte, error) { 90 | spkiASN1, err := x509.MarshalPKIXPublicKey(pubKey) 91 | if err != nil { 92 | return nil, err 93 | } 94 | 95 | var spki struct { 96 | Algorithm pkix.AlgorithmIdentifier 97 | SubjectPublicKey asn1.BitString 98 | } 99 | _, err = asn1.Unmarshal(spkiASN1, &spki) 100 | if err != nil { 101 | return nil, err 102 | } 103 | skid := sha1.Sum(spki.SubjectPublicKey.Bytes) 104 | return skid[:], nil 105 | } 106 | 107 | func sign(t *testing.T, iss *issuer, cn string) (*rsa.PrivateKey, *x509.Certificate) { 108 | const op = "testsupport/sign" 109 | t.Helper() 110 | 111 | if cn == "" { 112 | t.Fatalf("%s: no common name", op) 113 | } 114 | 115 | key := makeKey(t) 116 | serial, err := rand.Int(rand.Reader, big.NewInt(math.MaxInt64)) 117 | if err != nil { 118 | t.Fatalf("%s: %v", op, err) 119 | } 120 | 121 | template := &x509.Certificate{ 122 | Subject: pkix.Name{ 123 | CommonName: cn, 124 | }, 125 | SerialNumber: serial, 126 | NotBefore: time.Now(), 127 | NotAfter: time.Now().AddDate(0, 0, 30), 128 | 129 | KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment, 130 | ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, 131 | BasicConstraintsValid: true, 132 | IsCA: false, 133 | } 134 | der, err := x509.CreateCertificate(rand.Reader, template, iss.cert, key.Public(), iss.key) 135 | if err != nil { 136 | t.Fatalf("%s: %v", op, err) 137 | } 138 | cert, err := x509.ParseCertificate(der) 139 | if err != nil { 140 | t.Fatalf("%s: %v", op, err) 141 | } 142 | return key, cert 143 | } 144 | 145 | func encodePrivateKey(t *testing.T, key *rsa.PrivateKey) string { 146 | const op = "testsupport/encodePrivateKey" 147 | var buf bytes.Buffer 148 | 149 | t.Helper() 150 | 151 | bs := x509.MarshalPKCS1PrivateKey(key) 152 | err := pem.Encode(&buf, &pem.Block{ 153 | Type: "RSA PRIVATE KEY", 154 | Bytes: bs, 155 | }) 156 | if err != nil { 157 | t.Fatalf("%s: %v", op, err) 158 | } 159 | return buf.String() 160 | } 161 | 162 | func encodeCert(t *testing.T, cert *x509.Certificate) string { 163 | const op = "testsupport/encodeCert" 164 | var buf bytes.Buffer 165 | t.Helper() 166 | 167 | err := pem.Encode(&buf, &pem.Block{ 168 | Type: "CERTIFICATE", 169 | Bytes: cert.Raw, 170 | }) 171 | if err != nil { 172 | t.Fatalf("%s: %v", op, err) 173 | } 174 | return buf.String() 175 | } 176 | 177 | // TLSPair represents a TLS key and certificate pair. 178 | type TLSPair struct { 179 | Key string 180 | Cert string 181 | } 182 | 183 | // NewTLSPair generates new pair of TLS certificate and private key for the 184 | // passed common name. 185 | func NewTLSPair(t *testing.T, cn string) *TLSPair { 186 | t.Helper() 187 | 188 | iss := makeIssuer(t) 189 | key, cert := sign(t, iss, cn) 190 | return &TLSPair{ 191 | Key: encodePrivateKey(t, key), 192 | Cert: encodeCert(t, cert), 193 | } 194 | } 195 | -------------------------------------------------------------------------------- /internal/util/util.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "time" 7 | ) 8 | 9 | // GetEnvDuration returns the duration parsed from the environment variable with the given key and a potential error 10 | // parsing the var. Returns false if the env var is unset. 11 | func GetEnvDuration(key string) (time.Duration, error) { 12 | v := os.Getenv(key) 13 | if v == "" { 14 | return 0, nil 15 | } 16 | 17 | b, err := time.ParseDuration(v) 18 | if err != nil { 19 | return 0, fmt.Errorf("%s: %v", key, err) 20 | } 21 | 22 | return b, nil 23 | } 24 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // This file should be written by each cloud provider. 18 | // For an minimal working example, please refer to k8s.io/cloud-provider/sample/basic_main.go 19 | // For more details, please refer to k8s.io/kubernetes/cmd/cloud-controller-manager/main.go 20 | 21 | package main 22 | 23 | import ( 24 | "os" 25 | 26 | "github.com/spf13/pflag" 27 | _ "github.com/syself/hetzner-cloud-controller-manager/hcloud" 28 | "k8s.io/apimachinery/pkg/util/wait" 29 | cloudprovider "k8s.io/cloud-provider" 30 | "k8s.io/cloud-provider/app" 31 | "k8s.io/cloud-provider/app/config" 32 | "k8s.io/cloud-provider/names" 33 | "k8s.io/cloud-provider/options" 34 | cliflag "k8s.io/component-base/cli/flag" 35 | "k8s.io/component-base/logs" 36 | _ "k8s.io/component-base/metrics/prometheus/clientgo" 37 | _ "k8s.io/component-base/metrics/prometheus/version" 38 | "k8s.io/klog/v2" 39 | ) 40 | 41 | func main() { 42 | ccmOptions, err := options.NewCloudControllerManagerOptions() 43 | if err != nil { 44 | klog.Fatalf("unable to initialize command options: %v", err) 45 | } 46 | 47 | fss := cliflag.NamedFlagSets{} 48 | command := app.NewCloudControllerManagerCommand(ccmOptions, cloudInitializer, app.DefaultInitFuncConstructors, names.CCMControllerAliases(), fss, wait.NeverStop) 49 | 50 | pflag.CommandLine.SetNormalizeFunc(cliflag.WordSepNormalizeFunc) 51 | logs.InitLogs() 52 | defer logs.FlushLogs() 53 | 54 | if err := command.Execute(); err != nil { 55 | logs.FlushLogs() 56 | os.Exit(1) //nolint:gocritic 57 | } 58 | } 59 | 60 | func cloudInitializer(config *config.CompletedConfig) cloudprovider.Interface { 61 | cloudConfig := config.ComponentConfig.KubeCloudShared.CloudProvider 62 | 63 | // initialize cloud provider with the cloud provider name and config file provided 64 | cloud, err := cloudprovider.InitCloudProvider(cloudConfig.Name, cloudConfig.CloudConfigFile) 65 | if err != nil { 66 | klog.Fatalf("Cloud provider could not be initialized: %v", err) 67 | } 68 | if cloud == nil { 69 | klog.Fatalf("Cloud provider is nil") 70 | } 71 | 72 | if !cloud.HasClusterID() { 73 | if config.ComponentConfig.KubeCloudShared.AllowUntaggedCloud { 74 | klog.Warning("detected a cluster without a ClusterID. A ClusterID will be required in the future. Please tag your cluster to avoid any future issues") 75 | } else { 76 | klog.Fatalf("no ClusterID found. A ClusterID is required for the cloud provider to function properly. This check can be bypassed by setting the allow-untagged-cloud option") 77 | } 78 | } 79 | return cloud 80 | } 81 | --------------------------------------------------------------------------------