├── .conform.yaml ├── .dockerignore ├── .github ├── ISSUE_TEMPLATE │ ├── BUG_REPORT.md │ └── FEATURE_REQUEST.md ├── PULL_REQUEST_TEMPLATE.md ├── dependabot.yml └── workflows │ ├── build-edge.yaml │ ├── build-test.yaml │ ├── charts.yaml │ ├── conform.yaml │ ├── release-charts.yaml │ ├── release-pre.yaml │ ├── release.yaml │ └── stale.yaml ├── .gitignore ├── .golangci.yml ├── CHANGELOG.md ├── CONTRIBUTING.md ├── Dockerfile ├── LICENSE ├── Makefile ├── OWNERS ├── README.md ├── charts └── proxmox-cloud-controller-manager │ ├── .helmignore │ ├── Chart.yaml │ ├── README.md │ ├── README.md.gotmpl │ ├── ci │ └── values.yaml │ ├── icon.png │ ├── templates │ ├── NOTES.txt │ ├── _helpers.tpl │ ├── deployment.yaml │ ├── role.yaml │ ├── rolebinding.yaml │ ├── secrets.yaml │ └── serviceaccount.yaml │ ├── values.edge.yaml │ ├── values.talos.yaml │ └── values.yaml ├── cmd └── proxmox-cloud-controller-manager │ └── main.go ├── docs ├── cosign.md ├── deploy │ ├── cloud-controller-manager-daemonset.yml │ ├── cloud-controller-manager-talos.yml │ └── cloud-controller-manager.yml ├── faq.md ├── install.md ├── loadbalancer.md ├── metrics.md └── release.md ├── go.mod ├── go.sum ├── hack ├── CHANGELOG.tpl.md ├── chglog-config.yml ├── ct.yml └── proxmox-config.yaml └── pkg ├── cluster ├── client.go ├── client_test.go ├── cloud_config.go └── cloud_config_test.go ├── metrics ├── metrics.go └── metrics_api.go ├── provider ├── provider.go └── provider_test.go └── proxmox ├── cloud.go ├── cloud_test.go ├── instances.go └── instances_test.go /.conform.yaml: -------------------------------------------------------------------------------- 1 | policies: 2 | - type: commit 3 | spec: 4 | header: 5 | length: 89 6 | imperative: true 7 | case: lower 8 | invalidLastCharacters: . 9 | body: 10 | required: true 11 | dco: true 12 | gpg: false 13 | spellcheck: 14 | locale: US 15 | maximumOfOneCommit: false 16 | conventional: 17 | types: 18 | - build 19 | - chore 20 | - ci 21 | - docs 22 | - perf 23 | - refactor 24 | - revert 25 | - style 26 | - test 27 | scopes: 28 | - deps 29 | - main 30 | - chart 31 | descriptionLength: 72 32 | - type: license 33 | spec: 34 | skipPaths: 35 | - .git/ 36 | includeSuffixes: 37 | - .go 38 | excludeSuffixes: 39 | - .pb.go 40 | allowPrecedingComments: false 41 | header: | 42 | /* 43 | Copyright 2023 The Kubernetes Authors. 44 | 45 | Licensed under the Apache License, Version 2.0 (the "License"); 46 | you may not use this file except in compliance with the License. 47 | You may obtain a copy of the License at 48 | 49 | http://www.apache.org/licenses/LICENSE-2.0 50 | 51 | Unless required by applicable law or agreed to in writing, software 52 | distributed under the License is distributed on an "AS IS" BASIS, 53 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 54 | See the License for the specific language governing permissions and 55 | limitations under the License. 56 | */ 57 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | .vscode/ 2 | .github/ 3 | .git/ 4 | **/.gitignore 5 | # 6 | bin/ 7 | charts/ 8 | docs/ 9 | hack/ 10 | Dockerfile 11 | 12 | # other 13 | *.md 14 | *.yml 15 | *.zip 16 | *.sql 17 | 18 | # cosign 19 | /cosign.key 20 | /cosign.pub 21 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/BUG_REPORT.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug Report 3 | about: Report a bug. 4 | title: "" 5 | labels: "" 6 | assignees: "" 7 | --- 8 | 9 | ## Bug Report 10 | 11 | ### Description 12 | 13 | ### Logs 14 | 15 | ### Environment 16 | 17 | - Plugin version: 18 | - Kubernetes version: [`kubectl version --short`] 19 | - Node describe: [`kubectl describe node `] 20 | - OS version [`cat /etc/os-release`] 21 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/FEATURE_REQUEST.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature Requests 3 | about: Create a feature request. 4 | title: "" 5 | labels: "" 6 | assignees: "" 7 | --- 8 | 9 | ## Feature Request 10 | 11 | ### Description 12 | 13 | ### Community Note 14 | 15 | * Please vote on this issue by adding a 👍 reaction to the original issue to help the community and maintainers prioritize this request 16 | * Please do not leave "+1" or other comments that do not add relevant new information or questions, they generate extra noise for issue followers and do not help prioritize the request 17 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | # Pull Request 2 | 3 | 9 | 10 | ## What? (description) 11 | 12 | ## Why? (reasoning) 13 | 14 | ## Acceptance 15 | 16 | Please use the following checklist: 17 | 18 | - [ ] you linked an issue (if applicable) 19 | - [ ] you included tests (if applicable) 20 | - [ ] you linted your code (`make lint`) 21 | - [ ] you linted your code (`make unit`) 22 | 23 | > See `make help` for a description of the available targets. 24 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | # See https://docs.github.com/en/github/administering-a-repository/configuration-options-for-dependency-updates 4 | 5 | version: 2 6 | updates: 7 | - package-ecosystem: "github-actions" 8 | directory: "/" 9 | commit-message: 10 | prefix: "chore:" 11 | open-pull-requests-limit: 8 12 | rebase-strategy: disabled 13 | schedule: 14 | interval: "monthly" 15 | day: "monday" 16 | time: "08:00" 17 | timezone: "UTC" 18 | 19 | - package-ecosystem: "gomod" 20 | directory: "/" 21 | commit-message: 22 | prefix: "chore:" 23 | open-pull-requests-limit: 8 24 | rebase-strategy: disabled 25 | schedule: 26 | interval: "monthly" 27 | day: "monday" 28 | time: "07:00" 29 | timezone: "UTC" 30 | groups: 31 | k8s.io: 32 | patterns: 33 | - "k8s.io/api" 34 | - "k8s.io/apimachinery" 35 | - "k8s.io/apiserver" 36 | - "k8s.io/client-go" 37 | - "k8s.io/cloud-provider" 38 | - "k8s.io/component-base" 39 | - "k8s.io/component-helpers" 40 | - "k8s.io/controller-manager" 41 | 42 | - package-ecosystem: "docker" 43 | directory: "/" 44 | commit-message: 45 | prefix: "chore:" 46 | open-pull-requests-limit: 8 47 | rebase-strategy: disabled 48 | schedule: 49 | interval: "monthly" 50 | day: "monday" 51 | time: "07:00" 52 | timezone: "UTC" 53 | -------------------------------------------------------------------------------- /.github/workflows/build-edge.yaml: -------------------------------------------------------------------------------- 1 | name: Build edge 2 | 3 | on: 4 | workflow_dispatch: 5 | push: 6 | branches: 7 | - main 8 | paths: 9 | - 'go.mod' 10 | - 'go.sum' 11 | - 'cmd/**' 12 | - 'pkg/**' 13 | - 'Dockerfile' 14 | 15 | jobs: 16 | build-publish: 17 | name: "Build image and publish" 18 | timeout-minutes: 15 19 | runs-on: ubuntu-latest 20 | permissions: 21 | contents: read 22 | packages: write 23 | id-token: write 24 | steps: 25 | - name: Checkout 26 | uses: actions/checkout@v4 27 | with: 28 | ref: main 29 | - name: Unshallow 30 | run: git fetch --prune --unshallow 31 | 32 | - name: Install Cosign 33 | uses: sigstore/cosign-installer@v3.8.2 34 | - name: Set up QEMU 35 | uses: docker/setup-qemu-action@v3 36 | with: 37 | platforms: arm64 38 | - name: Set up docker buildx 39 | uses: docker/setup-buildx-action@v3 40 | 41 | - name: Github registry login 42 | uses: docker/login-action@v3 43 | with: 44 | registry: ghcr.io 45 | username: ${{ github.actor }} 46 | password: ${{ secrets.GITHUB_TOKEN }} 47 | 48 | - name: Build and push 49 | timeout-minutes: 10 50 | run: make images 51 | env: 52 | USERNAME: ${{ github.repository_owner }} 53 | PUSH: "true" 54 | TAG: "edge" 55 | - name: Sign images 56 | timeout-minutes: 4 57 | run: make images-cosign 58 | env: 59 | USERNAME: ${{ github.repository_owner }} 60 | TAG: "edge" 61 | -------------------------------------------------------------------------------- /.github/workflows/build-test.yaml: -------------------------------------------------------------------------------- 1 | name: Build check 2 | 3 | on: 4 | pull_request: 5 | branches: 6 | - main 7 | paths: 8 | - 'go.mod' 9 | - 'go.sum' 10 | - 'cmd/**' 11 | - 'pkg/**' 12 | - 'Dockerfile' 13 | 14 | jobs: 15 | build: 16 | name: Build 17 | timeout-minutes: 15 18 | runs-on: ubuntu-latest 19 | permissions: 20 | contents: read 21 | steps: 22 | - name: Checkout 23 | uses: actions/checkout@v4 24 | 25 | - name: Set up go 26 | timeout-minutes: 5 27 | uses: actions/setup-go@v5 28 | with: 29 | go-version-file: 'go.mod' 30 | 31 | - name: Lint 32 | uses: golangci/golangci-lint-action@v7 33 | with: 34 | version: v2.0.2 35 | args: --timeout=5m --config=.golangci.yml 36 | - name: Unit 37 | run: make unit 38 | - name: Build 39 | timeout-minutes: 10 40 | run: make build 41 | -------------------------------------------------------------------------------- /.github/workflows/charts.yaml: -------------------------------------------------------------------------------- 1 | name: Helm chart check 2 | 3 | on: 4 | pull_request: 5 | branches: 6 | - main 7 | paths: 8 | - 'charts/**' 9 | 10 | jobs: 11 | helm-lint: 12 | name: Helm chart check 13 | timeout-minutes: 5 14 | runs-on: ubuntu-latest 15 | steps: 16 | - name: Checkout 17 | uses: actions/checkout@v4 18 | - name: Unshallow 19 | run: git fetch --prune --unshallow 20 | 21 | - name: Install chart-testing tools 22 | id: lint 23 | uses: helm/chart-testing-action@v2.7.0 24 | 25 | - name: Run helm chart linter 26 | run: ct --config hack/ct.yml lint 27 | - name: Run helm template 28 | run: make helm-unit 29 | -------------------------------------------------------------------------------- /.github/workflows/conform.yaml: -------------------------------------------------------------------------------- 1 | name: Conformance check 2 | 3 | on: 4 | pull_request: 5 | branches: 6 | - main 7 | 8 | jobs: 9 | conform: 10 | name: Conformance 11 | timeout-minutes: 5 12 | runs-on: ubuntu-latest 13 | steps: 14 | - name: Checkout 15 | uses: actions/checkout@v4 16 | with: 17 | fetch-depth: 0 18 | ref: ${{ github.event.pull_request.head.sha }} 19 | - name: Checkout main branch 20 | run: git fetch --no-tags origin main:main 21 | 22 | - name: Conform action 23 | uses: talos-systems/conform@v0.1.0-alpha.30 24 | -------------------------------------------------------------------------------- /.github/workflows/release-charts.yaml: -------------------------------------------------------------------------------- 1 | name: HelmChart Release 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | paths: 8 | - 'charts/**' 9 | 10 | jobs: 11 | build-publish: 12 | name: "Publish helm chart" 13 | timeout-minutes: 10 14 | runs-on: ubuntu-latest 15 | permissions: 16 | contents: read 17 | packages: write 18 | id-token: write 19 | steps: 20 | - name: Checkout 21 | uses: actions/checkout@v4 22 | with: 23 | fetch-depth: 0 24 | 25 | - name: Install Helm 26 | uses: azure/setup-helm@v4 27 | with: 28 | version: v3.13.3 29 | - name: Install Cosign 30 | uses: sigstore/cosign-installer@v3.8.2 31 | 32 | - name: Github registry login 33 | uses: docker/login-action@v3 34 | with: 35 | registry: ghcr.io 36 | username: ${{ github.actor }} 37 | password: ${{ secrets.GITHUB_TOKEN }} 38 | 39 | - name: Helm release 40 | timeout-minutes: 5 41 | run: make helm-login helm-release 42 | env: 43 | HELM_TOKEN: ${{ secrets.GITHUB_TOKEN }} 44 | -------------------------------------------------------------------------------- /.github/workflows/release-pre.yaml: -------------------------------------------------------------------------------- 1 | name: Release check 2 | 3 | on: 4 | pull_request: 5 | branches: 6 | - main 7 | 8 | jobs: 9 | build-publish: 10 | name: "Check release docs" 11 | runs-on: ubuntu-latest 12 | if: startsWith(github.head_ref, 'release-') 13 | permissions: 14 | contents: read 15 | packages: write 16 | steps: 17 | - name: Checkout 18 | uses: actions/checkout@v4 19 | - name: Unshallow 20 | run: git fetch --prune --unshallow 21 | 22 | - name: Release version 23 | shell: bash 24 | id: release 25 | run: | 26 | echo "TAG=v${GITHUB_HEAD_REF:8}" >> "$GITHUB_ENV" 27 | 28 | - name: Helm docs 29 | uses: gabe565/setup-helm-docs-action@v1 30 | 31 | - name: Generate 32 | run: make docs 33 | - name: Check 34 | run: git diff --exit-code 35 | -------------------------------------------------------------------------------- /.github/workflows/release.yaml: -------------------------------------------------------------------------------- 1 | name: Release 2 | 3 | on: 4 | push: 5 | tags: 6 | - 'v*' 7 | 8 | jobs: 9 | build-publish: 10 | name: "Build image and publish" 11 | timeout-minutes: 15 12 | runs-on: ubuntu-latest 13 | permissions: 14 | contents: read 15 | packages: write 16 | id-token: write 17 | steps: 18 | - name: Checkout 19 | uses: actions/checkout@v4 20 | - name: Unshallow 21 | run: git fetch --prune --unshallow 22 | 23 | - name: Install Cosign 24 | uses: sigstore/cosign-installer@v3.8.2 25 | - name: Set up QEMU 26 | uses: docker/setup-qemu-action@v3 27 | with: 28 | platforms: arm64 29 | - name: Set up docker buildx 30 | uses: docker/setup-buildx-action@v3 31 | 32 | - name: Github registry login 33 | uses: docker/login-action@v3 34 | with: 35 | registry: ghcr.io 36 | username: ${{ github.actor }} 37 | password: ${{ secrets.GITHUB_TOKEN }} 38 | 39 | - name: Build and push edge 40 | timeout-minutes: 10 41 | run: make images 42 | env: 43 | PUSH: "true" 44 | TAG: "edge" 45 | - name: Sign images 46 | timeout-minutes: 4 47 | run: make images-cosign 48 | env: 49 | TAG: "edge" 50 | 51 | - name: Build and push 52 | timeout-minutes: 10 53 | run: make images 54 | env: 55 | PUSH: "true" 56 | - name: Sign images 57 | timeout-minutes: 4 58 | run: make images-cosign 59 | -------------------------------------------------------------------------------- /.github/workflows/stale.yaml: -------------------------------------------------------------------------------- 1 | name: Close stale issues 2 | 3 | on: 4 | schedule: 5 | - cron: '30 8 * * *' 6 | 7 | jobs: 8 | stale: 9 | name: Check stale issues 10 | runs-on: ubuntu-latest 11 | permissions: 12 | issues: write 13 | pull-requests: write 14 | steps: 15 | - uses: actions/stale@v9 16 | with: 17 | stale-issue-message: This issue is stale because it has been open 180 days with no activity. Remove stale label or comment or this will be closed in 14 days. 18 | close-issue-message: This issue was closed because it has been stalled for 14 days with no activity. 19 | days-before-issue-stale: 180 20 | days-before-issue-close: 14 21 | days-before-pr-close: -1 # never close PRs 22 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # 2 | /bin/ 3 | /charts/proxmox-cloud-controller-manager/values-dev.yaml 4 | /proxmox-cloud-controller-manager* 5 | /kubeconfig 6 | /kubeconfig* 7 | /proxmox-config.yaml 8 | # 9 | 10 | # cosign 11 | /cosign.key 12 | /cosign.pub 13 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | version: "2" 2 | run: 3 | build-tags: 4 | - integration 5 | - integration_api 6 | - integration_cli 7 | - integration_k8s 8 | - integration_provision 9 | issues-exit-code: 1 10 | tests: true 11 | output: 12 | formats: 13 | text: 14 | path: stdout 15 | print-linter-name: true 16 | print-issued-lines: true 17 | colors: false 18 | linters: 19 | default: all 20 | disable: 21 | - depguard 22 | - errorlint 23 | - exhaustruct 24 | - err113 25 | - forbidigo 26 | - forcetypeassert 27 | - funlen 28 | - gochecknoglobals 29 | - gochecknoinits 30 | - gocognit 31 | - godox 32 | - godot 33 | - gosec 34 | - inamedparam 35 | - ireturn 36 | - maintidx 37 | - mnd 38 | - musttag 39 | - nakedret 40 | - nestif 41 | - nilnil 42 | - nolintlint 43 | - nonamedreturns 44 | - paralleltest 45 | - perfsprint 46 | - promlinter 47 | - protogetter 48 | - recvcheck 49 | - tagalign 50 | - tagliatelle 51 | - testifylint 52 | - testpackage 53 | - thelper 54 | - varnamelen 55 | - wrapcheck 56 | 57 | # temporarily disabled linters 58 | - copyloopvar 59 | - intrange 60 | settings: 61 | cyclop: 62 | max-complexity: 30 63 | dupl: 64 | threshold: 100 65 | errcheck: 66 | check-type-assertions: false 67 | check-blank: true 68 | exclude-functions: 69 | - fmt.Fprintln 70 | - fmt.Fprintf 71 | - fmt.Fprint 72 | goconst: 73 | min-len: 3 74 | min-occurrences: 3 75 | gocyclo: 76 | min-complexity: 30 77 | gomoddirectives: 78 | replace-local: true 79 | replace-allow-list: [] 80 | retract-allow-no-explanation: false 81 | exclude-forbidden: true 82 | lll: 83 | line-length: 200 84 | tab-width: 1 85 | misspell: 86 | locale: US 87 | nolintlint: 88 | require-explanation: false 89 | require-specific: true 90 | allow-unused: false 91 | prealloc: 92 | simple: true 93 | range-loops: true 94 | for-loops: false 95 | staticcheck: 96 | checks: 97 | [ 98 | "all", 99 | "-ST1000", 100 | "-ST1003", 101 | "-ST1016", 102 | "-ST1020", 103 | "-ST1021", 104 | "-ST1022", 105 | "-QF1001", 106 | "-QF1008", 107 | ] 108 | unused: 109 | local-variables-are-used: false 110 | issues: 111 | max-issues-per-linter: 0 112 | max-same-issues: 0 113 | uniq-by-line: true 114 | new: false 115 | formatters: 116 | enable: 117 | - gci 118 | - gofmt 119 | - gofumpt 120 | - goimports 121 | settings: 122 | gci: 123 | sections: 124 | - standard # Captures all standard packages if they do not match another section. 125 | - default # Contains all imports that could not be matched to another section type. 126 | - prefix(github.com/sergelogvinov) # Groups all imports with the specified Prefix. 127 | - prefix(k8s.io) # Groups all imports with the specified Prefix. 128 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## [v0.8.0](https://github.com/sergelogvinov/proxmox-cloud-controller-manager/compare/v0.7.0...v0.8.0) (2025-04-12) 4 | 5 | Welcome to the v0.8.0 release of Kubernetes cloud controller manager for Proxmox! 6 | 7 | ### Bug Fixes 8 | 9 | - find node by name 10 | 11 | ### Features 12 | 13 | - custom instance type 14 | - **chart:** extra envs values 15 | 16 | ### Changelog 17 | 18 | * 646d776 feat(chart): extra envs values 19 | * 19e1f44 chore: bump deps 20 | * 0f0374c feat: custom instance type 21 | * 3a34fb9 fix: find node by name 22 | * 8a2f518 chore: bump deps 23 | * ca452ad chore: bump deps 24 | 25 | 26 | ## [v0.7.0](https://github.com/sergelogvinov/proxmox-cloud-controller-manager/compare/v0.6.0...v0.7.0) (2025-01-08) 27 | 28 | Welcome to the v0.7.0 release of Kubernetes cloud controller manager for Proxmox! 29 | 30 | ### Features 31 | 32 | - enable support for capmox This makes ccm compatible with cluster api and cluster api provider proxmox (capmox) 33 | 34 | ### Changelog 35 | 36 | * bb868bc chore: release v0.7.0 37 | * 956a30a feat: enable support for capmox This makes ccm compatible with cluster api and cluster api provider proxmox (capmox) 38 | 39 | 40 | ## [v0.6.0](https://github.com/sergelogvinov/proxmox-cloud-controller-manager/compare/v0.5.1...v0.6.0) (2025-01-01) 41 | 42 | Welcome to the v0.6.0 release of Kubernetes cloud controller manager for Proxmox! 43 | 44 | ### Changelog 45 | 46 | * 63eef87 chore: release v0.6.0 47 | * 710dc1b chore: bump deps 48 | * 5ea7b73 chore: bump deps 49 | * 2bfb088 chore: bump deps 50 | * 87baa50 docs: add faq 51 | * 7ec2617 docs: install 52 | * 64fc662 docs: kubelet flags 53 | 54 | 55 | ## [v0.5.1](https://github.com/sergelogvinov/proxmox-cloud-controller-manager/compare/v0.5.0...v0.5.1) (2024-09-23) 56 | 57 | Welcome to the v0.5.1 release of Kubernetes cloud controller manager for Proxmox! 58 | 59 | ### Bug Fixes 60 | 61 | - instance type 62 | 63 | ### Changelog 64 | 65 | * b3767b5 chore: release v0.5.1 66 | * 10f3e36 fix: instance type 67 | * 2b64352 chore(chart): update readme 68 | 69 | 70 | ## [v0.5.0](https://github.com/sergelogvinov/proxmox-cloud-controller-manager/compare/v0.4.2...v0.5.0) (2024-09-16) 71 | 72 | Welcome to the v0.5.0 release of Kubernetes cloud controller manager for Proxmox! 73 | 74 | ### Features 75 | 76 | - find node by uuid 77 | - prometheus metrics 78 | 79 | ### Changelog 80 | 81 | * 63b6907 chore: release v0.5.0 82 | * 4d79e4e docs: install instruction 83 | * 5876cd4 feat: find node by uuid 84 | * b81ad14 feat: prometheus metrics 85 | * e31b24c refactor: contextual logging 86 | * e1e5263 chore: bump deps 87 | 88 | 89 | ## [v0.4.2](https://github.com/sergelogvinov/proxmox-cloud-controller-manager/compare/v0.4.1...v0.4.2) (2024-05-04) 90 | 91 | Welcome to the v0.4.2 release of Kubernetes cloud controller manager for Proxmox! 92 | 93 | ### Changelog 94 | 95 | * 76dae87 chore: release v0.4.2 96 | 97 | 98 | ## [v0.4.1](https://github.com/sergelogvinov/proxmox-cloud-controller-manager/compare/v0.4.0...v0.4.1) (2024-05-04) 99 | 100 | Welcome to the v0.4.1 release of Kubernetes cloud controller manager for Proxmox! 101 | 102 | ### Features 103 | 104 | - **chart:** add daemonset mode 105 | - **chart:** add hostAliases and initContainers 106 | 107 | ### Changelog 108 | 109 | * c02bc2f chore: release v0.4.1 110 | * ce92b3e feat(chart): add daemonset mode 111 | * 4771769 chore: bump deps 112 | * 12d2858 ci: update multi arch build init 113 | * 3c7cd44 ci: update multi arch build init 114 | * 36757fc ci: update multi arch build init 115 | * c1ab34c chore: bump deps 116 | * d1e6e70 docs: update helm install command 117 | * 9ba9ff2 feat(chart): add hostAliases and initContainers 118 | 119 | 120 | ## [v0.4.0](https://github.com/sergelogvinov/proxmox-cloud-controller-manager/compare/v0.3.0...v0.4.0) (2024-02-16) 121 | 122 | Welcome to the v0.4.0 release of Kubernetes cloud controller manager for Proxmox! 123 | 124 | ### Bug Fixes 125 | 126 | - init provider 127 | 128 | ### Features 129 | 130 | - kubelet dualstack support 131 | 132 | ### Changelog 133 | 134 | * 677e6cc chore: release v0.4.0 135 | * a752d10 feat: kubelet dualstack support 136 | * de55986 fix: init provider 137 | * 10592d1 chore: bump deps 138 | * 7b73b5f refactor: move providerID to the package 139 | 140 | 141 | ## [v0.3.0](https://github.com/sergelogvinov/proxmox-cloud-controller-manager/compare/v0.2.0...v0.3.0) (2024-01-03) 142 | 143 | Welcome to the v0.3.0 release of Kubernetes cloud controller manager for Proxmox! 144 | 145 | ### Bug Fixes 146 | 147 | - namespace for extension-apiserver-authentication rolebinding 148 | 149 | ### Features 150 | 151 | - can use user/password 152 | - **chart:** add extraVolumes + extraVolumeMounts 153 | 154 | ### Changelog 155 | 156 | * 6f0c667 chore: release v0.3.0 157 | * ac2f564 feat: can use user/password 158 | * 41a7f8d chore: bump deps 159 | * 74d8c78 chore: bump deps 160 | * a76b7c2 chore: replace nodeSelector with nodeAffinity in chart + manifests 161 | * 93d8edc chore: bump deps 162 | * 4f7aaeb chore: bump deps 163 | * eef9c9c chore: bump deps 164 | * d54368e feat(chart): add extraVolumes + extraVolumeMounts 165 | * 3a3c070 chore: bump deps 166 | * 5c1a382 fix: namespace for extension-apiserver-authentication rolebinding 167 | * 75ead90 chore: bump deps 168 | 169 | 170 | ## [v0.2.0](https://github.com/sergelogvinov/proxmox-cloud-controller-manager/compare/v0.1.1...v0.2.0) (2023-09-20) 171 | 172 | Welcome to the v0.2.0 release of Kubernetes cloud controller manager for Proxmox! 173 | 174 | ### Features 175 | 176 | - cosign images 177 | - helm oci release 178 | 179 | ### Changelog 180 | 181 | * d2da2e8 chore: release v0.2.0 182 | * 4e641a1 chore: bump deps 183 | * 591b88d chore: bump actions/checkout from 3 to 4 184 | * 45e3aeb chore: bump sigstore/cosign-installer from 3.1.1 to 3.1.2 185 | * 8076eee chore: bump github actions deps 186 | * bc879ab feat: cosign images 187 | * abd63a2 chore: bump deps 188 | * f8d1712 feat: helm oci release 189 | * dfd7c5f chore: bump deps 190 | * 38da18f ci: fix git tag 191 | * d8c6bed chore: bump deps 192 | 193 | 194 | ## [v0.1.1](https://github.com/sergelogvinov/proxmox-cloud-controller-manager/compare/v0.1.0...v0.1.1) (2023-05-12) 195 | 196 | Welcome to the v0.1.1 release of Kubernetes cloud controller manager for Proxmox! 197 | 198 | ### Changelog 199 | 200 | * 6d79605 chore: release v0.1.1 201 | * f8c32e1 test: cloud config 202 | * c051d38 ci: build trigger 203 | * a1e7cd0 chore: bump deps 204 | * f813f30 ci: add git version 205 | 206 | 207 | ## [v0.1.0](https://github.com/sergelogvinov/proxmox-cloud-controller-manager/compare/v0.0.1...v0.1.0) (2023-05-07) 208 | 209 | Welcome to the v0.1.0 release of Kubernetes cloud controller manager for Proxmox! 210 | 211 | ### Changelog 212 | 213 | * 3796b9a chore: release v0.1.0 214 | * 2fb410d docs: update readme 215 | * fb96218 test: more tests 216 | * b776e54 test: mock proxmox api 217 | * 641509b doc: helm chart readme 218 | * 90b66dc test: basic test 219 | 220 | 221 | ## v0.0.1 (2023-04-29) 222 | 223 | Welcome to the v0.0.1 release of Kubernetes cloud controller manager for Proxmox! 224 | 225 | ### Features 226 | 227 | - add controllers 228 | 229 | ### Changelog 230 | 231 | * bf10985 chore: release v0.0.1 232 | * 0d89bf5 ci: add github checks 233 | * cc2dc17 refactor: proxmox cloud config 234 | * 850dcd4 chore: bump deps 235 | * 0173d67 doc: update readme 236 | * 5677ba3 doc: deploy 237 | * d99a5f0 doc: update 238 | * 8212493 feat: add controllers 239 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | ## Developer Certificate of Origin 4 | 5 | All commits require a [DCO](https://developercertificate.org/) sign-off. 6 | This is done by committing with the `--signoff` flag. 7 | 8 | ## Development 9 | 10 | The build process for this project is designed to run entirely in containers. 11 | To get started, run `make help` and follow the instructions. 12 | 13 | ## Conformance 14 | 15 | To verify conformance status, run `make conformance`. 16 | This runs a series of tests on the working tree and is required to pass before a contribution is accepted. 17 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # syntax = docker/dockerfile:1.15 2 | ######################################## 3 | 4 | FROM --platform=${BUILDPLATFORM} golang:1.24.2-alpine AS builder 5 | RUN apk update && apk add --no-cache make 6 | ENV GO111MODULE=on 7 | WORKDIR /src 8 | 9 | COPY go.mod go.sum /src 10 | RUN go mod download && go mod verify 11 | 12 | COPY . . 13 | ARG VERSION 14 | ARG TAG 15 | ARG SHA 16 | RUN make build-all-archs 17 | 18 | ######################################## 19 | 20 | FROM --platform=${TARGETARCH} scratch AS release 21 | LABEL org.opencontainers.image.source="https://github.com/sergelogvinov/proxmox-cloud-controller-manager" \ 22 | org.opencontainers.image.licenses="Apache-2.0" \ 23 | org.opencontainers.image.description="Proxmox VE CCM for Kubernetes" 24 | 25 | COPY --from=gcr.io/distroless/static-debian12:nonroot . . 26 | ARG TARGETARCH 27 | COPY --from=builder /src/bin/proxmox-cloud-controller-manager-${TARGETARCH} /bin/proxmox-cloud-controller-manager 28 | 29 | ENTRYPOINT ["/bin/proxmox-cloud-controller-manager"] 30 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | REGISTRY ?= ghcr.io 2 | USERNAME ?= sergelogvinov 3 | PROJECT ?= proxmox-cloud-controller-manager 4 | IMAGE ?= $(REGISTRY)/$(USERNAME)/$(PROJECT) 5 | HELMREPO ?= $(REGISTRY)/$(USERNAME)/charts 6 | PLATFORM ?= linux/arm64,linux/amd64 7 | PUSH ?= false 8 | 9 | VERSION ?= $(shell git describe --dirty --tag --match='v*') 10 | SHA ?= $(shell git describe --match=none --always --abbrev=7 --dirty) 11 | TAG ?= $(VERSION) 12 | 13 | GO_LDFLAGS := -s -w 14 | GO_LDFLAGS += -X k8s.io/component-base/version.gitVersion=$(VERSION) 15 | 16 | OS ?= $(shell go env GOOS) 17 | ARCH ?= $(shell go env GOARCH) 18 | ARCHS = amd64 arm64 19 | 20 | TESTARGS ?= "-v" 21 | 22 | BUILD_ARGS := --platform=$(PLATFORM) 23 | ifeq ($(PUSH),true) 24 | BUILD_ARGS += --push=$(PUSH) 25 | BUILD_ARGS += --output type=image,annotation-index.org.opencontainers.image.source="https://github.com/$(USERNAME)/$(PROJECT)",annotation-index.org.opencontainers.image.description="Proxmox VE CCM for Kubernetes" 26 | else 27 | BUILD_ARGS += --output type=docker 28 | endif 29 | 30 | COSING_ARGS ?= 31 | 32 | ############ 33 | 34 | # Help Menu 35 | 36 | define HELP_MENU_HEADER 37 | # Getting Started 38 | 39 | To build this project, you must have the following installed: 40 | 41 | - git 42 | - make 43 | - golang 1.20+ 44 | - golangci-lint 45 | 46 | endef 47 | 48 | export HELP_MENU_HEADER 49 | 50 | help: ## This help menu. 51 | @echo "$$HELP_MENU_HEADER" 52 | @grep -E '^[a-zA-Z0-9%_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' 53 | 54 | ############ 55 | # 56 | # Build Abstractions 57 | # 58 | 59 | build-all-archs: 60 | @for arch in $(ARCHS); do $(MAKE) ARCH=$${arch} build ; done 61 | 62 | .PHONY: clean 63 | clean: ## Clean 64 | rm -rf bin 65 | 66 | .PHONY: build 67 | build: ## Build 68 | CGO_ENABLED=0 GOOS=$(OS) GOARCH=$(ARCH) go build -ldflags "$(GO_LDFLAGS)" \ 69 | -o bin/proxmox-cloud-controller-manager-$(ARCH) ./cmd/proxmox-cloud-controller-manager 70 | 71 | .PHONY: run 72 | run: build ## Run 73 | ./bin/proxmox-cloud-controller-manager-$(ARCH) --v=5 --kubeconfig=kubeconfig --cloud-config=proxmox-config.yaml --controllers=cloud-node,cloud-node-lifecycle \ 74 | --use-service-account-credentials --leader-elect=false --bind-address=127.0.0.1 --authorization-always-allow-paths=/healthz,/livez,/readyz,/metrics 75 | 76 | .PHONY: lint 77 | lint: ## Lint Code 78 | golangci-lint run --config .golangci.yml 79 | 80 | .PHONY: unit 81 | unit: ## Unit Tests 82 | go test -tags=unit $(shell go list ./...) $(TESTARGS) 83 | 84 | ############ 85 | 86 | .PHONY: helm-unit 87 | helm-unit: ## Helm Unit Tests 88 | @helm lint charts/proxmox-cloud-controller-manager 89 | @helm template -f charts/proxmox-cloud-controller-manager/ci/values.yaml \ 90 | proxmox-cloud-controller-manager charts/proxmox-cloud-controller-manager >/dev/null 91 | 92 | .PHONY: helm-login 93 | helm-login: ## Helm Login 94 | @echo "${HELM_TOKEN}" | helm registry login $(REGISTRY) --username $(USERNAME) --password-stdin 95 | 96 | .PHONY: helm-release 97 | helm-release: ## Helm Release 98 | @rm -rf dist/ 99 | @helm package charts/proxmox-cloud-controller-manager -d dist 100 | @helm push dist/proxmox-cloud-controller-manager-*.tgz oci://$(HELMREPO) 2>&1 | tee dist/.digest 101 | @cosign sign --yes $(COSING_ARGS) $(HELMREPO)/proxmox-cloud-controller-manager@$$(cat dist/.digest | awk -F "[, ]+" '/Digest/{print $$NF}') 102 | 103 | ############ 104 | 105 | .PHONY: docs 106 | docs: 107 | yq -i '.appVersion = "$(TAG)"' charts/proxmox-cloud-controller-manager/Chart.yaml 108 | helm template -n kube-system proxmox-cloud-controller-manager \ 109 | -f charts/proxmox-cloud-controller-manager/values.edge.yaml \ 110 | --set-string image.tag=$(TAG) \ 111 | charts/proxmox-cloud-controller-manager > docs/deploy/cloud-controller-manager.yml 112 | helm template -n kube-system proxmox-cloud-controller-manager \ 113 | -f charts/proxmox-cloud-controller-manager/values.talos.yaml \ 114 | --set-string image.tag=$(TAG) \ 115 | charts/proxmox-cloud-controller-manager > docs/deploy/cloud-controller-manager-talos.yml 116 | helm template -n kube-system proxmox-cloud-controller-manager \ 117 | --set-string image.tag=$(TAG) \ 118 | --set useDaemonSet=true \ 119 | charts/proxmox-cloud-controller-manager > docs/deploy/cloud-controller-manager-daemonset.yml 120 | helm-docs --sort-values-order=file charts/proxmox-cloud-controller-manager 121 | 122 | release-update: 123 | git-chglog --config hack/chglog-config.yml -o CHANGELOG.md 124 | 125 | ############ 126 | # 127 | # Docker Abstractions 128 | # 129 | 130 | docker-init: 131 | @docker run --rm --privileged multiarch/qemu-user-static -p yes ||: 132 | 133 | @docker context create multiarch ||: 134 | @docker buildx create --name multiarch --driver docker-container --use ||: 135 | @docker context use multiarch 136 | @docker buildx inspect --bootstrap multiarch 137 | 138 | .PHONY: images 139 | images: ## Build images 140 | docker buildx build $(BUILD_ARGS) \ 141 | --build-arg VERSION="$(VERSION)" \ 142 | --build-arg TAG="$(TAG)" \ 143 | --build-arg SHA="$(SHA)" \ 144 | -t $(IMAGE):$(TAG) \ 145 | -f Dockerfile . 146 | 147 | .PHONY: images-checks 148 | images-checks: images 149 | trivy image --exit-code 1 --ignore-unfixed --severity HIGH,CRITICAL --no-progress $(IMAGE):$(TAG) 150 | 151 | .PHONY: images-cosign 152 | images-cosign: 153 | @cosign sign --yes $(COSING_ARGS) --recursive $(IMAGE):$(TAG) 154 | -------------------------------------------------------------------------------- /OWNERS: -------------------------------------------------------------------------------- 1 | approvers: 2 | - sergelogvinov 3 | reviewers: 4 | - sergelogvinov 5 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Kubernetes cloud controller manager for Proxmox 2 | 3 | To me, it seems like Proxmox is a bit old-fashioned when it comes to creating virtual machines. 4 | It doesn't have a lot of automation built-in, so you have to do a lot of things manually. 5 | Proxmox is a good option if you have a static infrastructure or don't create new virtual machines very often. 6 | 7 | I use Terraform to launch my Kubernetes nodes. 8 | However, when I need to scale down the cluster, I have to manually delete the corresponding node resource in Kubernetes. 9 | That's why I created the CCM (Cloud Controller Manager) for Proxmox. 10 | Originally, it was designed to work with [Talos CCM](https://github.com/siderolabs/talos-cloud-controller-manager), but it was not difficult to make it as standalone solution. 11 | 12 | The CCM does a few things: it initialises new nodes, applies common labels to them, and removes them when they're deleted. It also supports multiple clusters, meaning you can have one kubernetes cluster across multiple Proxmox clusters. 13 | 14 | The basic definitions: 15 | * kubernetes label `topology.kubernetes.io/region` is a Proxmox cluster `clusters[].region` 16 | * kubernetes label `topology.kubernetes.io/zone` is a hypervisor host machine name 17 | 18 | This makes it possible for me to use pods affinity/anti-affinity. 19 | 20 | ## Example 21 | 22 | ```yaml 23 | # cloud provider config 24 | clusters: 25 | - url: https://cluster-api-1.exmple.com:8006/api2/json 26 | insecure: false 27 | # Proxox auth token 28 | token_id: "user!token-id" 29 | token_secret: "secret" 30 | # Uniq region name 31 | region: cluster-1 32 | - url: https://cluster-api-2.exmple.com:8006/api2/json 33 | insecure: false 34 | token_id: "user!token-id" 35 | token_secret: "secret" 36 | region: cluster-2 37 | ``` 38 | 39 | Node spec result: 40 | 41 | ```yaml 42 | apiVersion: v1 43 | kind: Node 44 | metadata: 45 | labels: 46 | ... 47 | # Type generated base on CPU and RAM 48 | node.kubernetes.io/instance-type: 2VCPU-2GB 49 | # Proxmox cluster name as in the config 50 | topology.kubernetes.io/region: cluster-1 51 | # Proxmox hypervisor host machine name 52 | topology.kubernetes.io/zone: pve-node-1 53 | name: worker-1 54 | spec: 55 | ... 56 | # providerID - magic string: 57 | # cluster-1 - cluster name as in the config 58 | # 123 - Proxmox VM ID 59 | providerID: proxmox://cluster-1/123 60 | status: 61 | addresses: 62 | - address: 172.16.0.31 63 | type: InternalIP 64 | - address: worker-1 65 | type: Hostname 66 | ``` 67 | 68 | ## Install 69 | 70 | See [Install](docs/install.md) for installation instructions. 71 | 72 | ## Controllers 73 | 74 | Support controllers: 75 | 76 | * cloud-node 77 | * Updates node resource. 78 | * Assigns labels and taints based on Proxmox VM configuration. 79 | * cloud-node-lifecycle 80 | * Cleans up node resource when Proxmox VM is deleted. 81 | 82 | ## FAQ 83 | 84 | See [FAQ](docs/faq.md) for answers to common questions. 85 | 86 | ## Contributing 87 | 88 | Contributions are welcomed and appreciated! 89 | See [Contributing](CONTRIBUTING.md) for our guidelines. 90 | 91 | ## License 92 | 93 | Licensed under the Apache License, Version 2.0 (the "License"); 94 | you may not use this file except in compliance with the License. 95 | You may obtain a copy of the License at 96 | 97 | [http://www.apache.org/licenses/LICENSE-2.0](http://www.apache.org/licenses/LICENSE-2.0) 98 | 99 | Unless required by applicable law or agreed to in writing, software 100 | distributed under the License is distributed on an "AS IS" BASIS, 101 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 102 | See the License for the specific language governing permissions and 103 | limitations under the License. 104 | 105 | --- 106 | 107 | `Proxmox®` is a registered trademark of [Proxmox Server Solutions GmbH](https://www.proxmox.com/en/about/company). 108 | -------------------------------------------------------------------------------- /charts/proxmox-cloud-controller-manager/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *.orig 18 | *~ 19 | # Various IDEs 20 | .project 21 | .idea/ 22 | *.tmproj 23 | .vscode/ 24 | -------------------------------------------------------------------------------- /charts/proxmox-cloud-controller-manager/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: proxmox-cloud-controller-manager 3 | description: Cloud Controller Manager plugin for Proxmox 4 | type: application 5 | home: https://github.com/sergelogvinov/proxmox-cloud-controller-manager 6 | icon: https://raw.githubusercontent.com/sergelogvinov/proxmox-cloud-controller-manager/main/charts/proxmox-cloud-controller-manager/icon.png 7 | sources: 8 | - https://github.com/sergelogvinov/proxmox-cloud-controller-manager 9 | keywords: 10 | - ccm 11 | - proxmox 12 | - kubernetes 13 | maintainers: 14 | - name: sergelogvinov 15 | url: https://github.com/sergelogvinov 16 | # This is the chart version. This version number should be incremented each time you make changes 17 | # to the chart and its templates, including the app version. 18 | # Versions are expected to follow Semantic Versioning (https://semver.org/) 19 | version: 0.2.13 20 | # This is the version number of the application being deployed. This version number should be 21 | # incremented each time you make changes to the application. Versions are not expected to 22 | # follow Semantic Versioning. They should reflect the version the application is using. 23 | # It is recommended to use it with quotes. 24 | appVersion: v0.8.0 25 | -------------------------------------------------------------------------------- /charts/proxmox-cloud-controller-manager/README.md: -------------------------------------------------------------------------------- 1 | # proxmox-cloud-controller-manager 2 | 3 | ![Version: 0.2.13](https://img.shields.io/badge/Version-0.2.13-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v0.8.0](https://img.shields.io/badge/AppVersion-v0.8.0-informational?style=flat-square) 4 | 5 | Cloud Controller Manager plugin for Proxmox 6 | 7 | The Cloud Controller Manager (CCM) is responsible for managing node resources in cloud-based Kubernetes environments. 8 | 9 | Key functions of the Cloud Controller Manager: 10 | - `Node Management`: It manages nodes by initializing new nodes when they join the cluster (e.g., during scaling up) and removing nodes when they are no longer needed (e.g., during scaling down). 11 | - `Cloud-Specific Operations`: The CCM ensures that the cloud provider's API is integrated into the Kubernetes cluster to control and automate tasks like load balancing, storage provisioning, and node lifecycle management. 12 | 13 | **Homepage:** 14 | 15 | ## Maintainers 16 | 17 | | Name | Email | Url | 18 | | ---- | ------ | --- | 19 | | sergelogvinov | | | 20 | 21 | ## Source Code 22 | 23 | * 24 | 25 | ## Requirements 26 | 27 | You need to set `--cloud-provider=external` in the kubelet argument for all nodes in the cluster. 28 | 29 | ## Proxmox permissions 30 | 31 | ```shell 32 | # Create role CCM 33 | pveum role add CCM -privs "VM.Audit" 34 | # Create user and grant permissions 35 | pveum user add kubernetes@pve 36 | pveum aclmod / -user kubernetes@pve -role CCM 37 | pveum user token add kubernetes@pve ccm -privsep 0 38 | ``` 39 | 40 | ## Helm values example 41 | 42 | ```yaml 43 | # proxmox-ccm.yaml 44 | 45 | config: 46 | clusters: 47 | - url: https://cluster-api-1.exmple.com:8006/api2/json 48 | insecure: false 49 | token_id: "kubernetes@pve!csi" 50 | token_secret: "key" 51 | region: cluster-1 52 | 53 | enabledControllers: 54 | # Remove `cloud-node` if you use it with Talos CCM 55 | - cloud-node 56 | - cloud-node-lifecycle 57 | 58 | # Deploy CCM only on control-plane nodes 59 | affinity: 60 | nodeAffinity: 61 | requiredDuringSchedulingIgnoredDuringExecution: 62 | nodeSelectorTerms: 63 | - matchExpressions: 64 | - key: node-role.kubernetes.io/control-plane 65 | operator: Exists 66 | tolerations: 67 | - key: node-role.kubernetes.io/control-plane 68 | effect: NoSchedule 69 | ``` 70 | 71 | Deploy chart: 72 | 73 | ```shell 74 | helm upgrade -i --namespace=kube-system -f proxmox-ccm.yaml \ 75 | proxmox-cloud-controller-manager oci://ghcr.io/sergelogvinov/charts/proxmox-cloud-controller-manager 76 | ``` 77 | 78 | ## Values 79 | 80 | | Key | Type | Default | Description | 81 | |-----|------|---------|-------------| 82 | | replicaCount | int | `1` | | 83 | | image.repository | string | `"ghcr.io/sergelogvinov/proxmox-cloud-controller-manager"` | Proxmox CCM image. | 84 | | image.pullPolicy | string | `"IfNotPresent"` | Always or IfNotPresent | 85 | | image.tag | string | `""` | Overrides the image tag whose default is the chart appVersion. | 86 | | imagePullSecrets | list | `[]` | | 87 | | nameOverride | string | `""` | | 88 | | fullnameOverride | string | `""` | | 89 | | extraEnvs | list | `[]` | Any extra environments for talos-cloud-controller-manager | 90 | | extraArgs | list | `[]` | Any extra arguments for talos-cloud-controller-manager | 91 | | enabledControllers | list | `["cloud-node","cloud-node-lifecycle"]` | List of controllers should be enabled. Use '*' to enable all controllers. Support only `cloud-node,cloud-node-lifecycle` controllers. | 92 | | logVerbosityLevel | int | `2` | Log verbosity level. See https://github.com/kubernetes/community/blob/master/contributors/devel/sig-instrumentation/logging.md for description of individual verbosity levels. | 93 | | existingConfigSecret | string | `nil` | Proxmox cluster config stored in secrets. | 94 | | existingConfigSecretKey | string | `"config.yaml"` | Proxmox cluster config stored in secrets key. | 95 | | config | object | `{"clusters":[],"features":{"provider":"default"}}` | Proxmox cluster config. | 96 | | serviceAccount | object | `{"annotations":{},"create":true,"name":""}` | Pods Service Account. ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ | 97 | | priorityClassName | string | `"system-cluster-critical"` | CCM pods' priorityClassName. | 98 | | initContainers | list | `[]` | Add additional init containers to the CCM pods. ref: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ | 99 | | hostAliases | list | `[]` | hostAliases Deployment pod host aliases ref: https://kubernetes.io/docs/tasks/network/customize-hosts-file-for-pods/ | 100 | | podAnnotations | object | `{}` | Annotations for data pods. ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ | 101 | | podSecurityContext | object | `{"fsGroup":10258,"fsGroupChangePolicy":"OnRootMismatch","runAsGroup":10258,"runAsNonRoot":true,"runAsUser":10258}` | Pods Security Context. ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod | 102 | | securityContext | object | `{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"seccompProfile":{"type":"RuntimeDefault"}}` | Container Security Context. ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod | 103 | | resources | object | `{"requests":{"cpu":"10m","memory":"32Mi"}}` | Resource requests and limits. ref: https://kubernetes.io/docs/user-guide/compute-resources/ | 104 | | useDaemonSet | bool | `false` | Deploy CCM in Daemonset mode. CCM will use hostNetwork. It allows to use CCM without CNI plugins. | 105 | | updateStrategy | object | `{"rollingUpdate":{"maxUnavailable":1},"type":"RollingUpdate"}` | Deployment update strategy type. ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#updating-a-deployment | 106 | | nodeSelector | object | `{}` | Node labels for data pods assignment. ref: https://kubernetes.io/docs/user-guide/node-selection/ | 107 | | tolerations | list | `[{"effect":"NoSchedule","key":"node-role.kubernetes.io/control-plane","operator":"Exists"},{"effect":"NoSchedule","key":"node.cloudprovider.kubernetes.io/uninitialized","operator":"Exists"}]` | Tolerations for data pods assignment. ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ | 108 | | affinity | object | `{}` | Affinity for data pods assignment. ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity | 109 | | extraVolumes | list | `[]` | Additional volumes for Pods | 110 | | extraVolumeMounts | list | `[]` | Additional volume mounts for Pods | 111 | -------------------------------------------------------------------------------- /charts/proxmox-cloud-controller-manager/README.md.gotmpl: -------------------------------------------------------------------------------- 1 | {{ template "chart.header" . }} 2 | 3 | {{ template "chart.deprecationWarning" . }} 4 | 5 | {{ template "chart.badgesSection" . }} 6 | 7 | {{ template "chart.description" . }} 8 | 9 | The Cloud Controller Manager (CCM) is responsible for managing node resources in cloud-based Kubernetes environments. 10 | 11 | Key functions of the Cloud Controller Manager: 12 | - `Node Management`: It manages nodes by initializing new nodes when they join the cluster (e.g., during scaling up) and removing nodes when they are no longer needed (e.g., during scaling down). 13 | - `Cloud-Specific Operations`: The CCM ensures that the cloud provider's API is integrated into the Kubernetes cluster to control and automate tasks like load balancing, storage provisioning, and node lifecycle management. 14 | 15 | {{ template "chart.homepageLine" . }} 16 | 17 | {{ template "chart.maintainersSection" . }} 18 | 19 | {{ template "chart.sourcesSection" . }} 20 | 21 | {{ template "chart.requirementsSection" . }} 22 | 23 | ## Requirements 24 | 25 | You need to set `--cloud-provider=external` in the kubelet argument for all nodes in the cluster. 26 | 27 | ## Proxmox permissions 28 | 29 | ```shell 30 | # Create role CCM 31 | pveum role add CCM -privs "VM.Audit" 32 | # Create user and grant permissions 33 | pveum user add kubernetes@pve 34 | pveum aclmod / -user kubernetes@pve -role CCM 35 | pveum user token add kubernetes@pve ccm -privsep 0 36 | ``` 37 | 38 | ## Helm values example 39 | 40 | ```yaml 41 | # proxmox-ccm.yaml 42 | 43 | config: 44 | clusters: 45 | - url: https://cluster-api-1.exmple.com:8006/api2/json 46 | insecure: false 47 | token_id: "kubernetes@pve!csi" 48 | token_secret: "key" 49 | region: cluster-1 50 | 51 | enabledControllers: 52 | # Remove `cloud-node` if you use it with Talos CCM 53 | - cloud-node 54 | - cloud-node-lifecycle 55 | 56 | # Deploy CCM only on control-plane nodes 57 | affinity: 58 | nodeAffinity: 59 | requiredDuringSchedulingIgnoredDuringExecution: 60 | nodeSelectorTerms: 61 | - matchExpressions: 62 | - key: node-role.kubernetes.io/control-plane 63 | operator: Exists 64 | tolerations: 65 | - key: node-role.kubernetes.io/control-plane 66 | effect: NoSchedule 67 | ``` 68 | 69 | Deploy chart: 70 | 71 | ```shell 72 | helm upgrade -i --namespace=kube-system -f proxmox-ccm.yaml \ 73 | proxmox-cloud-controller-manager oci://ghcr.io/sergelogvinov/charts/proxmox-cloud-controller-manager 74 | ``` 75 | 76 | {{ template "chart.valuesSection" . }} 77 | -------------------------------------------------------------------------------- /charts/proxmox-cloud-controller-manager/ci/values.yaml: -------------------------------------------------------------------------------- 1 | image: 2 | repository: ghcr.io/sergelogvinov/proxmox-cloud-controller-manager 3 | pullPolicy: Always 4 | tag: edge 5 | 6 | affinity: 7 | nodeAffinity: 8 | requiredDuringSchedulingIgnoredDuringExecution: 9 | nodeSelectorTerms: 10 | - matchExpressions: 11 | - key: node-role.kubernetes.io/control-plane 12 | operator: Exists 13 | 14 | logVerbosityLevel: 4 15 | 16 | extraEnvs: 17 | - name: KUBERNETES_SERVICE_HOST 18 | value: 127.0.0.1 19 | 20 | enabledControllers: 21 | - cloud-node 22 | - cloud-node-lifecycle 23 | 24 | config: 25 | clusters: 26 | - url: https://cluster-api-1.exmple.com:8006/api2/json 27 | insecure: false 28 | token_id: "user!token-id" 29 | token_secret: "secret" 30 | region: cluster-1 31 | - url: https://cluster-api-2.exmple.com:8006/api2/json 32 | insecure: false 33 | token_id: "user!token-id" 34 | token_secret: "secret" 35 | region: cluster-2 36 | -------------------------------------------------------------------------------- /charts/proxmox-cloud-controller-manager/icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sergelogvinov/proxmox-cloud-controller-manager/efb753c9deea476da509fd2e4a1af9b238f70b5d/charts/proxmox-cloud-controller-manager/icon.png -------------------------------------------------------------------------------- /charts/proxmox-cloud-controller-manager/templates/NOTES.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sergelogvinov/proxmox-cloud-controller-manager/efb753c9deea476da509fd2e4a1af9b238f70b5d/charts/proxmox-cloud-controller-manager/templates/NOTES.txt -------------------------------------------------------------------------------- /charts/proxmox-cloud-controller-manager/templates/_helpers.tpl: -------------------------------------------------------------------------------- 1 | {{/* 2 | Expand the name of the chart. 3 | */}} 4 | {{- define "proxmox-cloud-controller-manager.name" -}} 5 | {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} 6 | {{- end }} 7 | 8 | {{/* 9 | Create a default fully qualified app name. 10 | We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). 11 | If release name contains chart name it will be used as a full name. 12 | */}} 13 | {{- define "proxmox-cloud-controller-manager.fullname" -}} 14 | {{- if .Values.fullnameOverride }} 15 | {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} 16 | {{- else }} 17 | {{- $name := default .Chart.Name .Values.nameOverride }} 18 | {{- if contains $name .Release.Name }} 19 | {{- .Release.Name | trunc 63 | trimSuffix "-" }} 20 | {{- else }} 21 | {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} 22 | {{- end }} 23 | {{- end }} 24 | {{- end }} 25 | 26 | {{/* 27 | Create chart name and version as used by the chart label. 28 | */}} 29 | {{- define "proxmox-cloud-controller-manager.chart" -}} 30 | {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} 31 | {{- end }} 32 | 33 | {{/* 34 | Common labels 35 | */}} 36 | {{- define "proxmox-cloud-controller-manager.labels" -}} 37 | helm.sh/chart: {{ include "proxmox-cloud-controller-manager.chart" . }} 38 | {{ include "proxmox-cloud-controller-manager.selectorLabels" . }} 39 | {{- if .Chart.AppVersion }} 40 | app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} 41 | {{- end }} 42 | app.kubernetes.io/managed-by: {{ .Release.Service }} 43 | {{- end }} 44 | 45 | {{/* 46 | Selector labels 47 | */}} 48 | {{- define "proxmox-cloud-controller-manager.selectorLabels" -}} 49 | app.kubernetes.io/name: {{ include "proxmox-cloud-controller-manager.name" . }} 50 | app.kubernetes.io/instance: {{ .Release.Name }} 51 | {{- end }} 52 | 53 | {{/* 54 | Create the name of the service account to use 55 | */}} 56 | {{- define "proxmox-cloud-controller-manager.serviceAccountName" -}} 57 | {{- if .Values.serviceAccount.create }} 58 | {{- default (include "proxmox-cloud-controller-manager.fullname" .) .Values.serviceAccount.name }} 59 | {{- else }} 60 | {{- default "default" .Values.serviceAccount.name }} 61 | {{- end }} 62 | {{- end }} 63 | 64 | {{/* 65 | Generate string of enabled controllers. Might have a trailing comma (,) which needs to be trimmed. 66 | */}} 67 | {{- define "proxmox-cloud-controller-manager.enabledControllers" }} 68 | {{- range .Values.enabledControllers -}}{{ . }},{{- end -}} 69 | {{- end }} 70 | -------------------------------------------------------------------------------- /charts/proxmox-cloud-controller-manager/templates/deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | {{- if .Values.useDaemonSet }} 3 | kind: DaemonSet 4 | {{- else }} 5 | kind: Deployment 6 | {{- end }} 7 | metadata: 8 | name: {{ include "proxmox-cloud-controller-manager.fullname" . }} 9 | labels: 10 | {{- include "proxmox-cloud-controller-manager.labels" . | nindent 4 }} 11 | namespace: {{ .Release.Namespace }} 12 | spec: 13 | {{- if not .Values.useDaemonSet }} 14 | replicas: {{ .Values.replicaCount }} 15 | strategy: 16 | type: {{ .Values.updateStrategy.type }} 17 | {{- else }} 18 | updateStrategy: 19 | type: {{ .Values.updateStrategy.type }} 20 | {{- end }} 21 | selector: 22 | matchLabels: 23 | {{- include "proxmox-cloud-controller-manager.selectorLabels" . | nindent 6 }} 24 | template: 25 | metadata: 26 | annotations: 27 | {{- if .Values.config }} 28 | checksum/config: {{ toJson .Values.config | sha256sum }} 29 | {{- end }} 30 | {{- with .Values.podAnnotations }} 31 | {{- toYaml . | nindent 8 }} 32 | {{- end }} 33 | labels: 34 | {{- include "proxmox-cloud-controller-manager.selectorLabels" . | nindent 8 }} 35 | spec: 36 | enableServiceLinks: false 37 | {{- if .Values.priorityClassName }} 38 | priorityClassName: {{ .Values.priorityClassName }} 39 | {{- end }} 40 | {{- with .Values.imagePullSecrets }} 41 | imagePullSecrets: 42 | {{- toYaml . | nindent 8 }} 43 | {{- end }} 44 | serviceAccountName: {{ include "proxmox-cloud-controller-manager.serviceAccountName" . }} 45 | securityContext: 46 | {{- toYaml .Values.podSecurityContext | nindent 8 }} 47 | {{- if .Values.useDaemonSet }} 48 | dnsPolicy: ClusterFirstWithHostNet 49 | hostNetwork: true 50 | {{- end }} 51 | {{- with .Values.hostAliases }} 52 | hostAliases: 53 | {{- toYaml . | nindent 8 }} 54 | {{- end }} 55 | initContainers: {{- toYaml .Values.initContainers | nindent 8 }} 56 | containers: 57 | - name: {{ .Chart.Name }} 58 | securityContext: 59 | {{- toYaml .Values.securityContext | nindent 12 }} 60 | image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" 61 | imagePullPolicy: {{ .Values.image.pullPolicy }} 62 | args: 63 | - --v={{ .Values.logVerbosityLevel }} 64 | - --cloud-provider=proxmox 65 | - --cloud-config=/etc/proxmox/config.yaml 66 | - --controllers={{- trimAll "," (include "proxmox-cloud-controller-manager.enabledControllers" . ) }} 67 | - --leader-elect-resource-name=cloud-controller-manager-proxmox 68 | - --use-service-account-credentials 69 | - --secure-port=10258 70 | - --authorization-always-allow-paths=/healthz,/livez,/readyz,/metrics 71 | {{- with .Values.extraArgs }} 72 | {{- toYaml . | nindent 12 }} 73 | {{- end }} 74 | {{- with .Values.extraEnvs }} 75 | env: 76 | {{- toYaml . | nindent 12 }} 77 | {{- end }} 78 | ports: 79 | - name: metrics 80 | containerPort: 10258 81 | protocol: TCP 82 | livenessProbe: 83 | httpGet: 84 | path: /healthz 85 | port: metrics 86 | scheme: HTTPS 87 | initialDelaySeconds: 20 88 | periodSeconds: 30 89 | timeoutSeconds: 5 90 | resources: 91 | {{- toYaml .Values.resources | nindent 12 }} 92 | volumeMounts: 93 | - name: cloud-config 94 | mountPath: /etc/proxmox 95 | readOnly: true 96 | {{- with .Values.extraVolumeMounts }} 97 | {{- toYaml . | nindent 12 }} 98 | {{- end }} 99 | {{- with .Values.nodeSelector }} 100 | nodeSelector: 101 | {{- toYaml . | nindent 8 }} 102 | {{- end }} 103 | affinity: 104 | {{- with .Values.affinity }} 105 | {{- toYaml . | nindent 8 }} 106 | {{- else }} 107 | podAntiAffinity: 108 | preferredDuringSchedulingIgnoredDuringExecution: 109 | - podAffinityTerm: 110 | labelSelector: 111 | matchLabels: 112 | {{- include "proxmox-cloud-controller-manager.selectorLabels" . | nindent 20 }} 113 | topologyKey: topology.kubernetes.io/zone 114 | weight: 1 115 | {{- end }} 116 | tolerations: 117 | {{- with .Values.tolerations }} 118 | {{- toYaml . | nindent 8 }} 119 | {{- end }} 120 | {{- if .Values.useDaemonSet }} 121 | - effect: NoSchedule 122 | key: node.kubernetes.io/not-ready 123 | operator: Exists 124 | {{- end }} 125 | {{- if not .Values.useDaemonSet }} 126 | topologySpreadConstraints: 127 | - maxSkew: 1 128 | topologyKey: kubernetes.io/hostname 129 | whenUnsatisfiable: DoNotSchedule 130 | labelSelector: 131 | matchLabels: 132 | {{- include "proxmox-cloud-controller-manager.selectorLabels" . | nindent 14 }} 133 | {{- end }} 134 | volumes: 135 | {{- if .Values.existingConfigSecret }} 136 | - name: cloud-config 137 | secret: 138 | secretName: {{ .Values.existingConfigSecret }} 139 | items: 140 | - key: {{ .Values.existingConfigSecretKey }} 141 | path: config.yaml 142 | defaultMode: 416 143 | {{- else }} 144 | - name: cloud-config 145 | secret: 146 | secretName: {{ include "proxmox-cloud-controller-manager.fullname" . }} 147 | defaultMode: 416 148 | {{- end }} 149 | {{- with .Values.extraVolumes }} 150 | {{- toYaml . | nindent 8 }} 151 | {{- end }} 152 | -------------------------------------------------------------------------------- /charts/proxmox-cloud-controller-manager/templates/role.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRole 3 | metadata: 4 | name: system:{{ include "proxmox-cloud-controller-manager.fullname" . }} 5 | labels: 6 | {{- include "proxmox-cloud-controller-manager.labels" . | nindent 4 }} 7 | rules: 8 | - apiGroups: 9 | - coordination.k8s.io 10 | resources: 11 | - leases 12 | verbs: 13 | - get 14 | - create 15 | - update 16 | - apiGroups: 17 | - "" 18 | resources: 19 | - events 20 | verbs: 21 | - create 22 | - patch 23 | - update 24 | - apiGroups: 25 | - "" 26 | resources: 27 | - nodes 28 | verbs: 29 | - get 30 | - list 31 | - watch 32 | - update 33 | - patch 34 | - delete 35 | - apiGroups: 36 | - "" 37 | resources: 38 | - nodes/status 39 | verbs: 40 | - patch 41 | - apiGroups: 42 | - "" 43 | resources: 44 | - serviceaccounts 45 | verbs: 46 | - create 47 | - get 48 | - apiGroups: 49 | - "" 50 | resources: 51 | - serviceaccounts/token 52 | verbs: 53 | - create 54 | -------------------------------------------------------------------------------- /charts/proxmox-cloud-controller-manager/templates/rolebinding.yaml: -------------------------------------------------------------------------------- 1 | kind: ClusterRoleBinding 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | metadata: 4 | name: system:{{ include "proxmox-cloud-controller-manager.fullname" . }} 5 | roleRef: 6 | apiGroup: rbac.authorization.k8s.io 7 | kind: ClusterRole 8 | name: system:{{ include "proxmox-cloud-controller-manager.fullname" . }} 9 | subjects: 10 | - kind: ServiceAccount 11 | name: {{ include "proxmox-cloud-controller-manager.fullname" . }} 12 | namespace: {{ .Release.Namespace }} 13 | --- 14 | apiVersion: rbac.authorization.k8s.io/v1 15 | kind: RoleBinding 16 | metadata: 17 | name: system:{{ include "proxmox-cloud-controller-manager.fullname" . }}:extension-apiserver-authentication-reader 18 | namespace: kube-system 19 | roleRef: 20 | apiGroup: rbac.authorization.k8s.io 21 | kind: Role 22 | name: extension-apiserver-authentication-reader 23 | subjects: 24 | - kind: ServiceAccount 25 | name: {{ include "proxmox-cloud-controller-manager.fullname" . }} 26 | namespace: {{ .Release.Namespace }} 27 | -------------------------------------------------------------------------------- /charts/proxmox-cloud-controller-manager/templates/secrets.yaml: -------------------------------------------------------------------------------- 1 | {{- if ne (len .Values.config.clusters) 0 }} 2 | apiVersion: v1 3 | kind: Secret 4 | metadata: 5 | name: {{ include "proxmox-cloud-controller-manager.fullname" . }} 6 | labels: 7 | {{- include "proxmox-cloud-controller-manager.labels" . | nindent 4 }} 8 | namespace: {{ .Release.Namespace }} 9 | data: 10 | config.yaml: {{ toYaml .Values.config | b64enc | quote }} 11 | {{- end }} 12 | -------------------------------------------------------------------------------- /charts/proxmox-cloud-controller-manager/templates/serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.serviceAccount.create -}} 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | name: {{ include "proxmox-cloud-controller-manager.serviceAccountName" . }} 6 | labels: 7 | {{- include "proxmox-cloud-controller-manager.labels" . | nindent 4 }} 8 | {{- with .Values.serviceAccount.annotations }} 9 | annotations: 10 | {{- toYaml . | nindent 4 }} 11 | {{- end }} 12 | namespace: {{ .Release.Namespace }} 13 | {{- end }} 14 | -------------------------------------------------------------------------------- /charts/proxmox-cloud-controller-manager/values.edge.yaml: -------------------------------------------------------------------------------- 1 | image: 2 | pullPolicy: Always 3 | tag: edge 4 | 5 | affinity: 6 | nodeAffinity: 7 | requiredDuringSchedulingIgnoredDuringExecution: 8 | nodeSelectorTerms: 9 | - matchExpressions: 10 | - key: node-role.kubernetes.io/control-plane 11 | operator: Exists 12 | 13 | logVerbosityLevel: 4 14 | 15 | enabledControllers: 16 | - cloud-node 17 | - cloud-node-lifecycle 18 | -------------------------------------------------------------------------------- /charts/proxmox-cloud-controller-manager/values.talos.yaml: -------------------------------------------------------------------------------- 1 | affinity: 2 | nodeAffinity: 3 | requiredDuringSchedulingIgnoredDuringExecution: 4 | nodeSelectorTerms: 5 | - matchExpressions: 6 | - key: node-role.kubernetes.io/control-plane 7 | operator: Exists 8 | 9 | logVerbosityLevel: 4 10 | 11 | enabledControllers: 12 | - cloud-node-lifecycle 13 | -------------------------------------------------------------------------------- /charts/proxmox-cloud-controller-manager/values.yaml: -------------------------------------------------------------------------------- 1 | # Default values for proxmox-cloud-controller-manager. 2 | # This is a YAML-formatted file. 3 | # Declare variables to be passed into your templates. 4 | 5 | replicaCount: 1 6 | 7 | image: 8 | # -- Proxmox CCM image. 9 | repository: ghcr.io/sergelogvinov/proxmox-cloud-controller-manager 10 | # -- Always or IfNotPresent 11 | pullPolicy: IfNotPresent 12 | # -- Overrides the image tag whose default is the chart appVersion. 13 | tag: "" 14 | 15 | imagePullSecrets: [] 16 | nameOverride: "" 17 | fullnameOverride: "" 18 | 19 | # -- Any extra environments for talos-cloud-controller-manager 20 | extraEnvs: 21 | [] 22 | # - name: KUBERNETES_SERVICE_HOST 23 | # value: 127.0.0.1 24 | 25 | # -- Any extra arguments for talos-cloud-controller-manager 26 | extraArgs: 27 | [] 28 | # - --cluster-name=kubernetes 29 | 30 | # -- List of controllers should be enabled. 31 | # Use '*' to enable all controllers. 32 | # Support only `cloud-node,cloud-node-lifecycle` controllers. 33 | enabledControllers: 34 | - cloud-node 35 | - cloud-node-lifecycle 36 | # - route 37 | # - service 38 | 39 | # -- Log verbosity level. See https://github.com/kubernetes/community/blob/master/contributors/devel/sig-instrumentation/logging.md 40 | # for description of individual verbosity levels. 41 | logVerbosityLevel: 2 42 | 43 | # -- Proxmox cluster config stored in secrets. 44 | existingConfigSecret: ~ 45 | # -- Proxmox cluster config stored in secrets key. 46 | existingConfigSecretKey: config.yaml 47 | 48 | # -- Proxmox cluster config. 49 | config: 50 | features: 51 | # specify provider: proxmox if you are using capmox (cluster api provider for proxmox) 52 | provider: "default" 53 | clusters: [] 54 | # - url: https://cluster-api-1.exmple.com:8006/api2/json 55 | # insecure: false 56 | # token_id: "login!name" 57 | # token_secret: "secret" 58 | # region: cluster-1 59 | 60 | # -- Pods Service Account. 61 | # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ 62 | serviceAccount: 63 | # Specifies whether a service account should be created 64 | create: true 65 | # Annotations to add to the service account 66 | annotations: {} 67 | # The name of the service account to use. 68 | # If not set and create is true, a name is generated using the fullname template 69 | name: "" 70 | 71 | # -- CCM pods' priorityClassName. 72 | priorityClassName: system-cluster-critical 73 | 74 | # -- Add additional init containers to the CCM pods. 75 | # ref: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ 76 | initContainers: 77 | [] 78 | # - name: loadbalancer 79 | # restartPolicy: Always 80 | # image: ghcr.io/sergelogvinov/haproxy:2.8.3-alpine3.18 81 | # imagePullPolicy: IfNotPresent 82 | # env: 83 | # - name: SVC 84 | # value: "proxmox.domain.com" 85 | # - name: PORT 86 | # value: "8006" 87 | # securityContext: 88 | # runAsUser: 99 89 | # runAsGroup: 99 90 | # resources: 91 | # limits: 92 | # cpu: 50m 93 | # memory: 64Mi 94 | # requests: 95 | # cpu: 50m 96 | # memory: 32Mi 97 | 98 | # -- hostAliases Deployment pod host aliases 99 | # ref: https://kubernetes.io/docs/tasks/network/customize-hosts-file-for-pods/ 100 | hostAliases: 101 | [] 102 | # - ip: 127.0.0.1 103 | # hostnames: 104 | # - proxmox.domain.com 105 | 106 | # -- Annotations for data pods. 107 | # ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ 108 | podAnnotations: {} 109 | 110 | # -- Pods Security Context. 111 | # ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod 112 | podSecurityContext: 113 | runAsNonRoot: true 114 | runAsUser: 10258 115 | runAsGroup: 10258 116 | fsGroup: 10258 117 | fsGroupChangePolicy: "OnRootMismatch" 118 | 119 | # -- Container Security Context. 120 | # ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod 121 | securityContext: 122 | allowPrivilegeEscalation: false 123 | capabilities: 124 | drop: 125 | - ALL 126 | seccompProfile: 127 | type: RuntimeDefault 128 | 129 | # -- Resource requests and limits. 130 | # ref: https://kubernetes.io/docs/user-guide/compute-resources/ 131 | resources: 132 | # We usually recommend not to specify default resources and to leave this as a conscious 133 | # choice for the user. This also increases chances charts run on environments with little 134 | # resources, such as Minikube. If you do want to specify resources, uncomment the following 135 | # lines, adjust them as necessary, and remove the curly braces after 'resources:'. 136 | # limits: 137 | # cpu: 100m 138 | # memory: 128Mi 139 | requests: 140 | cpu: 10m 141 | memory: 32Mi 142 | 143 | # -- Deploy CCM in Daemonset mode. 144 | # CCM will use hostNetwork. 145 | # It allows to use CCM without CNI plugins. 146 | useDaemonSet: false 147 | 148 | # -- Deployment update strategy type. 149 | # ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#updating-a-deployment 150 | updateStrategy: 151 | type: RollingUpdate 152 | rollingUpdate: 153 | maxUnavailable: 1 154 | 155 | # -- Node labels for data pods assignment. 156 | # ref: https://kubernetes.io/docs/user-guide/node-selection/ 157 | nodeSelector: 158 | {} 159 | # node-role.kubernetes.io/control-plane: "" 160 | 161 | # -- Tolerations for data pods assignment. 162 | # ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ 163 | tolerations: 164 | - effect: NoSchedule 165 | key: node-role.kubernetes.io/control-plane 166 | operator: Exists 167 | - effect: NoSchedule 168 | key: node.cloudprovider.kubernetes.io/uninitialized 169 | operator: Exists 170 | 171 | # -- Affinity for data pods assignment. 172 | # ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity 173 | affinity: {} 174 | # nodeAffinity: 175 | # requiredDuringSchedulingIgnoredDuringExecution: 176 | # nodeSelectorTerms: 177 | # - matchExpressions: 178 | # - key: node-role.kubernetes.io/control-plane 179 | # operator: Exists 180 | 181 | # -- Additional volumes for Pods 182 | extraVolumes: [] 183 | # - name: ca 184 | # secret: 185 | # secretName: my-ca 186 | # -- Additional volume mounts for Pods 187 | extraVolumeMounts: [] 188 | # - mountPath: /etc/ssl/certs/ca-certificates.crt 189 | # name: ca 190 | # subPath: ca.crt 191 | -------------------------------------------------------------------------------- /cmd/proxmox-cloud-controller-manager/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // This file should be written by each cloud provider. 18 | // For an minimal working example, please refer to k8s.io/cloud-provider/sample/basic_main.go 19 | // For more details, please refer to k8s.io/kubernetes/cmd/cloud-controller-manager/main.go 20 | 21 | // Package main provides the CCM implementation. 22 | package main 23 | 24 | import ( 25 | "os" 26 | 27 | "github.com/spf13/pflag" 28 | 29 | "github.com/sergelogvinov/proxmox-cloud-controller-manager/pkg/proxmox" 30 | 31 | "k8s.io/apimachinery/pkg/util/wait" 32 | cloudprovider "k8s.io/cloud-provider" 33 | "k8s.io/cloud-provider/app" 34 | "k8s.io/cloud-provider/app/config" 35 | "k8s.io/cloud-provider/names" 36 | "k8s.io/cloud-provider/options" 37 | "k8s.io/component-base/cli" 38 | cliflag "k8s.io/component-base/cli/flag" 39 | _ "k8s.io/component-base/metrics/prometheus/clientgo" // load all the prometheus client-go plugins 40 | _ "k8s.io/component-base/metrics/prometheus/version" // for version metric registration 41 | "k8s.io/klog/v2" 42 | ) 43 | 44 | func main() { 45 | ccmOptions, err := options.NewCloudControllerManagerOptions() 46 | if err != nil { 47 | klog.ErrorS(err, "unable to initialize command options") 48 | klog.FlushAndExit(klog.ExitFlushTimeout, 1) 49 | } 50 | 51 | fss := cliflag.NamedFlagSets{} 52 | command := app.NewCloudControllerManagerCommand(ccmOptions, cloudInitializer, app.DefaultInitFuncConstructors, names.CCMControllerAliases(), fss, wait.NeverStop) 53 | 54 | command.Flags().VisitAll(func(flag *pflag.Flag) { 55 | if flag.Name == "cloud-provider" { 56 | if err := flag.Value.Set(proxmox.ProviderName); err != nil { 57 | klog.ErrorS(err, "unable to set cloud-provider flag value") 58 | klog.FlushAndExit(klog.ExitFlushTimeout, 1) 59 | } 60 | } 61 | }) 62 | 63 | code := cli.Run(command) 64 | os.Exit(code) 65 | } 66 | 67 | func cloudInitializer(config *config.CompletedConfig) cloudprovider.Interface { 68 | cloudConfig := config.ComponentConfig.KubeCloudShared.CloudProvider 69 | 70 | // initialize cloud provider with the cloud provider name and config file provided 71 | cloud, err := cloudprovider.InitCloudProvider(cloudConfig.Name, cloudConfig.CloudConfigFile) 72 | if err != nil { 73 | klog.ErrorS(err, "Cloud provider could not be initialized") 74 | klog.FlushAndExit(klog.ExitFlushTimeout, 1) 75 | } 76 | 77 | if cloud == nil { 78 | klog.InfoS("Cloud provider is nil") 79 | klog.FlushAndExit(klog.ExitFlushTimeout, 1) 80 | } 81 | 82 | if !cloud.HasClusterID() { 83 | if config.ComponentConfig.KubeCloudShared.AllowUntaggedCloud { 84 | klog.InfoS("detected a cluster without a ClusterID. A ClusterID will be required in the future. Please tag your cluster to avoid any future issues") 85 | } else { 86 | klog.InfoS("no ClusterID found. A ClusterID is required for the cloud provider to function properly. This check can be bypassed by setting the allow-untagged-cloud option") 87 | klog.FlushAndExit(klog.ExitFlushTimeout, 1) 88 | } 89 | } 90 | 91 | return cloud 92 | } 93 | -------------------------------------------------------------------------------- /docs/cosign.md: -------------------------------------------------------------------------------- 1 | # Verify images 2 | 3 | We'll be employing [Cosing's](https://github.com/sigstore/cosign) keyless verifications to ensure that images were built in Github Actions. 4 | 5 | ## Verify Helm chart 6 | 7 | We will verify the keyless signature using the Cosign protocol. 8 | 9 | ```shell 10 | cosign verify ghcr.io/sergelogvinov/charts/proxmox-cloud-controller-manager:0.1.5 --certificate-identity https://github.com/sergelogvinov/proxmox-cloud-controller-manager/.github/workflows/release-charts.yaml@refs/heads/main --certificate-oidc-issuer https://token.actions.githubusercontent.com 11 | ``` 12 | 13 | ## Verify containers 14 | 15 | We will verify the keyless signature using the Cosign protocol. 16 | 17 | ```shell 18 | # Edge version 19 | cosign verify ghcr.io/sergelogvinov/proxmox-cloud-controller-manager:edge --certificate-identity https://github.com/sergelogvinov/proxmox-cloud-controller-manager/.github/workflows/build-edge.yaml@refs/heads/main --certificate-oidc-issuer https://token.actions.githubusercontent.com 20 | 21 | # Releases 22 | cosign verify ghcr.io/sergelogvinov/proxmox-cloud-controller-manager:v0.2.0 --certificate-identity https://github.com/sergelogvinov/proxmox-cloud-controller-manager/.github/workflows/release.yaml@refs/heads/main --certificate-oidc-issuer https://token.actions.githubusercontent.com 23 | ``` 24 | -------------------------------------------------------------------------------- /docs/deploy/cloud-controller-manager-daemonset.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Source: proxmox-cloud-controller-manager/templates/serviceaccount.yaml 3 | apiVersion: v1 4 | kind: ServiceAccount 5 | metadata: 6 | name: proxmox-cloud-controller-manager 7 | labels: 8 | helm.sh/chart: proxmox-cloud-controller-manager-0.2.13 9 | app.kubernetes.io/name: proxmox-cloud-controller-manager 10 | app.kubernetes.io/instance: proxmox-cloud-controller-manager 11 | app.kubernetes.io/version: "v0.8.0" 12 | app.kubernetes.io/managed-by: Helm 13 | namespace: kube-system 14 | --- 15 | # Source: proxmox-cloud-controller-manager/templates/role.yaml 16 | apiVersion: rbac.authorization.k8s.io/v1 17 | kind: ClusterRole 18 | metadata: 19 | name: system:proxmox-cloud-controller-manager 20 | labels: 21 | helm.sh/chart: proxmox-cloud-controller-manager-0.2.13 22 | app.kubernetes.io/name: proxmox-cloud-controller-manager 23 | app.kubernetes.io/instance: proxmox-cloud-controller-manager 24 | app.kubernetes.io/version: "v0.8.0" 25 | app.kubernetes.io/managed-by: Helm 26 | rules: 27 | - apiGroups: 28 | - coordination.k8s.io 29 | resources: 30 | - leases 31 | verbs: 32 | - get 33 | - create 34 | - update 35 | - apiGroups: 36 | - "" 37 | resources: 38 | - events 39 | verbs: 40 | - create 41 | - patch 42 | - update 43 | - apiGroups: 44 | - "" 45 | resources: 46 | - nodes 47 | verbs: 48 | - get 49 | - list 50 | - watch 51 | - update 52 | - patch 53 | - delete 54 | - apiGroups: 55 | - "" 56 | resources: 57 | - nodes/status 58 | verbs: 59 | - patch 60 | - apiGroups: 61 | - "" 62 | resources: 63 | - serviceaccounts 64 | verbs: 65 | - create 66 | - get 67 | - apiGroups: 68 | - "" 69 | resources: 70 | - serviceaccounts/token 71 | verbs: 72 | - create 73 | --- 74 | # Source: proxmox-cloud-controller-manager/templates/rolebinding.yaml 75 | kind: ClusterRoleBinding 76 | apiVersion: rbac.authorization.k8s.io/v1 77 | metadata: 78 | name: system:proxmox-cloud-controller-manager 79 | roleRef: 80 | apiGroup: rbac.authorization.k8s.io 81 | kind: ClusterRole 82 | name: system:proxmox-cloud-controller-manager 83 | subjects: 84 | - kind: ServiceAccount 85 | name: proxmox-cloud-controller-manager 86 | namespace: kube-system 87 | --- 88 | # Source: proxmox-cloud-controller-manager/templates/rolebinding.yaml 89 | apiVersion: rbac.authorization.k8s.io/v1 90 | kind: RoleBinding 91 | metadata: 92 | name: system:proxmox-cloud-controller-manager:extension-apiserver-authentication-reader 93 | namespace: kube-system 94 | roleRef: 95 | apiGroup: rbac.authorization.k8s.io 96 | kind: Role 97 | name: extension-apiserver-authentication-reader 98 | subjects: 99 | - kind: ServiceAccount 100 | name: proxmox-cloud-controller-manager 101 | namespace: kube-system 102 | --- 103 | # Source: proxmox-cloud-controller-manager/templates/deployment.yaml 104 | apiVersion: apps/v1 105 | kind: DaemonSet 106 | metadata: 107 | name: proxmox-cloud-controller-manager 108 | labels: 109 | helm.sh/chart: proxmox-cloud-controller-manager-0.2.13 110 | app.kubernetes.io/name: proxmox-cloud-controller-manager 111 | app.kubernetes.io/instance: proxmox-cloud-controller-manager 112 | app.kubernetes.io/version: "v0.8.0" 113 | app.kubernetes.io/managed-by: Helm 114 | namespace: kube-system 115 | spec: 116 | updateStrategy: 117 | type: RollingUpdate 118 | selector: 119 | matchLabels: 120 | app.kubernetes.io/name: proxmox-cloud-controller-manager 121 | app.kubernetes.io/instance: proxmox-cloud-controller-manager 122 | template: 123 | metadata: 124 | annotations: 125 | checksum/config: ce080eff0c26b50fe73bf9fcda017c8ad47c1000729fd0c555cfe3535c6d6222 126 | labels: 127 | app.kubernetes.io/name: proxmox-cloud-controller-manager 128 | app.kubernetes.io/instance: proxmox-cloud-controller-manager 129 | spec: 130 | enableServiceLinks: false 131 | priorityClassName: system-cluster-critical 132 | serviceAccountName: proxmox-cloud-controller-manager 133 | securityContext: 134 | fsGroup: 10258 135 | fsGroupChangePolicy: OnRootMismatch 136 | runAsGroup: 10258 137 | runAsNonRoot: true 138 | runAsUser: 10258 139 | dnsPolicy: ClusterFirstWithHostNet 140 | hostNetwork: true 141 | initContainers: 142 | [] 143 | containers: 144 | - name: proxmox-cloud-controller-manager 145 | securityContext: 146 | allowPrivilegeEscalation: false 147 | capabilities: 148 | drop: 149 | - ALL 150 | seccompProfile: 151 | type: RuntimeDefault 152 | image: "ghcr.io/sergelogvinov/proxmox-cloud-controller-manager:v0.8.0" 153 | imagePullPolicy: IfNotPresent 154 | args: 155 | - --v=2 156 | - --cloud-provider=proxmox 157 | - --cloud-config=/etc/proxmox/config.yaml 158 | - --controllers=cloud-node,cloud-node-lifecycle 159 | - --leader-elect-resource-name=cloud-controller-manager-proxmox 160 | - --use-service-account-credentials 161 | - --secure-port=10258 162 | - --authorization-always-allow-paths=/healthz,/livez,/readyz,/metrics 163 | ports: 164 | - name: metrics 165 | containerPort: 10258 166 | protocol: TCP 167 | livenessProbe: 168 | httpGet: 169 | path: /healthz 170 | port: metrics 171 | scheme: HTTPS 172 | initialDelaySeconds: 20 173 | periodSeconds: 30 174 | timeoutSeconds: 5 175 | resources: 176 | requests: 177 | cpu: 10m 178 | memory: 32Mi 179 | volumeMounts: 180 | - name: cloud-config 181 | mountPath: /etc/proxmox 182 | readOnly: true 183 | affinity: 184 | podAntiAffinity: 185 | preferredDuringSchedulingIgnoredDuringExecution: 186 | - podAffinityTerm: 187 | labelSelector: 188 | matchLabels: 189 | app.kubernetes.io/name: proxmox-cloud-controller-manager 190 | app.kubernetes.io/instance: proxmox-cloud-controller-manager 191 | topologyKey: topology.kubernetes.io/zone 192 | weight: 1 193 | tolerations: 194 | - effect: NoSchedule 195 | key: node-role.kubernetes.io/control-plane 196 | operator: Exists 197 | - effect: NoSchedule 198 | key: node.cloudprovider.kubernetes.io/uninitialized 199 | operator: Exists 200 | - effect: NoSchedule 201 | key: node.kubernetes.io/not-ready 202 | operator: Exists 203 | volumes: 204 | - name: cloud-config 205 | secret: 206 | secretName: proxmox-cloud-controller-manager 207 | defaultMode: 416 208 | -------------------------------------------------------------------------------- /docs/deploy/cloud-controller-manager-talos.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Source: proxmox-cloud-controller-manager/templates/serviceaccount.yaml 3 | apiVersion: v1 4 | kind: ServiceAccount 5 | metadata: 6 | name: proxmox-cloud-controller-manager 7 | labels: 8 | helm.sh/chart: proxmox-cloud-controller-manager-0.2.13 9 | app.kubernetes.io/name: proxmox-cloud-controller-manager 10 | app.kubernetes.io/instance: proxmox-cloud-controller-manager 11 | app.kubernetes.io/version: "v0.8.0" 12 | app.kubernetes.io/managed-by: Helm 13 | namespace: kube-system 14 | --- 15 | # Source: proxmox-cloud-controller-manager/templates/role.yaml 16 | apiVersion: rbac.authorization.k8s.io/v1 17 | kind: ClusterRole 18 | metadata: 19 | name: system:proxmox-cloud-controller-manager 20 | labels: 21 | helm.sh/chart: proxmox-cloud-controller-manager-0.2.13 22 | app.kubernetes.io/name: proxmox-cloud-controller-manager 23 | app.kubernetes.io/instance: proxmox-cloud-controller-manager 24 | app.kubernetes.io/version: "v0.8.0" 25 | app.kubernetes.io/managed-by: Helm 26 | rules: 27 | - apiGroups: 28 | - coordination.k8s.io 29 | resources: 30 | - leases 31 | verbs: 32 | - get 33 | - create 34 | - update 35 | - apiGroups: 36 | - "" 37 | resources: 38 | - events 39 | verbs: 40 | - create 41 | - patch 42 | - update 43 | - apiGroups: 44 | - "" 45 | resources: 46 | - nodes 47 | verbs: 48 | - get 49 | - list 50 | - watch 51 | - update 52 | - patch 53 | - delete 54 | - apiGroups: 55 | - "" 56 | resources: 57 | - nodes/status 58 | verbs: 59 | - patch 60 | - apiGroups: 61 | - "" 62 | resources: 63 | - serviceaccounts 64 | verbs: 65 | - create 66 | - get 67 | - apiGroups: 68 | - "" 69 | resources: 70 | - serviceaccounts/token 71 | verbs: 72 | - create 73 | --- 74 | # Source: proxmox-cloud-controller-manager/templates/rolebinding.yaml 75 | kind: ClusterRoleBinding 76 | apiVersion: rbac.authorization.k8s.io/v1 77 | metadata: 78 | name: system:proxmox-cloud-controller-manager 79 | roleRef: 80 | apiGroup: rbac.authorization.k8s.io 81 | kind: ClusterRole 82 | name: system:proxmox-cloud-controller-manager 83 | subjects: 84 | - kind: ServiceAccount 85 | name: proxmox-cloud-controller-manager 86 | namespace: kube-system 87 | --- 88 | # Source: proxmox-cloud-controller-manager/templates/rolebinding.yaml 89 | apiVersion: rbac.authorization.k8s.io/v1 90 | kind: RoleBinding 91 | metadata: 92 | name: system:proxmox-cloud-controller-manager:extension-apiserver-authentication-reader 93 | namespace: kube-system 94 | roleRef: 95 | apiGroup: rbac.authorization.k8s.io 96 | kind: Role 97 | name: extension-apiserver-authentication-reader 98 | subjects: 99 | - kind: ServiceAccount 100 | name: proxmox-cloud-controller-manager 101 | namespace: kube-system 102 | --- 103 | # Source: proxmox-cloud-controller-manager/templates/deployment.yaml 104 | apiVersion: apps/v1 105 | kind: Deployment 106 | metadata: 107 | name: proxmox-cloud-controller-manager 108 | labels: 109 | helm.sh/chart: proxmox-cloud-controller-manager-0.2.13 110 | app.kubernetes.io/name: proxmox-cloud-controller-manager 111 | app.kubernetes.io/instance: proxmox-cloud-controller-manager 112 | app.kubernetes.io/version: "v0.8.0" 113 | app.kubernetes.io/managed-by: Helm 114 | namespace: kube-system 115 | spec: 116 | replicas: 1 117 | strategy: 118 | type: RollingUpdate 119 | selector: 120 | matchLabels: 121 | app.kubernetes.io/name: proxmox-cloud-controller-manager 122 | app.kubernetes.io/instance: proxmox-cloud-controller-manager 123 | template: 124 | metadata: 125 | annotations: 126 | checksum/config: ce080eff0c26b50fe73bf9fcda017c8ad47c1000729fd0c555cfe3535c6d6222 127 | labels: 128 | app.kubernetes.io/name: proxmox-cloud-controller-manager 129 | app.kubernetes.io/instance: proxmox-cloud-controller-manager 130 | spec: 131 | enableServiceLinks: false 132 | priorityClassName: system-cluster-critical 133 | serviceAccountName: proxmox-cloud-controller-manager 134 | securityContext: 135 | fsGroup: 10258 136 | fsGroupChangePolicy: OnRootMismatch 137 | runAsGroup: 10258 138 | runAsNonRoot: true 139 | runAsUser: 10258 140 | initContainers: 141 | [] 142 | containers: 143 | - name: proxmox-cloud-controller-manager 144 | securityContext: 145 | allowPrivilegeEscalation: false 146 | capabilities: 147 | drop: 148 | - ALL 149 | seccompProfile: 150 | type: RuntimeDefault 151 | image: "ghcr.io/sergelogvinov/proxmox-cloud-controller-manager:v0.8.0" 152 | imagePullPolicy: IfNotPresent 153 | args: 154 | - --v=4 155 | - --cloud-provider=proxmox 156 | - --cloud-config=/etc/proxmox/config.yaml 157 | - --controllers=cloud-node-lifecycle 158 | - --leader-elect-resource-name=cloud-controller-manager-proxmox 159 | - --use-service-account-credentials 160 | - --secure-port=10258 161 | - --authorization-always-allow-paths=/healthz,/livez,/readyz,/metrics 162 | ports: 163 | - name: metrics 164 | containerPort: 10258 165 | protocol: TCP 166 | livenessProbe: 167 | httpGet: 168 | path: /healthz 169 | port: metrics 170 | scheme: HTTPS 171 | initialDelaySeconds: 20 172 | periodSeconds: 30 173 | timeoutSeconds: 5 174 | resources: 175 | requests: 176 | cpu: 10m 177 | memory: 32Mi 178 | volumeMounts: 179 | - name: cloud-config 180 | mountPath: /etc/proxmox 181 | readOnly: true 182 | affinity: 183 | nodeAffinity: 184 | requiredDuringSchedulingIgnoredDuringExecution: 185 | nodeSelectorTerms: 186 | - matchExpressions: 187 | - key: node-role.kubernetes.io/control-plane 188 | operator: Exists 189 | tolerations: 190 | - effect: NoSchedule 191 | key: node-role.kubernetes.io/control-plane 192 | operator: Exists 193 | - effect: NoSchedule 194 | key: node.cloudprovider.kubernetes.io/uninitialized 195 | operator: Exists 196 | topologySpreadConstraints: 197 | - maxSkew: 1 198 | topologyKey: kubernetes.io/hostname 199 | whenUnsatisfiable: DoNotSchedule 200 | labelSelector: 201 | matchLabels: 202 | app.kubernetes.io/name: proxmox-cloud-controller-manager 203 | app.kubernetes.io/instance: proxmox-cloud-controller-manager 204 | volumes: 205 | - name: cloud-config 206 | secret: 207 | secretName: proxmox-cloud-controller-manager 208 | defaultMode: 416 209 | -------------------------------------------------------------------------------- /docs/deploy/cloud-controller-manager.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Source: proxmox-cloud-controller-manager/templates/serviceaccount.yaml 3 | apiVersion: v1 4 | kind: ServiceAccount 5 | metadata: 6 | name: proxmox-cloud-controller-manager 7 | labels: 8 | helm.sh/chart: proxmox-cloud-controller-manager-0.2.13 9 | app.kubernetes.io/name: proxmox-cloud-controller-manager 10 | app.kubernetes.io/instance: proxmox-cloud-controller-manager 11 | app.kubernetes.io/version: "v0.8.0" 12 | app.kubernetes.io/managed-by: Helm 13 | namespace: kube-system 14 | --- 15 | # Source: proxmox-cloud-controller-manager/templates/role.yaml 16 | apiVersion: rbac.authorization.k8s.io/v1 17 | kind: ClusterRole 18 | metadata: 19 | name: system:proxmox-cloud-controller-manager 20 | labels: 21 | helm.sh/chart: proxmox-cloud-controller-manager-0.2.13 22 | app.kubernetes.io/name: proxmox-cloud-controller-manager 23 | app.kubernetes.io/instance: proxmox-cloud-controller-manager 24 | app.kubernetes.io/version: "v0.8.0" 25 | app.kubernetes.io/managed-by: Helm 26 | rules: 27 | - apiGroups: 28 | - coordination.k8s.io 29 | resources: 30 | - leases 31 | verbs: 32 | - get 33 | - create 34 | - update 35 | - apiGroups: 36 | - "" 37 | resources: 38 | - events 39 | verbs: 40 | - create 41 | - patch 42 | - update 43 | - apiGroups: 44 | - "" 45 | resources: 46 | - nodes 47 | verbs: 48 | - get 49 | - list 50 | - watch 51 | - update 52 | - patch 53 | - delete 54 | - apiGroups: 55 | - "" 56 | resources: 57 | - nodes/status 58 | verbs: 59 | - patch 60 | - apiGroups: 61 | - "" 62 | resources: 63 | - serviceaccounts 64 | verbs: 65 | - create 66 | - get 67 | - apiGroups: 68 | - "" 69 | resources: 70 | - serviceaccounts/token 71 | verbs: 72 | - create 73 | --- 74 | # Source: proxmox-cloud-controller-manager/templates/rolebinding.yaml 75 | kind: ClusterRoleBinding 76 | apiVersion: rbac.authorization.k8s.io/v1 77 | metadata: 78 | name: system:proxmox-cloud-controller-manager 79 | roleRef: 80 | apiGroup: rbac.authorization.k8s.io 81 | kind: ClusterRole 82 | name: system:proxmox-cloud-controller-manager 83 | subjects: 84 | - kind: ServiceAccount 85 | name: proxmox-cloud-controller-manager 86 | namespace: kube-system 87 | --- 88 | # Source: proxmox-cloud-controller-manager/templates/rolebinding.yaml 89 | apiVersion: rbac.authorization.k8s.io/v1 90 | kind: RoleBinding 91 | metadata: 92 | name: system:proxmox-cloud-controller-manager:extension-apiserver-authentication-reader 93 | namespace: kube-system 94 | roleRef: 95 | apiGroup: rbac.authorization.k8s.io 96 | kind: Role 97 | name: extension-apiserver-authentication-reader 98 | subjects: 99 | - kind: ServiceAccount 100 | name: proxmox-cloud-controller-manager 101 | namespace: kube-system 102 | --- 103 | # Source: proxmox-cloud-controller-manager/templates/deployment.yaml 104 | apiVersion: apps/v1 105 | kind: Deployment 106 | metadata: 107 | name: proxmox-cloud-controller-manager 108 | labels: 109 | helm.sh/chart: proxmox-cloud-controller-manager-0.2.13 110 | app.kubernetes.io/name: proxmox-cloud-controller-manager 111 | app.kubernetes.io/instance: proxmox-cloud-controller-manager 112 | app.kubernetes.io/version: "v0.8.0" 113 | app.kubernetes.io/managed-by: Helm 114 | namespace: kube-system 115 | spec: 116 | replicas: 1 117 | strategy: 118 | type: RollingUpdate 119 | selector: 120 | matchLabels: 121 | app.kubernetes.io/name: proxmox-cloud-controller-manager 122 | app.kubernetes.io/instance: proxmox-cloud-controller-manager 123 | template: 124 | metadata: 125 | annotations: 126 | checksum/config: ce080eff0c26b50fe73bf9fcda017c8ad47c1000729fd0c555cfe3535c6d6222 127 | labels: 128 | app.kubernetes.io/name: proxmox-cloud-controller-manager 129 | app.kubernetes.io/instance: proxmox-cloud-controller-manager 130 | spec: 131 | enableServiceLinks: false 132 | priorityClassName: system-cluster-critical 133 | serviceAccountName: proxmox-cloud-controller-manager 134 | securityContext: 135 | fsGroup: 10258 136 | fsGroupChangePolicy: OnRootMismatch 137 | runAsGroup: 10258 138 | runAsNonRoot: true 139 | runAsUser: 10258 140 | initContainers: 141 | [] 142 | containers: 143 | - name: proxmox-cloud-controller-manager 144 | securityContext: 145 | allowPrivilegeEscalation: false 146 | capabilities: 147 | drop: 148 | - ALL 149 | seccompProfile: 150 | type: RuntimeDefault 151 | image: "ghcr.io/sergelogvinov/proxmox-cloud-controller-manager:v0.8.0" 152 | imagePullPolicy: Always 153 | args: 154 | - --v=4 155 | - --cloud-provider=proxmox 156 | - --cloud-config=/etc/proxmox/config.yaml 157 | - --controllers=cloud-node,cloud-node-lifecycle 158 | - --leader-elect-resource-name=cloud-controller-manager-proxmox 159 | - --use-service-account-credentials 160 | - --secure-port=10258 161 | - --authorization-always-allow-paths=/healthz,/livez,/readyz,/metrics 162 | ports: 163 | - name: metrics 164 | containerPort: 10258 165 | protocol: TCP 166 | livenessProbe: 167 | httpGet: 168 | path: /healthz 169 | port: metrics 170 | scheme: HTTPS 171 | initialDelaySeconds: 20 172 | periodSeconds: 30 173 | timeoutSeconds: 5 174 | resources: 175 | requests: 176 | cpu: 10m 177 | memory: 32Mi 178 | volumeMounts: 179 | - name: cloud-config 180 | mountPath: /etc/proxmox 181 | readOnly: true 182 | affinity: 183 | nodeAffinity: 184 | requiredDuringSchedulingIgnoredDuringExecution: 185 | nodeSelectorTerms: 186 | - matchExpressions: 187 | - key: node-role.kubernetes.io/control-plane 188 | operator: Exists 189 | tolerations: 190 | - effect: NoSchedule 191 | key: node-role.kubernetes.io/control-plane 192 | operator: Exists 193 | - effect: NoSchedule 194 | key: node.cloudprovider.kubernetes.io/uninitialized 195 | operator: Exists 196 | topologySpreadConstraints: 197 | - maxSkew: 1 198 | topologyKey: kubernetes.io/hostname 199 | whenUnsatisfiable: DoNotSchedule 200 | labelSelector: 201 | matchLabels: 202 | app.kubernetes.io/name: proxmox-cloud-controller-manager 203 | app.kubernetes.io/instance: proxmox-cloud-controller-manager 204 | volumes: 205 | - name: cloud-config 206 | secret: 207 | secretName: proxmox-cloud-controller-manager 208 | defaultMode: 416 209 | -------------------------------------------------------------------------------- /docs/faq.md: -------------------------------------------------------------------------------- 1 | # Fast answers to common questions 2 | 3 | ## Dose CCM support online VM migration? 4 | 5 | No. 6 | Proxmox CCM uses [Cloud-Provider](https://github.com/kubernetes/cloud-provider.git) framework, which does not support label updates after the node initialization. 7 | 8 | Kuernetes has node drain feature, which can be used to move pods from one node to another. 9 | -------------------------------------------------------------------------------- /docs/install.md: -------------------------------------------------------------------------------- 1 | # Install 2 | 3 | Proxmox Cloud Controller Manager (CCM) supports controllers: 4 | * cloud-node 5 | * cloud-node-lifecycle 6 | 7 | `cloud-node` - detects new node launched in the cluster and registers them in the cluster. 8 | Assigns labels and taints based on Proxmox VM configuration. 9 | 10 | `cloud-node-lifecycle` - detects node deletion on Proxmox side and removes them from the cluster. 11 | 12 | ## Requirements 13 | 14 | You need to set `--cloud-provider=external` in the kubelet argument for all nodes in the cluster. 15 | The flag informs the kubelet to offload cloud-specific responsibilities to this external component like Proxmox CCM. 16 | 17 | ```shell 18 | kubelet --cloud-provider=external 19 | ``` 20 | 21 | Otherwise, kubelet will attempt to manage the node's lifecycle by itself, which can cause issues in environments using an external Cloud Controller Manager (CCM). 22 | 23 | ### Optional 24 | 25 | ```shell 26 | # ${IP} can be single or comma-separated list of two IPs (dual stack). 27 | kubelet --node-ip=${IP} 28 | ``` 29 | If your node has __multiple IP addresses__, you may need to set the `--node-ip` flag in the kubelet arguments to specify which IP address the kubelet should use. 30 | This ensures that the correct IP address is used for communication between the node and other components in the Kubernetes cluster, especially in environments where multiple network interfaces or IP addresses are present. 31 | 32 | ```shell 33 | # ${ID} has format proxmox://$REGION/$VMID. 34 | kubelet --provider-id=${ID} 35 | ``` 36 | If CCM cannot define VMID, you may need to set the `--provider-id` flag in the kubelet arguments to specify the VM ID in Proxmox. This ensures that the CCM can manage the node by VM ID. 37 | 38 | ```shell 39 | # ${NODENAME} is the name of the node. 40 | kubelet --hostname-override=${NODENAME} 41 | ``` 42 | If your node has a different hostname than the one registered in the cluster, you may need to set the `--hostname-override` flag in the kubelet arguments to specify the correct hostname. 43 | 44 | 45 | ## Create a Proxmox token 46 | 47 | Official [documentation](https://pve.proxmox.com/wiki/User_Management) 48 | 49 | ```shell 50 | # Create role CCM 51 | pveum role add CCM -privs "VM.Audit" 52 | # Create user and grant permissions 53 | pveum user add kubernetes@pve 54 | pveum aclmod / -user kubernetes@pve -role CCM 55 | pveum user token add kubernetes@pve ccm -privsep 0 56 | ``` 57 | 58 | ## Deploy CCM 59 | 60 | Create the proxmox credentials config file: 61 | 62 | ```yaml 63 | clusters: 64 | # List of Proxmox clusters, region mast be unique 65 | - url: https://cluster-api-1.exmple.com:8006/api2/json 66 | insecure: false 67 | token_id: "kubernetes@pve!ccm" 68 | # Token from the previous step 69 | token_secret: "secret" 70 | # Region name, can be any string, it will use as for kubernetes topology.kubernetes.io/region label 71 | region: cluster-1 72 | ``` 73 | 74 | ### Method 1: kubectl 75 | 76 | Upload it to the kubernetes: 77 | 78 | ```shell 79 | kubectl -n kube-system create secret generic proxmox-cloud-controller-manager --from-file=config.yaml 80 | ``` 81 | 82 | Deploy Proxmox CCM with `cloud-node,cloud-node-lifecycle` controllers 83 | 84 | ```shell 85 | kubectl apply -f https://raw.githubusercontent.com/sergelogvinov/proxmox-cloud-controller-manager/main/docs/deploy/cloud-controller-manager.yml 86 | ``` 87 | 88 | Deploy Proxmox CCM with `cloud-node-lifecycle` controller (for Talos) 89 | 90 | ```shell 91 | kubectl apply -f https://raw.githubusercontent.com/sergelogvinov/proxmox-cloud-controller-manager/main/docs/deploy/cloud-controller-manager-talos.yml 92 | ``` 93 | 94 | ### Method 2: helm chart 95 | 96 | Create the config file 97 | 98 | ```yaml 99 | # proxmox-ccm.yaml 100 | config: 101 | clusters: 102 | - url: https://cluster-api-1.exmple.com:8006/api2/json 103 | insecure: false 104 | token_id: "kubernetes@pve!ccm" 105 | token_secret: "secret" 106 | region: cluster-1 107 | ``` 108 | 109 | Deploy Proxmox CCM (deployment mode) 110 | 111 | ```shell 112 | helm upgrade -i --namespace=kube-system -f proxmox-ccm.yaml \ 113 | proxmox-cloud-controller-manager \ 114 | oci://ghcr.io/sergelogvinov/charts/proxmox-cloud-controller-manager 115 | ``` 116 | 117 | Deploy Proxmox CCM (daemonset mode) 118 | 119 | It makes sense to deploy on all control-plane nodes. Do not forget to set the nodeSelector. 120 | 121 | ```shell 122 | helm upgrade -i --namespace=kube-system -f proxmox-ccm.yaml \ 123 | --set useDaemonSet=true \ 124 | proxmox-cloud-controller-manager \ 125 | oci://ghcr.io/sergelogvinov/charts/proxmox-cloud-controller-manager 126 | ``` 127 | 128 | More options you can find [here](charts/proxmox-cloud-controller-manager) 129 | 130 | ## Deploy CCM (Rancher) 131 | 132 | Official [documentation](https://ranchermanager.docs.rancher.com/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters) 133 | 134 | Rancher RKE2 configuration: 135 | 136 | ```yaml 137 | machineGlobalConfig: 138 | # Kubelet predefined value --cloud-provider=external 139 | cloud-provider-name: external 140 | # Disable Rancher CCM 141 | disable-cloud-controller: true 142 | ``` 143 | 144 | Create the helm values file: 145 | 146 | ```yaml 147 | # proxmox-ccm.yaml 148 | config: 149 | clusters: 150 | - url: https://cluster-api-1.exmple.com:8006/api2/json 151 | insecure: false 152 | token_id: "kubernetes@pve!ccm" 153 | token_secret: "secret" 154 | region: cluster-1 155 | 156 | # Use host resolv.conf to resolve proxmox connection url 157 | useDaemonSet: true 158 | 159 | # Set nodeSelector in daemonset mode is required 160 | nodeSelector: 161 | node-role.kubernetes.io/control-plane: "" 162 | ``` 163 | 164 | Deploy Proxmox CCM (daemondset mode) 165 | 166 | ```shell 167 | helm upgrade -i --namespace=kube-system -f proxmox-ccm.yaml \ 168 | proxmox-cloud-controller-manager \ 169 | oci://ghcr.io/sergelogvinov/charts/proxmox-cloud-controller-manager 170 | ``` 171 | 172 | ## Deploy CCM with load balancer (optional) 173 | 174 | This optional setup to improve the Proxmox API availability. 175 | 176 | See [load balancer](loadbalancer.md) for installation instructions. 177 | 178 | ## Troubleshooting 179 | 180 | How `kubelet` works with flag `cloud-provider=external`: 181 | 182 | 1. kubelet join the cluster and send the `Node` object to the API server. 183 | Node object has values: 184 | * `node.cloudprovider.kubernetes.io/uninitialized` taint. 185 | * `alpha.kubernetes.io/provided-node-ip` annotation with the node IP. 186 | * `nodeInfo` field with system information. 187 | 2. CCM detects the new node and sends a request to the Proxmox API to get the VM configuration. Like VMID, hostname, etc. 188 | 3. CCM updates the `Node` object with labels, taints and `providerID` field. The `providerID` is immutable and has the format `proxmox://$REGION/$VMID`, it cannot be changed after the first update. 189 | 4. CCM removes the `node.cloudprovider.kubernetes.io/uninitialized` taint. 190 | 191 | If `kubelet` does not have `cloud-provider=external` flag, kubelet will expect that no external CCM is running and will try to manage the node lifecycle by itself. 192 | This can cause issues with Proxmox CCM. 193 | So, CCM will skip the node and will not update the `Node` object. 194 | 195 | If you modify the `kubelet` flags, it's recommended to check all workloads in the cluster. 196 | Please __delete__ the node resource first, and __restart__ the kubelet. 197 | 198 | The steps to troubleshoot the Proxmox CCM: 199 | 1. scale down the CCM deployment to 1 replica. 200 | 2. set log level to `--v=5` in the deployment. 201 | 3. check the logs 202 | 4. check kubelet flag `--cloud-provider=external`, delete the node resource and restart the kubelet. 203 | 5. check the logs 204 | 6. wait for 1 minute. If CCM cannot reach the Proxmox API, it will log the error. 205 | 7. check tains, labels, and providerID in the `Node` object. 206 | -------------------------------------------------------------------------------- /docs/loadbalancer.md: -------------------------------------------------------------------------------- 1 | # Loadbalancer on top of the Proxmox cluster 2 | 3 | Set up a load balancer to distribute traffic across multiple proxmox nodes. 4 | We use the [haproxy](https://hub.docker.com/_/haproxy) image to create a simple load balancer on top of the proxmox cluster. 5 | First, we need to create a headless service and set endpoints. 6 | 7 | ```yaml 8 | # proxmox-service.yaml 9 | --- 10 | apiVersion: v1 11 | kind: Service 12 | metadata: 13 | name: proxmox 14 | namespace: kube-system 15 | spec: 16 | clusterIP: None 17 | ports: 18 | - name: https 19 | protocol: TCP 20 | port: 8006 21 | targetPort: 8006 22 | --- 23 | apiVersion: v1 24 | kind: Endpoints 25 | metadata: 26 | name: proxmox 27 | namespace: kube-system 28 | subsets: 29 | - addresses: 30 | - ip: 192.168.0.1 31 | - ip: 192.168.0.2 32 | ports: 33 | - port: 8006 34 | ``` 35 | 36 | Apply the configuration to the cluster. 37 | 38 | ```bash 39 | kubectl apply -f proxmox-service.yaml 40 | ``` 41 | 42 | Second, we need to deploy proxmox CCM with sidecar load balancer. 43 | Haproxy will resolve the `proxmox.kube-system.svc.cluster.local` service and uses IPs from the endpoints to distribute traffic. 44 | Proxmox CCM will use the `proxmox.domain.com` domain to connect to the proxmox cluster which is resolved to the load balancer IP (127.0.0.1). 45 | 46 | ```yaml 47 | # CCM helm chart values 48 | 49 | config: 50 | clusters: 51 | - region: cluster 52 | url: https://proxmox.domain.com:8006/api2/json 53 | insecure: true 54 | token_id: kubernetes@pve!ccm 55 | token_secret: 11111111-1111-1111-1111-111111111111 56 | 57 | hostAliases: 58 | - ip: 127.0.0.1 59 | hostnames: 60 | - proxmox.domain.com 61 | 62 | initContainers: 63 | - name: loadbalancer 64 | restartPolicy: Always 65 | image: ghcr.io/sergelogvinov/haproxy:2.8.6-alpine3.19 66 | imagePullPolicy: IfNotPresent 67 | env: 68 | - name: SVC 69 | value: proxmox.kube-system.svc.cluster.local 70 | - name: PORT 71 | value: "8006" 72 | securityContext: 73 | runAsUser: 99 74 | runAsGroup: 99 75 | resources: 76 | limits: 77 | cpu: 50m 78 | memory: 64Mi 79 | requests: 80 | cpu: 50m 81 | memory: 32Mi 82 | ``` 83 | -------------------------------------------------------------------------------- /docs/metrics.md: -------------------------------------------------------------------------------- 1 | # Metrics documentation 2 | 3 | This document is a reflection of the current state of the exposed metrics of the Proxmox CCM. 4 | 5 | ## Gather metrics 6 | 7 | By default, the Proxmox CCM exposes metrics on the `https://localhost:10258/metrics` endpoint. 8 | 9 | ```yaml 10 | proxmox-cloud-controller-manager --authorization-always-allow-paths="/metrics" --secure-port=10258 11 | ``` 12 | 13 | ### Helm chart values 14 | 15 | The following values can be set in the Helm chart to expose the metrics of the Talos CCM. 16 | 17 | ```yaml 18 | podAnnotations: 19 | prometheus.io/scrape: "true" 20 | prometheus.io/scheme: "https" 21 | prometheus.io/port: "10258" 22 | ``` 23 | 24 | ## Metrics exposed by the CCM 25 | 26 | ### Proxmox API calls 27 | 28 | |Metric name|Metric type|Labels/tags| 29 | |-----------|-----------|-----------| 30 | |proxmox_api_request_duration_seconds|Histogram|`request`=| 31 | |proxmox_api_request_errors_total|Counter|`request`=| 32 | 33 | Example output: 34 | 35 | ```txt 36 | proxmox_api_request_duration_seconds_bucket{request="getVmInfo",le="0.1"} 13 37 | proxmox_api_request_duration_seconds_bucket{request="getVmInfo",le="0.25"} 172 38 | proxmox_api_request_duration_seconds_bucket{request="getVmInfo",le="0.5"} 199 39 | proxmox_api_request_duration_seconds_bucket{request="getVmInfo",le="1"} 210 40 | proxmox_api_request_duration_seconds_bucket{request="getVmInfo",le="2.5"} 210 41 | proxmox_api_request_duration_seconds_bucket{request="getVmInfo",le="5"} 210 42 | proxmox_api_request_duration_seconds_bucket{request="getVmInfo",le="10"} 210 43 | proxmox_api_request_duration_seconds_bucket{request="getVmInfo",le="30"} 210 44 | proxmox_api_request_duration_seconds_bucket{request="getVmInfo",le="+Inf"} 210 45 | proxmox_api_request_duration_seconds_sum{request="getVmInfo"} 39.698945394000006 46 | proxmox_api_request_duration_seconds_count{request="getVmInfo"} 210 47 | ``` 48 | -------------------------------------------------------------------------------- /docs/release.md: -------------------------------------------------------------------------------- 1 | # Make release 2 | 3 | ```shell 4 | git checkout -b release-0.0.2 5 | git tag v0.0.2 6 | 7 | make helm-unit docs 8 | make release-update 9 | 10 | git add . 11 | git commit 12 | ``` 13 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/sergelogvinov/proxmox-cloud-controller-manager 2 | 3 | go 1.24.2 4 | 5 | require ( 6 | github.com/Telmate/proxmox-api-go v0.0.0-20250202141955-0f3daee49334 7 | github.com/jarcoal/httpmock v1.4.0 8 | github.com/spf13/pflag v1.0.6 9 | github.com/stretchr/testify v1.10.0 10 | gopkg.in/yaml.v3 v3.0.1 11 | k8s.io/api v0.33.0 12 | k8s.io/apimachinery v0.33.0 13 | k8s.io/client-go v0.33.0 14 | k8s.io/cloud-provider v0.33.0 15 | k8s.io/component-base v0.33.0 16 | k8s.io/klog/v2 v2.130.1 17 | ) 18 | 19 | require ( 20 | cel.dev/expr v0.23.1 // indirect 21 | github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect 22 | github.com/NYTimes/gziphandler v1.1.1 // indirect 23 | github.com/antlr4-go/antlr/v4 v4.13.1 // indirect 24 | github.com/beorn7/perks v1.0.1 // indirect 25 | github.com/blang/semver/v4 v4.0.0 // indirect 26 | github.com/cenkalti/backoff/v4 v4.3.0 // indirect 27 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 28 | github.com/coreos/go-semver v0.3.1 // indirect 29 | github.com/coreos/go-systemd/v22 v22.5.0 // indirect 30 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect 31 | github.com/emicklei/go-restful/v3 v3.12.2 // indirect 32 | github.com/felixge/httpsnoop v1.0.4 // indirect 33 | github.com/fsnotify/fsnotify v1.9.0 // indirect 34 | github.com/fxamacker/cbor/v2 v2.8.0 // indirect 35 | github.com/go-logr/logr v1.4.2 // indirect 36 | github.com/go-logr/stdr v1.2.2 // indirect 37 | github.com/go-openapi/jsonpointer v0.21.1 // indirect 38 | github.com/go-openapi/jsonreference v0.21.0 // indirect 39 | github.com/go-openapi/swag v0.23.1 // indirect 40 | github.com/gogo/protobuf v1.3.2 // indirect 41 | github.com/golang/protobuf v1.5.4 // indirect 42 | github.com/google/btree v1.1.3 // indirect 43 | github.com/google/cel-go v0.25.0 // indirect 44 | github.com/google/gnostic-models v0.6.9 // indirect 45 | github.com/google/go-cmp v0.7.0 // indirect 46 | github.com/google/uuid v1.6.0 // indirect 47 | github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect 48 | github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 // indirect 49 | github.com/inconshreveable/mousetrap v1.1.0 // indirect 50 | github.com/josharian/intern v1.0.0 // indirect 51 | github.com/json-iterator/go v1.1.12 // indirect 52 | github.com/kylelemons/godebug v1.1.0 // indirect 53 | github.com/mailru/easyjson v0.9.0 // indirect 54 | github.com/moby/term v0.5.2 // indirect 55 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 56 | github.com/modern-go/reflect2 v1.0.2 // indirect 57 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 58 | github.com/pkg/errors v0.9.1 // indirect 59 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect 60 | github.com/prometheus/client_golang v1.22.0 // indirect 61 | github.com/prometheus/client_model v0.6.2 // indirect 62 | github.com/prometheus/common v0.63.0 // indirect 63 | github.com/prometheus/procfs v0.16.1 // indirect 64 | github.com/spf13/cobra v1.9.1 // indirect 65 | github.com/stoewer/go-strcase v1.3.0 // indirect 66 | github.com/x448/float16 v0.8.4 // indirect 67 | go.etcd.io/etcd/api/v3 v3.5.21 // indirect 68 | go.etcd.io/etcd/client/pkg/v3 v3.5.21 // indirect 69 | go.etcd.io/etcd/client/v3 v3.5.21 // indirect 70 | go.opentelemetry.io/auto/sdk v1.1.0 // indirect 71 | go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.58.0 // indirect 72 | go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 // indirect 73 | go.opentelemetry.io/otel v1.35.0 // indirect 74 | go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0 // indirect 75 | go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0 // indirect 76 | go.opentelemetry.io/otel/metric v1.35.0 // indirect 77 | go.opentelemetry.io/otel/sdk v1.33.0 // indirect 78 | go.opentelemetry.io/otel/trace v1.35.0 // indirect 79 | go.opentelemetry.io/proto/otlp v1.4.0 // indirect 80 | go.uber.org/multierr v1.11.0 // indirect 81 | go.uber.org/zap v1.27.0 // indirect 82 | golang.org/x/crypto v0.37.0 // indirect 83 | golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0 // indirect 84 | golang.org/x/net v0.39.0 // indirect 85 | golang.org/x/oauth2 v0.30.0 // indirect 86 | golang.org/x/sync v0.14.0 // indirect 87 | golang.org/x/sys v0.33.0 // indirect 88 | golang.org/x/term v0.31.0 // indirect 89 | golang.org/x/text v0.24.0 // indirect 90 | golang.org/x/time v0.11.0 // indirect 91 | google.golang.org/genproto v0.0.0-20240814211410-ddb44dafa142 // indirect 92 | google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb // indirect 93 | google.golang.org/genproto/googleapis/rpc v0.0.0-20250303144028-a0af3efb3deb // indirect 94 | google.golang.org/grpc v1.70.0 // indirect 95 | google.golang.org/protobuf v1.36.6 // indirect 96 | gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect 97 | gopkg.in/inf.v0 v0.9.1 // indirect 98 | gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect 99 | k8s.io/apiserver v0.33.0 // indirect 100 | k8s.io/component-helpers v0.33.0 // indirect 101 | k8s.io/controller-manager v0.33.0 // indirect 102 | k8s.io/kms v0.33.0 // indirect 103 | k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect 104 | k8s.io/utils v0.0.0-20250502105355-0f33e8f1c979 // indirect 105 | sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.32.0 // indirect 106 | sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect 107 | sigs.k8s.io/randfill v1.0.0 // indirect 108 | sigs.k8s.io/structured-merge-diff/v4 v4.7.0 // indirect 109 | sigs.k8s.io/yaml v1.4.0 // indirect 110 | ) 111 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | cel.dev/expr v0.23.1 h1:K4KOtPCJQjVggkARsjG9RWXP6O4R73aHeJMa/dmCQQg= 2 | cel.dev/expr v0.23.1/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw= 3 | github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= 4 | github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= 5 | github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= 6 | github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= 7 | github.com/Telmate/proxmox-api-go v0.0.0-20250202141955-0f3daee49334 h1:Zp0bWj3jviVC9ppfqOwYkmMESBcIFLlzHEBK7DduBNA= 8 | github.com/Telmate/proxmox-api-go v0.0.0-20250202141955-0f3daee49334/go.mod h1:6qNnkqdMB+22ytC/5qGAIIqtdK9egN1b/Sqs9tB/i1Y= 9 | github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= 10 | github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= 11 | github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= 12 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= 13 | github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= 14 | github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= 15 | github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= 16 | github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= 17 | github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= 18 | github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 19 | github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= 20 | github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= 21 | github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= 22 | github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= 23 | github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= 24 | github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= 25 | github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= 26 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 27 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 28 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= 29 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 30 | github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= 31 | github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= 32 | github.com/emicklei/go-restful/v3 v3.12.2 h1:DhwDP0vY3k8ZzE0RunuJy8GhNpPL6zqLkDf9B/a0/xU= 33 | github.com/emicklei/go-restful/v3 v3.12.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= 34 | github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= 35 | github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= 36 | github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= 37 | github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= 38 | github.com/fxamacker/cbor/v2 v2.8.0 h1:fFtUGXUzXPHTIUdne5+zzMPTfffl3RD5qYnkY40vtxU= 39 | github.com/fxamacker/cbor/v2 v2.8.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= 40 | github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= 41 | github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= 42 | github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= 43 | github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= 44 | github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= 45 | github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= 46 | github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= 47 | github.com/go-openapi/jsonpointer v0.21.1 h1:whnzv/pNXtK2FbX/W9yJfRmE2gsmkfahjMKB0fZvcic= 48 | github.com/go-openapi/jsonpointer v0.21.1/go.mod h1:50I1STOfbY1ycR8jGz8DaMeLCdXiI6aDteEdRNNzpdk= 49 | github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ= 50 | github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4= 51 | github.com/go-openapi/swag v0.23.1 h1:lpsStH0n2ittzTnbaSloVZLuB5+fvSY/+hnagBjSNZU= 52 | github.com/go-openapi/swag v0.23.1/go.mod h1:STZs8TbRvEQQKUA+JZNAm3EWlgaOBGpyFDqQnDHMef0= 53 | github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= 54 | github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= 55 | github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= 56 | github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= 57 | github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= 58 | github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= 59 | github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= 60 | github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= 61 | github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= 62 | github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= 63 | github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= 64 | github.com/google/cel-go v0.25.0 h1:jsFw9Fhn+3y2kBbltZR4VEz5xKkcIFRPDnuEzAGv5GY= 65 | github.com/google/cel-go v0.25.0/go.mod h1:hjEb6r5SuOSlhCHmFoLzu8HGCERvIsDAbxDAyNU/MmI= 66 | github.com/google/gnostic-models v0.6.9 h1:MU/8wDLif2qCXZmzncUQ/BOfxWfthHi63KqpoNbWqVw= 67 | github.com/google/gnostic-models v0.6.9/go.mod h1:CiWsm0s6BSQd1hRn8/QmxqB6BesYcbSZxsz9b0KuDBw= 68 | github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 69 | github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= 70 | github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= 71 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 72 | github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= 73 | github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= 74 | github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= 75 | github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 76 | github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo= 77 | github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA= 78 | github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= 79 | github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= 80 | github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= 81 | github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= 82 | github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= 83 | github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= 84 | github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 h1:5ZPtiqj0JL5oKWmcsq4VMaAW5ukBEgSGXEN89zeH1Jo= 85 | github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3/go.mod h1:ndYquD05frm2vACXE1nsccT4oJzjhw2arTS2cpUD1PI= 86 | github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= 87 | github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= 88 | github.com/jarcoal/httpmock v1.4.0 h1:BvhqnH0JAYbNudL2GMJKgOHe2CtKlzJ/5rWKyp+hc2k= 89 | github.com/jarcoal/httpmock v1.4.0/go.mod h1:ftW1xULwo+j0R0JJkJIIi7UKigZUXCLLanykgjwBXL0= 90 | github.com/jonboulle/clockwork v0.4.0 h1:p4Cf1aMWXnXAUh8lVfewRBx1zaTSYKrKMF2g3ST4RZ4= 91 | github.com/jonboulle/clockwork v0.4.0/go.mod h1:xgRqUGwRcjKCO1vbZUEtSLrqKoPSsUpK7fnezOII0kc= 92 | github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= 93 | github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= 94 | github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= 95 | github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= 96 | github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= 97 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 98 | github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= 99 | github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= 100 | github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= 101 | github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= 102 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 103 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 104 | github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= 105 | github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= 106 | github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4= 107 | github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= 108 | github.com/maxatome/go-testdeep v1.14.0 h1:rRlLv1+kI8eOI3OaBXZwb3O7xY3exRzdW5QyX48g9wI= 109 | github.com/maxatome/go-testdeep v1.14.0/go.mod h1:lPZc/HAcJMP92l7yI6TRz1aZN5URwUBUAfUNvrclaNM= 110 | github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ= 111 | github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc= 112 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 113 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= 114 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 115 | github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= 116 | github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= 117 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= 118 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= 119 | github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM= 120 | github.com/onsi/ginkgo/v2 v2.21.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= 121 | github.com/onsi/gomega v1.35.1 h1:Cwbd75ZBPxFSuZ6T+rN/WCb/gOc6YgFBXLlZLhC7Ds4= 122 | github.com/onsi/gomega v1.35.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= 123 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 124 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 125 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 126 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= 127 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 128 | github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= 129 | github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= 130 | github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= 131 | github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= 132 | github.com/prometheus/common v0.63.0 h1:YR/EIY1o3mEFP/kZCD7iDMnLPlGyuU2Gb3HIcXnA98k= 133 | github.com/prometheus/common v0.63.0/go.mod h1:VVFF/fBIoToEnWRVkYoXEkq3R3paCoxG9PXP74SnV18= 134 | github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= 135 | github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= 136 | github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= 137 | github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= 138 | github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= 139 | github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= 140 | github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= 141 | github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= 142 | github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= 143 | github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= 144 | github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= 145 | github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= 146 | github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= 147 | github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs= 148 | github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= 149 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 150 | github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= 151 | github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= 152 | github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= 153 | github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= 154 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 155 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 156 | github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= 157 | github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= 158 | github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= 159 | github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 160 | github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7otjonDflCTK0BCfls4SPy3NcCVb5dqqmbRknE= 161 | github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk= 162 | github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= 163 | github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= 164 | github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 h1:S2dVYn90KE98chqDkyE9Z4N61UnQd+KOfgp5Iu53llk= 165 | github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= 166 | github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 167 | github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 168 | go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0= 169 | go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I= 170 | go.etcd.io/etcd/api/v3 v3.5.21 h1:A6O2/JDb3tvHhiIz3xf9nJ7REHvtEFJJ3veW3FbCnS8= 171 | go.etcd.io/etcd/api/v3 v3.5.21/go.mod h1:c3aH5wcvXv/9dqIw2Y810LDXJfhSYdHQ0vxmP3CCHVY= 172 | go.etcd.io/etcd/client/pkg/v3 v3.5.21 h1:lPBu71Y7osQmzlflM9OfeIV2JlmpBjqBNlLtcoBqUTc= 173 | go.etcd.io/etcd/client/pkg/v3 v3.5.21/go.mod h1:BgqT/IXPjK9NkeSDjbzwsHySX3yIle2+ndz28nVsjUs= 174 | go.etcd.io/etcd/client/v2 v2.305.21 h1:eLiFfexc2mE+pTLz9WwnoEsX5JTTpLCYVivKkmVXIRA= 175 | go.etcd.io/etcd/client/v2 v2.305.21/go.mod h1:OKkn4hlYNf43hpjEM3Ke3aRdUkhSl8xjKjSf8eCq2J8= 176 | go.etcd.io/etcd/client/v3 v3.5.21 h1:T6b1Ow6fNjOLOtM0xSoKNQt1ASPCLWrF9XMHcH9pEyY= 177 | go.etcd.io/etcd/client/v3 v3.5.21/go.mod h1:mFYy67IOqmbRf/kRUvsHixzo3iG+1OF2W2+jVIQRAnU= 178 | go.etcd.io/etcd/pkg/v3 v3.5.21 h1:jUItxeKyrDuVuWhdh0HtjUANwyuzcb7/FAeUfABmQsk= 179 | go.etcd.io/etcd/pkg/v3 v3.5.21/go.mod h1:wpZx8Egv1g4y+N7JAsqi2zoUiBIUWznLjqJbylDjWgU= 180 | go.etcd.io/etcd/raft/v3 v3.5.21 h1:dOmE0mT55dIUsX77TKBLq+RgyumsQuYeiRQnW/ylugk= 181 | go.etcd.io/etcd/raft/v3 v3.5.21/go.mod h1:fmcuY5R2SNkklU4+fKVBQi2biVp5vafMrWUEj4TJ4Cs= 182 | go.etcd.io/etcd/server/v3 v3.5.21 h1:9w0/k12majtgarGmlMVuhwXRI2ob3/d1Ik3X5TKo0yU= 183 | go.etcd.io/etcd/server/v3 v3.5.21/go.mod h1:G1mOzdwuzKT1VRL7SqRchli/qcFrtLBTAQ4lV20sXXo= 184 | go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= 185 | go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= 186 | go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.58.0 h1:PS8wXpbyaDJQ2VDHHncMe9Vct0Zn1fEjpsjrLxGJoSc= 187 | go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.58.0/go.mod h1:HDBUsEjOuRC0EzKZ1bSaRGZWUBAzo+MhAcUUORSr4D0= 188 | go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 h1:yd02MEjBdJkG3uabWP9apV+OuWRIXGDuJEUJbOHmCFU= 189 | go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0/go.mod h1:umTcuxiv1n/s/S6/c2AT/g2CQ7u5C59sHDNmfSwgz7Q= 190 | go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ= 191 | go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y= 192 | go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0 h1:Vh5HayB/0HHfOQA7Ctx69E/Y/DcQSMPpKANYVMQ7fBA= 193 | go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0/go.mod h1:cpgtDBaqD/6ok/UG0jT15/uKjAY8mRA53diogHBg3UI= 194 | go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0 h1:5pojmb1U1AogINhN3SurB+zm/nIcusopeBNp42f45QM= 195 | go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0/go.mod h1:57gTHJSE5S1tqg+EKsLPlTWhpHMsWlVmer+LA926XiA= 196 | go.opentelemetry.io/otel/metric v1.35.0 h1:0znxYu2SNyuMSQT4Y9WDWej0VpcsxkuklLa4/siN90M= 197 | go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE= 198 | go.opentelemetry.io/otel/sdk v1.33.0 h1:iax7M131HuAm9QkZotNHEfstof92xM+N8sr3uHXc2IM= 199 | go.opentelemetry.io/otel/sdk v1.33.0/go.mod h1:A1Q5oi7/9XaMlIWzPSxLRWOI8nG3FnzHJNbiENQuihM= 200 | go.opentelemetry.io/otel/sdk/metric v1.32.0 h1:rZvFnvmvawYb0alrYkjraqJq0Z4ZUJAiyYCU9snn1CU= 201 | go.opentelemetry.io/otel/sdk/metric v1.32.0/go.mod h1:PWeZlq0zt9YkYAp3gjKZ0eicRYvOh1Gd+X99x6GHpCQ= 202 | go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs= 203 | go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc= 204 | go.opentelemetry.io/proto/otlp v1.4.0 h1:TA9WRvW6zMwP+Ssb6fLoUIuirti1gGbP28GcKG1jgeg= 205 | go.opentelemetry.io/proto/otlp v1.4.0/go.mod h1:PPBWZIP98o2ElSqI35IHfu7hIhSwvc5N38Jw8pXuGFY= 206 | go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= 207 | go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= 208 | go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= 209 | go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= 210 | go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= 211 | go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= 212 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 213 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 214 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 215 | golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= 216 | golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= 217 | golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0 h1:R84qjqJb5nVJMxqWYb3np9L5ZsaDtB+a39EqjV0JSUM= 218 | golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0/go.mod h1:S9Xr4PYopiDyqSyp5NjCrhFrqg6A5zA2E/iPHPhqnS8= 219 | golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 220 | golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 221 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 222 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 223 | golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 224 | golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= 225 | golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY= 226 | golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E= 227 | golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= 228 | golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= 229 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 230 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 231 | golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 232 | golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ= 233 | golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= 234 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 235 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 236 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 237 | golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 238 | golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= 239 | golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= 240 | golang.org/x/term v0.31.0 h1:erwDkOK1Msy6offm1mOgvspSkslFnIGsFnxOKoufg3o= 241 | golang.org/x/term v0.31.0/go.mod h1:R4BeIy7D95HzImkxGkTW1UQTtP54tio2RyHz7PwK0aw= 242 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 243 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 244 | golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= 245 | golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= 246 | golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= 247 | golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= 248 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 249 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 250 | golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= 251 | golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= 252 | golang.org/x/tools v0.32.0 h1:Q7N1vhpkQv7ybVzLFtTjvQya2ewbwNDZzUgfXGqtMWU= 253 | golang.org/x/tools v0.32.0/go.mod h1:ZxrU41P/wAbZD8EDa6dDCa6XfpkhJ7HFMjHJXfBDu8s= 254 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 255 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 256 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 257 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 258 | google.golang.org/genproto v0.0.0-20240814211410-ddb44dafa142 h1:oLiyxGgE+rt22duwci1+TG7bg2/L1LQsXwfjPlmuJA0= 259 | google.golang.org/genproto v0.0.0-20240814211410-ddb44dafa142/go.mod h1:G11eXq53iI5Q+kyNOmCvnzBaxEA2Q/Ik5Tj7nqBE8j4= 260 | google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb h1:p31xT4yrYrSM/G4Sn2+TNUkVhFCbG9y8itM2S6Th950= 261 | google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb/go.mod h1:jbe3Bkdp+Dh2IrslsFCklNhweNTBgSYanP1UXhJDhKg= 262 | google.golang.org/genproto/googleapis/rpc v0.0.0-20250303144028-a0af3efb3deb h1:TLPQVbx1GJ8VKZxz52VAxl1EBgKXXbTiU9Fc5fZeLn4= 263 | google.golang.org/genproto/googleapis/rpc v0.0.0-20250303144028-a0af3efb3deb/go.mod h1:LuRYeWDFV6WOn90g357N17oMCaxpgCnbi/44qJvDn2I= 264 | google.golang.org/grpc v1.70.0 h1:pWFv03aZoHzlRKHWicjsZytKAiYCtNS0dHbXnIdq7jQ= 265 | google.golang.org/grpc v1.70.0/go.mod h1:ofIJqVKDXx/JiXrwr2IG4/zwdH9txy3IlF40RmcJSQw= 266 | google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= 267 | google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= 268 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 269 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= 270 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= 271 | gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= 272 | gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= 273 | gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= 274 | gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= 275 | gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= 276 | gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= 277 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 278 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 279 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 280 | k8s.io/api v0.33.0 h1:yTgZVn1XEe6opVpP1FylmNrIFWuDqe2H0V8CT5gxfIU= 281 | k8s.io/api v0.33.0/go.mod h1:CTO61ECK/KU7haa3qq8sarQ0biLq2ju405IZAd9zsiM= 282 | k8s.io/apimachinery v0.33.0 h1:1a6kHrJxb2hs4t8EE5wuR/WxKDwGN1FKH3JvDtA0CIQ= 283 | k8s.io/apimachinery v0.33.0/go.mod h1:BHW0YOu7n22fFv/JkYOEfkUYNRN0fj0BlvMFWA7b+SM= 284 | k8s.io/apiserver v0.33.0 h1:QqcM6c+qEEjkOODHppFXRiw/cE2zP85704YrQ9YaBbc= 285 | k8s.io/apiserver v0.33.0/go.mod h1:EixYOit0YTxt8zrO2kBU7ixAtxFce9gKGq367nFmqI8= 286 | k8s.io/client-go v0.33.0 h1:UASR0sAYVUzs2kYuKn/ZakZlcs2bEHaizrrHUZg0G98= 287 | k8s.io/client-go v0.33.0/go.mod h1:kGkd+l/gNGg8GYWAPr0xF1rRKvVWvzh9vmZAMXtaKOg= 288 | k8s.io/cloud-provider v0.33.0 h1:nVU2Q9QK7O50yaNx+pE61oDPqflsSsKygN43f5js9+I= 289 | k8s.io/cloud-provider v0.33.0/go.mod h1:2reyEBbsimZJKHF325vxLBD5fcJGNeJHeLjJ+jGM8Qg= 290 | k8s.io/component-base v0.33.0 h1:Ot4PyJI+0JAD9covDhwLp9UNkUja209OzsJ4FzScBNk= 291 | k8s.io/component-base v0.33.0/go.mod h1:aXYZLbw3kihdkOPMDhWbjGCO6sg+luw554KP51t8qCU= 292 | k8s.io/component-helpers v0.33.0 h1:0AdW0A0mIgljLgtG0hJDdJl52PPqTrtMgOgtm/9i/Ys= 293 | k8s.io/component-helpers v0.33.0/go.mod h1:9SRiXfLldPw9lEEuSsapMtvT8j/h1JyFFapbtybwKvU= 294 | k8s.io/controller-manager v0.33.0 h1:O9LnTjffOe62d66gMcKLuPXsBjY5sqETWEIzg+DVL8w= 295 | k8s.io/controller-manager v0.33.0/go.mod h1:vQwAQnroav4+UyE2acW1Rj6CSsHPzr2/018kgRLYqlI= 296 | k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= 297 | k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= 298 | k8s.io/kms v0.33.0 h1:fhQSW/vyaWDhMp0vDuO/sLg2RlGZf4F77beSXcB4/eE= 299 | k8s.io/kms v0.33.0/go.mod h1:C1I8mjFFBNzfUZXYt9FZVJ8MJl7ynFbGgZFbBzkBJ3E= 300 | k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff h1:/usPimJzUKKu+m+TE36gUyGcf03XZEP0ZIKgKj35LS4= 301 | k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff/go.mod h1:5jIi+8yX4RIb8wk3XwBo5Pq2ccx4FP10ohkbSKCZoK8= 302 | k8s.io/utils v0.0.0-20250502105355-0f33e8f1c979 h1:jgJW5IePPXLGB8e/1wvd0Ich9QE97RvvF3a8J3fP/Lg= 303 | k8s.io/utils v0.0.0-20250502105355-0f33e8f1c979/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= 304 | sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.32.0 h1:XotDXzqvJ8Nx5eiZZueLpTuafJz8SiodgOemI+w87QU= 305 | sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.32.0/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= 306 | sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7npUVDpVXUUOFfW6CgAqE= 307 | sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= 308 | sigs.k8s.io/randfill v0.0.0-20250304075658-069ef1bbf016/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= 309 | sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= 310 | sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= 311 | sigs.k8s.io/structured-merge-diff/v4 v4.7.0 h1:qPeWmscJcXP0snki5IYF79Z8xrl8ETFxgMd7wez1XkI= 312 | sigs.k8s.io/structured-merge-diff/v4 v4.7.0/go.mod h1:dDy58f92j70zLsuZVuUX5Wp9vtxXpaZnkPGWeqDfCps= 313 | sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= 314 | sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= 315 | -------------------------------------------------------------------------------- /hack/CHANGELOG.tpl.md: -------------------------------------------------------------------------------- 1 | {{ range .Versions }} 2 | 3 | ## {{ if .Tag.Previous }}[{{ .Tag.Name }}]({{ $.Info.RepositoryURL }}/compare/{{ .Tag.Previous.Name }}...{{ .Tag.Name }}){{ else }}{{ .Tag.Name }}{{ end }} ({{ datetime "2006-01-02" .Tag.Date }}) 4 | 5 | Welcome to the {{ .Tag.Name }} release of Kubernetes cloud controller manager for Proxmox! 6 | 7 | {{ if .CommitGroups -}} 8 | {{ range .CommitGroups -}} 9 | ### {{ .Title }} 10 | 11 | {{ range .Commits -}} 12 | - {{ if .Scope }}**{{ .Scope }}:** {{ end }}{{ .Subject }} 13 | {{ end }} 14 | {{ end -}} 15 | {{ end -}} 16 | 17 | 18 | ### Changelog 19 | 20 | {{ range .Commits -}}{{ if ne .Subject "" -}} 21 | * {{ .Hash.Short }} {{ .Header }} 22 | {{ end }}{{ end }} 23 | 24 | {{- if .NoteGroups -}} 25 | {{ range .NoteGroups -}} 26 | ### {{ .Title }} 27 | 28 | {{ range .Notes }} 29 | {{ .Body }} 30 | {{ end }} 31 | {{ end -}} 32 | {{ end -}} 33 | {{ end -}} 34 | -------------------------------------------------------------------------------- /hack/chglog-config.yml: -------------------------------------------------------------------------------- 1 | style: github 2 | template: CHANGELOG.tpl.md 3 | info: 4 | title: CHANGELOG 5 | repository_url: https://github.com/sergelogvinov/proxmox-cloud-controller-manager 6 | options: 7 | commits: 8 | filters: 9 | Type: 10 | - feat 11 | - fix 12 | commit_groups: 13 | title_maps: 14 | feat: Features 15 | fix: Bug Fixes 16 | header: 17 | pattern: "^(\\w*)(?:\\(([\\w\\$\\.\\-\\*\\s]*)\\))?\\:\\s(.*)$" 18 | pattern_maps: 19 | - Type 20 | - Scope 21 | - Subject 22 | notes: 23 | keywords: 24 | - BREAKING CHANGE 25 | -------------------------------------------------------------------------------- /hack/ct.yml: -------------------------------------------------------------------------------- 1 | helm-extra-args: --timeout 300s 2 | check-version-increment: true 3 | debug: true 4 | chart-dirs: 5 | - charts 6 | validate-maintainers: true 7 | namespace: default 8 | release-label: test 9 | target-branch: main 10 | -------------------------------------------------------------------------------- /hack/proxmox-config.yaml: -------------------------------------------------------------------------------- 1 | clusters: 2 | - url: https://cluster-api-1.exmple.com:8006/api2/json 3 | insecure: false 4 | token_id: "user!token-id" 5 | token_secret: "secret" 6 | region: cluster-1 7 | - url: https://cluster-api-2.exmple.com:8006/api2/json 8 | insecure: false 9 | token_id: "user!token-id" 10 | token_secret: "secret" 11 | region: cluster-2 12 | -------------------------------------------------------------------------------- /pkg/cluster/client.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Package cluster implements the multi-cloud provider interface for Proxmox. 18 | package cluster 19 | 20 | import ( 21 | "context" 22 | "crypto/tls" 23 | "encoding/base64" 24 | "fmt" 25 | "net/http" 26 | "net/url" 27 | "os" 28 | "strings" 29 | 30 | pxapi "github.com/Telmate/proxmox-api-go/proxmox" 31 | 32 | v1 "k8s.io/api/core/v1" 33 | "k8s.io/klog/v2" 34 | ) 35 | 36 | // Cluster is a Proxmox client. 37 | type Cluster struct { 38 | config *ClustersConfig 39 | proxmox map[string]*pxapi.Client 40 | } 41 | 42 | // NewCluster creates a new Proxmox cluster client. 43 | func NewCluster(config *ClustersConfig, hclient *http.Client) (*Cluster, error) { 44 | clusters := len(config.Clusters) 45 | if clusters > 0 { 46 | proxmox := make(map[string]*pxapi.Client, clusters) 47 | 48 | for _, cfg := range config.Clusters { 49 | tlsconf := &tls.Config{InsecureSkipVerify: true} 50 | if !cfg.Insecure { 51 | tlsconf = nil 52 | } 53 | 54 | client, err := pxapi.NewClient(cfg.URL, hclient, os.Getenv("PM_HTTP_HEADERS"), tlsconf, "", 600) 55 | if err != nil { 56 | return nil, err 57 | } 58 | 59 | if cfg.Username != "" && cfg.Password != "" { 60 | if err := client.Login(context.Background(), cfg.Username, cfg.Password, ""); err != nil { 61 | return nil, err 62 | } 63 | } else { 64 | client.SetAPIToken(cfg.TokenID, cfg.TokenSecret) 65 | } 66 | 67 | proxmox[cfg.Region] = client 68 | } 69 | 70 | return &Cluster{ 71 | config: config, 72 | proxmox: proxmox, 73 | }, nil 74 | } 75 | 76 | return nil, fmt.Errorf("no Proxmox clusters found") 77 | } 78 | 79 | // CheckClusters checks if the Proxmox connection is working. 80 | func (c *Cluster) CheckClusters(ctx context.Context) error { 81 | for region, client := range c.proxmox { 82 | if _, err := client.GetVersion(ctx); err != nil { 83 | return fmt.Errorf("failed to initialized proxmox client in region %s, error: %v", region, err) 84 | } 85 | 86 | vmlist, err := client.GetVmList(ctx) 87 | if err != nil { 88 | return fmt.Errorf("failed to get list of VMs in region %s, error: %v", region, err) 89 | } 90 | 91 | vms, ok := vmlist["data"].([]interface{}) 92 | if !ok { 93 | return fmt.Errorf("failed to cast response to list of VMs in region %s, error: %v", region, err) 94 | } 95 | 96 | if len(vms) > 0 { 97 | klog.V(4).InfoS("Proxmox cluster has VMs", "region", region, "count", len(vms)) 98 | } else { 99 | klog.InfoS("Proxmox cluster has no VMs, or check the account permission", "region", region) 100 | } 101 | } 102 | 103 | return nil 104 | } 105 | 106 | // GetProxmoxCluster returns a Proxmox cluster client in a given region. 107 | func (c *Cluster) GetProxmoxCluster(region string) (*pxapi.Client, error) { 108 | if c.proxmox[region] != nil { 109 | return c.proxmox[region], nil 110 | } 111 | 112 | return nil, fmt.Errorf("proxmox cluster %s not found", region) 113 | } 114 | 115 | // FindVMByNode find a VM by kubernetes node resource in all Proxmox clusters. 116 | func (c *Cluster) FindVMByNode(ctx context.Context, node *v1.Node) (*pxapi.VmRef, string, error) { 117 | for region, px := range c.proxmox { 118 | vmrs, err := px.GetVmRefsByName(ctx, node.Name) 119 | if err != nil { 120 | if strings.Contains(err.Error(), "not found") { 121 | continue 122 | } 123 | 124 | return nil, "", err 125 | } 126 | 127 | for _, vmr := range vmrs { 128 | config, err := px.GetVmConfig(ctx, vmr) 129 | if err != nil { 130 | return nil, "", err 131 | } 132 | 133 | if c.GetVMUUID(config) == node.Status.NodeInfo.SystemUUID { 134 | return vmr, region, nil 135 | } 136 | } 137 | } 138 | 139 | return nil, "", fmt.Errorf("vm '%s' not found", node.Name) 140 | } 141 | 142 | // FindVMByName find a VM by name in all Proxmox clusters. 143 | func (c *Cluster) FindVMByName(ctx context.Context, name string) (*pxapi.VmRef, string, error) { 144 | for region, px := range c.proxmox { 145 | vmr, err := px.GetVmRefByName(ctx, name) 146 | if err != nil { 147 | if strings.Contains(err.Error(), "not found") { 148 | continue 149 | } 150 | 151 | return nil, "", err 152 | } 153 | 154 | return vmr, region, nil 155 | } 156 | 157 | return nil, "", fmt.Errorf("vm '%s' not found", name) 158 | } 159 | 160 | // FindVMByUUID find a VM by uuid in all Proxmox clusters. 161 | func (c *Cluster) FindVMByUUID(ctx context.Context, uuid string) (*pxapi.VmRef, string, error) { 162 | for region, px := range c.proxmox { 163 | vms, err := px.GetResourceList(ctx, "vm") 164 | if err != nil { 165 | return nil, "", fmt.Errorf("error get resources %v", err) 166 | } 167 | 168 | for vmii := range vms { 169 | vm, ok := vms[vmii].(map[string]interface{}) 170 | if !ok { 171 | return nil, "", fmt.Errorf("failed to cast response to map, vm: %v", vm) 172 | } 173 | 174 | if vm["type"].(string) != "qemu" { //nolint:errcheck 175 | continue 176 | } 177 | 178 | vmr := pxapi.NewVmRef(int(vm["vmid"].(float64))) //nolint:errcheck 179 | vmr.SetNode(vm["node"].(string)) //nolint:errcheck 180 | vmr.SetVmType("qemu") 181 | 182 | config, err := px.GetVmConfig(ctx, vmr) 183 | if err != nil { 184 | return nil, "", err 185 | } 186 | 187 | if config["smbios1"] != nil { 188 | if c.getSMBSetting(config, "uuid") == uuid { 189 | return vmr, region, nil 190 | } 191 | } 192 | } 193 | } 194 | 195 | return nil, "", fmt.Errorf("vm with uuid '%s' not found", uuid) 196 | } 197 | 198 | // GetVMName returns the VM name. 199 | func (c *Cluster) GetVMName(vmInfo map[string]interface{}) string { 200 | if vmInfo["name"] != nil { 201 | return vmInfo["name"].(string) //nolint:errcheck 202 | } 203 | 204 | return "" 205 | } 206 | 207 | // GetVMUUID returns the VM UUID. 208 | func (c *Cluster) GetVMUUID(vmInfo map[string]interface{}) string { 209 | if vmInfo["smbios1"] != nil { 210 | return c.getSMBSetting(vmInfo, "uuid") 211 | } 212 | 213 | return "" 214 | } 215 | 216 | // GetVMSKU returns the VM instance type name. 217 | func (c *Cluster) GetVMSKU(vmInfo map[string]interface{}) string { 218 | if vmInfo["smbios1"] != nil { 219 | return c.getSMBSetting(vmInfo, "sku") 220 | } 221 | 222 | return "" 223 | } 224 | 225 | func (c *Cluster) getSMBSetting(vmInfo map[string]interface{}, name string) string { 226 | smbios, ok := vmInfo["smbios1"].(string) 227 | if !ok { 228 | return "" 229 | } 230 | 231 | for _, l := range strings.Split(smbios, ",") { 232 | if l == "" || l == "base64=1" { 233 | continue 234 | } 235 | 236 | parsedParameter, err := url.ParseQuery(l) 237 | if err != nil { 238 | return "" 239 | } 240 | 241 | for k, v := range parsedParameter { 242 | if k == name { 243 | decodedString, err := base64.StdEncoding.DecodeString(v[0]) 244 | if err != nil { 245 | decodedString = []byte(v[0]) 246 | } 247 | 248 | return string(decodedString) 249 | } 250 | } 251 | } 252 | 253 | return "" 254 | } 255 | -------------------------------------------------------------------------------- /pkg/cluster/client_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package cluster_test 18 | 19 | import ( 20 | "fmt" 21 | "net/http" 22 | "strings" 23 | "testing" 24 | 25 | "github.com/jarcoal/httpmock" 26 | "github.com/stretchr/testify/assert" 27 | 28 | "github.com/sergelogvinov/proxmox-cloud-controller-manager/pkg/cluster" 29 | ) 30 | 31 | func newClusterEnv() (*cluster.ClustersConfig, error) { 32 | cfg, err := cluster.ReadCloudConfig(strings.NewReader(` 33 | clusters: 34 | - url: https://127.0.0.1:8006/api2/json 35 | insecure: false 36 | token_id: "user!token-id" 37 | token_secret: "secret" 38 | region: cluster-1 39 | - url: https://127.0.0.2:8006/api2/json 40 | insecure: false 41 | token_id: "user!token-id" 42 | token_secret: "secret" 43 | region: cluster-2 44 | `)) 45 | 46 | return &cfg, err 47 | } 48 | 49 | func TestNewClient(t *testing.T) { 50 | cfg, err := newClusterEnv() 51 | assert.Nil(t, err) 52 | assert.NotNil(t, cfg) 53 | 54 | client, err := cluster.NewCluster(&cluster.ClustersConfig{}, nil) 55 | assert.NotNil(t, err) 56 | assert.Nil(t, client) 57 | 58 | client, err = cluster.NewCluster(cfg, nil) 59 | assert.Nil(t, err) 60 | assert.NotNil(t, client) 61 | } 62 | 63 | func TestCheckClusters(t *testing.T) { 64 | cfg, err := newClusterEnv() 65 | assert.Nil(t, err) 66 | assert.NotNil(t, cfg) 67 | 68 | client, err := cluster.NewCluster(cfg, nil) 69 | assert.Nil(t, err) 70 | assert.NotNil(t, client) 71 | 72 | pxapi, err := client.GetProxmoxCluster("test") 73 | assert.NotNil(t, err) 74 | assert.Nil(t, pxapi) 75 | assert.Equal(t, "proxmox cluster test not found", err.Error()) 76 | 77 | pxapi, err = client.GetProxmoxCluster("cluster-1") 78 | assert.Nil(t, err) 79 | assert.NotNil(t, pxapi) 80 | 81 | err = client.CheckClusters(t.Context()) 82 | assert.NotNil(t, err) 83 | assert.Contains(t, err.Error(), "failed to initialized proxmox client in region") 84 | } 85 | 86 | func TestFindVMByNameNonExist(t *testing.T) { 87 | cfg, err := newClusterEnv() 88 | assert.Nil(t, err) 89 | assert.NotNil(t, cfg) 90 | 91 | httpmock.Activate() 92 | defer httpmock.DeactivateAndReset() 93 | 94 | httpmock.RegisterResponder("GET", "https://127.0.0.1:8006/api2/json/cluster/resources", 95 | func(_ *http.Request) (*http.Response, error) { 96 | return httpmock.NewJsonResponse(200, map[string]interface{}{ 97 | "data": []interface{}{ 98 | map[string]interface{}{ 99 | "node": "node-1", 100 | "type": "qemu", 101 | "vmid": 100, 102 | "name": "test1-vm", 103 | }, 104 | }, 105 | }) 106 | }, 107 | ) 108 | 109 | httpmock.RegisterResponder("GET", "https://127.0.0.2:8006/api2/json/cluster/resources", 110 | func(_ *http.Request) (*http.Response, error) { 111 | return httpmock.NewJsonResponse(200, map[string]interface{}{ 112 | "data": []interface{}{ 113 | map[string]interface{}{ 114 | "node": "node-2", 115 | "type": "qemu", 116 | "vmid": 100, 117 | "name": "test2-vm", 118 | }, 119 | }, 120 | }) 121 | }, 122 | ) 123 | 124 | client, err := cluster.NewCluster(cfg, &http.Client{}) 125 | assert.Nil(t, err) 126 | assert.NotNil(t, client) 127 | 128 | vmr, cluster, err := client.FindVMByName(t.Context(), "non-existing-vm") 129 | assert.NotNil(t, err) 130 | assert.Equal(t, "", cluster) 131 | assert.Nil(t, vmr) 132 | assert.Contains(t, err.Error(), "vm 'non-existing-vm' not found") 133 | } 134 | 135 | func TestFindVMByNameExist(t *testing.T) { 136 | cfg, err := newClusterEnv() 137 | assert.Nil(t, err) 138 | assert.NotNil(t, cfg) 139 | 140 | httpmock.Activate() 141 | defer httpmock.DeactivateAndReset() 142 | 143 | httpmock.RegisterResponder("GET", "https://127.0.0.1:8006/api2/json/cluster/resources", 144 | httpmock.NewJsonResponderOrPanic(200, map[string]interface{}{ 145 | "data": []interface{}{ 146 | map[string]interface{}{ 147 | "node": "node-1", 148 | "type": "qemu", 149 | "vmid": 100, 150 | "name": "test1-vm", 151 | }, 152 | }, 153 | }), 154 | ) 155 | 156 | httpmock.RegisterResponder("GET", "https://127.0.0.2:8006/api2/json/cluster/resources", 157 | func(_ *http.Request) (*http.Response, error) { 158 | return httpmock.NewJsonResponse(200, map[string]interface{}{ 159 | "data": []interface{}{ 160 | map[string]interface{}{ 161 | "node": "node-2", 162 | "type": "qemu", 163 | "vmid": 100, 164 | "name": "test2-vm", 165 | }, 166 | }, 167 | }) 168 | }, 169 | ) 170 | 171 | client, err := cluster.NewCluster(cfg, &http.Client{}) 172 | assert.Nil(t, err) 173 | assert.NotNil(t, client) 174 | 175 | tests := []struct { 176 | msg string 177 | vmName string 178 | expectedError error 179 | expectedVMID int 180 | expectedCluster string 181 | }{ 182 | { 183 | msg: "vm not found", 184 | vmName: "non-existing-vm", 185 | expectedError: fmt.Errorf("vm 'non-existing-vm' not found"), 186 | }, 187 | { 188 | msg: "Test1-VM", 189 | vmName: "test1-vm", 190 | expectedVMID: 100, 191 | expectedCluster: "cluster-1", 192 | }, 193 | { 194 | msg: "Test2-VM", 195 | vmName: "test2-vm", 196 | expectedVMID: 100, 197 | expectedCluster: "cluster-2", 198 | }, 199 | } 200 | 201 | for _, testCase := range tests { 202 | testCase := testCase 203 | 204 | t.Run(fmt.Sprint(testCase.msg), func(t *testing.T) { 205 | vmr, cluster, err := client.FindVMByName(t.Context(), testCase.vmName) 206 | 207 | if testCase.expectedError == nil { 208 | assert.Nil(t, err) 209 | assert.NotNil(t, vmr) 210 | assert.Equal(t, testCase.expectedVMID, vmr.VmId()) 211 | assert.Equal(t, testCase.expectedCluster, cluster) 212 | } else { 213 | assert.NotNil(t, err) 214 | assert.Equal(t, "", cluster) 215 | assert.Nil(t, vmr) 216 | assert.Contains(t, err.Error(), "vm 'non-existing-vm' not found") 217 | } 218 | }) 219 | } 220 | } 221 | -------------------------------------------------------------------------------- /pkg/cluster/cloud_config.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package cluster 18 | 19 | import ( 20 | "fmt" 21 | "io" 22 | "os" 23 | "path/filepath" 24 | "strings" 25 | 26 | yaml "gopkg.in/yaml.v3" 27 | ) 28 | 29 | // Provider specifies the provider. Can be 'default' or 'capmox' 30 | type Provider string 31 | 32 | // ProviderDefault is the default provider 33 | const ProviderDefault Provider = "default" 34 | 35 | // ProviderCapmox is the Provider for capmox 36 | const ProviderCapmox Provider = "capmox" 37 | 38 | // ClustersConfig is proxmox multi-cluster cloud config. 39 | type ClustersConfig struct { 40 | Features struct { 41 | Provider Provider `yaml:"provider,omitempty"` 42 | } `yaml:"features,omitempty"` 43 | Clusters []struct { 44 | URL string `yaml:"url"` 45 | Insecure bool `yaml:"insecure,omitempty"` 46 | TokenID string `yaml:"token_id,omitempty"` 47 | TokenSecret string `yaml:"token_secret,omitempty"` 48 | Username string `yaml:"username,omitempty"` 49 | Password string `yaml:"password,omitempty"` 50 | Region string `yaml:"region,omitempty"` 51 | } `yaml:"clusters,omitempty"` 52 | } 53 | 54 | // ReadCloudConfig reads cloud config from a reader. 55 | func ReadCloudConfig(config io.Reader) (ClustersConfig, error) { 56 | cfg := ClustersConfig{} 57 | 58 | if config != nil { 59 | if err := yaml.NewDecoder(config).Decode(&cfg); err != nil { 60 | return ClustersConfig{}, err 61 | } 62 | } 63 | 64 | for idx, c := range cfg.Clusters { 65 | if c.Username != "" && c.Password != "" { 66 | if c.TokenID != "" || c.TokenSecret != "" { 67 | return ClustersConfig{}, fmt.Errorf("cluster #%d: token_id and token_secret are not allowed when username and password are set", idx+1) 68 | } 69 | } else if c.TokenID == "" || c.TokenSecret == "" { 70 | return ClustersConfig{}, fmt.Errorf("cluster #%d: either username and password or token_id and token_secret are required", idx+1) 71 | } 72 | 73 | if c.Region == "" { 74 | return ClustersConfig{}, fmt.Errorf("cluster #%d: region is required", idx+1) 75 | } 76 | 77 | if c.URL == "" || !strings.HasPrefix(c.URL, "http") { 78 | return ClustersConfig{}, fmt.Errorf("cluster #%d: url is required", idx+1) 79 | } 80 | } 81 | 82 | if cfg.Features.Provider == "" { 83 | cfg.Features.Provider = ProviderDefault 84 | } 85 | 86 | return cfg, nil 87 | } 88 | 89 | // ReadCloudConfigFromFile reads cloud config from a file. 90 | func ReadCloudConfigFromFile(file string) (ClustersConfig, error) { 91 | f, err := os.Open(filepath.Clean(file)) 92 | if err != nil { 93 | return ClustersConfig{}, fmt.Errorf("error reading %s: %v", file, err) 94 | } 95 | defer f.Close() // nolint: errcheck 96 | 97 | return ReadCloudConfig(f) 98 | } 99 | -------------------------------------------------------------------------------- /pkg/cluster/cloud_config_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package cluster_test 18 | 19 | import ( 20 | "strings" 21 | "testing" 22 | 23 | "github.com/stretchr/testify/assert" 24 | 25 | "github.com/sergelogvinov/proxmox-cloud-controller-manager/pkg/cluster" 26 | ) 27 | 28 | func TestReadCloudConfig(t *testing.T) { 29 | cfg, err := cluster.ReadCloudConfig(nil) 30 | assert.Nil(t, err) 31 | assert.NotNil(t, cfg) 32 | 33 | // Empty config 34 | cfg, err = cluster.ReadCloudConfig(strings.NewReader(` 35 | clusters: 36 | `)) 37 | assert.Nil(t, err) 38 | assert.NotNil(t, cfg) 39 | 40 | // Wrong config 41 | cfg, err = cluster.ReadCloudConfig(strings.NewReader(` 42 | clusters: 43 | test: false 44 | `)) 45 | 46 | assert.NotNil(t, err) 47 | assert.NotNil(t, cfg) 48 | 49 | // Non full config 50 | cfg, err = cluster.ReadCloudConfig(strings.NewReader(` 51 | clusters: 52 | - url: abcd 53 | region: cluster-1 54 | `)) 55 | 56 | assert.NotNil(t, err) 57 | assert.NotNil(t, cfg) 58 | 59 | // Valid config with one cluster 60 | cfg, err = cluster.ReadCloudConfig(strings.NewReader(` 61 | clusters: 62 | - url: https://example.com 63 | insecure: false 64 | token_id: "user!token-id" 65 | token_secret: "secret" 66 | region: cluster-1 67 | `)) 68 | assert.Nil(t, err) 69 | assert.NotNil(t, cfg) 70 | assert.Equal(t, 1, len(cfg.Clusters)) 71 | 72 | // Valid config with one cluster (username/password), implicit default provider 73 | cfg, err = cluster.ReadCloudConfig(strings.NewReader(` 74 | clusters: 75 | - url: https://example.com 76 | insecure: false 77 | username: "user@pam" 78 | password: "secret" 79 | region: cluster-1 80 | `)) 81 | assert.Nil(t, err) 82 | assert.NotNil(t, cfg) 83 | assert.Equal(t, 1, len(cfg.Clusters)) 84 | assert.Equal(t, cluster.ProviderDefault, cfg.Features.Provider) 85 | 86 | // Valid config with one cluster (username/password), explicit provider default 87 | cfg, err = cluster.ReadCloudConfig(strings.NewReader(` 88 | features: 89 | provider: 'default' 90 | clusters: 91 | - url: https://example.com 92 | insecure: false 93 | username: "user@pam" 94 | password: "secret" 95 | region: cluster-1 96 | `)) 97 | assert.Nil(t, err) 98 | assert.NotNil(t, cfg) 99 | assert.Equal(t, 1, len(cfg.Clusters)) 100 | assert.Equal(t, cluster.ProviderDefault, cfg.Features.Provider) 101 | 102 | // Valid config with one cluster (username/password), explicit provider capmox 103 | cfg, err = cluster.ReadCloudConfig(strings.NewReader(` 104 | features: 105 | provider: 'capmox' 106 | clusters: 107 | - url: https://example.com 108 | insecure: false 109 | username: "user@pam" 110 | password: "secret" 111 | region: cluster-1 112 | `)) 113 | assert.Nil(t, err) 114 | assert.NotNil(t, cfg) 115 | assert.Equal(t, 1, len(cfg.Clusters)) 116 | assert.Equal(t, cluster.ProviderCapmox, cfg.Features.Provider) 117 | } 118 | 119 | func TestReadCloudConfigFromFile(t *testing.T) { 120 | cfg, err := cluster.ReadCloudConfigFromFile("testdata/cloud-config.yaml") 121 | assert.NotNil(t, err) 122 | assert.EqualError(t, err, "error reading testdata/cloud-config.yaml: open testdata/cloud-config.yaml: no such file or directory") 123 | assert.NotNil(t, cfg) 124 | 125 | cfg, err = cluster.ReadCloudConfigFromFile("../../hack/proxmox-config.yaml") 126 | assert.Nil(t, err) 127 | assert.NotNil(t, cfg) 128 | assert.Equal(t, 2, len(cfg.Clusters)) 129 | } 130 | -------------------------------------------------------------------------------- /pkg/metrics/metrics.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Package metrics collects metrics. 18 | package metrics 19 | 20 | import ( 21 | "time" 22 | ) 23 | 24 | // MetricContext indicates the context for Talos client metrics. 25 | type MetricContext struct { 26 | start time.Time 27 | attributes []string 28 | } 29 | 30 | // NewMetricContext creates a new MetricContext. 31 | func NewMetricContext(resource string) *MetricContext { 32 | return &MetricContext{ 33 | start: time.Now(), 34 | attributes: []string{resource}, 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /pkg/metrics/metrics_api.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package metrics 18 | 19 | import ( 20 | "time" 21 | 22 | "k8s.io/component-base/metrics" 23 | "k8s.io/component-base/metrics/legacyregistry" 24 | ) 25 | 26 | // CSIMetrics contains the metrics for Talos API calls. 27 | type CSIMetrics struct { 28 | Duration *metrics.HistogramVec 29 | Errors *metrics.CounterVec 30 | } 31 | 32 | var apiMetrics = registerAPIMetrics() 33 | 34 | // ObserveRequest records the request latency and counts the errors. 35 | func (mc *MetricContext) ObserveRequest(err error) error { 36 | apiMetrics.Duration.WithLabelValues(mc.attributes...).Observe( 37 | time.Since(mc.start).Seconds()) 38 | 39 | if err != nil { 40 | apiMetrics.Errors.WithLabelValues(mc.attributes...).Inc() 41 | } 42 | 43 | return err 44 | } 45 | 46 | func registerAPIMetrics() *CSIMetrics { 47 | metrics := &CSIMetrics{ 48 | Duration: metrics.NewHistogramVec( 49 | &metrics.HistogramOpts{ 50 | Name: "proxmox_api_request_duration_seconds", 51 | Help: "Latency of an Proxmox API call", 52 | Buckets: []float64{.1, .25, .5, 1, 2.5, 5, 10, 30}, 53 | }, []string{"request"}), 54 | Errors: metrics.NewCounterVec( 55 | &metrics.CounterOpts{ 56 | Name: "proxmox_api_request_errors_total", 57 | Help: "Total number of errors for an Proxmox API call", 58 | }, []string{"request"}), 59 | } 60 | 61 | legacyregistry.MustRegister( 62 | metrics.Duration, 63 | metrics.Errors, 64 | ) 65 | 66 | return metrics 67 | } 68 | -------------------------------------------------------------------------------- /pkg/provider/provider.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Package provider implements the providerID interface for Proxmox. 18 | package provider 19 | 20 | import ( 21 | "fmt" 22 | "regexp" 23 | "strconv" 24 | "strings" 25 | 26 | pxapi "github.com/Telmate/proxmox-api-go/proxmox" 27 | ) 28 | 29 | const ( 30 | // ProviderName is the name of the Proxmox provider. 31 | ProviderName = "proxmox" 32 | ) 33 | 34 | var providerIDRegexp = regexp.MustCompile(`^` + ProviderName + `://([^/]*)/([^/]+)$`) 35 | 36 | // GetProviderID returns the magic providerID for kubernetes node. 37 | func GetProviderID(region string, vmr *pxapi.VmRef) string { 38 | return fmt.Sprintf("%s://%s/%d", ProviderName, region, vmr.VmId()) 39 | } 40 | 41 | // GetProviderIDFromUUID returns the magic providerID for kubernetes node. 42 | func GetProviderIDFromUUID(uuid string) string { 43 | return fmt.Sprintf("%s://%s", ProviderName, uuid) 44 | } 45 | 46 | // GetVMID returns the VM ID from the providerID. 47 | func GetVMID(providerID string) (int, error) { 48 | if !strings.HasPrefix(providerID, ProviderName) { 49 | return 0, fmt.Errorf("foreign providerID or empty \"%s\"", providerID) 50 | } 51 | 52 | matches := providerIDRegexp.FindStringSubmatch(providerID) 53 | if len(matches) != 3 { 54 | return 0, fmt.Errorf("providerID \"%s\" didn't match expected format \"%s://region/InstanceID\"", providerID, ProviderName) 55 | } 56 | 57 | vmID, err := strconv.Atoi(matches[2]) 58 | if err != nil { 59 | return 0, fmt.Errorf("InstanceID have to be a number, but got \"%s\"", matches[2]) 60 | } 61 | 62 | return vmID, nil 63 | } 64 | 65 | // ParseProviderID returns the VmRef and region from the providerID. 66 | func ParseProviderID(providerID string) (*pxapi.VmRef, string, error) { 67 | if !strings.HasPrefix(providerID, ProviderName) { 68 | return nil, "", fmt.Errorf("foreign providerID or empty \"%s\"", providerID) 69 | } 70 | 71 | matches := providerIDRegexp.FindStringSubmatch(providerID) 72 | if len(matches) != 3 { 73 | return nil, "", fmt.Errorf("providerID \"%s\" didn't match expected format \"%s://region/InstanceID\"", providerID, ProviderName) 74 | } 75 | 76 | vmID, err := strconv.Atoi(matches[2]) 77 | if err != nil { 78 | return nil, "", fmt.Errorf("InstanceID have to be a number, but got \"%s\"", matches[2]) 79 | } 80 | 81 | return pxapi.NewVmRef(vmID), matches[1], nil 82 | } 83 | -------------------------------------------------------------------------------- /pkg/provider/provider_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package provider_test 18 | 19 | import ( 20 | "fmt" 21 | "testing" 22 | 23 | pxapi "github.com/Telmate/proxmox-api-go/proxmox" 24 | "github.com/stretchr/testify/assert" 25 | 26 | provider "github.com/sergelogvinov/proxmox-cloud-controller-manager/pkg/provider" 27 | ) 28 | 29 | func TestGetProviderID(t *testing.T) { 30 | t.Parallel() 31 | 32 | tests := []struct { 33 | msg string 34 | region string 35 | vmID int 36 | expectedProviderID string 37 | }{ 38 | { 39 | msg: "Valid providerID", 40 | region: "region", 41 | vmID: 123, 42 | expectedProviderID: "proxmox://region/123", 43 | }, 44 | { 45 | msg: "No region", 46 | region: "", 47 | vmID: 123, 48 | expectedProviderID: "proxmox:///123", 49 | }, 50 | } 51 | 52 | for _, testCase := range tests { 53 | testCase := testCase 54 | 55 | t.Run(fmt.Sprint(testCase.msg), func(t *testing.T) { 56 | t.Parallel() 57 | 58 | providerID := provider.GetProviderID(testCase.region, pxapi.NewVmRef(testCase.vmID)) 59 | 60 | assert.Equal(t, testCase.expectedProviderID, providerID) 61 | }) 62 | } 63 | } 64 | 65 | func TestGetVmID(t *testing.T) { 66 | t.Parallel() 67 | 68 | tests := []struct { 69 | msg string 70 | providerID string 71 | expectedError error 72 | expectedvmID int 73 | }{ 74 | { 75 | msg: "Valid VMID", 76 | providerID: "proxmox://region/123", 77 | expectedError: nil, 78 | expectedvmID: 123, 79 | }, 80 | { 81 | msg: "Valid VMID with empty region", 82 | providerID: "proxmox:///123", 83 | expectedError: nil, 84 | expectedvmID: 123, 85 | }, 86 | { 87 | msg: "Invalid providerID format", 88 | providerID: "proxmox://123", 89 | expectedError: fmt.Errorf("providerID \"proxmox://123\" didn't match expected format \"proxmox://region/InstanceID\""), 90 | }, 91 | { 92 | msg: "Non proxmox providerID", 93 | providerID: "cloud:///123", 94 | expectedError: fmt.Errorf("foreign providerID or empty \"cloud:///123\""), 95 | expectedvmID: 123, 96 | }, 97 | { 98 | msg: "Non proxmox providerID", 99 | providerID: "cloud://123", 100 | expectedError: fmt.Errorf("foreign providerID or empty \"cloud://123\""), 101 | expectedvmID: 123, 102 | }, 103 | { 104 | msg: "InValid VMID", 105 | providerID: "proxmox://region/abc", 106 | expectedError: fmt.Errorf("InstanceID have to be a number, but got \"abc\""), 107 | expectedvmID: 0, 108 | }, 109 | } 110 | 111 | for _, testCase := range tests { 112 | testCase := testCase 113 | 114 | t.Run(fmt.Sprint(testCase.msg), func(t *testing.T) { 115 | t.Parallel() 116 | 117 | VMID, err := provider.GetVMID(testCase.providerID) 118 | 119 | if testCase.expectedError != nil { 120 | assert.NotNil(t, err) 121 | assert.Equal(t, err.Error(), testCase.expectedError.Error()) 122 | } else { 123 | assert.Equal(t, testCase.expectedvmID, VMID) 124 | } 125 | }) 126 | } 127 | } 128 | 129 | func TestParseProviderID(t *testing.T) { 130 | t.Parallel() 131 | 132 | tests := []struct { 133 | msg string 134 | providerID string 135 | expectedError error 136 | expectedvmID int 137 | expectedRegion string 138 | }{ 139 | { 140 | msg: "Valid VMID", 141 | providerID: "proxmox://region/123", 142 | expectedError: nil, 143 | expectedvmID: 123, 144 | expectedRegion: "region", 145 | }, 146 | { 147 | msg: "Valid VMID with empty region", 148 | providerID: "proxmox:///123", 149 | expectedError: nil, 150 | expectedvmID: 123, 151 | expectedRegion: "", 152 | }, 153 | { 154 | msg: "Invalid providerID format", 155 | providerID: "proxmox://123", 156 | expectedError: fmt.Errorf("providerID \"proxmox://123\" didn't match expected format \"proxmox://region/InstanceID\""), 157 | }, 158 | { 159 | msg: "Non proxmox providerID", 160 | providerID: "cloud:///123", 161 | expectedError: fmt.Errorf("foreign providerID or empty \"cloud:///123\""), 162 | }, 163 | { 164 | msg: "Non proxmox providerID", 165 | providerID: "cloud://123", 166 | expectedError: fmt.Errorf("foreign providerID or empty \"cloud://123\""), 167 | }, 168 | { 169 | msg: "InValid VMID", 170 | providerID: "proxmox://region/abc", 171 | expectedError: fmt.Errorf("InstanceID have to be a number, but got \"abc\""), 172 | }, 173 | } 174 | 175 | for _, testCase := range tests { 176 | testCase := testCase 177 | 178 | t.Run(fmt.Sprint(testCase.msg), func(t *testing.T) { 179 | t.Parallel() 180 | 181 | vmr, region, err := provider.ParseProviderID(testCase.providerID) 182 | 183 | if testCase.expectedError != nil { 184 | assert.NotNil(t, err) 185 | assert.Equal(t, err.Error(), testCase.expectedError.Error()) 186 | } else { 187 | assert.NotNil(t, vmr) 188 | assert.Equal(t, testCase.expectedvmID, vmr.VmId()) 189 | assert.Equal(t, testCase.expectedRegion, region) 190 | } 191 | }) 192 | } 193 | } 194 | -------------------------------------------------------------------------------- /pkg/proxmox/cloud.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Package proxmox is main CCM definition. 18 | package proxmox 19 | 20 | import ( 21 | "context" 22 | "io" 23 | 24 | "github.com/sergelogvinov/proxmox-cloud-controller-manager/pkg/cluster" 25 | provider "github.com/sergelogvinov/proxmox-cloud-controller-manager/pkg/provider" 26 | 27 | clientkubernetes "k8s.io/client-go/kubernetes" 28 | cloudprovider "k8s.io/cloud-provider" 29 | "k8s.io/klog/v2" 30 | ) 31 | 32 | const ( 33 | // ProviderName is the name of the Proxmox provider. 34 | ProviderName = provider.ProviderName 35 | 36 | // ServiceAccountName is the service account name used in kube-system namespace. 37 | ServiceAccountName = provider.ProviderName + "-cloud-controller-manager" 38 | ) 39 | 40 | type cloud struct { 41 | client *cluster.Cluster 42 | kclient clientkubernetes.Interface 43 | instancesV2 cloudprovider.InstancesV2 44 | 45 | ctx context.Context //nolint:containedctx 46 | stop func() 47 | } 48 | 49 | func init() { 50 | cloudprovider.RegisterCloudProvider(provider.ProviderName, func(config io.Reader) (cloudprovider.Interface, error) { 51 | cfg, err := cluster.ReadCloudConfig(config) 52 | if err != nil { 53 | klog.ErrorS(err, "failed to read config") 54 | 55 | return nil, err 56 | } 57 | 58 | return newCloud(&cfg) 59 | }) 60 | } 61 | 62 | func newCloud(config *cluster.ClustersConfig) (cloudprovider.Interface, error) { 63 | client, err := cluster.NewCluster(config, nil) 64 | if err != nil { 65 | return nil, err 66 | } 67 | 68 | instancesInterface := newInstances(client, config.Features.Provider) 69 | 70 | return &cloud{ 71 | client: client, 72 | instancesV2: instancesInterface, 73 | }, nil 74 | } 75 | 76 | // Initialize provides the cloud with a kubernetes client builder and may spawn goroutines 77 | // to perform housekeeping or run custom controllers specific to the cloud provider. 78 | // Any tasks started here should be cleaned up when the stop channel closes. 79 | func (c *cloud) Initialize(clientBuilder cloudprovider.ControllerClientBuilder, stop <-chan struct{}) { 80 | c.kclient = clientBuilder.ClientOrDie(ServiceAccountName) 81 | 82 | klog.InfoS("clientset initialized") 83 | 84 | ctx, cancel := context.WithCancel(context.Background()) 85 | c.ctx = ctx 86 | c.stop = cancel 87 | 88 | err := c.client.CheckClusters(ctx) 89 | if err != nil { 90 | klog.ErrorS(err, "failed to check proxmox cluster") 91 | } 92 | 93 | // Broadcast the upstream stop signal to all provider-level goroutines 94 | // watching the provider's context for cancellation. 95 | go func(provider *cloud) { 96 | <-stop 97 | klog.V(3).InfoS("received cloud provider termination signal") 98 | provider.stop() 99 | }(c) 100 | 101 | klog.InfoS("proxmox initialized") 102 | } 103 | 104 | // LoadBalancer returns a balancer interface. 105 | // Also returns true if the interface is supported, false otherwise. 106 | func (c *cloud) LoadBalancer() (cloudprovider.LoadBalancer, bool) { 107 | return nil, false 108 | } 109 | 110 | // Instances returns an instances interface. 111 | // Also returns true if the interface is supported, false otherwise. 112 | func (c *cloud) Instances() (cloudprovider.Instances, bool) { 113 | return nil, false 114 | } 115 | 116 | // InstancesV2 is an implementation for instances and should only be implemented by external cloud providers. 117 | // Implementing InstancesV2 is behaviorally identical to Instances but is optimized to significantly reduce 118 | // API calls to the cloud provider when registering and syncing nodes. 119 | // Also returns true if the interface is supported, false otherwise. 120 | func (c *cloud) InstancesV2() (cloudprovider.InstancesV2, bool) { 121 | return c.instancesV2, c.instancesV2 != nil 122 | } 123 | 124 | // Zones returns a zones interface. 125 | // Also returns true if the interface is supported, false otherwise. 126 | func (c *cloud) Zones() (cloudprovider.Zones, bool) { 127 | return nil, false 128 | } 129 | 130 | // Clusters is not implemented. 131 | func (c *cloud) Clusters() (cloudprovider.Clusters, bool) { 132 | return nil, false 133 | } 134 | 135 | // Routes is not implemented. 136 | func (c *cloud) Routes() (cloudprovider.Routes, bool) { 137 | return nil, false 138 | } 139 | 140 | // ProviderName returns the cloud provider ID. 141 | func (c *cloud) ProviderName() string { 142 | return provider.ProviderName 143 | } 144 | 145 | // HasClusterID is not implemented. 146 | func (c *cloud) HasClusterID() bool { 147 | return true 148 | } 149 | -------------------------------------------------------------------------------- /pkg/proxmox/cloud_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package proxmox 18 | 19 | import ( 20 | "strings" 21 | "testing" 22 | 23 | "github.com/stretchr/testify/assert" 24 | 25 | "github.com/sergelogvinov/proxmox-cloud-controller-manager/pkg/cluster" 26 | provider "github.com/sergelogvinov/proxmox-cloud-controller-manager/pkg/provider" 27 | ) 28 | 29 | func TestNewCloudError(t *testing.T) { 30 | cloud, err := newCloud(&cluster.ClustersConfig{}) 31 | assert.NotNil(t, err) 32 | assert.Nil(t, cloud) 33 | assert.EqualError(t, err, "no Proxmox clusters found") 34 | } 35 | 36 | func TestCloud(t *testing.T) { 37 | cfg, err := cluster.ReadCloudConfig(strings.NewReader(` 38 | clusters: 39 | - url: https://example.com 40 | insecure: false 41 | token_id: "user!token-id" 42 | token_secret: "secret" 43 | region: cluster-1 44 | `)) 45 | assert.Nil(t, err) 46 | assert.NotNil(t, cfg) 47 | 48 | cloud, err := newCloud(&cfg) 49 | assert.Nil(t, err) 50 | assert.NotNil(t, cloud) 51 | 52 | lb, res := cloud.LoadBalancer() 53 | assert.Nil(t, lb) 54 | assert.Equal(t, res, false) 55 | 56 | ins, res := cloud.Instances() 57 | assert.Nil(t, ins) 58 | assert.Equal(t, res, false) 59 | 60 | ins2, res := cloud.InstancesV2() 61 | assert.NotNil(t, ins2) 62 | assert.Equal(t, res, true) 63 | 64 | zone, res := cloud.Zones() 65 | assert.Nil(t, zone) 66 | assert.Equal(t, res, false) 67 | 68 | cl, res := cloud.Clusters() 69 | assert.Nil(t, cl) 70 | assert.Equal(t, res, false) 71 | 72 | route, res := cloud.Routes() 73 | assert.Nil(t, route) 74 | assert.Equal(t, res, false) 75 | 76 | pName := cloud.ProviderName() 77 | assert.Equal(t, pName, provider.ProviderName) 78 | 79 | clID := cloud.HasClusterID() 80 | assert.Equal(t, clID, true) 81 | } 82 | -------------------------------------------------------------------------------- /pkg/proxmox/instances.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package proxmox 18 | 19 | import ( 20 | "context" 21 | "fmt" 22 | "regexp" 23 | "strconv" 24 | "strings" 25 | 26 | pxapi "github.com/Telmate/proxmox-api-go/proxmox" 27 | 28 | "github.com/sergelogvinov/proxmox-cloud-controller-manager/pkg/cluster" 29 | metrics "github.com/sergelogvinov/proxmox-cloud-controller-manager/pkg/metrics" 30 | provider "github.com/sergelogvinov/proxmox-cloud-controller-manager/pkg/provider" 31 | 32 | v1 "k8s.io/api/core/v1" 33 | cloudprovider "k8s.io/cloud-provider" 34 | cloudproviderapi "k8s.io/cloud-provider/api" 35 | "k8s.io/klog/v2" 36 | ) 37 | 38 | type instances struct { 39 | c *cluster.Cluster 40 | provider cluster.Provider 41 | } 42 | 43 | var instanceTypeNameRegexp = regexp.MustCompile(`(^[a-zA-Z0-9_.-]+)$`) 44 | 45 | func newInstances(client *cluster.Cluster, provider cluster.Provider) *instances { 46 | return &instances{ 47 | c: client, 48 | provider: provider, 49 | } 50 | } 51 | 52 | // InstanceExists returns true if the instance for the given node exists according to the cloud provider. 53 | // Use the node.name or node.spec.providerID field to find the node in the cloud provider. 54 | func (i *instances) InstanceExists(ctx context.Context, node *v1.Node) (bool, error) { 55 | klog.V(4).InfoS("instances.InstanceExists() called", "node", klog.KRef("", node.Name)) 56 | 57 | if node.Spec.ProviderID == "" { 58 | klog.V(4).InfoS("instances.InstanceExists() empty providerID, omitting unmanaged node", "node", klog.KObj(node)) 59 | 60 | return true, nil 61 | } 62 | 63 | if !strings.HasPrefix(node.Spec.ProviderID, provider.ProviderName) { 64 | klog.V(4).InfoS("instances.InstanceExists() omitting unmanaged node", "node", klog.KObj(node), "providerID", node.Spec.ProviderID) 65 | 66 | return true, nil 67 | } 68 | 69 | mc := metrics.NewMetricContext("getVmInfo") 70 | if _, _, err := i.getInstance(ctx, node); mc.ObserveRequest(err) != nil { 71 | if err == cloudprovider.InstanceNotFound { 72 | klog.V(4).InfoS("instances.InstanceExists() instance not found", "node", klog.KObj(node), "providerID", node.Spec.ProviderID) 73 | 74 | return false, nil 75 | } 76 | 77 | return false, err 78 | } 79 | 80 | return true, nil 81 | } 82 | 83 | // InstanceShutdown returns true if the instance is shutdown according to the cloud provider. 84 | // Use the node.name or node.spec.providerID field to find the node in the cloud provider. 85 | func (i *instances) InstanceShutdown(ctx context.Context, node *v1.Node) (bool, error) { 86 | klog.V(4).InfoS("instances.InstanceShutdown() called", "node", klog.KRef("", node.Name)) 87 | 88 | if node.Spec.ProviderID == "" { 89 | klog.V(4).InfoS("instances.InstanceShutdown() empty providerID, omitting unmanaged node", "node", klog.KObj(node)) 90 | 91 | return false, nil 92 | } 93 | 94 | if !strings.HasPrefix(node.Spec.ProviderID, provider.ProviderName) { 95 | klog.V(4).InfoS("instances.InstanceShutdown() omitting unmanaged node", "node", klog.KObj(node), "providerID", node.Spec.ProviderID) 96 | 97 | return false, nil 98 | } 99 | 100 | vmr, region, err := provider.ParseProviderID(node.Spec.ProviderID) 101 | if err != nil { 102 | klog.ErrorS(err, "instances.InstanceShutdown() failed to parse providerID", "providerID", node.Spec.ProviderID) 103 | 104 | return false, nil 105 | } 106 | 107 | px, err := i.c.GetProxmoxCluster(region) 108 | if err != nil { 109 | klog.ErrorS(err, "instances.InstanceShutdown() failed to get Proxmox cluster", "region", region) 110 | 111 | return false, nil 112 | } 113 | 114 | mc := metrics.NewMetricContext("getVmState") 115 | 116 | vmState, err := px.GetVmState(ctx, vmr) 117 | if mc.ObserveRequest(err) != nil { 118 | return false, err 119 | } 120 | 121 | if vmState["status"].(string) == "stopped" { //nolint:errcheck 122 | return true, nil 123 | } 124 | 125 | return false, nil 126 | } 127 | 128 | // InstanceMetadata returns the instance's metadata. The values returned in InstanceMetadata are 129 | // translated into specific fields in the Node object on registration. 130 | // Use the node.name or node.spec.providerID field to find the node in the cloud provider. 131 | func (i *instances) InstanceMetadata(ctx context.Context, node *v1.Node) (*cloudprovider.InstanceMetadata, error) { 132 | klog.V(4).InfoS("instances.InstanceMetadata() called", "node", klog.KRef("", node.Name)) 133 | 134 | if providedIP, ok := node.ObjectMeta.Annotations[cloudproviderapi.AnnotationAlphaProvidedIPAddr]; ok { 135 | var ( 136 | vmRef *pxapi.VmRef 137 | region string 138 | err error 139 | ) 140 | 141 | providerID := node.Spec.ProviderID 142 | if providerID == "" { 143 | uuid := node.Status.NodeInfo.SystemUUID 144 | 145 | klog.V(4).InfoS("instances.InstanceMetadata() empty providerID, trying find node", "node", klog.KObj(node), "uuid", uuid) 146 | 147 | mc := metrics.NewMetricContext("findVmByName") 148 | 149 | vmRef, region, err = i.c.FindVMByNode(ctx, node) 150 | if mc.ObserveRequest(err) != nil { 151 | mc := metrics.NewMetricContext("findVmByUUID") 152 | 153 | vmRef, region, err = i.c.FindVMByUUID(ctx, uuid) 154 | if mc.ObserveRequest(err) != nil { 155 | return nil, fmt.Errorf("instances.InstanceMetadata() - failed to find instance by name/uuid %s: %v, skipped", node.Name, err) 156 | } 157 | } 158 | 159 | if i.provider == cluster.ProviderCapmox { 160 | providerID = provider.GetProviderIDFromUUID(uuid) 161 | } else { 162 | providerID = provider.GetProviderID(region, vmRef) 163 | } 164 | } else if !strings.HasPrefix(node.Spec.ProviderID, provider.ProviderName) { 165 | klog.V(4).InfoS("instances.InstanceMetadata() omitting unmanaged node", "node", klog.KObj(node), "providerID", node.Spec.ProviderID) 166 | 167 | return &cloudprovider.InstanceMetadata{}, nil 168 | } 169 | 170 | if vmRef == nil { 171 | mc := metrics.NewMetricContext("getVmInfo") 172 | 173 | vmRef, region, err = i.getInstance(ctx, node) 174 | if mc.ObserveRequest(err) != nil { 175 | return nil, err 176 | } 177 | } 178 | 179 | addresses := []v1.NodeAddress{} 180 | 181 | for _, ip := range strings.Split(providedIP, ",") { 182 | addresses = append(addresses, v1.NodeAddress{Type: v1.NodeInternalIP, Address: ip}) 183 | } 184 | 185 | addresses = append(addresses, v1.NodeAddress{Type: v1.NodeHostName, Address: node.Name}) 186 | 187 | instanceType, err := i.getInstanceType(ctx, vmRef, region) 188 | if err != nil { 189 | instanceType = vmRef.GetVmType() 190 | } 191 | 192 | return &cloudprovider.InstanceMetadata{ 193 | ProviderID: providerID, 194 | NodeAddresses: addresses, 195 | InstanceType: instanceType, 196 | Zone: vmRef.Node().String(), 197 | Region: region, 198 | }, nil 199 | } 200 | 201 | klog.InfoS("instances.InstanceMetadata() is kubelet has args: --cloud-provider=external on the node?", node, klog.KRef("", node.Name)) 202 | 203 | return &cloudprovider.InstanceMetadata{}, nil 204 | } 205 | 206 | func (i *instances) getInstance(ctx context.Context, node *v1.Node) (*pxapi.VmRef, string, error) { 207 | klog.V(4).InfoS("instances.getInstance() called", "node", klog.KRef("", node.Name), "provider", i.provider) 208 | 209 | if i.provider == cluster.ProviderCapmox { 210 | uuid := node.Status.NodeInfo.SystemUUID 211 | 212 | vmRef, region, err := i.c.FindVMByUUID(ctx, uuid) 213 | if err != nil { 214 | return nil, "", fmt.Errorf("instances.getInstance() error: %v", err) 215 | } 216 | 217 | return vmRef, region, nil 218 | } 219 | 220 | vmRef, region, err := provider.ParseProviderID(node.Spec.ProviderID) 221 | if err != nil { 222 | return nil, "", fmt.Errorf("instances.getInstance() error: %v", err) 223 | } 224 | 225 | px, err := i.c.GetProxmoxCluster(region) 226 | if err != nil { 227 | return nil, "", fmt.Errorf("instances.getInstance() error: %v", err) 228 | } 229 | 230 | mc := metrics.NewMetricContext("getVmInfo") 231 | 232 | vmConfig, err := px.GetVmConfig(ctx, vmRef) 233 | if mc.ObserveRequest(err) != nil { 234 | if strings.Contains(err.Error(), "not found") { 235 | return nil, "", cloudprovider.InstanceNotFound 236 | } 237 | 238 | return nil, "", err 239 | } 240 | 241 | if i.c.GetVMName(vmConfig) != node.Name || i.c.GetVMUUID(vmConfig) != node.Status.NodeInfo.SystemUUID { 242 | klog.Errorf("instances.getInstance() vm.name(%s) != node.name(%s) with uuid=%s", i.c.GetVMName(vmConfig), node.Name, node.Status.NodeInfo.SystemUUID) 243 | 244 | return nil, "", cloudprovider.InstanceNotFound 245 | } 246 | 247 | klog.V(5).Infof("instances.getInstance() vmConfig %+v", vmConfig) 248 | 249 | return vmRef, region, nil 250 | } 251 | 252 | func (i *instances) getInstanceType(ctx context.Context, vmRef *pxapi.VmRef, region string) (string, error) { 253 | px, err := i.c.GetProxmoxCluster(region) 254 | if err != nil { 255 | return "", err 256 | } 257 | 258 | mc := metrics.NewMetricContext("getVmInfo") 259 | 260 | vmConfig, err := px.GetVmConfig(ctx, vmRef) 261 | if mc.ObserveRequest(err) != nil { 262 | return "", err 263 | } 264 | 265 | sku := i.c.GetVMSKU(vmConfig) 266 | if sku != "" && instanceTypeNameRegexp.MatchString(sku) { 267 | return sku, nil 268 | } 269 | 270 | if vmConfig["cores"] == nil || vmConfig["memory"] == nil { 271 | return "", fmt.Errorf("instances.getInstanceType() failed to get instance type") 272 | } 273 | 274 | memory, err := strconv.Atoi(vmConfig["memory"].(string)) //nolint:errcheck 275 | if err != nil { 276 | return "", err 277 | } 278 | 279 | return fmt.Sprintf("%.0fVCPU-%.0fGB", 280 | vmConfig["cores"].(float64), //nolint:errcheck 281 | float64(memory)/1024), nil 282 | } 283 | -------------------------------------------------------------------------------- /pkg/proxmox/instances_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package proxmox 18 | 19 | import ( 20 | "context" 21 | "fmt" 22 | "net/http" 23 | "strings" 24 | "testing" 25 | 26 | pxapi "github.com/Telmate/proxmox-api-go/proxmox" 27 | "github.com/jarcoal/httpmock" 28 | "github.com/stretchr/testify/assert" 29 | "github.com/stretchr/testify/suite" 30 | 31 | proxmoxcluster "github.com/sergelogvinov/proxmox-cloud-controller-manager/pkg/cluster" 32 | "github.com/sergelogvinov/proxmox-cloud-controller-manager/pkg/provider" 33 | 34 | v1 "k8s.io/api/core/v1" 35 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 36 | cloudprovider "k8s.io/cloud-provider" 37 | cloudproviderapi "k8s.io/cloud-provider/api" 38 | ) 39 | 40 | type ccmTestSuite struct { 41 | suite.Suite 42 | 43 | i *instances 44 | } 45 | 46 | func (ts *ccmTestSuite) SetupTest() { 47 | cfg, err := proxmoxcluster.ReadCloudConfig(strings.NewReader(` 48 | clusters: 49 | - url: https://127.0.0.1:8006/api2/json 50 | insecure: false 51 | token_id: "user!token-id" 52 | token_secret: "secret" 53 | region: cluster-1 54 | - url: https://127.0.0.2:8006/api2/json 55 | insecure: false 56 | token_id: "user!token-id" 57 | token_secret: "secret" 58 | region: cluster-2 59 | `)) 60 | if err != nil { 61 | ts.T().Fatalf("failed to read config: %v", err) 62 | } 63 | 64 | httpmock.RegisterResponder("GET", "https://127.0.0.1:8006/api2/json/cluster/resources", 65 | func(_ *http.Request) (*http.Response, error) { 66 | return httpmock.NewJsonResponse(200, map[string]interface{}{ 67 | "data": []interface{}{ 68 | map[string]interface{}{ 69 | "node": "pve-1", 70 | "type": "qemu", 71 | "vmid": 100, 72 | "name": "cluster-1-node-1", 73 | "maxcpu": 4, 74 | "maxmem": 10 * 1024 * 1024 * 1024, 75 | }, 76 | map[string]interface{}{ 77 | "node": "pve-2", 78 | "type": "qemu", 79 | "vmid": 101, 80 | "name": "cluster-1-node-2", 81 | "maxcpu": 2, 82 | "maxmem": 5 * 1024 * 1024 * 1024, 83 | }, 84 | }, 85 | }) 86 | }, 87 | ) 88 | 89 | httpmock.RegisterResponder("GET", "https://127.0.0.2:8006/api2/json/cluster/resources", 90 | func(_ *http.Request) (*http.Response, error) { 91 | return httpmock.NewJsonResponse(200, map[string]interface{}{ 92 | "data": []interface{}{ 93 | map[string]interface{}{ 94 | "node": "pve-3", 95 | "type": "qemu", 96 | "vmid": 100, 97 | "name": "cluster-2-node-1", 98 | "maxcpu": 1, 99 | "maxmem": 2 * 1024 * 1024 * 1024, 100 | }, 101 | }, 102 | }) 103 | }, 104 | ) 105 | 106 | httpmock.RegisterResponder("GET", "https://127.0.0.1:8006/api2/json/nodes/pve-1/qemu/100/config", 107 | func(_ *http.Request) (*http.Response, error) { 108 | return httpmock.NewJsonResponse(200, map[string]interface{}{ 109 | "data": map[string]interface{}{ 110 | "name": "cluster-1-node-1", 111 | "node": "pve-1", 112 | "type": "qemu", 113 | "vmid": 100, 114 | "cores": 4, 115 | "memory": "10240", 116 | "smbios1": "uuid=8af7110d-bfad-407a-a663-9527d10a6583", 117 | }, 118 | }) 119 | }, 120 | ) 121 | 122 | httpmock.RegisterResponder("GET", "https://127.0.0.1:8006/api2/json/nodes/pve-2/qemu/101/config", 123 | func(_ *http.Request) (*http.Response, error) { 124 | return httpmock.NewJsonResponse(200, map[string]interface{}{ 125 | "data": map[string]interface{}{ 126 | "name": "cluster-1-node-2", 127 | "node": "pve-2", 128 | "type": "qemu", 129 | "vmid": 101, 130 | "cores": 2, 131 | "memory": "5120", 132 | "smbios1": "uuid=5d04cb23-ea78-40a3-af2e-dd54798dc887", 133 | }, 134 | }) 135 | }, 136 | ) 137 | 138 | httpmock.RegisterResponder("GET", "https://127.0.0.2:8006/api2/json/nodes/pve-3/qemu/100/config", 139 | func(_ *http.Request) (*http.Response, error) { 140 | return httpmock.NewJsonResponse(200, map[string]interface{}{ 141 | "data": map[string]interface{}{ 142 | "name": "cluster-2-node-1", 143 | "node": "pve-3", 144 | "type": "qemu", 145 | "vmid": 100, 146 | "cores": 1, 147 | "memory": "2048", 148 | "smbios1": "uuid=3d3db687-89dd-473e-8463-6599f25b36a8,sku=YzEubWVkaXVt", 149 | }, 150 | }) 151 | }, 152 | ) 153 | 154 | httpmock.RegisterResponder("GET", "https://127.0.0.1:8006/api2/json/nodes/pve-1/qemu/100/status/current", 155 | func(_ *http.Request) (*http.Response, error) { 156 | return httpmock.NewJsonResponse(200, map[string]interface{}{ 157 | "data": map[string]interface{}{ 158 | "status": "running", 159 | }, 160 | }) 161 | }, 162 | ) 163 | 164 | httpmock.RegisterResponder("GET", "https://127.0.0.2:8006/api2/json/nodes/pve-3/qemu/100/status/current", 165 | func(_ *http.Request) (*http.Response, error) { 166 | return httpmock.NewJsonResponse(200, map[string]interface{}{ 167 | "data": map[string]interface{}{ 168 | "status": "stopped", 169 | }, 170 | }) 171 | }, 172 | ) 173 | 174 | cluster, err := proxmoxcluster.NewCluster(&cfg, &http.Client{}) 175 | if err != nil { 176 | ts.T().Fatalf("failed to create cluster client: %v", err) 177 | } 178 | 179 | ts.i = newInstances(cluster, proxmoxcluster.ProviderDefault) 180 | } 181 | 182 | func (ts *ccmTestSuite) TearDownTest() { 183 | } 184 | 185 | func TestSuiteCCM(t *testing.T) { 186 | suite.Run(t, new(ccmTestSuite)) 187 | } 188 | 189 | // nolint:dupl 190 | func (ts *ccmTestSuite) TestInstanceExists() { 191 | httpmock.Activate() 192 | defer httpmock.DeactivateAndReset() 193 | 194 | tests := []struct { 195 | msg string 196 | node *v1.Node 197 | expectedError string 198 | expected bool 199 | }{ 200 | { 201 | msg: "NodeForeignProviderID", 202 | node: &v1.Node{ 203 | ObjectMeta: metav1.ObjectMeta{ 204 | Name: "test-node-1", 205 | }, 206 | Spec: v1.NodeSpec{ 207 | ProviderID: "foreign://provider-id", 208 | }, 209 | }, 210 | expected: true, 211 | }, 212 | { 213 | msg: "NodeWrongCluster", 214 | node: &v1.Node{ 215 | ObjectMeta: metav1.ObjectMeta{ 216 | Name: "cluster-3-node-1", 217 | }, 218 | Spec: v1.NodeSpec{ 219 | ProviderID: "proxmox://cluster-3/100", 220 | }, 221 | }, 222 | expected: false, 223 | expectedError: "instances.getInstance() error: proxmox cluster cluster-3 not found", 224 | }, 225 | { 226 | msg: "NodeNotExists", 227 | node: &v1.Node{ 228 | ObjectMeta: metav1.ObjectMeta{ 229 | Name: "cluster-1-node-500", 230 | }, 231 | Spec: v1.NodeSpec{ 232 | ProviderID: "proxmox://cluster-1/500", 233 | }, 234 | }, 235 | expected: false, 236 | }, 237 | { 238 | msg: "NodeExists", 239 | node: &v1.Node{ 240 | ObjectMeta: metav1.ObjectMeta{ 241 | Name: "cluster-1-node-1", 242 | }, 243 | Spec: v1.NodeSpec{ 244 | ProviderID: "proxmox://cluster-1/100", 245 | }, 246 | Status: v1.NodeStatus{ 247 | NodeInfo: v1.NodeSystemInfo{ 248 | SystemUUID: "8af7110d-bfad-407a-a663-9527d10a6583", 249 | }, 250 | }, 251 | }, 252 | expected: true, 253 | }, 254 | { 255 | msg: "NodeExistsWithDifferentName", 256 | node: &v1.Node{ 257 | ObjectMeta: metav1.ObjectMeta{ 258 | Name: "cluster-1-node-3", 259 | }, 260 | Spec: v1.NodeSpec{ 261 | ProviderID: "proxmox://cluster-1/100", 262 | }, 263 | Status: v1.NodeStatus{ 264 | NodeInfo: v1.NodeSystemInfo{ 265 | SystemUUID: "8af7110d-bfad-407a-a663-9527d10a6583", 266 | }, 267 | }, 268 | }, 269 | expected: false, 270 | }, 271 | { 272 | msg: "NodeExistsWithDifferentUUID", 273 | node: &v1.Node{ 274 | ObjectMeta: metav1.ObjectMeta{ 275 | Name: "cluster-1-node-1", 276 | }, 277 | Spec: v1.NodeSpec{ 278 | ProviderID: "proxmox://cluster-1/100", 279 | }, 280 | Status: v1.NodeStatus{ 281 | NodeInfo: v1.NodeSystemInfo{ 282 | SystemUUID: "8af7110d-0000-0000-0000-9527d10a6583", 283 | }, 284 | }, 285 | }, 286 | expected: false, 287 | }, 288 | { 289 | msg: "NodeExistsWithDifferentNameAndUUID", 290 | node: &v1.Node{ 291 | ObjectMeta: metav1.ObjectMeta{ 292 | Name: "cluster-1-node-3", 293 | }, 294 | Spec: v1.NodeSpec{ 295 | ProviderID: "proxmox://cluster-1/100", 296 | }, 297 | Status: v1.NodeStatus{ 298 | NodeInfo: v1.NodeSystemInfo{ 299 | SystemUUID: "8af7110d-0000-0000-0000-9527d10a6583", 300 | }, 301 | }, 302 | }, 303 | expected: false, 304 | }, 305 | } 306 | 307 | for _, testCase := range tests { 308 | testCase := testCase 309 | 310 | ts.Run(fmt.Sprint(testCase.msg), func() { 311 | exists, err := ts.i.InstanceExists(context.Background(), testCase.node) 312 | 313 | if testCase.expectedError != "" { 314 | ts.Require().Error(err) 315 | ts.Require().False(exists) 316 | ts.Require().Contains(err.Error(), testCase.expectedError) 317 | } else { 318 | ts.Require().NoError(err) 319 | ts.Require().Equal(testCase.expected, exists) 320 | } 321 | }) 322 | } 323 | } 324 | 325 | // nolint:dupl 326 | func (ts *ccmTestSuite) TestInstanceShutdown() { 327 | httpmock.Activate() 328 | defer httpmock.DeactivateAndReset() 329 | 330 | tests := []struct { 331 | msg string 332 | node *v1.Node 333 | expectedError string 334 | expected bool 335 | }{ 336 | { 337 | msg: "NodeForeignProviderID", 338 | node: &v1.Node{ 339 | ObjectMeta: metav1.ObjectMeta{ 340 | Name: "test-node-1", 341 | }, 342 | Spec: v1.NodeSpec{ 343 | ProviderID: "foreign://provider-id", 344 | }, 345 | }, 346 | expected: false, 347 | }, 348 | { 349 | msg: "NodeWrongCluster", 350 | node: &v1.Node{ 351 | ObjectMeta: metav1.ObjectMeta{ 352 | Name: "cluster-3-node-1", 353 | }, 354 | Spec: v1.NodeSpec{ 355 | ProviderID: "proxmox://cluster-3/100", 356 | }, 357 | }, 358 | expected: false, 359 | }, 360 | { 361 | msg: "NodeNotExists", 362 | node: &v1.Node{ 363 | ObjectMeta: metav1.ObjectMeta{ 364 | Name: "cluster-1-node-500", 365 | }, 366 | Spec: v1.NodeSpec{ 367 | ProviderID: "proxmox://cluster-1/500", 368 | }, 369 | }, 370 | expected: false, 371 | expectedError: "vm '500' not found", 372 | }, 373 | { 374 | msg: "NodeExists", 375 | node: &v1.Node{ 376 | ObjectMeta: metav1.ObjectMeta{ 377 | Name: "cluster-1-node-1", 378 | }, 379 | Spec: v1.NodeSpec{ 380 | ProviderID: "proxmox://cluster-1/100", 381 | }, 382 | Status: v1.NodeStatus{ 383 | NodeInfo: v1.NodeSystemInfo{ 384 | SystemUUID: "8af7110d-bfad-407a-a663-9527d10a6583", 385 | }, 386 | }, 387 | }, 388 | expected: false, 389 | }, 390 | { 391 | msg: "NodeExistsStopped", 392 | node: &v1.Node{ 393 | ObjectMeta: metav1.ObjectMeta{ 394 | Name: "cluster-1-node-3", 395 | }, 396 | Spec: v1.NodeSpec{ 397 | ProviderID: "proxmox://cluster-2/100", 398 | }, 399 | }, 400 | expected: true, 401 | }, 402 | { 403 | msg: "NodeExistsWithDifferentName", 404 | node: &v1.Node{ 405 | ObjectMeta: metav1.ObjectMeta{ 406 | Name: "cluster-1-node-3", 407 | }, 408 | Spec: v1.NodeSpec{ 409 | ProviderID: "proxmox://cluster-1/100", 410 | }, 411 | Status: v1.NodeStatus{ 412 | NodeInfo: v1.NodeSystemInfo{ 413 | SystemUUID: "8af7110d-bfad-407a-a663-9527d10a6583", 414 | }, 415 | }, 416 | }, 417 | expected: false, 418 | }, 419 | { 420 | msg: "NodeExistsWithDifferentUUID", 421 | node: &v1.Node{ 422 | ObjectMeta: metav1.ObjectMeta{ 423 | Name: "cluster-1-node-1", 424 | }, 425 | Spec: v1.NodeSpec{ 426 | ProviderID: "proxmox://cluster-1/100", 427 | }, 428 | Status: v1.NodeStatus{ 429 | NodeInfo: v1.NodeSystemInfo{ 430 | SystemUUID: "8af7110d-0000-0000-0000-9527d10a6583", 431 | }, 432 | }, 433 | }, 434 | expected: false, 435 | }, 436 | { 437 | msg: "NodeExistsWithDifferentNameAndUUID", 438 | node: &v1.Node{ 439 | ObjectMeta: metav1.ObjectMeta{ 440 | Name: "cluster-1-node-3", 441 | }, 442 | Spec: v1.NodeSpec{ 443 | ProviderID: "proxmox://cluster-1/100", 444 | }, 445 | Status: v1.NodeStatus{ 446 | NodeInfo: v1.NodeSystemInfo{ 447 | SystemUUID: "8af7110d-0000-0000-0000-9527d10a6583", 448 | }, 449 | }, 450 | }, 451 | expected: false, 452 | }, 453 | } 454 | 455 | for _, testCase := range tests { 456 | testCase := testCase 457 | 458 | ts.Run(fmt.Sprint(testCase.msg), func() { 459 | exists, err := ts.i.InstanceShutdown(context.Background(), testCase.node) 460 | 461 | if testCase.expectedError != "" { 462 | ts.Require().Error(err) 463 | ts.Require().False(exists) 464 | ts.Require().Contains(err.Error(), testCase.expectedError) 465 | } else { 466 | ts.Require().NoError(err) 467 | ts.Require().Equal(testCase.expected, exists) 468 | } 469 | }) 470 | } 471 | } 472 | 473 | func (ts *ccmTestSuite) TestInstanceMetadata() { 474 | httpmock.Activate() 475 | defer httpmock.DeactivateAndReset() 476 | 477 | tests := []struct { 478 | msg string 479 | node *v1.Node 480 | expectedError string 481 | expected *cloudprovider.InstanceMetadata 482 | }{ 483 | { 484 | msg: "NodeAnnotations", 485 | node: &v1.Node{ 486 | ObjectMeta: metav1.ObjectMeta{ 487 | Name: "test-node-1", 488 | }, 489 | }, 490 | expected: &cloudprovider.InstanceMetadata{}, 491 | }, 492 | { 493 | msg: "NodeForeignProviderID", 494 | node: &v1.Node{ 495 | ObjectMeta: metav1.ObjectMeta{ 496 | Name: "test-node-1", 497 | Annotations: map[string]string{ 498 | cloudproviderapi.AnnotationAlphaProvidedIPAddr: "1.2.3.4", 499 | }, 500 | }, 501 | Spec: v1.NodeSpec{ 502 | ProviderID: "foreign://provider-id", 503 | }, 504 | }, 505 | expected: &cloudprovider.InstanceMetadata{}, 506 | }, 507 | { 508 | msg: "NodeWrongCluster", 509 | node: &v1.Node{ 510 | ObjectMeta: metav1.ObjectMeta{ 511 | Name: "cluster-3-node-1", 512 | Annotations: map[string]string{ 513 | cloudproviderapi.AnnotationAlphaProvidedIPAddr: "1.2.3.4", 514 | }, 515 | }, 516 | Spec: v1.NodeSpec{ 517 | ProviderID: "proxmox://cluster-3/100", 518 | }, 519 | }, 520 | expected: &cloudprovider.InstanceMetadata{}, 521 | expectedError: "instances.getInstance() error: proxmox cluster cluster-3 not found", 522 | }, 523 | { 524 | msg: "NodeNotExists", 525 | node: &v1.Node{ 526 | ObjectMeta: metav1.ObjectMeta{ 527 | Name: "cluster-1-node-500", 528 | Annotations: map[string]string{ 529 | cloudproviderapi.AnnotationAlphaProvidedIPAddr: "1.2.3.4", 530 | }, 531 | }, 532 | Spec: v1.NodeSpec{ 533 | ProviderID: "proxmox://cluster-1/500", 534 | }, 535 | }, 536 | expected: &cloudprovider.InstanceMetadata{}, 537 | expectedError: cloudprovider.InstanceNotFound.Error(), 538 | }, 539 | { 540 | msg: "NodeExists", 541 | node: &v1.Node{ 542 | ObjectMeta: metav1.ObjectMeta{ 543 | Name: "cluster-1-node-1", 544 | Annotations: map[string]string{ 545 | cloudproviderapi.AnnotationAlphaProvidedIPAddr: "1.2.3.4", 546 | }, 547 | }, 548 | Status: v1.NodeStatus{ 549 | NodeInfo: v1.NodeSystemInfo{ 550 | SystemUUID: "8af7110d-bfad-407a-a663-9527d10a6583", 551 | }, 552 | }, 553 | }, 554 | expected: &cloudprovider.InstanceMetadata{ 555 | ProviderID: "proxmox://cluster-1/100", 556 | NodeAddresses: []v1.NodeAddress{ 557 | { 558 | Type: v1.NodeInternalIP, 559 | Address: "1.2.3.4", 560 | }, 561 | { 562 | Type: v1.NodeHostName, 563 | Address: "cluster-1-node-1", 564 | }, 565 | }, 566 | InstanceType: "4VCPU-10GB", 567 | Region: "cluster-1", 568 | Zone: "pve-1", 569 | }, 570 | }, 571 | { 572 | msg: "NodeExistsDualstack", 573 | node: &v1.Node{ 574 | ObjectMeta: metav1.ObjectMeta{ 575 | Name: "cluster-1-node-1", 576 | Annotations: map[string]string{ 577 | cloudproviderapi.AnnotationAlphaProvidedIPAddr: "1.2.3.4,2001::1", 578 | }, 579 | }, 580 | Status: v1.NodeStatus{ 581 | NodeInfo: v1.NodeSystemInfo{ 582 | SystemUUID: "8af7110d-bfad-407a-a663-9527d10a6583", 583 | }, 584 | }, 585 | }, 586 | expected: &cloudprovider.InstanceMetadata{ 587 | ProviderID: "proxmox://cluster-1/100", 588 | NodeAddresses: []v1.NodeAddress{ 589 | { 590 | Type: v1.NodeInternalIP, 591 | Address: "1.2.3.4", 592 | }, 593 | { 594 | Type: v1.NodeInternalIP, 595 | Address: "2001::1", 596 | }, 597 | { 598 | Type: v1.NodeHostName, 599 | Address: "cluster-1-node-1", 600 | }, 601 | }, 602 | InstanceType: "4VCPU-10GB", 603 | Region: "cluster-1", 604 | Zone: "pve-1", 605 | }, 606 | }, 607 | { 608 | msg: "NodeExistsCluster2", 609 | node: &v1.Node{ 610 | ObjectMeta: metav1.ObjectMeta{ 611 | Name: "cluster-2-node-1", 612 | Annotations: map[string]string{ 613 | cloudproviderapi.AnnotationAlphaProvidedIPAddr: "1.2.3.4", 614 | }, 615 | }, 616 | Status: v1.NodeStatus{ 617 | NodeInfo: v1.NodeSystemInfo{ 618 | SystemUUID: "3d3db687-89dd-473e-8463-6599f25b36a8", 619 | }, 620 | }, 621 | }, 622 | expected: &cloudprovider.InstanceMetadata{ 623 | ProviderID: "proxmox://cluster-2/100", 624 | NodeAddresses: []v1.NodeAddress{ 625 | { 626 | Type: v1.NodeInternalIP, 627 | Address: "1.2.3.4", 628 | }, 629 | { 630 | Type: v1.NodeHostName, 631 | Address: "cluster-2-node-1", 632 | }, 633 | }, 634 | InstanceType: "c1.medium", 635 | Region: "cluster-2", 636 | Zone: "pve-3", 637 | }, 638 | }, 639 | } 640 | 641 | for _, testCase := range tests { 642 | testCase := testCase 643 | 644 | ts.Run(fmt.Sprint(testCase.msg), func() { 645 | meta, err := ts.i.InstanceMetadata(context.Background(), testCase.node) 646 | 647 | if testCase.expectedError != "" { 648 | ts.Require().Error(err) 649 | ts.Require().Contains(err.Error(), testCase.expectedError) 650 | } else { 651 | ts.Require().NoError(err) 652 | ts.Require().Equal(testCase.expected, meta) 653 | } 654 | }) 655 | } 656 | } 657 | 658 | func TestGetProviderID(t *testing.T) { 659 | t.Parallel() 660 | 661 | tests := []struct { 662 | msg string 663 | region string 664 | vmr *pxapi.VmRef 665 | expected string 666 | }{ 667 | { 668 | msg: "empty region", 669 | region: "", 670 | vmr: pxapi.NewVmRef(100), 671 | expected: "proxmox:///100", 672 | }, 673 | { 674 | msg: "region", 675 | region: "cluster1", 676 | vmr: pxapi.NewVmRef(100), 677 | expected: "proxmox://cluster1/100", 678 | }, 679 | } 680 | 681 | for _, testCase := range tests { 682 | testCase := testCase 683 | 684 | t.Run(fmt.Sprint(testCase.msg), func(t *testing.T) { 685 | t.Parallel() 686 | 687 | expected := provider.GetProviderID(testCase.region, testCase.vmr) 688 | assert.Equal(t, expected, testCase.expected) 689 | }) 690 | } 691 | } 692 | 693 | func TestParseProviderID(t *testing.T) { 694 | t.Parallel() 695 | 696 | tests := []struct { 697 | msg string 698 | magic string 699 | expectedCluster string 700 | expectedVmr *pxapi.VmRef 701 | expectedError error 702 | }{ 703 | { 704 | msg: "Empty magic string", 705 | magic: "", 706 | expectedError: fmt.Errorf("foreign providerID or empty \"\""), 707 | }, 708 | { 709 | msg: "Wrong provider", 710 | magic: "provider://region/100", 711 | expectedError: fmt.Errorf("foreign providerID or empty \"provider://region/100\""), 712 | }, 713 | { 714 | msg: "Empty region", 715 | magic: "proxmox:///100", 716 | expectedCluster: "", 717 | expectedVmr: pxapi.NewVmRef(100), 718 | }, 719 | { 720 | msg: "Empty region", 721 | magic: "proxmox://100", 722 | expectedError: fmt.Errorf("providerID \"proxmox://100\" didn't match expected format \"proxmox://region/InstanceID\""), 723 | }, 724 | { 725 | msg: "Cluster and InstanceID", 726 | magic: "proxmox://cluster/100", 727 | expectedCluster: "cluster", 728 | expectedVmr: pxapi.NewVmRef(100), 729 | }, 730 | { 731 | msg: "Cluster and wrong InstanceID", 732 | magic: "proxmox://cluster/name", 733 | expectedError: fmt.Errorf("InstanceID have to be a number, but got \"name\""), 734 | }, 735 | } 736 | 737 | for _, testCase := range tests { 738 | testCase := testCase 739 | 740 | t.Run(fmt.Sprint(testCase.msg), func(t *testing.T) { 741 | t.Parallel() 742 | 743 | vmr, cluster, err := provider.ParseProviderID(testCase.magic) 744 | 745 | if testCase.expectedError != nil { 746 | assert.Equal(t, testCase.expectedError, err) 747 | } else { 748 | assert.Equal(t, testCase.expectedVmr, vmr) 749 | assert.Equal(t, testCase.expectedCluster, cluster) 750 | } 751 | }) 752 | } 753 | } 754 | --------------------------------------------------------------------------------