├── .conform.yaml ├── .dockerignore ├── .github ├── ISSUE_TEMPLATE │ ├── BUG_REPORT.md │ └── FEATURE_REQUEST.md ├── PULL_REQUEST_TEMPLATE.md ├── dependabot.yml └── workflows │ ├── build-edge.yaml │ ├── build-test.yaml │ ├── charts.yaml │ ├── conform.yaml │ ├── release-charts.yaml │ ├── release-please.yml │ ├── release-pre.yaml │ ├── release.yaml │ └── stale.yaml ├── .gitignore ├── .golangci.yml ├── .goreleaser.yml ├── ADOPTERS.md ├── CHANGELOG.md ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── Dockerfile ├── LICENSE ├── Makefile ├── OWNERS ├── README.md ├── charts └── proxmox-csi-plugin │ ├── .helmignore │ ├── Chart.yaml │ ├── README.md │ ├── README.md.gotmpl │ ├── ci │ └── values.yaml │ ├── icon.png │ ├── templates │ ├── NOTES.txt │ ├── _helpers.tpl │ ├── _storage.tpl │ ├── controller-clusterrole.yaml │ ├── controller-deployment.yaml │ ├── controller-role.yaml │ ├── controller-rolebinding.yaml │ ├── csidriver.yaml │ ├── namespace.yaml │ ├── node-clusterrole.yaml │ ├── node-deployment.yaml │ ├── node-rolebinding.yaml │ ├── secrets.yaml │ ├── serviceaccount.yaml │ └── storageclass.yaml │ ├── values.edge.yaml │ ├── values.talos.yaml │ └── values.yaml ├── cmd ├── controller │ └── main.go ├── node │ └── main.go └── pvecsictl │ ├── main.go │ ├── migrate.go │ ├── rename.go │ ├── swap.go │ └── utils.go ├── docker-compose.yml ├── docs ├── architecture.md ├── benchmark.md ├── cosign.md ├── deploy │ ├── debug.yaml │ ├── proxmox-csi-plugin-release.yml │ ├── proxmox-csi-plugin-talos.yml │ ├── proxmox-csi-plugin.yml │ ├── pvc.yaml │ ├── test-pod-ephemeral.yaml │ ├── test-pod-secret-ephemeral.yaml │ ├── test-statefulset-raw.yaml │ └── test-statefulset.yaml ├── eraser.md ├── faq.md ├── install.md ├── metrics.md ├── options.md ├── proxmox-lvm-secret.yaml ├── proxmox-lvm.yaml ├── proxmox-rbd.yaml ├── proxmox-regions.gif ├── proxmox-regions.jpeg ├── proxmox-zfs.yaml ├── pvecsictl.md ├── release.md ├── vm-disks.png └── volume-attributes.yaml ├── go.mod ├── go.sum ├── hack ├── CHANGELOG.tpl.md ├── chglog-config.yml ├── ct.yml ├── e2e-tests.md ├── release-please-config.json ├── release-please-manifest.json └── testdata │ └── cloud-config.yaml ├── pkg ├── csi │ ├── controller.go │ ├── controller_test.go │ ├── driver.go │ ├── helper.go │ ├── helper_test.go │ ├── identity.go │ ├── identity_test.go │ ├── node.go │ ├── node_test.go │ ├── parameters.go │ ├── parameters_test.go │ ├── utils.go │ └── utils_test.go ├── helpers │ └── ptr │ │ └── ptr.go ├── log │ └── log.go ├── metrics │ ├── metrics.go │ └── metrics_api.go ├── proxmox │ ├── doc.go │ ├── vm.go │ └── volume.go ├── tools │ ├── config.go │ ├── nodes.go │ └── pv.go └── volume │ ├── volume.go │ └── volume_test.go └── tools ├── deps-check.sh └── deps.sh /.conform.yaml: -------------------------------------------------------------------------------- 1 | policies: 2 | - type: commit 3 | spec: 4 | header: 5 | length: 89 6 | imperative: true 7 | case: lower 8 | invalidLastCharacters: . 9 | body: 10 | required: true 11 | dco: true 12 | gpg: false 13 | spellcheck: 14 | locale: US 15 | maximumOfOneCommit: true 16 | conventional: 17 | types: 18 | - build 19 | - chore 20 | - ci 21 | - docs 22 | - perf 23 | - refactor 24 | - revert 25 | - style 26 | - test 27 | scopes: 28 | - deps 29 | - main 30 | - chart 31 | descriptionLength: 72 32 | - type: license 33 | spec: 34 | skipPaths: 35 | - .git/ 36 | - pkg/helpers/ptr/ 37 | includeSuffixes: 38 | - .go 39 | excludeSuffixes: 40 | - .pb.go 41 | allowPrecedingComments: false 42 | header: | 43 | /* 44 | Copyright 2023 The Kubernetes Authors. 45 | 46 | Licensed under the Apache License, Version 2.0 (the "License"); 47 | you may not use this file except in compliance with the License. 48 | You may obtain a copy of the License at 49 | 50 | http://www.apache.org/licenses/LICENSE-2.0 51 | 52 | Unless required by applicable law or agreed to in writing, software 53 | distributed under the License is distributed on an "AS IS" BASIS, 54 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 55 | See the License for the specific language governing permissions and 56 | limitations under the License. 57 | */ 58 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | .vscode/ 2 | .github/ 3 | .git/ 4 | **/.gitignore 5 | # 6 | bin/ 7 | charts/ 8 | docs/ 9 | dist/ 10 | hack/ 11 | docker-compose.yml 12 | Dockerfile 13 | 14 | # other 15 | *.md 16 | *.yml 17 | *.zip 18 | *.sql 19 | 20 | # cosign 21 | /cosign.key 22 | /cosign.pub 23 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/BUG_REPORT.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug Report 3 | about: Report a bug. 4 | title: "" 5 | labels: "" 6 | assignees: "" 7 | --- 8 | 9 | ## Bug Report 10 | 11 | ### Description 12 | 13 | ### Logs 14 | 15 | Controller: [`kubectl logs -c proxmox-csi-plugin-controller proxmox-csi-plugin-controller-...`] 16 | 17 | Node: [`kubectl logs -c proxmox-csi-plugin-node proxmox-csi-plugin-node-...`] 18 | 19 | ### Environment 20 | 21 | - Plugin version: 22 | - Kubernetes version: [`kubectl version --short`] 23 | - CSI capasity: [`kubectl get csistoragecapacities -ocustom-columns=CLASS:.storageClassName,AVAIL:.capacity,ZONE:.nodeTopology.matchLabels -A`] 24 | - CSI resource on the node: [`kubectl get CSINode -oyaml`] 25 | - Node describe: [`kubectl describe node `] 26 | - OS version [`cat /etc/os-release`] 27 | 28 | ### Community Note 29 | 30 | * Please vote on this issue by adding a 👍 reaction to the original issue to help the community and maintainers prioritize this request 31 | * Please do not leave "+1" or other comments that do not add relevant new information or questions, they generate extra noise for issue followers and do not help prioritize the request 32 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/FEATURE_REQUEST.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature Requests 3 | about: Create a feature request. 4 | title: "" 5 | labels: "" 6 | assignees: "" 7 | --- 8 | 9 | ## Feature Request 10 | 11 | ### Description 12 | 13 | ### Community Note 14 | 15 | * Please vote on this issue by adding a 👍 reaction to the original issue to help the community and maintainers prioritize this request 16 | * Please do not leave "+1" or other comments that do not add relevant new information or questions, they generate extra noise for issue followers and do not help prioritize the request 17 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | # Pull Request 2 | 3 | 9 | 10 | ## What? (description) 11 | 12 | ## Why? (reasoning) 13 | 14 | ## Acceptance 15 | 16 | Please use the following checklist: 17 | 18 | - [ ] you linked an issue (if applicable) 19 | - [ ] you included tests (if applicable) 20 | - [ ] you ran conformance (`make conformance`) 21 | - [ ] you linted your code (`make lint`) 22 | - [ ] you linted your code (`make unit`) 23 | 24 | > See `make help` for a description of the available targets. 25 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | # See https://docs.github.com/en/github/administering-a-repository/configuration-options-for-dependency-updates 4 | 5 | version: 2 6 | updates: 7 | - package-ecosystem: "github-actions" 8 | directory: "/" 9 | commit-message: 10 | prefix: "chore:" 11 | open-pull-requests-limit: 8 12 | rebase-strategy: disabled 13 | schedule: 14 | interval: "monthly" 15 | day: "monday" 16 | time: "08:00" 17 | timezone: "UTC" 18 | 19 | - package-ecosystem: "gomod" 20 | directory: "/" 21 | commit-message: 22 | prefix: "chore:" 23 | open-pull-requests-limit: 8 24 | rebase-strategy: disabled 25 | schedule: 26 | interval: "monthly" 27 | day: "monday" 28 | time: "07:00" 29 | timezone: "UTC" 30 | groups: 31 | k8s.io: 32 | patterns: 33 | - "k8s.io/api" 34 | - "k8s.io/apimachinery" 35 | - "k8s.io/client-go" 36 | - "k8s.io/cloud-provider" 37 | - "k8s.io/component-base" 38 | - "k8s.io/mount-utils" 39 | 40 | - package-ecosystem: "docker" 41 | directory: "/" 42 | commit-message: 43 | prefix: "chore:" 44 | open-pull-requests-limit: 8 45 | rebase-strategy: disabled 46 | schedule: 47 | interval: "monthly" 48 | day: "monday" 49 | time: "07:00" 50 | timezone: "UTC" 51 | -------------------------------------------------------------------------------- /.github/workflows/build-edge.yaml: -------------------------------------------------------------------------------- 1 | name: Build edge 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | paths: 8 | - 'go.mod' 9 | - 'go.sum' 10 | - 'cmd/**' 11 | - 'pkg/**' 12 | - 'Dockerfile' 13 | 14 | jobs: 15 | build-publish: 16 | name: "Build image and publish" 17 | timeout-minutes: 15 18 | runs-on: ubuntu-24.04 19 | permissions: 20 | contents: read 21 | packages: write 22 | id-token: write 23 | steps: 24 | - name: Checkout 25 | uses: actions/checkout@v4 26 | - name: Unshallow 27 | run: git fetch --prune --unshallow 28 | 29 | - name: Install Cosign 30 | uses: sigstore/cosign-installer@v3.8.2 31 | - name: Set up QEMU 32 | uses: docker/setup-qemu-action@v3 33 | with: 34 | platforms: arm64 35 | - name: Set up docker buildx 36 | uses: docker/setup-buildx-action@v3 37 | 38 | - name: Github registry login 39 | uses: docker/login-action@v3 40 | with: 41 | registry: ghcr.io 42 | username: ${{ github.actor }} 43 | password: ${{ secrets.GITHUB_TOKEN }} 44 | 45 | - name: Build and push 46 | timeout-minutes: 10 47 | run: make images 48 | env: 49 | USERNAME: ${{ github.repository_owner }} 50 | PUSH: "true" 51 | TAG: "edge" 52 | - name: Sign images 53 | timeout-minutes: 4 54 | run: make images-cosign 55 | env: 56 | USERNAME: ${{ github.repository_owner }} 57 | TAG: "edge" 58 | -------------------------------------------------------------------------------- /.github/workflows/build-test.yaml: -------------------------------------------------------------------------------- 1 | name: Build check 2 | 3 | on: 4 | pull_request: 5 | branches: 6 | - main 7 | paths: 8 | - 'go.mod' 9 | - 'go.sum' 10 | - 'cmd/**' 11 | - 'pkg/**' 12 | - 'Dockerfile' 13 | 14 | jobs: 15 | build: 16 | name: Build 17 | timeout-minutes: 15 18 | runs-on: ubuntu-24.04 19 | if: github.event.pull_request.draft == false 20 | permissions: 21 | contents: read 22 | steps: 23 | - name: Checkout 24 | uses: actions/checkout@v4 25 | 26 | - name: Set up go 27 | timeout-minutes: 5 28 | uses: actions/setup-go@v5 29 | with: 30 | go-version-file: 'go.mod' 31 | 32 | - name: Lint 33 | uses: golangci/golangci-lint-action@v8 34 | with: 35 | version: v2.1.6 36 | args: --timeout=5m --config=.golangci.yml 37 | - name: Unit 38 | run: make unit 39 | - name: Build 40 | timeout-minutes: 10 41 | run: make images 42 | env: 43 | PLATFORM: linux/amd64 44 | - name: Check node tools 45 | timeout-minutes: 5 46 | run: make image-tools-check 47 | env: 48 | PLATFORM: linux/amd64 49 | -------------------------------------------------------------------------------- /.github/workflows/charts.yaml: -------------------------------------------------------------------------------- 1 | name: Helm chart check 2 | 3 | on: 4 | pull_request: 5 | branches: 6 | - main 7 | paths: 8 | - 'charts/**' 9 | 10 | jobs: 11 | helm-lint: 12 | name: Helm chart check 13 | timeout-minutes: 5 14 | runs-on: ubuntu-24.04 15 | steps: 16 | - name: Checkout 17 | uses: actions/checkout@v4 18 | - name: Unshallow 19 | run: git fetch --prune --unshallow 20 | 21 | - name: Install chart-testing tools 22 | id: lint 23 | uses: helm/chart-testing-action@v2.7.0 24 | 25 | - name: Run helm chart linter 26 | run: ct --config hack/ct.yml lint 27 | - name: Run helm template 28 | run: make helm-unit 29 | -------------------------------------------------------------------------------- /.github/workflows/conform.yaml: -------------------------------------------------------------------------------- 1 | name: Conformance check 2 | 3 | on: 4 | pull_request: 5 | branches: 6 | - main 7 | 8 | jobs: 9 | conform: 10 | name: Conformance 11 | timeout-minutes: 5 12 | runs-on: ubuntu-24.04 13 | if: github.event.pull_request.draft == false 14 | steps: 15 | - name: Checkout 16 | uses: actions/checkout@v4 17 | with: 18 | fetch-depth: 0 19 | ref: ${{ github.event.pull_request.head.sha }} 20 | - name: Checkout main branch 21 | run: git fetch --no-tags origin main:main 22 | 23 | - name: Conform action 24 | uses: talos-systems/conform@v0.1.0-alpha.30 25 | with: 26 | token: ${{ secrets.GITHUB_TOKEN }} 27 | -------------------------------------------------------------------------------- /.github/workflows/release-charts.yaml: -------------------------------------------------------------------------------- 1 | name: HelmChart Release 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | paths: 8 | - 'charts/**' 9 | 10 | jobs: 11 | build-publish: 12 | name: "Publish helm chart" 13 | timeout-minutes: 10 14 | runs-on: ubuntu-24.04 15 | permissions: 16 | contents: read 17 | packages: write 18 | id-token: write 19 | steps: 20 | - name: Checkout 21 | uses: actions/checkout@v4 22 | with: 23 | fetch-depth: 0 24 | 25 | - name: Install Helm 26 | uses: azure/setup-helm@v4 27 | with: 28 | version: v3.12.2 29 | - name: Install Cosign 30 | uses: sigstore/cosign-installer@v3.8.2 31 | 32 | - name: Github registry login 33 | uses: docker/login-action@v3 34 | with: 35 | registry: ghcr.io 36 | username: ${{ github.actor }} 37 | password: ${{ secrets.GITHUB_TOKEN }} 38 | 39 | - name: Helm release 40 | timeout-minutes: 5 41 | run: make helm-login helm-release 42 | env: 43 | HELM_TOKEN: ${{ secrets.GITHUB_TOKEN }} 44 | -------------------------------------------------------------------------------- /.github/workflows/release-please.yml: -------------------------------------------------------------------------------- 1 | name: Release please 2 | 3 | on: 4 | workflow_dispatch: {} 5 | push: 6 | branches: 7 | - main 8 | 9 | jobs: 10 | release-please: 11 | runs-on: ubuntu-24.04 12 | permissions: 13 | contents: write 14 | pull-requests: write 15 | 16 | steps: 17 | - name: Create release PR 18 | id: release 19 | uses: googleapis/release-please-action@v4 20 | with: 21 | config-file: hack/release-please-config.json 22 | manifest-file: hack/release-please-manifest.json 23 | -------------------------------------------------------------------------------- /.github/workflows/release-pre.yaml: -------------------------------------------------------------------------------- 1 | name: Release check 2 | 3 | on: 4 | pull_request: 5 | branches: 6 | - main 7 | 8 | jobs: 9 | build-publish: 10 | name: "Check release docs" 11 | timeout-minutes: 15 12 | runs-on: ubuntu-24.04 13 | if: startsWith(github.head_ref, 'release-') 14 | permissions: 15 | contents: read 16 | packages: write 17 | steps: 18 | - name: Checkout 19 | uses: actions/checkout@v4 20 | - name: Unshallow 21 | run: git fetch --prune --unshallow 22 | 23 | - name: Release version 24 | if: startsWith(github.head_ref, 'release-please') 25 | run: jq -r '"TAG=v"+.[]' hack/release-please-manifest.json >> "$GITHUB_ENV" 26 | 27 | - name: Helm docs 28 | uses: gabe565/setup-helm-docs-action@v1 29 | with: 30 | version: v1.11.3 31 | 32 | - name: Generate 33 | run: make docs 34 | - name: Check 35 | run: git diff --exit-code 36 | 37 | build-publish-cli: 38 | name: "Check cli tool" 39 | timeout-minutes: 15 40 | runs-on: ubuntu-24.04 41 | if: startsWith(github.head_ref, 'release-') 42 | steps: 43 | - name: Checkout 44 | uses: actions/checkout@v4 45 | - name: Unshallow 46 | run: git fetch --prune --unshallow 47 | 48 | - name: Run GoReleaser 49 | uses: goreleaser/goreleaser-action@v6 50 | with: 51 | version: '~> v2' 52 | args: check 53 | env: 54 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 55 | -------------------------------------------------------------------------------- /.github/workflows/release.yaml: -------------------------------------------------------------------------------- 1 | name: Release 2 | 3 | on: 4 | workflow_dispatch: {} 5 | push: 6 | tags: 7 | - 'v*' 8 | 9 | jobs: 10 | build-publish: 11 | name: "Build image and publish" 12 | timeout-minutes: 15 13 | runs-on: ubuntu-24.04 14 | permissions: 15 | contents: read 16 | packages: write 17 | id-token: write 18 | steps: 19 | - name: Checkout 20 | uses: actions/checkout@v4 21 | - name: Unshallow 22 | run: git fetch --prune --unshallow 23 | 24 | - name: Install Cosign 25 | uses: sigstore/cosign-installer@v3.8.2 26 | - name: Set up QEMU 27 | uses: docker/setup-qemu-action@v3 28 | with: 29 | platforms: arm64 30 | - name: Set up docker buildx 31 | uses: docker/setup-buildx-action@v3 32 | 33 | - name: Github registry login 34 | uses: docker/login-action@v3 35 | with: 36 | registry: ghcr.io 37 | username: ${{ github.actor }} 38 | password: ${{ secrets.GITHUB_TOKEN }} 39 | 40 | - name: Build and push 41 | timeout-minutes: 10 42 | run: make images 43 | env: 44 | PUSH: "true" 45 | TAG: "edge" 46 | - name: Sign images 47 | timeout-minutes: 4 48 | run: make images-cosign 49 | env: 50 | TAG: "edge" 51 | 52 | - name: Build and push 53 | timeout-minutes: 10 54 | run: make images 55 | env: 56 | PUSH: "true" 57 | - name: Sign images 58 | timeout-minutes: 4 59 | run: make images-cosign 60 | 61 | build-publish-cli: 62 | name: "Publish cli tool" 63 | timeout-minutes: 15 64 | runs-on: ubuntu-24.04 65 | permissions: 66 | contents: write 67 | steps: 68 | - name: Checkout 69 | uses: actions/checkout@v4 70 | - name: Unshallow 71 | run: git fetch --prune --unshallow 72 | 73 | - name: Set up go 74 | timeout-minutes: 5 75 | uses: actions/setup-go@v5 76 | with: 77 | go-version-file: 'go.mod' 78 | 79 | - name: Generate token 80 | uses: actions/create-github-app-token@df432ceedc7162793a195dd1713ff69aefc7379e # v2.0.6 81 | id: token 82 | with: 83 | app-id: "${{ secrets.BOT_APP_ID }}" 84 | private-key: "${{ secrets.BOT_APP_PRIVATE_KEY }}" 85 | owner: "${{ github.repository_owner }}" 86 | repositories: homebrew-tap 87 | - name: Run GoReleaser 88 | uses: goreleaser/goreleaser-action@v6 89 | with: 90 | version: '~> v2' 91 | args: release --clean 92 | env: 93 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 94 | HOMEBREW_TOKEN: ${{ steps.token.outputs.token }} 95 | -------------------------------------------------------------------------------- /.github/workflows/stale.yaml: -------------------------------------------------------------------------------- 1 | name: Close stale issues 2 | 3 | on: 4 | schedule: 5 | - cron: '30 8 * * *' 6 | 7 | jobs: 8 | stale: 9 | name: Check stale issues 10 | runs-on: ubuntu-24.04 11 | permissions: 12 | issues: write 13 | pull-requests: write 14 | steps: 15 | - uses: actions/stale@v9 16 | with: 17 | stale-issue-message: This issue is stale because it has been open 180 days with no activity. Remove stale label or comment or this will be closed in 14 days. 18 | close-issue-message: This issue was closed because it has been stalled for 14 days with no activity. 19 | days-before-issue-stale: 180 20 | days-before-issue-close: 14 21 | days-before-pr-close: -1 # never close PRs 22 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # If you prefer the allow list template instead of the deny list, see community template: 2 | # https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore 3 | # 4 | # Binaries for programs and plugins 5 | *.exe 6 | *.exe~ 7 | *.dll 8 | *.so 9 | *.dylib 10 | bin/ 11 | dist/ 12 | 13 | # Test binary, built with `go test -c` 14 | *.test 15 | 16 | # Output of the go coverage tool, specifically when used with LiteIDE 17 | *.out 18 | 19 | # Dependency directories (remove the comment below to include it) 20 | # vendor/ 21 | 22 | # Go workspace file 23 | go.work 24 | 25 | # 26 | /charts/proxmox-csi-plugin/values.*.yaml 27 | /hack/cloud-config.yaml 28 | /hack/kubeconfig 29 | /hack/kubeconfig* 30 | kubeconfig 31 | .cache/ 32 | /log 33 | 34 | # cosign 35 | /cosign.key 36 | /cosign.pub 37 | 38 | .idea -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | version: "2" 2 | run: 3 | build-tags: 4 | - integration 5 | - integration_api 6 | - integration_cli 7 | - integration_k8s 8 | - integration_provision 9 | issues-exit-code: 1 10 | tests: true 11 | output: 12 | formats: 13 | text: 14 | path: stdout 15 | print-linter-name: true 16 | print-issued-lines: true 17 | colors: false 18 | linters: 19 | default: all 20 | disable: 21 | - depguard 22 | - errorlint 23 | - exhaustruct 24 | - err113 25 | - forbidigo 26 | - forcetypeassert 27 | - funlen 28 | - gochecknoglobals 29 | - gochecknoinits 30 | - gocognit 31 | - godox 32 | - godot 33 | - gosec 34 | - inamedparam 35 | - ireturn 36 | - maintidx 37 | - mnd 38 | - musttag 39 | - nakedret 40 | - nestif 41 | - nilnil 42 | - nolintlint 43 | - nonamedreturns 44 | - paralleltest 45 | - perfsprint 46 | - promlinter 47 | - protogetter 48 | - recvcheck 49 | - tagalign 50 | - tagliatelle 51 | - testifylint 52 | - testpackage 53 | - thelper 54 | - varnamelen 55 | - wrapcheck 56 | 57 | # temporarily disabled linters 58 | - copyloopvar 59 | - intrange 60 | settings: 61 | cyclop: 62 | max-complexity: 30 63 | dupl: 64 | threshold: 100 65 | errcheck: 66 | check-type-assertions: false 67 | check-blank: true 68 | exclude-functions: 69 | - fmt.Fprintln 70 | - fmt.Fprintf 71 | - fmt.Fprint 72 | goconst: 73 | min-len: 3 74 | min-occurrences: 3 75 | gocyclo: 76 | min-complexity: 30 77 | gomoddirectives: 78 | replace-local: true 79 | replace-allow-list: [] 80 | retract-allow-no-explanation: false 81 | exclude-forbidden: true 82 | lll: 83 | line-length: 200 84 | tab-width: 1 85 | misspell: 86 | locale: US 87 | nolintlint: 88 | require-explanation: false 89 | require-specific: true 90 | allow-unused: false 91 | prealloc: 92 | simple: true 93 | range-loops: true 94 | for-loops: false 95 | staticcheck: 96 | checks: 97 | [ 98 | "all", 99 | "-ST1000", 100 | "-ST1003", 101 | "-ST1016", 102 | "-ST1020", 103 | "-ST1021", 104 | "-ST1022", 105 | "-QF1001", 106 | "-QF1008", 107 | ] 108 | unused: 109 | local-variables-are-used: false 110 | 111 | exclusions: 112 | paths: 113 | - pkg/helpers/ptr/ptr.go 114 | issues: 115 | max-issues-per-linter: 0 116 | max-same-issues: 0 117 | uniq-by-line: true 118 | new: false 119 | formatters: 120 | enable: 121 | - gci 122 | - gofmt 123 | - gofumpt 124 | - goimports 125 | settings: 126 | gci: 127 | sections: 128 | - standard # Captures all standard packages if they do not match another section. 129 | - default # Contains all imports that could not be matched to another section type. 130 | - prefix(github.com/sergelogvinov) # Groups all imports with the specified Prefix. 131 | - prefix(k8s.io) # Groups all imports with the specified Prefix. 132 | -------------------------------------------------------------------------------- /.goreleaser.yml: -------------------------------------------------------------------------------- 1 | # yaml-language-server: $schema=https://goreleaser.com/static/schema.json 2 | version: 2 3 | project_name: pvecsictl 4 | 5 | before: 6 | hooks: 7 | - go mod download 8 | 9 | dist: bin 10 | builds: 11 | - dir: cmd/pvecsictl 12 | binary: pvecsictl-{{ .Os }}-{{ .Arch }} 13 | no_unique_dist_dir: true 14 | env: 15 | - CGO_ENABLED=0 16 | goos: 17 | - linux 18 | - darwin 19 | goarch: 20 | - amd64 21 | - arm64 22 | 23 | archives: 24 | - formats: 25 | - binary 26 | name_template: "{{ .Binary }}" 27 | 28 | checksum: 29 | name_template: "checksums.txt" 30 | 31 | snapshot: 32 | version_template: edge 33 | 34 | # dockers: 35 | # - use: buildx 36 | # image_templates: 37 | # - ghcr.io/sergelogvinov/pvecsictl:{{ .Version }}-amd64 38 | # goos: linux 39 | # goarch: amd64 40 | # build_flag_templates: 41 | # - "--label=org.opencontainers.image.version={{.Version}}" 42 | # - "--target=pvecsictl-goreleaser" 43 | # - "--platform=linux/amd64" 44 | # - use: buildx 45 | # image_templates: 46 | # - ghcr.io/sergelogvinov/pvecsictl:{{ .Version }}-arm64 47 | # goos: linux 48 | # goarch: arm64 49 | # build_flag_templates: 50 | # - "--label=org.opencontainers.image.version={{.Version}}" 51 | # - "--target=pvecsictl-goreleaser" 52 | # - "--platform=linux/arm64" 53 | # docker_manifests: 54 | # - name_template: ghcr.io/sergelogvinov/{{ .ProjectName }}:{{ .Version }} 55 | # image_templates: 56 | # - ghcr.io/sergelogvinov/{{ .ProjectName }}:{{ .Version }}-amd64 57 | # - ghcr.io/sergelogvinov/{{ .ProjectName }}:{{ .Version }}-arm64 58 | 59 | brews: 60 | - name: pvecsictl 61 | directory: Formula 62 | homepage: https://github.com/sergelogvinov/proxmox-csi-plugin 63 | description: "Proxmox VE CSI Mutate tool" 64 | license: Apache-2.0 65 | 66 | commit_author: 67 | name: sergelogvinov 68 | email: 5407715+sergelogvinov@users.noreply.github.com 69 | repository: 70 | owner: sergelogvinov 71 | name: homebrew-tap 72 | branch: main 73 | token: "{{ .Env.HOMEBREW_TOKEN }}" 74 | 75 | test: | 76 | system "#{bin}/pvecsictl -v" 77 | install: | 78 | bin.install "{{ .Binary }}" => "pvecsictl" 79 | -------------------------------------------------------------------------------- /ADOPTERS.md: -------------------------------------------------------------------------------- 1 | # Adopters 2 | 3 | This is a list of organizations or projects that have adopted the Proxmox CSI driver. 4 | 5 | ## Adopters (listed alphabetically) 6 | 7 | * **[Boriss Novickovs](https://github.com/kubebn/talos-proxmox-kaas)** 8 | Kubernetes As a Service in Proxmox. Homelab using Talos and Proxmox, including base add-ons and GitOps practices. Proxmox CCM/CSI to manage VMs and storages. 9 | 10 | * **[Serge Logvinov](https://github.com/sergelogvinov/terraform-talos/tree/main/proxmox)** 11 | Terraform example for Talos on Proxmox. I am using the Proxmox CSI plugin for GitHub Actions (for ephemeral storage) and databases on Proxmox. _Not so well documented_. 12 | 13 | * **[Vegard Hagen](https://blog.stonegarden.dev/articles/2024/06/k8s-proxmox-csi/)** 14 | Article explaining of to get Proxmox CSI Plugin working along with some testing. 15 | 16 | ### Template 17 | 18 | * **[Project/User name](Project URL)** 19 | Description & Use cases 20 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## [0.10.0](https://github.com/sergelogvinov/proxmox-csi-plugin/compare/v0.9.0...v0.10.0) (2025-01-20) 2 | 3 | 4 | ### Features 5 | 6 | * enable support for capmox ([6145c7d](https://github.com/sergelogvinov/proxmox-csi-plugin/commit/6145c7d91cfc47c131ac453e2a90a915e5694b2b)) 7 | 8 | ## [0.11.0](https://github.com/sergelogvinov/proxmox-csi-plugin/compare/v0.10.0...v0.11.0) (2025-02-08) 9 | 10 | 11 | ### Features 12 | 13 | * allow ovverid backup attribute ([2fada12](https://github.com/sergelogvinov/proxmox-csi-plugin/commit/2fada12f0a0305a7083debff4c94b088b721cf04)) 14 | * support different disk id ([e3a25c2](https://github.com/sergelogvinov/proxmox-csi-plugin/commit/e3a25c26a2152d8605fef42e8b0c7e2b3b3c26c4)) 15 | * support volume attributes class ([bab93fb](https://github.com/sergelogvinov/proxmox-csi-plugin/commit/bab93fb05355f4e65995b60a7ec003b129fbe984)) 16 | * volume replication ([0b66712](https://github.com/sergelogvinov/proxmox-csi-plugin/commit/0b667121a527b01652a773f3274af7f65dc7b7f6)) 17 | * zfs storage migration ([37d7fb0](https://github.com/sergelogvinov/proxmox-csi-plugin/commit/37d7fb09f2e76fa4ea5b40377777a19e8832f09e)) 18 | 19 | 20 | ### Bug Fixes 21 | 22 | * parametes attributes ([820cb7e](https://github.com/sergelogvinov/proxmox-csi-plugin/commit/820cb7ea11e09d1d8c7c6feff176350b20135f62)) 23 | 24 | ## [v0.9.0](https://github.com/sergelogvinov/proxmox-csi-plugin/compare/v0.8.2...v0.9.0) (2025-01-01) 25 | 26 | Welcome to the v0.9.0 release of Proxmox CSI Plugin! 27 | 28 | ### Bug Fixes 29 | 30 | - volume size (b08a592) 31 | 32 | ### Features 33 | 34 | - minimal chunk size (898f6e7) 35 | 36 | ### Miscellaneous 37 | 38 | - release v0.9.0 (1555d55) 39 | - bump deps (a30235b) 40 | - bump deps (db61132) 41 | - bump deps (0695c22) 42 | - bump deps (2351ca2) 43 | - release v0.8.2 (0cd72b0) 44 | - **chart:** update csi sidecar (d3b2b84) 45 | 46 | 47 | ## [v0.8.2](https://github.com/sergelogvinov/proxmox-csi-plugin/compare/v0.8.1...v0.8.2) (2024-09-28) 48 | 49 | Welcome to the v0.8.2 release of Proxmox CSI Plugin! 50 | 51 | ### Bug Fixes 52 | 53 | - log sanitizer (474e734) 54 | 55 | ### Miscellaneous 56 | 57 | - release v0.8.2 (0274c03) 58 | 59 | 60 | ## [v0.8.1](https://github.com/sergelogvinov/proxmox-csi-plugin/compare/v0.8.0...v0.8.1) (2024-09-24) 61 | 62 | Welcome to the v0.8.1 release of Proxmox CSI Plugin! 63 | 64 | ### Bug Fixes 65 | 66 | - release please (593f605) 67 | - goreleaser (4e0e87a) 68 | 69 | ### Miscellaneous 70 | 71 | - release v0.8.1 (3f8bd85) 72 | 73 | 74 | ## [v0.8.0](https://github.com/sergelogvinov/proxmox-csi-plugin/compare/v0.7.0...v0.8.0) (2024-09-23) 75 | 76 | Welcome to the v0.8.0 release of Proxmox CSI Plugin! 77 | 78 | ### Bug Fixes 79 | 80 | - check rbac permission (57a6b0d) 81 | - helm chart metrics option (e5ef1b1) 82 | - allow nfs shared storages (04cfb97) 83 | - helm chart podAnnotation (b935d88) 84 | 85 | ### Features 86 | 87 | - expose metrics (4bbe65d) 88 | - add unsafe env (36fa532) 89 | 90 | ### Miscellaneous 91 | 92 | - release v0.8.0 (589de9c) 93 | - bump deps (9a0161b) 94 | - bump deps (3c3c122) 95 | - bump deps (c5769c1) 96 | - **chart:** update readme (c76555a) 97 | 98 | 99 | ## [v0.7.0](https://github.com/sergelogvinov/proxmox-csi-plugin/compare/v0.6.1...v0.7.0) (2024-06-14) 100 | 101 | Welcome to the v0.7.0 release of Proxmox CSI Plugin! 102 | 103 | ### Bug Fixes 104 | 105 | - implement structured logging (cb5fb4e) 106 | - pv force migration (8ecf990) 107 | 108 | ### Features 109 | 110 | - wait volume to be detached (3683d96) 111 | - swap pv in already created pvc (76c899e) 112 | 113 | ### Miscellaneous 114 | 115 | - release v0.7.0 (9424c06) 116 | - release v0.7.0 (7362940) 117 | - bump deps (5bf0677) 118 | - bump deps (89adec9) 119 | - release v0.6.1 (ac1ef92) 120 | 121 | 122 | ## [v0.6.1](https://github.com/sergelogvinov/proxmox-csi-plugin/compare/v0.6.0...v0.6.1) (2024-04-13) 123 | 124 | Welcome to the v0.6.1 release of Proxmox CSI Plugin! 125 | 126 | ### Bug Fixes 127 | 128 | - build release (facdec5) 129 | - release doc (215c366) 130 | 131 | ### Miscellaneous 132 | 133 | - release v0.6.1 (e7dfde2) 134 | 135 | 136 | ## [v0.6.0](https://github.com/sergelogvinov/proxmox-csi-plugin/compare/v0.5.0...v0.6.0) (2024-04-13) 137 | 138 | Welcome to the v0.6.0 release of Proxmox CSI Plugin! 139 | 140 | ### Bug Fixes 141 | 142 | - pvc migration (ddfc362) 143 | - deps update (657ad00) 144 | - cli migration (41b19bd) 145 | - goreleaser (04a40f4) 146 | 147 | ### Features 148 | 149 | - remove udev dependency (1810ec7) 150 | - **chart:** support setting annotations and labels on storageClasses (a5f5add) 151 | - **chart:** add initContainers and hostAliases (769c008) 152 | 153 | ### Miscellaneous 154 | 155 | - release v0.6.0 (0b13bd0) 156 | - bump deps (67dc34c) 157 | - bump deps (2f9f17a) 158 | - **chart:** update sidecar deps (5f16e6b) 159 | 160 | 161 | ## [v0.5.0](https://github.com/sergelogvinov/proxmox-csi-plugin/compare/v0.4.1...v0.5.0) (2024-02-20) 162 | 163 | Welcome to the v0.5.0 release of Proxmox CSI Plugin! 164 | 165 | ### Bug Fixes 166 | 167 | - add delay before unattach device (ff575d1) 168 | - release please (ffad744) 169 | - **chart:** detect safe mounted behavior (5580695) 170 | 171 | ### Features 172 | 173 | - prefer providerID (7dcde72) 174 | - pv/pvc cli helper (d97bc32) 175 | - use release please tool (39c4b22) 176 | - use readonly root (ca00846) 177 | - raw block device (1be660b) 178 | - **chart:** add support to mount a custom CA (9b94627) 179 | 180 | ### Miscellaneous 181 | 182 | - release v0.5.0 (a361ce9) 183 | - bump deps (ac4ddd0) 184 | 185 | 186 | ## [v0.4.1](https://github.com/sergelogvinov/proxmox-csi-plugin/compare/v0.4.0...v0.4.1) (2024-01-01) 187 | 188 | Welcome to the v0.4.1 release of Proxmox CSI Plugin! 189 | 190 | ### Bug Fixes 191 | 192 | - publish shared volumes (a681b2b) 193 | - find zone by region (4eae22d) 194 | 195 | ### Features 196 | 197 | - **chart:** add value to customize kubeletDir (bbb627f) 198 | - **chart:** add allowedTopologies (41cb02a) 199 | 200 | ### Miscellaneous 201 | 202 | - release v0.4.1 (fd8d14f) 203 | - bump deps (2a86bd7) 204 | - bump deps (d8c98ea) 205 | - bump deps (9054282) 206 | 207 | 208 | ## [v0.4.0](https://github.com/sergelogvinov/proxmox-csi-plugin/compare/v0.3.0...v0.4.0) (2023-10-24) 209 | 210 | Welcome to the v0.4.0 release of Proxmox CSI Plugin! 211 | 212 | ### Bug Fixes 213 | 214 | - check volume existence (aba0ca8) 215 | - helm create namespace (364b8be) 216 | - remove nocloud label (74e42b2) 217 | 218 | ### Features 219 | 220 | - mkfs block/inode size options (88f4ebc) 221 | - disk speed limit (c464dab) 222 | - **chart:** make StorageClass parameters/mountOptions configurable (a78e338) 223 | 224 | ### Miscellaneous 225 | 226 | - release v0.4.0 (764b741) 227 | - bump deps (9e5a139) 228 | - bump deps (a243ffb) 229 | 230 | 231 | ## [v0.3.0](https://github.com/sergelogvinov/proxmox-csi-plugin/compare/v0.2.0...v0.3.0) (2023-09-19) 232 | 233 | Welcome to the v0.3.0 release of Proxmox CSI Plugin! 234 | 235 | ### Features 236 | 237 | - storage encryption (26c1928) 238 | - volume capability (1088dbb) 239 | - regional block devices (c7d1541) 240 | 241 | ### Miscellaneous 242 | 243 | - release v0.3.0 (324ad91) 244 | - bump deps (5f5d781) 245 | - bump actions/checkout from 3 to 4 (f75bfff) 246 | - bump sigstore/cosign-installer from 3.1.1 to 3.1.2 (51419d3) 247 | - bump deps (ae63a06) 248 | - bump deps (4ceef77) 249 | 250 | 251 | ## [v0.2.0](https://github.com/sergelogvinov/proxmox-csi-plugin/compare/v0.1.1...v0.2.0) (2023-08-07) 252 | 253 | Welcome to the v0.2.0 release of Proxmox CSI Plugin! 254 | 255 | ### Bug Fixes 256 | 257 | - skip lxc containers on resize process (a24d24e) 258 | - helm liveness context (e1ed889) 259 | - detach volume error (dc128d1) 260 | - kubectl apply in readme (bc2f88b) 261 | 262 | ### Features 263 | 264 | - noatime flag for ssd (cd4f3f7) 265 | - cosign images (5e13f3f) 266 | - pin version (e81d8e3) 267 | - helm oci release (c438712) 268 | - drop node capabilities (927f664) 269 | - trim filesystem (dc7dbbd) 270 | 271 | ### Miscellaneous 272 | 273 | - release v0.2.0 (6a2d98a) 274 | - bump actions versions (b477132) 275 | - bump deps (f6d726c) 276 | - bump deps (ecea2ad) 277 | - bump deps (28f0a72) 278 | - bump deps (f00f057) 279 | 280 | 281 | ## [v0.1.1](https://github.com/sergelogvinov/proxmox-csi-plugin/compare/v0.1.0...v0.1.1) (2023-05-12) 282 | 283 | Welcome to the v0.1.1 release of Proxmox CSI Plugin! 284 | 285 | ### Features 286 | 287 | - switch to distroless (ff1c9bf) 288 | - decrease node image (93a04b6) 289 | 290 | ### Miscellaneous 291 | 292 | - release v0.1.1 (429a420) 293 | - bump deps (4e80caf) 294 | - bump deps (be954c9) 295 | 296 | 297 | ## [v0.1.0](https://github.com/sergelogvinov/proxmox-csi-plugin/compare/v0.0.2...v0.1.0) (2023-05-04) 298 | 299 | Welcome to the v0.1.0 release of Proxmox CSI Plugin! 300 | 301 | ### Bug Fixes 302 | 303 | - release check (c3bd4e7) 304 | 305 | ### Miscellaneous 306 | 307 | - release v0.1.0 (449bddf) 308 | 309 | 310 | ## [v0.0.2](https://github.com/sergelogvinov/proxmox-csi-plugin/compare/v0.01...v0.0.2) (2023-04-29) 311 | 312 | Welcome to the v0.0.2 release of Proxmox CSI Plugin! 313 | 314 | ### Miscellaneous 315 | 316 | - release v0.0.2 (8390a9f) 317 | 318 | 319 | ## v0.01 (2023-04-29) 320 | 321 | Welcome to the v0.01 release of Proxmox CSI Plugin! 322 | 323 | ### Bug Fixes 324 | 325 | - raise condition during volume attach (3bf3ef5) 326 | - cluster schema (494a82b) 327 | 328 | ### Features 329 | 330 | - resize pvc (bd2c653) 331 | - node daemon (54dec7d) 332 | - node daemonsets (269c708) 333 | - controller (9f0f7a3) 334 | 335 | ### Miscellaneous 336 | 337 | - release v0.0.1 (56b4297) 338 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | ## Code of Conduct 2 | 3 | ### Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as 6 | contributors and maintainers pledge to making participation in our project and 7 | our community a harassment-free experience for everyone, regardless of age, body 8 | size, disability, ethnicity, sex characteristics, gender identity and expression, 9 | level of experience, education, socio-economic status, nationality, personal 10 | appearance, race, religion, or sexual identity and orientation. 11 | 12 | ### Our Standards 13 | 14 | Examples of behavior that contributes to creating a positive environment 15 | include: 16 | 17 | * Using welcoming and inclusive language 18 | * Being respectful of differing viewpoints and experiences 19 | * Gracefully accepting constructive criticism 20 | * Focusing on what is best for the community 21 | * Showing empathy towards other community members 22 | 23 | Examples of unacceptable behavior by participants include: 24 | 25 | * The use of sexualized language or imagery and unwelcome sexual attention or 26 | advances 27 | * Trolling, insulting/derogatory comments, and personal or political attacks 28 | * Public or private harassment 29 | * Publishing others' private information, such as a physical or electronic 30 | address, without explicit permission 31 | * Other conduct which could reasonably be considered inappropriate in a 32 | professional setting 33 | 34 | ### Our Responsibilities 35 | 36 | Project maintainers are responsible for clarifying the standards of acceptable 37 | behavior and are expected to take appropriate and fair corrective action in 38 | response to any instances of unacceptable behavior. 39 | 40 | Project maintainers have the right and responsibility to remove, edit, or 41 | reject comments, commits, code, wiki edits, issues, and other contributions 42 | that are not aligned to this Code of Conduct, or to ban temporarily or 43 | permanently any contributor for other behaviors that they deem inappropriate, 44 | threatening, offensive, or harmful. 45 | 46 | ### Scope 47 | 48 | This Code of Conduct applies both within project spaces and in public spaces 49 | when an individual is representing the project or its community. Examples of 50 | representing a project or community include using an official project e-mail 51 | address, posting via an official social media account, or acting as an appointed 52 | representative at an online or offline event. Representation of a project may be 53 | further defined and clarified by project maintainers. 54 | 55 | ### Enforcement 56 | 57 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 58 | reported by contacting the project team at [INSERT EMAIL ADDRESS]. All 59 | complaints will be reviewed and investigated and will result in a response that 60 | is deemed necessary and appropriate to the circumstances. The project team is 61 | obligated to maintain confidentiality with regard to the reporter of an incident. 62 | Further details of specific enforcement policies may be posted separately. 63 | 64 | Project maintainers who do not follow or enforce the Code of Conduct in good 65 | faith may face temporary or permanent repercussions as determined by other 66 | members of the project's leadership. 67 | 68 | ### Attribution 69 | 70 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 71 | available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html 72 | 73 | [homepage]: https://www.contributor-covenant.org 74 | 75 | For answers to common questions about this code of conduct, see 76 | https://www.contributor-covenant.org/faq 77 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | ## Pull Requests 4 | 5 | All PRs require a single commit. 6 | 7 | Having one commit in a Pull Request is very important for several reasons: 8 | * A single commit per PR keeps the git history clean and readable. 9 | It helps reviewers and future developers understand the change as one atomic unit of work, instead of sifting through many intermediate or redundant commits. 10 | * One commit is easier to cherry-pick into another branch or to track in changelogs. 11 | * Squashing into one meaningful commit ensures the final PR only contains what matters. 12 | 13 | ## Developer Certificate of Origin 14 | 15 | All commits require a [DCO](https://developercertificate.org/) sign-off. 16 | This is done by committing with the `--signoff` flag. 17 | 18 | ## Development 19 | 20 | The build process for this project is designed to run entirely in containers. 21 | To get started, run `make help` and follow the instructions. 22 | 23 | ## Conformance 24 | 25 | To verify conformance status, run `make conformance`. 26 | This runs a series of tests on the working tree and is required to pass before a contribution is accepted. 27 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # syntax = docker/dockerfile:1.15 2 | ######################################## 3 | 4 | FROM golang:1.24-bookworm AS develop 5 | 6 | WORKDIR /src 7 | COPY ["go.mod", "go.sum", "/src"] 8 | RUN go mod download 9 | 10 | ######################################## 11 | 12 | FROM --platform=${BUILDPLATFORM} golang:1.24.3-alpine3.22 AS builder 13 | RUN apk update && apk add --no-cache make 14 | ENV GO111MODULE=on 15 | WORKDIR /src 16 | 17 | COPY ["go.mod", "go.sum", "/src"] 18 | RUN go mod download && go mod verify 19 | 20 | COPY . . 21 | ARG TAG 22 | ARG SHA 23 | RUN make build-all-archs 24 | 25 | ######################################## 26 | 27 | FROM --platform=${TARGETARCH} scratch AS proxmox-csi-controller 28 | LABEL org.opencontainers.image.source="https://github.com/sergelogvinov/proxmox-csi-plugin" \ 29 | org.opencontainers.image.licenses="Apache-2.0" \ 30 | org.opencontainers.image.description="Proxmox VE CSI plugin" 31 | 32 | COPY --from=gcr.io/distroless/static-debian12:nonroot . . 33 | ARG TARGETARCH 34 | COPY --from=builder /src/bin/proxmox-csi-controller-${TARGETARCH} /bin/proxmox-csi-controller 35 | 36 | ENTRYPOINT ["/bin/proxmox-csi-controller"] 37 | 38 | ######################################## 39 | 40 | FROM --platform=${TARGETARCH} debian:12.11 AS tools 41 | 42 | RUN apt-get update && apt-get install -y --no-install-recommends \ 43 | bash \ 44 | mount \ 45 | udev \ 46 | e2fsprogs \ 47 | xfsprogs \ 48 | util-linux \ 49 | cryptsetup \ 50 | rsync 51 | 52 | COPY tools /tools 53 | RUN /tools/deps.sh 54 | 55 | ######################################## 56 | 57 | FROM --platform=${TARGETARCH} gcr.io/distroless/base-debian12 AS tools-check 58 | 59 | COPY --from=tools /bin/sh /bin/sh 60 | COPY --from=tools /tools /tools 61 | COPY --from=tools /dest / 62 | 63 | SHELL ["/bin/sh"] 64 | RUN /tools/deps-check.sh 65 | 66 | ######################################## 67 | 68 | FROM --platform=${TARGETARCH} scratch AS proxmox-csi-node 69 | LABEL org.opencontainers.image.source="https://github.com/sergelogvinov/proxmox-csi-plugin" \ 70 | org.opencontainers.image.licenses="Apache-2.0" \ 71 | org.opencontainers.image.description="Proxmox VE CSI plugin" 72 | 73 | COPY --from=gcr.io/distroless/base-debian12 . . 74 | COPY --from=tools /dest / 75 | 76 | ARG TARGETARCH 77 | COPY --from=builder /src/bin/proxmox-csi-node-${TARGETARCH} /bin/proxmox-csi-node 78 | 79 | ENTRYPOINT ["/bin/proxmox-csi-node"] 80 | 81 | ######################################## 82 | 83 | FROM alpine:3.22 AS pvecsictl 84 | LABEL org.opencontainers.image.source="https://github.com/sergelogvinov/proxmox-csi-plugin" \ 85 | org.opencontainers.image.licenses="Apache-2.0" \ 86 | org.opencontainers.image.description="Proxmox VE CSI tools" 87 | 88 | ARG TARGETARCH 89 | COPY --from=builder /src/bin/pvecsictl-${TARGETARCH} /bin/pvecsictl 90 | 91 | ENTRYPOINT ["/bin/pvecsictl"] 92 | 93 | ######################################## 94 | 95 | FROM alpine:3.22 AS pvecsictl-goreleaser 96 | LABEL org.opencontainers.image.source="https://github.com/sergelogvinov/proxmox-csi-plugin" \ 97 | org.opencontainers.image.licenses="Apache-2.0" \ 98 | org.opencontainers.image.description="Proxmox VE CSI tools" 99 | 100 | ARG TARGETARCH 101 | COPY pvecsictl-linux-${TARGETARCH} /bin/pvecsictl 102 | 103 | ENTRYPOINT ["/bin/pvecsictl"] 104 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | REGISTRY ?= ghcr.io 2 | USERNAME ?= sergelogvinov 3 | OCIREPO ?= $(REGISTRY)/$(USERNAME) 4 | HELMREPO ?= $(REGISTRY)/$(USERNAME)/charts 5 | PLATFORM ?= linux/arm64,linux/amd64 6 | PUSH ?= false 7 | 8 | SHA ?= $(shell git describe --match=none --always --abbrev=7 --dirty) 9 | TAG ?= $(shell git describe --tag --always --match v[0-9]\*) 10 | GO_LDFLAGS := -ldflags "-w -s -X main.version=$(TAG) -X main.commit=$(SHA)" 11 | 12 | OS ?= $(shell go env GOOS) 13 | ARCH ?= $(shell go env GOARCH) 14 | ARCHS = amd64 arm64 15 | 16 | BUILD_ARGS := --platform=$(PLATFORM) 17 | ifeq ($(PUSH),true) 18 | BUILD_ARGS += --push=$(PUSH) 19 | BUILD_ARGS += --output type=image,annotation-index.org.opencontainers.image.source="https://github.com/$(USERNAME)/proxmox-csi-plugin",annotation-index.org.opencontainers.image.description="Proxmox VE CSI plugin" 20 | else 21 | BUILD_ARGS += --output type=docker 22 | endif 23 | 24 | COSING_ARGS ?= 25 | 26 | ############ 27 | 28 | # Help Menu 29 | 30 | define HELP_MENU_HEADER 31 | # Getting Started 32 | 33 | To build this project, you must have the following installed: 34 | 35 | - git 36 | - make 37 | - golang 1.20+ 38 | - golangci-lint 39 | 40 | endef 41 | 42 | export HELP_MENU_HEADER 43 | 44 | help: ## This help menu 45 | @echo "$$HELP_MENU_HEADER" 46 | @grep -E '^[a-zA-Z0-9%_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' 47 | 48 | ############ 49 | # 50 | # Build Abstractions 51 | # 52 | 53 | build-all-archs: 54 | @for arch in $(ARCHS); do $(MAKE) ARCH=$${arch} build ; done 55 | 56 | .PHONY: clean 57 | clean: ## Clean 58 | rm -rf bin .cache 59 | 60 | build-pvecsictl: 61 | CGO_ENABLED=0 GOOS=$(OS) GOARCH=$(ARCH) go build $(GO_LDFLAGS) \ 62 | -o bin/pvecsictl-$(ARCH) ./cmd/pvecsictl 63 | 64 | build-%: 65 | CGO_ENABLED=0 GOOS=$(OS) GOARCH=$(ARCH) go build $(GO_LDFLAGS) \ 66 | -o bin/proxmox-csi-$*-$(ARCH) ./cmd/$* 67 | 68 | .PHONY: build 69 | build: build-controller build-node build-pvecsictl ## Build 70 | 71 | .PHONY: run 72 | run: build-controller ## Run 73 | ./bin/proxmox-csi-controller-$(ARCH) --cloud-config=hack/cloud-config.yaml -v=5 --metrics-address=:8080 74 | 75 | .PHONY: lint 76 | lint: ## Lint Code 77 | golangci-lint run --config .golangci.yml 78 | 79 | .PHONY: unit 80 | unit: ## Unit Tests 81 | go test -tags=unit $(shell go list ./...) $(TESTARGS) 82 | 83 | .PHONY: conformance 84 | conformance: ## Conformance 85 | docker run --rm -it -v $(PWD):/src -w /src ghcr.io/siderolabs/conform:v0.1.0-alpha.27 enforce 86 | 87 | ############ 88 | 89 | .PHONY: helm-unit 90 | helm-unit: ## Helm Unit Tests 91 | @helm lint charts/proxmox-csi-plugin 92 | @helm template -f charts/proxmox-csi-plugin/ci/values.yaml proxmox-csi-plugin charts/proxmox-csi-plugin >/dev/null 93 | 94 | .PHONY: helm-login 95 | helm-login: ## Helm Login 96 | @echo "${HELM_TOKEN}" | helm registry login $(REGISTRY) --username $(USERNAME) --password-stdin 97 | 98 | .PHONY: helm-release 99 | helm-release: ## Helm Release 100 | @rm -rf dist/ 101 | @helm package charts/proxmox-csi-plugin -d dist 102 | @helm push dist/proxmox-csi-plugin-*.tgz oci://$(HELMREPO) 2>&1 | tee dist/.digest 103 | @cosign sign --yes $(COSING_ARGS) $(HELMREPO)/proxmox-csi-plugin@$$(cat dist/.digest | awk -F "[, ]+" '/Digest/{print $$NF}') 104 | 105 | ############ 106 | 107 | .PHONY: docs 108 | docs: 109 | yq -i '.appVersion = "$(TAG)"' charts/proxmox-csi-plugin/Chart.yaml 110 | helm template -n csi-proxmox proxmox-csi-plugin \ 111 | -f charts/proxmox-csi-plugin/values.edge.yaml \ 112 | charts/proxmox-csi-plugin > docs/deploy/proxmox-csi-plugin.yml 113 | helm template -n csi-proxmox proxmox-csi-plugin \ 114 | --set-string image.tag=$(TAG) \ 115 | --set createNamespace=true \ 116 | charts/proxmox-csi-plugin > docs/deploy/proxmox-csi-plugin-release.yml 117 | helm template -n csi-proxmox proxmox-csi-plugin \ 118 | -f charts/proxmox-csi-plugin/values.talos.yaml \ 119 | --set-string image.tag=$(TAG) \ 120 | charts/proxmox-csi-plugin > docs/deploy/proxmox-csi-plugin-talos.yml 121 | helm-docs --sort-values-order=file charts/proxmox-csi-plugin 122 | 123 | release-update: 124 | git-chglog --config hack/chglog-config.yml -o CHANGELOG.md 125 | 126 | ############ 127 | # 128 | # Docker Abstractions 129 | # 130 | 131 | .PHONY: docker-init 132 | docker-init: 133 | docker run --rm --privileged multiarch/qemu-user-static:register --reset 134 | 135 | docker context create multiarch ||: 136 | docker buildx create --name multiarch --driver docker-container --use ||: 137 | docker context use multiarch 138 | docker buildx inspect --bootstrap multiarch 139 | 140 | image-%: 141 | docker buildx build $(BUILD_ARGS) \ 142 | --build-arg TAG=$(TAG) \ 143 | --build-arg SHA=$(SHA) \ 144 | -t $(OCIREPO)/$*:$(TAG) \ 145 | --target $* \ 146 | -f Dockerfile . 147 | 148 | .PHONY: images-checks 149 | images-checks: images image-tools-check 150 | trivy image --exit-code 1 --ignore-unfixed --severity HIGH,CRITICAL --no-progress $(OCIREPO)/proxmox-csi-controller:$(TAG) 151 | trivy image --exit-code 1 --ignore-unfixed --severity HIGH,CRITICAL --no-progress $(OCIREPO)/proxmox-csi-node:$(TAG) 152 | trivy image --exit-code 1 --ignore-unfixed --severity HIGH,CRITICAL --no-progress $(OCIREPO)/pvecsictl:$(TAG) 153 | 154 | .PHONY: images-cosign 155 | images-cosign: 156 | @cosign sign --yes $(COSING_ARGS) --recursive $(OCIREPO)/proxmox-csi-controller:$(TAG) 157 | @cosign sign --yes $(COSING_ARGS) --recursive $(OCIREPO)/proxmox-csi-node:$(TAG) 158 | @cosign sign --yes $(COSING_ARGS) --recursive $(OCIREPO)/pvecsictl:$(TAG) 159 | 160 | .PHONY: images 161 | images: image-proxmox-csi-controller image-proxmox-csi-node image-pvecsictl ## Build images 162 | -------------------------------------------------------------------------------- /OWNERS: -------------------------------------------------------------------------------- 1 | approvers: 2 | - sergelogvinov 3 | reviewers: 4 | - sergelogvinov 5 | -------------------------------------------------------------------------------- /charts/proxmox-csi-plugin/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *.orig 18 | *~ 19 | # Various IDEs 20 | .project 21 | .idea/ 22 | *.tmproj 23 | .vscode/ 24 | -------------------------------------------------------------------------------- /charts/proxmox-csi-plugin/Chart.yaml: -------------------------------------------------------------------------------- 1 | # yaml-language-server: $schema=https://json.schemastore.org/chart.json 2 | apiVersion: v2 3 | name: proxmox-csi-plugin 4 | description: Container Storage Interface plugin for Proxmox 5 | type: application 6 | home: https://github.com/sergelogvinov/proxmox-csi-plugin 7 | icon: https://raw.githubusercontent.com/sergelogvinov/proxmox-csi-plugin/main/charts/proxmox-csi-plugin/icon.png 8 | sources: 9 | - https://github.com/sergelogvinov/proxmox-csi-plugin 10 | keywords: 11 | - csi 12 | - storage 13 | - block-storage 14 | - volume 15 | - proxmox 16 | maintainers: 17 | - name: sergelogvinov 18 | url: https://github.com/sergelogvinov 19 | # 20 | # Versions are expected to follow Semantic Versioning (https://semver.org/) 21 | version: 0.3.7 22 | # This is the version number of the application being deployed. This version number should be 23 | # incremented each time you make changes to the application. Versions are not expected to 24 | # follow Semantic Versioning. They should reflect the version the application is using. 25 | # It is recommended to use it with quotes. 26 | appVersion: v0.11.0 27 | -------------------------------------------------------------------------------- /charts/proxmox-csi-plugin/README.md.gotmpl: -------------------------------------------------------------------------------- 1 | {{ template "chart.header" . }} 2 | 3 | {{ template "chart.deprecationWarning" . }} 4 | 5 | {{ template "chart.badgesSection" . }} 6 | 7 | {{ template "chart.description" . }} 8 | 9 | The Container Storage Interface (CSI) plugin is a specification designed to standardize the way container orchestration systems like Kubernetes, interact with different storage systems. The CSI plugin abstracts the underlying storage, enabling the seamless integration of different storage solutions (such as local block devices, file systems, or cloud-based storage) with containerized applications. 10 | 11 | This plugin allows Kubernetes to use `Proxmox VE` storage as a persistent storage solution for stateful applications. 12 | Supported storage types: 13 | - Directory 14 | - LVM 15 | - LVM-thin 16 | - ZFS 17 | - NFS 18 | - Ceph 19 | 20 | {{ template "chart.homepageLine" . }} 21 | 22 | {{ template "chart.maintainersSection" . }} 23 | 24 | {{ template "chart.sourcesSection" . }} 25 | 26 | {{ template "chart.requirementsSection" . }} 27 | 28 | ## Proxmox permissions 29 | 30 | ```shell 31 | # Create role CSI 32 | pveum role add CSI -privs "VM.Audit VM.Config.Disk Datastore.Allocate Datastore.AllocateSpace Datastore.Audit" 33 | # Create user and grant permissions 34 | pveum user add kubernetes-csi@pve 35 | pveum aclmod / -user kubernetes-csi@pve -role CSI 36 | pveum user token add kubernetes-csi@pve csi -privsep 0 37 | ``` 38 | 39 | ## Helm values example 40 | 41 | ```yaml 42 | # proxmox-csi.yaml 43 | 44 | config: 45 | clusters: 46 | - url: https://cluster-api-1.exmple.com:8006/api2/json 47 | insecure: false 48 | token_id: "kubernetes-csi@pve!csi" 49 | token_secret: "key" 50 | region: cluster-1 51 | 52 | # Deploy Node CSI driver only on proxmox nodes 53 | node: 54 | nodeSelector: 55 | # It will work only with Talos CCM, remove it overwise 56 | node.cloudprovider.kubernetes.io/platform: nocloud 57 | tolerations: 58 | - operator: Exists 59 | 60 | # Deploy CSI controller only on control-plane nodes 61 | nodeSelector: 62 | node-role.kubernetes.io/control-plane: "" 63 | tolerations: 64 | - key: node-role.kubernetes.io/control-plane 65 | effect: NoSchedule 66 | 67 | # Define storage classes 68 | # See https://pve.proxmox.com/wiki/Storage 69 | storageClass: 70 | - name: proxmox-data-xfs 71 | storage: data 72 | reclaimPolicy: Delete 73 | fstype: xfs 74 | - name: proxmox-data 75 | storage: data 76 | reclaimPolicy: Delete 77 | fstype: ext4 78 | cache: writethrough 79 | ``` 80 | 81 | ## Deploy 82 | 83 | ```shell 84 | # Prepare namespace 85 | kubectl create ns csi-proxmox 86 | kubectl label ns csi-proxmox pod-security.kubernetes.io/enforce=privileged 87 | # Install Proxmox CSI plugin 88 | helm upgrade -i --namespace=csi-proxmox -f proxmox-csi.yaml \ 89 | proxmox-csi-plugin oci://ghcr.io/sergelogvinov/charts/proxmox-csi-plugin 90 | ``` 91 | 92 | {{ template "chart.valuesSection" . }} 93 | -------------------------------------------------------------------------------- /charts/proxmox-csi-plugin/ci/values.yaml: -------------------------------------------------------------------------------- 1 | 2 | options: 3 | enableCapacity: false 4 | 5 | node: 6 | nodeSelector: 7 | node.cloudprovider.kubernetes.io/platform: nocloud 8 | tolerations: 9 | - operator: Exists 10 | 11 | nodeSelector: 12 | node-role.kubernetes.io/control-plane: "" 13 | tolerations: 14 | - key: node-role.kubernetes.io/control-plane 15 | effect: NoSchedule 16 | 17 | storageClass: 18 | - name: proxmox-data-xfs 19 | storage: data 20 | reclaimPolicy: Delete 21 | fstype: xfs 22 | - name: proxmox-data 23 | storage: data 24 | reclaimPolicy: Delete 25 | ssd: true 26 | -------------------------------------------------------------------------------- /charts/proxmox-csi-plugin/icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sergelogvinov/proxmox-csi-plugin/41a3934c48cc81ac45fce25144afe881a249d6a1/charts/proxmox-csi-plugin/icon.png -------------------------------------------------------------------------------- /charts/proxmox-csi-plugin/templates/NOTES.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sergelogvinov/proxmox-csi-plugin/41a3934c48cc81ac45fce25144afe881a249d6a1/charts/proxmox-csi-plugin/templates/NOTES.txt -------------------------------------------------------------------------------- /charts/proxmox-csi-plugin/templates/_helpers.tpl: -------------------------------------------------------------------------------- 1 | {{/* 2 | Expand the name of the chart. 3 | */}} 4 | {{- define "proxmox-csi-plugin.name" -}} 5 | {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} 6 | {{- end }} 7 | 8 | {{/* 9 | Create a default fully qualified app name. 10 | We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). 11 | If release name contains chart name it will be used as a full name. 12 | */}} 13 | {{- define "proxmox-csi-plugin.fullname" -}} 14 | {{- if .Values.fullnameOverride }} 15 | {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} 16 | {{- else }} 17 | {{- $name := default .Chart.Name .Values.nameOverride }} 18 | {{- if contains $name .Release.Name }} 19 | {{- .Release.Name | trunc 63 | trimSuffix "-" }} 20 | {{- else }} 21 | {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} 22 | {{- end }} 23 | {{- end }} 24 | {{- end }} 25 | 26 | {{/* 27 | Create chart name and version as used by the chart label. 28 | */}} 29 | {{- define "proxmox-csi-plugin.chart" -}} 30 | {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} 31 | {{- end }} 32 | 33 | {{/* 34 | Common labels 35 | */}} 36 | {{- define "proxmox-csi-plugin.labels" -}} 37 | helm.sh/chart: {{ include "proxmox-csi-plugin.chart" . }} 38 | app.kubernetes.io/name: {{ include "proxmox-csi-plugin.name" . }} 39 | app.kubernetes.io/instance: {{ .Release.Name }} 40 | {{- if .Chart.AppVersion }} 41 | app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} 42 | {{- end }} 43 | app.kubernetes.io/managed-by: {{ .Release.Service }} 44 | {{- end }} 45 | 46 | {{/* 47 | Selector labels 48 | */}} 49 | {{- define "proxmox-csi-plugin.selectorLabels" -}} 50 | app.kubernetes.io/name: {{ include "proxmox-csi-plugin.name" . }} 51 | app.kubernetes.io/instance: {{ .Release.Name }} 52 | app.kubernetes.io/component: controller 53 | {{- end }} 54 | 55 | {{- define "proxmox-csi-plugin-node.selectorLabels" -}} 56 | app.kubernetes.io/name: {{ include "proxmox-csi-plugin.name" . }} 57 | app.kubernetes.io/instance: {{ .Release.Name }} 58 | app.kubernetes.io/component: node 59 | {{- end }} 60 | 61 | 62 | {{/* 63 | Create the name of the service account to use 64 | */}} 65 | {{- define "proxmox-csi-plugin.serviceAccountName" -}} 66 | {{- if .Values.serviceAccount.create }} 67 | {{- default (include "proxmox-csi-plugin.fullname" .) .Values.serviceAccount.name }} 68 | {{- else }} 69 | {{- default "default" .Values.serviceAccount.name }} 70 | {{- end }} 71 | {{- end }} 72 | -------------------------------------------------------------------------------- /charts/proxmox-csi-plugin/templates/_storage.tpl: -------------------------------------------------------------------------------- 1 | {{/* 2 | storageClass parameters uses to merge the default parameters with the user provided parameters. 3 | */}} 4 | {{- define "storageClass.parameters" -}} 5 | csi.storage.k8s.io/fstype: {{ default "ext4" .fstype }} 6 | storage: {{ .storage | required "Proxmox Storage name must be provided." }} 7 | {{- with .cache }} 8 | cache: {{ . }} 9 | {{- end }} 10 | {{- if .ssd }} 11 | ssd: "true" 12 | {{- end }} 13 | {{- end }} 14 | -------------------------------------------------------------------------------- /charts/proxmox-csi-plugin/templates/controller-clusterrole.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRole 3 | metadata: 4 | name: {{ include "proxmox-csi-plugin.fullname" . }}-controller 5 | namespace: {{ .Release.Namespace }} 6 | labels: 7 | {{- include "proxmox-csi-plugin.labels" . | nindent 4 }} 8 | rules: 9 | - apiGroups: [""] 10 | resources: ["persistentvolumes"] 11 | verbs: ["get", "list", "watch", "create", "patch", "delete"] 12 | - apiGroups: [""] 13 | resources: ["persistentvolumeclaims"] 14 | verbs: ["get", "list", "watch", "update"] 15 | - apiGroups: [""] 16 | resources: ["persistentvolumeclaims/status"] 17 | verbs: ["patch"] 18 | - apiGroups: [""] 19 | resources: ["events"] 20 | verbs: ["get","list", "watch", "create", "update", "patch"] 21 | 22 | - apiGroups: ["storage.k8s.io"] 23 | resources: ["storageclasses"] 24 | verbs: ["get", "list", "watch"] 25 | - apiGroups: ["storage.k8s.io"] 26 | resources: ["csinodes"] 27 | verbs: ["get", "list", "watch"] 28 | - apiGroups: ["storage.k8s.io"] 29 | resources: ["volumeattributesclasses"] 30 | verbs: ["get", "list", "watch"] 31 | - apiGroups: [""] 32 | resources: ["nodes"] 33 | verbs: ["get", "list", "watch"] 34 | 35 | - apiGroups: ["storage.k8s.io"] 36 | resources: ["volumeattachments"] 37 | verbs: ["get", "list", "watch", "patch"] 38 | - apiGroups: ["storage.k8s.io"] 39 | resources: ["volumeattachments/status"] 40 | verbs: ["patch"] 41 | -------------------------------------------------------------------------------- /charts/proxmox-csi-plugin/templates/controller-deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: {{ include "proxmox-csi-plugin.fullname" . }}-controller 5 | namespace: {{ .Release.Namespace }} 6 | labels: 7 | {{- include "proxmox-csi-plugin.labels" . | nindent 4 }} 8 | spec: 9 | replicas: {{ .Values.replicaCount }} 10 | strategy: 11 | type: {{ .Values.updateStrategy.type }} 12 | rollingUpdate: 13 | {{- toYaml .Values.updateStrategy.rollingUpdate | nindent 6 }} 14 | selector: 15 | matchLabels: 16 | {{- include "proxmox-csi-plugin.selectorLabels" . | nindent 6 }} 17 | template: 18 | metadata: 19 | annotations: 20 | checksum/config: {{ toJson .Values.config | sha256sum }} 21 | {{- with default .Values.podAnnotations .Values.controller.podAnnotations }} 22 | {{- toYaml . | nindent 8 }} 23 | {{- end }} 24 | {{- if and .Values.metrics.enabled (eq .Values.metrics.type "annotation") }} 25 | prometheus.io/scrape: "true" 26 | prometheus.io/port: {{ .Values.metrics.port | quote }} 27 | {{- end }} 28 | labels: 29 | {{- include "proxmox-csi-plugin.selectorLabels" . | nindent 8 }} 30 | spec: 31 | {{- if .Values.priorityClassName }} 32 | priorityClassName: {{ .Values.priorityClassName }} 33 | {{- end }} 34 | {{- with .Values.imagePullSecrets }} 35 | imagePullSecrets: 36 | {{- toYaml . | nindent 8 }} 37 | {{- end }} 38 | enableServiceLinks: false 39 | serviceAccountName: {{ include "proxmox-csi-plugin.serviceAccountName" . }}-controller 40 | securityContext: 41 | {{- toYaml .Values.podSecurityContext | nindent 8 }} 42 | hostAliases: {{- toYaml .Values.hostAliases | nindent 8 }} 43 | initContainers: {{- toYaml .Values.initContainers | nindent 8 }} 44 | containers: 45 | - name: {{ include "proxmox-csi-plugin.fullname" . }}-controller 46 | securityContext: 47 | {{- toYaml .Values.securityContext | nindent 12 }} 48 | image: "{{ .Values.controller.plugin.image.repository }}:{{ .Values.controller.plugin.image.tag | default .Chart.AppVersion }}" 49 | imagePullPolicy: {{ .Values.controller.plugin.image.pullPolicy }} 50 | args: 51 | - "-v={{ .Values.logVerbosityLevel }}" 52 | - "--csi-address=unix:///csi/csi.sock" 53 | - "--cloud-config={{ .Values.configFile }}" 54 | {{- if .Values.metrics.enabled }} 55 | - "--metrics-address=:{{ .Values.metrics.port }}" 56 | {{- end }} 57 | ports: 58 | {{- if .Values.metrics.enabled }} 59 | - name: metrics 60 | containerPort: {{ .Values.metrics.port }} 61 | protocol: TCP 62 | {{- end }} 63 | resources: 64 | {{- toYaml .Values.controller.plugin.resources | nindent 12 }} 65 | volumeMounts: 66 | - name: socket-dir 67 | mountPath: /csi 68 | - name: cloud-config 69 | mountPath: /etc/proxmox/ 70 | {{- with .Values.extraVolumeMounts }} 71 | {{- toYaml . | nindent 12 }} 72 | {{- end }} 73 | - name: csi-attacher 74 | securityContext: 75 | {{- toYaml .Values.securityContext | nindent 12 }} 76 | image: "{{ .Values.controller.attacher.image.repository }}:{{ .Values.controller.attacher.image.tag }}" 77 | imagePullPolicy: {{ .Values.controller.attacher.image.pullPolicy }} 78 | args: 79 | - "-v={{ .Values.logVerbosityLevel }}" 80 | - "--csi-address=unix:///csi/csi.sock" 81 | - "--timeout={{ .Values.timeout }}" 82 | - "--leader-election" 83 | {{- range .Values.controller.attacher.args }} 84 | - {{ . | quote }} 85 | {{- end }} 86 | volumeMounts: 87 | - name: socket-dir 88 | mountPath: /csi 89 | resources: {{ toYaml .Values.controller.attacher.resources | nindent 12 }} 90 | - name: csi-provisioner 91 | securityContext: 92 | {{- toYaml .Values.securityContext | nindent 12 }} 93 | image: "{{ .Values.controller.provisioner.image.repository }}:{{ .Values.controller.provisioner.image.tag }}" 94 | imagePullPolicy: {{ .Values.controller.provisioner.image.pullPolicy }} 95 | args: 96 | - "-v={{ .Values.logVerbosityLevel }}" 97 | - "--csi-address=unix:///csi/csi.sock" 98 | - "--timeout={{ .Values.timeout }}" 99 | - "--leader-election" 100 | {{- if .Values.options.enableCapacity }} 101 | - "--enable-capacity" 102 | - "--capacity-ownerref-level=2" 103 | {{- end }} 104 | {{- range .Values.controller.provisioner.args }} 105 | - {{ . | quote }} 106 | {{- end }} 107 | env: 108 | - name: NAMESPACE 109 | valueFrom: 110 | fieldRef: 111 | fieldPath: metadata.namespace 112 | - name: POD_NAME 113 | valueFrom: 114 | fieldRef: 115 | fieldPath: metadata.name 116 | volumeMounts: 117 | - name: socket-dir 118 | mountPath: /csi 119 | resources: {{ toYaml .Values.controller.provisioner.resources | nindent 12 }} 120 | - name: csi-resizer 121 | securityContext: 122 | {{- toYaml .Values.securityContext | nindent 12 }} 123 | image: "{{ .Values.controller.resizer.image.repository }}:{{ .Values.controller.resizer.image.tag }}" 124 | imagePullPolicy: {{ .Values.controller.resizer.image.pullPolicy }} 125 | args: 126 | - "-v={{ .Values.logVerbosityLevel }}" 127 | - "--csi-address=unix:///csi/csi.sock" 128 | - "--timeout={{ .Values.timeout }}" 129 | - "--handle-volume-inuse-error=false" 130 | - "--leader-election" 131 | {{- range .Values.controller.resizer.args }} 132 | - {{ . | quote }} 133 | {{- end }} 134 | volumeMounts: 135 | - name: socket-dir 136 | mountPath: /csi 137 | resources: {{ toYaml .Values.controller.resizer.resources | nindent 12 }} 138 | - name: liveness-probe 139 | securityContext: 140 | {{- toYaml .Values.securityContext | nindent 12 }} 141 | image: "{{ .Values.livenessprobe.image.repository }}:{{ .Values.livenessprobe.image.tag }}" 142 | imagePullPolicy: {{ .Values.livenessprobe.image.pullPolicy }} 143 | args: 144 | - "-v={{ .Values.logVerbosityLevel }}" 145 | - "--csi-address=unix:///csi/csi.sock" 146 | volumeMounts: 147 | - name: socket-dir 148 | mountPath: /csi 149 | resources: {{ toYaml .Values.livenessprobe.resources | nindent 12 }} 150 | volumes: 151 | - name: socket-dir 152 | emptyDir: {} 153 | {{- if .Values.existingConfigSecret }} 154 | - name: cloud-config 155 | secret: 156 | secretName: {{ .Values.existingConfigSecret }} 157 | items: 158 | - key: {{ .Values.existingConfigSecretKey }} 159 | path: config.yaml 160 | {{- else }} 161 | - name: cloud-config 162 | secret: 163 | secretName: {{ include "proxmox-csi-plugin.fullname" . }} 164 | {{- end }} 165 | {{- with .Values.extraVolumes }} 166 | {{- toYaml . | nindent 8 }} 167 | {{- end }} 168 | {{- with .Values.nodeSelector }} 169 | nodeSelector: 170 | {{- toYaml . | nindent 8 }} 171 | {{- end }} 172 | {{- with .Values.affinity }} 173 | affinity: 174 | {{- toYaml . | nindent 8 }} 175 | {{- end }} 176 | {{- with .Values.tolerations }} 177 | tolerations: 178 | {{- toYaml . | nindent 8 }} 179 | {{- end }} 180 | topologySpreadConstraints: 181 | - maxSkew: 1 182 | topologyKey: kubernetes.io/hostname 183 | whenUnsatisfiable: DoNotSchedule 184 | labelSelector: 185 | matchLabels: 186 | {{- include "proxmox-csi-plugin.selectorLabels" . | nindent 14 }} 187 | -------------------------------------------------------------------------------- /charts/proxmox-csi-plugin/templates/controller-role.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: Role 3 | metadata: 4 | name: {{ include "proxmox-csi-plugin.fullname" . }}-controller 5 | namespace: {{ .Release.Namespace }} 6 | labels: 7 | {{- include "proxmox-csi-plugin.labels" . | nindent 4 }} 8 | rules: 9 | - apiGroups: ["coordination.k8s.io"] 10 | resources: ["leases"] 11 | verbs: ["get", "watch", "list", "delete", "update", "create"] 12 | 13 | - apiGroups: ["storage.k8s.io"] 14 | resources: ["csistoragecapacities"] 15 | verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] 16 | - apiGroups: [""] 17 | resources: ["pods"] 18 | verbs: ["get"] 19 | - apiGroups: ["apps"] 20 | resources: ["replicasets"] 21 | verbs: ["get"] 22 | -------------------------------------------------------------------------------- /charts/proxmox-csi-plugin/templates/controller-rolebinding.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRoleBinding 3 | metadata: 4 | name: {{ include "proxmox-csi-plugin.fullname" . }}-controller 5 | roleRef: 6 | apiGroup: rbac.authorization.k8s.io 7 | kind: ClusterRole 8 | name: {{ include "proxmox-csi-plugin.fullname" . }}-controller 9 | subjects: 10 | - kind: ServiceAccount 11 | name: {{ include "proxmox-csi-plugin.serviceAccountName" . }}-controller 12 | namespace: {{ .Release.Namespace }} 13 | --- 14 | apiVersion: rbac.authorization.k8s.io/v1 15 | kind: RoleBinding 16 | metadata: 17 | name: {{ include "proxmox-csi-plugin.fullname" . }}-controller 18 | namespace: {{ .Release.Namespace }} 19 | roleRef: 20 | apiGroup: rbac.authorization.k8s.io 21 | kind: Role 22 | name: {{ include "proxmox-csi-plugin.fullname" . }}-controller 23 | subjects: 24 | - kind: ServiceAccount 25 | name: {{ include "proxmox-csi-plugin.serviceAccountName" . }}-controller 26 | namespace: {{ .Release.Namespace }} 27 | -------------------------------------------------------------------------------- /charts/proxmox-csi-plugin/templates/csidriver.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: storage.k8s.io/v1 2 | kind: CSIDriver 3 | metadata: 4 | name: {{ .Values.provisionerName }} 5 | spec: 6 | attachRequired: true 7 | podInfoOnMount: true 8 | storageCapacity: true 9 | volumeLifecycleModes: 10 | - Persistent 11 | -------------------------------------------------------------------------------- /charts/proxmox-csi-plugin/templates/namespace.yaml: -------------------------------------------------------------------------------- 1 | {{- if and .Values.createNamespace (ne .Release.Namespace "kube-system") }} 2 | apiVersion: v1 3 | kind: Namespace 4 | metadata: 5 | name: {{ .Release.Namespace }} 6 | labels: 7 | pod-security.kubernetes.io/enforce: privileged 8 | pod-security.kubernetes.io/audit: baseline 9 | pod-security.kubernetes.io/warn: baseline 10 | {{- end }} 11 | -------------------------------------------------------------------------------- /charts/proxmox-csi-plugin/templates/node-clusterrole.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRole 3 | metadata: 4 | name: {{ include "proxmox-csi-plugin.fullname" . }}-node 5 | namespace: {{ .Release.Namespace }} 6 | labels: 7 | {{- include "proxmox-csi-plugin.labels" . | nindent 4 }} 8 | rules: 9 | - apiGroups: 10 | - "" 11 | resources: 12 | - nodes 13 | verbs: 14 | - get 15 | -------------------------------------------------------------------------------- /charts/proxmox-csi-plugin/templates/node-deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: DaemonSet 3 | metadata: 4 | name: {{ include "proxmox-csi-plugin.fullname" . }}-node 5 | namespace: {{ .Release.Namespace }} 6 | labels: 7 | {{- include "proxmox-csi-plugin.labels" . | nindent 4 }} 8 | spec: 9 | updateStrategy: 10 | type: {{ .Values.updateStrategy.type }} 11 | selector: 12 | matchLabels: 13 | {{- include "proxmox-csi-plugin-node.selectorLabels" . | nindent 6 }} 14 | template: 15 | metadata: 16 | {{- with .Values.podAnnotations }} 17 | annotations: 18 | {{- toYaml . | nindent 8 }} 19 | {{- end }} 20 | labels: 21 | {{- include "proxmox-csi-plugin-node.selectorLabels" . | nindent 8 }} 22 | spec: 23 | priorityClassName: system-node-critical 24 | {{- with .Values.imagePullSecrets }} 25 | imagePullSecrets: 26 | {{- toYaml . | nindent 8 }} 27 | {{- end }} 28 | enableServiceLinks: false 29 | serviceAccountName: {{ include "proxmox-csi-plugin.serviceAccountName" . }}-node 30 | securityContext: 31 | runAsUser: 0 32 | runAsGroup: 0 33 | containers: 34 | - name: {{ include "proxmox-csi-plugin.fullname" . }}-node 35 | securityContext: 36 | privileged: true 37 | capabilities: 38 | drop: 39 | - ALL 40 | add: 41 | - SYS_ADMIN 42 | - CHOWN 43 | - DAC_OVERRIDE 44 | seccompProfile: 45 | type: RuntimeDefault 46 | image: "{{ .Values.node.plugin.image.repository }}:{{ .Values.node.plugin.image.tag | default .Chart.AppVersion }}" 47 | imagePullPolicy: {{ .Values.node.plugin.image.pullPolicy }} 48 | args: 49 | - "-v={{ .Values.logVerbosityLevel }}" 50 | - "--csi-address=unix:///csi/csi.sock" 51 | - "--node-id=$(NODE_NAME)" 52 | env: 53 | - name: NODE_NAME 54 | valueFrom: 55 | fieldRef: 56 | fieldPath: spec.nodeName 57 | resources: {{- toYaml .Values.node.plugin.resources | nindent 12 }} 58 | volumeMounts: 59 | - name: socket 60 | mountPath: /csi 61 | - name: kubelet 62 | mountPath: {{ .Values.node.kubeletDir }} 63 | mountPropagation: Bidirectional 64 | - name: dev 65 | mountPath: /dev 66 | - name: sys 67 | mountPath: /sys 68 | - name: csi-node-driver-registrar 69 | securityContext: 70 | allowPrivilegeEscalation: false 71 | capabilities: 72 | drop: 73 | - ALL 74 | readOnlyRootFilesystem: true 75 | seccompProfile: 76 | type: RuntimeDefault 77 | image: "{{ .Values.node.driverRegistrar.image.repository }}:{{ .Values.node.driverRegistrar.image.tag }}" 78 | imagePullPolicy: {{ .Values.node.driverRegistrar.image.pullPolicy }} 79 | args: 80 | - "-v={{ .Values.logVerbosityLevel }}" 81 | - "--csi-address=unix:///csi/csi.sock" 82 | - "--kubelet-registration-path={{ .Values.node.kubeletDir }}/plugins/{{ .Values.provisionerName }}/csi.sock" 83 | volumeMounts: 84 | - name: socket 85 | mountPath: /csi 86 | - name: registration 87 | mountPath: /registration 88 | resources: {{- toYaml .Values.node.driverRegistrar.resources | nindent 12 }} 89 | - name: liveness-probe 90 | securityContext: 91 | allowPrivilegeEscalation: false 92 | capabilities: 93 | drop: 94 | - ALL 95 | readOnlyRootFilesystem: true 96 | seccompProfile: 97 | type: RuntimeDefault 98 | image: "{{ .Values.livenessprobe.image.repository }}:{{ .Values.livenessprobe.image.tag }}" 99 | imagePullPolicy: {{ .Values.livenessprobe.image.pullPolicy }} 100 | args: 101 | - "-v={{ .Values.logVerbosityLevel }}" 102 | - "--csi-address=unix:///csi/csi.sock" 103 | volumeMounts: 104 | - name: socket 105 | mountPath: /csi 106 | resources: {{- toYaml .Values.livenessprobe.resources | nindent 12 }} 107 | volumes: 108 | - name: socket 109 | hostPath: 110 | path: {{ .Values.node.kubeletDir }}/plugins/{{ .Values.provisionerName }}/ 111 | type: DirectoryOrCreate 112 | - name: registration 113 | hostPath: 114 | path: {{ .Values.node.kubeletDir }}/plugins_registry/ 115 | type: Directory 116 | - name: kubelet 117 | hostPath: 118 | path: {{ .Values.node.kubeletDir }} 119 | type: Directory 120 | - name: dev 121 | hostPath: 122 | path: /dev 123 | type: Directory 124 | - name: sys 125 | hostPath: 126 | path: /sys 127 | type: Directory 128 | {{- with .Values.node.nodeSelector }} 129 | nodeSelector: 130 | {{- toYaml . | nindent 8 }} 131 | {{- end }} 132 | {{- with .Values.node.tolerations }} 133 | tolerations: 134 | {{- toYaml . | nindent 8 }} 135 | {{- end }} 136 | -------------------------------------------------------------------------------- /charts/proxmox-csi-plugin/templates/node-rolebinding.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRoleBinding 3 | metadata: 4 | name: {{ include "proxmox-csi-plugin.fullname" . }}-node 5 | roleRef: 6 | apiGroup: rbac.authorization.k8s.io 7 | kind: ClusterRole 8 | name: {{ include "proxmox-csi-plugin.fullname" . }}-node 9 | subjects: 10 | - kind: ServiceAccount 11 | name: {{ include "proxmox-csi-plugin.serviceAccountName" . }}-node 12 | namespace: {{ .Release.Namespace }} 13 | -------------------------------------------------------------------------------- /charts/proxmox-csi-plugin/templates/secrets.yaml: -------------------------------------------------------------------------------- 1 | {{- if ne (len .Values.config.clusters) 0 }} 2 | apiVersion: v1 3 | kind: Secret 4 | metadata: 5 | name: {{ include "proxmox-csi-plugin.fullname" . }} 6 | namespace: {{ .Release.Namespace }} 7 | labels: 8 | {{- include "proxmox-csi-plugin.labels" . | nindent 4 }} 9 | type: Opaque 10 | data: 11 | config.yaml: {{ toYaml .Values.config | b64enc | quote }} 12 | {{- end }} 13 | -------------------------------------------------------------------------------- /charts/proxmox-csi-plugin/templates/serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.serviceAccount.create -}} 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | name: {{ include "proxmox-csi-plugin.serviceAccountName" . }}-controller 6 | namespace: {{ .Release.Namespace }} 7 | labels: 8 | {{- include "proxmox-csi-plugin.labels" . | nindent 4 }} 9 | {{- with .Values.serviceAccount.annotations }} 10 | annotations: 11 | {{- toYaml . | nindent 4 }} 12 | {{- end }} 13 | --- 14 | apiVersion: v1 15 | kind: ServiceAccount 16 | metadata: 17 | name: {{ include "proxmox-csi-plugin.serviceAccountName" . }}-node 18 | namespace: {{ .Release.Namespace }} 19 | labels: 20 | {{- include "proxmox-csi-plugin.labels" . | nindent 4 }} 21 | {{- with .Values.serviceAccount.annotations }} 22 | annotations: 23 | {{- toYaml . | nindent 4 }} 24 | {{- end }} 25 | {{- end }} 26 | -------------------------------------------------------------------------------- /charts/proxmox-csi-plugin/templates/storageclass.yaml: -------------------------------------------------------------------------------- 1 | {{- range $storage := .Values.storageClass }} 2 | apiVersion: storage.k8s.io/v1 3 | kind: StorageClass 4 | metadata: 5 | name: {{ $storage.name | required "StorageClass name must be provided." }} 6 | {{- with $storage.labels }} 7 | labels: 8 | {{- toYaml . | nindent 4 }} 9 | {{- end }} 10 | {{- with $storage.annotations }} 11 | annotations: 12 | {{- toYaml . | nindent 4 }} 13 | {{- end }} 14 | provisioner: {{ $.Values.provisionerName }} 15 | allowVolumeExpansion: true 16 | volumeBindingMode: WaitForFirstConsumer 17 | reclaimPolicy: {{ default "Delete" $storage.reclaimPolicy }} 18 | parameters: 19 | {{- mustMergeOverwrite (default (dict) $storage.extraParameters) (include "storageClass.parameters" . | fromYaml) | toYaml | nindent 2 -}} 20 | {{- with $storage.mountOptions }} 21 | mountOptions: 22 | {{- . | toYaml | nindent 2 }} 23 | {{- end }} 24 | {{- with $storage.allowedTopologies }} 25 | allowedTopologies: 26 | {{- . | toYaml | nindent 2 }} 27 | {{- end }} 28 | --- 29 | {{- end }} 30 | -------------------------------------------------------------------------------- /charts/proxmox-csi-plugin/values.edge.yaml: -------------------------------------------------------------------------------- 1 | 2 | createNamespace: true 3 | 4 | controller: 5 | plugin: 6 | image: 7 | pullPolicy: Always 8 | tag: edge 9 | 10 | node: 11 | plugin: 12 | image: 13 | pullPolicy: Always 14 | tag: edge 15 | 16 | nodeSelector: 17 | node-role.kubernetes.io/control-plane: "" 18 | tolerations: 19 | - key: node-role.kubernetes.io/control-plane 20 | effect: NoSchedule 21 | 22 | storageClass: 23 | - name: proxmox-data-xfs 24 | storage: data 25 | reclaimPolicy: Delete 26 | fstype: xfs 27 | - name: proxmox-data 28 | storage: data 29 | ssd: true 30 | -------------------------------------------------------------------------------- /charts/proxmox-csi-plugin/values.talos.yaml: -------------------------------------------------------------------------------- 1 | 2 | createNamespace: true 3 | 4 | node: 5 | nodeSelector: 6 | node.cloudprovider.kubernetes.io/platform: nocloud 7 | tolerations: 8 | - operator: Exists 9 | 10 | nodeSelector: 11 | node-role.kubernetes.io/control-plane: "" 12 | tolerations: 13 | - key: node-role.kubernetes.io/control-plane 14 | effect: NoSchedule 15 | 16 | storageClass: 17 | - name: proxmox-data-xfs 18 | storage: data 19 | reclaimPolicy: Delete 20 | fstype: xfs 21 | - name: proxmox-data 22 | storage: data 23 | reclaimPolicy: Delete 24 | -------------------------------------------------------------------------------- /cmd/controller/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Proxmox CSI Plugin Controller 18 | package main 19 | 20 | import ( 21 | "context" 22 | "flag" 23 | "net" 24 | "net/http" 25 | "os" 26 | 27 | proto "github.com/container-storage-interface/spec/lib/go/csi" 28 | "google.golang.org/grpc" 29 | 30 | "github.com/sergelogvinov/proxmox-csi-plugin/pkg/csi" 31 | "github.com/sergelogvinov/proxmox-csi-plugin/pkg/tools" 32 | 33 | clientkubernetes "k8s.io/client-go/kubernetes" 34 | "k8s.io/component-base/metrics/legacyregistry" 35 | "k8s.io/klog/v2" 36 | ) 37 | 38 | var ( 39 | version string 40 | commit string 41 | 42 | showVersion = flag.Bool("version", false, "Print the version and exit.") 43 | csiEndpoint = flag.String("csi-address", "unix:///csi/csi.sock", "CSI Endpoint") 44 | 45 | metricsAddress = flag.String("metrics-address", "", "The TCP network address where the HTTP server for metrics, will listen (example: `:8080`). By default the server is disabled.") 46 | metricsPath = flag.String("metrics-path", "/metrics", "The HTTP path where prometheus metrics will be exposed.") 47 | 48 | cloudconfig = flag.String("cloud-config", "", "The path to the CSI driver cloud config.") 49 | kubeconfig = flag.String("kubeconfig", "", "Absolute path to the kubeconfig file. Either this or master needs to be set if the provisioner is being run out of cluster.") 50 | ) 51 | 52 | func main() { 53 | klog.InitFlags(nil) 54 | flag.Set("logtostderr", "true") //nolint: errcheck 55 | flag.Parse() 56 | 57 | klog.V(2).InfoS("Version", "version", csi.DriverVersion, "csiVersion", csi.DriverSpecVersion, "gitVersion", version, "gitCommit", commit) 58 | 59 | if *showVersion { 60 | klog.Infof("Driver version %v, GitVersion %s", csi.DriverVersion, version) 61 | os.Exit(0) 62 | } 63 | 64 | if *csiEndpoint == "" { 65 | klog.Error("csi-address must be provided") 66 | klog.FlushAndExit(klog.ExitFlushTimeout, 1) 67 | } 68 | 69 | if *cloudconfig == "" { 70 | klog.Error("cloud-config must be provided") 71 | klog.FlushAndExit(klog.ExitFlushTimeout, 1) 72 | } 73 | 74 | kconfig, _, err := tools.BuildConfig(*kubeconfig, "") 75 | if err != nil { 76 | klog.Error(err, "Failed to build a Kubernetes config") 77 | klog.FlushAndExit(klog.ExitFlushTimeout, 1) 78 | } 79 | 80 | clientset, err := clientkubernetes.NewForConfig(kconfig) 81 | if err != nil { 82 | klog.Error(err, "Failed to create a Clientset") 83 | klog.FlushAndExit(klog.ExitFlushTimeout, 1) 84 | } 85 | 86 | scheme, addr, err := csi.ParseEndpoint(*csiEndpoint) 87 | if err != nil { 88 | klog.Error(err, "Failed to parse endpoint") 89 | klog.FlushAndExit(klog.ExitFlushTimeout, 1) 90 | } 91 | 92 | listener, err := net.Listen(scheme, addr) 93 | if err != nil { 94 | klog.ErrorS(err, "Failed to listen", "address", *csiEndpoint) 95 | klog.FlushAndExit(klog.ExitFlushTimeout, 1) 96 | } 97 | 98 | logErr := func(ctx context.Context, req interface{}, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { 99 | resp, rpcerr := handler(ctx, req) 100 | if rpcerr != nil { 101 | klog.ErrorS(rpcerr, "GRPC error") 102 | } 103 | 104 | return resp, rpcerr 105 | } 106 | 107 | opts := []grpc.ServerOption{ 108 | grpc.UnaryInterceptor(logErr), 109 | } 110 | 111 | // Prepare http endpoint for metrics 112 | mux := http.NewServeMux() 113 | if *metricsAddress != "" { 114 | mux.Handle("/metrics", legacyregistry.Handler()) 115 | 116 | go func() { 117 | klog.V(2).InfoS("Metrics listening", "address", *metricsAddress, "metricsPath", *metricsPath) 118 | 119 | err := http.ListenAndServe(*metricsAddress, mux) 120 | if err != nil { 121 | klog.ErrorS(err, "Failed to start HTTP server at specified address and metrics path", "address", addr, "metricsPath", *metricsPath) 122 | } 123 | }() 124 | } 125 | 126 | srv := grpc.NewServer(opts...) 127 | 128 | identityService := csi.NewIdentityService() 129 | 130 | controllerService, err := csi.NewControllerService(clientset, *cloudconfig) 131 | if err != nil { 132 | klog.ErrorS(err, "Failed to create controller service") 133 | klog.FlushAndExit(klog.ExitFlushTimeout, 1) 134 | } 135 | 136 | proto.RegisterControllerServer(srv, controllerService) 137 | proto.RegisterIdentityServer(srv, identityService) 138 | 139 | klog.InfoS("Listening for connection on address", "address", listener.Addr()) 140 | 141 | if err := srv.Serve(listener); err != nil { 142 | klog.ErrorS(err, "Failed to run driver") 143 | klog.FlushAndExit(klog.ExitFlushTimeout, 1) 144 | } 145 | } 146 | -------------------------------------------------------------------------------- /cmd/node/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Main package for the node driver. 18 | package main 19 | 20 | import ( 21 | "context" 22 | "flag" 23 | "net" 24 | "os" 25 | 26 | proto "github.com/container-storage-interface/spec/lib/go/csi" 27 | "google.golang.org/grpc" 28 | 29 | "github.com/sergelogvinov/proxmox-csi-plugin/pkg/csi" 30 | 31 | clientkubernetes "k8s.io/client-go/kubernetes" 32 | "k8s.io/client-go/rest" 33 | "k8s.io/client-go/tools/clientcmd" 34 | "k8s.io/klog/v2" 35 | ) 36 | 37 | var ( 38 | version string 39 | commit string 40 | 41 | showVersion = flag.Bool("version", false, "Print the version and exit.") 42 | csiEndpoint = flag.String("csi-address", "unix:///csi/csi.sock", "CSI Endpoint") 43 | nodeID = flag.String("node-id", "", "Node name") 44 | 45 | master = flag.String("master", "", "Master URL to build a client config from. Either this or kubeconfig needs to be set if the provisioner is being run out of cluster.") 46 | kubeconfig = flag.String("kubeconfig", "", "Absolute path to the kubeconfig file. Either this or master needs to be set if the provisioner is being run out of cluster.") 47 | ) 48 | 49 | func main() { 50 | klog.InitFlags(nil) 51 | flag.Set("logtostderr", "true") //nolint: errcheck 52 | flag.Parse() 53 | 54 | klog.V(2).Infof("Driver version %v, GitVersion %s, GitCommit %s", csi.DriverVersion, version, commit) 55 | klog.V(2).Info("Driver CSI Spec version: ", csi.DriverSpecVersion) 56 | 57 | if *showVersion { 58 | klog.Infof("Driver version %v, GitVersion %s", csi.DriverVersion, version) 59 | klog.Info("Driver CSI Spec version: ", csi.DriverSpecVersion) 60 | os.Exit(0) 61 | } 62 | 63 | kubeconfigEnv := os.Getenv("KUBECONFIG") 64 | if kubeconfigEnv != "" { 65 | klog.Infof("Found KUBECONFIG environment variable set, using that..") 66 | 67 | kubeconfig = &kubeconfigEnv 68 | } 69 | 70 | var ( 71 | config *rest.Config 72 | err error 73 | ) 74 | 75 | if *master != "" || *kubeconfig != "" { 76 | klog.Infof("Either master or kubeconfig specified. building kube config from that..") 77 | 78 | config, err = clientcmd.BuildConfigFromFlags(*master, *kubeconfig) 79 | if err != nil { 80 | klog.Fatal(err) 81 | } 82 | } else { 83 | klog.Infof("Building kube configs for running in cluster...") 84 | 85 | config, err = rest.InClusterConfig() 86 | if err != nil { 87 | klog.Fatal(err) 88 | } 89 | } 90 | 91 | clientset, err := clientkubernetes.NewForConfig(config) 92 | if err != nil { 93 | klog.Fatalf("Failed to create client: %v", err) 94 | } 95 | 96 | if *csiEndpoint == "" { 97 | klog.Fatalln("csi-address must be provided") 98 | } 99 | 100 | nodeName := *nodeID 101 | if nodeName == "" { 102 | nodeName = os.Getenv("NODE_NAME") 103 | 104 | if nodeName == "" { 105 | klog.Fatalln("node-id or NODE_NAME environment must be provided") 106 | } 107 | } 108 | 109 | scheme, addr, err := csi.ParseEndpoint(*csiEndpoint) 110 | if err != nil { 111 | klog.Fatalf("Failed to parse endpoint: %v", err) 112 | } 113 | 114 | listener, err := net.Listen(scheme, addr) 115 | if err != nil { 116 | klog.Fatalf("Failed to listen on %s: %v", *csiEndpoint, err) 117 | } 118 | 119 | logErr := func(ctx context.Context, req interface{}, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { 120 | resp, rpcerr := handler(ctx, req) 121 | if rpcerr != nil { 122 | klog.Errorf("GRPC error: %v", rpcerr) 123 | } 124 | 125 | return resp, rpcerr 126 | } 127 | 128 | opts := []grpc.ServerOption{ 129 | grpc.UnaryInterceptor(logErr), 130 | } 131 | 132 | srv := grpc.NewServer(opts...) 133 | 134 | identityService := csi.NewIdentityService() 135 | nodeService := csi.NewNodeService(nodeName, clientset) 136 | 137 | proto.RegisterIdentityServer(srv, identityService) 138 | proto.RegisterNodeServer(srv, nodeService) 139 | 140 | klog.Infof("Listening for connection on address: %#v", listener.Addr()) 141 | 142 | if err := srv.Serve(listener); err != nil { 143 | klog.Fatalf("Failed to serve: %v", err) 144 | } 145 | } 146 | -------------------------------------------------------------------------------- /cmd/pvecsictl/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Proxmox PV Migrate utility 18 | package main 19 | 20 | import ( 21 | "context" 22 | "fmt" 23 | "os" 24 | "strings" 25 | 26 | log "github.com/sirupsen/logrus" 27 | cobra "github.com/spf13/cobra" 28 | 29 | clilog "github.com/sergelogvinov/proxmox-csi-plugin/pkg/log" 30 | ) 31 | 32 | var ( 33 | command = "pvecsictl" 34 | version = "v0.0.0" 35 | commit = "none" 36 | 37 | cloudconfig string 38 | kubeconfig string 39 | 40 | flagLogLevel = "log-level" 41 | 42 | flagProxmoxConfig = "config" 43 | flagKubeConfig = "kubeconfig" 44 | 45 | logger *log.Entry 46 | ) 47 | 48 | func main() { 49 | if exitCode := run(); exitCode != 0 { 50 | os.Exit(exitCode) 51 | } 52 | } 53 | 54 | func run() int { 55 | ctx, cancel := context.WithCancel(context.Background()) 56 | defer cancel() 57 | 58 | l := log.New() 59 | l.SetOutput(os.Stdout) 60 | l.SetLevel(log.InfoLevel) 61 | 62 | logger = l.WithContext(ctx) 63 | 64 | cmd := cobra.Command{ 65 | Use: command, 66 | Version: fmt.Sprintf("%s (commit: %s)", version, commit), 67 | Short: "A command-line utility to manipulate PersistentVolume/PersistentVolumeClaim on Proxmox VE", 68 | PersistentPreRunE: func(cmd *cobra.Command, _ []string) error { 69 | f := cmd.Flags() 70 | loglvl, _ := f.GetString(flagLogLevel) //nolint: errcheck 71 | 72 | clilog.Configure(logger, loglvl) 73 | 74 | return nil 75 | }, 76 | SilenceUsage: true, 77 | SilenceErrors: true, 78 | } 79 | 80 | cmd.PersistentFlags().String(flagLogLevel, clilog.LevelInfo, 81 | fmt.Sprintf("log level, must be one of: %s", strings.Join(clilog.Levels, ", "))) 82 | 83 | cmd.PersistentFlags().StringVar(&cloudconfig, flagProxmoxConfig, "", "proxmox cluster config file") 84 | cmd.PersistentFlags().StringVar(&kubeconfig, flagKubeConfig, "", "kubernetes config file") 85 | 86 | cmd.AddCommand(buildMigrateCmd()) 87 | cmd.AddCommand(buildRenameCmd()) 88 | cmd.AddCommand(buildSwapCmd()) 89 | 90 | err := cmd.ExecuteContext(ctx) 91 | if err != nil { 92 | errorString := err.Error() 93 | if strings.Contains(errorString, "arg(s)") || strings.Contains(errorString, "flag") || strings.Contains(errorString, "command") { 94 | fmt.Fprintf(os.Stderr, "Error: %s\n\n", errorString) 95 | fmt.Fprintln(os.Stderr, cmd.UsageString()) 96 | } else { 97 | logger.Errorf("Error: %s\n", errorString) 98 | } 99 | 100 | return 1 101 | } 102 | 103 | return 0 104 | } 105 | -------------------------------------------------------------------------------- /cmd/pvecsictl/migrate.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "context" 21 | "fmt" 22 | "strings" 23 | "time" 24 | 25 | cobra "github.com/spf13/cobra" 26 | 27 | proxmox "github.com/sergelogvinov/proxmox-cloud-controller-manager/pkg/cluster" 28 | "github.com/sergelogvinov/proxmox-csi-plugin/pkg/csi" 29 | vm "github.com/sergelogvinov/proxmox-csi-plugin/pkg/proxmox" 30 | tools "github.com/sergelogvinov/proxmox-csi-plugin/pkg/tools" 31 | volume "github.com/sergelogvinov/proxmox-csi-plugin/pkg/volume" 32 | 33 | rbacv1 "k8s.io/api/authorization/v1" 34 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 35 | clientkubernetes "k8s.io/client-go/kubernetes" 36 | ) 37 | 38 | type migrateCmd struct { 39 | pclient *proxmox.Cluster 40 | kclient *clientkubernetes.Clientset 41 | namespace string 42 | } 43 | 44 | func buildMigrateCmd() *cobra.Command { 45 | c := &migrateCmd{} 46 | 47 | cmd := cobra.Command{ 48 | Use: "migrate pvc proxmox-node", 49 | Aliases: []string{"m"}, 50 | Short: "Migrate data from one Proxmox node to another", 51 | Args: cobra.ExactArgs(2), 52 | PreRunE: c.migrationValidate, 53 | RunE: c.runMigration, 54 | SilenceUsage: true, 55 | SilenceErrors: true, 56 | } 57 | 58 | setMigrateCmdFlags(&cmd) 59 | 60 | return &cmd 61 | } 62 | 63 | func setMigrateCmdFlags(cmd *cobra.Command) { 64 | flags := cmd.Flags() 65 | 66 | flags.StringP("namespace", "n", "", "namespace of the persistentvolumeclaims") 67 | 68 | flags.BoolP("force", "f", false, "force migration even if the persistentvolumeclaims is in use") 69 | flags.Int("timeout", 7200, "task timeout in seconds") 70 | } 71 | 72 | // nolint: cyclop, gocyclo 73 | func (c *migrateCmd) runMigration(cmd *cobra.Command, args []string) error { 74 | flags := cmd.Flags() 75 | force, _ := flags.GetBool("force") //nolint: errcheck 76 | 77 | var err error 78 | 79 | ctx := context.Background() 80 | pvc := args[0] 81 | node := args[1] 82 | 83 | kubePVC, kubePV, err := tools.PVCResources(ctx, c.kclient, c.namespace, pvc) 84 | if err != nil { 85 | return fmt.Errorf("failed to get resources: %v", err) 86 | } 87 | 88 | vol, err := volume.NewVolumeFromVolumeID(kubePV.Spec.CSI.VolumeHandle) 89 | if err != nil { 90 | return fmt.Errorf("failed to parse volume ID: %v", err) 91 | } 92 | 93 | if vol.Node() == node { 94 | return fmt.Errorf("persistentvolumeclaims %s is already on proxmox node %s", pvc, node) 95 | } 96 | 97 | cluster, err := c.pclient.GetProxmoxCluster(vol.Cluster()) 98 | if err != nil { 99 | return fmt.Errorf("failed to get Proxmox cluster: %v", err) 100 | } 101 | 102 | pods, vmName, err := tools.PVCPodUsage(ctx, c.kclient, c.namespace, pvc) 103 | if err != nil { 104 | return fmt.Errorf("failed to find pods using pvc: %v", err) 105 | } 106 | 107 | cordonedNodes := []string{} 108 | 109 | if len(pods) > 0 { 110 | if force { 111 | logger.Infof("persistentvolumeclaims is using by pods: %s on node %s, trying to force migration\n", strings.Join(pods, ","), vmName) 112 | 113 | var csiNodes []string 114 | 115 | csiNodes, err = tools.CSINodes(ctx, c.kclient, csi.DriverName) 116 | if err != nil { 117 | return err 118 | } 119 | 120 | cordonedNodes = append(cordonedNodes, csiNodes...) 121 | 122 | logger.Infof("cordoning nodes: %s", strings.Join(cordonedNodes, ",")) 123 | 124 | if _, err = tools.CondonNodes(ctx, c.kclient, cordonedNodes); err != nil { 125 | return fmt.Errorf("failed to cordon nodes: %v", err) 126 | } 127 | 128 | logger.Infof("terminated pods: %s", strings.Join(pods, ",")) 129 | 130 | for _, pod := range pods { 131 | if err = c.kclient.CoreV1().Pods(c.namespace).Delete(ctx, pod, metav1.DeleteOptions{}); err != nil { 132 | return fmt.Errorf("failed to delete pod: %v", err) 133 | } 134 | } 135 | 136 | for { 137 | p, _, e := tools.PVCPodUsage(ctx, c.kclient, c.namespace, pvc) 138 | if e != nil { 139 | return fmt.Errorf("failed to find pods using pvc: %v", e) 140 | } 141 | 142 | if len(p) == 0 { 143 | break 144 | } 145 | 146 | logger.Infof("waiting pods: %s", strings.Join(p, " ")) 147 | 148 | time.Sleep(2 * time.Second) 149 | } 150 | 151 | time.Sleep(5 * time.Second) 152 | } else { 153 | return fmt.Errorf("persistentvolumeclaims is using by pods: %s on node %s, cannot move volume", strings.Join(pods, ","), vmName) 154 | } 155 | } 156 | 157 | if err = vm.WaitForVolumeDetach(cluster, vmName, vol.Disk()); err != nil { 158 | return fmt.Errorf("failed to wait for volume detach: %v", err) 159 | } 160 | 161 | logger.Infof("moving disk %s to proxmox node %s", vol.Disk(), node) 162 | 163 | taskTimeout, _ := flags.GetInt("timeout") //nolint: errcheck 164 | if err = vm.MoveQemuDisk(cluster, vol, node, taskTimeout); err != nil { 165 | return fmt.Errorf("failed to move disk: %v", err) 166 | } 167 | 168 | logger.Infof("replacing persistentvolume topology") 169 | 170 | if err = replacePVTopology(ctx, c.kclient, c.namespace, kubePVC, kubePV, vol, node); err != nil { 171 | return fmt.Errorf("failed to replace PV topology: %v", err) 172 | } 173 | 174 | if force { 175 | logger.Infof("uncordoning nodes: %s", strings.Join(cordonedNodes, ",")) 176 | 177 | if err = tools.UncondonNodes(ctx, c.kclient, cordonedNodes); err != nil { 178 | return fmt.Errorf("failed to uncordon nodes: %v", err) 179 | } 180 | } 181 | 182 | logger.Infof("persistentvolumeclaims %s has been migrated to proxmox node %s", pvc, node) 183 | 184 | return nil 185 | } 186 | 187 | // nolint: dupl 188 | func (c *migrateCmd) migrationValidate(cmd *cobra.Command, _ []string) error { 189 | flags := cmd.Flags() 190 | 191 | cfg, err := proxmox.ReadCloudConfigFromFile(cloudconfig) 192 | if err != nil { 193 | return fmt.Errorf("failed to read config: %v", err) 194 | } 195 | 196 | for _, c := range cfg.Clusters { 197 | if c.Username == "" || c.Password == "" { 198 | return fmt.Errorf("this command requires Proxmox root account, please provide username and password in config file (cluster=%s)", c.Region) 199 | } 200 | } 201 | 202 | c.pclient, err = proxmox.NewCluster(&cfg, nil) 203 | if err != nil { 204 | return fmt.Errorf("failed to create Proxmox cluster client: %v", err) 205 | } 206 | 207 | if err = c.pclient.CheckClusters(); err != nil { 208 | return fmt.Errorf("failed to initialize Proxmox clusters: %v", err) 209 | } 210 | 211 | namespace, _ := flags.GetString("namespace") //nolint: errcheck 212 | 213 | kclientConfig, namespace, err := tools.BuildConfig(kubeconfig, namespace) 214 | if err != nil { 215 | return fmt.Errorf("failed to create kubernetes config: %v", err) 216 | } 217 | 218 | c.kclient, err = clientkubernetes.NewForConfig(kclientConfig) 219 | if err != nil { 220 | return fmt.Errorf("failed to create kubernetes client: %v", err) 221 | } 222 | 223 | c.namespace = namespace 224 | 225 | accessCheck := []rbacv1.ResourceAttributes{ 226 | {Group: "", Namespace: "", Resource: "persistentvolumeclaims", Verb: "create"}, 227 | {Group: "", Namespace: "", Resource: "persistentvolumeclaims", Verb: "delete"}, 228 | {Group: "", Namespace: "", Resource: "persistentvolumes", Verb: "create"}, 229 | {Group: "", Namespace: "", Resource: "persistentvolumes", Verb: "delete"}, 230 | {Group: "", Namespace: "", Resource: "pods", Verb: "delete"}, 231 | {Group: "", Namespace: "", Resource: "nodes", Verb: "patch"}, 232 | } 233 | 234 | return checkPermissions(context.TODO(), c.kclient, accessCheck) 235 | } 236 | -------------------------------------------------------------------------------- /cmd/pvecsictl/rename.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "context" 21 | "fmt" 22 | "strings" 23 | "time" 24 | 25 | cobra "github.com/spf13/cobra" 26 | 27 | tools "github.com/sergelogvinov/proxmox-csi-plugin/pkg/tools" 28 | 29 | rbacv1 "k8s.io/api/authorization/v1" 30 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 31 | clientkubernetes "k8s.io/client-go/kubernetes" 32 | ) 33 | 34 | type renameCmd struct { 35 | kclient *clientkubernetes.Clientset 36 | namespace string 37 | } 38 | 39 | func buildRenameCmd() *cobra.Command { 40 | c := &renameCmd{} 41 | 42 | cmd := cobra.Command{ 43 | Use: "rename pvc-old pvc-new", 44 | Aliases: []string{"re"}, 45 | Short: "Rename PersistentVolumeClaim", 46 | Args: cobra.ExactArgs(2), 47 | PreRunE: c.renameValidate, 48 | RunE: c.runRename, 49 | SilenceUsage: true, 50 | SilenceErrors: true, 51 | } 52 | 53 | setrenameCmdFlags(&cmd) 54 | 55 | return &cmd 56 | } 57 | 58 | func setrenameCmdFlags(cmd *cobra.Command) { 59 | flags := cmd.Flags() 60 | 61 | flags.StringP("namespace", "n", "", "namespace of the PersistentVolumeClaims") 62 | 63 | flags.BoolP("force", "f", false, "force migration even if the PersistentVolumeClaims is in use") 64 | } 65 | 66 | // nolint: cyclop, gocyclo 67 | func (c *renameCmd) runRename(cmd *cobra.Command, args []string) error { 68 | flags := cmd.Flags() 69 | force, _ := flags.GetBool("force") //nolint: errcheck 70 | 71 | var err error 72 | 73 | ctx := context.Background() 74 | 75 | srcPVC, srcPV, err := tools.PVCResources(ctx, c.kclient, c.namespace, args[0]) 76 | if err != nil { 77 | return fmt.Errorf("failed to get resources: %v", err) 78 | } 79 | 80 | pods, vmName, err := tools.PVCPodUsage(ctx, c.kclient, c.namespace, args[0]) 81 | if err != nil { 82 | return fmt.Errorf("failed to find pods using pvc: %v", err) 83 | } 84 | 85 | cordonedNodes := []string{} 86 | 87 | defer func() { 88 | if len(cordonedNodes) > 0 { 89 | logger.Infof("uncordoning nodes: %s", strings.Join(cordonedNodes, ",")) 90 | 91 | if err = tools.UncondonNodes(ctx, c.kclient, cordonedNodes); err != nil { 92 | logger.Errorf("failed to uncordon nodes: %v", err) 93 | } 94 | } 95 | }() 96 | 97 | if len(pods) > 0 { 98 | if force { 99 | if srcPV.Spec.CSI == nil { 100 | return fmt.Errorf("only CSI PersistentVolumes can be swapped in force mode") 101 | } 102 | 103 | logger.Infof("persistentvolumeclaims is using by pods: %s on node %s, trying to force migration\n", strings.Join(pods, ","), vmName) 104 | 105 | cordonedNodes, err = cordoneNodeWithPVs(ctx, c.kclient, srcPV) 106 | if err != nil { 107 | return fmt.Errorf("failed to cordon nodes: %v", err) 108 | } 109 | 110 | logger.Infof("cordoned nodes: %s", strings.Join(cordonedNodes, ",")) 111 | logger.Infof("terminated pods: %s", strings.Join(pods, ",")) 112 | 113 | for _, pod := range pods { 114 | if err = c.kclient.CoreV1().Pods(c.namespace).Delete(ctx, pod, metav1.DeleteOptions{}); err != nil { 115 | return fmt.Errorf("failed to delete pod: %v", err) 116 | } 117 | } 118 | 119 | for { 120 | p, _, e := tools.PVCPodUsage(ctx, c.kclient, c.namespace, args[0]) 121 | if e != nil { 122 | return fmt.Errorf("failed to find pods using pvc: %v", e) 123 | } 124 | 125 | if len(p) == 0 { 126 | break 127 | } 128 | 129 | logger.Infof("waiting pods: %s", strings.Join(p, " ")) 130 | 131 | time.Sleep(2 * time.Second) 132 | } 133 | 134 | time.Sleep(5 * time.Second) 135 | } else { 136 | return fmt.Errorf("persistentvolumeclaims is using by pods: %s on node %s, cannot move volume", strings.Join(pods, ","), vmName) 137 | } 138 | } 139 | 140 | err = renamePVC(ctx, c.kclient, c.namespace, srcPVC, srcPV, args[1]) 141 | if err != nil { 142 | cordonedNodes = []string{} 143 | 144 | return fmt.Errorf("failed to rename persistentvolumeclaims: %v", err) 145 | } 146 | 147 | logger.Infof("persistentvolumeclaims %s has been renamed", args[0]) 148 | 149 | return nil 150 | } 151 | 152 | // nolint: dupl 153 | func (c *renameCmd) renameValidate(cmd *cobra.Command, _ []string) error { 154 | flags := cmd.Flags() 155 | 156 | namespace, _ := flags.GetString("namespace") //nolint: errcheck 157 | 158 | kclientConfig, namespace, err := tools.BuildConfig(kubeconfig, namespace) 159 | if err != nil { 160 | return fmt.Errorf("failed to create kubernetes config: %v", err) 161 | } 162 | 163 | c.kclient, err = clientkubernetes.NewForConfig(kclientConfig) 164 | if err != nil { 165 | return fmt.Errorf("failed to create kubernetes client: %v", err) 166 | } 167 | 168 | c.namespace = namespace 169 | 170 | accessCheck := []rbacv1.ResourceAttributes{ 171 | {Group: "", Namespace: "", Resource: "persistentvolumeclaims", Verb: "create"}, 172 | {Group: "", Namespace: "", Resource: "persistentvolumeclaims", Verb: "delete"}, 173 | {Group: "", Namespace: "", Resource: "persistentvolumes", Verb: "create"}, 174 | {Group: "", Namespace: "", Resource: "persistentvolumes", Verb: "delete"}, 175 | {Group: "", Namespace: "", Resource: "pods", Verb: "delete"}, 176 | {Group: "", Namespace: "", Resource: "nodes", Verb: "patch"}, 177 | } 178 | 179 | return checkPermissions(context.TODO(), c.kclient, accessCheck) 180 | } 181 | -------------------------------------------------------------------------------- /cmd/pvecsictl/swap.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "context" 21 | "fmt" 22 | "strings" 23 | "time" 24 | 25 | cobra "github.com/spf13/cobra" 26 | 27 | tools "github.com/sergelogvinov/proxmox-csi-plugin/pkg/tools" 28 | 29 | rbacv1 "k8s.io/api/authorization/v1" 30 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 31 | clientkubernetes "k8s.io/client-go/kubernetes" 32 | ) 33 | 34 | type swapCmd struct { 35 | kclient *clientkubernetes.Clientset 36 | namespace string 37 | } 38 | 39 | func buildSwapCmd() *cobra.Command { 40 | c := &swapCmd{} 41 | 42 | cmd := cobra.Command{ 43 | Use: "swap pvc-a pvc-b", 44 | Aliases: []string{"sw"}, 45 | Short: "Swap PersistentVolumes between two PersistentVolumeClaims", 46 | Args: cobra.ExactArgs(2), 47 | PreRunE: c.swapValidate, 48 | RunE: c.runSwap, 49 | SilenceUsage: true, 50 | SilenceErrors: true, 51 | } 52 | 53 | setSwapCmdFlags(&cmd) 54 | 55 | return &cmd 56 | } 57 | 58 | func setSwapCmdFlags(cmd *cobra.Command) { 59 | flags := cmd.Flags() 60 | 61 | flags.StringP("namespace", "n", "", "namespace of the PersistentVolumeClaims") 62 | 63 | flags.BoolP("force", "f", false, "force migration even if the PersistentVolumeClaims is in use") 64 | } 65 | 66 | // nolint: cyclop, gocyclo 67 | func (c *swapCmd) runSwap(cmd *cobra.Command, args []string) error { 68 | flags := cmd.Flags() 69 | force, _ := flags.GetBool("force") //nolint: errcheck 70 | 71 | var err error 72 | 73 | ctx := context.Background() 74 | 75 | srcPVC, srcPV, err := tools.PVCResources(ctx, c.kclient, c.namespace, args[0]) 76 | if err != nil { 77 | return fmt.Errorf("failed to get resources: %v", err) 78 | } 79 | 80 | srcPods, srcVMName, err := tools.PVCPodUsage(ctx, c.kclient, c.namespace, args[0]) 81 | if err != nil { 82 | return fmt.Errorf("failed to find pods using pvc: %v", err) 83 | } 84 | 85 | dstPVC, dstPV, err := tools.PVCResources(ctx, c.kclient, c.namespace, args[1]) 86 | if err != nil { 87 | return fmt.Errorf("failed to get resources: %v", err) 88 | } 89 | 90 | dstPods, dstVMName, err := tools.PVCPodUsage(ctx, c.kclient, c.namespace, args[1]) 91 | if err != nil { 92 | return fmt.Errorf("failed to find pods using pvc: %v", err) 93 | } 94 | 95 | cordonedNodes := []string{} 96 | 97 | defer func() { 98 | if len(cordonedNodes) > 0 { 99 | logger.Infof("uncordoning nodes: %s", strings.Join(cordonedNodes, ",")) 100 | 101 | if err = tools.UncondonNodes(ctx, c.kclient, cordonedNodes); err != nil { 102 | logger.Errorf("failed to uncordon nodes: %v", err) 103 | } 104 | } 105 | }() 106 | 107 | if len(srcPods) > 0 || len(dstPods) > 0 { 108 | if force { 109 | var csiNodes []string 110 | 111 | if srcPV.Spec.CSI == nil || dstPV.Spec.CSI == nil { 112 | return fmt.Errorf("only CSI PersistentVolumes can be swapped in force mode") 113 | } 114 | 115 | if len(srcPods) > 0 { 116 | logger.Infof("persistentvolumeclaims is using by pods: %s on node %s, trying to force swap\n", strings.Join(srcPods, ","), srcVMName) 117 | 118 | csiNodes, err = cordoneNodeWithPVs(ctx, c.kclient, srcPV) 119 | if err != nil { 120 | return fmt.Errorf("failed to cordon nodes: %v", err) 121 | } 122 | 123 | cordonedNodes = append(cordonedNodes, csiNodes...) 124 | } 125 | 126 | if len(dstPods) > 0 { 127 | logger.Infof("persistentvolumeclaims is using by pods: %s on node %s, trying to force swap\n", strings.Join(dstPods, ","), dstVMName) 128 | 129 | csiNodes, err = cordoneNodeWithPVs(ctx, c.kclient, dstPV) 130 | if err != nil { 131 | return fmt.Errorf("failed to cordon nodes: %v", err) 132 | } 133 | 134 | cordonedNodes = append(cordonedNodes, csiNodes...) 135 | } 136 | 137 | logger.Infof("cordoned nodes: %s", strings.Join(cordonedNodes, ",")) 138 | 139 | pods := srcPods 140 | pods = append(pods, dstPods...) 141 | 142 | logger.Infof("terminated pods: %s", strings.Join(pods, ",")) 143 | 144 | for _, pod := range pods { 145 | if err = c.kclient.CoreV1().Pods(c.namespace).Delete(ctx, pod, metav1.DeleteOptions{}); err != nil { 146 | return fmt.Errorf("failed to delete pod: %v", err) 147 | } 148 | } 149 | 150 | waitPods := func(pod string) error { 151 | for { 152 | p, _, e := tools.PVCPodUsage(ctx, c.kclient, c.namespace, pod) 153 | if e != nil { 154 | return fmt.Errorf("failed to find pods using pvc: %v", e) 155 | } 156 | 157 | if len(p) == 0 { 158 | break 159 | } 160 | 161 | logger.Infof("waiting pods: %s", strings.Join(p, " ")) 162 | 163 | time.Sleep(2 * time.Second) 164 | } 165 | 166 | return nil 167 | } 168 | 169 | if err = waitPods(args[0]); err != nil { 170 | return err 171 | } 172 | 173 | if err = waitPods(args[1]); err != nil { 174 | return err 175 | } 176 | 177 | time.Sleep(5 * time.Second) 178 | } else { 179 | if len(srcPods) > 0 { 180 | return fmt.Errorf("persistentvolumeclaims is using by pods: %s on the node %s, cannot swap pvc", strings.Join(srcPods, ","), srcVMName) 181 | } 182 | 183 | if len(dstPods) > 0 { 184 | return fmt.Errorf("persistentvolumeclaims is using by pods: %s on the node %s, cannot swap pvc", strings.Join(dstPods, ","), dstVMName) 185 | } 186 | } 187 | } 188 | 189 | err = swapPVC(ctx, c.kclient, c.namespace, srcPVC, srcPV, dstPVC, dstPV) 190 | if err != nil { 191 | cordonedNodes = []string{} 192 | 193 | return fmt.Errorf("failed to swap persistentvolumeclaims: %v", err) 194 | } 195 | 196 | logger.Infof("persistentvolumeclaims %s,%s has been swapped", args[0], args[1]) 197 | 198 | return nil 199 | } 200 | 201 | // nolint: dupl 202 | func (c *swapCmd) swapValidate(cmd *cobra.Command, _ []string) error { 203 | flags := cmd.Flags() 204 | 205 | namespace, _ := flags.GetString("namespace") //nolint: errcheck 206 | 207 | kclientConfig, namespace, err := tools.BuildConfig(kubeconfig, namespace) 208 | if err != nil { 209 | return fmt.Errorf("failed to create kubernetes config: %v", err) 210 | } 211 | 212 | c.kclient, err = clientkubernetes.NewForConfig(kclientConfig) 213 | if err != nil { 214 | return fmt.Errorf("failed to create kubernetes client: %v", err) 215 | } 216 | 217 | c.namespace = namespace 218 | 219 | accessCheck := []rbacv1.ResourceAttributes{ 220 | {Group: "", Namespace: "", Resource: "persistentvolumeclaims", Verb: "create"}, 221 | {Group: "", Namespace: "", Resource: "persistentvolumeclaims", Verb: "delete"}, 222 | {Group: "", Namespace: "", Resource: "persistentvolumes", Verb: "create"}, 223 | {Group: "", Namespace: "", Resource: "persistentvolumes", Verb: "delete"}, 224 | {Group: "", Namespace: "", Resource: "pods", Verb: "delete"}, 225 | {Group: "", Namespace: "", Resource: "nodes", Verb: "patch"}, 226 | } 227 | 228 | return checkPermissions(context.TODO(), c.kclient, accessCheck) 229 | } 230 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | base: 3 | image: k8s.gcr.io/pause:3.6 4 | ports: 5 | - "8080:8080" 6 | plugin: 7 | build: 8 | context: . 9 | target: develop 10 | network_mode: "service:base" 11 | environment: 12 | # NODE_NAME: worker-11 13 | KUBECONFIG: /etc/kubernetes/kubeconfig 14 | # UNSAFEMOUNT: "true" 15 | command: 16 | - "make" 17 | - "run" 18 | volumes: 19 | - type: volume 20 | source: socket-dir 21 | target: /csi 22 | - type: bind 23 | source: ./hack 24 | target: /etc/kubernetes 25 | - type: bind 26 | source: ./ 27 | target: /src 28 | csi-attacher: 29 | image: registry.k8s.io/sig-storage/csi-attacher:v4.8.0 30 | restart: always 31 | network_mode: "service:base" 32 | command: 33 | - "--v=5" 34 | - "--csi-address=unix:///csi/csi.sock" 35 | - "--leader-election=false" 36 | - "--default-fstype=ext4" 37 | - "--kubeconfig=/etc/kubernetes/kubeconfig" 38 | volumes: 39 | - type: volume 40 | source: socket-dir 41 | target: /csi 42 | - type: bind 43 | source: ./hack 44 | target: /etc/kubernetes 45 | csi-resizer: 46 | image: registry.k8s.io/sig-storage/csi-resizer:v1.13.1 47 | restart: always 48 | network_mode: "service:base" 49 | command: 50 | - "--v=5" 51 | - "--workers=1" 52 | - "--csi-address=unix:///csi/csi.sock" 53 | - "--leader-election=false" 54 | - "--kubeconfig=/etc/kubernetes/kubeconfig" 55 | # - "--feature-gates=VolumeAttributesClass=true" 56 | volumes: 57 | - type: volume 58 | source: socket-dir 59 | target: /csi 60 | - type: bind 61 | source: ./hack 62 | target: /etc/kubernetes 63 | csi-provisioner: 64 | image: registry.k8s.io/sig-storage/csi-provisioner:v5.1.0 65 | restart: always 66 | network_mode: "service:base" 67 | command: 68 | - "--v=5" 69 | - "--csi-address=unix:///csi/csi.sock" 70 | - "--leader-election=false" 71 | - "--default-fstype=ext4" 72 | # - "--feature-gates=VolumeAttributesClass=true" 73 | - "--enable-capacity" 74 | - "--capacity-ownerref-level=-1" 75 | - "--capacity-poll-interval=2m" 76 | # - "--extra-create-metadata=true" 77 | # - "--node-deployment" 78 | - "--kubeconfig=/etc/kubernetes/kubeconfig" 79 | environment: 80 | NAMESPACE: csi-proxmox 81 | POD_NAME: csi-provisioner 82 | # NODE_NAME: worker-11 83 | volumes: 84 | - type: volume 85 | source: socket-dir 86 | target: /csi 87 | - type: bind 88 | source: ./hack 89 | target: /etc/kubernetes 90 | # csi-node-driver-registrar: 91 | # image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.11.1 92 | # network_mode: "service:base" 93 | # command: 94 | # - "--v=5" 95 | # - "--csi-address=unix:///csi/csi.sock" 96 | # - "--kubelet-registration-path=/var/lib/kubelet/plugins/csi.proxmox.sinextra.dev/csi.sock" 97 | # environment: 98 | # KUBE_NODE_NAME: worker-11 99 | # volumes: 100 | # - type: volume 101 | # source: socket-dir 102 | # target: /csi 103 | # - type: bind 104 | # source: ./hack 105 | # target: /etc/kubernetes 106 | 107 | volumes: 108 | socket-dir: 109 | -------------------------------------------------------------------------------- /docs/architecture.md: -------------------------------------------------------------------------------- 1 | # Architecture 2 | 3 | Proxmox cluster with local storage like: lvm, lvm-thin, zfs, xfs, ext4, etc. 4 | 5 | ![ProxmoxClusers!](/docs/proxmox-regions.jpeg) 6 | 7 | - Each Proxmox cluster has predefined in cloud-config the region name (see `clusters[].region` below). 8 | - Each Proxmox Cluster has many Proxmox Nodes. In kubernetes scope it is called as `zone`. The name of `zone` is the name of Proxmox node. 9 | - Pods can easily migrate between Kubernetes nodes on the same physical Proxmox node (`zone`). 10 | The PV will automatically be moved by the CSI Plugin. 11 | - Pods with PVC `cannot` automatically migrate across zones (Proxmox nodes). 12 | You can manually move PVs across zones using [pvecsictl](docs/pvecsictl.md) to migrate Pods across zones. 13 | 14 | 15 | ```mermaid 16 | --- 17 | title: Automatic Pod migration within zone 18 | --- 19 | flowchart LR 20 | subgraph cluster1["Proxmox Cluster (Region 1)"] 21 | subgraph node11["Proxmox Node (zone 1)"] 22 | direction BT 23 | subgraph vm1["VM (worker 1)"] 24 | pod11(["Pod (pv-1)"]) 25 | end 26 | subgraph vm2["VM (worker 2)"] 27 | pod12(["Pod (pv-1)"]) 28 | end 29 | pv11[("Disk (pv-1)")] 30 | end 31 | subgraph node12["Proxmox Node (zone 2)"] 32 | direction BT 33 | subgraph vm3["VM (worker 3)"] 34 | pod22(["Pod (pv-2)"]) 35 | end 36 | pv22[("Disk (pv-2)")] 37 | end 38 | end 39 | pv11 .-> vm1 40 | pv11 -->|automatic| vm2 41 | pod11 -->|migrate| pod12 42 | 43 | pv22 --> vm3 44 | ``` 45 | ```mermaid 46 | --- 47 | title: Manual migration using pvecsictl across zones 48 | --- 49 | flowchart 50 | subgraph cluster1["Proxmox Cluster (Region 1)"] 51 | direction BT 52 | subgraph node11["Proxmox Node (zone 1)"] 53 | subgraph vm1["VM (worker 1)"] 54 | pod11["Pod (pv-1)"] 55 | end 56 | subgraph vm2["VM (worker 2)"] 57 | pod21["Pod (pv-2)"] 58 | end 59 | pv11[("Disk (pv-1)")] 60 | pv21[("Disk (pv-2)")] 61 | end 62 | subgraph node12["Proxmox Node (zone 2)"] 63 | direction TB 64 | subgraph vm3["VM (worker 3)"] 65 | pod22["Pod (pv-2)"] 66 | end 67 | pv22[("Disk (pv-2)")] 68 | end 69 | end 70 | pv11 --> vm1 71 | pv21 .-> vm2 72 | 73 | pv22 --> vm3 74 | pod21 -->|migrate| pod22 75 | pv21 -->|pvecsictl| pv22 76 | ``` 77 | -------------------------------------------------------------------------------- /docs/cosign.md: -------------------------------------------------------------------------------- 1 | # Verify images 2 | 3 | We'll be employing [Cosing's](https://github.com/sigstore/cosign) keyless verifications to ensure that images were built in Github Actions. 4 | 5 | ## Verify Helm chart 6 | 7 | We will verify the keyless signature using the Cosign protocol. 8 | 9 | ```shell 10 | cosign verify ghcr.io/sergelogvinov/charts/proxmox-csi-plugin:0.1.4 --certificate-identity https://github.com/sergelogvinov/proxmox-csi-plugin/.github/workflows/release-charts.yaml@refs/heads/main --certificate-oidc-issuer https://token.actions.githubusercontent.com 11 | ``` 12 | 13 | ## Verify containers 14 | 15 | We will verify the keyless signature using the Cosign protocol. 16 | 17 | ```shell 18 | # Edge version 19 | cosign verify ghcr.io/sergelogvinov/proxmox-csi-controller:edge --certificate-identity https://github.com/sergelogvinov/proxmox-csi-plugin/.github/workflows/build-edge.yaml@refs/heads/main --certificate-oidc-issuer https://token.actions.githubusercontent.com 20 | 21 | cosign verify ghcr.io/sergelogvinov/proxmox-csi-node:edge --certificate-identity https://github.com/sergelogvinov/proxmox-csi-plugin/.github/workflows/build-edge.yaml@refs/heads/main --certificate-oidc-issuer https://token.actions.githubusercontent.com 22 | 23 | # Releases 24 | cosign verify ghcr.io/sergelogvinov/proxmox-csi-controller:v0.2.0 --certificate-identity https://github.com/sergelogvinov/proxmox-csi-plugin/.github/workflows/release.yaml@refs/tags/v0.2.0 --certificate-oidc-issuer https://token.actions.githubusercontent.com 25 | 26 | cosign verify ghcr.io/sergelogvinov/proxmox-csi-node:v0.2.0 --certificate-identity https://github.com/sergelogvinov/proxmox-csi-plugin/.github/workflows/release.yaml@refs/tags/v0.2.0 --certificate-oidc-issuer https://token.actions.githubusercontent.com 27 | ``` 28 | -------------------------------------------------------------------------------- /docs/deploy/debug.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: ubuntu 5 | namespace: kube-system 6 | spec: 7 | hostname: ubuntu 8 | subdomain: default 9 | hostPID: true 10 | hostNetwork: true 11 | containers: 12 | - image: ubuntu 13 | command: 14 | - sleep 15 | - "14d" 16 | name: ubuntu 17 | securityContext: 18 | privileged: true 19 | capabilities: 20 | add: 21 | - SYS_RAWIO 22 | volumeMounts: 23 | - name: dev 24 | mountPath: /dev 25 | - name: sys 26 | mountPath: /sys 27 | - name: root 28 | mountPath: /mnt/root 29 | readOnly: true 30 | - mountPath: /lib/modules 31 | name: lib-modules 32 | readOnly: true 33 | - name: tmp 34 | mountPath: /tmp 35 | tolerations: 36 | - operator: Exists 37 | volumes: 38 | - name: dev 39 | hostPath: 40 | path: /dev 41 | - name: sys 42 | hostPath: 43 | path: /sys 44 | - name: root 45 | hostPath: 46 | path: / 47 | - hostPath: 48 | path: /lib/modules 49 | name: lib-modules 50 | - name: tmp 51 | emptyDir: 52 | medium: Memory 53 | nodeSelector: 54 | kubernetes.io/hostname: kube-11 55 | -------------------------------------------------------------------------------- /docs/deploy/pvc.yaml: -------------------------------------------------------------------------------- 1 | # Create a PersistentVolumeClaim resource of already existing Disk Volume 2 | # 3 | # Cluster name: region-1 4 | # Proxmox Node name: hvm-1 5 | # Data Storage name: data 6 | # Existing Volume: vm-9999-pvc-369fec02-bb4a-4d2f-853c-507dc4ff131e 7 | # Storage Class: proxmox 8 | # 9 | --- 10 | apiVersion: v1 11 | kind: PersistentVolume 12 | metadata: 13 | name: pvc-369fec02-bb4a-4d2f-853c-507dc4ff131e 14 | spec: 15 | accessModes: 16 | - ReadWriteOnce 17 | capacity: 18 | storage: 250Gi 19 | csi: 20 | driver: csi.proxmox.sinextra.dev 21 | fsType: xfs 22 | volumeHandle: region-1/hvm-1/data/vm-9999-pvc-369fec02-bb4a-4d2f-853c-507dc4ff131e 23 | nodeAffinity: 24 | required: 25 | nodeSelectorTerms: 26 | - matchExpressions: 27 | - key: topology.kubernetes.io/region 28 | operator: In 29 | values: 30 | - region-1 31 | - key: topology.kubernetes.io/zone 32 | operator: In 33 | values: 34 | - hvm-1 35 | storageClassName: proxmox 36 | --- 37 | apiVersion: v1 38 | kind: PersistentVolumeClaim 39 | metadata: 40 | name: data-clickhouse-0 41 | namespace: clickhouse 42 | spec: 43 | accessModes: 44 | - ReadWriteOnce 45 | resources: 46 | requests: 47 | storage: 250Gi 48 | storageClassName: proxmox 49 | volumeName: pvc-369fec02-bb4a-4d2f-853c-507dc4ff131e 50 | -------------------------------------------------------------------------------- /docs/deploy/test-pod-ephemeral.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: test 5 | namespace: default 6 | spec: 7 | tolerations: 8 | - effect: NoSchedule 9 | key: node-role.kubernetes.io/control-plane 10 | nodeSelector: 11 | kubernetes.io/hostname: kube-11 12 | containers: 13 | - name: alpine 14 | image: alpine 15 | command: ["sleep", "6000"] 16 | volumeMounts: 17 | - name: pvc 18 | mountPath: /mnt 19 | securityContext: 20 | allowPrivilegeEscalation: false 21 | capabilities: 22 | drop: 23 | - ALL 24 | seccompProfile: 25 | type: RuntimeDefault 26 | runAsNonRoot: true 27 | terminationGracePeriodSeconds: 1 28 | securityContext: 29 | fsGroup: 65534 30 | runAsGroup: 65534 31 | runAsUser: 65534 32 | volumes: 33 | - name: pvc 34 | ephemeral: 35 | volumeClaimTemplate: 36 | metadata: 37 | labels: 38 | type: pvc-volume 39 | spec: 40 | accessModes: ["ReadWriteOnce"] 41 | storageClassName: proxmox 42 | resources: 43 | requests: 44 | storage: 64Mi 45 | -------------------------------------------------------------------------------- /docs/deploy/test-pod-secret-ephemeral.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: test 5 | namespace: default 6 | spec: 7 | tolerations: 8 | - effect: NoSchedule 9 | key: node-role.kubernetes.io/control-plane 10 | nodeSelector: 11 | # topology.kubernetes.io/zone: hvm-1 12 | kubernetes.io/hostname: kube-11 13 | containers: 14 | - name: alpine 15 | image: alpine 16 | command: ["sleep","6000"] 17 | volumeMounts: 18 | - name: pvc 19 | mountPath: /mnt 20 | securityContext: 21 | allowPrivilegeEscalation: false 22 | capabilities: 23 | drop: 24 | - ALL 25 | seccompProfile: 26 | type: RuntimeDefault 27 | runAsNonRoot: true 28 | terminationGracePeriodSeconds: 1 29 | securityContext: 30 | fsGroup: 65534 31 | runAsGroup: 65534 32 | runAsUser: 65534 33 | volumes: 34 | - name: pvc 35 | ephemeral: 36 | volumeClaimTemplate: 37 | metadata: 38 | labels: 39 | type: pvc-volume 40 | spec: 41 | accessModes: [ "ReadWriteOnce" ] 42 | storageClassName: proxmox-secret 43 | resources: 44 | requests: 45 | storage: 5Gi 46 | -------------------------------------------------------------------------------- /docs/deploy/test-statefulset-raw.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: StatefulSet 3 | metadata: 4 | name: test 5 | namespace: default 6 | labels: 7 | app: alpine 8 | spec: 9 | podManagementPolicy: Parallel # default is OrderedReady 10 | serviceName: test 11 | replicas: 1 12 | template: 13 | metadata: 14 | labels: 15 | app: alpine 16 | spec: 17 | terminationGracePeriodSeconds: 3 18 | tolerations: 19 | - effect: NoSchedule 20 | key: node-role.kubernetes.io/control-plane 21 | nodeSelector: 22 | # kubernetes.io/hostname: kube-21 23 | # topology.kubernetes.io/zone: hvm-1 24 | containers: 25 | - name: alpine 26 | image: alpine 27 | command: ["sleep","1d"] 28 | securityContext: 29 | privileged: true 30 | capabilities: 31 | drop: 32 | - ALL 33 | add: 34 | - SYS_ADMIN 35 | - CHOWN 36 | - DAC_OVERRIDE 37 | seccompProfile: 38 | type: RuntimeDefault 39 | volumeDevices: 40 | - name: storage 41 | devicePath: /dev/xvda 42 | updateStrategy: 43 | type: RollingUpdate 44 | selector: 45 | matchLabels: 46 | app: alpine 47 | volumeClaimTemplates: 48 | - metadata: 49 | name: storage 50 | spec: 51 | accessModes: ["ReadWriteOnce"] 52 | volumeMode: Block 53 | resources: 54 | requests: 55 | storage: 1Gi 56 | storageClassName: proxmox-zfs 57 | -------------------------------------------------------------------------------- /docs/deploy/test-statefulset.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: StatefulSet 3 | metadata: 4 | name: test 5 | namespace: default 6 | labels: 7 | app: alpine 8 | spec: 9 | podManagementPolicy: Parallel # default is OrderedReady 10 | serviceName: test 11 | replicas: 1 12 | template: 13 | metadata: 14 | labels: 15 | app: alpine 16 | spec: 17 | terminationGracePeriodSeconds: 3 18 | tolerations: 19 | - effect: NoSchedule 20 | key: node-role.kubernetes.io/control-plane 21 | nodeSelector: 22 | # kubernetes.io/hostname: kube-01a 23 | # topology.kubernetes.io/zone: hvm-1 24 | containers: 25 | - name: alpine 26 | image: alpine 27 | command: ["sleep", "1d"] 28 | securityContext: 29 | seccompProfile: 30 | type: RuntimeDefault 31 | capabilities: 32 | drop: ["ALL"] 33 | volumeMounts: 34 | - name: storage 35 | mountPath: /mnt 36 | updateStrategy: 37 | type: RollingUpdate 38 | selector: 39 | matchLabels: 40 | app: alpine 41 | volumeClaimTemplates: 42 | - metadata: 43 | name: storage 44 | spec: 45 | accessModes: ["ReadWriteOnce"] 46 | resources: 47 | requests: 48 | storage: 50Gi 49 | storageClassName: proxmox-lvm 50 | # volumeAttributesClassName: test 51 | -------------------------------------------------------------------------------- /docs/eraser.md: -------------------------------------------------------------------------------- 1 | direction down 2 | styleMode: plain 3 | 4 | CLUSTER1 [label: "Proxmox Cluster (Region-1)"] { 5 | Node 1 [label: "Proxmox Node 1 (zone 1)"] { 6 | VM1 [label: "VM-1"] { 7 | pod2 [icon: k8s-pod, label: "pod", colorMode: outline, color: black] { 8 | pvc1 [icon: k8s-pvc, label: "pvc", colorMode: outline, color: black]} 9 | } 10 | VM2 [label: "VM-2"] { 11 | pod3 [icon: k8s-pod, label: "pod", colorMode: outline, color: black] { 12 | pvc3 [icon: k8s-pvc, label: "pvc", colorMode: outline, color: black]} 13 | } 14 | DiskNode1 [icon:azure-disks, label:"Local Disk"] 15 | } 16 | Node 2 [label: "Proxmox Node 2 (zone 2)"] { 17 | VM3 [label: "VM-3"] { 18 | pod4 [icon: k8s-pod, label: "pod", colorMode: bold] { 19 | pvc4 [icon: k8s-pvc, label: "pvc", colorMode: bold] 20 | } 21 | } 22 | DiskNode2 [icon:azure-disks, label:"Local Disk"] 23 | } 24 | DiskSH1 [icon:azure-disk-pool, label:"Shared Disk"] 25 | } 26 | 27 | pvc1 -- DiskNode1 28 | pvc3 -- DiskNode1 29 | pvc4 -- DiskNode2 30 | pvc1 -- DiskSH1 31 | pvc3 -- DiskSH1 32 | pvc4 - DiskSH1 33 | -------------------------------------------------------------------------------- /docs/faq.md: -------------------------------------------------------------------------------- 1 | # Fast answers to common questions 2 | 3 | ## Can PV/PVC migrate between Proxmox nodes? 4 | 5 | The __local storages__ can't be migrated between Proxmox nodes automatically. 6 | But you can do it manually by following tool [pvecsictl](../docs/pvecsictl.md) 7 | 8 | The __shared storages__ like nfs, ceph can be migrated between Proxmox nodes automatically. 9 | 10 | ## Create a PV/PVC with already existing disk 11 | 12 | If you have a disk already created in Proxmox, you can use it with the CSI plugin. 13 | First, you need to create a PV/PVC with special disk name and the storage class name. 14 | The size of the PersistentVolume must be the same as the disk size. 15 | 16 | ```yaml 17 | apiVersion: v1 18 | kind: PersistentVolume 19 | metadata: 20 | name: pvc-test 21 | spec: 22 | accessModes: 23 | - ReadWriteOnce 24 | capacity: 25 | storage: 10Gi 26 | csi: 27 | driver: csi.proxmox.sinextra.dev 28 | fsType: xfs 29 | volumeAttributes: 30 | storage: zfs 31 | volumeHandle: dev-1/pve-m-4/zfs/vm-100-disk-1 32 | storageClassName: proxmox-zfs 33 | --- 34 | apiVersion: v1 35 | kind: PersistentVolumeClaim 36 | metadata: 37 | name: storage-test-0 38 | spec: 39 | accessModes: 40 | - ReadWriteOnce 41 | resources: 42 | requests: 43 | storage: 10Gi 44 | storageClassName: proxmox-zfs 45 | volumeName: pvc-test 46 | ``` 47 | 48 | The disk name is `vm-100-disk-1`, and the storage class name is `proxmox-zfs`. 49 | 50 | ## How to change encrypted disk secret key? 51 | 52 | The secret key cannot be change through kubernetes API, but you can use the following instructions. 53 | Before starting, read the good explanation of the [cryptsetup](https://wiki.archlinux.org/title/Dm-crypt/Device_encryption) tool. 54 | 55 | First, you need to run the pod with secured PVC, then you need to define the csi-plugin pod running on the same node as the pod with the PVC. 56 | 57 | Lets say the csi-plugin pod name is `proxmox-csi-plugin-node-hjchn` and `/dev/sdb` is the disk you want to change the secret key for. 58 | 59 | ```shell 60 | # Check the disk 61 | kubectl -n csi-proxmox exec -ti proxmox-csi-plugin-node-hjchn -- /sbin/cryptsetup luksDump /dev/sdb 62 | # Check the passphrase 63 | kubectl -n csi-proxmox exec -ti proxmox-csi-plugin-node-hjchn -- /sbin/cryptsetup luksOpen --test-passphrase -v /dev/sdb 64 | ``` 65 | 66 | Add the new passphrase to the disk 67 | 68 | ```shell 69 | # Add the new passphrase 70 | kubectl -n csi-proxmox exec -ti proxmox-csi-plugin-node-hjchn -- /sbin/cryptsetup luksAddKey /dev/sdb 71 | # Check the new passphrase 72 | kubectl -n csi-proxmox exec -ti proxmox-csi-plugin-node-hjchn -- /sbin/cryptsetup luksOpen --test-passphrase -v /dev/sdb 73 | # Check the disk 74 | kubectl -n csi-proxmox exec -ti proxmox-csi-plugin-node-hjchn -- /sbin/cryptsetup luksDump /dev/sdb 75 | ``` 76 | 77 | After you have added the new passphrase, you can remove the old one. 78 | And change the passphrase in kubernetes secret resource. 79 | 80 | ```shell 81 | # Remove the old passphrase 82 | kubectl -n csi-proxmox exec -ti proxmox-csi-plugin-node-hjchn -- /sbin/cryptsetup luksRemoveKey /dev/sdb 83 | ``` 84 | -------------------------------------------------------------------------------- /docs/install.md: -------------------------------------------------------------------------------- 1 | # Install plugin 2 | 3 | This plugin allows Kubernetes to use `Proxmox VE` storage as a persistent storage solution for stateful applications. 4 | Supported storage types: 5 | - Directory 6 | - LVM 7 | - LVM-thin 8 | - ZFS 9 | - NFS 10 | - Ceph 11 | 12 | ## Proxmox configuration 13 | 14 | Proxmox CSI Plugin requires the correct privileges in order to allocate and attach disks. 15 | 16 | Create `CSI` role in Proxmox: 17 | 18 | ```shell 19 | pveum role add CSI -privs "VM.Audit VM.Config.Disk Datastore.Allocate Datastore.AllocateSpace Datastore.Audit" 20 | ``` 21 | 22 | Next create a user `kubernetes-csi@pve` for the CSI plugin and grant it the above role 23 | 24 | ```shell 25 | pveum user add kubernetes-csi@pve 26 | pveum aclmod / -user kubernetes-csi@pve -role CSI 27 | pveum user token add kubernetes-csi@pve csi -privsep 0 28 | ``` 29 | 30 | All VMs in the cluster must have the `SCSI Controller` set to `VirtIO SCSI single` or `VirtIO SCSI` type to be able to attach disks. 31 | 32 | ## Install CSI Driver 33 | 34 | Create a namespace `csi-proxmox` for the plugin and grant it the `privileged` permissions 35 | 36 | ```shell 37 | kubectl create ns csi-proxmox 38 | kubectl label ns csi-proxmox pod-security.kubernetes.io/enforce=privileged 39 | ``` 40 | 41 | ### Install the plugin by using kubectl 42 | 43 | Create a Proxmox cloud config to connect to your cluster with the Proxmox user you just created 44 | 45 | ```yaml 46 | # config.yaml 47 | clusters: 48 | # List of Proxmox clusters 49 | 50 | - url: https://cluster-api-1.exmple.com:8006/api2/json 51 | # Skip the certificate verification, if needed 52 | insecure: false 53 | # Proxmox api token 54 | token_id: "kubernetes-csi@pve!csi" 55 | token_secret: "secret" 56 | # Region name, which is cluster name 57 | region: Region-1 58 | 59 | # Add more clusters if needed 60 | - url: https://cluster-api-2.exmple.com:8006/api2/json 61 | insecure: false 62 | token_id: "kubernetes-csi@pve!csi" 63 | token_secret: "secret" 64 | region: Region-2 65 | ``` 66 | 67 | Upload the configuration to the Kubernetes as a secret 68 | 69 | ```shell 70 | kubectl -n csi-proxmox create secret generic proxmox-csi-plugin --from-file=config.yaml 71 | ``` 72 | 73 | Install latest release version 74 | 75 | ```shell 76 | kubectl apply -f https://raw.githubusercontent.com/sergelogvinov/proxmox-csi-plugin/main/docs/deploy/proxmox-csi-plugin-release.yml 77 | ``` 78 | 79 | Or install latest stable version (edge) 80 | 81 | ```shell 82 | kubectl apply -f https://raw.githubusercontent.com/sergelogvinov/proxmox-csi-plugin/main/docs/deploy/proxmox-csi-plugin.yml 83 | ``` 84 | 85 | ### Install the plugin by using Helm 86 | 87 | Create the helm values file, for more information see [values.yaml](../charts/proxmox-csi-plugin/values.yaml) 88 | 89 | ```yaml 90 | # proxmox-csi.yaml 91 | config: 92 | clusters: 93 | - url: https://cluster-api-1.exmple.com:8006/api2/json 94 | insecure: false 95 | token_id: "kubernetes-csi@pve!csi" 96 | token_secret: "secret" 97 | region: Region-1 98 | # Add more clusters if needed 99 | - url: https://cluster-api-2.exmple.com:8006/api2/json 100 | insecure: false 101 | token_id: "kubernetes-csi@pve!csi" 102 | token_secret: "secret" 103 | region: Region-2 104 | 105 | # Define the storage classes 106 | storageClass: 107 | - name: proxmox-data-xfs 108 | storage: data 109 | reclaimPolicy: Delete 110 | fstype: xfs 111 | # Define the storage class as default 112 | annotations: 113 | storageclass.kubernetes.io/is-default-class: "true" 114 | ``` 115 | 116 | Install the plugin. You need to prepare the `csi-proxmox` namespace first, see above 117 | 118 | ```shell 119 | helm upgrade -i -n csi-proxmox -f proxmox-csi.yaml proxmox-csi-plugin oci://ghcr.io/sergelogvinov/charts/proxmox-csi-plugin 120 | ``` 121 | 122 | #### Option for k0s 123 | 124 | If you're running [k0s](https://k0sproject.io/) you need to add extra value to the helm chart 125 | 126 | ```yaml 127 | kubeletDir: /var/lib/k0s/kubelet 128 | ``` 129 | 130 | #### Option for microk8s 131 | 132 | If you're running [microk8s](https://microk8s.io/) you need to add extra value to the helm chart 133 | 134 | ```yaml 135 | kubeletDir: /var/snap/microk8s/common/var/lib/kubelet 136 | ``` 137 | 138 | ### Install the plugin by using Talos machine config 139 | 140 | If you're running [Talos](https://www.talos.dev/) you can install Proxmox CSI plugin using the machine config 141 | 142 | ```yaml 143 | cluster: 144 | externalCloudProvider: 145 | enabled: true 146 | manifests: 147 | - https://raw.githubusercontent.com/sergelogvinov/proxmox-csi-plugin/main/docs/deploy/proxmox-csi-plugin.yml 148 | ``` 149 | 150 | Or all together with the Proxmox Cloud Controller Manager 151 | 152 | * Proxmox CCM will label the nodes 153 | * Proxmox CSI will use the labeled nodes to define the regions and zones 154 | 155 | ```yaml 156 | cluster: 157 | inlineManifests: 158 | - name: proxmox-cloud-controller-manager 159 | contents: |- 160 | apiVersion: v1 161 | kind: Secret 162 | type: Opaque 163 | metadata: 164 | name: proxmox-cloud-controller-manager 165 | namespace: kube-system 166 | stringData: 167 | config.yaml: | 168 | clusters: 169 | - url: https://cluster-api-1.exmple.com:8006/api2/json 170 | insecure: false 171 | token_id: "kubernetes-csi@pve!ccm" 172 | token_secret: "secret" 173 | region: Region-1 174 | - name: proxmox-csi-plugin 175 | contents: |- 176 | apiVersion: v1 177 | kind: Secret 178 | type: Opaque 179 | metadata: 180 | name: proxmox-csi-plugin 181 | namespace: csi-proxmox 182 | stringData: 183 | config.yaml: | 184 | clusters: 185 | - url: https://cluster-api-1.exmple.com:8006/api2/json 186 | insecure: false 187 | token_id: "kubernetes-csi@pve!csi" 188 | token_secret: "secret" 189 | region: Region-1 190 | externalCloudProvider: 191 | enabled: true 192 | manifests: 193 | - https://raw.githubusercontent.com/sergelogvinov/proxmox-cloud-controller-manager/main/docs/deploy/cloud-controller-manager.yml 194 | - https://raw.githubusercontent.com/sergelogvinov/proxmox-csi-plugin/main/docs/deploy/proxmox-csi-plugin.yml 195 | ``` 196 | -------------------------------------------------------------------------------- /docs/metrics.md: -------------------------------------------------------------------------------- 1 | # Metrics documentation 2 | 3 | This document is a reflection of the current state of the exposed metrics of the Proxmox CSI controller. 4 | 5 | ## Gather metrics 6 | 7 | Enabling the metrics is done by setting the `--metrics-address` flag to the desired address and port. 8 | 9 | ```yaml 10 | proxmox-csi-controller 11 | --metrics-address=8080 12 | ``` 13 | 14 | ### Helm chart values 15 | 16 | The following values can be set in the Helm chart to expose the metrics of the Talos CCM. 17 | 18 | ```yaml 19 | controller: 20 | podAnnotations: 21 | prometheus.io/scrape: "true" 22 | prometheus.io/port: "8080" 23 | ``` 24 | 25 | ## Metrics exposed by the CSI controller 26 | 27 | ### Proxmox API calls 28 | 29 | |Metric name|Metric type|Labels/tags| 30 | |-----------|-----------|-----------| 31 | |proxmox_api_request_duration_seconds|Histogram|`request`=| 32 | |proxmox_api_request_errors_total|Counter|`request`=| 33 | 34 | Example output: 35 | 36 | ```txt 37 | proxmox_api_request_duration_seconds_bucket{request="storageStatus",le="0.1"} 13 38 | proxmox_api_request_duration_seconds_bucket{request="storageStatus",le="0.25"} 172 39 | proxmox_api_request_duration_seconds_bucket{request="storageStatus",le="0.5"} 199 40 | proxmox_api_request_duration_seconds_bucket{request="storageStatus",le="1"} 210 41 | proxmox_api_request_duration_seconds_bucket{request="storageStatus",le="2.5"} 210 42 | proxmox_api_request_duration_seconds_bucket{request="storageStatus",le="5"} 210 43 | proxmox_api_request_duration_seconds_bucket{request="storageStatus",le="10"} 210 44 | proxmox_api_request_duration_seconds_bucket{request="storageStatus",le="30"} 210 45 | proxmox_api_request_duration_seconds_bucket{request="storageStatus",le="+Inf"} 210 46 | proxmox_api_request_duration_seconds_sum{request="storageStatus"} 39.698945394000006 47 | proxmox_api_request_duration_seconds_count{request="storageStatus"} 210 48 | ``` 49 | -------------------------------------------------------------------------------- /docs/options.md: -------------------------------------------------------------------------------- 1 | # Node 2 | 3 | Individual Kubernetes Nodes can be configured using a set of labels specific to the driver. 4 | 5 | ```yaml 6 | apiVersion: v1 7 | kind: Node 8 | metadata: 9 | ... 10 | labels: 11 | # Maximum number of volumes that can be attached to this node, default is 24 12 | # Note: Currently, there is a maximum limit of 30 virtio iscsi volumes *total*, including root disks, that can be attached to a single VM in QEMU/Proxmox. 13 | csi.proxmox.sinextra.dev/max-volume-attachments = "24" 14 | ... 15 | ``` 16 | 17 | # Storage Class 18 | 19 | A Kubernetes StorageClass is an object that defines the storage "classes" or tiers available for dynamic provisioning of storage volumes in a Kubernetes cluster. It abstracts the underlying storage infrastructure, making it easier for developers and administrators to manage persistent storage for applications running in Kubernetes. 20 | 21 | Deploy examples you can find [here](deploy/). 22 | 23 | ```yaml 24 | apiVersion: storage.k8s.io/v1 25 | kind: StorageClass 26 | metadata: 27 | name: proxmox-storage-class-name 28 | annotations: 29 | # If you need to set it as default 30 | storageclass.kubernetes.io/is-default-class: "true" 31 | parameters: 32 | # Pre defined options 33 | ## File system format (default: ext4) 34 | csi.storage.k8s.io/fstype: ext4|xfs 35 | 36 | ## Optional: If you want to encrypt the disk 37 | csi.storage.k8s.io/node-stage-secret-name: "proxmox-csi-secret" 38 | csi.storage.k8s.io/node-stage-secret-namespace: "kube-system" 39 | ## Have to be the same as node-stage-secret-* if you want to expand the volume 40 | csi.storage.k8s.io/node-expand-secret-name: "proxmox-csi-secret" 41 | csi.storage.k8s.io/node-expand-secret-namespace: "kube-system" 42 | 43 | ## Optional: File system format options 44 | blockSize: "4096" 45 | inodeSize: "256" 46 | 47 | # Proxmox csi options 48 | ## Proxmox storage ID 49 | storage: data 50 | 51 | ## Optional: Proxmox csi options 52 | cache: directsync|none|writeback|writethrough 53 | ssd: "true|false" 54 | 55 | ## Optional: Proxmox disk speed limit 56 | diskIOPS: "4000" 57 | diskMBps: "1000" 58 | 59 | ## Optional: Zone replication 60 | replicate: "true" 61 | replicateSchedule: "*/15" 62 | replicateZones: "pve-1,pve-3" 63 | 64 | # Optional: This field allows you to specify additional mount options to be applied when the volume is mounted on the node 65 | mountOptions: 66 | # Common for ssd 67 | - noatime 68 | 69 | # Optional: Allowed topologies restricts what nodes this StorageClass can be used on 70 | allowedTopologies: 71 | - matchLabelExpressions: 72 | # Region have to be exist, if you want to use allowedTopologies 73 | - key: topology.kubernetes.io/region 74 | values: 75 | - Region-1 76 | # Better to set zone, otherwise it will be used random node in the region 77 | - key: topology.kubernetes.io/zone 78 | values: 79 | - pve-1 80 | - pve-3 81 | 82 | provisioner: csi.proxmox.sinextra.dev 83 | allowVolumeExpansion: true 84 | reclaimPolicy: Delete|Retain 85 | volumeBindingMode: WaitForFirstConsumer|Immediate 86 | ``` 87 | 88 | You can use `VolumeAttributesClass` to redefine the mutable parameters of the `StorageClass`. 89 | 90 | ```yaml 91 | apiVersion: storage.k8s.io/v1beta1 92 | kind: VolumeAttributesClass 93 | metadata: 94 | name: proxmox-attributes 95 | driverName: csi.proxmox.sinextra.dev 96 | parameters: 97 | ## Optional: Proxmox disk speed limit 98 | diskIOPS: "4000" 99 | diskMBps: "1000" 100 | 101 | ## Optional: Backup disk with VM 102 | backup: "true" 103 | 104 | ## Optional: Zone replication 105 | replicateSchedule: "*/30" 106 | replicateZones: "rnd-1,rnd-2" 107 | ``` 108 | 109 | ## Parameters: 110 | 111 | * `node-stage-secret-name`/`node-expand-secret-name`, `node-stage-secret-namespace`/`node-expand-secret-namespace` - Refer to the name and namespace of the Secret object in the Kubernetes API. The secrets key name is `encryption-passphrase`. [Official documentation](https://kubernetes-csi.github.io/docs/secrets-and-credentials-storage-class.html) 112 | 113 | ```yaml 114 | apiVersion: v1 115 | data: 116 | encryption-passphrase: base64-encode 117 | kind: Secret 118 | metadata: 119 | name: proxmox-csi-secret 120 | namespace: kube-system 121 | ``` 122 | 123 | * `blockSize` - specify the size of blocks in bytes. 124 | * `inodeSize` - Specify the size of each inode in bytes. 125 | 126 | * `storage` - proxmox storage ID 127 | * `cache` - qemu cache param: `directsync`, `none`, `writeback`, `writethrough` [Official documentation](https://pve.proxmox.com/wiki/Performance_Tweaks) 128 | * `ssd` - set true if SSD/NVME disk 129 | 130 | * `diskIOPS` - maximum r/w I/O in operations per second 131 | * `diskMBps` - maximum r/w throughput in megabytes per second 132 | 133 | * `backup` - set true if you want to backup the disk with VM. Dangerous option! Do not use it unless you fully understand how to use it in the recovery process. 134 | 135 | * `replicate` - set true if you want to replicate the disk to another zone 136 | * `replicateSchedule` - replication schedule [in systemd calendar format](https://pve.proxmox.com/pve-docs/pve-admin-guide.html#pvesr_schedule_time_format) (default: `*/15`) 137 | * `replicateZones` - zones where the disk will be replicated, separated by commas 138 | 139 | ## AllowVolumeExpansion 140 | 141 | Allow you to resize (expand) the PVC in future. 142 | 143 | ## ReclaimPolicy 144 | 145 | It defines what happens to the storage volume when the associated PersistentVolumeClaim (PVC) is deleted. There are three reclaim policies: 146 | 147 | * `Retain`: The storage volume is not deleted when the PVC is released, and it must be manually reclaimed by an administrator. 148 | * `Delete`: The storage volume is deleted when the PVC is released. 149 | 150 | ## VolumeBindingMode 151 | 152 | It specifies how volumes should be bound to PVs (Persistent Volumes). There are two modes: 153 | 154 | * `Immediate`: PVs are bound as soon as a PVC is created, even if a suitable storage volume isn't immediately available. This is suitable for scenarios where waiting for storage is not an option. 155 | * `WaitForFirstConsumer`: PVs are bound only when a pod using the PVC is scheduled. This is useful when you want to ensure that storage is provisioned only when it's actually needed. 156 | -------------------------------------------------------------------------------- /docs/proxmox-lvm-secret.yaml: -------------------------------------------------------------------------------- 1 | # dd if=/dev/urandom bs=1 count=16 2>/dev/null | hexdump -e '"%00x"' > csi-secret.secret 2 | # kubectl -n kube-system create secret generic proxmox-csi-secret --from-file=encryption-passphrase=csi-secret.secret --dry-run=client -oyaml 3 | --- 4 | apiVersion: v1 5 | data: 6 | encryption-passphrase: MWYwMzkyODAzM2RkYTJlNGZkMzQ3ZTQ0MjY2Y2ZiYw== 7 | kind: Secret 8 | metadata: 9 | creationTimestamp: null 10 | name: proxmox-csi-secret 11 | namespace: kube-system 12 | --- 13 | apiVersion: storage.k8s.io/v1 14 | kind: StorageClass 15 | metadata: 16 | name: proxmox-secret 17 | parameters: 18 | csi.storage.k8s.io/fstype: xfs 19 | csi.storage.k8s.io/node-stage-secret-name: "proxmox-csi-secret" 20 | csi.storage.k8s.io/node-stage-secret-namespace: "kube-system" 21 | csi.storage.k8s.io/node-expand-secret-name: "proxmox-csi-secret" 22 | csi.storage.k8s.io/node-expand-secret-namespace: "kube-system" 23 | storage: lvm 24 | mountOptions: 25 | - discard 26 | provisioner: csi.proxmox.sinextra.dev 27 | reclaimPolicy: Delete 28 | volumeBindingMode: WaitForFirstConsumer 29 | allowVolumeExpansion: true 30 | -------------------------------------------------------------------------------- /docs/proxmox-lvm.yaml: -------------------------------------------------------------------------------- 1 | allowVolumeExpansion: true 2 | apiVersion: storage.k8s.io/v1 3 | kind: StorageClass 4 | metadata: 5 | name: proxmox-lvm 6 | parameters: 7 | csi.storage.k8s.io/fstype: xfs 8 | # blockSize: "1024" 9 | # inodeSize: "512" 10 | # 11 | storage: lvm 12 | # diskIOPS: "400" 13 | # diskMBps: "120" 14 | provisioner: csi.proxmox.sinextra.dev 15 | reclaimPolicy: Delete 16 | volumeBindingMode: WaitForFirstConsumer 17 | -------------------------------------------------------------------------------- /docs/proxmox-rbd.yaml: -------------------------------------------------------------------------------- 1 | allowVolumeExpansion: true 2 | apiVersion: storage.k8s.io/v1 3 | kind: StorageClass 4 | metadata: 5 | name: proxmox-rbd 6 | parameters: 7 | csi.storage.k8s.io/fstype: xfs 8 | storage: rbd 9 | provisioner: csi.proxmox.sinextra.dev 10 | reclaimPolicy: Delete 11 | volumeBindingMode: WaitForFirstConsumer 12 | -------------------------------------------------------------------------------- /docs/proxmox-regions.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sergelogvinov/proxmox-csi-plugin/41a3934c48cc81ac45fce25144afe881a249d6a1/docs/proxmox-regions.gif -------------------------------------------------------------------------------- /docs/proxmox-regions.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sergelogvinov/proxmox-csi-plugin/41a3934c48cc81ac45fce25144afe881a249d6a1/docs/proxmox-regions.jpeg -------------------------------------------------------------------------------- /docs/proxmox-zfs.yaml: -------------------------------------------------------------------------------- 1 | allowVolumeExpansion: true 2 | apiVersion: storage.k8s.io/v1 3 | kind: StorageClass 4 | metadata: 5 | name: proxmox-zfs 6 | parameters: 7 | csi.storage.k8s.io/fstype: xfs 8 | # 9 | storage: zfs 10 | # 11 | replicate: "true" 12 | replicateSchedule: "*/15" 13 | replicateZones: "pve-1,pve-2" 14 | provisioner: csi.proxmox.sinextra.dev 15 | reclaimPolicy: Delete 16 | volumeBindingMode: WaitForFirstConsumer 17 | -------------------------------------------------------------------------------- /docs/release.md: -------------------------------------------------------------------------------- 1 | # Make release 2 | 3 | ```shell 4 | git branch -D release-please--branches--main 5 | git checkout release-please--branches--main 6 | git tag v0.0.2 7 | 8 | make helm-unit docs 9 | 10 | git add . 11 | git commit 12 | ``` 13 | -------------------------------------------------------------------------------- /docs/vm-disks.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sergelogvinov/proxmox-csi-plugin/41a3934c48cc81ac45fce25144afe881a249d6a1/docs/vm-disks.png -------------------------------------------------------------------------------- /docs/volume-attributes.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: storage.k8s.io/v1beta1 2 | kind: VolumeAttributesClass 3 | metadata: 4 | name: test 5 | driverName: csi.proxmox.sinextra.dev 6 | parameters: 7 | diskIOPS: "400" 8 | diskMBps: "120" 9 | --- 10 | apiVersion: storage.k8s.io/v1beta1 11 | kind: VolumeAttributesClass 12 | metadata: 13 | name: test-2 14 | driverName: csi.proxmox.sinextra.dev 15 | parameters: 16 | diskIOPS: "500" 17 | diskMBps: "200" 18 | --- 19 | apiVersion: storage.k8s.io/v1 20 | kind: StorageClass 21 | metadata: 22 | name: proxmox-lvm 23 | parameters: 24 | csi.storage.k8s.io/fstype: xfs 25 | # blockSize: "1024" 26 | # inodeSize: "512" 27 | # 28 | storage: lvm 29 | ssd: "true" 30 | # diskIOPS: "100" 31 | # diskMBps: "100" 32 | provisioner: csi.proxmox.sinextra.dev 33 | reclaimPolicy: Delete 34 | volumeBindingMode: WaitForFirstConsumer 35 | allowVolumeExpansion: true 36 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/sergelogvinov/proxmox-csi-plugin 2 | 3 | go 1.24.3 4 | 5 | require ( 6 | github.com/Telmate/proxmox-api-go v0.0.0-20241127232213-af1f4e86b570 7 | github.com/container-storage-interface/spec v1.11.0 8 | github.com/golang/protobuf v1.5.4 9 | github.com/jarcoal/httpmock v1.4.0 10 | github.com/kubernetes-csi/csi-lib-utils v0.22.0 11 | github.com/sergelogvinov/proxmox-cloud-controller-manager v0.7.0 12 | github.com/siderolabs/go-blockdevice v0.4.8 13 | github.com/siderolabs/go-retry v0.3.3 14 | github.com/sirupsen/logrus v1.9.3 15 | github.com/spf13/cobra v1.9.1 16 | github.com/stretchr/testify v1.10.0 17 | google.golang.org/grpc v1.72.2 18 | k8s.io/api v0.33.1 19 | k8s.io/apimachinery v0.33.1 20 | k8s.io/client-go v0.33.1 21 | k8s.io/cloud-provider-openstack v1.32.0 22 | k8s.io/component-base v0.33.1 23 | k8s.io/klog/v2 v2.130.1 24 | k8s.io/mount-utils v0.33.1 25 | k8s.io/utils v0.0.0-20250502105355-0f33e8f1c979 26 | ) 27 | 28 | require ( 29 | github.com/armon/circbuf v0.0.0-20190214190532-5111143e8da2 // indirect 30 | github.com/beorn7/perks v1.0.1 // indirect 31 | github.com/blang/semver/v4 v4.0.0 // indirect 32 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 33 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect 34 | github.com/emicklei/go-restful/v3 v3.12.2 // indirect 35 | github.com/fxamacker/cbor/v2 v2.8.0 // indirect 36 | github.com/go-logr/logr v1.4.3 // indirect 37 | github.com/go-openapi/jsonpointer v0.21.1 // indirect 38 | github.com/go-openapi/jsonreference v0.21.0 // indirect 39 | github.com/go-openapi/swag v0.23.1 // indirect 40 | github.com/gogo/protobuf v1.3.2 // indirect 41 | github.com/google/gnostic-models v0.6.9 // indirect 42 | github.com/google/go-cmp v0.7.0 // indirect 43 | github.com/google/uuid v1.6.0 // indirect 44 | github.com/inconshreveable/mousetrap v1.1.0 // indirect 45 | github.com/josharian/intern v1.0.0 // indirect 46 | github.com/json-iterator/go v1.1.12 // indirect 47 | github.com/mailru/easyjson v0.9.0 // indirect 48 | github.com/moby/sys/mountinfo v0.7.2 // indirect 49 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 50 | github.com/modern-go/reflect2 v1.0.2 // indirect 51 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 52 | github.com/pkg/errors v0.9.1 // indirect 53 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect 54 | github.com/prometheus/client_golang v1.22.0 // indirect 55 | github.com/prometheus/client_model v0.6.2 // indirect 56 | github.com/prometheus/common v0.64.0 // indirect 57 | github.com/prometheus/procfs v0.16.1 // indirect 58 | github.com/siderolabs/go-cmd v0.1.3 // indirect 59 | github.com/spf13/pflag v1.0.6 // indirect 60 | github.com/stretchr/objx v0.5.2 // indirect 61 | github.com/x448/float16 v0.8.4 // indirect 62 | go.opentelemetry.io/otel v1.35.0 // indirect 63 | go.opentelemetry.io/otel/trace v1.35.0 // indirect 64 | golang.org/x/net v0.40.0 // indirect 65 | golang.org/x/oauth2 v0.30.0 // indirect 66 | golang.org/x/sys v0.33.0 // indirect 67 | golang.org/x/term v0.32.0 // indirect 68 | golang.org/x/text v0.25.0 // indirect 69 | golang.org/x/time v0.11.0 // indirect 70 | google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a // indirect 71 | google.golang.org/protobuf v1.36.6 // indirect 72 | gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect 73 | gopkg.in/inf.v0 v0.9.1 // indirect 74 | gopkg.in/yaml.v3 v3.0.1 // indirect 75 | k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect 76 | sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect 77 | sigs.k8s.io/randfill v1.0.0 // indirect 78 | sigs.k8s.io/structured-merge-diff/v4 v4.7.0 // indirect 79 | sigs.k8s.io/yaml v1.4.0 // indirect 80 | ) 81 | -------------------------------------------------------------------------------- /hack/CHANGELOG.tpl.md: -------------------------------------------------------------------------------- 1 | {{ range .Versions }} 2 | ## {{ if .Tag.Previous }}[{{ .Tag.Name }}]({{ $.Info.RepositoryURL }}/compare/{{ .Tag.Previous.Name }}...{{ .Tag.Name }}){{ else }}{{ .Tag.Name }}{{ end }} ({{ datetime "2006-01-02" .Tag.Date }}) 3 | 4 | Welcome to the {{ .Tag.Name }} release of Proxmox CSI Plugin! 5 | 6 | {{ if .CommitGroups -}} 7 | {{ range .CommitGroups -}} 8 | ### {{ .Title }} 9 | 10 | {{ range .Commits -}} 11 | - {{ if .Scope }}**{{ .Scope }}:** {{ end }}{{ .Subject }} ({{ .Hash.Short }}) 12 | {{ end }} 13 | {{ end -}} 14 | {{ end -}} 15 | 16 | {{- if .NoteGroups -}} 17 | {{ range .NoteGroups -}} 18 | ### {{ .Title }} 19 | 20 | {{ range .Notes }} 21 | {{ .Body }} 22 | {{ end }} 23 | {{ end -}} 24 | {{ end -}} 25 | {{ end -}} 26 | -------------------------------------------------------------------------------- /hack/chglog-config.yml: -------------------------------------------------------------------------------- 1 | style: github 2 | template: CHANGELOG.tpl.md 3 | info: 4 | title: CHANGELOG 5 | repository_url: https://github.com/sergelogvinov/proxmox-csi-plugin 6 | options: 7 | commits: 8 | filters: 9 | Type: 10 | - feat 11 | - fix 12 | - chore 13 | commit_groups: 14 | title_maps: 15 | feat: Features 16 | fix: Bug Fixes 17 | chore: Miscellaneous 18 | header: 19 | pattern: "^(\\w*)(?:\\(([\\w\\$\\.\\-\\*\\s]*)\\))?\\:\\s(.*)$" 20 | pattern_maps: 21 | - Type 22 | - Scope 23 | - Subject 24 | notes: 25 | keywords: 26 | - BREAKING CHANGE 27 | -------------------------------------------------------------------------------- /hack/ct.yml: -------------------------------------------------------------------------------- 1 | helm-extra-args: --timeout 300s 2 | check-version-increment: true 3 | debug: true 4 | chart-dirs: 5 | - charts 6 | validate-maintainers: true 7 | namespace: default 8 | release-label: test 9 | target-branch: main 10 | -------------------------------------------------------------------------------- /hack/e2e-tests.md: -------------------------------------------------------------------------------- 1 | # Integration tests 2 | 3 | ## Manual integration tests 4 | 5 | ### Encrypted PVs 6 | 7 | Create PV secret and deploy a pod that uses it. 8 | 9 | ```yaml 10 | --- 11 | apiVersion: v1 12 | data: 13 | # echo 1f03928033dda2e4fd347e44266cfbc | base64 14 | encryption-passphrase: MWYwMzkyODAzM2RkYTJlNGZkMzQ3ZTQ0MjY2Y2ZiYw== 15 | kind: Secret 16 | metadata: 17 | creationTimestamp: null 18 | name: proxmox-csi-secret 19 | namespace: kube-system 20 | --- 21 | apiVersion: storage.k8s.io/v1 22 | kind: StorageClass 23 | metadata: 24 | name: proxmox-secret 25 | parameters: 26 | csi.storage.k8s.io/fstype: xfs 27 | csi.storage.k8s.io/node-stage-secret-name: "proxmox-csi-secret" 28 | csi.storage.k8s.io/node-stage-secret-namespace: "kube-system" 29 | csi.storage.k8s.io/node-expand-secret-name: "proxmox-csi-secret" 30 | csi.storage.k8s.io/node-expand-secret-namespace: "kube-system" 31 | storage: lvm 32 | provisioner: csi.proxmox.sinextra.dev 33 | reclaimPolicy: Delete 34 | volumeBindingMode: WaitForFirstConsumer 35 | allowVolumeExpansion: true 36 | --- 37 | apiVersion: apps/v1 38 | kind: StatefulSet 39 | metadata: 40 | name: test 41 | namespace: default 42 | labels: 43 | app: alpine 44 | spec: 45 | podManagementPolicy: Parallel 46 | serviceName: test 47 | replicas: 1 48 | template: 49 | metadata: 50 | labels: 51 | app: alpine 52 | spec: 53 | terminationGracePeriodSeconds: 3 54 | tolerations: 55 | - effect: NoSchedule 56 | key: node-role.kubernetes.io/control-plane 57 | nodeSelector: 58 | # kubernetes.io/hostname: kube-store-02a 59 | # topology.kubernetes.io/zone: hvm-1 60 | containers: 61 | - name: alpine 62 | image: alpine 63 | command: ["sleep","1d"] 64 | securityContext: 65 | seccompProfile: 66 | type: RuntimeDefault 67 | capabilities: 68 | drop: ["ALL"] 69 | volumeMounts: 70 | - name: storage 71 | mountPath: /mnt 72 | updateStrategy: 73 | type: RollingUpdate 74 | selector: 75 | matchLabels: 76 | app: alpine 77 | volumeClaimTemplates: 78 | - metadata: 79 | name: storage 80 | spec: 81 | accessModes: ["ReadWriteOnce"] 82 | resources: 83 | requests: 84 | storage: 1Gi 85 | storageClassName: proxmox-secret 86 | ``` 87 | 88 | Run the statefulset, wait for it to be running and exec into the proxmox-csi-plugin-node pod to check the passphrase. 89 | 90 | ```bash 91 | echo -n "1f03928033dda2e4fd347e44266cfbc" | kube -n csi-proxmox exec -ti proxmox-csi-plugin-node-srm6v -- /sbin/cryptsetup luksOpen --debug --test-passphrase -v /dev/sdb --key-file=- 92 | ``` 93 | -------------------------------------------------------------------------------- /hack/release-please-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://raw.githubusercontent.com/googleapis/release-please/main/schemas/config.json", 3 | "pull-request-header": ":robot: I have created a release", 4 | "pull-request-title-pattern": "chore: release v${version}", 5 | "group-pull-request-title-pattern": "chore: release v${version}", 6 | "packages": { 7 | ".": { 8 | "changelog-path": "CHANGELOG.md", 9 | "release-type": "go", 10 | "skip-github-release": false, 11 | "bump-minor-pre-major": true, 12 | "include-v-in-tag": true, 13 | "draft": false, 14 | "draft-pull-request": true, 15 | "prerelease": false, 16 | "changelog-sections": [ 17 | { 18 | "type": "feat", 19 | "section": "Features", 20 | "hidden": false 21 | }, 22 | { 23 | "type": "fix", 24 | "section": "Bug Fixes", 25 | "hidden": false 26 | }, 27 | { 28 | "type": "*", 29 | "section": "Changelog", 30 | "hidden": false 31 | } 32 | ] 33 | } 34 | } 35 | } -------------------------------------------------------------------------------- /hack/release-please-manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | ".": "0.11.0" 3 | } -------------------------------------------------------------------------------- /hack/testdata/cloud-config.yaml: -------------------------------------------------------------------------------- 1 | clusters: 2 | - url: https://cluster-api-1.exmple.com:8006/api2/json 3 | insecure: false 4 | token_id: "login!name" 5 | token_secret: "secret" 6 | region: Region-1 7 | - url: https://cluster-api-2.exmple.com:8006/api2/json 8 | insecure: false 9 | token_id: "login!name" 10 | token_secret: "secret" 11 | region: Region-2 12 | -------------------------------------------------------------------------------- /pkg/csi/driver.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Package csi contains the CSI driver implementation 18 | package csi 19 | 20 | const ( 21 | // DriverName is the name of the CSI driver 22 | DriverName = "csi.proxmox.sinextra.dev" 23 | // DriverVersion is the version of the CSI driver 24 | DriverVersion = "0.5.0" 25 | // DriverSpecVersion CSI spec version 26 | DriverSpecVersion = "1.11.0" 27 | 28 | // DefaultMaxVolumesPerNode is the default maximum number of volumes that can be attached to a node 29 | DefaultMaxVolumesPerNode = 24 30 | // DefaultVolumeSizeBytes is the default size of a volume 31 | DefaultVolumeSizeBytes = 10 * GiB 32 | // MinChunkSizeBytes is the minimum size of a volume chunk 33 | MinChunkSizeBytes = 512 * MiB 34 | 35 | // EncryptionPassphraseKey is the encryption passphrase secret key 36 | EncryptionPassphraseKey = "encryption-passphrase" 37 | ) 38 | 39 | // constants for fstypes 40 | const ( 41 | // FSTypeExt4 represents the ext4 filesystem type 42 | FSTypeExt4 = "ext4" 43 | // FSTypeXfs represents the xfs filesystem type 44 | FSTypeXfs = "xfs" 45 | ) 46 | 47 | // constants for node labels 48 | const ( 49 | NodeLabelMaxVolumeAttachments = DriverName + "/max-volume-attachments" 50 | ) 51 | -------------------------------------------------------------------------------- /pkg/csi/helper.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package csi 18 | 19 | import ( 20 | "fmt" 21 | "net/url" 22 | "os" 23 | "path" 24 | "path/filepath" 25 | "strings" 26 | 27 | proto "github.com/container-storage-interface/spec/lib/go/csi" 28 | 29 | corev1 "k8s.io/api/core/v1" 30 | ) 31 | 32 | // ParseEndpoint parses the endpoint string and returns the scheme and address 33 | func ParseEndpoint(endpoint string) (string, string, error) { 34 | u, err := url.Parse(endpoint) 35 | if err != nil { 36 | return "", "", fmt.Errorf("could not parse endpoint: %v", err) 37 | } 38 | 39 | addr := path.Join(u.Host, filepath.FromSlash(u.Path)) 40 | 41 | scheme := strings.ToLower(u.Scheme) 42 | switch scheme { 43 | case "tcp": 44 | case "unix": 45 | addr = path.Join("/", addr) 46 | if err := os.Remove(addr); err != nil && !os.IsNotExist(err) { 47 | return "", "", fmt.Errorf("could not remove unix domain socket %q: %v", addr, err) 48 | } 49 | default: 50 | return "", "", fmt.Errorf("unsupported protocol: %s", scheme) 51 | } 52 | 53 | return scheme, addr, nil 54 | } 55 | 56 | func locationFromTopologyRequirement(tr *proto.TopologyRequirement) (region, zone string) { 57 | if tr == nil { 58 | return "", "" 59 | } 60 | 61 | for _, top := range tr.GetPreferred() { 62 | segment := top.GetSegments() 63 | 64 | tsr := segment[corev1.LabelTopologyRegion] 65 | tsz := segment[corev1.LabelTopologyZone] 66 | 67 | if tsr != "" && tsz != "" { 68 | return tsr, tsz 69 | } 70 | 71 | if tsr != "" && region == "" { 72 | region = tsr 73 | } 74 | } 75 | 76 | for _, top := range tr.GetRequisite() { 77 | segment := top.GetSegments() 78 | 79 | tsr := segment[corev1.LabelTopologyRegion] 80 | tsz := segment[corev1.LabelTopologyZone] 81 | 82 | if tsr != "" && tsz != "" { 83 | return tsr, tsz 84 | } 85 | 86 | if tsr != "" && region == "" { 87 | region = tsr 88 | } 89 | } 90 | 91 | return region, "" 92 | } 93 | -------------------------------------------------------------------------------- /pkg/csi/helper_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package csi 18 | 19 | import ( 20 | "fmt" 21 | "testing" 22 | 23 | proto "github.com/container-storage-interface/spec/lib/go/csi" 24 | "github.com/stretchr/testify/assert" 25 | 26 | corev1 "k8s.io/api/core/v1" 27 | ) 28 | 29 | func TestParseEndpoint(t *testing.T) { 30 | t.Parallel() 31 | 32 | tests := []struct { 33 | msg string 34 | endpoint string 35 | expectedScheme string 36 | expectedAddr string 37 | expectedError error 38 | }{ 39 | { 40 | msg: "unix socket", 41 | endpoint: "unix://tmp/csi.sock", 42 | expectedScheme: "unix", 43 | expectedAddr: "/tmp/csi.sock", 44 | }, 45 | { 46 | msg: "http", 47 | endpoint: "http://tmp/csi.sock", 48 | expectedError: fmt.Errorf("unsupported protocol: http"), 49 | }, 50 | } 51 | 52 | for _, testCase := range tests { 53 | testCase := testCase 54 | 55 | t.Run(fmt.Sprint(testCase.msg), func(t *testing.T) { 56 | t.Parallel() 57 | 58 | scheme, addr, err := ParseEndpoint(testCase.endpoint) 59 | 60 | if testCase.expectedError != nil { 61 | assert.NotNil(t, err) 62 | assert.Equal(t, err.Error(), testCase.expectedError.Error()) 63 | } else { 64 | assert.Nil(t, err) 65 | assert.NotNil(t, scheme, testCase.expectedScheme) 66 | assert.Equal(t, addr, testCase.expectedAddr) 67 | } 68 | }) 69 | } 70 | } 71 | 72 | func TestLocationFromTopologyRequirement(t *testing.T) { 73 | t.Parallel() 74 | 75 | tests := []struct { 76 | msg string 77 | topology *proto.TopologyRequirement 78 | expectedRegion string 79 | expectedZone string 80 | }{ 81 | { 82 | msg: "EmptyTopologyRequirement", 83 | topology: &proto.TopologyRequirement{}, 84 | expectedRegion: "", 85 | expectedZone: "", 86 | }, 87 | { 88 | msg: "EmptyTopologyPreferredZone", 89 | topology: &proto.TopologyRequirement{ 90 | Preferred: []*proto.Topology{ 91 | { 92 | Segments: map[string]string{ 93 | corev1.LabelTopologyRegion: "region1", 94 | }, 95 | }, 96 | }, 97 | }, 98 | expectedRegion: "region1", 99 | expectedZone: "", 100 | }, 101 | { 102 | msg: "EmptyTopologyRequisiteZone", 103 | topology: &proto.TopologyRequirement{ 104 | Requisite: []*proto.Topology{ 105 | { 106 | Segments: map[string]string{ 107 | corev1.LabelTopologyRegion: "region1", 108 | }, 109 | }, 110 | }, 111 | }, 112 | expectedRegion: "region1", 113 | expectedZone: "", 114 | }, 115 | { 116 | msg: "EmptyTopologyPreferredRegion", 117 | topology: &proto.TopologyRequirement{ 118 | Preferred: []*proto.Topology{ 119 | { 120 | Segments: map[string]string{ 121 | corev1.LabelTopologyZone: "zone1", 122 | }, 123 | }, 124 | }, 125 | }, 126 | expectedRegion: "", 127 | expectedZone: "", 128 | }, 129 | { 130 | msg: "TopologyPreferred", 131 | topology: &proto.TopologyRequirement{ 132 | Preferred: []*proto.Topology{ 133 | { 134 | Segments: map[string]string{ 135 | corev1.LabelTopologyRegion: "region1", 136 | corev1.LabelTopologyZone: "zone1", 137 | }, 138 | }, 139 | }, 140 | }, 141 | expectedRegion: "region1", 142 | expectedZone: "zone1", 143 | }, 144 | { 145 | msg: "TopologyRequisite", 146 | topology: &proto.TopologyRequirement{ 147 | Requisite: []*proto.Topology{ 148 | { 149 | Segments: map[string]string{ 150 | corev1.LabelTopologyRegion: "region1", 151 | corev1.LabelTopologyZone: "zone1", 152 | }, 153 | }, 154 | }, 155 | }, 156 | expectedRegion: "region1", 157 | expectedZone: "zone1", 158 | }, 159 | } 160 | 161 | for _, testCase := range tests { 162 | testCase := testCase 163 | 164 | t.Run(fmt.Sprint(testCase.msg), func(t *testing.T) { 165 | t.Parallel() 166 | 167 | region, zone := locationFromTopologyRequirement(testCase.topology) 168 | 169 | assert.Equal(t, testCase.expectedRegion, region) 170 | assert.Equal(t, testCase.expectedZone, zone) 171 | }) 172 | } 173 | } 174 | -------------------------------------------------------------------------------- /pkg/csi/identity.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package csi 18 | 19 | import ( 20 | "context" 21 | 22 | "github.com/container-storage-interface/spec/lib/go/csi" 23 | "github.com/golang/protobuf/ptypes/wrappers" 24 | 25 | "k8s.io/klog/v2" 26 | ) 27 | 28 | // IdentityService is the identity service for the CSI driver 29 | type IdentityService struct { 30 | csi.UnimplementedIdentityServer 31 | } 32 | 33 | // NewIdentityService returns a new identity service 34 | func NewIdentityService() *IdentityService { 35 | return &IdentityService{} 36 | } 37 | 38 | // GetPluginInfo returns the name and version of the plugin 39 | func (d *IdentityService) GetPluginInfo(context.Context, *csi.GetPluginInfoRequest) (*csi.GetPluginInfoResponse, error) { 40 | klog.V(5).InfoS("GetPluginInfo: called") 41 | 42 | return &csi.GetPluginInfoResponse{ 43 | Name: DriverName, 44 | VendorVersion: DriverVersion, 45 | }, nil 46 | } 47 | 48 | // GetPluginCapabilities returns the capabilities of the plugin 49 | func (d *IdentityService) GetPluginCapabilities(context.Context, *csi.GetPluginCapabilitiesRequest) (*csi.GetPluginCapabilitiesResponse, error) { 50 | klog.V(5).InfoS("GetPluginCapabilities: called") 51 | 52 | resp := &csi.GetPluginCapabilitiesResponse{ 53 | Capabilities: []*csi.PluginCapability{ 54 | { 55 | Type: &csi.PluginCapability_Service_{ 56 | Service: &csi.PluginCapability_Service{ 57 | Type: csi.PluginCapability_Service_CONTROLLER_SERVICE, 58 | }, 59 | }, 60 | }, 61 | { 62 | Type: &csi.PluginCapability_Service_{ 63 | Service: &csi.PluginCapability_Service{ 64 | Type: csi.PluginCapability_Service_VOLUME_ACCESSIBILITY_CONSTRAINTS, 65 | }, 66 | }, 67 | }, 68 | { 69 | Type: &csi.PluginCapability_VolumeExpansion_{ 70 | VolumeExpansion: &csi.PluginCapability_VolumeExpansion{ 71 | Type: csi.PluginCapability_VolumeExpansion_ONLINE, 72 | }, 73 | }, 74 | }, 75 | }, 76 | } 77 | 78 | return resp, nil 79 | } 80 | 81 | // Probe returns the health and readiness of the plugin 82 | func (d *IdentityService) Probe(_ context.Context, _ *csi.ProbeRequest) (*csi.ProbeResponse, error) { 83 | klog.V(5).InfoS("Probe: called") 84 | 85 | return &csi.ProbeResponse{ 86 | Ready: &wrappers.BoolValue{Value: true}, 87 | }, nil 88 | } 89 | -------------------------------------------------------------------------------- /pkg/csi/identity_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package csi_test 18 | 19 | import ( 20 | "testing" 21 | 22 | proto "github.com/container-storage-interface/spec/lib/go/csi" 23 | "github.com/stretchr/testify/assert" 24 | 25 | "github.com/sergelogvinov/proxmox-csi-plugin/pkg/csi" 26 | ) 27 | 28 | var _ proto.IdentityServer = (*csi.IdentityService)(nil) 29 | 30 | type identityServiceTestEnv struct { 31 | service *csi.IdentityService 32 | } 33 | 34 | func newIdentityServerTestEnv() identityServiceTestEnv { 35 | return identityServiceTestEnv{ 36 | service: csi.NewIdentityService(), 37 | } 38 | } 39 | 40 | func TestGetPluginInfo(t *testing.T) { 41 | env := newIdentityServerTestEnv() 42 | 43 | resp, err := env.service.GetPluginInfo(t.Context(), &proto.GetPluginInfoRequest{}) 44 | assert.Nil(t, err) 45 | assert.NotNil(t, resp) 46 | 47 | assert.Equal(t, resp.GetName(), csi.DriverName) 48 | assert.Equal(t, resp.GetVendorVersion(), csi.DriverVersion) 49 | } 50 | 51 | func TestGetPluginCapabilities(t *testing.T) { 52 | env := newIdentityServerTestEnv() 53 | 54 | resp, err := env.service.GetPluginCapabilities(t.Context(), &proto.GetPluginCapabilitiesRequest{}) 55 | assert.Nil(t, err) 56 | assert.NotNil(t, resp) 57 | assert.NotNil(t, resp.GetCapabilities()) 58 | 59 | for _, capability := range resp.GetCapabilities() { 60 | if capability.GetVolumeExpansion() != nil { 61 | switch capability.GetVolumeExpansion().GetType() { //nolint:exhaustive 62 | case proto.PluginCapability_VolumeExpansion_ONLINE: 63 | case proto.PluginCapability_VolumeExpansion_OFFLINE: 64 | default: 65 | t.Fatalf("Unknown capability: %v", capability.GetVolumeExpansion().GetType()) 66 | } 67 | } 68 | 69 | if capability.GetService() != nil { 70 | switch capability.GetService().GetType() { //nolint:exhaustive 71 | case proto.PluginCapability_Service_CONTROLLER_SERVICE: 72 | case proto.PluginCapability_Service_VOLUME_ACCESSIBILITY_CONSTRAINTS: 73 | default: 74 | t.Fatalf("Unknown capability: %v", capability.GetService().GetType()) 75 | } 76 | } 77 | } 78 | } 79 | 80 | func TestProbe(t *testing.T) { 81 | env := newIdentityServerTestEnv() 82 | 83 | resp, err := env.service.Probe(t.Context(), &proto.ProbeRequest{}) 84 | assert.Nil(t, err) 85 | assert.NotNil(t, resp) 86 | } 87 | -------------------------------------------------------------------------------- /pkg/csi/parameters_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package csi_test 18 | 19 | import ( 20 | "fmt" 21 | "testing" 22 | 23 | "github.com/stretchr/testify/assert" 24 | 25 | "github.com/sergelogvinov/proxmox-csi-plugin/pkg/csi" 26 | "github.com/sergelogvinov/proxmox-csi-plugin/pkg/helpers/ptr" 27 | ) 28 | 29 | func Test_ExtractAndDefaultParameters(t *testing.T) { 30 | t.Parallel() 31 | 32 | tests := []struct { 33 | msg string 34 | params map[string]string 35 | storage csi.StorageParameters 36 | }{ 37 | { 38 | msg: "Empty params", 39 | params: map[string]string{ 40 | csi.StorageIDKey: "local-lvm", 41 | }, 42 | storage: csi.StorageParameters{ 43 | Backup: ptr.Ptr(false), 44 | IOThread: true, 45 | }, 46 | }, 47 | { 48 | msg: "SSD disk", 49 | params: map[string]string{ 50 | csi.StorageIDKey: "local-lvm", 51 | csi.StorageSSDKey: "true", 52 | csi.StorageCacheKey: "directsync", 53 | }, 54 | storage: csi.StorageParameters{ 55 | Cache: "directsync", 56 | Backup: ptr.Ptr(false), 57 | IOThread: true, 58 | SSD: ptr.Ptr(true), 59 | Discard: "on", 60 | }, 61 | }, 62 | { 63 | msg: "disk limits", 64 | params: map[string]string{ 65 | csi.StorageIDKey: "local-lvm", 66 | csi.StorageSSDKey: "true", 67 | csi.StorageDiskIOPSKey: "100", 68 | }, 69 | storage: csi.StorageParameters{ 70 | Backup: ptr.Ptr(false), 71 | IOThread: true, 72 | SSD: ptr.Ptr(true), 73 | Discard: "on", 74 | Iops: ptr.Ptr(100), 75 | IopsRead: ptr.Ptr(100), 76 | IopsWrite: ptr.Ptr(100), 77 | }, 78 | }, 79 | { 80 | msg: "ovverid disk backup", 81 | params: map[string]string{ 82 | csi.StorageIDKey: "local-lvm", 83 | csi.StorageSSDKey: "true", 84 | csi.StorageDiskIOPSKey: "100", 85 | "backup": "true", 86 | }, 87 | storage: csi.StorageParameters{ 88 | Backup: ptr.Ptr(true), 89 | IOThread: true, 90 | SSD: ptr.Ptr(true), 91 | Discard: "on", 92 | Iops: ptr.Ptr(100), 93 | IopsRead: ptr.Ptr(100), 94 | IopsWrite: ptr.Ptr(100), 95 | }, 96 | }, 97 | { 98 | msg: "replication disk", 99 | params: map[string]string{ 100 | csi.StorageIDKey: "local-lvm", 101 | "replicate": "true", 102 | "replicateZones": "zone1,zone2", 103 | }, 104 | storage: csi.StorageParameters{ 105 | Backup: ptr.Ptr(false), 106 | IOThread: true, 107 | Replicate: ptr.Ptr(true), 108 | ReplicateZones: "zone1,zone2", 109 | }, 110 | }, 111 | } 112 | 113 | for _, testCase := range tests { 114 | testCase := testCase 115 | 116 | t.Run(fmt.Sprint(testCase.msg), func(t *testing.T) { 117 | t.Parallel() 118 | 119 | storage, err := csi.ExtractAndDefaultParameters(testCase.params) 120 | 121 | assert.Nil(t, err) 122 | assert.Equal(t, testCase.storage, storage) 123 | }) 124 | } 125 | } 126 | 127 | func Test_ToMap(t *testing.T) { 128 | t.Parallel() 129 | 130 | tests := []struct { 131 | msg string 132 | storage csi.StorageParameters 133 | params map[string]string 134 | }{ 135 | { 136 | msg: "Empty params", 137 | storage: csi.StorageParameters{}, 138 | params: map[string]string{ 139 | "iothread": "0", 140 | }, 141 | }, 142 | { 143 | msg: "Params with IOThread and limits", 144 | storage: csi.StorageParameters{ 145 | Cache: "directsync", 146 | IOThread: true, 147 | IopsRead: ptr.Ptr(100), 148 | IopsWrite: ptr.Ptr(100), 149 | }, 150 | params: map[string]string{ 151 | "cache": "directsync", 152 | "iothread": "1", 153 | "iops_rd": "100", 154 | "iops_wr": "100", 155 | }, 156 | }, 157 | { 158 | msg: "Params with replication", 159 | storage: csi.StorageParameters{ 160 | Cache: "directsync", 161 | IOThread: true, 162 | Replicate: ptr.Ptr(true), 163 | ReplicateZones: "zone1,zone2", 164 | ReplicateSchedule: "*/30", 165 | }, 166 | params: map[string]string{ 167 | "cache": "directsync", 168 | "iothread": "1", 169 | "replicate": "1", 170 | }, 171 | }, 172 | } 173 | 174 | for _, testCase := range tests { 175 | testCase := testCase 176 | 177 | t.Run(fmt.Sprint(testCase.msg), func(t *testing.T) { 178 | t.Parallel() 179 | 180 | storage := testCase.storage.ToMap() 181 | assert.Equal(t, testCase.params, storage) 182 | }) 183 | } 184 | } 185 | 186 | func Test_ExtractModifyVolumeParameters(t *testing.T) { 187 | t.Parallel() 188 | 189 | tests := []struct { 190 | msg string 191 | params map[string]string 192 | storage csi.ModifyVolumeParameters 193 | }{ 194 | { 195 | msg: "Empty params", 196 | params: map[string]string{}, 197 | storage: csi.ModifyVolumeParameters{}, 198 | }, 199 | { 200 | msg: "Backup volume", 201 | params: map[string]string{ 202 | "backup": "true", 203 | }, 204 | storage: csi.ModifyVolumeParameters{ 205 | Backup: ptr.Ptr(true), 206 | }, 207 | }, 208 | { 209 | msg: "BW limits", 210 | params: map[string]string{ 211 | "diskIOPS": "100", 212 | "diskMBps": "100", 213 | }, 214 | storage: csi.ModifyVolumeParameters{ 215 | Iops: ptr.Ptr(100), 216 | IopsRead: ptr.Ptr(100), 217 | IopsWrite: ptr.Ptr(100), 218 | SpeedMbps: ptr.Ptr(100), 219 | ReadSpeedMbps: ptr.Ptr(100), 220 | WriteSpeedMbps: ptr.Ptr(100), 221 | }, 222 | }, 223 | } 224 | 225 | for _, testCase := range tests { 226 | testCase := testCase 227 | 228 | t.Run(fmt.Sprint(testCase.msg), func(t *testing.T) { 229 | t.Parallel() 230 | 231 | storage, err := csi.ExtractModifyVolumeParameters(testCase.params) 232 | 233 | assert.Nil(t, err) 234 | assert.Equal(t, testCase.storage, storage) 235 | }) 236 | } 237 | } 238 | 239 | func Test_MergeMap(t *testing.T) { 240 | t.Parallel() 241 | 242 | tests := []struct { 243 | msg string 244 | storage csi.ModifyVolumeParameters 245 | params map[string]string 246 | expected map[string]string 247 | }{ 248 | { 249 | msg: "Empty modify params", 250 | storage: csi.ModifyVolumeParameters{}, 251 | params: map[string]string{ 252 | "storage": "lvm", 253 | "ssd": "true", 254 | }, 255 | expected: map[string]string{ 256 | "storage": "lvm", 257 | "ssd": "true", 258 | }, 259 | }, 260 | { 261 | msg: "Backup param", 262 | storage: csi.ModifyVolumeParameters{ 263 | Backup: ptr.Ptr(true), 264 | }, 265 | params: map[string]string{ 266 | "storage": "lvm", 267 | "ssd": "true", 268 | "blockSize": "1024", 269 | }, 270 | expected: map[string]string{ 271 | "backup": "1", 272 | "storage": "lvm", 273 | "ssd": "true", 274 | "blockSize": "1024", 275 | }, 276 | }, 277 | } 278 | 279 | for _, testCase := range tests { 280 | testCase := testCase 281 | 282 | t.Run(fmt.Sprint(testCase.msg), func(t *testing.T) { 283 | t.Parallel() 284 | 285 | storage := testCase.storage.MergeMap(testCase.params) 286 | assert.Equal(t, testCase.expected, storage) 287 | }) 288 | } 289 | } 290 | -------------------------------------------------------------------------------- /pkg/csi/utils_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package csi 18 | 19 | import ( 20 | "fmt" 21 | "testing" 22 | 23 | "github.com/stretchr/testify/assert" 24 | ) 25 | 26 | func TestIsVolumeAttached(t *testing.T) { 27 | t.Parallel() 28 | 29 | tests := []struct { 30 | msg string 31 | vmConfig map[string]interface{} 32 | pvc string 33 | expectedLun int 34 | expectedExist bool 35 | }{ 36 | { 37 | msg: "Empty VM config", 38 | vmConfig: map[string]interface{}{}, 39 | pvc: "", 40 | expectedLun: 0, 41 | expectedExist: false, 42 | }, 43 | { 44 | msg: "Empty PVC", 45 | vmConfig: map[string]interface{}{ 46 | "ide2": "local:iso/ubuntu-20.04.1-live-server-amd64.iso,media=cdrom", 47 | "scsihw": "virtio-scsi-single", 48 | "scsi0": "local-lvm:vm-100-disk-0,size=8G", 49 | "scsi5": "local-lvm:vm-100-pvc-123,size=8G", 50 | }, 51 | pvc: "", 52 | expectedLun: 0, 53 | expectedExist: false, 54 | }, 55 | { 56 | msg: "LUN 5", 57 | vmConfig: map[string]interface{}{ 58 | "ide2": "local:iso/ubuntu-20.04.1-live-server-amd64.iso,media=cdrom", 59 | "scsihw": "virtio-scsi-single", 60 | "scsi0": "local-lvm:vm-100-disk-0,size=8G", 61 | "scsi5": "local-lvm:vm-100-pvc-123,size=8G", 62 | }, 63 | pvc: "pvc-123", 64 | expectedLun: 5, 65 | expectedExist: true, 66 | }, 67 | } 68 | 69 | for _, testCase := range tests { 70 | testCase := testCase 71 | 72 | t.Run(fmt.Sprint(testCase.msg), func(t *testing.T) { 73 | t.Parallel() 74 | 75 | lun, exist := isVolumeAttached(testCase.vmConfig, testCase.pvc) 76 | 77 | if testCase.expectedExist { 78 | assert.True(t, exist) 79 | assert.Equal(t, testCase.expectedLun, lun) 80 | } else { 81 | assert.False(t, exist) 82 | assert.Equal(t, 0, lun) 83 | } 84 | }) 85 | } 86 | } 87 | 88 | func TestRoundUpSizeBytes(t *testing.T) { 89 | t.Parallel() 90 | 91 | tests := []struct { 92 | msg string 93 | volumeSize int64 94 | allocationUnitBytes int64 95 | expected int64 96 | }{ 97 | { 98 | msg: "Zero size", 99 | volumeSize: 0, 100 | allocationUnitBytes: GiB, 101 | expected: 1024 * 1024 * 1024, 102 | }, 103 | { 104 | msg: "KiB", 105 | volumeSize: 123, 106 | allocationUnitBytes: KiB, 107 | expected: 1024, 108 | }, 109 | { 110 | msg: "MiB", 111 | volumeSize: 123, 112 | allocationUnitBytes: MiB, 113 | expected: 1024 * 1024, 114 | }, 115 | { 116 | msg: "GiB", 117 | volumeSize: 123, 118 | allocationUnitBytes: GiB, 119 | expected: 1024 * 1024 * 1024, 120 | }, 121 | { 122 | msg: "256MiB -> GiB", 123 | volumeSize: 256 * 1024 * 1024, 124 | allocationUnitBytes: GiB, 125 | expected: 1024 * 1024 * 1024, 126 | }, 127 | { 128 | msg: "256MiB -> GiB/2", 129 | volumeSize: 256 * 1024 * 1024, 130 | allocationUnitBytes: 512 * MiB, 131 | expected: 512 * 1024 * 1024, 132 | }, 133 | } 134 | 135 | for _, testCase := range tests { 136 | testCase := testCase 137 | 138 | t.Run(fmt.Sprint(testCase.msg), func(t *testing.T) { 139 | t.Parallel() 140 | 141 | expected := RoundUpSizeBytes(testCase.volumeSize, testCase.allocationUnitBytes) 142 | assert.Equal(t, testCase.expected, expected) 143 | }) 144 | } 145 | } 146 | -------------------------------------------------------------------------------- /pkg/helpers/ptr/ptr.go: -------------------------------------------------------------------------------- 1 | /* 2 | * This Source Code Form is subject to the terms of the Mozilla Public 3 | * License, v. 2.0. If a copy of the MPL was not distributed with this 4 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. 5 | */ 6 | 7 | package ptr 8 | 9 | // Ptr creates a ptr from a value to use it inline. 10 | func Ptr[T any](val T) *T { 11 | return &val 12 | } 13 | 14 | // Or will dereference a pointer and return the given value if it's nil. 15 | func Or[T any](p *T, or T) T { 16 | if p != nil { 17 | return *p 18 | } 19 | 20 | return or 21 | } 22 | -------------------------------------------------------------------------------- /pkg/log/log.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Package log is cli logger configurator. 18 | package log 19 | 20 | import ( 21 | "os" 22 | 23 | log "github.com/sirupsen/logrus" 24 | ) 25 | 26 | const ( 27 | // LevelTrace is the log level for tracing 28 | LevelTrace = "trace" 29 | // LevelDebug is the log level for debugging 30 | LevelDebug = "debug" 31 | // LevelInfo is the log level for informational messages 32 | LevelInfo = "info" 33 | // LevelWarn is the log level for warnings 34 | LevelWarn = "warn" 35 | // LevelError is the log level for errors 36 | LevelError = "error" 37 | // LevelFatal is the log level for fatal errors 38 | LevelFatal = "fatal" 39 | // LevelPanic is the log level for panics 40 | LevelPanic = "panic" 41 | ) 42 | 43 | // Levels is a slice of all log levels 44 | var Levels = []string{ 45 | LevelTrace, LevelDebug, LevelInfo, LevelWarn, 46 | LevelError, LevelFatal, LevelPanic, 47 | } 48 | 49 | // Configure configures the logger. 50 | func Configure(entry *log.Entry, level string) { 51 | logger := entry.Logger 52 | logger.SetOutput(os.Stdout) 53 | 54 | logger.SetFormatter(&log.TextFormatter{ 55 | // DisableColors: true, 56 | // FullTimestamp: true, 57 | DisableTimestamp: true, 58 | DisableLevelTruncation: true, 59 | }) 60 | logger.SetLevel(logLevel(level)) 61 | } 62 | 63 | func logLevel(level string) log.Level { 64 | switch level { 65 | case LevelTrace: 66 | return log.TraceLevel 67 | case LevelDebug: 68 | return log.DebugLevel 69 | case LevelInfo: 70 | return log.InfoLevel 71 | case LevelWarn: 72 | return log.WarnLevel 73 | case LevelError: 74 | return log.ErrorLevel 75 | case LevelFatal: 76 | return log.FatalLevel 77 | case LevelPanic: 78 | return log.PanicLevel 79 | } 80 | 81 | return log.DebugLevel 82 | } 83 | -------------------------------------------------------------------------------- /pkg/metrics/metrics.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Package metrics collects metrics. 18 | package metrics 19 | 20 | import ( 21 | "time" 22 | ) 23 | 24 | // MetricContext indicates the context for Talos client metrics. 25 | type MetricContext struct { 26 | start time.Time 27 | attributes []string 28 | } 29 | 30 | // NewMetricContext creates a new MetricContext. 31 | func NewMetricContext(resource string) *MetricContext { 32 | return &MetricContext{ 33 | start: time.Now(), 34 | attributes: []string{resource}, 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /pkg/metrics/metrics_api.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package metrics 18 | 19 | import ( 20 | "time" 21 | 22 | "k8s.io/component-base/metrics" 23 | "k8s.io/component-base/metrics/legacyregistry" 24 | ) 25 | 26 | // CSIMetrics contains the metrics for Talos API calls. 27 | type CSIMetrics struct { 28 | Duration *metrics.HistogramVec 29 | Errors *metrics.CounterVec 30 | } 31 | 32 | var apiMetrics = registerAPIMetrics() 33 | 34 | // ObserveRequest records the request latency and counts the errors. 35 | func (mc *MetricContext) ObserveRequest(err error) error { 36 | apiMetrics.Duration.WithLabelValues(mc.attributes...).Observe( 37 | time.Since(mc.start).Seconds()) 38 | 39 | if err != nil { 40 | apiMetrics.Errors.WithLabelValues(mc.attributes...).Inc() 41 | } 42 | 43 | return err 44 | } 45 | 46 | func registerAPIMetrics() *CSIMetrics { 47 | metrics := &CSIMetrics{ 48 | Duration: metrics.NewHistogramVec( 49 | &metrics.HistogramOpts{ 50 | Name: "proxmox_api_request_duration_seconds", 51 | Help: "Latency of an Proxmox API call", 52 | Buckets: []float64{.1, .25, .5, 1, 2.5, 5, 10, 30}, 53 | }, []string{"request"}), 54 | Errors: metrics.NewCounterVec( 55 | &metrics.CounterOpts{ 56 | Name: "proxmox_api_request_errors_total", 57 | Help: "Total number of errors for an Proxmox API call", 58 | }, []string{"request"}), 59 | } 60 | 61 | legacyregistry.MustRegister( 62 | metrics.Duration, 63 | metrics.Errors, 64 | ) 65 | 66 | return metrics 67 | } 68 | -------------------------------------------------------------------------------- /pkg/proxmox/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Package proxmox implements tools to work with Proxmox VM. 18 | package proxmox 19 | -------------------------------------------------------------------------------- /pkg/proxmox/vm.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package proxmox 18 | 19 | import ( 20 | "fmt" 21 | 22 | pxapi "github.com/Telmate/proxmox-api-go/proxmox" 23 | ) 24 | 25 | // CreateQemuVM creates a new simple Qemu VM on the given node with the given name. 26 | func CreateQemuVM(client *pxapi.Client, vmr *pxapi.VmRef, name string) error { 27 | vm := map[string]interface{}{} 28 | vm["vmid"] = vmr.VmId() 29 | vm["node"] = vmr.Node() 30 | vm["name"] = name 31 | vm["boot"] = "order=scsi0" 32 | vm["agent"] = "0" 33 | vm["machine"] = "pc" 34 | vm["cores"] = "1" 35 | vm["memory"] = "512" 36 | vm["scsihw"] = "virtio-scsi-single" 37 | 38 | _, err := client.CreateQemuVm(vmr.Node(), vm) 39 | if err != nil { 40 | return fmt.Errorf("failed to create vm: %v", err) 41 | } 42 | 43 | return nil 44 | } 45 | 46 | // DeleteQemuVM delete the Qemu VM on the given node with the given name. 47 | func DeleteQemuVM(client *pxapi.Client, vmr *pxapi.VmRef) error { 48 | params := map[string]interface{}{} 49 | params["purge"] = "1" 50 | 51 | if _, err := client.DeleteVmParams(vmr, params); err != nil { 52 | return fmt.Errorf("failed to delete vm %d: %v", vmr.VmId(), err) 53 | } 54 | 55 | return nil 56 | } 57 | 58 | // SetQemuVMReplication sets the replication configuration for the given VM. 59 | func SetQemuVMReplication(client *pxapi.Client, vmr *pxapi.VmRef, node string, schedule string) error { 60 | if schedule == "" { 61 | schedule = "*/15" 62 | } 63 | 64 | vmParams := map[string]interface{}{ 65 | "id": fmt.Sprintf("%d-0", vmr.VmId()), 66 | "type": "local", 67 | "target": node, 68 | "schedule": schedule, 69 | "comment": "CSI Replication for PV", 70 | } 71 | 72 | // POST https://pve.proxmox.com/pve-docs/api-viewer/index.html#/cluster/replication 73 | _, err := client.CreateItemReturnStatus(vmParams, "/cluster/replication") 74 | if err != nil { 75 | return fmt.Errorf("failed to create replication: %v, vmParams=%+v", err, vmParams) 76 | } 77 | 78 | return nil 79 | } 80 | -------------------------------------------------------------------------------- /pkg/proxmox/volume.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package proxmox 18 | 19 | import ( 20 | "encoding/json" 21 | "fmt" 22 | "strconv" 23 | "strings" 24 | "time" 25 | 26 | pxapi "github.com/Telmate/proxmox-api-go/proxmox" 27 | 28 | volume "github.com/sergelogvinov/proxmox-csi-plugin/pkg/volume" 29 | ) 30 | 31 | // WaitForVolumeDetach waits for the volume to be detached from the VM. 32 | func WaitForVolumeDetach(client *pxapi.Client, vmName string, disk string) error { 33 | if vmName == "" { 34 | return nil 35 | } 36 | 37 | vmr, err := client.GetVmRefsByName(vmName) 38 | if err != nil || len(vmr) == 0 { 39 | return fmt.Errorf("failed to get vmID") 40 | } 41 | 42 | for { 43 | time.Sleep(5 * time.Second) 44 | 45 | vmConfig, err := client.GetVmConfig(vmr[0]) 46 | if err != nil { 47 | return fmt.Errorf("failed to get vm config: %v", err) 48 | } 49 | 50 | found := false 51 | 52 | for lun := 1; lun < 30; lun++ { 53 | device := fmt.Sprintf("scsi%d", lun) 54 | 55 | if vmConfig[device] != nil && strings.Contains(vmConfig[device].(string), disk) { //nolint:errcheck 56 | found = true 57 | 58 | break 59 | } 60 | } 61 | 62 | if !found { 63 | return nil 64 | } 65 | } 66 | } 67 | 68 | // MoveQemuDisk moves the volume from one node to another. 69 | func MoveQemuDisk(cluster *pxapi.Client, vol *volume.Volume, node string, taskTimeout int) error { 70 | vmParams := map[string]interface{}{ 71 | "node": vol.Node(), 72 | "target": vol.Disk(), 73 | "target_node": node, 74 | "volume": vol.Disk(), 75 | } 76 | 77 | oldTimeout := cluster.TaskTimeout 78 | cluster.TaskTimeout = taskTimeout 79 | 80 | // POST https://pve.proxmox.com/pve-docs/api-viewer/index.html#/nodes/{node}/storage/{storage}/content/{volume} 81 | // Copy a volume. This is experimental code - do not use. 82 | resp, err := cluster.CreateItemReturnStatus(vmParams, "/nodes/"+vol.Node()+"/storage/"+vol.Storage()+"/content/"+vol.Disk()) 83 | if err != nil { 84 | return fmt.Errorf("failed to move pvc: %v, vmParams=%+v", err, vmParams) 85 | } 86 | 87 | var taskResponse map[string]interface{} 88 | 89 | if err = json.Unmarshal([]byte(resp), &taskResponse); err != nil { 90 | return fmt.Errorf("failed to parse response: %v", err) 91 | } 92 | 93 | for range 3 { 94 | if _, err = cluster.WaitForCompletion(taskResponse); err != nil { 95 | time.Sleep(2 * time.Second) 96 | 97 | continue 98 | } 99 | 100 | break 101 | } 102 | 103 | if err != nil { 104 | return fmt.Errorf("failed to wait for task completion: %v", err) 105 | } 106 | 107 | cluster.TaskTimeout = oldTimeout 108 | 109 | return nil 110 | } 111 | 112 | // DeleteDisk delete the volume from all nodes. 113 | func DeleteDisk(cluster *pxapi.Client, vol *volume.Volume) error { 114 | data, err := cluster.GetNodeList() 115 | if err != nil { 116 | return fmt.Errorf("failed to get node list: %v", err) 117 | } 118 | 119 | if data["data"] == nil { 120 | return fmt.Errorf("failed to parce node list: %v", err) 121 | } 122 | 123 | id, err := strconv.Atoi(vol.VMID()) 124 | if err != nil { 125 | return fmt.Errorf("failed to parse volume vm id: %v", err) 126 | } 127 | 128 | for _, item := range data["data"].([]interface{}) { //nolint:errcheck 129 | node, ok := item.(map[string]interface{}) 130 | if !ok { 131 | continue 132 | } 133 | 134 | vmr := pxapi.NewVmRef(id) 135 | vmr.SetNode(node["node"].(string)) //nolint:errcheck 136 | vmr.SetVmType("qemu") 137 | 138 | content, err := cluster.GetStorageContent(vmr, vol.Storage()) 139 | if err != nil { 140 | return fmt.Errorf("failed to get storage content: %v", err) 141 | } 142 | 143 | images, ok := content["data"].([]interface{}) 144 | if !ok { 145 | return fmt.Errorf("failed to cast images to map: %v", err) 146 | } 147 | 148 | volid := fmt.Sprintf("%s:%s", vol.Storage(), vol.Disk()) 149 | 150 | for i := range images { 151 | image, ok := images[i].(map[string]interface{}) 152 | if !ok { 153 | return fmt.Errorf("failed to cast image to map: %v", err) 154 | } 155 | 156 | if image["volid"].(string) == volid && image["size"] != nil { //nolint:errcheck 157 | if _, err := cluster.DeleteVolume(vmr, vol.Storage(), vol.Disk()); err != nil { 158 | return fmt.Errorf("failed to delete volume: %s", vol.Disk()) 159 | } 160 | } 161 | } 162 | } 163 | 164 | return nil 165 | } 166 | -------------------------------------------------------------------------------- /pkg/tools/config.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Package tools implements tools to work with kubeernetes. 18 | package tools 19 | 20 | import ( 21 | "fmt" 22 | 23 | "k8s.io/client-go/rest" 24 | "k8s.io/client-go/tools/clientcmd" 25 | ) 26 | 27 | // BuildConfig returns a kubernetes client configuration and namespace. 28 | func BuildConfig(kubeconfig, namespace string) (k *rest.Config, ns string, err error) { 29 | clientConfigLoadingRules := clientcmd.NewDefaultClientConfigLoadingRules() 30 | 31 | if kubeconfig != "" { 32 | clientConfigLoadingRules.ExplicitPath = kubeconfig 33 | } 34 | 35 | config := clientcmd.NewNonInteractiveDeferredLoadingClientConfig( 36 | clientConfigLoadingRules, &clientcmd.ConfigOverrides{}) 37 | 38 | if namespace == "" { 39 | namespace, _, err = config.Namespace() 40 | if err != nil { 41 | return nil, "", fmt.Errorf("failed to get namespace from kubeconfig: %w", err) 42 | } 43 | } 44 | 45 | k, err = config.ClientConfig() 46 | 47 | return k, namespace, err 48 | } 49 | -------------------------------------------------------------------------------- /pkg/tools/nodes.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package tools 18 | 19 | import ( 20 | "context" 21 | "fmt" 22 | 23 | "github.com/sergelogvinov/proxmox-cloud-controller-manager/pkg/provider" 24 | 25 | corev1 "k8s.io/api/core/v1" 26 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 27 | "k8s.io/apimachinery/pkg/types" 28 | clientkubernetes "k8s.io/client-go/kubernetes" 29 | ) 30 | 31 | // CSINodes returns a list of nodes that have the specified CSI driver name. 32 | func CSINodes(ctx context.Context, kclient *clientkubernetes.Clientset, csiDriverName string) ([]string, error) { 33 | nodes := []string{} 34 | 35 | csinodes, err := kclient.StorageV1().CSINodes().List(ctx, metav1.ListOptions{}) 36 | if err != nil { 37 | return nil, fmt.Errorf("failed to list CSINodes: %v", err) 38 | } 39 | 40 | for _, csinode := range csinodes.Items { 41 | for _, driver := range csinode.Spec.Drivers { 42 | if driver.Name == csiDriverName { 43 | nodes = append(nodes, driver.NodeID) 44 | 45 | break 46 | } 47 | } 48 | } 49 | 50 | return nodes, nil 51 | } 52 | 53 | // CondonNodes condones the specified nodes. 54 | func CondonNodes(ctx context.Context, kclient *clientkubernetes.Clientset, nodes []string) ([]string, error) { 55 | cordonedNodes := []string{} 56 | patch := []byte(`{"spec":{"unschedulable":true}}`) 57 | 58 | for _, node := range nodes { 59 | nodeStatus, err := kclient.CoreV1().Nodes().Get(ctx, node, metav1.GetOptions{}) 60 | if err != nil { 61 | return nil, fmt.Errorf("failed to get node status: %v", err) 62 | } 63 | 64 | if !nodeStatus.Spec.Unschedulable { 65 | _, err = kclient.CoreV1().Nodes().Patch(ctx, node, types.MergePatchType, patch, metav1.PatchOptions{}) 66 | if err != nil { 67 | return nil, fmt.Errorf("failed to cordon node: %v", err) 68 | } 69 | 70 | cordonedNodes = append(cordonedNodes, node) 71 | } 72 | } 73 | 74 | return cordonedNodes, nil 75 | } 76 | 77 | // UncondonNodes uncondones the specified nodes. 78 | func UncondonNodes(ctx context.Context, kclient *clientkubernetes.Clientset, nodes []string) error { 79 | patch := []byte(`{"spec":{"unschedulable":false}}`) 80 | 81 | for _, node := range nodes { 82 | _, err := kclient.CoreV1().Nodes().Patch(ctx, node, types.MergePatchType, patch, metav1.PatchOptions{}) 83 | if err != nil { 84 | return fmt.Errorf("failed to uncordon node: %v", err) 85 | } 86 | } 87 | 88 | return nil 89 | } 90 | 91 | // ProxmoxVMIDbyProviderID returns the Proxmox VM ID from the specified kubernetes node name. 92 | func ProxmoxVMIDbyProviderID(_ context.Context, node *corev1.Node) (int, string, error) { 93 | vmID, err := provider.GetVMID(node.Spec.ProviderID) 94 | 95 | return vmID, node.Labels[corev1.LabelTopologyZone], err 96 | } 97 | -------------------------------------------------------------------------------- /pkg/tools/pv.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package tools 18 | 19 | import ( 20 | "context" 21 | "encoding/json" 22 | "fmt" 23 | "time" 24 | 25 | corev1 "k8s.io/api/core/v1" 26 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 27 | "k8s.io/apimachinery/pkg/types" 28 | "k8s.io/apimachinery/pkg/watch" 29 | clientkubernetes "k8s.io/client-go/kubernetes" 30 | ) 31 | 32 | // PVCResources returns the PersistentVolumeClaim and PersistentVolume resources. 33 | func PVCResources(ctx context.Context, clientset *clientkubernetes.Clientset, namespace, pvcName string) (*corev1.PersistentVolumeClaim, *corev1.PersistentVolume, error) { 34 | pvc, err := clientset.CoreV1().PersistentVolumeClaims(namespace).Get(ctx, pvcName, metav1.GetOptions{}) 35 | if err != nil { 36 | return nil, nil, fmt.Errorf("failed to get PersistentVolumeClaims: %v", err) 37 | } 38 | 39 | pv, err := clientset.CoreV1().PersistentVolumes().Get(ctx, pvc.Spec.VolumeName, metav1.GetOptions{}) 40 | if err != nil { 41 | return nil, nil, fmt.Errorf("failed to get PersistentVolumes: %v", err) 42 | } 43 | 44 | return pvc, pv, nil 45 | } 46 | 47 | // PVCPodUsage returns the list of pods and the node that are using the specified PersistentVolumeClaim. 48 | func PVCPodUsage(ctx context.Context, clientset *clientkubernetes.Clientset, namespace, pvcName string) (pods []string, node string, err error) { 49 | podList, err := clientset.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{}) 50 | if err != nil { 51 | return nil, "", fmt.Errorf("failed to list pods: %v", err) 52 | } 53 | 54 | for _, pod := range podList.Items { 55 | if pod.Status.Phase != corev1.PodPending { 56 | for _, volume := range pod.Spec.Volumes { 57 | if volume.PersistentVolumeClaim != nil && volume.PersistentVolumeClaim.ClaimName == pvcName { 58 | pods = append(pods, pod.Name) 59 | node = pod.Spec.NodeName 60 | 61 | break 62 | } 63 | } 64 | } 65 | } 66 | 67 | return pods, node, nil 68 | } 69 | 70 | // PVCCreateOrUpdate creates or updates the specified PersistentVolumeClaim resource. 71 | func PVCCreateOrUpdate( 72 | ctx context.Context, 73 | clientset *clientkubernetes.Clientset, 74 | pvc *corev1.PersistentVolumeClaim, 75 | ) (*corev1.PersistentVolumeClaim, error) { 76 | res, err := clientset.CoreV1().PersistentVolumeClaims(pvc.Namespace).Create(ctx, pvc, metav1.CreateOptions{}) 77 | if err != nil { 78 | patch := corev1.PersistentVolumeClaim{ 79 | Spec: corev1.PersistentVolumeClaimSpec{ 80 | VolumeName: pvc.Spec.VolumeName, 81 | }, 82 | } 83 | 84 | patchBytes, err := json.Marshal(&patch) 85 | if err != nil { 86 | return nil, fmt.Errorf("failed to json.Marshal PVC: %w", err) 87 | } 88 | 89 | return clientset.CoreV1().PersistentVolumeClaims(pvc.Namespace).Patch(ctx, pvc.Name, types.MergePatchType, patchBytes, metav1.PatchOptions{}) 90 | } 91 | 92 | return res, err 93 | } 94 | 95 | // PVWaitDelete waits for the specified PersistentVolume to be deleted. 96 | func PVWaitDelete(ctx context.Context, clientset *clientkubernetes.Clientset, pvName string) error { 97 | _, err := clientset.CoreV1().PersistentVolumes().Get(ctx, pvName, metav1.GetOptions{}) 98 | if err != nil { 99 | return nil //nolint: nilerr 100 | } 101 | 102 | watcher, err := clientset.CoreV1().PersistentVolumes().Watch(ctx, metav1.ListOptions{ 103 | FieldSelector: "metadata.name=" + pvName, 104 | }) 105 | if err != nil { 106 | return err 107 | } 108 | 109 | defer watcher.Stop() 110 | 111 | timeout := time.After(10 * time.Minute) 112 | 113 | for { 114 | select { 115 | case event, ok := <-watcher.ResultChan(): 116 | if !ok { 117 | return fmt.Errorf("watch channel closed unexpectedly") 118 | } 119 | 120 | if event.Type == watch.Deleted { 121 | return nil 122 | } 123 | 124 | case <-timeout: 125 | return fmt.Errorf("timeout waiting for PersistentVolume %s to be deleted", pvName) 126 | } 127 | } 128 | } 129 | -------------------------------------------------------------------------------- /pkg/volume/volume.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Package volume implements the volume ID type and functions. 18 | package volume 19 | 20 | import ( 21 | "fmt" 22 | "strings" 23 | ) 24 | 25 | // Volume is the volume ID type. 26 | type Volume struct { 27 | region string 28 | zone string 29 | storage string 30 | disk string 31 | } 32 | 33 | // NewVolume creates a new volume ID. 34 | func NewVolume(region, zone, storage, disk string) *Volume { 35 | return &Volume{ 36 | region: region, 37 | zone: zone, 38 | storage: storage, 39 | disk: disk, 40 | } 41 | } 42 | 43 | // NewVolumeFromVolumeID creates a new volume ID from a volume magic string. 44 | func NewVolumeFromVolumeID(volume string) (*Volume, error) { 45 | return parseVolumeID(volume) 46 | } 47 | 48 | func parseVolumeID(vol string) (*Volume, error) { 49 | parts := strings.SplitN(vol, "/", 4) 50 | if len(parts) != 4 { 51 | return nil, fmt.Errorf("VolumeID must be in the format of region/zone/storageName/diskName") 52 | } 53 | 54 | return &Volume{ 55 | region: parts[0], 56 | zone: parts[1], 57 | storage: parts[2], 58 | disk: parts[3], 59 | }, nil 60 | } 61 | 62 | // VolumeID function returns the volume magic string. 63 | func (v *Volume) VolumeID() string { 64 | return v.region + "/" + v.zone + "/" + v.storage + "/" + v.disk 65 | } 66 | 67 | // VolumeSharedID function returns the shared volume magic string. 68 | func (v *Volume) VolumeSharedID() string { 69 | return v.region + "//" + v.storage + "/" + v.disk 70 | } 71 | 72 | // Region function returns the region in which the volume was created. 73 | func (v *Volume) Region() string { 74 | return v.region 75 | } 76 | 77 | // Zone function returns the zone in which the volume was created. 78 | func (v *Volume) Zone() string { 79 | return v.zone 80 | } 81 | 82 | // Storage function returns the Proxmox storage name. 83 | func (v *Volume) Storage() string { 84 | return v.storage 85 | } 86 | 87 | // Disk function returns the Proxmox disk name. 88 | func (v *Volume) Disk() string { 89 | return v.disk 90 | } 91 | 92 | // Cluster function returns the cluster name in which the volume was created. 93 | func (v *Volume) Cluster() string { 94 | return v.region 95 | } 96 | 97 | // Node function returns the node name in which the volume was created. 98 | func (v *Volume) Node() string { 99 | return v.zone 100 | } 101 | 102 | // VMID function returns the vmID in which the volume was created. 103 | func (v *Volume) VMID() string { 104 | parts := strings.SplitN(v.disk, "-", 3) 105 | if len(parts) != 3 { 106 | return "" 107 | } 108 | 109 | return parts[1] 110 | } 111 | -------------------------------------------------------------------------------- /pkg/volume/volume_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package volume_test 18 | 19 | import ( 20 | "testing" 21 | 22 | "github.com/stretchr/testify/assert" 23 | 24 | "github.com/sergelogvinov/proxmox-csi-plugin/pkg/volume" 25 | ) 26 | 27 | func TestNewVolume(t *testing.T) { 28 | v := volume.NewVolume("region", "zone", "storage", "disk") 29 | assert.NotNil(t, v) 30 | 31 | assert.Equal(t, "region", v.Cluster()) 32 | assert.Equal(t, "zone", v.Node()) 33 | 34 | assert.Equal(t, "region", v.Region()) 35 | assert.Equal(t, "zone", v.Zone()) 36 | assert.Equal(t, "storage", v.Storage()) 37 | assert.Equal(t, "disk", v.Disk()) 38 | assert.Equal(t, "region/zone/storage/disk", v.VolumeID()) 39 | } 40 | 41 | func TestNewVolumeFromVolumeID(t *testing.T) { 42 | v, err := volume.NewVolumeFromVolumeID("region/zone/storage/vm-1000-disk") 43 | assert.Nil(t, err) 44 | assert.NotNil(t, v) 45 | assert.Equal(t, "region", v.Cluster()) 46 | assert.Equal(t, "zone", v.Node()) 47 | assert.Equal(t, "storage", v.Storage()) 48 | assert.Equal(t, "vm-1000-disk", v.Disk()) 49 | assert.Equal(t, "1000", v.VMID()) 50 | } 51 | 52 | func TestNewVolumeFromVolumeIDWithFolder(t *testing.T) { 53 | v, err := volume.NewVolumeFromVolumeID("region/zone/storage/folder/disk") 54 | assert.Nil(t, err) 55 | assert.NotNil(t, v) 56 | assert.Equal(t, "region", v.Cluster()) 57 | assert.Equal(t, "zone", v.Node()) 58 | assert.Equal(t, "storage", v.Storage()) 59 | assert.Equal(t, "folder/disk", v.Disk()) 60 | } 61 | 62 | func TestNewVolumeFromVolumeIDError(t *testing.T) { 63 | _, err := volume.NewVolumeFromVolumeID("region/storage/disk") 64 | assert.NotNil(t, err) 65 | assert.Equal(t, "VolumeID must be in the format of region/zone/storageName/diskName", err.Error()) 66 | } 67 | -------------------------------------------------------------------------------- /tools/deps-check.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Copyright 2022 The Kubernetes Authors. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | set -o errexit 18 | 19 | # We will check all necessary utils in the image. 20 | # They all have to launch without errors. 21 | 22 | # This utils are using by 23 | # go mod k8s.io/mount-utils 24 | /bin/mount -V 25 | /bin/umount -V 26 | /sbin/blkid -V 27 | /sbin/blockdev -V 28 | /sbin/dumpe2fs -V 29 | /sbin/fsck --version 30 | /sbin/mke2fs -V 31 | /sbin/mkfs.ext4 -V 32 | /sbin/mkfs.xfs -V 33 | /usr/sbin/xfs_io -V 34 | # /sbin/resize2fs 35 | /sbin/xfs_repair -V 36 | /usr/sbin/xfs_growfs -V 37 | 38 | # This utils are using by 39 | # go mod pkg/csi/node.go 40 | /sbin/fstrim -V 41 | /sbin/cryptsetup -V 42 | 43 | # This utils are using by 44 | # go mod k8s.io/cloud-provider-openstack/pkg/util/mount 45 | /bin/udevadm --version 46 | /bin/findmnt -V 47 | -------------------------------------------------------------------------------- /tools/deps.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Copyright 2022 The Kubernetes Authors. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | set -o errexit 18 | set -o nounset 19 | 20 | # 21 | # We will copy all dependencies for CSI Node driver to /dest directory 22 | # all utils are using by csi-plugin 23 | # to format/mount/unmount/resize the volumes. 24 | # 25 | # It is very important to have slim image, 26 | # because it runs as root (privileged mode) on the nodes 27 | # 28 | 29 | DEST=/dest 30 | 31 | copy_deps() { 32 | PROG="$1" 33 | 34 | mkdir -p "${DEST}$(dirname $PROG)" 35 | 36 | if [ -d "${PROG}" ]; then 37 | rsync -av "${PROG}/" "${DEST}${PROG}/" 38 | else 39 | cp -Lv "$PROG" "${DEST}${PROG}" 40 | fi 41 | 42 | if [ -x ${PROG} -o $(/usr/bin/ldd "$PROG" >/dev/null) ]; then 43 | DEPS="$(/usr/bin/ldd "$PROG" | /bin/grep '=>' | /usr/bin/awk '{ print $3 }')" 44 | 45 | for d in $DEPS; do 46 | mkdir -p "${DEST}$(dirname $d)" 47 | cp -Lv "$d" "${DEST}${d}" 48 | done 49 | fi 50 | } 51 | 52 | # This utils are using by 53 | # go mod k8s.io/mount-utils 54 | copy_deps /etc/mke2fs.conf 55 | copy_deps /bin/mount 56 | copy_deps /bin/umount 57 | copy_deps /sbin/blkid 58 | copy_deps /sbin/blockdev 59 | copy_deps /sbin/dumpe2fs 60 | copy_deps /sbin/fsck 61 | copy_deps /sbin/fsck.xfs 62 | cp /sbin/fsck* ${DEST}/sbin/ 63 | copy_deps /sbin/e2fsck 64 | # from pkg e2fsprogs - e2image, e2label, e2scrub and etc. 65 | cp /sbin/e* ${DEST}/sbin/ 66 | copy_deps /sbin/mke2fs 67 | copy_deps /sbin/resize2fs 68 | cp /sbin/mkfs* ${DEST}/sbin/ 69 | copy_deps /sbin/mkfs.xfs 70 | copy_deps /sbin/xfs_repair 71 | copy_deps /usr/sbin/xfs_growfs 72 | copy_deps /usr/sbin/xfs_io 73 | cp /usr/sbin/xfs* ${DEST}/usr/sbin/ 74 | 75 | # This utils are using by 76 | # go mod pkg/csi/node.go 77 | copy_deps /sbin/fstrim 78 | copy_deps /sbin/cryptsetup 79 | ARCH=$(uname -m) 80 | mkdir -p ${DEST}/lib/${ARCH}-linux-gnu && cp /lib/${ARCH}-linux-gnu/libgcc_s.so.* ${DEST}/lib/${ARCH}-linux-gnu/ 81 | 82 | # hack for fsck https://github.com/sergelogvinov/proxmox-csi-plugin/issues/59 83 | copy_deps /bin/true 84 | rm -f ${DEST}/sbin/fsck.xfs 85 | ln -s /bin/true ${DEST}/sbin/fsck.xfs 86 | 87 | # This utils are using by 88 | # go mod k8s.io/cloud-provider-openstack/pkg/util/mount 89 | copy_deps /bin/udevadm 90 | copy_deps /lib/udev/rules.d 91 | copy_deps /bin/findmnt 92 | --------------------------------------------------------------------------------