├── .dockerignore ├── example ├── zfs-namespace.yml ├── pvc-nfs.yml ├── pvc-delete.yml ├── pvc-retain.yml ├── storageclass-hostpath.yml ├── storageclass-nfs.yml └── pod.yml ├── test ├── zfs ├── update-permissions └── provision_integration_test.go ├── charts ├── kubernetes-zfs-provisioner │ ├── test │ │ ├── values │ │ │ ├── deployment_1.yaml │ │ │ ├── storageclass_1.yaml │ │ │ ├── deployment_2.yaml │ │ │ ├── secret_3.yaml │ │ │ ├── secret_1.yaml │ │ │ └── secret_2.yaml │ │ ├── main_test.go │ │ ├── secret_test.go │ │ ├── deployment_test.go │ │ └── storageclass_test.go │ ├── templates │ │ ├── serviceaccount.yaml │ │ ├── secret.yaml │ │ ├── storageclass.yaml │ │ ├── rbac.yaml │ │ ├── _helpers.tpl │ │ └── deployment.yaml │ ├── .helmignore │ ├── Chart.yaml │ ├── README.gotmpl.md │ ├── values.yaml │ └── README.md ├── tools.go ├── charts.mk └── go.mod ├── .github ├── helm-docs-footer.gotmpl.md ├── render-charts.sh ├── workflows │ ├── build.yml │ ├── lint.yml │ ├── chart-test.yml │ ├── test.yml │ ├── chart-lint.yml │ ├── release.yml │ └── chart-release.yml ├── changelog-charts.sh ├── helm-docs-header.gotmpl.md ├── changelog-configuration.json ├── changelog-charts.json └── PULL_REQUEST_TEMPLATE.md ├── .gitignore ├── docker ├── zfs.sh ├── Dockerfile └── update-permissions.sh ├── renovate.json ├── Makefile.vars.mk ├── pkg ├── provisioner │ ├── delete.go │ ├── provisioner_test.go │ ├── provisioner.go │ ├── delete_test.go │ ├── parameters_test.go │ ├── parameters.go │ ├── provision.go │ └── provision_test.go └── zfs │ └── zfs.go ├── Makefile ├── .goreleaser.yml ├── go.mod ├── main.go ├── README.md ├── LICENSE.txt ├── architecture.nfs.drawio.svg ├── go.sum └── architecture.hostpath.drawio.svg /.dockerignore: -------------------------------------------------------------------------------- 1 | .* 2 | * 3 | !kubernetes-zfs-provisioner 4 | !docker/* 5 | -------------------------------------------------------------------------------- /example/zfs-namespace.yml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: zfs-system -------------------------------------------------------------------------------- /test/zfs: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -eo pipefail 4 | 5 | zfs_bin=${ZFS_BIN:-/sbin/zfs} 6 | 7 | ${zfs_bin} ${@} 8 | -------------------------------------------------------------------------------- /charts/kubernetes-zfs-provisioner/test/values/deployment_1.yaml: -------------------------------------------------------------------------------- 1 | env: 2 | KEY1: value 3 | ANOTHER_KEY: another value 4 | -------------------------------------------------------------------------------- /.github/helm-docs-footer.gotmpl.md: -------------------------------------------------------------------------------- 1 | 2 | {{ template "chart.sourcesSection" . }} 3 | 4 | {{ template "chart.requirementsSection" . }} 5 | -------------------------------------------------------------------------------- /charts/kubernetes-zfs-provisioner/test/values/storageclass_1.yaml: -------------------------------------------------------------------------------- 1 | storageClass: 2 | create: true 3 | classes: 4 | - name: test 5 | -------------------------------------------------------------------------------- /charts/kubernetes-zfs-provisioner/test/main_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | var ( 4 | helmChartPath = ".." 5 | releaseName = "test-release" 6 | ) 7 | -------------------------------------------------------------------------------- /charts/kubernetes-zfs-provisioner/test/values/deployment_2.yaml: -------------------------------------------------------------------------------- 1 | ssh: 2 | identities: 3 | id_rsa: | 4 | ----BEGIN---- 5 | ----END---- 6 | -------------------------------------------------------------------------------- /charts/kubernetes-zfs-provisioner/test/values/secret_3.yaml: -------------------------------------------------------------------------------- 1 | ssh: 2 | externalSecretName: "" 3 | knownHosts: 4 | - host: test 5 | pubKey: ssh-rsa asdf 6 | -------------------------------------------------------------------------------- /charts/kubernetes-zfs-provisioner/test/values/secret_1.yaml: -------------------------------------------------------------------------------- 1 | ssh: 2 | externalSecretName: "" 3 | config: | 4 | Host test 5 | IdentityFile ~/.ssh/id_ed25519 6 | -------------------------------------------------------------------------------- /test/update-permissions: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -eo pipefail 4 | 5 | zfs_mod="${ZFS_MOD:-g+w}" 6 | chmod_bin=${ZFS_CHOWN_BIN:-chmod} 7 | 8 | zfs_mountpoint="${1}" 9 | 10 | ${chmod_bin} ${zfs_mod} ${zfs_mountpoint} 11 | -------------------------------------------------------------------------------- /.github/render-charts.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -eo pipefail 4 | 5 | chartYaml="${1}" 6 | chartName=$(dirname "${chartYaml}") 7 | 8 | echo "::group::Render chart ${chartName}" 9 | helm template "${chartName}" 10 | echo "::endgroup::" 11 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Output of the go coverage tool 2 | /*.out 3 | 4 | vendor 5 | *.img 6 | bin 7 | .vscode 8 | .DS_Store 9 | .env 10 | kube.conf 11 | 12 | /dist 13 | /kubernetes-zfs-provisioner 14 | /.zpool 15 | /.work 16 | /.github/release-notes.md 17 | -------------------------------------------------------------------------------- /charts/kubernetes-zfs-provisioner/test/values/secret_2.yaml: -------------------------------------------------------------------------------- 1 | ssh: 2 | externalSecretName: "" 3 | identities: 4 | id_ed25519: | 5 | ---- 6 | PRIVATE_KEY 7 | ---- 8 | id_rsa: | 9 | ---- 10 | PRIVATE_KEY 11 | ---- 12 | -------------------------------------------------------------------------------- /example/pvc-nfs.yml: -------------------------------------------------------------------------------- 1 | kind: PersistentVolumeClaim 2 | apiVersion: v1 3 | metadata: 4 | name: example-nfs 5 | namespace: zfs-system 6 | spec: 7 | storageClassName: zfs-nfs 8 | accessModes: 9 | - ReadWriteMany 10 | resources: 11 | requests: 12 | storage: 128M 13 | -------------------------------------------------------------------------------- /example/pvc-delete.yml: -------------------------------------------------------------------------------- 1 | kind: PersistentVolumeClaim 2 | apiVersion: v1 3 | metadata: 4 | name: example-delete 5 | namespace: zfs-system 6 | spec: 7 | storageClassName: zfs-delete 8 | accessModes: 9 | - ReadWriteMany 10 | resources: 11 | requests: 12 | storage: 128M 13 | -------------------------------------------------------------------------------- /example/pvc-retain.yml: -------------------------------------------------------------------------------- 1 | kind: PersistentVolumeClaim 2 | apiVersion: v1 3 | metadata: 4 | name: example-retain 5 | namespace: zfs-system 6 | spec: 7 | storageClassName: zfs-retain 8 | accessModes: 9 | - ReadWriteMany 10 | resources: 11 | requests: 12 | storage: 128M 13 | -------------------------------------------------------------------------------- /charts/tools.go: -------------------------------------------------------------------------------- 1 | //go:build tools 2 | 3 | // Package tools is a place to put any tooling dependencies as imports. 4 | // Go modules will be forced to download and install them. 5 | package tools 6 | 7 | import ( 8 | // helm-docs 9 | _ "github.com/norwoodj/helm-docs/cmd/helm-docs" 10 | ) 11 | -------------------------------------------------------------------------------- /docker/zfs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -eo pipefail 4 | 5 | zfs_bin=${ZFS_BIN:-sudo -H zfs} 6 | 7 | # Do not try to manually modify these Env vars, they will be updated by the provisioner just before invoking the script. 8 | zfs_host="${ZFS_HOST}" 9 | 10 | ssh "${zfs_host}" "${zfs_bin} ${*}" 11 | -------------------------------------------------------------------------------- /example/storageclass-hostpath.yml: -------------------------------------------------------------------------------- 1 | kind: StorageClass 2 | apiVersion: storage.k8s.io/v1 3 | metadata: 4 | name: zfs-retain 5 | provisioner: pv.kubernetes.io/zfs 6 | reclaimPolicy: Retain 7 | parameters: 8 | parentDataset: tank/services/kubernetes 9 | hostname: zfs-host.intern 10 | type: hostpath 11 | node: zfs-host 12 | -------------------------------------------------------------------------------- /example/storageclass-nfs.yml: -------------------------------------------------------------------------------- 1 | kind: StorageClass 2 | apiVersion: storage.k8s.io/v1 3 | metadata: 4 | name: zfs-nfs 5 | provisioner: pv.kubernetes.io/zfs 6 | reclaimPolicy: Delete 7 | parameters: 8 | parentDataset: tank/services/kubernetes 9 | hostname: pve-2.intern 10 | type: nfs 11 | shareProperties: rw,no_root_squash 12 | -------------------------------------------------------------------------------- /docker/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM docker.io/library/alpine:3.20 as runtime 2 | 3 | ENTRYPOINT ["kubernetes-zfs-provisioner"] 4 | 5 | RUN \ 6 | apk add --no-cache curl bash openssh && \ 7 | adduser -S zfs -G root 8 | 9 | COPY docker/zfs.sh /usr/bin/zfs 10 | COPY docker/update-permissions.sh /usr/bin/update-permissions 11 | COPY kubernetes-zfs-provisioner /usr/bin/ 12 | 13 | USER zfs:root 14 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: Build 2 | 3 | on: 4 | pull_request: 5 | branches: 6 | - master 7 | 8 | jobs: 9 | docker: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v4 13 | 14 | - uses: actions/setup-go@v5 15 | with: 16 | go-version-file: go.mod 17 | 18 | - name: Build 19 | run: make build:docker 20 | -------------------------------------------------------------------------------- /.github/workflows/lint.yml: -------------------------------------------------------------------------------- 1 | name: Lint 2 | 3 | on: 4 | pull_request: {} 5 | push: 6 | branches: 7 | - master 8 | 9 | jobs: 10 | lint: 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/checkout@v4 14 | 15 | - uses: actions/setup-go@v5 16 | with: 17 | go-version-file: go.mod 18 | 19 | - name: Run linters 20 | run: make lint 21 | -------------------------------------------------------------------------------- /docker/update-permissions.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -eo pipefail 4 | 5 | zfs_mod="${ZFS_MOD:-g+w}" 6 | chmod_bin=${ZFS_CHOWN_BIN:-sudo -H chmod} 7 | 8 | zfs_mountpoint="${1}" 9 | 10 | # Do not try to manually modify these Env vars, they will be updated by the provisioner just before invoking the script. 11 | zfs_host="${ZFS_HOST}" 12 | 13 | ssh "${zfs_host}" "${chmod_bin} ${zfs_mod} ${zfs_mountpoint}" 14 | -------------------------------------------------------------------------------- /.github/changelog-charts.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -eo pipefail 4 | 5 | chart="${1}" 6 | 7 | tagPattern="${chart}-(.+)" 8 | chartLabel="chart:${chart}" 9 | 10 | echo ::group::Configuring changelog generator 11 | jq '.tag_resolver.filter.pattern="'$tagPattern'" | .tag_resolver.transformer.pattern="'$tagPattern'" | .categories[].labels += ["'$chartLabel'"]' \ 12 | .github/changelog-charts.json | tee .github/configuration.json 13 | echo ::endgroup:: 14 | -------------------------------------------------------------------------------- /.github/workflows/chart-test.yml: -------------------------------------------------------------------------------- 1 | name: ChartTest 2 | 3 | on: 4 | pull_request: 5 | # only run when there are chart changes 6 | paths: 7 | - 'charts/**' 8 | 9 | jobs: 10 | test: 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/checkout@v4 14 | 15 | - uses: actions/setup-go@v5 16 | with: 17 | go-version-file: charts/go.mod 18 | 19 | - name: Run chart tests 20 | run: make chart-test 21 | -------------------------------------------------------------------------------- /charts/kubernetes-zfs-provisioner/templates/serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.serviceAccount.create -}} 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | name: {{ include "kubernetes-zfs-provisioner.serviceAccountName" . }} 6 | labels: 7 | {{- include "kubernetes-zfs-provisioner.labels" . | nindent 4 }} 8 | {{- with .Values.serviceAccount.annotations }} 9 | annotations: 10 | {{- toYaml . | nindent 4 }} 11 | {{- end }} 12 | {{- end -}} 13 | -------------------------------------------------------------------------------- /renovate.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": [ 3 | "config:base", 4 | ":gitSignOff", 5 | ":disableDependencyDashboard" 6 | ], 7 | "labels": [ 8 | "kind:dependency" 9 | ], 10 | "postUpdateOptions": [ 11 | "gomodTidy" 12 | ], 13 | "prBodyNotes": [ 14 | "- [ ] PR contains the label that identifies the area, one of: `area:provisioner`, `area:chart`\n- [ ] If the PR is targeting a Helm chart, add the chart label, e.g. `chart:kubernetes-zfs-provisioner`" 15 | ] 16 | } 17 | -------------------------------------------------------------------------------- /Makefile.vars.mk: -------------------------------------------------------------------------------- 1 | PROJECT_DIR ?= $(shell pwd) 2 | WORK_DIR = $(PROJECT_DIR)/.work 3 | 4 | ZPOOL_SIZE=1 # in GB 5 | zpool_dir := .zpool 6 | zpool_disk := $(zpool_dir)/zpool.img 7 | zpool_name_file := $(zpool_dir)/zpool.nfo 8 | zpool_name := $(shell bash -c "cat .zpool/zpool.nfo || echo test$$RANDOM") 9 | zfs_dataset := $(zpool_name)/zfs-provisioner 10 | 11 | binary ?= kubernetes-zfs-provisioner 12 | 13 | IMAGE_REGISTRY ?= ghcr.io 14 | IMAGE_REPOSITORY ?= $(IMAGE_REGISTRY)/ccremer/zfs-provisioner 15 | IMAGE_TAG ?= latest 16 | -------------------------------------------------------------------------------- /charts/kubernetes-zfs-provisioner/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *.orig 18 | *~ 19 | # Various IDEs 20 | .project 21 | .idea/ 22 | *.tmproj 23 | .vscode/ 24 | 25 | # Unit tests 26 | test/ 27 | *gotmpl* 28 | 29 | -------------------------------------------------------------------------------- /.github/helm-docs-header.gotmpl.md: -------------------------------------------------------------------------------- 1 | {{ template "chart.header" . }} 2 | {{ template "chart.deprecationWarning" . }} 3 | 4 | {{ template "chart.versionBadge" . }}{{ template "chart.typeBadge" . }}{{ template "chart.appVersionBadge" . }} 5 | 6 | {{ template "chart.description" . }} 7 | 8 | {{ template "chart.homepageLine" . }} 9 | 10 | ## Installation 11 | 12 | ```bash 13 | helm repo add zfs-provisioner https://ccremer.github.io/kubernetes-zfs-provisioner 14 | helm install {{ template "chart.name" . }} zfs-provisioner/{{ template "chart.name" . }} 15 | ``` 16 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | tags-ignore: 8 | - "*" 9 | paths-ignore: 10 | - charts/** 11 | - docs/** 12 | pull_request: 13 | branches: 14 | - master 15 | paths-ignore: 16 | - charts/** 17 | - docs/** 18 | 19 | jobs: 20 | test: 21 | runs-on: ubuntu-latest 22 | steps: 23 | - uses: actions/checkout@v4 24 | 25 | - uses: actions/setup-go@v5 26 | with: 27 | go-version-file: go.mod 28 | 29 | - name: Run unit tests 30 | run: make test 31 | 32 | - name: Run integration tests 33 | run: make install:zfs test:integration 34 | -------------------------------------------------------------------------------- /charts/kubernetes-zfs-provisioner/templates/secret.yaml: -------------------------------------------------------------------------------- 1 | {{- if not .Values.ssh.externalSecretName -}} 2 | apiVersion: v1 3 | kind: Secret 4 | metadata: 5 | name: {{ include "kubernetes-zfs-provisioner.fullname" . }} 6 | labels: 7 | {{- include "kubernetes-zfs-provisioner.labels" . | nindent 4 }} 8 | stringData: 9 | {{- with .Values.ssh.knownHosts }} 10 | known_hosts: | 11 | {{- range . }} 12 | {{ .host }} {{ .pubKey }} 13 | {{ end }} 14 | {{- end }} 15 | {{- with .Values.ssh.config }} 16 | config: | 17 | {{- . | nindent 4 }} 18 | {{- end }} 19 | {{- range $key,$value := .Values.ssh.identities }} 20 | {{ $key -}}: {{ $value | toYaml | indent 2 }} 21 | {{- end }} 22 | {{- end }} 23 | -------------------------------------------------------------------------------- /.github/workflows/chart-lint.yml: -------------------------------------------------------------------------------- 1 | name: ChartLint 2 | 3 | on: 4 | pull_request: 5 | # only run when there are chart changes 6 | paths: 7 | - 'charts/**' 8 | - '!charts/charts.mk' 9 | - '!charts/go*' 10 | 11 | jobs: 12 | lint: 13 | runs-on: ubuntu-latest 14 | steps: 15 | - uses: actions/checkout@v4 16 | 17 | - uses: actions/setup-go@v5 18 | with: 19 | go-version-file: charts/go.mod 20 | 21 | - name: Verify charts are upt-do-date 22 | run: make chart-lint 23 | 24 | template: 25 | runs-on: ubuntu-latest 26 | steps: 27 | - uses: actions/checkout@v4 28 | 29 | - name: Render Helm charts 30 | run: find charts -type f -name Chart.yaml -exec .github/render-charts.sh {} \; 31 | -------------------------------------------------------------------------------- /example/pod.yml: -------------------------------------------------------------------------------- 1 | kind: Pod 2 | apiVersion: v1 3 | metadata: 4 | name: zfs-example 5 | namespace: zfs-system 6 | spec: 7 | securityContext: 8 | #runAsUser: 1000 9 | fsGroup: 0 10 | #runAsGroup: 1000 11 | containers: 12 | - name: example 13 | image: busybox 14 | securityContext: 15 | runAsUser: 1000 16 | command: 17 | - "/bin/sh" 18 | env: 19 | - name: DIR 20 | value: /tmp/pv 21 | - name: FILE 22 | value: /tmp/pv/SUCCESS 23 | args: 24 | - "-c" 25 | - "id; mount; ls -lah ${DIR}; sleep 5; date > ${FILE} && chmod +w ${FILE}" 26 | volumeMounts: 27 | - name: zfs-volume 28 | mountPath: "/tmp/pv" 29 | restartPolicy: "Never" 30 | volumes: 31 | - name: zfs-volume 32 | persistentVolumeClaim: 33 | claimName: example-nfs 34 | -------------------------------------------------------------------------------- /charts/kubernetes-zfs-provisioner/templates/storageclass.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.storageClass.create -}} 2 | {{- range .Values.storageClass.classes }} 3 | --- 4 | kind: StorageClass 5 | apiVersion: storage.k8s.io/v1 6 | metadata: 7 | name: {{ .name }} 8 | labels: 9 | {{- include "kubernetes-zfs-provisioner.labels" $ | nindent 4 }} 10 | {{- with .annotations }} 11 | annotations: 12 | {{ toYaml . | nindent 4 }} 13 | {{- end }} 14 | provisioner: {{ $.Values.provisioner.instance }} 15 | reclaimPolicy: {{ .policy | default "Delete" }} 16 | parameters: 17 | parentDataset: {{ .parentDataset }} 18 | hostname: {{ .hostName }} 19 | type: {{ .type | default "nfs" }} 20 | node: {{ .node | default "''" }} 21 | shareProperties: {{ .shareProperties | default "''" }} 22 | {{- if kindIs "bool" .reserveSpace }} 23 | reserveSpace: {{ quote .reserveSpace }} 24 | {{- end }} 25 | {{- end }} 26 | {{- end }} 27 | -------------------------------------------------------------------------------- /.github/changelog-configuration.json: -------------------------------------------------------------------------------- 1 | { 2 | "pr_template": "- ${{TITLE}} (#${{NUMBER}})", 3 | "categories": [ 4 | { 5 | "title": "## 🚀 Features", 6 | "labels": [ 7 | "kind:enhancement" 8 | ] 9 | }, 10 | { 11 | "title": "## 🛠️ Minor Changes", 12 | "labels": [ 13 | "kind:change" 14 | ] 15 | }, 16 | { 17 | "title": "## 🔎 Breaking Changes", 18 | "labels": [ 19 | "kind:breaking" 20 | ] 21 | }, 22 | { 23 | "title": "## 🐛 Fixes", 24 | "labels": [ 25 | "kind:bug" 26 | ] 27 | }, 28 | { 29 | "title": "## 📄 Documentation", 30 | "labels": [ 31 | "kind:documentation" 32 | ] 33 | }, 34 | { 35 | "title": "## 🔗 Dependency Updates", 36 | "labels": [ 37 | "kind:dependency" 38 | ] 39 | } 40 | ], 41 | "template": "${{CATEGORIZED_COUNT}} changes since ${{FROM_TAG}}\n\n${{CHANGELOG}}" 42 | } 43 | -------------------------------------------------------------------------------- /pkg/provisioner/delete.go: -------------------------------------------------------------------------------- 1 | package provisioner 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/ccremer/kubernetes-zfs-provisioner/pkg/zfs" 8 | core "k8s.io/api/core/v1" 9 | ) 10 | 11 | // Delete removes a given volume from the server 12 | func (p *ZFSProvisioner) Delete(ctx context.Context, volume *core.PersistentVolume) error { 13 | for _, annotation := range []string{DatasetPathAnnotation, ZFSHostAnnotation} { 14 | value := volume.ObjectMeta.Annotations[annotation] 15 | if value == "" { 16 | return fmt.Errorf("annotation '%s' not found or empty, cannot determine which ZFS dataset to destroy", annotation) 17 | } 18 | } 19 | datasetPath := volume.ObjectMeta.Annotations[DatasetPathAnnotation] 20 | zfsHost := volume.ObjectMeta.Annotations[ZFSHostAnnotation] 21 | 22 | err := p.zfs.DestroyDataset(&zfs.Dataset{Name: datasetPath, Hostname: zfsHost}, zfs.DestroyRecursively) 23 | if err != nil { 24 | return fmt.Errorf("error destroying dataset: %w", err) 25 | } 26 | 27 | p.log.Info("dataset destroyed", "dataset", datasetPath) 28 | return nil 29 | } 30 | -------------------------------------------------------------------------------- /pkg/provisioner/provisioner_test.go: -------------------------------------------------------------------------------- 1 | package provisioner 2 | 3 | import ( 4 | "github.com/stretchr/testify/mock" 5 | 6 | "github.com/ccremer/kubernetes-zfs-provisioner/pkg/zfs" 7 | ) 8 | 9 | type ( 10 | zfsStub struct { 11 | mock.Mock 12 | } 13 | ) 14 | 15 | func (z *zfsStub) GetDataset(name string, hostname string) (*zfs.Dataset, error) { 16 | args := z.Called(name, hostname) 17 | return args.Get(0).(*zfs.Dataset), args.Error(1) 18 | } 19 | 20 | func (z *zfsStub) CreateDataset(name string, hostname string, properties map[string]string) (*zfs.Dataset, error) { 21 | args := z.Called(name, properties) 22 | return args.Get(0).(*zfs.Dataset), args.Error(1) 23 | } 24 | 25 | func (z *zfsStub) DestroyDataset(dataset *zfs.Dataset, flag zfs.DestroyFlag) error { 26 | args := z.Called(dataset, flag) 27 | return args.Error(0) 28 | } 29 | 30 | func (z *zfsStub) SetPermissions(dataset *zfs.Dataset) error { 31 | args := z.Called(dataset) 32 | return args.Error(0) 33 | } 34 | 35 | func NewZFSProvisionerStub(stub *zfsStub) (*ZFSProvisioner, error) { 36 | return &ZFSProvisioner{ 37 | zfs: stub, 38 | InstanceName: "test", 39 | }, nil 40 | } 41 | -------------------------------------------------------------------------------- /pkg/provisioner/provisioner.go: -------------------------------------------------------------------------------- 1 | package provisioner 2 | 3 | import ( 4 | "github.com/ccremer/kubernetes-zfs-provisioner/pkg/zfs" 5 | "k8s.io/klog/v2" 6 | ) 7 | 8 | const ( 9 | DatasetPathAnnotation = "zfs.pv.kubernetes.io/zfs-dataset-path" 10 | ZFSHostAnnotation = "zfs.pv.kubernetes.io/zfs-host" 11 | 12 | RefQuotaProperty = "refquota" 13 | RefReservationProperty = "refreservation" 14 | ShareNfsProperty = "sharenfs" 15 | ManagedByProperty = "io.kubernetes.pv.zfs:managed_by" 16 | ReclaimPolicyProperty = "io.kubernetes.pv.zfs:reclaim_policy" 17 | ) 18 | 19 | // ZFSProvisioner implements the Provisioner interface to create and export ZFS volumes 20 | type ZFSProvisioner struct { 21 | zfs zfs.Interface 22 | log klog.Logger 23 | InstanceName string 24 | } 25 | 26 | // NewZFSProvisioner returns a new ZFSProvisioner based on a given optional 27 | // zap.Logger. If none given, zaps default production logger is used. 28 | func NewZFSProvisioner(instanceName string, logger klog.Logger) (*ZFSProvisioner, error) { 29 | return &ZFSProvisioner{ 30 | log: logger, 31 | zfs: zfs.NewInterface(), InstanceName: instanceName, 32 | }, nil 33 | } 34 | -------------------------------------------------------------------------------- /charts/charts.mk: -------------------------------------------------------------------------------- 1 | helm_docs_bin := $(WORK_DIR)/helm-docs 2 | 3 | # Prepare binary 4 | # We need to set the Go arch since the binary is meant for the user's OS. 5 | $(helm_docs_bin): export GOOS = $(shell go env GOOS) 6 | $(helm_docs_bin): export GOARCH = $(shell go env GOARCH) 7 | $(helm_docs_bin): 8 | @mkdir -p $(WORK_DIR) 9 | cd charts && go build -o $@ github.com/norwoodj/helm-docs/cmd/helm-docs 10 | 11 | # This executes `make clean prepare` for every dir found in charts/ that has a Makefile. 12 | .PHONY: chart-prepare 13 | chart-prepare: ## Prepare the Helm charts 14 | @find charts -type f -name Makefile | sed 's|/[^/]*$$||' | xargs -I '%' make -C '%' clean prepare 15 | 16 | .PHONY: chart-docs 17 | chart-docs: $(helm_docs_bin) ## Creates the Chart READMEs from template and values.yaml files 18 | @$(helm_docs_bin) \ 19 | --template-files ./.github/helm-docs-header.gotmpl.md \ 20 | --template-files README.gotmpl.md \ 21 | --template-files ./.github/helm-docs-footer.gotmpl.md 22 | 23 | .PHONY: chart-lint 24 | chart-lint: chart-prepare chart-docs ## Lint charts 25 | @echo 'Check for uncommitted changes ...' 26 | git diff --exit-code 27 | 28 | .PHONY: chart-test 29 | chart-test: ## Run unit tests for charts 30 | cd charts && go test ./... 31 | -------------------------------------------------------------------------------- /charts/kubernetes-zfs-provisioner/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: kubernetes-zfs-provisioner 3 | description: Dynamic ZFS persistent volume provisioner for Kubernetes 4 | 5 | # A chart can be either an 'application' or a 'library' chart. 6 | # 7 | # Application charts are a collection of templates that can be packaged into versioned archives 8 | # to be deployed. 9 | # 10 | # Library charts provide useful utilities or functions for the chart developer. They're included as 11 | # a dependency of application charts to inject those utilities and functions into the rendering 12 | # pipeline. Library charts do not define any templates and therefore cannot be deployed. 13 | # type: application 14 | 15 | # This is the chart version. This version number should be incremented each time you make changes 16 | # to the chart and its templates, including the app version. 17 | version: 2.2.1 18 | 19 | # This is the version number of the application being deployed. This version number should be 20 | # incremented each time you make changes to the application. 21 | #appVersion: see image.tag 22 | 23 | # Licensing of https://commons.wikimedia.org/wiki/File:Openzfs.svg applies 24 | icon: https://upload.wikimedia.org/wikipedia/commons/thumb/7/7f/Openzfs.svg/257px-Openzfs.svg.png 25 | -------------------------------------------------------------------------------- /.github/changelog-charts.json: -------------------------------------------------------------------------------- 1 | { 2 | "pr_template": "- ${{TITLE}} (#${{NUMBER}})", 3 | "categories": [ 4 | { 5 | "title": "## 🚀 Features", 6 | "labels": [ 7 | "kind:enhancement", 8 | "area:chart" 9 | ], 10 | "exhaustive": true 11 | }, 12 | { 13 | "title": "## 🛠️ Minor Changes", 14 | "labels": [ 15 | "kind:change", 16 | "area:chart" 17 | ], 18 | "exhaustive": true 19 | }, 20 | { 21 | "title": "## 🔎 Breaking Changes", 22 | "labels": [ 23 | "kind:breaking", 24 | "area:chart" 25 | ], 26 | "exhaustive": true 27 | }, 28 | { 29 | "title": "## 🐛 Fixes", 30 | "labels": [ 31 | "kind:bug", 32 | "area:chart" 33 | ], 34 | "exhaustive": true 35 | }, 36 | { 37 | "title": "## 📄 Documentation", 38 | "labels": [ 39 | "kind:documentation", 40 | "area:chart" 41 | ], 42 | "exhaustive": true 43 | }, 44 | { 45 | "title": "## 🔗 Dependency Updates", 46 | "labels": [ 47 | "kind:dependency", 48 | "area:chart" 49 | ], 50 | "exhaustive": true 51 | } 52 | ], 53 | "template": "This release contains _only_ Helm chart changes\n\n${{CATEGORIZED_COUNT}} changes since ${{FROM_TAG}}\n\n${{CHANGELOG}}" 54 | } 55 | -------------------------------------------------------------------------------- /pkg/provisioner/delete_test.go: -------------------------------------------------------------------------------- 1 | package provisioner 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | gozfs "github.com/mistifyio/go-zfs/v3" 8 | "github.com/stretchr/testify/assert" 9 | "github.com/stretchr/testify/require" 10 | core "k8s.io/api/core/v1" 11 | v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 12 | 13 | "github.com/ccremer/kubernetes-zfs-provisioner/pkg/zfs" 14 | ) 15 | 16 | func TestDelete_GivenVolume_WhenAnnotationCorrect_ThenDeleteZfsDataset(t *testing.T) { 17 | expectedDataset := "test/volumes/pv-testcreate" 18 | expectedHost := "host" 19 | dataset := &zfs.Dataset{ 20 | Name: expectedDataset, 21 | Hostname: expectedHost, 22 | } 23 | stub := new(zfsStub) 24 | stub.On("DestroyDataset", dataset, zfs.DestroyFlag(gozfs.DestroyRecursive)). 25 | Return(nil) 26 | p, _ := NewZFSProvisionerStub(stub) 27 | pv := core.PersistentVolume{ObjectMeta: v1.ObjectMeta{Annotations: map[string]string{ 28 | DatasetPathAnnotation: expectedDataset, 29 | ZFSHostAnnotation: expectedHost, 30 | }}} 31 | result := p.Delete(context.Background(), &pv) 32 | require.NoError(t, result) 33 | stub.AssertExpectations(t) 34 | } 35 | 36 | func TestDelete_GivenVolume_WhenAnnotationMissing_ThenThrowError(t *testing.T) { 37 | stub := new(zfsStub) 38 | p, _ := NewZFSProvisionerStub(stub) 39 | pv := core.PersistentVolume{} 40 | err := p.Delete(context.Background(), &pv) 41 | require.Error(t, err) 42 | stub.AssertExpectations(t) 43 | assert.Contains(t, err.Error(), "annotation") 44 | } 45 | -------------------------------------------------------------------------------- /charts/kubernetes-zfs-provisioner/README.gotmpl.md: -------------------------------------------------------------------------------- 1 | 2 | 7 | 8 | 13 | {{ template "chart.valuesSection" . }} 14 | 15 | ## Upgrading from 0.x to 1.x charts 16 | 17 | There are some breaking changes from 0.x to 1.x versions. 18 | 19 | * The `storageclass.classes` array is now empty. 20 | Where it previously contained an example, the example is removed as a default value. 21 | The example is still in `values.yaml` in form of YAML comments. 22 | * The `image.registry` has changed from `docker.io` to `quay.io` due to Docker Hub's pull limit. 23 | * Bumped `image.tag` to `v1.0.0` 24 | 25 | ## Upgrading from 1.x to 2.x charts 26 | 27 | Due to the migration of the [chart from ccremer/charts](https://github.com/ccremer/charts/tree/master/charts/kubernetes-zfs-provisioner) to this repo, a breaking change was made for the chart. 28 | Only chart archives from version 2.x can be downloaded from the https://ccremer.github.io/kubernetes-zfs-provisioner index. 29 | No 1.x or 0.x chart releases will be migrated from the `ccremer/charts` Helm repo. 30 | 31 | * The `image.registry` has changed from `quay.io` to `ghcr.io`. 32 | * The `image.tag` has changed from to `v1.1.0` to `v1`. 33 | * The `image.pullPolicy` has changed from `IfNotPresent` to `Always`. 34 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release 2 | 3 | on: 4 | push: 5 | tags: 6 | - "v*" 7 | 8 | jobs: 9 | goreleaser: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v4 13 | with: 14 | fetch-depth: 0 15 | 16 | - uses: actions/setup-go@v5 17 | with: 18 | go-version-file: go.mod 19 | 20 | - name: Set up QEMU 21 | uses: docker/setup-qemu-action@v3 22 | - name: Set up Docker Buildx 23 | uses: docker/setup-buildx-action@v3 24 | 25 | - name: Login to ghcr.io 26 | uses: docker/login-action@v3 27 | with: 28 | registry: ghcr.io 29 | username: ${{ github.repository_owner }} 30 | password: ${{ secrets.GITHUB_TOKEN }} 31 | 32 | - name: Build changelog from PRs with labels 33 | id: build_changelog 34 | uses: mikepenz/release-changelog-builder-action@v4 35 | with: 36 | configuration: ".github/changelog-configuration.json" 37 | # PreReleases still get a changelog, but the next full release gets a diff since the last full release, 38 | # combining possible changelogs of all previous PreReleases in between. PreReleases show a partial changelog 39 | # since last PreRelease. 40 | ignorePreReleases: "${{ !contains(github.ref, '-rc') }}" 41 | outputFile: .github/release-notes.md 42 | env: 43 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 44 | - uses: goreleaser/goreleaser-action@v6 45 | with: 46 | args: release --release-notes .github/release-notes.md 47 | env: 48 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 49 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ## Summary 2 | 3 | * Short summary of what's included in the PR 4 | * Give special note to breaking changes: List the exact changes or provide links to documentation. 5 | 6 | ## Checklist 7 | 8 | 12 | 13 | ### For Code changes 14 | 15 | - [ ] Categorize the PR by setting a good title and adding one of the labels: 16 | `kind:bug`, `kind:enhancement`, `kind:documentation`, `kind:change`, `kind:breaking`, `kind:dependency` 17 | as they show up in the changelog 18 | - [ ] PR contains the label `area:provisioner` 19 | - [ ] Link this PR to related issues 20 | - [ ] I have not made _any_ changes in the `charts/` directory. 21 | 22 | ### For Helm Chart changes 23 | 24 | - [ ] Categorize the PR by setting a good title and adding one of the labels: 25 | `kind:bug`, `kind:enhancement`, `kind:documentation`, `kind:change`, `kind:breaking`, `kind:dependency` 26 | as they show up in the changelog 27 | - [ ] PR contains the label `area:chart` 28 | - [ ] PR contains the chart label, e.g. `chart:kubernetes-zfs-provisioner` 29 | - [ ] Variables are documented in the values.yaml using the format required by [Helm-Docs](https://github.com/norwoodj/helm-docs#valuesyaml-metadata). 30 | - [ ] Chart Version bumped if immediate release after merging is planned 31 | - [ ] I have run `make chart-docs` 32 | - [ ] Link this PR to related code release or other issues. 33 | 34 | 41 | -------------------------------------------------------------------------------- /charts/kubernetes-zfs-provisioner/test/secret_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/gruntwork-io/terratest/modules/helm" 7 | "github.com/stretchr/testify/assert" 8 | v1 "k8s.io/api/core/v1" 9 | ) 10 | 11 | var tplSecret = []string{"templates/secret.yaml"} 12 | 13 | func Test_Secret_GivenNoExternalSecret_WhenConfigSet_ThenRenderConfigFile(t *testing.T) { 14 | options := &helm.Options{ 15 | ValuesFiles: []string{"values/secret_1.yaml"}, 16 | } 17 | 18 | output := helm.RenderTemplate(t, options, helmChartPath, releaseName, tplSecret) 19 | 20 | var secret v1.Secret 21 | helm.UnmarshalK8SYaml(t, output, &secret) 22 | 23 | config := secret.StringData["config"] 24 | assert.Equal(t, `Host test 25 | IdentityFile ~/.ssh/id_ed25519`, config) 26 | } 27 | 28 | func Test_Secret_GivenNoExternalSecret_WhenIdentitiesSet_ThenRenderPrivateKeys(t *testing.T) { 29 | options := &helm.Options{ 30 | ValuesFiles: []string{"values/secret_2.yaml"}, 31 | } 32 | 33 | output := helm.RenderTemplate(t, options, helmChartPath, releaseName, tplSecret) 34 | 35 | var secret v1.Secret 36 | helm.UnmarshalK8SYaml(t, output, &secret) 37 | 38 | expected := "----\nPRIVATE_KEY\n----" 39 | 40 | assert.Contains(t, secret.StringData["id_rsa"], expected) 41 | assert.Contains(t, secret.StringData["id_ed25519"], expected) 42 | } 43 | 44 | func Test_Secret_GivenNoExternalSecret_WhenKnownHostsSet_ThenRenderHostKeys(t *testing.T) { 45 | options := &helm.Options{ 46 | ValuesFiles: []string{"values/secret_3.yaml"}, 47 | } 48 | 49 | output := helm.RenderTemplate(t, options, helmChartPath, releaseName, tplSecret) 50 | 51 | var secret v1.Secret 52 | helm.UnmarshalK8SYaml(t, output, &secret) 53 | 54 | expectedHost := "test" 55 | expectedPubKey := "ssh-rsa asdf" 56 | 57 | assert.Contains(t, secret.StringData["known_hosts"], expectedHost+" "+expectedPubKey) 58 | } 59 | -------------------------------------------------------------------------------- /charts/kubernetes-zfs-provisioner/test/deployment_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "testing" 5 | "unsafe" 6 | 7 | "github.com/stretchr/testify/assert" 8 | "github.com/stretchr/testify/require" 9 | v1 "k8s.io/api/core/v1" 10 | 11 | "github.com/gruntwork-io/terratest/modules/helm" 12 | appv1 "k8s.io/api/apps/v1" 13 | ) 14 | 15 | var tplDeployment = []string{"templates/deployment.yaml"} 16 | 17 | func Test_Deployment_ShouldRender_EnvironmentVariables(t *testing.T) { 18 | options := &helm.Options{ 19 | ValuesFiles: []string{"values/deployment_1.yaml"}, 20 | } 21 | 22 | expectedKeys := []string{"KEY1", "ANOTHER_KEY", "ZFS_PROVISIONER_INSTANCE"} 23 | expectedValues := []string{"value", "another value", "pv.kubernetes.io/zfs"} 24 | 25 | output := helm.RenderTemplate(t, options, helmChartPath, releaseName, tplDeployment) 26 | 27 | var deployment appv1.Deployment 28 | helm.UnmarshalK8SYaml(t, output, &deployment) 29 | 30 | envs := deployment.Spec.Template.Spec.Containers[0].Env 31 | assert.Equal(t, len(envs), 3) 32 | for i, _ := range envs { 33 | require.Contains(t, envs, v1.EnvVar{ 34 | Name: expectedKeys[i], 35 | Value: expectedValues[i], 36 | }) 37 | } 38 | } 39 | 40 | func Test_Deployment_ShouldRender_SshVolumes_IfEnabled(t *testing.T) { 41 | options := &helm.Options{ 42 | ValuesFiles: []string{"values/deployment_2.yaml"}, 43 | } 44 | 45 | output := helm.RenderTemplate(t, options, helmChartPath, releaseName, tplDeployment) 46 | 47 | var deployment appv1.Deployment 48 | helm.UnmarshalK8SYaml(t, output, &deployment) 49 | 50 | volumeMounts := deployment.Spec.Template.Spec.Containers[0].VolumeMounts 51 | require.Contains(t, volumeMounts, v1.VolumeMount{ 52 | Name: "ssh", 53 | MountPath: "/home/zfs/.ssh", 54 | }) 55 | 56 | volumes := deployment.Spec.Template.Spec.Volumes 57 | require.Contains(t, volumes, v1.Volume{ 58 | Name: "ssh", 59 | VolumeSource: v1.VolumeSource{ 60 | Secret: &v1.SecretVolumeSource{ 61 | SecretName: releaseName + "-kubernetes-zfs-provisioner", 62 | DefaultMode: getIntPointer(0600), 63 | }, 64 | }, 65 | }) 66 | } 67 | 68 | func getIntPointer(mode int) *int32 { 69 | i := *((*int32)(unsafe.Pointer(&mode))) 70 | return &i 71 | } 72 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | SHELL := /usr/bin/env bash 2 | 3 | # Disable built-in rules 4 | MAKEFLAGS += --no-builtin-rules 5 | MAKEFLAGS += --no-builtin-variables 6 | .SUFFIXES: 7 | .SECONDARY: 8 | .DEFAULT_GOAL := help 9 | 10 | include Makefile.vars.mk 11 | -include charts/charts.mk 12 | 13 | .PHONY: help 14 | help: ## Show this help 15 | @grep -E -h '\s##\s' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = "(: ).*?## "}; {gsub(/\\:/,":",$$1)}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' 16 | 17 | .PHONY: build 18 | build: export CGO_ENABLED = 0 19 | build: ## Builds the binary 20 | go build -o $(binary) main.go 21 | 22 | .PHONY: build\:docker 23 | build\:docker: build ## Builds the docker image 24 | docker build -t $(IMAGE_REPOSITORY):$(IMAGE_TAG) -f docker/Dockerfile . 25 | 26 | .PHONY: install\:zfs 27 | install\:zfs: ## Installs zfs-on-linux and nfs-kernel-server (requires sudo) 28 | sudo apt update 29 | sudo apt install -y zfsutils-linux nfs-kernel-server 30 | 31 | $(zpool_dir): 32 | mkdir $(zpool_dir) 33 | 34 | $(zpool_name_file): $(zpool_dir) 35 | # Create a disk image 36 | dd if=/dev/zero bs=1024M count=$(ZPOOL_SIZE) of=$$(pwd)/$(zpool_disk) 37 | echo $(zpool_name) 38 | sudo zpool create $(zpool_name) $$(pwd)/$(zpool_disk) 39 | echo "$(zpool_name)" > $(zpool_name_file) 40 | 41 | /$(zfs_dataset): $(zpool_name_file) 42 | sudo zfs create $(zfs_dataset) 43 | sudo zfs allow -e create,destroy,snapshot,refreservation,refquota,share,sharenfs $(zfs_dataset) 44 | 45 | .PHONY: prepare 46 | prepare: /$(zfs_dataset) ## Prepares the zfs zpool for integration test 47 | 48 | .PHONY: clean\:zfs 49 | clean\:zfs: ## Cleans the zfs pool (requires sudo) 50 | sudo zpool destroy $(zpool_name) 51 | rm -rfv $(zpool_dir) 52 | 53 | .PHONY: clean 54 | clean: clean\:zfs ## Cleans everything 55 | rm -rf c.out $(binary) dist 56 | 57 | .PHONY: test 58 | test: ## Runs the unit tests 59 | go test -coverprofile c.out ./... 60 | 61 | .PHONY: test\:integration 62 | test\:integration: prepare ## Runs the integration tests with zfs (requires sudo) 63 | sudo sh -c "export PATH=$$PATH:$$(go env GOROOT)/bin && go test -tags=integration -v ./test/... -parentDataset $(zfs_dataset)" 64 | 65 | .PHONY: fmt 66 | fmt: ## Run go fmt against code 67 | go fmt ./... 68 | 69 | .PHONY: vet 70 | vet: ## Run go vet against code 71 | go vet ./... 72 | 73 | .PHONY: lint 74 | lint: fmt vet ## Invokes the fmt, vet and checks for uncommitted changes 75 | @echo 'Check for uncommitted changes ...' 76 | git diff --exit-code 77 | -------------------------------------------------------------------------------- /charts/kubernetes-zfs-provisioner/templates/rbac.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.rbac.create -}} 2 | --- 3 | apiVersion: rbac.authorization.k8s.io/v1 4 | kind: ClusterRole 5 | metadata: 6 | name: '{{ include "kubernetes-zfs-provisioner.fullname" . }}-controller' 7 | labels: 8 | {{- include "kubernetes-zfs-provisioner.labels" . | nindent 4 }} 9 | rules: 10 | # leader election 11 | - apiGroups: 12 | - coordination.k8s.io 13 | resources: 14 | - leases 15 | verbs: 16 | - create 17 | - get 18 | - list 19 | - update 20 | 21 | # system:controller:endpoint-controller 22 | - apiGroups: 23 | - '' 24 | resources: 25 | - pods 26 | - services 27 | verbs: 28 | - get 29 | - list 30 | - watch 31 | - apiGroups: 32 | - '' 33 | resources: 34 | - endpoints 35 | verbs: 36 | - create 37 | - delete 38 | - get 39 | - list 40 | - update 41 | - apiGroups: 42 | - '' 43 | resources: 44 | - endpoints/restricted 45 | verbs: 46 | - create 47 | - apiGroups: 48 | - '' 49 | - events.k8s.io 50 | resources: 51 | - events 52 | verbs: 53 | - create 54 | - patch 55 | - update 56 | - watch 57 | 58 | # system:persistent-volume-provisioner (deduplicated) 59 | - apiGroups: 60 | - '' 61 | resources: 62 | - persistentvolumes 63 | verbs: 64 | - create 65 | - delete 66 | - get 67 | - list 68 | - watch 69 | - apiGroups: 70 | - '' 71 | resources: 72 | - persistentvolumeclaims 73 | verbs: 74 | - get 75 | - list 76 | - update 77 | - watch 78 | - apiGroups: 79 | - storage.k8s.io 80 | resources: 81 | - storageclasses 82 | verbs: 83 | - get 84 | - list 85 | - watch 86 | --- 87 | apiVersion: rbac.authorization.k8s.io/v1 88 | kind: ClusterRoleBinding 89 | metadata: 90 | name: {{ include "kubernetes-zfs-provisioner.fullname" . }} 91 | labels: 92 | {{- include "kubernetes-zfs-provisioner.labels" . | nindent 4 }} 93 | roleRef: 94 | apiGroup: rbac.authorization.k8s.io 95 | kind: ClusterRole 96 | name: '{{ include "kubernetes-zfs-provisioner.fullname" . }}-controller' 97 | subjects: 98 | - kind: ServiceAccount 99 | name: {{ include "kubernetes-zfs-provisioner.serviceAccountName" . }} 100 | namespace: {{ .Release.Namespace }} 101 | {{- end -}} 102 | -------------------------------------------------------------------------------- /charts/kubernetes-zfs-provisioner/templates/_helpers.tpl: -------------------------------------------------------------------------------- 1 | {{/* vim: set filetype=mustache: */}} 2 | {{/* 3 | Expand the name of the chart. 4 | */}} 5 | {{- define "kubernetes-zfs-provisioner.name" -}} 6 | {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} 7 | {{- end -}} 8 | 9 | {{/* 10 | Create a default fully qualified app name. 11 | We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). 12 | If release name contains chart name it will be used as a full name. 13 | */}} 14 | {{- define "kubernetes-zfs-provisioner.fullname" -}} 15 | {{- if .Values.fullnameOverride -}} 16 | {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} 17 | {{- else -}} 18 | {{- $name := default .Chart.Name .Values.nameOverride -}} 19 | {{- if contains $name .Release.Name -}} 20 | {{- .Release.Name | trunc 63 | trimSuffix "-" -}} 21 | {{- else -}} 22 | {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} 23 | {{- end -}} 24 | {{- end -}} 25 | {{- end -}} 26 | 27 | {{/* 28 | Create chart name and version as used by the chart label. 29 | */}} 30 | {{- define "kubernetes-zfs-provisioner.chart" -}} 31 | {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} 32 | {{- end -}} 33 | 34 | {{/* 35 | Common labels 36 | */}} 37 | {{- define "kubernetes-zfs-provisioner.labels" -}} 38 | helm.sh/chart: {{ include "kubernetes-zfs-provisioner.chart" . }} 39 | {{ include "kubernetes-zfs-provisioner.selectorLabels" . }} 40 | {{- if .Chart.AppVersion }} 41 | app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} 42 | {{- end }} 43 | app.kubernetes.io/managed-by: {{ .Release.Service }} 44 | {{- end -}} 45 | 46 | {{/* 47 | Selector labels 48 | */}} 49 | {{- define "kubernetes-zfs-provisioner.selectorLabels" -}} 50 | app.kubernetes.io/name: {{ include "kubernetes-zfs-provisioner.name" . }} 51 | app.kubernetes.io/instance: {{ .Release.Name }} 52 | {{- end -}} 53 | 54 | {{/* 55 | Create the name of the service account to use 56 | */}} 57 | {{- define "kubernetes-zfs-provisioner.serviceAccountName" -}} 58 | {{- if .Values.serviceAccount.create -}} 59 | {{ default (include "kubernetes-zfs-provisioner.fullname" .) .Values.serviceAccount.name }} 60 | {{- else -}} 61 | {{ default "default" .Values.serviceAccount.name }} 62 | {{- end -}} 63 | {{- end -}} 64 | 65 | {{/* 66 | Create the name of the secret to use 67 | */}} 68 | {{- define "kubernetes-zfs-provisioner.secretName" -}} 69 | {{ default (include "kubernetes-zfs-provisioner.fullname" .) .Values.ssh.externalSecretName }} 70 | {{- end -}} 71 | -------------------------------------------------------------------------------- /.goreleaser.yml: -------------------------------------------------------------------------------- 1 | # This is an example goreleaser.yaml file with some sane defaults. 2 | # Make sure to check the documentation at http://goreleaser.com 3 | 4 | # Blog to build multiarch images with GH, docker buildx and manifests 5 | # https://carlosbecker.com/posts/multi-platform-docker-images-goreleaser-gh-actions/ 6 | 7 | # Setup Multiarch builds with docker and QEMU, restart docker service! 8 | # https://github.com/docker/buildx#building-multi-platform-images 9 | version: 2 10 | builds: 11 | - env: 12 | - CGO_ENABLED=0 # this is needed to build the single binary 13 | goarch: 14 | - amd64 15 | - arm64 16 | goos: 17 | - linux 18 | 19 | archives: 20 | - format: binary 21 | name_template: "{{ .Binary }}_{{ .Os }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}" 22 | 23 | checksum: 24 | name_template: 'checksums.txt' 25 | 26 | snapshot: 27 | name_template: "{{ .Tag }}-snapshot" 28 | 29 | dockers: 30 | - image_templates: 31 | - "ghcr.io/ccremer/zfs-provisioner:v{{ .Version }}-amd64" 32 | 33 | extra_files: 34 | - docker/zfs.sh 35 | - docker/update-permissions.sh 36 | goarch: amd64 37 | dockerfile: docker/Dockerfile 38 | use: buildx 39 | build_flag_templates: 40 | - "--platform=linux/amd64" 41 | 42 | - image_templates: 43 | - "ghcr.io/ccremer/zfs-provisioner:v{{ .Version }}-arm64" 44 | 45 | extra_files: 46 | - docker/zfs.sh 47 | - docker/update-permissions.sh 48 | goarch: arm64 49 | dockerfile: docker/Dockerfile 50 | use: buildx 51 | build_flag_templates: 52 | - "--platform=linux/arm64/v8" 53 | 54 | docker_manifests: 55 | # For prereleases, updating `latest` and the floating tags of the major version does not make sense. 56 | # Only the image for the exact version should be pushed. 57 | 58 | - name_template: "{{ if not .Prerelease }}ghcr.io/ccremer/zfs-provisioner:latest{{ end }}" 59 | image_templates: 60 | - "ghcr.io/ccremer/zfs-provisioner:v{{ .Version }}-amd64" 61 | - "ghcr.io/ccremer/zfs-provisioner:v{{ .Version }}-arm64" 62 | 63 | - name_template: "{{ if not .Prerelease }}ghcr.io/ccremer/zfs-provisioner:v{{ .Major }}{{ end }}" 64 | image_templates: 65 | - "ghcr.io/ccremer/zfs-provisioner:v{{ .Version }}-amd64" 66 | - "ghcr.io/ccremer/zfs-provisioner:v{{ .Version }}-arm64" 67 | 68 | - name_template: "ghcr.io/ccremer/zfs-provisioner:v{{ .Version }}" 69 | image_templates: 70 | - "ghcr.io/ccremer/zfs-provisioner:v{{ .Version }}-amd64" 71 | - "ghcr.io/ccremer/zfs-provisioner:v{{ .Version }}-arm64" 72 | 73 | release: 74 | prerelease: auto 75 | github: 76 | owner: ccremer 77 | name: kubernetes-zfs-provisioner 78 | -------------------------------------------------------------------------------- /charts/kubernetes-zfs-provisioner/templates/deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: {{ include "kubernetes-zfs-provisioner.fullname" . }} 5 | labels: 6 | {{- include "kubernetes-zfs-provisioner.labels" . | nindent 4 }} 7 | spec: 8 | replicas: {{ .Values.replicaCount }} 9 | selector: 10 | matchLabels: 11 | {{- include "kubernetes-zfs-provisioner.selectorLabels" . | nindent 6 }} 12 | template: 13 | metadata: 14 | labels: 15 | {{- include "kubernetes-zfs-provisioner.selectorLabels" . | nindent 8 }} 16 | spec: 17 | {{- with .Values.imagePullSecrets }} 18 | imagePullSecrets: 19 | {{- toYaml . | nindent 8 }} 20 | {{- end }} 21 | serviceAccountName: {{ include "kubernetes-zfs-provisioner.serviceAccountName" . }} 22 | securityContext: 23 | {{- toYaml .Values.podSecurityContext | nindent 8 }} 24 | containers: 25 | - name: provisioner 26 | securityContext: 27 | {{- toYaml .Values.securityContext | nindent 12 }} 28 | image: "{{ .Values.image.registry }}/{{ .Values.image.repository }}:{{ .Values.image.tag }}" 29 | imagePullPolicy: {{ .Values.image.pullPolicy }} 30 | env: 31 | - name: ZFS_PROVISIONER_INSTANCE 32 | value: {{ .Values.provisioner.instance }} 33 | {{- with .Values.env }} 34 | {{- range $key, $value := . }} 35 | - name: {{ $key }} 36 | value: {{ $value }} 37 | {{- end }} 38 | {{- end }} 39 | ports: 40 | - name: http 41 | containerPort: 8080 42 | volumeMounts: 43 | - name: ssh 44 | mountPath: {{ .Values.ssh.mountPath }} 45 | resources: 46 | {{- toYaml .Values.resources | nindent 12 }} 47 | {{- with .Values.hostAliases }} 48 | hostAliases: 49 | {{- range $ip, $hostnames := . }} 50 | - ip: {{ $ip }} 51 | hostnames: 52 | {{- range . }} 53 | - {{ . }} 54 | {{- end }} 55 | {{- end }} 56 | {{- end }} 57 | volumes: 58 | {{- with .Values.ssh }} 59 | - name: ssh 60 | secret: 61 | secretName: {{ include "kubernetes-zfs-provisioner.secretName" $ }} 62 | defaultMode: 0600 63 | {{- end }} 64 | {{- with .Values.nodeSelector }} 65 | nodeSelector: 66 | {{- toYaml . | nindent 8 }} 67 | {{- end }} 68 | {{- with .Values.affinity }} 69 | affinity: 70 | {{- toYaml . | nindent 8 }} 71 | {{- end }} 72 | {{- with .Values.tolerations }} 73 | tolerations: 74 | {{- toYaml . | nindent 8 }} 75 | {{- end }} 76 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/ccremer/kubernetes-zfs-provisioner 2 | 3 | go 1.22.0 4 | 5 | toolchain go1.22.5 6 | 7 | require ( 8 | github.com/mistifyio/go-zfs/v3 v3.0.1 9 | github.com/stretchr/testify v1.9.0 10 | k8s.io/api v0.30.3 11 | k8s.io/apimachinery v0.30.3 12 | k8s.io/client-go v0.30.3 13 | k8s.io/klog/v2 v2.130.1 14 | sigs.k8s.io/sig-storage-lib-external-provisioner/v10 v10.0.0 15 | ) 16 | 17 | require ( 18 | github.com/beorn7/perks v1.0.1 // indirect 19 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 20 | github.com/davecgh/go-spew v1.1.1 // indirect 21 | github.com/emicklei/go-restful/v3 v3.11.0 // indirect 22 | github.com/go-logr/logr v1.4.2 // indirect 23 | github.com/go-openapi/jsonpointer v0.19.6 // indirect 24 | github.com/go-openapi/jsonreference v0.20.2 // indirect 25 | github.com/go-openapi/swag v0.22.3 // indirect 26 | github.com/gogo/protobuf v1.3.2 // indirect 27 | github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect 28 | github.com/golang/protobuf v1.5.4 // indirect 29 | github.com/google/gnostic-models v0.6.8 // indirect 30 | github.com/google/go-cmp v0.6.0 // indirect 31 | github.com/google/gofuzz v1.2.0 // indirect 32 | github.com/google/uuid v1.3.0 // indirect 33 | github.com/imdario/mergo v0.3.6 // indirect 34 | github.com/josharian/intern v1.0.0 // indirect 35 | github.com/json-iterator/go v1.1.12 // indirect 36 | github.com/mailru/easyjson v0.7.7 // indirect 37 | github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect 38 | github.com/miekg/dns v1.1.41 // indirect 39 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 40 | github.com/modern-go/reflect2 v1.0.2 // indirect 41 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 42 | github.com/pmezard/go-difflib v1.0.0 // indirect 43 | github.com/prometheus/client_golang v1.11.1 // indirect 44 | github.com/prometheus/client_model v0.2.0 // indirect 45 | github.com/prometheus/common v0.26.0 // indirect 46 | github.com/prometheus/procfs v0.6.0 // indirect 47 | github.com/spf13/pflag v1.0.5 // indirect 48 | github.com/stretchr/objx v0.5.2 // indirect 49 | golang.org/x/net v0.27.0 // indirect 50 | golang.org/x/oauth2 v0.10.0 // indirect 51 | golang.org/x/sys v0.22.0 // indirect 52 | golang.org/x/term v0.22.0 // indirect 53 | golang.org/x/text v0.16.0 // indirect 54 | golang.org/x/time v0.3.0 // indirect 55 | google.golang.org/appengine v1.6.7 // indirect 56 | google.golang.org/protobuf v1.34.2 // indirect 57 | gopkg.in/inf.v0 v0.9.1 // indirect 58 | gopkg.in/yaml.v2 v2.4.0 // indirect 59 | gopkg.in/yaml.v3 v3.0.1 // indirect 60 | k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect 61 | k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect 62 | sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect 63 | sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect 64 | sigs.k8s.io/yaml v1.3.0 // indirect 65 | ) 66 | -------------------------------------------------------------------------------- /.github/workflows/chart-release.yml: -------------------------------------------------------------------------------- 1 | name: ChartRelease 2 | 3 | on: 4 | push: 5 | tags: 6 | - "chart/[a-z0-9]+-*" # match tags following the 'chart/$chartname-x.y.z' 7 | 8 | jobs: 9 | gh-pages: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - name: Download cr 13 | uses: giantswarm/install-binary-action@v3.0.0 14 | with: 15 | binary: cr 16 | version: "1.4.0" 17 | download_url: https://github.com/helm/chart-releaser/releases/download/v${version}/chart-releaser_${version}_linux_amd64.tar.gz 18 | tarball_binary_path: "${binary}" 19 | smoke_test: "${binary} version" 20 | 21 | - name: Checkout 22 | uses: actions/checkout@v4 23 | with: 24 | fetch-depth: '0' 25 | 26 | - uses: actions/setup-go@v5 27 | with: 28 | go-version-file: charts/go.mod 29 | 30 | - name: Configure Git 31 | run: | 32 | git config user.name "$GITHUB_ACTOR" 33 | git config user.email "$GITHUB_ACTOR@users.noreply.github.com" 34 | 35 | - name: Get chart name 36 | run: echo "CHART_NAME=$(echo ${GITHUB_REF##*/} | grep --perl-regexp --only-matching '^([a-zA-Z0-9-]+)(?![0-9.]+)')" >> $GITHUB_ENV 37 | 38 | - name: Get chart versions 39 | run: | 40 | echo "CHART_VERSION=$(yq e '.version' charts/${CHART_NAME}/Chart.yaml)" >> $GITHUB_ENV 41 | echo "PREVIOUS_CHART_VERSION=$(git tag --sort=taggerdate --list "chart/${CHART_NAME}-*" | tail -n 2 | head -n 1 | rev | cut -d - -f 1 | rev)" >> $GITHUB_ENV 42 | 43 | - name: Prepare changelog config 44 | run: .github/changelog-charts.sh ${CHART_NAME} 45 | 46 | - name: Generate additional artifacts 47 | run: make chart-prepare 48 | 49 | - name: Package Helm chart 50 | run: | 51 | mkdir -p .cr-index 52 | cr package charts/${CHART_NAME} 53 | cr upload 54 | cr index --push 55 | env: 56 | CR_TOKEN: ${{ secrets.GITHUB_TOKEN }} 57 | CR_OWNER: ${{ github.repository_owner }} 58 | CR_GIT_REPO: ${{ github.event.repository.name }} 59 | 60 | - name: Build changelog from PRs with labels 61 | id: build_changelog 62 | uses: mikepenz/release-changelog-builder-action@v4 63 | with: 64 | configuration: ".github/configuration.json" 65 | ignorePreReleases: true 66 | outputFile: .github/release-notes.md 67 | fromTag: chart/${{ env.CHART_NAME }}-${{ env.PREVIOUS_CHART_VERSION }} 68 | env: 69 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 70 | 71 | # there doesn't seem to be any maintained GitHub actions that allow uploading assets after release has been made. 72 | - name: Update release 73 | run: | 74 | gh release edit ${CHART_NAME}-${CHART_VERSION} --notes-file .github/release-notes.md 75 | env: 76 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 77 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/ccremer/kubernetes-zfs-provisioner/pkg/provisioner" 7 | "k8s.io/klog/v2" 8 | "net/http" 9 | "os" 10 | "strconv" 11 | 12 | "k8s.io/client-go/kubernetes" 13 | "k8s.io/client-go/tools/clientcmd" 14 | "sigs.k8s.io/sig-storage-lib-external-provisioner/v10/controller" 15 | ) 16 | 17 | const ( 18 | metricsAddrKey = "METRICS_ADDR" 19 | metricsPortKey = "METRICS_PORT" 20 | kubeConfigPathKey = "KUBE_CONFIG_PATH" 21 | provisionerInstanceKey = "PROVISIONER_INSTANCE" 22 | ) 23 | 24 | type Settings struct { 25 | MetricsAddr string 26 | MetricsPort int 27 | KubeConfigPath string 28 | ProvisionerInstance string 29 | } 30 | 31 | var ( 32 | // These will be populated by Goreleaser at build time 33 | version = "snapshot" 34 | commit = "dirty" 35 | 36 | settings Settings 37 | ) 38 | 39 | func main() { 40 | loadEnvironmentVariables() 41 | 42 | log := klog.NewKlogr() 43 | 44 | log.Info("Using configuration", "config", settings) 45 | 46 | config, err := clientcmd.BuildConfigFromFlags("", settings.KubeConfigPath) 47 | if err != nil { 48 | klog.Fatalf("Couldn't get in-cluster or kubectl config: %v", err) 49 | } 50 | 51 | // Retrieve config 52 | clientset, err := kubernetes.NewForConfig(config) 53 | if err != nil { 54 | klog.Fatalf("Failed to create kubernetes client: %v", err) 55 | } 56 | 57 | log.Info("Connected to cluster", "host", config.Host) 58 | p, err := provisioner.NewZFSProvisioner(settings.ProvisionerInstance, log) 59 | if err != nil { 60 | klog.Fatalf("Failed to create ZFS provisioner: %v", err) 61 | } 62 | 63 | http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { 64 | http.Redirect(w, r, "/metrics", http.StatusMovedPermanently) 65 | }) 66 | 67 | pc := controller.NewProvisionController( 68 | log, 69 | clientset, 70 | settings.ProvisionerInstance, 71 | p, 72 | controller.MetricsAddress(settings.MetricsAddr), 73 | controller.MetricsPort(int32(settings.MetricsPort)), 74 | ) 75 | 76 | log.Info("Starting provisioner", "version", version, "commit", commit) 77 | pc.Run(context.Background()) 78 | } 79 | 80 | func loadEnvironmentVariables() { 81 | prefix := "ZFS_" 82 | 83 | defaults := map[string]string{ 84 | metricsPortKey: "8080", 85 | metricsAddrKey: "0.0.0.0", 86 | kubeConfigPathKey: "", 87 | provisionerInstanceKey: "pv.kubernetes.io/zfs", 88 | } 89 | 90 | for key, _ := range defaults { 91 | value, found := os.LookupEnv(fmt.Sprintf("%s%s", prefix, key)) 92 | if found { 93 | defaults[key] = value 94 | } 95 | } 96 | settings = Settings{ 97 | MetricsAddr: defaults[metricsAddrKey], 98 | MetricsPort: parseInt(defaults[metricsPortKey]), 99 | KubeConfigPath: defaults[kubeConfigPathKey], 100 | ProvisionerInstance: defaults[provisionerInstanceKey], 101 | } 102 | } 103 | 104 | func parseInt(s string) int { 105 | i, err := strconv.Atoi(s) 106 | if err != nil { 107 | klog.Fatalf("Failed to convert metrics port to integer: %v", err) 108 | } 109 | return i 110 | } 111 | -------------------------------------------------------------------------------- /pkg/provisioner/parameters_test.go: -------------------------------------------------------------------------------- 1 | package provisioner 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestNewStorageClassParameters(t *testing.T) { 10 | type args struct { 11 | parameters map[string]string 12 | } 13 | tests := []struct { 14 | name string 15 | args args 16 | want *ZFSStorageClassParameters 17 | errContains string 18 | }{ 19 | { 20 | name: "GivenWrongSpec_WhenParentDatasetEmpty_ThenThrowError", 21 | args: args{ 22 | parameters: map[string]string{ 23 | HostnameParameter: "host", 24 | }, 25 | }, 26 | errContains: ParentDatasetParameter, 27 | }, 28 | { 29 | name: "GivenWrongSpec_WhenParentDatasetBeginsWithSlash_ThenThrowError", 30 | args: args{ 31 | parameters: map[string]string{ 32 | ParentDatasetParameter: "/tank", 33 | HostnameParameter: "host", 34 | TypeParameter: "nfs", 35 | }, 36 | }, 37 | errContains: ParentDatasetParameter, 38 | }, 39 | { 40 | name: "GivenWrongSpec_WhenParentDatasetEndsWithSlash_ThenThrowError", 41 | args: args{ 42 | parameters: map[string]string{ 43 | ParentDatasetParameter: "/tank/volume/", 44 | HostnameParameter: "host", 45 | TypeParameter: "nfs", 46 | }, 47 | }, 48 | errContains: ParentDatasetParameter, 49 | }, 50 | { 51 | name: "GivenWrongSpec_WhenHostnameEmpty_ThenThrowError", 52 | args: args{ 53 | parameters: map[string]string{ 54 | ParentDatasetParameter: "tank", 55 | }, 56 | }, 57 | errContains: HostnameParameter, 58 | }, 59 | { 60 | name: "GivenWrongSpec_WhenTypeInvalid_ThenThrowError", 61 | args: args{ 62 | parameters: map[string]string{ 63 | ParentDatasetParameter: "tank", 64 | HostnameParameter: "host", 65 | TypeParameter: "invalid", 66 | }, 67 | }, 68 | errContains: TypeParameter, 69 | }, 70 | { 71 | name: "GivenCorrectSpec_WhenTypeNfs_ThenReturnNfsParameters", 72 | args: args{ 73 | parameters: map[string]string{ 74 | ParentDatasetParameter: "tank", 75 | HostnameParameter: "host", 76 | TypeParameter: "nfs", 77 | SharePropertiesParameter: "rw", 78 | }, 79 | }, 80 | want: &ZFSStorageClassParameters{NFSShareProperties: "rw"}, 81 | }, 82 | { 83 | name: "GivenCorrectSpec_WhenTypeNfsWithoutProperties_ThenReturnNfsParametersWithDefault", 84 | args: args{ 85 | parameters: map[string]string{ 86 | ParentDatasetParameter: "tank", 87 | HostnameParameter: "host", 88 | TypeParameter: "nfs", 89 | }, 90 | }, 91 | want: &ZFSStorageClassParameters{NFSShareProperties: "on"}, 92 | }, 93 | { 94 | name: "GivenCorrectSpec_WhenTypeHostPath_ThenReturnHostPathParameters", 95 | args: args{ 96 | parameters: map[string]string{ 97 | ParentDatasetParameter: "tank", 98 | HostnameParameter: "host", 99 | TypeParameter: "hostpath", 100 | NodeNameParameter: "my-node", 101 | }, 102 | }, 103 | want: &ZFSStorageClassParameters{HostPathNodeName: "my-node"}, 104 | }, 105 | } 106 | for _, tt := range tests { 107 | t.Run(tt.name, func(t *testing.T) { 108 | result, err := NewStorageClassParameters(tt.args.parameters) 109 | if tt.errContains != "" { 110 | assert.Error(t, err) 111 | assert.Contains(t, err.Error(), tt.errContains) 112 | return 113 | } 114 | assert.NoError(t, err) 115 | assert.Equal(t, tt.want.NFSShareProperties, result.NFSShareProperties) 116 | assert.Equal(t, tt.want.HostPathNodeName, result.HostPathNodeName) 117 | }) 118 | } 119 | } 120 | -------------------------------------------------------------------------------- /pkg/provisioner/parameters.go: -------------------------------------------------------------------------------- 1 | package provisioner 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | ) 7 | 8 | const ( 9 | ParentDatasetParameter = "parentDataset" 10 | SharePropertiesParameter = "shareProperties" 11 | HostnameParameter = "hostname" 12 | TypeParameter = "type" 13 | NodeNameParameter = "node" 14 | ReserveSpaceParameter = "reserveSpace" 15 | ) 16 | 17 | // StorageClass Parameters are expected in the following schema: 18 | /* 19 | parameters: 20 | parentDataset: tank/volumes 21 | hostname: my-zfs-host.localdomain 22 | type: nfs|hostPath|auto 23 | shareProperties: rw=10.0.0.0/8,no_root_squash 24 | node: my-zfs-host 25 | reserveSpace: true|false 26 | */ 27 | 28 | type ProvisioningType string 29 | 30 | const ( 31 | Nfs ProvisioningType = "nfs" 32 | HostPath ProvisioningType = "hostPath" 33 | Auto ProvisioningType = "auto" 34 | ) 35 | 36 | type ( 37 | // ZFSStorageClassParameters represents the parameters on the `StorageClass` 38 | // object. It is used to ease access and validate those parameters at run time. 39 | ZFSStorageClassParameters struct { 40 | // ParentDataset of the zpool. Needs to be existing on the target ZFS host. 41 | ParentDataset string 42 | // Hostname of the target ZFS host. Will be used to connect over SSH. 43 | Hostname string 44 | Type ProvisioningType 45 | // NFSShareProperties specifies additional properties to pass to 'zfs create sharenfs=%s'. 46 | NFSShareProperties string 47 | // HostPathNodeName overrides the hostname if the Kubernetes node name is different than the ZFS target host. Used for Affinity 48 | HostPathNodeName string 49 | ReserveSpace bool 50 | } 51 | ) 52 | 53 | // NewStorageClassParameters takes a storage class parameters, validates it for invalid configuration and returns a 54 | // ZFSStorageClassParameters on success. 55 | func NewStorageClassParameters(parameters map[string]string) (*ZFSStorageClassParameters, error) { 56 | for _, parameter := range []string{ParentDatasetParameter, HostnameParameter, TypeParameter} { 57 | value := parameters[parameter] 58 | if value == "" { 59 | return nil, fmt.Errorf("undefined required parameter: %s", parameter) 60 | } 61 | } 62 | parentDataset := parameters[ParentDatasetParameter] 63 | if strings.HasPrefix(parentDataset, "/") || strings.HasSuffix(parentDataset, "/") { 64 | return nil, fmt.Errorf("%s must not begin or end with '/': %s", ParentDatasetParameter, parentDataset) 65 | } 66 | 67 | reserveSpaceValue, reserveSpaceValuePresent := parameters[ReserveSpaceParameter] 68 | var reserveSpace bool 69 | if !reserveSpaceValuePresent || strings.EqualFold(reserveSpaceValue, "true") { 70 | reserveSpace = true 71 | } else if strings.EqualFold(reserveSpaceValue, "false") { 72 | reserveSpace = false 73 | } else { 74 | return nil, fmt.Errorf("invalid '%s' parameter value: %s", ReserveSpaceParameter, parameters[ReserveSpaceParameter]) 75 | } 76 | 77 | p := &ZFSStorageClassParameters{ 78 | ParentDataset: parentDataset, 79 | Hostname: parameters[HostnameParameter], 80 | ReserveSpace: reserveSpace, 81 | } 82 | typeParam := parameters[TypeParameter] 83 | switch typeParam { 84 | case "hostpath", "hostPath", "HostPath", "Hostpath", "HOSTPATH": 85 | p.Type = HostPath 86 | case "nfs", "Nfs", "NFS": 87 | p.Type = Nfs 88 | case "auto", "Auto", "AUTO": 89 | p.Type = Auto 90 | default: 91 | return nil, fmt.Errorf("invalid '%s' parameter value: %s", TypeParameter, typeParam) 92 | } 93 | 94 | if p.Type == HostPath || p.Type == Auto { 95 | p.HostPathNodeName = parameters[NodeNameParameter] 96 | } 97 | 98 | if p.Type == Nfs || p.Type == Auto { 99 | shareProps := parameters[SharePropertiesParameter] 100 | if shareProps == "" { 101 | shareProps = "on" 102 | } 103 | p.NFSShareProperties = shareProps 104 | } 105 | 106 | return p, nil 107 | } 108 | -------------------------------------------------------------------------------- /charts/kubernetes-zfs-provisioner/values.yaml: -------------------------------------------------------------------------------- 1 | # Default values for kubernetes-zfs-provisioner. 2 | # This is a YAML-formatted file. 3 | # Declare variables to be passed into your templates. 4 | 5 | # -- Usually `1` is fine 6 | replicaCount: 1 7 | 8 | image: 9 | # -- Location of the container image 10 | repository: ccremer/zfs-provisioner 11 | # -- Container image registry 12 | registry: ghcr.io 13 | # -- Container image tag 14 | tag: v1 15 | pullPolicy: Always 16 | 17 | imagePullSecrets: [] 18 | nameOverride: "" 19 | fullnameOverride: "" 20 | 21 | provisioner: 22 | # -- Provisoner instance name if multiple are running (multiple 23 | # instances are not required for managing multiple ZFS hosts) 24 | instance: pv.kubernetes.io/zfs 25 | 26 | storageClass: 27 | # -- Whether to create storage classes for this provisioner. 28 | create: false 29 | # -- Storage classes to create. See [values.yaml](values.yaml) for an example. 30 | classes: [] 31 | # - name: zfs 32 | # # -- The provisioners connects through SSH to this ZFS host 33 | # hostName: storage-1.domain.tld 34 | # # -- Existing dataset on the target ZFS host 35 | # parentDataset: tank/kubernetes 36 | # # -- The reclaim policy supported by the provisioner 37 | # policy: "Delete" 38 | # # -- NFS export properties (see `exports(5)`) 39 | # shareProperties: "" 40 | # # -- Provision type, one of [`nfs`, `hostpath`, `auto`] 41 | # type: "nfs" 42 | # # -- Override `kubernetes.io/hostname` from `hostName` parameter for 43 | # # `HostPath` node affinity 44 | # node: "" 45 | # # -- Reserve space for created datasets. Default is true. Use false to enable thin provisioning 46 | # reserveSpace: false 47 | # # -- Annotations for the storage class 48 | # # annotations: 49 | # # storageclass.kubernetes.io/is-default-class: "true" 50 | 51 | ssh: 52 | # -- If SSH secrets are managed externally, specify the name 53 | externalSecretName: "" 54 | # -- The path where the SSH config and identities are mounted 55 | mountPath: "/home/zfs/.ssh" 56 | # -- **Required.** ssh_config(5)-compatible file content to configure SSH options when connecting 57 | config: "" 58 | # config: | 59 | # Host my-host 60 | # IdentityFile ~/.ssh/id_ed25519 61 | # User zfs 62 | 63 | # -- **Required.** Provide a private key for each SSH identity. 64 | # See [values.yaml](./values.yaml) for an example 65 | identities: {} 66 | # id_ed25519: | 67 | # -----BEGIN OPENSSH PRIVATE KEY----- 68 | # ... 69 | # -----END OPENSSH PRIVATE KEY----- 70 | 71 | # -- **Required.** List of {host, pubKey} dicts where the public key of each host is configured 72 | knownHosts: [] 73 | # - host: my-host 74 | # pubKey: ssh-ed25519 AAAAC3NzaC... 75 | 76 | # -- A dict with KEY: VALUE pairs 77 | env: {} 78 | 79 | # -- A dict with `{ip, hostnames array}` to configure custom entries in /etc/hosts. 80 | # See [values.yaml](./values.yaml) for an example. 81 | hostAliases: {} 82 | # 192.168.1.1: 83 | # - my-custom-host.name 84 | 85 | serviceAccount: 86 | # -- Specifies whether a service account should be created 87 | create: true 88 | # -- Annotations to add to the service account 89 | annotations: {} 90 | # -- The name of the service account to use. 91 | # If not set and create is true, a name is generated using the fullname template 92 | name: "" 93 | 94 | rbac: 95 | # -- **Required for first time deployments** Grant the service account 96 | # the necessary permissions, 97 | create: false 98 | 99 | # -- If you encounter **issues with SSH, set `podSecurityContext.fsGroup=100`**, as the SSH 100 | # files might not be readable to the container user `zfs` with uid 100. 101 | podSecurityContext: {} 102 | # fsGroup: 100 103 | 104 | securityContext: {} 105 | # capabilities: 106 | # drop: 107 | # - ALL 108 | # readOnlyRootFilesystem: true 109 | # runAsNonRoot: true 110 | # runAsUser: 100 111 | 112 | resources: 113 | limits: 114 | memory: 40Mi 115 | requests: 116 | cpu: 50m 117 | memory: 20Mi 118 | 119 | # -- Reminder: This has no effect on any PVs, but maybe you want the provisioner pod running 120 | # on certain nodes. 121 | nodeSelector: {} 122 | 123 | tolerations: [] 124 | 125 | affinity: {} 126 | -------------------------------------------------------------------------------- /charts/kubernetes-zfs-provisioner/test/storageclass_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/gruntwork-io/terratest/modules/helm" 7 | "github.com/stretchr/testify/assert" 8 | core "k8s.io/api/core/v1" 9 | v1 "k8s.io/api/storage/v1" 10 | ) 11 | 12 | var tplStorageclass = []string{"templates/storageclass.yaml"} 13 | 14 | func Test_Storageclass_GivenClassesEnabled_WhenNoPolicyDefined_ThenRenderDefault(t *testing.T) { 15 | options := &helm.Options{ 16 | ValuesFiles: []string{"values/storageclass_1.yaml"}, 17 | SetValues: map[string]string{ 18 | "storageClass.classes[0].policy": "", 19 | }, 20 | } 21 | 22 | output := helm.RenderTemplate(t, options, helmChartPath, releaseName, tplStorageclass) 23 | 24 | var class v1.StorageClass 25 | helm.UnmarshalK8SYaml(t, output, &class) 26 | 27 | expectedPolicy := core.PersistentVolumeReclaimDelete 28 | assert.Equal(t, &expectedPolicy, class.ReclaimPolicy) 29 | } 30 | 31 | func Test_StorageClass_GivenClassesEnabled_WhenNoTypeDefined_ThenRenderDefault(t *testing.T) { 32 | options := &helm.Options{ 33 | ValuesFiles: []string{"values/storageclass_1.yaml"}, 34 | } 35 | 36 | output := helm.RenderTemplate(t, options, helmChartPath, releaseName, tplStorageclass) 37 | 38 | var class v1.StorageClass 39 | helm.UnmarshalK8SYaml(t, output, &class) 40 | 41 | assert.Equal(t, "nfs", class.Parameters["type"]) 42 | } 43 | 44 | func Test_StorageClass_GivenClassesEnabled_WhenNodeDefined_ThenRenderNodeName(t *testing.T) { 45 | options := &helm.Options{ 46 | SetValues: map[string]string{ 47 | "storageClass.create": "true", 48 | "storageClass.classes[0].node": "host", 49 | "storageClass.classes[0].type": "hostpath", 50 | }, 51 | } 52 | 53 | output := helm.RenderTemplate(t, options, helmChartPath, releaseName, tplStorageclass) 54 | 55 | var class v1.StorageClass 56 | helm.UnmarshalK8SYaml(t, output, &class) 57 | 58 | assert.Equal(t, "host", class.Parameters["node"]) 59 | assert.Equal(t, "hostpath", class.Parameters["type"]) 60 | } 61 | 62 | func Test_StorageClass_GivenClassesEnabled_WhenReserveSpaceUndefined_ThenRenderDefault(t *testing.T) { 63 | options := &helm.Options{ 64 | ValuesFiles: []string{"values/storageclass_1.yaml"}, 65 | } 66 | 67 | output := helm.RenderTemplate(t, options, helmChartPath, releaseName, tplStorageclass) 68 | 69 | var class v1.StorageClass 70 | helm.UnmarshalK8SYaml(t, output, &class) 71 | 72 | value, exists := class.Parameters["reserveSpace"] 73 | assert.False(t, exists) 74 | assert.Empty(t, value) 75 | } 76 | 77 | func Test_StorageClass_GivenClassesEnabled_WhenReserveSpaceFalse_ThenRenderReserveSpace(t *testing.T) { 78 | options := &helm.Options{ 79 | SetValues: map[string]string{ 80 | "storageClass.create": "true", 81 | "storageClass.classes[0].reserveSpace": "false", 82 | }, 83 | } 84 | 85 | output := helm.RenderTemplate(t, options, helmChartPath, releaseName, tplStorageclass) 86 | 87 | var class v1.StorageClass 88 | helm.UnmarshalK8SYaml(t, output, &class) 89 | 90 | assert.Equal(t, "false", class.Parameters["reserveSpace"]) 91 | } 92 | 93 | func Test_StorageClass_GivenClassesEnabled_WhenReserveSpaceTrue_ThenRenderReserveSpace(t *testing.T) { 94 | options := &helm.Options{ 95 | SetValues: map[string]string{ 96 | "storageClass.create": "true", 97 | "storageClass.classes[0].reserveSpace": "true", 98 | }, 99 | } 100 | 101 | output := helm.RenderTemplate(t, options, helmChartPath, releaseName, tplStorageclass) 102 | 103 | var class v1.StorageClass 104 | helm.UnmarshalK8SYaml(t, output, &class) 105 | 106 | assert.Equal(t, "true", class.Parameters["reserveSpace"]) 107 | } 108 | 109 | func Test_StorageClass_GivenClassesEnabled_WhenAdditionalParametersUndefined_ThenRenderEmptyValues(t *testing.T) { 110 | options := &helm.Options{ 111 | ValuesFiles: []string{"values/storageclass_1.yaml"}, 112 | SetValues: map[string]string{ 113 | "storageClass.classes[0].node": "", 114 | "storageClass.classes[0].shareProperties": "", 115 | }, 116 | } 117 | 118 | output := helm.RenderTemplate(t, options, helmChartPath, releaseName, tplStorageclass) 119 | 120 | var class v1.StorageClass 121 | helm.UnmarshalK8SYaml(t, output, &class) 122 | 123 | assert.Equal(t, "", class.Parameters["shareProperties"]) 124 | assert.Equal(t, "", class.Parameters["node"]) 125 | } 126 | -------------------------------------------------------------------------------- /charts/go.mod: -------------------------------------------------------------------------------- 1 | module charts 2 | 3 | go 1.19 4 | 5 | require ( 6 | github.com/gruntwork-io/terratest v0.43.12 7 | github.com/norwoodj/helm-docs v1.11.1 8 | github.com/stretchr/testify v1.8.4 9 | k8s.io/api v0.27.2 10 | ) 11 | 12 | require ( 13 | github.com/Masterminds/goutils v1.1.1 // indirect 14 | github.com/Masterminds/semver/v3 v3.1.1 // indirect 15 | github.com/Masterminds/sprig/v3 v3.2.2 // indirect 16 | github.com/aws/aws-sdk-go v1.44.122 // indirect 17 | github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect 18 | github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect 19 | github.com/davecgh/go-spew v1.1.1 // indirect 20 | github.com/emicklei/go-restful/v3 v3.9.0 // indirect 21 | github.com/fsnotify/fsnotify v1.4.9 // indirect 22 | github.com/ghodss/yaml v1.0.0 // indirect 23 | github.com/go-errors/errors v1.0.2-0.20180813162953-d98b870cc4e0 // indirect 24 | github.com/go-logr/logr v1.2.3 // indirect 25 | github.com/go-openapi/jsonpointer v0.19.6 // indirect 26 | github.com/go-openapi/jsonreference v0.20.1 // indirect 27 | github.com/go-openapi/swag v0.22.3 // indirect 28 | github.com/go-sql-driver/mysql v1.4.1 // indirect 29 | github.com/gobwas/glob v0.2.3 // indirect 30 | github.com/gogo/protobuf v1.3.2 // indirect 31 | github.com/golang/protobuf v1.5.3 // indirect 32 | github.com/google/gnostic v0.5.7-v3refs // indirect 33 | github.com/google/go-cmp v0.5.9 // indirect 34 | github.com/google/gofuzz v1.1.0 // indirect 35 | github.com/google/uuid v1.3.0 // indirect 36 | github.com/gruntwork-io/go-commons v0.8.0 // indirect 37 | github.com/hashicorp/errwrap v1.0.0 // indirect 38 | github.com/hashicorp/go-multierror v1.1.0 // indirect 39 | github.com/hashicorp/hcl v1.0.0 // indirect 40 | github.com/huandu/xstrings v1.3.1 // indirect 41 | github.com/imdario/mergo v0.3.11 // indirect 42 | github.com/inconshreveable/mousetrap v1.0.0 // indirect 43 | github.com/jmespath/go-jmespath v0.4.0 // indirect 44 | github.com/josharian/intern v1.0.0 // indirect 45 | github.com/json-iterator/go v1.1.12 // indirect 46 | github.com/magiconair/properties v1.8.5 // indirect 47 | github.com/mailru/easyjson v0.7.7 // indirect 48 | github.com/mattn/go-zglob v0.0.2-0.20190814121620-e3c945676326 // indirect 49 | github.com/mitchellh/copystructure v1.2.0 // indirect 50 | github.com/mitchellh/go-homedir v1.1.0 // indirect 51 | github.com/mitchellh/mapstructure v1.4.1 // indirect 52 | github.com/mitchellh/reflectwalk v1.0.2 // indirect 53 | github.com/moby/spdystream v0.2.0 // indirect 54 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 55 | github.com/modern-go/reflect2 v1.0.2 // indirect 56 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 57 | github.com/pelletier/go-toml v1.9.3 // indirect 58 | github.com/pmezard/go-difflib v1.0.0 // indirect 59 | github.com/pquerna/otp v1.2.0 // indirect 60 | github.com/russross/blackfriday/v2 v2.1.0 // indirect 61 | github.com/shopspring/decimal v1.2.0 // indirect 62 | github.com/sirupsen/logrus v1.8.1 // indirect 63 | github.com/spf13/afero v1.6.0 // indirect 64 | github.com/spf13/cast v1.3.1 // indirect 65 | github.com/spf13/cobra v1.2.1 // indirect 66 | github.com/spf13/jwalterweatherman v1.1.0 // indirect 67 | github.com/spf13/pflag v1.0.5 // indirect 68 | github.com/spf13/viper v1.8.1 // indirect 69 | github.com/subosito/gotenv v1.2.0 // indirect 70 | github.com/urfave/cli v1.22.2 // indirect 71 | golang.org/x/crypto v0.1.0 // indirect 72 | golang.org/x/net v0.8.0 // indirect 73 | golang.org/x/oauth2 v0.1.0 // indirect 74 | golang.org/x/sys v0.6.0 // indirect 75 | golang.org/x/term v0.6.0 // indirect 76 | golang.org/x/text v0.8.0 // indirect 77 | golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect 78 | google.golang.org/appengine v1.6.7 // indirect 79 | google.golang.org/protobuf v1.31.0 // indirect 80 | gopkg.in/inf.v0 v0.9.1 // indirect 81 | gopkg.in/ini.v1 v1.62.0 // indirect 82 | gopkg.in/yaml.v2 v2.4.0 // indirect 83 | gopkg.in/yaml.v3 v3.0.1 // indirect 84 | k8s.io/apimachinery v0.27.2 // indirect 85 | k8s.io/client-go v0.27.2 // indirect 86 | k8s.io/helm v2.14.3+incompatible // indirect 87 | k8s.io/klog/v2 v2.90.1 // indirect 88 | k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect 89 | k8s.io/utils v0.0.0-20230209194617-a36077c30491 // indirect 90 | sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect 91 | sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect 92 | sigs.k8s.io/yaml v1.3.0 // indirect 93 | ) 94 | -------------------------------------------------------------------------------- /pkg/zfs/zfs.go: -------------------------------------------------------------------------------- 1 | package zfs 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "os" 7 | "os/exec" 8 | "path/filepath" 9 | "sync" 10 | 11 | gozfs "github.com/mistifyio/go-zfs/v3" 12 | "k8s.io/klog/v2" 13 | ) 14 | 15 | type ( 16 | // Interface abstracts the underlying ZFS library with the minimum functionality implemented 17 | Interface interface { 18 | GetDataset(name string, hostname string) (*Dataset, error) 19 | CreateDataset(name string, hostname string, properties map[string]string) (*Dataset, error) 20 | DestroyDataset(dataset *Dataset, flag DestroyFlag) error 21 | SetPermissions(dataset *Dataset) error 22 | } 23 | // Dataset is a representation of a ZFS dataset 24 | Dataset struct { 25 | datasetImpl *gozfs.Dataset 26 | 27 | Name string 28 | Mountpoint string 29 | Hostname string 30 | } 31 | DestroyFlag int 32 | zfsImpl struct{} 33 | ) 34 | 35 | const ( 36 | DestroyRecursively DestroyFlag = 2 37 | HostEnvVar = "ZFS_HOST" 38 | ) 39 | 40 | var ( 41 | globalLock = sync.Mutex{} 42 | ) 43 | 44 | func (z *zfsImpl) GetDataset(name string, hostname string) (*Dataset, error) { 45 | klog.V(3).Info("acquiring lock...") 46 | globalLock.Lock() 47 | defer globalLock.Unlock() 48 | if err := setEnvironmentVars(hostname); err != nil { 49 | return nil, err 50 | } 51 | dataset, err := gozfs.GetDataset(name) 52 | if err != nil { 53 | return nil, err 54 | } 55 | return &Dataset{ 56 | datasetImpl: dataset, 57 | Name: dataset.Name, 58 | Mountpoint: dataset.Mountpoint, 59 | Hostname: hostname, 60 | }, err 61 | } 62 | 63 | func (z *zfsImpl) CreateDataset(name string, hostname string, properties map[string]string) (*Dataset, error) { 64 | klog.V(3).Info("acquiring lock...") 65 | globalLock.Lock() 66 | defer globalLock.Unlock() 67 | if err := setEnvironmentVars(hostname); err != nil { 68 | return nil, err 69 | } 70 | klog.V(3).InfoS("creating dataset", "name", name, "host", hostname) 71 | dataset, err := gozfs.CreateFilesystem(name, properties) 72 | if err != nil { 73 | return nil, err 74 | } 75 | return &Dataset{ 76 | datasetImpl: dataset, 77 | Name: dataset.Name, 78 | Mountpoint: dataset.Mountpoint, 79 | Hostname: hostname, 80 | }, err 81 | } 82 | 83 | func (z *zfsImpl) DestroyDataset(dataset *Dataset, flag DestroyFlag) error { 84 | if err := validateDataset(dataset); err != nil { 85 | return err 86 | } 87 | if dataset.datasetImpl == nil { 88 | ds, err := z.GetDataset(dataset.Name, dataset.Hostname) 89 | if err != nil { 90 | return err 91 | } 92 | dataset.datasetImpl = ds.datasetImpl 93 | } 94 | var destrFlag gozfs.DestroyFlag 95 | switch flag { 96 | case DestroyRecursively: 97 | destrFlag = gozfs.DestroyRecursive 98 | break 99 | default: 100 | return fmt.Errorf("programmer error: flag not implemented: %d", flag) 101 | } 102 | klog.V(3).Info("acquiring lock...") 103 | globalLock.Lock() 104 | defer globalLock.Unlock() 105 | if err := setEnvironmentVars(dataset.Hostname); err != nil { 106 | return err 107 | } 108 | return dataset.datasetImpl.Destroy(destrFlag) 109 | } 110 | 111 | func (z *zfsImpl) SetPermissions(dataset *Dataset) error { 112 | if err := validateDataset(dataset); err != nil { 113 | return err 114 | } 115 | if dataset.Mountpoint == "" { 116 | return fmt.Errorf("undefined mountpoint for dataset: %s", dataset.Name) 117 | } 118 | 119 | globalLock.Lock() 120 | defer globalLock.Unlock() 121 | if err := setEnvironmentVars(dataset.Hostname); err != nil { 122 | return err 123 | } 124 | cmd := exec.Command("update-permissions", dataset.Mountpoint) 125 | if filepath.IsAbs(cmd.Path) { 126 | out, err := cmd.CombinedOutput() 127 | if err != nil { 128 | return fmt.Errorf("could not update permissions on '%s': %w: %s", dataset.Hostname, err, out) 129 | } 130 | return nil 131 | } 132 | 133 | // update-permissions executable not found in PATH 134 | st, err := os.Lstat(dataset.Mountpoint) 135 | if err != nil { 136 | return err 137 | } 138 | 139 | // Add group write bit 140 | if err := os.Chmod(dataset.Mountpoint, st.Mode()|0o020); err != nil { 141 | return err 142 | } 143 | 144 | return nil 145 | } 146 | 147 | func NewInterface() *zfsImpl { 148 | return &zfsImpl{} 149 | } 150 | 151 | func setEnvironmentVars(hostName string) error { 152 | return os.Setenv(HostEnvVar, hostName) 153 | } 154 | 155 | func validateDataset(dataset *Dataset) error { 156 | if dataset.Name == "" { 157 | return errors.New("undefined dataset name") 158 | } 159 | if dataset.Hostname == "" { 160 | return fmt.Errorf("required hostname parameter not given for dataset '%s'", dataset.Name) 161 | } 162 | return nil 163 | } 164 | -------------------------------------------------------------------------------- /charts/kubernetes-zfs-provisioner/README.md: -------------------------------------------------------------------------------- 1 | # kubernetes-zfs-provisioner 2 | 3 | ![Version: 2.2.1](https://img.shields.io/badge/Version-2.2.1-informational?style=flat-square) 4 | 5 | Dynamic ZFS persistent volume provisioner for Kubernetes 6 | 7 | ## Installation 8 | 9 | ```bash 10 | helm repo add zfs-provisioner https://ccremer.github.io/kubernetes-zfs-provisioner 11 | helm install kubernetes-zfs-provisioner zfs-provisioner/kubernetes-zfs-provisioner 12 | ``` 13 | 14 | 19 | 20 | 25 | ## Values 26 | 27 | | Key | Type | Default | Description | 28 | |-----|------|---------|-------------| 29 | | affinity | object | `{}` | | 30 | | env | object | `{}` | A dict with KEY: VALUE pairs | 31 | | fullnameOverride | string | `""` | | 32 | | hostAliases | object | `{}` | A dict with `{ip, hostnames array}` to configure custom entries in /etc/hosts. See [values.yaml](./values.yaml) for an example. | 33 | | image.pullPolicy | string | `"Always"` | | 34 | | image.registry | string | `"ghcr.io"` | Container image registry | 35 | | image.repository | string | `"ccremer/zfs-provisioner"` | Location of the container image | 36 | | image.tag | string | `"v1"` | Container image tag | 37 | | imagePullSecrets | list | `[]` | | 38 | | nameOverride | string | `""` | | 39 | | nodeSelector | object | `{}` | Reminder: This has no effect on any PVs, but maybe you want the provisioner pod running on certain nodes. | 40 | | podSecurityContext | object | `{}` | If you encounter **issues with SSH, set `podSecurityContext.fsGroup=100`**, as the SSH files might not be readable to the container user `zfs` with uid 100. | 41 | | provisioner.instance | string | `"pv.kubernetes.io/zfs"` | Provisoner instance name if multiple are running (multiple instances are not required for managing multiple ZFS hosts) | 42 | | rbac.create | bool | `false` | **Required for first time deployments** Grant the service account the necessary permissions, | 43 | | replicaCount | int | `1` | Usually `1` is fine | 44 | | resources.limits.memory | string | `"40Mi"` | | 45 | | resources.requests.cpu | string | `"50m"` | | 46 | | resources.requests.memory | string | `"20Mi"` | | 47 | | securityContext | object | `{}` | | 48 | | serviceAccount.annotations | object | `{}` | Annotations to add to the service account | 49 | | serviceAccount.create | bool | `true` | Specifies whether a service account should be created | 50 | | serviceAccount.name | string | `""` | The name of the service account to use. If not set and create is true, a name is generated using the fullname template | 51 | | ssh.config | string | `""` | **Required.** ssh_config(5)-compatible file content to configure SSH options when connecting | 52 | | ssh.externalSecretName | string | `""` | If SSH secrets are managed externally, specify the name | 53 | | ssh.identities | object | `{}` | **Required.** Provide a private key for each SSH identity. See [values.yaml](./values.yaml) for an example | 54 | | ssh.knownHosts | list | `[]` | **Required.** List of {host, pubKey} dicts where the public key of each host is configured | 55 | | ssh.mountPath | string | `"/home/zfs/.ssh"` | The path where the SSH config and identities are mounted | 56 | | storageClass.classes | list | `[]` | Storage classes to create. See [values.yaml](values.yaml) for an example. | 57 | | storageClass.create | bool | `false` | Whether to create storage classes for this provisioner. | 58 | | tolerations | list | `[]` | | 59 | 60 | ## Upgrading from 0.x to 1.x charts 61 | 62 | There are some breaking changes from 0.x to 1.x versions. 63 | 64 | * The `storageclass.classes` array is now empty. 65 | Where it previously contained an example, the example is removed as a default value. 66 | The example is still in `values.yaml` in form of YAML comments. 67 | * The `image.registry` has changed from `docker.io` to `quay.io` due to Docker Hub's pull limit. 68 | * Bumped `image.tag` to `v1.0.0` 69 | 70 | ## Upgrading from 1.x to 2.x charts 71 | 72 | Due to the migration of the [chart from ccremer/charts](https://github.com/ccremer/charts/tree/master/charts/kubernetes-zfs-provisioner) to this repo, a breaking change was made for the chart. 73 | Only chart archives from version 2.x can be downloaded from the https://ccremer.github.io/kubernetes-zfs-provisioner index. 74 | No 1.x or 0.x chart releases will be migrated from the `ccremer/charts` Helm repo. 75 | 76 | * The `image.registry` has changed from `quay.io` to `ghcr.io`. 77 | * The `image.tag` has changed from to `v1.1.0` to `v1`. 78 | * The `image.pullPolicy` has changed from `IfNotPresent` to `Always`. 79 | -------------------------------------------------------------------------------- /pkg/provisioner/provision.go: -------------------------------------------------------------------------------- 1 | package provisioner 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "slices" 7 | "strconv" 8 | 9 | "github.com/ccremer/kubernetes-zfs-provisioner/pkg/zfs" 10 | 11 | v1 "k8s.io/api/core/v1" 12 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 13 | "sigs.k8s.io/sig-storage-lib-external-provisioner/v10/controller" 14 | ) 15 | 16 | // Provision creates a PersistentVolume, sets quota and shares it via NFS. 17 | func (p *ZFSProvisioner) Provision(ctx context.Context, options controller.ProvisionOptions) (*v1.PersistentVolume, controller.ProvisioningState, error) { 18 | parameters, err := NewStorageClassParameters(options.StorageClass.Parameters) 19 | if err != nil { 20 | return nil, controller.ProvisioningNoChange, fmt.Errorf("failed to parse StorageClass parameters: %w", err) 21 | } 22 | 23 | datasetPath := fmt.Sprintf("%s/%s", parameters.ParentDataset, options.PVName) 24 | properties := make(map[string]string) 25 | 26 | useHostPath := canUseHostPath(parameters, options) 27 | if !useHostPath { 28 | properties[ShareNfsProperty] = parameters.NFSShareProperties 29 | } 30 | 31 | var reclaimPolicy v1.PersistentVolumeReclaimPolicy 32 | if options.StorageClass.ReclaimPolicy == nil { 33 | // Default is delete, see https://kubernetes.io/docs/concepts/storage/storage-classes/#reclaim-policy 34 | reclaimPolicy = v1.PersistentVolumeReclaimDelete 35 | } else if *options.StorageClass.ReclaimPolicy == v1.PersistentVolumeReclaimRecycle { 36 | return nil, controller.ProvisioningFinished, fmt.Errorf("unsupported reclaim policy of this provisioner: %s", v1.PersistentVolumeReclaimRecycle) 37 | } else { 38 | reclaimPolicy = *options.StorageClass.ReclaimPolicy 39 | } 40 | 41 | storageRequest := options.PVC.Spec.Resources.Requests[v1.ResourceStorage] 42 | storageRequestBytes := strconv.FormatInt(storageRequest.Value(), 10) 43 | properties[RefQuotaProperty] = storageRequestBytes 44 | properties[ManagedByProperty] = p.InstanceName 45 | properties[ReclaimPolicyProperty] = string(reclaimPolicy) 46 | 47 | if parameters.ReserveSpace { 48 | properties[RefReservationProperty] = storageRequestBytes 49 | } 50 | 51 | dataset, err := p.zfs.CreateDataset(datasetPath, parameters.Hostname, properties) 52 | if err != nil { 53 | return nil, controller.ProvisioningFinished, fmt.Errorf("creating ZFS dataset failed: %w", err) 54 | } 55 | if err := p.zfs.SetPermissions(dataset); err != nil { 56 | return nil, controller.ProvisioningFinished, err 57 | } 58 | p.log.Info("dataset created", "dataset", dataset.Name) 59 | 60 | // Copy the annotations from the claim and overwrite with ours 61 | if options.PVC.Annotations == nil { 62 | options.PVC.Annotations = make(map[string]string) 63 | } 64 | annotations := options.PVC.Annotations 65 | annotations[DatasetPathAnnotation] = dataset.Name 66 | annotations[ZFSHostAnnotation] = parameters.Hostname 67 | 68 | pv := &v1.PersistentVolume{ 69 | ObjectMeta: metav1.ObjectMeta{ 70 | Name: options.PVName, 71 | Labels: options.PVC.Labels, 72 | Annotations: annotations, 73 | }, 74 | Spec: v1.PersistentVolumeSpec{ 75 | PersistentVolumeReclaimPolicy: reclaimPolicy, 76 | AccessModes: createAccessModes(options, useHostPath), 77 | Capacity: v1.ResourceList{ 78 | v1.ResourceStorage: options.PVC.Spec.Resources.Requests[v1.ResourceStorage], 79 | }, 80 | PersistentVolumeSource: createVolumeSource(parameters, dataset, useHostPath), 81 | NodeAffinity: createNodeAffinity(parameters, useHostPath), 82 | }, 83 | } 84 | return pv, controller.ProvisioningFinished, nil 85 | } 86 | 87 | func canUseHostPath(parameters *ZFSStorageClassParameters, options controller.ProvisionOptions) bool { 88 | switch parameters.Type { 89 | case Nfs: 90 | return false 91 | case HostPath: 92 | return true 93 | case Auto: 94 | if !slices.Contains(options.PVC.Spec.AccessModes, v1.ReadOnlyMany) && !slices.Contains(options.PVC.Spec.AccessModes, v1.ReadWriteMany) { 95 | return true 96 | } 97 | } 98 | return false 99 | } 100 | 101 | func createAccessModes(options controller.ProvisionOptions, useHostPath bool) []v1.PersistentVolumeAccessMode { 102 | if slices.Contains(options.PVC.Spec.AccessModes, v1.ReadWriteOncePod) { 103 | return []v1.PersistentVolumeAccessMode{v1.ReadWriteOncePod} 104 | } 105 | accessModes := []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce} 106 | if !useHostPath { 107 | accessModes = append(accessModes, v1.ReadOnlyMany, v1.ReadWriteMany) 108 | } 109 | return accessModes 110 | } 111 | 112 | func createVolumeSource(parameters *ZFSStorageClassParameters, dataset *zfs.Dataset, useHostPath bool) v1.PersistentVolumeSource { 113 | if useHostPath { 114 | hostPathType := v1.HostPathDirectory 115 | return v1.PersistentVolumeSource{ 116 | HostPath: &v1.HostPathVolumeSource{ 117 | Path: dataset.Mountpoint, 118 | Type: &hostPathType, 119 | }, 120 | } 121 | } 122 | 123 | return v1.PersistentVolumeSource{ 124 | NFS: &v1.NFSVolumeSource{ 125 | Server: parameters.Hostname, 126 | Path: dataset.Mountpoint, 127 | ReadOnly: false, 128 | }, 129 | } 130 | } 131 | 132 | func createNodeAffinity(parameters *ZFSStorageClassParameters, useHostPath bool) *v1.VolumeNodeAffinity { 133 | if !useHostPath { 134 | return nil 135 | } 136 | 137 | node := parameters.HostPathNodeName 138 | if node == "" { 139 | node = parameters.Hostname 140 | } 141 | return &v1.VolumeNodeAffinity{Required: &v1.NodeSelector{NodeSelectorTerms: []v1.NodeSelectorTerm{ 142 | { 143 | MatchExpressions: []v1.NodeSelectorRequirement{ 144 | { 145 | Values: []string{node}, 146 | Operator: v1.NodeSelectorOpIn, 147 | Key: v1.LabelHostname, 148 | }, 149 | }, 150 | }, 151 | }}} 152 | } 153 | -------------------------------------------------------------------------------- /pkg/provisioner/provision_test.go: -------------------------------------------------------------------------------- 1 | package provisioner 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/require" 8 | storagev1 "k8s.io/api/storage/v1" 9 | 10 | "github.com/ccremer/kubernetes-zfs-provisioner/pkg/zfs" 11 | 12 | "github.com/stretchr/testify/assert" 13 | v1 "k8s.io/api/core/v1" 14 | "k8s.io/apimachinery/pkg/api/resource" 15 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 16 | "sigs.k8s.io/sig-storage-lib-external-provisioner/v10/controller" 17 | ) 18 | 19 | func TestProvisionNfs(t *testing.T) { 20 | 21 | expectedShareProperties := "rw=@10.0.0.0/8" 22 | expectedHost := "host" 23 | expectedDatasetName := "test/volumes/pv-testcreate" 24 | expectedDataset := &zfs.Dataset{ 25 | Name: expectedDatasetName, 26 | Hostname: expectedHost, 27 | Mountpoint: "/" + expectedDatasetName, 28 | } 29 | stub := new(zfsStub) 30 | stub.On("CreateDataset", expectedDatasetName, map[string]string{ 31 | RefQuotaProperty: "1000000000", 32 | RefReservationProperty: "1000000000", 33 | "sharenfs": "rw=@10.0.0.0/8", 34 | ManagedByProperty: "test", 35 | ReclaimPolicyProperty: string(v1.PersistentVolumeReclaimDelete), 36 | }).Return(expectedDataset, nil) 37 | stub.On("SetPermissions", expectedDataset).Return(nil) 38 | 39 | p, _ := NewZFSProvisionerStub(stub) 40 | options := controller.ProvisionOptions{ 41 | PVName: "pv-testcreate", 42 | PVC: newClaim(resource.MustParse("1G"), []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce, v1.ReadOnlyMany}), 43 | StorageClass: &storagev1.StorageClass{ 44 | Parameters: map[string]string{ 45 | ParentDatasetParameter: "test/volumes", 46 | HostnameParameter: expectedHost, 47 | TypeParameter: "nfs", 48 | SharePropertiesParameter: expectedShareProperties, 49 | }, 50 | }, 51 | } 52 | 53 | pv, _, err := p.Provision(context.Background(), options) 54 | require.NoError(t, err) 55 | assertBasics(t, stub, pv, expectedDatasetName, expectedHost) 56 | assert.Contains(t, pv.Spec.AccessModes, v1.ReadWriteOnce) 57 | // Pods located on other nodes can mount this PV 58 | assert.Contains(t, pv.Spec.AccessModes, v1.ReadOnlyMany) 59 | assert.Contains(t, pv.Spec.AccessModes, v1.ReadWriteMany) 60 | 61 | assert.Equal(t, v1.PersistentVolumeReclaimDelete, pv.Spec.PersistentVolumeReclaimPolicy) 62 | 63 | require.NotNil(t, pv.Spec.NFS) 64 | require.Nil(t, pv.Spec.HostPath) 65 | require.Nil(t, pv.Spec.NodeAffinity) 66 | assert.Equal(t, "/"+expectedDatasetName, pv.Spec.NFS.Path) 67 | assert.Equal(t, expectedHost, pv.Spec.NFS.Server) 68 | } 69 | 70 | func assertBasics(t *testing.T, stub *zfsStub, pv *v1.PersistentVolume, expectedDataset string, expectedHost string) { 71 | stub.AssertExpectations(t) 72 | 73 | assert.Contains(t, pv.Annotations, "my/annotation") 74 | assert.Equal(t, expectedDataset, pv.Annotations[DatasetPathAnnotation]) 75 | assert.Equal(t, expectedHost, pv.Annotations[ZFSHostAnnotation]) 76 | } 77 | 78 | func TestProvisionHostPath(t *testing.T) { 79 | 80 | expectedDatasetName := "test/volumes/pv-testcreate" 81 | expectedDataset := &zfs.Dataset{ 82 | Name: expectedDatasetName, 83 | Mountpoint: "/" + expectedDatasetName, 84 | } 85 | expectedHost := "host" 86 | policy := v1.PersistentVolumeReclaimRetain 87 | stub := new(zfsStub) 88 | stub.On("CreateDataset", expectedDatasetName, map[string]string{ 89 | RefQuotaProperty: "1000000000", 90 | RefReservationProperty: "1000000000", 91 | ManagedByProperty: "test", 92 | ReclaimPolicyProperty: string(policy), 93 | }).Return(expectedDataset, nil) 94 | stub.On("SetPermissions", expectedDataset).Return(nil) 95 | 96 | p, _ := NewZFSProvisionerStub(stub) 97 | options := controller.ProvisionOptions{ 98 | PVName: "pv-testcreate", 99 | PVC: newClaim(resource.MustParse("1G"), []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce, v1.ReadOnlyMany}), 100 | StorageClass: &storagev1.StorageClass{ 101 | Parameters: map[string]string{ 102 | ParentDatasetParameter: "test/volumes", 103 | HostnameParameter: expectedHost, 104 | TypeParameter: "hostpath", 105 | NodeNameParameter: "node", 106 | }, 107 | ReclaimPolicy: &policy, 108 | }, 109 | } 110 | 111 | pv, _, err := p.Provision(context.Background(), options) 112 | require.NoError(t, err) 113 | assertBasics(t, stub, pv, expectedDatasetName, expectedHost) 114 | assert.Contains(t, pv.Spec.AccessModes, v1.ReadWriteOnce) 115 | // Pods located on other nodes cannot mount this PV 116 | assert.NotContains(t, pv.Spec.AccessModes, v1.ReadOnlyMany) 117 | assert.NotContains(t, pv.Spec.AccessModes, v1.ReadWriteMany) 118 | 119 | assert.Equal(t, policy, pv.Spec.PersistentVolumeReclaimPolicy) 120 | 121 | hostPathType := v1.HostPathDirectory 122 | require.NotNil(t, pv.Spec.HostPath) 123 | require.Nil(t, pv.Spec.NFS) 124 | assert.Equal(t, "/"+expectedDatasetName, pv.Spec.HostPath.Path) 125 | assert.Equal(t, &hostPathType, pv.Spec.HostPath.Type) 126 | assert.Contains(t, pv.Spec.NodeAffinity.Required.NodeSelectorTerms[0].MatchExpressions[0].Values, "node") 127 | } 128 | 129 | func newClaim(capacity resource.Quantity, accessModes []v1.PersistentVolumeAccessMode) *v1.PersistentVolumeClaim { 130 | storageClassName := "zfs" 131 | claim := &v1.PersistentVolumeClaim{ 132 | ObjectMeta: metav1.ObjectMeta{ 133 | Annotations: map[string]string{ 134 | "my/annotation": "value", 135 | }, 136 | }, 137 | Spec: v1.PersistentVolumeClaimSpec{ 138 | AccessModes: accessModes, 139 | Resources: v1.VolumeResourceRequirements{ 140 | Requests: v1.ResourceList{ 141 | v1.ResourceStorage: capacity, 142 | }, 143 | }, 144 | StorageClassName: &storageClassName, 145 | }, 146 | Status: v1.PersistentVolumeClaimStatus{}, 147 | } 148 | return claim 149 | } 150 | -------------------------------------------------------------------------------- /test/provision_integration_test.go: -------------------------------------------------------------------------------- 1 | //go:build integration 2 | 3 | package test 4 | 5 | import ( 6 | "bufio" 7 | "context" 8 | "flag" 9 | "k8s.io/klog/v2" 10 | "math/rand" 11 | "os" 12 | "strconv" 13 | "strings" 14 | "testing" 15 | 16 | gozfs "github.com/mistifyio/go-zfs/v3" 17 | 18 | "github.com/stretchr/testify/assert" 19 | "github.com/stretchr/testify/require" 20 | "github.com/stretchr/testify/suite" 21 | v1 "k8s.io/api/core/v1" 22 | storagev1 "k8s.io/api/storage/v1" 23 | "k8s.io/apimachinery/pkg/api/resource" 24 | "sigs.k8s.io/sig-storage-lib-external-provisioner/v10/controller" 25 | 26 | "github.com/ccremer/kubernetes-zfs-provisioner/pkg/provisioner" 27 | "github.com/ccremer/kubernetes-zfs-provisioner/pkg/zfs" 28 | ) 29 | 30 | var ( 31 | parentDataset = flag.String("parentDataset", "", "parent dataset") 32 | ) 33 | 34 | type ProvisionTestSuit struct { 35 | suite.Suite 36 | p *provisioner.ZFSProvisioner 37 | datasetPrefix string 38 | createdDatasets []string 39 | } 40 | 41 | func TestProvisionSuite(t *testing.T) { 42 | s := ProvisionTestSuit{ 43 | datasetPrefix: "pv-test-" + strconv.Itoa(rand.Int()), 44 | createdDatasets: make([]string, 0), 45 | } 46 | suite.Run(t, &s) 47 | } 48 | 49 | func (suite *ProvisionTestSuit) SetupSuite() { 50 | path := os.Getenv("PATH") 51 | pwd, _ := os.Getwd() 52 | err := os.Setenv("PATH", pwd+":"+path) 53 | log := klog.NewKlogr() 54 | require.NoError(suite.T(), err) 55 | prov, err := provisioner.NewZFSProvisioner("pv.kubernetes.io/zfs", log) 56 | require.NoError(suite.T(), err) 57 | suite.p = prov 58 | } 59 | 60 | func (suite *ProvisionTestSuit) TearDownSuite() { 61 | for _, dataset := range suite.createdDatasets { 62 | err := zfs.NewInterface().DestroyDataset(&zfs.Dataset{ 63 | Name: *parentDataset + "/" + dataset, 64 | Hostname: "host", 65 | }, zfs.DestroyRecursively) 66 | require.NoError(suite.T(), err) 67 | } 68 | } 69 | 70 | func (suite *ProvisionTestSuit) TestDefaultProvisionDataset() { 71 | dataset := provisionDataset(suite, "default", map[string]string{ 72 | provisioner.ParentDatasetParameter: *parentDataset, 73 | provisioner.HostnameParameter: "localhost", 74 | provisioner.TypeParameter: "nfs", 75 | provisioner.SharePropertiesParameter: "rw,no_root_squash", 76 | }) 77 | assertZfsReservation(suite.T(), dataset, true) 78 | } 79 | 80 | func (suite *ProvisionTestSuit) TestThickProvisionDataset() { 81 | dataset := provisionDataset(suite, "thick", map[string]string{ 82 | provisioner.ParentDatasetParameter: *parentDataset, 83 | provisioner.HostnameParameter: "localhost", 84 | provisioner.TypeParameter: "nfs", 85 | provisioner.SharePropertiesParameter: "rw,no_root_squash", 86 | provisioner.ReserveSpaceParameter: "true", 87 | }) 88 | assertZfsReservation(suite.T(), dataset, true) 89 | } 90 | 91 | func (suite *ProvisionTestSuit) TestThinProvisionDataset() { 92 | dataset := provisionDataset(suite, "thin", map[string]string{ 93 | provisioner.ParentDatasetParameter: *parentDataset, 94 | provisioner.HostnameParameter: "localhost", 95 | provisioner.TypeParameter: "nfs", 96 | provisioner.SharePropertiesParameter: "rw,no_root_squash", 97 | provisioner.ReserveSpaceParameter: "false", 98 | }) 99 | assertZfsReservation(suite.T(), dataset, false) 100 | } 101 | 102 | func provisionDataset(suite *ProvisionTestSuit, name string, parameters map[string]string) string { 103 | t := suite.T() 104 | pvName := suite.datasetPrefix + "_" + name 105 | fullDataset := *parentDataset + "/" + pvName 106 | datasetDirectory := "/" + fullDataset 107 | policy := v1.PersistentVolumeReclaimRetain 108 | options := controller.ProvisionOptions{ 109 | PVName: pvName, 110 | PVC: newClaim(resource.MustParse("10M"), []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce, v1.ReadOnlyMany}), 111 | StorageClass: &storagev1.StorageClass{ 112 | Parameters: parameters, 113 | ReclaimPolicy: &policy, 114 | }, 115 | } 116 | 117 | _, _, err := suite.p.Provision(context.Background(), options) 118 | suite.createdDatasets = append(suite.createdDatasets, pvName) 119 | assert.NoError(t, err) 120 | require.DirExists(t, datasetDirectory) 121 | assertNfsExport(t, datasetDirectory) 122 | return fullDataset 123 | } 124 | 125 | func assertZfsReservation(t *testing.T, datasetName string, reserve bool) { 126 | dataset, err := gozfs.GetDataset(datasetName) 127 | assert.NoError(t, err) 128 | 129 | refreserved, err := dataset.GetProperty("refreservation") 130 | assert.NoError(t, err) 131 | 132 | refquota, err := dataset.GetProperty("refquota") 133 | assert.NoError(t, err) 134 | 135 | if reserve { 136 | assert.Equal(t, refquota, refreserved) 137 | } else { 138 | assert.Equal(t, "none", refreserved) 139 | } 140 | } 141 | 142 | func assertNfsExport(t *testing.T, fullDataset string) { 143 | file, err := os.Open("/var/lib/nfs/etab") 144 | require.NoError(t, err) 145 | defer file.Close() 146 | 147 | scanner := bufio.NewScanner(file) 148 | require.NoError(t, err) 149 | found := false 150 | for scanner.Scan() { 151 | line := scanner.Text() 152 | if strings.Contains(line, fullDataset) { 153 | found = true 154 | assert.Contains(t, line, "rw") 155 | assert.Contains(t, line, "no_root_squash") 156 | } 157 | } 158 | assert.True(t, found) 159 | } 160 | 161 | func newClaim(capacity resource.Quantity, accessModes []v1.PersistentVolumeAccessMode) *v1.PersistentVolumeClaim { 162 | storageClassName := "zfs" 163 | claim := &v1.PersistentVolumeClaim{ 164 | Spec: v1.PersistentVolumeClaimSpec{ 165 | AccessModes: accessModes, 166 | Resources: v1.VolumeResourceRequirements{ 167 | Requests: v1.ResourceList{ 168 | v1.ResourceStorage: capacity, 169 | }, 170 | }, 171 | StorageClassName: &storageClassName, 172 | }, 173 | } 174 | return claim 175 | } 176 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Looking for maintainer 2 | 3 | See [137](https://github.com/ccremer/kubernetes-zfs-provisioner/issues/137) 4 | 5 | # Dynamic ZFS provisioner for Kubernetes 6 | 7 | [![Build](https://img.shields.io/github/workflow/status/ccremer/kubernetes-zfs-provisioner/Test)][build] 8 | ![Go version](https://img.shields.io/github/go-mod/go-version/ccremer/kubernetes-zfs-provisioner) 9 | [![Version](https://img.shields.io/github/v/release/ccremer/kubernetes-zfs-provisioner)][releases] 10 | [![GitHub downloads](https://img.shields.io/github/downloads/ccremer/kubernetes-zfs-provisioner/total)][releases] 11 | 12 | kubernetes-zfs-provisioner is a dynamic ZFS persistent volume provisioner for Kubernetes. 13 | It creates ZFS datasets via SSH on remote hosts and shares them via [NFS][nfs] to make them mountable to pods. 14 | 15 | ![architecture with NFS](architecture.nfs.drawio.svg "Architecture with NFS provisioning") 16 | 17 | Alternatively, if the ZFS hosts are part of the cluster, [HostPath][hostpath] is also possible, 18 | but the `PersistentVolume` objects will have a [NodeAffinity][node affinity] configured. 19 | 20 | ![architecture with Hostpath](architecture.hostpath.drawio.svg "Architecture with Hostpath provisioning") 21 | 22 | As a third option, if the ZFS host is part of the cluster, you can let the provisioner choose 23 | whether [NFS][nfs] or [HostPath][hostpath] is used with the `Auto` mode. If the requested 24 | [AccessModes][access modes] in the Persistent Volume Claim contains `ReadWriteOnce` (the volume 25 | can only be accessed by pods running on the same node), or `ReadWriteOncePod` (the volume can only 26 | be accessed by one single Pod at any time), then [HostPath][hostpath] will be used and 27 | the [NodeAffinity][node affinity] will be configured on the `PersistentVolume` objects so the 28 | scheduler will automatically place the corresponding Pods onto the ZFS host. Otherwise 29 | [NFS][nfs] will be used and [NodeAffinity][node affinity] will not be set. If multiple (exclusive) 30 | [AccessModes][access modes] are given, [NFS][nfs] takes precedence. 31 | 32 | Currently all ZFS attributes are inherited from the parent dataset. 33 | 34 | For more information about external storage in kubernetes, see 35 | [kubernetes-sigs/sig-storage-lib-external-provisioner][lib provisioner]. 36 | 37 | ## Installation 38 | 39 | Recommended option is via [Helm][helm chart] 40 | 41 | ## Configuration 42 | 43 | The provisioner relies on an already set up Zpool and a dataset by the administrator. 44 | It also needs **SSH access** to the target ZFS hosts, i.e. mount the SSH private key and 45 | config to the container so that the executing user can find it. 46 | 47 | ### Provisioner 48 | 49 | By **default the container image should work out of the box** when installed in the cluster. 50 | The only thing to configure is SSH, the [Helm Chart][helm chart] should help you with that. 51 | 52 | The provisioner can be configured via the following environment variables: 53 | 54 | | Variable | Description | Default | 55 | | :------: | :---------- | :-----: | 56 | | `ZFS_METRICS_PORT` | Port on which to export Prometheus metrics. | `8080` | 57 | | `ZFS_METRICS_ADDR` | Interface binding address on which to export Prometheus metrics. | `0.0.0.0` | 58 | | `ZFS_KUBE_CONFIG_PATH` | Kubeconfig file path in which the credentials and API URL are defined. | `` | 59 | | `ZFS_PROVISIONER_INSTANCE` | The instance name needs to be unique if multiple provisioners are deployed. | `pv.kubernetes.io/zfs` | 60 | 61 | The provisioner instance name is also stored as a ZFS user property in the created 62 | dataset of the form `io.kubernetes.pv.zfs:managed_by` for system administrators, but is not 63 | further significant to the provisioner. 64 | 65 | ### Storage Classes 66 | 67 | The provisioner relies on properly configured storage classes. The following shows an example 68 | for the [HostPath][hostpath] type. 69 | 70 | ```yaml 71 | kind: StorageClass 72 | apiVersion: storage.k8s.io/v1 73 | metadata: 74 | name: zfs-hostpath 75 | provisioner: pv.kubernetes.io/zfs 76 | reclaimPolicy: Delete 77 | parameters: 78 | parentDataset: tank/kubernetes 79 | hostname: storage-1.domain.tld 80 | type: hostpath 81 | node: storage-1 # the kubernetes.io/hostname label if different than hostname parameter (optional) 82 | reserveSpace: true 83 | ``` 84 | 85 | Following example configures a storage class for ZFS over [NFS][nfs]: 86 | ```yaml 87 | kind: StorageClass 88 | apiVersion: storage.k8s.io/v1 89 | metadata: 90 | name: zfs-nfs 91 | provisioner: pv.kubernetes.io/zfs 92 | reclaimPolicy: Retain 93 | parameters: 94 | parentDataset: tank/kubernetes 95 | hostname: storage-1.domain.tld 96 | type: nfs 97 | shareProperties: rw,no_root_squash # no_root_squash by default sets mode to 'ro' 98 | reserveSpace: true 99 | ``` 100 | For NFS, you can also specify other options, as described in [exports(5)][man exports]. 101 | 102 | The following example configures a storage class using the `Auto` type. The provisioner 103 | will decide whether [HostPath][hostpath] or [NFS][nfs] will be used based on the 104 | [AccessModess][access modes] requested by the persistent volume claim. 105 | 106 | ```yaml 107 | kind: StorageClass 108 | apiVersion: storage.k8s.io/v1 109 | metadata: 110 | name: zfs-nfs 111 | provisioner: pv.kubernetes.io/zfs 112 | reclaimPolicy: Retain 113 | parameters: 114 | parentDataset: tank/kubernetes 115 | hostname: storage-1.domain.tld 116 | type: auto 117 | node: storage-1 # the name of the node where the ZFS datasets are located. 118 | shareProperties: rw,no_root_squash 119 | reserveSpace: true 120 | ``` 121 | 122 | ## Notes 123 | 124 | ### Reclaim policy 125 | 126 | This provisioner supports the `Delete` and `Retain` reclaim policies, with `Delete` being 127 | default if unspecified. The reclaim policy is also stored as ZFS user property of the form 128 | `io.kubernetes.pv.zfs:reclaim_policy` for system administrators, but is not 129 | further significant to the provisioner. 130 | 131 | ### Storage space 132 | 133 | By default, the provisioner uses the `refreservation` and `refquota` ZFS attributes 134 | to limit storage space for volumes. Each volume can not use more storage space than 135 | the given resource request and also reserves exactly that much. To disable this and 136 | enable thin provisioning, set `reserveSpace` to `false` in your storage class parameters. 137 | Snapshots **do not** account for the storage space limit, however this provisioner 138 | does not do any snapshots or backups. 139 | 140 | See [zfs(8)][man zfs] for more information. 141 | 142 | ### Security 143 | 144 | First of all, no warranties and use at own risk. 145 | 146 | Making a container image and creating ZFS datasets from a container is not exactly 147 | easy, as ZFS runs in kernel. While it's possible to pass `/dev/zfs` to a container 148 | so it can create and destroy datasets within the container, sharing the volume with NFS 149 | does not work. 150 | 151 | Setting `sharenfs` property to anything other than `off` invokes [exportfs(8)][man exportfs], 152 | that requires also running the NFS Server to reload its exports. Which is not the case 153 | in a container (see [zfs(8)][man zfs]). 154 | 155 | But most importantly: Mounting `/dev/zfs` inside the provisioner container would mean that 156 | the datasets will only be created on the same host as the container currently runs. 157 | 158 | So, in order to "break out" of the container the `zfs` calls are wrapped and redirected 159 | to another host over **SSH**. This requires SSH private keys to be mounted in the container 160 | for a SSH user with sufficient permissions to run `zfs` commands on the target host. 161 | 162 | Example sudoers file in `/etc/sudoers.d/zfs-provisioner` (On the ZFS host): 163 | ``` 164 | zfs-provisioner ALL=(ALL) NOPASSWD:/sbin/zfs *,/bin/chmod * 165 | ``` 166 | 167 | For increased performance and security install ZFS on all Kubernetes nodes thats should 168 | provide ZFS storage. Then it's possible to create `PersistentVolume` objects with [HostPath][hostpath]. 169 | This eliminates network latency over unencrypted NFS, but schedules the pods to the ZFS hosts only. 170 | 171 | ## Development 172 | 173 | ### Requirements 174 | 175 | * go 176 | * docker 177 | * ZFS and NFS (run `make install:zfs` on Debian/Ubuntu if not already installed) 178 | 179 | ### Building and Testing 180 | 181 | Run `make help` to see which target does what. 182 | 183 | ## Troubleshooting 184 | 185 | ### Filesystem created, but not shared 186 | 187 | ``` 188 | controller.go:920] error syncing claim "56ea786a-e376-4911-a4b1-7b040dc3537f": failed to provision volume 189 | with StorageClass "zfs-retain-pve-1": creating ZFS dataset failed: exit status 1: 190 | "/usr/bin/zfs zfs create -o sharenfs=rw,no_root_squash ... tank/kubernetes/ 191 | pvc-56ea786a-e376-4911-a4b1-7b040dc3537f" => cannot share 'tank/kubernetes/ 192 | pvc-56ea786a-e376-4911-a4b1-7b040dc3537f': share(1M) failed 193 | filesystem successfully created, but not shared 194 | ``` 195 | 196 | This happens when the dataset got created, but invoking `zfs share` is failing. 197 | Most likely because from [zfs(8)][man zfs] it's stated that [exportfs(8)][man exportfs] is invoked, which talks to the NFS server. 198 | 199 | So, have you got `nfs-kernel-server` installed on the host and is `exportfs` available? 200 | 201 | Once you solve this, destroy the dataset again, as the following retries will fail forever: 202 | 203 | ``` 204 | cannot create 'tank/services/kubernetes/pvc-56ea786a-e376-4911-a4b1-7b040dc3537f': dataset already exists 205 | ``` 206 | 207 | ## Credits 208 | 209 | Thanks to [Gentics][gentics] for open sourcing the [initial version][gentics repo]! 210 | 211 | I (@ccremer) have been allowed to take over maintenance for this repository. 212 | 213 | 214 | [build]: https://github.com/ccremer/kubernetes-zfs-provisioner/actions?query=workflow%3ATest 215 | [releases]: https://github.com/ccremer/kubernetes-zfs-provisioner/releases 216 | [node affinity]: https://kubernetes.io/docs/concepts/storage/persistent-volumes/#node-affinity 217 | [lib provisioner]: https://github.com/kubernetes-sigs/sig-storage-lib-external-provisioner 218 | [hostpath]: https://kubernetes.io/docs/concepts/storage/volumes/#hostpath 219 | [nfs]: https://kubernetes.io/docs/concepts/storage/volumes/#nfs 220 | [man zfs]: https://linux.die.net/man/8/zfs 221 | [man exportfs]: https://linux.die.net/man/8/exportfs 222 | [man exports]: https://linux.die.net/man/5/exports 223 | [helm chart]: https://github.com/ccremer/kubernetes-zfs-provisioner/blob/master/charts/kubernetes-zfs-provisioner/README.md 224 | [gentics]: https://www.gentics.com/genticscms/index.en.html 225 | [gentics repo]: https://github.com/gentics/kubernetes-zfs-provisioner 226 | [access modes]: https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes 227 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. -------------------------------------------------------------------------------- /architecture.nfs.drawio.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 |
10 |
11 |
12 | Kubernetes 13 |
14 |
15 |
16 | 17 | 18 | Kubernetes 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 |
30 |
31 |
32 | ZFS host 33 |
34 |
35 |
36 | 37 | 38 | ZFS host 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 |
47 |
48 |
49 | zfs-provisioner 50 |
51 |
52 |
53 | 54 | 55 | zfs-provisioner 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 |
65 |
66 |
67 | 2) Create Dataset 68 |
69 | (over ssh) 70 |
71 |
72 |
73 | 74 | 75 | 2) Create Dataset... 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 |
86 |
87 |
88 | controller 89 |
90 |
91 |
92 | 93 | 94 | controll... 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 |
103 |
104 |
105 | app 106 |
107 |
108 |
109 | 110 | 111 | app 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 |
121 |
122 |
123 | 1) Request PV 124 |
125 |
126 |
127 | 128 | 129 | 1) Request PV 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 |
140 |
141 |
142 | PVC 143 |
144 |
145 |
146 | 147 | 148 | PVC 149 | 150 | 151 | 152 | 153 | 154 | 155 | 156 | 157 |
158 |
159 |
160 | 4) Bind 161 |
162 |
163 |
164 | 165 | 166 | 4) Bind 167 | 168 | 169 | 170 | 171 | 172 | 173 | 174 | 175 |
176 |
177 |
178 | 5) Mount 179 |
180 |
181 |
182 | 183 | 184 | 5) Mount 185 | 186 | 187 | 188 | 189 | 190 | 191 | 192 | 193 | 194 |
195 |
196 |
197 | PV (NFS) 198 |
199 |
200 |
201 | 202 | 203 | PV (NFS) 204 | 205 | 206 | 207 | 208 | 209 | 210 | 211 | 212 |
213 |
214 |
215 | 3) Provision PV 216 |
217 |
218 |
219 | 220 | 221 | 3) Provision PV 222 | 223 | 224 | 225 | 226 | 227 | 228 | 229 | 230 | Viewer does not support full SVG 1.1 231 | 232 | 233 | 234 | 235 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 2 | github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= 3 | github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= 4 | github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= 5 | github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= 6 | github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= 7 | github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= 8 | github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= 9 | github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= 10 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= 11 | github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 12 | github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= 13 | github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 14 | github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= 15 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 16 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 17 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 18 | github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= 19 | github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= 20 | github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= 21 | github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= 22 | github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= 23 | github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= 24 | github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= 25 | github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= 26 | github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= 27 | github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= 28 | github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= 29 | github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= 30 | github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= 31 | github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= 32 | github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= 33 | github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= 34 | github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= 35 | github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= 36 | github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= 37 | github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= 38 | github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= 39 | github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= 40 | github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= 41 | github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= 42 | github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= 43 | github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= 44 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 45 | github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 46 | github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 47 | github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= 48 | github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= 49 | github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= 50 | github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= 51 | github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= 52 | github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= 53 | github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= 54 | github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= 55 | github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= 56 | github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= 57 | github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= 58 | github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 59 | github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 60 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 61 | github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 62 | github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 63 | github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 64 | github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= 65 | github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 66 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 67 | github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= 68 | github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 69 | github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= 70 | github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= 71 | github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 72 | github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= 73 | github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 74 | github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= 75 | github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= 76 | github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= 77 | github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= 78 | github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= 79 | github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= 80 | github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 81 | github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 82 | github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= 83 | github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= 84 | github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= 85 | github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= 86 | github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= 87 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 88 | github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 89 | github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 90 | github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= 91 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 92 | github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= 93 | github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= 94 | github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= 95 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 96 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 97 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 98 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 99 | github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= 100 | github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= 101 | github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= 102 | github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= 103 | github.com/miekg/dns v1.1.41 h1:WMszZWJG0XmzbK9FEmzH2TVcqYzFesusSIB41b8KHxY= 104 | github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= 105 | github.com/mistifyio/go-zfs/v3 v3.0.1 h1:YaoXgBePoMA12+S1u/ddkv+QqxcfiZK4prI6HPnkFiU= 106 | github.com/mistifyio/go-zfs/v3 v3.0.1/go.mod h1:CzVgeB0RvF2EGzQnytKVvVSDwmKJXxkOTUGbNrTja/k= 107 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 108 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= 109 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 110 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 111 | github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 112 | github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= 113 | github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= 114 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= 115 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= 116 | github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= 117 | github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= 118 | github.com/onsi/ginkgo/v2 v2.15.0 h1:79HwNRBAZHOEwrczrgSOPy+eFTTlIGELKy5as+ClttY= 119 | github.com/onsi/ginkgo/v2 v2.15.0/go.mod h1:HlxMHtYF57y6Dpf+mc5529KKmSq9h2FpCF+/ZkwUxKM= 120 | github.com/onsi/gomega v1.31.0 h1:54UJxxj6cPInHS3a35wm6BK/F9nHYueZ1NVujHDrnXE= 121 | github.com/onsi/gomega v1.31.0/go.mod h1:DW9aCi7U6Yi40wNVAvT6kzFnEVEI5n3DloYBiKiT6zk= 122 | github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 123 | github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 124 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 125 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 126 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 127 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 128 | github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= 129 | github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= 130 | github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= 131 | github.com/prometheus/client_golang v1.11.1 h1:+4eQaD7vAZ6DsfsxB15hbE0odUjGI5ARs9yskGu1v4s= 132 | github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= 133 | github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= 134 | github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 135 | github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= 136 | github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 137 | github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= 138 | github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= 139 | github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ= 140 | github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= 141 | github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= 142 | github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= 143 | github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= 144 | github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= 145 | github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= 146 | github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= 147 | github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= 148 | github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= 149 | github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= 150 | github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= 151 | github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= 152 | github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= 153 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 154 | github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 155 | github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= 156 | github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= 157 | github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= 158 | github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= 159 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 160 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 161 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 162 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 163 | github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= 164 | github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= 165 | github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= 166 | github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 167 | github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 168 | github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 169 | golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 170 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 171 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 172 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 173 | golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 174 | golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 175 | golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 176 | golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 177 | golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 178 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 179 | golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= 180 | golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 181 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 182 | golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 183 | golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= 184 | golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= 185 | golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= 186 | golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= 187 | golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= 188 | golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= 189 | golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8= 190 | golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI= 191 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 192 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 193 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 194 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 195 | golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 196 | golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 197 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 198 | golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= 199 | golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= 200 | golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 201 | golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 202 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 203 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 204 | golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 205 | golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 206 | golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 207 | golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 208 | golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 209 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 210 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 211 | golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 212 | golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 213 | golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 214 | golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= 215 | golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 216 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 217 | golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk= 218 | golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= 219 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 220 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 221 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 222 | golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= 223 | golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= 224 | golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= 225 | golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 226 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 227 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 228 | golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= 229 | golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= 230 | golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= 231 | golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= 232 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 233 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 234 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 235 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 236 | google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 237 | google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= 238 | google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= 239 | google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= 240 | google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= 241 | google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= 242 | google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= 243 | google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= 244 | google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 245 | google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= 246 | google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= 247 | google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= 248 | gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= 249 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 250 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 251 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= 252 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= 253 | gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= 254 | gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= 255 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 256 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 257 | gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 258 | gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 259 | gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 260 | gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 261 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= 262 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 263 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 264 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 265 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 266 | k8s.io/api v0.30.3 h1:ImHwK9DCsPA9uoU3rVh4QHAHHK5dTSv1nxJUapx8hoQ= 267 | k8s.io/api v0.30.3/go.mod h1:GPc8jlzoe5JG3pb0KJCSLX5oAFIW3/qNJITlDj8BH04= 268 | k8s.io/apimachinery v0.30.3 h1:q1laaWCmrszyQuSQCfNB8cFgCuDAoPszKY4ucAjDwHc= 269 | k8s.io/apimachinery v0.30.3/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= 270 | k8s.io/client-go v0.30.3 h1:bHrJu3xQZNXIi8/MoxYtZBBWQQXwy16zqJwloXXfD3k= 271 | k8s.io/client-go v0.30.3/go.mod h1:8d4pf8vYu665/kUbsxWAQ/JDBNWqfFeZnvFiVdmx89U= 272 | k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= 273 | k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= 274 | k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag= 275 | k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= 276 | k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= 277 | k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= 278 | sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= 279 | sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= 280 | sigs.k8s.io/sig-storage-lib-external-provisioner/v10 v10.0.0 h1:+OhFU21hL9Gq/sHKHfYxbc7M6RWV3UqTpnk5/wF9cP4= 281 | sigs.k8s.io/sig-storage-lib-external-provisioner/v10 v10.0.0/go.mod h1:mfQ2enu5yAHUhpNWsce9NmkqkRQsk70zQT+7KjZ+JMo= 282 | sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= 283 | sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= 284 | sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= 285 | sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= 286 | -------------------------------------------------------------------------------- /architecture.hostpath.drawio.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 |
10 |
11 |
12 | Kubernetes 13 |
14 |
15 |
16 | 17 | 18 | Kubernetes 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 |
27 |
28 |
29 | zfs-provisioner 30 |
31 |
32 |
33 | 34 | 35 | zfs-provisioner 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 |
45 |
46 |
47 | 2) Create Dataset 48 |
49 | (ssh) 50 |
51 |
52 |
53 | 54 | 55 | 2) Create Dataset... 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 |
66 |
67 |
68 | controller 69 |
70 |
71 |
72 | 73 | 74 | controll... 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 |
83 |
84 |
85 | app 86 |
87 |
88 |
89 | 90 | 91 | app 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 |
101 |
102 |
103 | 1) Request PV 104 |
105 |
106 |
107 | 108 | 109 | 1) Request PV 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 |
120 |
121 |
122 | PVC 123 |
124 |
125 |
126 | 127 | 128 | PVC 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 |
138 |
139 |
140 | 4) Bind 141 |
142 |
143 |
144 | 145 | 146 | 4) Bind 147 | 148 | 149 | 150 | 151 | 152 | 153 | 154 | 155 |
156 |
157 |
158 | 5) Consuming 159 |
160 | Pods scheduled 161 |
162 | to this node 163 |
164 |
165 |
166 | 167 | 168 | 5) Consuming... 169 | 170 | 171 | 172 | 173 | 174 | 175 | 176 | 177 | 178 |
179 |
180 |
181 | PV (Hostpath) 182 |
183 |
184 |
185 | 186 | 187 | PV (Host... 188 | 189 | 190 | 191 | 192 | 193 | 194 | 195 | 196 |
197 |
198 |
199 | 3) Provision PV 200 |
201 | with NodeAffinity 202 |
203 |
204 |
205 | 206 | 207 | 3) Provision PV... 208 | 209 | 210 | 211 | 212 | 213 | 214 | 215 | 216 | 217 |
218 |
219 |
220 | 221 | Kubernetes 222 | 223 | 224 | Node 225 |
226 | with local ZFS 227 | 228 |
229 |
230 |
231 | 232 | 233 | Kubernet... 234 | 235 | 236 | 237 | 238 | 239 | 240 | 241 | 242 | Viewer does not support full SVG 1.1 243 | 244 | 245 | 246 | 247 | --------------------------------------------------------------------------------