├── .github └── workflows │ ├── codeql-analysis.yaml │ ├── codespell.yml │ ├── controller-chart-release.yaml │ ├── csi-chart-release.yaml │ ├── linux.yaml │ ├── pluto.yaml │ └── test.yaml ├── .gitignore ├── .golangci.yml ├── CONTRIBUTING.md ├── LICENSE ├── Makefile ├── README.md ├── charts ├── README.md ├── sb-controller │ ├── index.yaml │ ├── latest │ │ ├── .helmignore │ │ ├── Chart.yaml │ │ ├── templates │ │ │ ├── caching-node-controller.yaml │ │ │ ├── config-map.yaml │ │ │ ├── controller-role.yaml │ │ │ ├── job.yaml │ │ │ └── storage-node-controller.yaml │ │ └── values.yaml │ └── v0.1.0 │ │ └── sb-controller-v0.1.0.tgz └── spdk-csi │ ├── index.yaml │ ├── latest │ └── spdk-csi │ │ ├── .helmignore │ │ ├── Chart.yaml │ │ ├── templates │ │ ├── NOTES.txt │ │ ├── _helpers.tpl │ │ ├── benchmark.yaml │ │ ├── caching-node-handler.yaml │ │ ├── caching-node.yaml │ │ ├── config-map.yaml │ │ ├── controller-rbac.yaml │ │ ├── controller.yaml │ │ ├── driver.yaml │ │ ├── node-rbac.yaml │ │ ├── node.yaml │ │ ├── nodeserver-config-map.yaml │ │ ├── rbac-snapshot-controller.yaml │ │ ├── secret.yaml │ │ ├── setup-snapshot-controller.yaml │ │ ├── snapshot.storage.k8s.io_volumesnapshotclasses.yaml │ │ ├── snapshot.storage.k8s.io_volumesnapshotcontents.yaml │ │ ├── snapshot.storage.k8s.io_volumesnapshots.yaml │ │ ├── snapshotclass.yaml │ │ ├── storage-node.yaml │ │ └── storageclass.yaml │ │ └── values.yaml │ ├── v0.3.0 │ └── spdk-csi-v0.3.0.tgz │ ├── v0.3.1 │ └── spdk-csi-v0.3.1.tgz │ ├── v0.3.10 │ └── spdk-csi-v0.3.10.tgz │ ├── v0.3.11 │ └── spdk-csi-v0.3.11.tgz │ ├── v0.3.12 │ └── spdk-csi-v0.3.12.tgz │ ├── v0.3.13 │ └── spdk-csi-v0.3.13.tgz │ ├── v0.3.14 │ └── spdk-csi-v0.3.14.tgz │ ├── v0.3.15 │ └── spdk-csi-v0.3.15.tgz │ ├── v0.3.16 │ └── spdk-csi-v0.3.16.tgz │ ├── v0.3.17 │ └── spdk-csi-v0.3.17.tgz │ ├── v0.3.18 │ └── spdk-csi-v0.3.18.tgz │ ├── v0.3.2 │ └── spdk-csi-v0.3.2.tgz │ ├── v0.3.3 │ └── spdk-csi-v0.3.3.tgz │ ├── v0.3.4 │ └── spdk-csi-v0.3.4.tgz │ ├── v0.3.5 │ └── spdk-csi-v0.3.5.tgz │ ├── v0.3.6 │ └── spdk-csi-v0.3.6.tgz │ ├── v0.3.7 │ └── spdk-csi-v0.3.7.tgz │ ├── v0.3.8 │ └── spdk-csi-v0.3.8.tgz │ └── v0.3.9 │ └── spdk-csi-v0.3.9.tgz ├── cmd └── main.go ├── deploy ├── image │ └── Dockerfile ├── kubernetes │ ├── caching-node.yaml │ ├── config-map.yaml │ ├── controller-rbac.yaml │ ├── controller.yaml │ ├── deploy.sh │ ├── driver.yaml │ ├── job.yaml │ ├── mysql-pvc.yaml │ ├── node-rbac.yaml │ ├── node.yaml │ ├── nodeserver-config-map.yaml │ ├── prepare-cachingnode.sh │ ├── rbac-snapshot-controller.yaml │ ├── secret.yaml │ ├── setup-snapshot-controller.yaml │ ├── snapshot.storage.k8s.io_volumesnapshotclasses.yaml │ ├── snapshot.storage.k8s.io_volumesnapshotcontents.yaml │ ├── snapshot.storage.k8s.io_volumesnapshots.yaml │ ├── snapshotclass.yaml │ ├── storageclass.yaml │ ├── testclone.yaml │ ├── testpod-cache.yaml │ ├── testpod.yaml │ ├── testresize.yaml │ ├── testrestore.yaml │ └── testsnapshot.yaml └── spdk │ ├── Dockerfile │ ├── README.md │ └── sma.yaml ├── docs ├── caching-nodes.md ├── csi-debug.md ├── encrypted-volumes.md ├── install-simplyblock-csi-driver.md ├── static-pvc.md ├── storage-nodes.md └── support-ports.md ├── e2e ├── clone.go ├── controlplane.go ├── e2e.go ├── e2e_test.go ├── nvmeof.go ├── restartnode.go ├── snapshot.go ├── templates │ ├── multi-pvc.yaml │ ├── pvc-cache.yaml │ ├── pvc.yaml │ ├── testpod-cache.yaml │ ├── testpod-clone.yaml │ ├── testpod-multi-pvc.yaml │ ├── testpod-snapshot.yaml │ ├── testpod-snapshot2.yaml │ └── testpod.yaml └── utils.go ├── go.mod ├── go.sum ├── pkg ├── csi-common │ ├── controllerserver-default.go │ ├── driver.go │ ├── identityserver-default.go │ ├── nodeserver-default.go │ ├── server.go │ └── utils.go ├── spdk │ ├── controllerserver.go │ ├── controllerserver_test.go │ ├── driver.go │ ├── identityserver.go │ └── nodeserver.go └── util │ ├── config.go │ ├── idlocker.go │ ├── idlocker_test.go │ ├── initiator.go │ ├── initiator_test.go │ ├── jsonrpc.go │ ├── nvmf.go │ ├── nvmf_test.go │ ├── util.go │ └── util_test.go ├── scripts ├── ci │ ├── common.sh │ ├── env │ ├── mdl_rules.rb │ ├── prepare.sh │ ├── spdkcsi-ci.yaml │ └── test.sh ├── config-apply.sh ├── config-gen-upgrade.sh ├── golangci.yml ├── install-helm.sh ├── install-snapshot.sh ├── minikube.sh ├── verify-helm-yamllint.sh └── yamllint.yml └── test └── sanity ├── README.md ├── params.yaml ├── run-test.sh └── secrets.yaml /.github/workflows/codeql-analysis.yaml: -------------------------------------------------------------------------------- 1 | 2 | # For most projects, this workflow file will not need changing; you simply need 3 | # to commit it to your repository. 4 | # 5 | # You may wish to alter this file to override the set of languages analyzed, 6 | # or to provide custom queries or build logic. 7 | # 8 | # ******** NOTE ******** 9 | # We have attempted to detect the languages in your repository. Please check 10 | # the `language` matrix defined below to confirm you have the correct set of 11 | # supported CodeQL languages. 12 | # 13 | name: "CodeQL" 14 | 15 | on: 16 | push: 17 | branches: [ master, 'dev', 'release-**' ] 18 | pull_request: 19 | # The branches below must be a subset of the branches above 20 | branches: [ master, 'dev', 'release-**' ] 21 | schedule: 22 | - cron: '0 */24 * * *' 23 | 24 | jobs: 25 | analyze: 26 | name: Analyze 27 | runs-on: ubuntu-latest 28 | permissions: 29 | actions: read 30 | contents: read 31 | security-events: write 32 | 33 | strategy: 34 | fail-fast: false 35 | matrix: 36 | language: [ 'go' ] 37 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] 38 | # Learn more about CodeQL language support at https://git.io/codeql-language-support 39 | 40 | steps: 41 | - name: Set up Go 1.x 42 | uses: actions/setup-go@v5 43 | with: 44 | go-version: ^1.18 45 | id: go 46 | 47 | - name: Checkout repository 48 | uses: actions/checkout@v4 49 | 50 | - name: Tidy Go modules 51 | run: | 52 | go mod tidy 53 | 54 | # Initializes the CodeQL tools for scanning. 55 | - name: Initialize CodeQL 56 | uses: github/codeql-action/init@v3 57 | with: 58 | languages: ${{ matrix.language }} 59 | # If you wish to specify custom queries, you can do so here or in a config file. 60 | # By default, queries listed here will override any specified in a config file. 61 | # Prefix the list here with "+" to use these queries and those in the config file. 62 | # queries: ./path/to/local/query, your-org/your-repo/queries@main 63 | 64 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 65 | # If this step fails, then you should remove it and run the build manually (see below) 66 | # - name: Autobuild 67 | # run: | 68 | # make all 69 | # - name: Perform CodeQL Analysis 70 | # uses: github/codeql-action/analyze@v3 71 | -------------------------------------------------------------------------------- /.github/workflows/codespell.yml: -------------------------------------------------------------------------------- 1 | # GitHub Action to automate the identification of common misspellings in text files. 2 | # https://github.com/codespell-project/actions-codespell 3 | # https://github.com/codespell-project/codespell 4 | name: codespell 5 | on: 6 | push: 7 | branches: ['master', 'dev'] 8 | pull_request: 9 | 10 | jobs: 11 | codespell: 12 | name: Check for spelling errors 13 | runs-on: ubuntu-latest 14 | steps: 15 | - uses: actions/checkout@v4 16 | - uses: codespell-project/actions-codespell@master 17 | with: 18 | check_filenames: true 19 | skip: ./.git,./.github/workflows/codespell.yml,.git,*.png,*.jpg,*.svg,*.sum,./vendor,go.sum 20 | ignore_words_list: "browseable,ro" 21 | -------------------------------------------------------------------------------- /.github/workflows/controller-chart-release.yaml: -------------------------------------------------------------------------------- 1 | name: SImplyBlock Controller Release Charts 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | paths: 8 | - 'charts/sb-controller/latest/Chart.yaml' 9 | 10 | jobs: 11 | package-and-update: 12 | runs-on: ubuntu-latest 13 | 14 | steps: 15 | - name: Checkout repo 16 | uses: actions/checkout@v3 17 | 18 | - name: Set up Helm 19 | uses: azure/setup-helm@v3 20 | with: 21 | version: v3.13.0 22 | 23 | - name: Extract version and package chart 24 | run: | 25 | cd charts/sb-controller/latest/ 26 | VERSION="$(grep '^version:' Chart.yaml | awk '{print $2}' | tr -d '"')" 27 | echo "VERSION=$VERSION" >> $GITHUB_ENV 28 | 29 | helm package . 30 | mkdir -p "../$VERSION" 31 | 32 | mv "sb-controller-$VERSION.tgz" "../$VERSION/" 33 | 34 | - name: Update index.yaml 35 | run: | 36 | cd charts/sb-controller 37 | helm repo index . --url https://github.com/simplyblock-io/spdk-csi/raw/master/charts/sb-controller 38 | 39 | - name: Commit and create PR 40 | run: | 41 | git config user.name "github-actions[bot]" 42 | git config user.email "github-actions[bot]@users.noreply.github.com" 43 | 44 | BRANCH="release-${{ env.VERSION }}" 45 | git checkout -b "$BRANCH" 46 | 47 | git add charts/sb-controller/index.yaml charts/sb-controller/${{ env.VERSION }}/ 48 | git commit -m "helm release for simplyblock csi ${{ env.VERSION }}" || echo "No changes to commit" 49 | git push origin "$BRANCH" 50 | 51 | gh pr create \ 52 | --title "Helm release: ${{ env.VERSION }}" \ 53 | --body "Automated Helm release for simplyblock storage controller ${{ env.VERSION}}" \ 54 | --head "$BRANCH" \ 55 | --base master 56 | env: 57 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 58 | -------------------------------------------------------------------------------- /.github/workflows/csi-chart-release.yaml: -------------------------------------------------------------------------------- 1 | name: CSI Release Charts 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | paths: 8 | - 'charts/spdk-csi/latest/spdk-csi/Chart.yaml' 9 | 10 | jobs: 11 | package-and-update: 12 | runs-on: ubuntu-latest 13 | 14 | steps: 15 | - name: Checkout repo 16 | uses: actions/checkout@v3 17 | 18 | - name: Set up Helm 19 | uses: azure/setup-helm@v3 20 | with: 21 | version: v3.13.0 22 | 23 | - name: Extract version and package chart 24 | run: | 25 | cd charts/spdk-csi/latest/ 26 | VERSION="$(grep '^version:' spdk-csi/Chart.yaml | awk '{print $2}' | tr -d '"')" 27 | echo "VERSION=$VERSION" >> $GITHUB_ENV 28 | 29 | helm package spdk-csi 30 | mkdir -p "../$VERSION" 31 | 32 | mv "spdk-csi-$VERSION.tgz" "../$VERSION/" 33 | 34 | - name: Update index.yaml 35 | run: | 36 | cd charts/spdk-csi 37 | helm repo index . --url https://github.com/simplyblock-io/spdk-csi/raw/master/charts/spdk-csi 38 | 39 | - name: Commit and create PR 40 | run: | 41 | git config user.name "github-actions[bot]" 42 | git config user.email "github-actions[bot]@users.noreply.github.com" 43 | 44 | BRANCH="release-${{ env.VERSION }}" 45 | git checkout -b "$BRANCH" 46 | 47 | git add charts/spdk-csi/index.yaml charts/spdk-csi/${{ env.VERSION }}/ 48 | git commit -m "helm release for simplyblock csi ${{ env.VERSION }}" || echo "No changes to commit" 49 | git push origin "$BRANCH" 50 | 51 | gh pr create \ 52 | --title "Helm release: ${{ env.VERSION }}" \ 53 | --body "Automated Helm release for simplyblock-csi ${{ env.VERSION}}" \ 54 | --head "$BRANCH" \ 55 | --base master 56 | env: 57 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 58 | -------------------------------------------------------------------------------- /.github/workflows/linux.yaml: -------------------------------------------------------------------------------- 1 | name: Linux Unit tests and docker push 2 | on: 3 | pull_request: 4 | push: 5 | branches: ['master', 'dev'] 6 | release: 7 | types: [created] 8 | 9 | jobs: 10 | 11 | build: 12 | name: Build 13 | runs-on: ubuntu-latest 14 | steps: 15 | - name: Set up Go 1.x 16 | uses: actions/setup-go@v5 17 | with: 18 | go-version: ^1.21 19 | id: go 20 | 21 | - name: Check out code into the Go module directory 22 | uses: actions/checkout@v4 23 | 24 | - name: Tidy Go modules 25 | run: | 26 | go mod tidy 27 | 28 | - name: Set up Docker Buildx 29 | uses: docker/setup-buildx-action@v2 30 | 31 | - name: Install QEMU 32 | uses: docker/setup-qemu-action@v2 33 | with: 34 | platforms: linux/arm64,linux/amd64 35 | 36 | - name: Enable Docker BuildKit and Buildx 37 | run: | 38 | export DOCKER_CLI_EXPERIMENTAL=enabled 39 | export DOCKER_BUILDKIT=1 40 | docker buildx create --use 41 | 42 | - name: Test 43 | run: | 44 | make test 45 | 46 | - name: Set docker image tag 47 | id: get_info 48 | run: | 49 | if [[ "${{ github.event_name }}" == "release" ]]; then 50 | echo "TAG=${{ github.event.release.tag_name }}" >> $GITHUB_ENV 51 | echo "BRANCH=release" >> $GITHUB_ENV 52 | echo "SHA=${{ github.event.release.target_commitish }}" >> $GITHUB_ENV 53 | elif [[ "${{ github.ref }}" == refs/pull/* ]]; then 54 | tag=${GITHUB_REF/\/merge/} 55 | echo "TAG=$(echo pr-${tag:10})" >> $GITHUB_ENV 56 | echo "BRANCH=${GITHUB_HEAD_REF}" >> $GITHUB_ENV 57 | echo "SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV 58 | else 59 | echo "TAG=$(echo ${GITHUB_REF#refs/*/} | sed 's/\//-/g')" >> $GITHUB_ENV 60 | echo "BRANCH=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV 61 | echo "SHA=$GITHUB_SHA" >> $GITHUB_ENV 62 | fi 63 | 64 | - name: Login to Docker Hub 65 | uses: docker/login-action@v1 66 | with: 67 | username: ${{ secrets.DOCKERHUB_USERNAME }} 68 | password: ${{ secrets.DOCKERHUB_TOKEN }} 69 | 70 | - name: Configure AWS Credentials 71 | uses: aws-actions/configure-aws-credentials@v1 72 | with: 73 | aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_GEOFFREY }} 74 | aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_GEOFFREY }} 75 | aws-region: us-east-1 76 | 77 | - name: Login to ECR 78 | id: login-ecr 79 | uses: aws-actions/amazon-ecr-login@v2 80 | with: 81 | registry-type: public 82 | 83 | - name: Build & Push 84 | run: | 85 | SHORT_SHA=$(echo ${{ env.SHA }} | head -c 8) 86 | make image 87 | 88 | # Docker Hub Tags 89 | docker tag simplyblock/spdkcsi:latest-amd64 simplyblock/spdkcsi:$TAG-amd64 90 | docker tag simplyblock/spdkcsi:latest-arm64 simplyblock/spdkcsi:$TAG-arm64 91 | 92 | 93 | # ECR Tags 94 | docker tag simplyblock/spdkcsi:latest-amd64 public.ecr.aws/simply-block/spdkcsi:$TAG-amd64 95 | docker tag simplyblock/spdkcsi:latest-arm64 public.ecr.aws/simply-block/spdkcsi:$TAG-arm64 96 | 97 | # Push to Docker Hub 98 | docker push simplyblock/spdkcsi:$TAG-amd64 99 | docker push simplyblock/spdkcsi:$TAG-arm64 100 | 101 | # Push to Amazon ECR 102 | docker push public.ecr.aws/simply-block/spdkcsi:$TAG-amd64 103 | docker push public.ecr.aws/simply-block/spdkcsi:$TAG-arm64 104 | 105 | 106 | # Create multi-arch manifests 107 | 108 | if [ "${{ github.event_name }}" = "release" ]; then 109 | docker buildx imagetools create -t simplyblock/spdkcsi:$TAG simplyblock/spdkcsi:$TAG-amd64 simplyblock/spdkcsi:$TAG-arm64 110 | docker buildx imagetools create -t public.ecr.aws/simply-block/spdkcsi:$TAG public.ecr.aws/simply-block/spdkcsi:$TAG-amd64 public.ecr.aws/simply-block/spdkcsi:$TAG-arm64 111 | 112 | elif [ "${{ github.ref }}" = "refs/heads/master" ]; then 113 | docker buildx imagetools create -t simplyblock/spdkcsi:latest simplyblock/spdkcsi:$TAG-amd64 simplyblock/spdkcsi:$TAG-arm64 114 | docker buildx imagetools create -t public.ecr.aws/simply-block/spdkcsi:latest public.ecr.aws/simply-block/spdkcsi:$TAG-amd64 public.ecr.aws/simply-block/spdkcsi:$TAG-arm64 115 | 116 | else 117 | docker buildx imagetools create -t simplyblock/spdkcsi:$TAG simplyblock/spdkcsi:$TAG-amd64 simplyblock/spdkcsi:$TAG-arm64 118 | docker buildx imagetools create -t public.ecr.aws/simply-block/spdkcsi:$TAG public.ecr.aws/simply-block/spdkcsi:$TAG-amd64 public.ecr.aws/simply-block/spdkcsi:$TAG-arm64 119 | docker buildx imagetools create -t simplyblock/spdkcsi:$TAG-$SHORT_SHA simplyblock/spdkcsi:$TAG-amd64 simplyblock/spdkcsi:$TAG-arm64 120 | docker buildx imagetools create -t public.ecr.aws/simply-block/spdkcsi:$TAG-$SHORT_SHA public.ecr.aws/simply-block/spdkcsi:$TAG-amd64 public.ecr.aws/simply-block/spdkcsi:$TAG-arm64 121 | fi 122 | -------------------------------------------------------------------------------- /.github/workflows/pluto.yaml: -------------------------------------------------------------------------------- 1 | name: k8s api version check 2 | on: 3 | pull_request: 4 | push: 5 | branches: ['master', 'dev'] 6 | 7 | jobs: 8 | 9 | build: 10 | name: Build 11 | runs-on: ubuntu-latest 12 | steps: 13 | 14 | - name: Checkout 15 | uses: actions/checkout@v4 16 | 17 | # https://pluto.docs.fairwinds.com/advanced/#display-options 18 | - name: Download pluto 19 | uses: FairwindsOps/pluto/github-action@master 20 | 21 | - name: Check deploy folder 22 | run: | 23 | pluto detect-files -d deploy --ignore-deprecations --ignore-removals 24 | - name: Check kubernetes folder 25 | run: | 26 | pluto detect-files -d deploy/kubernetes 27 | -------------------------------------------------------------------------------- /.github/workflows/test.yaml: -------------------------------------------------------------------------------- 1 | name: SPDK E2E TEST 2 | 3 | on: 4 | workflow_dispatch: 5 | inputs: 6 | uuid: 7 | description: '' 8 | required: true 9 | default: '79276661-5f8a-405d-ab6d-651b88326206' 10 | ip: 11 | description: '' 12 | required: true 13 | default: '18.218.243.112' 14 | secret: 15 | description: '' 16 | required: true 17 | default: '8K9BVlNfYo6aiLamu1c2' 18 | jobs: 19 | build: 20 | name: Build 21 | runs-on: ubuntu-latest 22 | steps: 23 | - name: Set up Go 1.x 24 | uses: actions/setup-go@v5 25 | with: 26 | go-version: ^1.21 27 | id: go 28 | 29 | - name: Check out code into the Go module directory 30 | uses: actions/checkout@v4 31 | 32 | - name: Install kubectl 33 | run: | 34 | curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" 35 | sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl 36 | 37 | - name: Install Helm 38 | run: | 39 | curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 40 | chmod 700 get_helm.sh 41 | ./get_helm.sh 42 | 43 | - name: Authenticate with Kubernetes cluster 44 | run: | 45 | mkdir -p "${HOME}/.kube" 46 | echo ${{ secrets.KUBE_CONFIG_DATA }} | base64 --decode > ${HOME}/.kube/config 47 | 48 | - name: Install SPDK-CSI using Helm 49 | run: | 50 | cd spdk-csi/charts/spdk-csi 51 | helm install spdk-csi ./ \ 52 | --set csiConfig.simplybk.uuid=${{ github.event.inputs.uuid }} \ 53 | --set csiConfig.simplybk.ip=${{ github.event.inputs.ip }} \ 54 | --set csiSecret.simplybk.secret=${{ github.event.inputs.secret }} \ 55 | --set logicalVolume.pool_name=testing1 56 | 57 | - name: Run tests 58 | run: make e2e-test 59 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /_out 2 | /vendor 3 | 4 | *.orig 5 | *.patch 6 | *.rej 7 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/simplyblock-io/simplyblock-csi/5f1784f36c6badb115f32dccda18a5083fb6e411/.golangci.yml -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | Please see the [SPDK development guide](http://www.spdk.io/development/) for information on how to contribute to SPDK. 4 | 5 | The code repo for SPDK-CSI is [https://review.spdk.io/spdk/spdk-csi](https://review.spdk.io/spdk/spdk-csi). 6 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Simplyblock CSI Driver for Kubernetes 2 | 3 | This repo contains Simplyblock CSI ([Container Storage Interface](https://github.com/container-storage-interface/)) 4 | plugin for Kubernetes. 5 | 6 | Simplyblock CSI plugin brings high performance block storage to Kubernetes. It provisions SPDK logical volumes on storage node dynamically and enables Pods to access SPDK storage backend through NVMe-oF . 7 | 8 | Most parts of the CSI driver are vey much similar to the original [SPDK CSI Design Document](https://docs.google.com/document/d/1aLi6SkNBp__wjG7YkrZu7DdhoftAquZiWiIOMy3hskY/) 9 | 10 | ### Supported Features 11 | - Dynamic Volume Provisioning 12 | - Dynamic Volume Provisioning for Caching nodes 13 | - Volume Snapshots 14 | 15 | ### Container Images & Kubernetes Compatibility: 16 | | driver version | supported k8s version | status | 17 | | -------------- | --------------------- | ------ | 18 | | master branch | 1.21+ | Stable | 19 | | v0.1.0 | 1.21+ | Beta | 20 | | v0.1.1 | 1.21+ | Stable | 21 | 22 | ### Install driver on a Kubernetes cluster 23 | - install via [helm charts](./charts) 24 | - install via [kubectl](./docs/install-simplyblock-csi-driver.md) 25 | 26 | ### Driver parameters 27 | Please refer to [`csi.simplyblock.io` driver parameters](./charts/README.md#driver-parameters) 28 | 29 | ### Troubleshooting 30 | - [CSI driver troubleshooting guide](./docs/csi-debug.md) 31 | 32 | ### Supported Worker node types 33 | 34 | The CSI driver is currently tested with various types of worker nodes. 35 | 36 | On AWS EKS the following worker nodes types are supported: 37 | * AmazonLinux2 (AL_2_x86_64) 38 | * AmazonLinux2023 (AL_2023_x86_64_STANDARD) 39 | 40 | On K3S the following Worker nodes are supported: 41 | * RHEL9 42 | * Ubuntu 22.04 43 | * AmazonLinux2 44 | * AmazonLinux2023 45 | -------------------------------------------------------------------------------- /charts/sb-controller/index.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | entries: 3 | sb-controller: 4 | - apiVersion: v1 5 | appVersion: latest 6 | created: "2025-05-30T10:50:16.464925568Z" 7 | description: Simplyblock Storage Controller for Kubernetes 8 | digest: 95f2805131c4d71212f375fc68e96e4544fe62cfd5f27734b541f7fc08d2442b 9 | home: https://github.com/simplyblock-io/spdk-csi 10 | keywords: 11 | - spdk 12 | - simplyblock 13 | - controller 14 | name: sb-controller 15 | sources: 16 | - https://github.com/simplyblock-io/spdk-csi/tree/master/charts/sb-controller 17 | urls: 18 | - https://github.com/simplyblock-io/spdk-csi/raw/master/charts/sb-controller/v0.1.0/sb-controller-v0.1.0.tgz 19 | version: v0.1.0 20 | generated: "2025-05-30T10:50:16.464472794Z" 21 | -------------------------------------------------------------------------------- /charts/sb-controller/latest/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *.orig 18 | *~ 19 | # Various IDEs 20 | .project 21 | .idea/ 22 | *.tmproj 23 | .vscode/ 24 | -------------------------------------------------------------------------------- /charts/sb-controller/latest/Chart.yaml: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | # Copyright (c) Arm Limited and Contributors 3 | # Copyright (c) Intel Corporation 4 | --- 5 | apiVersion: v1 6 | appVersion: latest 7 | description: Simplyblock Storage Controller for Kubernetes 8 | name: sb-controller 9 | version: "v0.1.0" 10 | keywords: 11 | - spdk 12 | - simplyblock 13 | - controller 14 | home: https://github.com/simplyblock-io/spdk-csi 15 | sources: 16 | - https://github.com/simplyblock-io/spdk-csi/tree/master/charts/sb-controller 17 | 18 | -------------------------------------------------------------------------------- /charts/sb-controller/latest/templates/caching-node-controller.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.cachingnode.create -}} 2 | --- 3 | apiVersion: v1 4 | kind: ServiceAccount 5 | metadata: 6 | name: simplyblock-caching-node-service-account 7 | 8 | --- 9 | apiVersion: apps/v1 10 | kind: Deployment 11 | metadata: 12 | name: simplyblock-caching-node-controller 13 | labels: 14 | app: caching-node-controller 15 | spec: 16 | replicas: 1 17 | selector: 18 | matchLabels: 19 | app: caching-node-controller 20 | template: 21 | metadata: 22 | labels: 23 | app: caching-node-controller 24 | spec: 25 | serviceAccountName: simplyblock-caching-node-service-account 26 | containers: 27 | - name: caching-node-controller 28 | image: "{{ .Values.image.cachingNode.repository }}:{{ .Values.image.cachingNode.tag }}" 29 | imagePullPolicy: "{{ .Values.image.cachingNode.pullPolicy }}" 30 | env: 31 | - name: SPDKCSI_SECRET 32 | valueFrom: 33 | secretKeyRef: 34 | name: simplyblock-csi-secret 35 | key: secret.json 36 | - name: CLUSTER_CONFIG 37 | valueFrom: 38 | configMapKeyRef: 39 | name: simplyblock-csi-cm 40 | key: config.json 41 | - name: IFNAME 42 | value: "{{ .Values.cachingnode.ifname }}" 43 | - name: CPUMASK 44 | value: "{{ .Values.cachingnode.cpuMask }}" 45 | - name: SPDKMEM 46 | value: "{{ .Values.cachingnode.spdkMem }}" 47 | - name: SPDKIMAGE 48 | value: "{{ .Values.cachingnode.spdkImage }}" 49 | - name: MULTIPATHING 50 | value: "{{ .Values.cachingnode.multipathing }}" 51 | 52 | - name: NAMESPACE 53 | valueFrom: 54 | fieldRef: 55 | fieldPath: metadata.namespace 56 | restartPolicy: Always 57 | 58 | {{- end -}} 59 | -------------------------------------------------------------------------------- /charts/sb-controller/latest/templates/config-map.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: simplyblock-mgmt-api-script-cm 5 | data: 6 | action.py: | 7 | import json 8 | import os 9 | import requests 10 | from time import sleep 11 | from kubernetes import client, config 12 | 13 | config.load_incluster_config() 14 | v1 = client.CoreV1Api() 15 | 16 | def get_node_statuses(cluster_ip, cluster_uuid, cluster_secret): 17 | url = f"{cluster_ip}/storagenode" 18 | headers = { 19 | "Content-Type": "application/json", 20 | "Authorization": f"{cluster_uuid} {cluster_secret}" 21 | } 22 | 23 | try: 24 | response = requests.get(url, headers=headers) 25 | if response.status_code == 200: 26 | results = response.json().get('results', []) 27 | return results 28 | else: 29 | print(f"Failed to get node statuses, Status Code: {response.status_code}") 30 | except Exception as e: 31 | print(f"Error occurred while getting node statuses: {e}") 32 | return [] 33 | 34 | def activate_cluster_if_needed(cluster_ip, cluster_uuid, cluster_secret, distr_ndcs, distr_npcs, label_selector): 35 | nodes = v1.list_node(label_selector=label_selector) 36 | 37 | total_nodes = int(len(nodes.items)) 38 | total_value = distr_ndcs + distr_npcs + 1 39 | retries = total_value * 60 40 | while retries > 0: 41 | node_statuses = get_node_statuses(cluster_ip, cluster_uuid, cluster_secret) 42 | online_nodes = [node for node in node_statuses if node.get('status') == 'online' and len(node.get('nvme_devices'))>0] 43 | 44 | if len(online_nodes) >= total_nodes and len(online_nodes) >= total_value : 45 | print("Proceeding with cluster activation.") 46 | url = f"{cluster_ip}/cluster/activate/{cluster_uuid}" 47 | headers = { 48 | "Content-Type": "application/json", 49 | "Authorization": f"{cluster_uuid} {cluster_secret}" 50 | } 51 | try: 52 | response = requests.put(url, headers=headers) 53 | if response.status_code == 200: 54 | print("Successfully activated the cluster.") 55 | return 56 | else: 57 | print(f"Failed to activate the cluster, Status Code: {response.status_code}") 58 | except Exception as e: 59 | print(f"Error occurred while activating the cluster: {e}") 60 | return 61 | 62 | retries -= 1 63 | print(f"Not enough 'online' nodes. Retrying in 5 seconds... Remaining retries: {retries}") 64 | sleep(5) 65 | 66 | print(f"Cluster not activated: Number of 'online' storage nodes is less than {total_value} after maximum retries.") 67 | 68 | print("Loaded environment variables") 69 | action_type = os.getenv("ACTION_TYPE") 70 | uuid = os.getenv("SNODE_UUID", "") 71 | distr_ndcs = int(os.getenv("DISTR_NDCS", 1)) if os.getenv("DISTR_NDCS", "").isdigit() else 1 72 | distr_npcs = int(os.getenv("DISTR_NPCS", 1)) if os.getenv("DISTR_NPCS", "").isdigit() else 1 73 | 74 | label_selector = "type=simplyblock-storage-plane" 75 | 76 | secret = json.loads(os.getenv("SPDKCSI_SECRET")) 77 | cluster_secret = secret['simplybk']['secret'] 78 | 79 | cluster_config = json.loads(os.getenv("CLUSTER_CONFIG")) 80 | namespace = os.getenv("NAMESPACE", "default") 81 | cluster_uuid = cluster_config['simplybk']['uuid'] 82 | cluster_ip = cluster_config['simplybk']['ip'] 83 | headers = { 84 | "Content-Type": "application/json", 85 | "Authorization": f"{cluster_uuid} {cluster_secret}" 86 | } 87 | 88 | print(f"action type: {action_type}. performing appropriate action") 89 | if action_type == "cl_activate": 90 | # Check if we should activate the cluster 91 | activate_cluster_if_needed(cluster_ip, cluster_uuid, cluster_secret, distr_ndcs, distr_npcs, label_selector) 92 | elif action_type in ["sn_restart", "sn_shutdown", "sn_remove", "make_sec_primary"] and uuid: 93 | if action_type == "sn_restart": 94 | url = f"{cluster_ip}/storagenode/restart/{uuid}" 95 | elif action_type == "sn_shutdown": 96 | url = f"{cluster_ip}/storagenode/shutdown/{uuid}?force=True" 97 | elif action_type == "sn_remove": 98 | url = f"{cluster_ip}/storagenode/remove/{uuid}" 99 | elif action_type == "make_sec_primary": 100 | url = f"{cluster_ip}/storagenode/make-sec-new-primary/{uuid}" 101 | 102 | try: 103 | response = requests.get(url, headers=headers) 104 | if response.status_code == 200: 105 | print(f"Successfully executed action: {action_type} for UUID: {uuid}") 106 | else: 107 | print(f"Failed to execute action: {action_type} for UUID: {uuid}, Status Code: {response.status_code}") 108 | except Exception as e: 109 | print(f"Error occurred: {e}") 110 | elif action_type == "sn_idle": 111 | print("No action needed for 'sn_idle'.") 112 | else: 113 | print(f"Invalid action type or UUID is missing for action: {action_type}") 114 | exit(1) 115 | -------------------------------------------------------------------------------- /charts/sb-controller/latest/templates/controller-role.yaml: -------------------------------------------------------------------------------- 1 | {{- if or .Values.cachingnode.create .Values.storagenode.create -}} 2 | kind: ClusterRoleBinding 3 | apiVersion: rbac.authorization.k8s.io/v1 4 | metadata: 5 | name: simplyblock-pod-listing-role-binding 6 | subjects: 7 | {{- if .Values.cachingnode.create }} 8 | - kind: ServiceAccount 9 | name: simplyblock-caching-node-service-account 10 | namespace: {{ .Release.Namespace }} 11 | {{- end }} 12 | {{- if .Values.storagenode.create }} 13 | - kind: ServiceAccount 14 | name: simplyblock-storage-node-service-account 15 | namespace: {{ .Release.Namespace }} 16 | {{- end }} 17 | roleRef: 18 | kind: ClusterRole 19 | name: simplyblock-pod-listing-role 20 | apiGroup: rbac.authorization.k8s.io 21 | 22 | --- 23 | kind: ClusterRole 24 | apiVersion: rbac.authorization.k8s.io/v1 25 | metadata: 26 | name: simplyblock-pod-listing-role 27 | rules: 28 | - apiGroups: [""] 29 | resources: ["pods"] 30 | verbs: ["get", "list", "watch"] 31 | - apiGroups: [""] 32 | resources: ["nodes"] 33 | verbs: ["get", "list", "watch"] 34 | - apiGroups: [""] 35 | resources: ["configmaps"] 36 | verbs: ["get", "list", "watch", "create", "update", "patch"] 37 | 38 | {{- end -}} 39 | -------------------------------------------------------------------------------- /charts/sb-controller/latest/templates/job.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.storagenode.create -}} 2 | --- 3 | apiVersion: batch/v1 4 | kind: Job 5 | metadata: 6 | name: simplyblock-mgmt-api-job 7 | spec: 8 | template: 9 | spec: 10 | volumes: 11 | - name: script-config 12 | configMap: 13 | name: simplyblock-mgmt-api-script-cm 14 | serviceAccountName: simplyblock-storage-node-service-account 15 | containers: 16 | - name: mgmt-api 17 | image: "{{ .Values.image.mgmtAPI.repository }}:{{ .Values.image.mgmtAPI.tag }}" 18 | command: ["/bin/sh", "-c"] 19 | imagePullPolicy: "{{ .Values.image.mgmtAPI.pullPolicy }}" 20 | args: 21 | - "pip install requests kubernetes && python3 /config/action.py" 22 | env: 23 | - name: SPDKCSI_SECRET 24 | valueFrom: 25 | secretKeyRef: 26 | name: simplyblock-csi-secret 27 | key: secret.json 28 | - name: CLUSTER_CONFIG 29 | valueFrom: 30 | configMapKeyRef: 31 | name: simplyblock-csi-cm 32 | key: config.json 33 | - name: NAMESPACE 34 | valueFrom: 35 | fieldRef: 36 | fieldPath: metadata.namespace 37 | - name: DISTR_NDCS 38 | value: "{{ .Values.storagenode.distr_ndcs }}" 39 | - name: DISTR_NPCS 40 | value: "{{ .Values.storagenode.distr_npcs }}" 41 | - name: ACTION_TYPE 42 | value: "cl_activate" #options: "sn_idle", "sn_restart", "sn_shutdown", "sn_remove", "cl_activate", "make_sec_primary" 43 | - name: SNODE_UUID 44 | value: "" 45 | volumeMounts: 46 | - name: script-config 47 | mountPath: /config 48 | restartPolicy: Never 49 | backoffLimit: 4 50 | 51 | {{- end -}} 52 | -------------------------------------------------------------------------------- /charts/sb-controller/latest/templates/storage-node-controller.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.storagenode.create -}} 2 | --- 3 | apiVersion: v1 4 | kind: ServiceAccount 5 | metadata: 6 | name: simplyblock-storage-node-service-account 7 | 8 | --- 9 | apiVersion: apps/v1 10 | kind: Deployment 11 | metadata: 12 | name: simplyblock-storage-node-controller 13 | labels: 14 | app: storage-node-controller 15 | spec: 16 | replicas: 1 17 | selector: 18 | matchLabels: 19 | app: storage-node-controller 20 | template: 21 | metadata: 22 | labels: 23 | app: storage-node-controller 24 | spec: 25 | serviceAccountName: simplyblock-storage-node-service-account 26 | containers: 27 | - name: storage-node-controller 28 | image: "{{ .Values.image.storageNode.repository }}:{{ .Values.image.storageNode.tag }}" 29 | imagePullPolicy: "{{ .Values.image.storageNode.pullPolicy }}" 30 | env: 31 | - name: SPDKCSI_SECRET 32 | valueFrom: 33 | secretKeyRef: 34 | name: simplyblock-csi-secret 35 | key: secret.json 36 | - name: CLUSTER_CONFIG 37 | valueFrom: 38 | configMapKeyRef: 39 | name: simplyblock-csi-cm 40 | key: config.json 41 | - name: IFNAME 42 | value: "{{ .Values.storagenode.ifname }}" 43 | - name: MAXSNAP 44 | value: "{{ .Values.storagenode.maxSnap }}" 45 | - name: JMPERCENT 46 | value: "{{ .Values.storagenode.jmPercent }}" 47 | - name: NUMPARTITIONS 48 | value: "{{ .Values.storagenode.numPartitions }}" 49 | - name: DISABLEHAJM 50 | value: "{{ .Values.storagenode.disableHAJM }}" 51 | - name: ENABLETESTDEVICE 52 | value: "{{ .Values.storagenode.enableTestDevice }}" 53 | - name: DATANICS 54 | value: "{{ .Values.storagenode.dataNics }}" 55 | - name: SPDKIMAGE 56 | value: "{{ .Values.storagenode.spdkImage }}" 57 | - name: NAMESPACE 58 | valueFrom: 59 | fieldRef: 60 | fieldPath: metadata.namespace 61 | restartPolicy: Always 62 | 63 | {{- end -}} 64 | -------------------------------------------------------------------------------- /charts/sb-controller/latest/values.yaml: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | # Copyright (c) Arm Limited and Contributors 3 | # Copyright (c) Intel Corporation 4 | --- 5 | 6 | image: 7 | storageNode: 8 | repository: simplyblock/storage-node-handler 9 | tag: v0.1.0 10 | pullPolicy: Always 11 | cachingNode: 12 | repository: simplyblock/caching-node-handler 13 | tag: v0.1.0 14 | pullPolicy: Always 15 | mgmtAPI: 16 | repository: python 17 | tag: "3.10" 18 | pullPolicy: Always 19 | serviceAccount: 20 | # Specifies whether a serviceAccount should be created 21 | create: true 22 | 23 | rbac: 24 | # Specifies whether RBAC resources should be created 25 | create: true 26 | 27 | cachingnode: 28 | create: false 29 | ifname: eth0 30 | cpuMask: 31 | spdkMem: 32 | spdkImage: 33 | multipathing: true 34 | 35 | storagenode: 36 | create: false 37 | ifname: eth0 38 | distr_ndcs: "1" 39 | distr_npcs: "1" 40 | spdkImage: 41 | maxSnap: 10 42 | jmPercent: 3 43 | numPartitions: 0 44 | disableHAJM: false 45 | enableTestDevice: false 46 | dataNics: 47 | -------------------------------------------------------------------------------- /charts/sb-controller/v0.1.0/sb-controller-v0.1.0.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/simplyblock-io/simplyblock-csi/5f1784f36c6badb115f32dccda18a5083fb6e411/charts/sb-controller/v0.1.0/sb-controller-v0.1.0.tgz -------------------------------------------------------------------------------- /charts/spdk-csi/latest/spdk-csi/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *.orig 18 | *~ 19 | # Various IDEs 20 | .project 21 | .idea/ 22 | *.tmproj 23 | .vscode/ 24 | -------------------------------------------------------------------------------- /charts/spdk-csi/latest/spdk-csi/Chart.yaml: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | # Copyright (c) Arm Limited and Contributors 3 | # Copyright (c) Intel Corporation 4 | --- 5 | apiVersion: v1 6 | appVersion: latest 7 | description: Simplyblock CSI Driver for Kubernetes 8 | name: spdk-csi 9 | version: "v0.3.18" 10 | keywords: 11 | - spdk 12 | - csi 13 | - spdk-csi 14 | home: https://github.com/simplyblock-io/spdk-csi 15 | sources: 16 | - https://github.com/simplyblock-io/spdk-csi/tree/master/charts/spdk-csi 17 | -------------------------------------------------------------------------------- /charts/spdk-csi/latest/spdk-csi/templates/NOTES.txt: -------------------------------------------------------------------------------- 1 | The Simplyblock SPDK Driver is getting deployed to your cluster. 2 | 3 | To check CSI SPDK Driver pods status, please run: 4 | 5 | kubectl --namespace={{ .Release.Namespace }} get pods --selector="release={{ .Release.Name }}" --watch 6 | -------------------------------------------------------------------------------- /charts/spdk-csi/latest/spdk-csi/templates/_helpers.tpl: -------------------------------------------------------------------------------- 1 | {{/* vim: set filetype=mustache: */}} 2 | 3 | {{/* labels for helm resources */}} 4 | {{- define "spdk.labels" -}} 5 | labels: 6 | heritage: "{{ .Release.Service }}" 7 | release: "{{ .Release.Name }}" 8 | revision: "{{ .Release.Revision }}" 9 | chart: "{{ .Chart.Name }}" 10 | chartVersion: "{{ .Chart.Version }}" 11 | {{- end -}} -------------------------------------------------------------------------------- /charts/spdk-csi/latest/spdk-csi/templates/benchmark.yaml: -------------------------------------------------------------------------------- 1 | {{- $benchmarks := .Values.benchmarks }} 2 | {{- range $i := until (int $benchmarks) }} 3 | 4 | --- 5 | apiVersion: batch/v1 6 | kind: Job 7 | metadata: 8 | name: simplyblock-fio-benchmark-job-{{ $i }} 9 | spec: 10 | backoffLimit: 4 11 | template: 12 | spec: 13 | containers: 14 | - name: fio-benchmark 15 | image: manoharbrm/fio:latest 16 | imagePullPolicy: Always 17 | command: ["fio", "/fio/fio.cfg"] 18 | volumeMounts: 19 | - mountPath: "/spdkvol" 20 | name: benchmark-volume 21 | - mountPath: "/fio" 22 | name: fio-config 23 | volumes: 24 | - name: benchmark-volume 25 | persistentVolumeClaim: 26 | claimName: simplyblock-csi-benchmark-pvc-{{ $i }} 27 | - name: fio-config 28 | configMap: 29 | name: simplyblock-fio-config 30 | restartPolicy: Never 31 | 32 | --- 33 | kind: PersistentVolumeClaim 34 | apiVersion: v1 35 | metadata: 36 | name: simplyblock-csi-benchmark-pvc-{{ $i }} 37 | spec: 38 | accessModes: 39 | - ReadWriteOnce 40 | resources: 41 | requests: 42 | storage: 30Gi 43 | storageClassName: simplyblock-csi-sc 44 | 45 | {{- end }} 46 | 47 | --- 48 | apiVersion: v1 49 | kind: ConfigMap 50 | metadata: 51 | name: simplyblock-fio-config 52 | data: 53 | fio.cfg: | 54 | [test] 55 | ioengine=aiolib 56 | direct=1 57 | iodepth=4 58 | time_based=1 59 | runtime=1000 60 | readwrite=randrw 61 | bs=4K,8K,16K,32K,64K,128K,256K 62 | nrfiles=4 63 | size=5G 64 | verify=md5 65 | numjobs=3 66 | directory=/spdkvol 67 | -------------------------------------------------------------------------------- /charts/spdk-csi/latest/spdk-csi/templates/caching-node-handler.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.cachingnode.create -}} 2 | --- 3 | apiVersion: v1 4 | kind: ServiceAccount 5 | metadata: 6 | name: simplyblock-caching-node-service-account 7 | 8 | --- 9 | apiVersion: apps/v1 10 | kind: Deployment 11 | metadata: 12 | name: simplyblock-caching-node-handler 13 | labels: 14 | app: caching-node-handler 15 | spec: 16 | replicas: 1 17 | selector: 18 | matchLabels: 19 | app: caching-node-handler 20 | template: 21 | metadata: 22 | labels: 23 | app: caching-node-handler 24 | spec: 25 | serviceAccountName: simplyblock-caching-node-service-account 26 | containers: 27 | - name: caching-node-handler 28 | image: "{{ .Values.image.cachingNode.repository }}:{{ .Values.image.cachingNode.tag }}" 29 | imagePullPolicy: "{{ .Values.image.cachingNode.pullPolicy }}" 30 | env: 31 | - name: SPDKCSI_SECRET 32 | valueFrom: 33 | secretKeyRef: 34 | name: simplyblock-csi-secret 35 | key: secret.json 36 | - name: CLUSTER_CONFIG 37 | valueFrom: 38 | configMapKeyRef: 39 | name: simplyblock-csi-cm 40 | key: config.json 41 | - name: IFNAME 42 | value: "{{ .Values.cachingnode.ifname }}" 43 | - name: CPUMASK 44 | value: "{{ .Values.cachingnode.cpuMask }}" 45 | - name: SPDKMEM 46 | value: "{{ .Values.cachingnode.spdkMem }}" 47 | - name: SPDKIMAGE 48 | value: "{{ .Values.cachingnode.spdkImage }}" 49 | - name: MULTIPATHING 50 | value: "{{ .Values.cachingnode.multipathing }}" 51 | 52 | - name: NAMESPACE 53 | valueFrom: 54 | fieldRef: 55 | fieldPath: metadata.namespace 56 | restartPolicy: Always 57 | 58 | {{- end -}} 59 | -------------------------------------------------------------------------------- /charts/spdk-csi/latest/spdk-csi/templates/caching-node.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.cachingnode.create -}} 2 | --- 3 | apiVersion: v1 4 | kind: ServiceAccount 5 | metadata: 6 | name: simplyblock-caching-node-sa 7 | 8 | --- 9 | kind: ClusterRole 10 | apiVersion: rbac.authorization.k8s.io/v1 11 | metadata: 12 | name: simplyblock-caching-node-role 13 | rules: 14 | - apiGroups: [""] 15 | resources: ["pods"] 16 | verbs: ["list"] 17 | - apiGroups: ["apps"] 18 | resources: ["deployments"] 19 | verbs: ["create", "delete"] 20 | 21 | --- 22 | kind: ClusterRoleBinding 23 | apiVersion: rbac.authorization.k8s.io/v1 24 | metadata: 25 | name: pods-list-cn 26 | subjects: 27 | - kind: ServiceAccount 28 | name: simplyblock-caching-node-sa 29 | namespace: {{ .Release.Namespace }} 30 | roleRef: 31 | kind: ClusterRole 32 | name: simplyblock-caching-node-role 33 | apiGroup: rbac.authorization.k8s.io 34 | 35 | --- 36 | apiVersion: apps/v1 37 | kind: DaemonSet 38 | metadata: 39 | name: simplyblock-caching-node-ds 40 | spec: 41 | selector: 42 | matchLabels: 43 | app: caching-node 44 | template: 45 | metadata: 46 | labels: 47 | app: caching-node 48 | spec: 49 | serviceAccountName: simplyblock-caching-node-sa 50 | nodeSelector: 51 | {{ .Values.cachingnode.nodeSelector.key }}: {{ .Values.cachingnode.nodeSelector.value }} 52 | volumes: 53 | - name: dev-vol 54 | hostPath: 55 | path: /dev 56 | 57 | hostNetwork: true 58 | {{- if .Values.cachingnode.tolerations.create }} 59 | tolerations: 60 | - operator: {{ .Values.cachingnode.tolerations.operator }} 61 | {{- if .Values.cachingnode.tolerations.effect }} 62 | effect: {{ .Values.cachingnode.tolerations.effect }} 63 | {{- end }} 64 | {{- if .Values.cachingnode.tolerations.key }} 65 | key: {{ .Values.cachingnode.tolerations.key }} 66 | {{- end }} 67 | {{- if .Values.cachingnode.tolerations.value }} 68 | value: {{ .Values.cachingnode.tolerations.value }} 69 | {{- end }} 70 | {{- end }} 71 | containers: 72 | - name: c-node-api-container 73 | image: "{{ .Values.image.simplyblock.repository }}:{{ .Values.image.simplyblock.tag }}" 74 | imagePullPolicy: "{{ .Values.image.simplyblock.pullPolicy }}" 75 | command: ["python", "simplyblock_web/node_webapp.py", "caching_kubernetes_node"] 76 | env: 77 | - name: HOSTNAME 78 | valueFrom: 79 | fieldRef: 80 | fieldPath: spec.nodeName 81 | securityContext: 82 | privileged: true 83 | volumeMounts: 84 | - name: dev-vol 85 | mountPath: /dev 86 | 87 | {{- end -}} 88 | -------------------------------------------------------------------------------- /charts/spdk-csi/latest/spdk-csi/templates/config-map.yaml: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | # Copyright (c) Arm Limited and Contributors 3 | # Copyright (c) Intel Corporation 4 | 5 | {{- if .Values.externallyManagedConfigmap }} 6 | --- 7 | apiVersion: v1 8 | kind: ConfigMap 9 | metadata: 10 | name: simplyblock-csi-cm 11 | data: 12 | # uuid: the simplyblock cluster UUID 13 | # ip: the management IP of the simplyblock cluster 14 | # targetAddr: target service IP 15 | config.json: |- 16 | {{ toJson .Values.csiConfig | indent 4 -}} 17 | {{- end }} 18 | 19 | --- 20 | apiVersion: v1 21 | kind: ConfigMap 22 | metadata: 23 | name: simplyblock-caching-node-restart-script-cm 24 | data: 25 | restart_script.py: | 26 | import json, os, requests 27 | 28 | secret = json.loads(os.getenv("SPDKCSI_SECRET")) 29 | cluster_secret = secret['simplybk']['secret'] 30 | 31 | cluster_config = json.loads(os.getenv("CLUSTER_CONFIG")) 32 | cluster_uuid = cluster_config['simplybk']['uuid'] 33 | cluster_ip = cluster_config['simplybk']['ip'] 34 | hostname = os.getenv('HOSTNAME') 35 | hostname = hostname.split(".")[0] 36 | 37 | url = f'{cluster_ip}/cachingnode/recreate/{hostname}' 38 | headers = { 39 | 'Authorization': f'{cluster_uuid} {cluster_secret}' 40 | } 41 | 42 | print(f"making GET request to: {url}") 43 | response = requests.get(url, headers=headers) 44 | 45 | print("Response Text:", response.text) 46 | print("Response Code:", response.status_code) 47 | -------------------------------------------------------------------------------- /charts/spdk-csi/latest/spdk-csi/templates/controller.yaml: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | # Copyright (c) Arm Limited and Contributors 3 | # Copyright (c) Intel Corporation 4 | --- 5 | kind: StatefulSet 6 | apiVersion: apps/v1 7 | metadata: 8 | name: simplyblock-csi-controller 9 | namespace: {{ .Release.Namespace }} 10 | {{ include "spdk.labels" . | indent 2 }} 11 | spec: 12 | serviceName: simplyblock-csi-controller 13 | replicas: {{ .Values.controller.replicas }} 14 | selector: 15 | matchLabels: 16 | app: csi-controller 17 | template: 18 | metadata: 19 | {{ include "spdk.labels" . | indent 6 }} 20 | app: csi-controller 21 | spec: 22 | serviceAccountName: simplyblock-csi-controller-sa 23 | hostNetwork: true 24 | containers: 25 | - name: csi-provisioner 26 | image: "{{ .Values.image.csiProvisioner.repository }}:{{ .Values.image.csiProvisioner.tag }}" 27 | imagePullPolicy: {{ .Values.image.csiProvisioner.pullPolicy }} 28 | args: 29 | - "--v=5" 30 | - "--csi-address=unix:///csi/csi-provisioner.sock" 31 | - "--timeout=30s" 32 | - "--retry-interval-start=500ms" 33 | - "--leader-election=false" 34 | - "--extra-create-metadata=true" 35 | - "--feature-gates=Topology=true" 36 | volumeMounts: 37 | - name: socket-dir 38 | mountPath: /csi 39 | - name: csi-snapshotter 40 | image: "{{ .Values.image.csiSnapshotter.repository }}:{{ .Values.image.csiSnapshotter.tag }}" 41 | args: 42 | - "--csi-address=unix:///csi/csi-provisioner.sock" 43 | - "--v=5" 44 | - "--timeout=150s" 45 | - "--leader-election=false" 46 | imagePullPolicy: {{ .Values.image.csiProvisioner.pullPolicy }} 47 | securityContext: 48 | privileged: true 49 | volumeMounts: 50 | - name: socket-dir 51 | mountPath: /csi 52 | - name: csi-attacher 53 | image: "{{ .Values.image.csiAttacher.repository }}:{{ .Values.image.csiAttacher.tag }}" 54 | imagePullPolicy: {{ .Values.image.csiAttacher.pullPolicy }} 55 | args: 56 | - "--v=5" 57 | - "--csi-address=unix:///csi/csi-provisioner.sock" 58 | - "--leader-election=false" 59 | volumeMounts: 60 | - name: socket-dir 61 | mountPath: /csi 62 | - name: csi-resizer 63 | image: "{{ .Values.image.csiResizer.repository }}:{{ .Values.image.csiResizer.tag }}" 64 | imagePullPolicy: {{ .Values.image.csiResizer.pullPolicy }} 65 | args: 66 | - "--v=5" 67 | - "--csi-address=unix:///csi/csi-provisioner.sock" 68 | - "--leader-election=false" 69 | volumeMounts: 70 | - name: socket-dir 71 | mountPath: /csi 72 | - name: csi-health-monitor 73 | image: "{{ .Values.image.csiHealthMonitor.repository }}:{{ .Values.image.csiHealthMonitor.tag }}" 74 | imagePullPolicy: {{ .Values.image.csiHealthMonitor.pullPolicy }} 75 | args: 76 | - "--v=5" 77 | - "--csi-address=unix:///csi/csi-provisioner.sock" 78 | - "--leader-election=false" 79 | # - "--http-endpoint=:8081" 80 | volumeMounts: 81 | - name: socket-dir 82 | mountPath: /csi 83 | ports: 84 | - containerPort: 8080 85 | name: http-endpoint 86 | protocol: TCP 87 | - name: csi-controller 88 | image: "{{ .Values.image.csi.repository }}:{{ .Values.image.csi.tag }}" 89 | imagePullPolicy: {{ .Values.image.csi.pullPolicy }} 90 | args: 91 | - "--v=5" 92 | - "--endpoint=unix:///csi/csi-provisioner.sock" 93 | - "--nodeid=$(NODE_ID)" 94 | - "--controller" 95 | env: 96 | - name: NODE_ID 97 | valueFrom: 98 | fieldRef: 99 | fieldPath: spec.nodeName 100 | volumeMounts: 101 | - name: socket-dir 102 | mountPath: /csi 103 | - name: csi-config 104 | mountPath: /etc/spdkcsi-config/ 105 | readOnly: true 106 | - name: csi-secret 107 | mountPath: /etc/spdkcsi-secret/ 108 | readOnly: true 109 | volumes: 110 | - name: socket-dir 111 | emptyDir: 112 | medium: "Memory" 113 | - name: csi-config 114 | configMap: 115 | name: simplyblock-csi-cm 116 | - name: csi-secret 117 | secret: 118 | secretName: simplyblock-csi-secret 119 | -------------------------------------------------------------------------------- /charts/spdk-csi/latest/spdk-csi/templates/driver.yaml: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | # Copyright (c) Arm Limited and Contributors 3 | # Copyright (c) Intel Corporation 4 | --- 5 | apiVersion: storage.k8s.io/v1 6 | kind: CSIDriver 7 | metadata: 8 | name: {{ .Values.driverName }} 9 | spec: 10 | attachRequired: true 11 | volumeLifecycleModes: 12 | - Persistent 13 | -------------------------------------------------------------------------------- /charts/spdk-csi/latest/spdk-csi/templates/node-rbac.yaml: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | # Copyright (c) Arm Limited and Contributors 3 | # Copyright (c) Intel Corporation 4 | 5 | {{- if .Values.serviceAccount.create -}} 6 | --- 7 | apiVersion: v1 8 | kind: ServiceAccount 9 | metadata: 10 | name: simplyblock-csi-node-sa 11 | {{- end -}} 12 | -------------------------------------------------------------------------------- /charts/spdk-csi/latest/spdk-csi/templates/node.yaml: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | # Copyright (c) Arm Limited and Contributors 3 | # Copyright (c) Intel Corporation 4 | --- 5 | kind: DaemonSet 6 | apiVersion: apps/v1 7 | metadata: 8 | name: simplyblock-csi-node 9 | namespace: {{ .Release.Namespace }} 10 | {{ include "spdk.labels" . | indent 2 }} 11 | spec: 12 | selector: 13 | matchLabels: 14 | app: simplyblock-csi-node 15 | template: 16 | metadata: 17 | {{ include "spdk.labels" . | indent 6 }} 18 | app: simplyblock-csi-node 19 | spec: 20 | serviceAccountName: simplyblock-csi-node-sa 21 | hostNetwork: true 22 | {{- if .Values.node.tolerations.create }} 23 | tolerations: 24 | - operator: {{ .Values.node.tolerations.operator }} 25 | {{- if .Values.node.tolerations.effect }} 26 | effect: {{ .Values.node.tolerations.effect }} 27 | {{- end }} 28 | {{- if .Values.node.tolerations.key }} 29 | key: {{ .Values.node.tolerations.key }} 30 | {{- end }} 31 | {{- if .Values.node.tolerations.value }} 32 | value: {{ .Values.node.tolerations.value }} 33 | {{- end }} 34 | {{- end }} 35 | containers: 36 | - name: csi-registrar 37 | securityContext: 38 | privileged: true 39 | image: "{{ .Values.image.nodeDriverRegistrar.repository }}:{{ .Values.image.nodeDriverRegistrar.tag }}" 40 | imagePullPolicy: {{ .Values.image.nodeDriverRegistrar.pullPolicy }} 41 | args: 42 | - "--v=5" 43 | - "--csi-address=unix:///csi/csi.sock" 44 | - "--kubelet-registration-path=/var/lib/kubelet/plugins/csi.simplyblock.io/csi.sock" 45 | - "--health-port=9809" 46 | ports: 47 | - containerPort: 9809 48 | name: healthz 49 | livenessProbe: 50 | httpGet: 51 | path: /healthz 52 | port: healthz 53 | initialDelaySeconds: 20 54 | timeoutSeconds: 10 55 | volumeMounts: 56 | - name: socket-dir 57 | mountPath: /csi 58 | - name: registration-dir 59 | mountPath: /registration 60 | - name: csi-node 61 | securityContext: 62 | privileged: true 63 | capabilities: 64 | add: ["SYS_ADMIN", "SYS_MODULE"] 65 | allowPrivilegeEscalation: true 66 | image: "{{ .Values.image.csi.repository }}:{{ .Values.image.csi.tag }}" 67 | imagePullPolicy: {{ .Values.image.csi.pullPolicy }} 68 | args: 69 | - "--v=5" 70 | - "--endpoint=unix:///csi/csi.sock" 71 | - "--nodeid=$(NODE_ID)" 72 | - "--node" 73 | env: 74 | - name: NODE_ID 75 | valueFrom: 76 | fieldRef: 77 | fieldPath: spec.nodeName 78 | lifecycle: 79 | postStart: 80 | exec: 81 | command: 82 | [ 83 | "/bin/sh", "-c", 84 | "sudo modprobe nvme-tcp || echo failed to modprobe nvme-tcp && \ 85 | if [ ! -f /var/lib/nvme/hostid ]; then uuidgen > /var/lib/nvme/hostid; fi && \ 86 | cp /var/lib/nvme/hostid /etc/nvme/hostid && \ 87 | echo \"nqn.2014-08.org.nvmexpress:uuid:$(cat /etc/nvme/hostid)\" > /etc/nvme/hostnqn" 88 | ] 89 | volumeMounts: 90 | - name: socket-dir 91 | mountPath: /csi 92 | - name: plugin-dir 93 | mountPath: /var/lib/kubelet/plugins 94 | mountPropagation: "Bidirectional" 95 | - name: pod-dir 96 | mountPath: /var/lib/kubelet/pods 97 | mountPropagation: "Bidirectional" 98 | - name: nvme-hostid-dir 99 | mountPath: /var/lib/nvme 100 | mountPropagation: "Bidirectional" 101 | - name: host-dev 102 | mountPath: /dev 103 | - name: host-sys 104 | mountPath: /sys 105 | - name: csi-nodeserver-config 106 | mountPath: /etc/spdkcsi-nodeserver-config/ 107 | readOnly: true 108 | - name: csi-config 109 | mountPath: /etc/spdkcsi-config/ 110 | readOnly: true 111 | - name: csi-secret 112 | mountPath: /etc/spdkcsi-secret/ 113 | readOnly: true 114 | volumes: 115 | - name: socket-dir 116 | hostPath: 117 | path: /var/lib/kubelet/plugins/csi.simplyblock.io 118 | type: DirectoryOrCreate 119 | - name: registration-dir 120 | hostPath: 121 | path: /var/lib/kubelet/plugins_registry/ 122 | type: Directory 123 | - name: plugin-dir 124 | hostPath: 125 | path: /var/lib/kubelet/plugins 126 | type: Directory 127 | - name: pod-dir 128 | hostPath: 129 | path: /var/lib/kubelet/pods 130 | type: Directory 131 | - name: nvme-hostid-dir 132 | hostPath: 133 | path: /var/lib/nvme 134 | type: DirectoryOrCreate 135 | - name: host-dev 136 | hostPath: 137 | path: /dev 138 | - name: host-sys 139 | hostPath: 140 | path: /sys 141 | - name: csi-nodeserver-config 142 | configMap: 143 | name: simplyblock-csi-nodeservercm 144 | optional: true 145 | - name: csi-config 146 | configMap: 147 | name: simplyblock-csi-cm 148 | - name: csi-secret 149 | secret: 150 | secretName: simplyblock-csi-secret 151 | -------------------------------------------------------------------------------- /charts/spdk-csi/latest/spdk-csi/templates/nodeserver-config-map.yaml: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | # Copyright (c) Arm Limited and Contributors 3 | # Copyright (c) Intel Corporation 4 | --- 5 | apiVersion: v1 6 | kind: ConfigMap 7 | metadata: 8 | name: simplyblock-csi-nodeservercm 9 | data: 10 | # xpu.targetType: 11 | # - SMA: xpu-sma-nvmftcp, xpu-sma-virtioblk, xpu-sma-nvme 12 | # xpu.targetAddr: 13 | # - URL to connect the xPU node through GRPC, IPADDR:PORT 14 | # - 127.0.0.1:5114 for SMA server by default 15 | # kvmPciBridges: 16 | # - used by sma-virtioblk, sma-nvme 17 | # - based on the configuration in "deploy/spdk/sma.yaml" and qemu VM 18 | # 19 | # example: 20 | # nodeserver-config.json: |- 21 | # { 22 | # "xpuList": [ 23 | # { 24 | # "name": "xPU0", 25 | # "targetType": "xpu-sma-nvmftcp", 26 | # "targetAddr": "127.0.0.1:5114" 27 | # } 28 | # ], 29 | # "kvmPciBridges": 2 30 | # } 31 | nodeserver-config.json: |- 32 | { 33 | "xpuList": [], 34 | "kvmPciBridges": null 35 | } 36 | -------------------------------------------------------------------------------- /charts/spdk-csi/latest/spdk-csi/templates/rbac-snapshot-controller.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.snapshotcontroller.create -}} 2 | --- 3 | apiVersion: v1 4 | kind: ServiceAccount 5 | metadata: 6 | name: simplyblock-snapshot-controller 7 | namespace: kube-system 8 | 9 | --- 10 | kind: ClusterRole 11 | apiVersion: rbac.authorization.k8s.io/v1 12 | metadata: 13 | name: simplyblock-snapshot-controller-runner 14 | rules: 15 | - apiGroups: [""] 16 | resources: ["persistentvolumes"] 17 | verbs: ["get", "list", "watch"] 18 | - apiGroups: [""] 19 | resources: ["persistentvolumeclaims"] 20 | verbs: ["get", "list", "watch", "update"] 21 | - apiGroups: [""] 22 | resources: ["events"] 23 | verbs: ["list", "watch", "create", "update", "patch"] 24 | - apiGroups: ["snapshot.storage.k8s.io"] 25 | resources: ["volumesnapshotclasses"] 26 | verbs: ["get", "list", "watch"] 27 | - apiGroups: ["snapshot.storage.k8s.io"] 28 | resources: ["volumesnapshotcontents"] 29 | verbs: ["create", "get", "list", "watch", "update", "delete", "patch"] 30 | - apiGroups: ["snapshot.storage.k8s.io"] 31 | resources: ["volumesnapshotcontents/status"] 32 | verbs: ["patch"] 33 | - apiGroups: ["snapshot.storage.k8s.io"] 34 | resources: ["volumesnapshots"] 35 | verbs: ["get", "list", "watch", "update", "patch", "delete"] 36 | - apiGroups: ["snapshot.storage.k8s.io"] 37 | resources: ["volumesnapshots/status"] 38 | verbs: ["update", "patch"] 39 | 40 | - apiGroups: ["groupsnapshot.storage.k8s.io"] 41 | resources: ["volumegroupsnapshotclasses"] 42 | verbs: ["get", "list", "watch"] 43 | - apiGroups: ["groupsnapshot.storage.k8s.io"] 44 | resources: ["volumegroupsnapshotcontents"] 45 | verbs: ["create", "get", "list", "watch", "update", "delete", "patch"] 46 | - apiGroups: ["groupsnapshot.storage.k8s.io"] 47 | resources: ["volumegroupsnapshotcontents/status"] 48 | verbs: ["patch"] 49 | - apiGroups: ["groupsnapshot.storage.k8s.io"] 50 | resources: ["volumegroupsnapshots"] 51 | verbs: ["get", "list", "watch", "update", "patch"] 52 | - apiGroups: ["groupsnapshot.storage.k8s.io"] 53 | resources: ["volumegroupsnapshots/status"] 54 | verbs: ["update", "patch"] 55 | 56 | # Enable this RBAC rule only when using distributed snapshotting, i.e. when the enable-distributed-snapshotting flag is set to true 57 | # - apiGroups: [""] 58 | # resources: ["nodes"] 59 | # verbs: ["get", "list", "watch"] 60 | --- 61 | kind: ClusterRoleBinding 62 | apiVersion: rbac.authorization.k8s.io/v1 63 | metadata: 64 | name: simplyblock-snapshot-controller-role 65 | subjects: 66 | - kind: ServiceAccount 67 | name: simplyblock-snapshot-controller 68 | namespace: kube-system 69 | roleRef: 70 | kind: ClusterRole 71 | name: simplyblock-snapshot-controller-runner 72 | apiGroup: rbac.authorization.k8s.io 73 | 74 | --- 75 | kind: Role 76 | apiVersion: rbac.authorization.k8s.io/v1 77 | metadata: 78 | name: simplyblock-snapshot-controller-leaderelection 79 | namespace: kube-system 80 | rules: 81 | - apiGroups: ["coordination.k8s.io"] 82 | resources: ["leases"] 83 | verbs: ["get", "watch", "list", "delete", "update", "create"] 84 | 85 | --- 86 | kind: RoleBinding 87 | apiVersion: rbac.authorization.k8s.io/v1 88 | metadata: 89 | name: simplyblock-snapshot-controller-leaderelection 90 | namespace: kube-system 91 | subjects: 92 | - kind: ServiceAccount 93 | name: simplyblock-snapshot-controller 94 | roleRef: 95 | kind: Role 96 | name: simplyblock-snapshot-controller-leaderelection 97 | apiGroup: rbac.authorization.k8s.io 98 | 99 | {{- end -}} 100 | -------------------------------------------------------------------------------- /charts/spdk-csi/latest/spdk-csi/templates/secret.yaml: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | # Copyright (c) Arm Limited and Contributors 3 | # Copyright (c) Intel Corporation 4 | 5 | {{- if .Values.externallyManagedSecret }} 6 | --- 7 | apiVersion: v1 8 | kind: Secret 9 | metadata: 10 | name: simplyblock-pvc-keys 11 | namespace: {{ .Release.Namespace }} 12 | data: 13 | crypto_key1: {{ .Values.csiSecret.simplybkPvc.crypto_key1 }} 14 | crypto_key2: {{ .Values.csiSecret.simplybkPvc.crypto_key2 }} 15 | 16 | --- 17 | apiVersion: v1 18 | kind: Secret 19 | metadata: 20 | name: simplyblock-csi-secret 21 | stringData: 22 | secret.json: |- 23 | {{ toJson .Values.csiSecret | indent 4 -}} 24 | {{- end }} 25 | -------------------------------------------------------------------------------- /charts/spdk-csi/latest/spdk-csi/templates/setup-snapshot-controller.yaml: -------------------------------------------------------------------------------- 1 | # This YAML file shows how to deploy the snapshot controller 2 | 3 | # The snapshot controller implements the control loop for CSI snapshot functionality. 4 | # It should be installed as part of the base Kubernetes distribution in an appropriate 5 | # namespace for components implementing base system functionality. For installing with 6 | # Vanilla Kubernetes, kube-system makes sense for the namespace. 7 | {{- if .Values.snapshotcontroller.create -}} 8 | --- 9 | kind: Deployment 10 | apiVersion: apps/v1 11 | metadata: 12 | name: simplyblock-snapshot-controller 13 | namespace: kube-system 14 | spec: 15 | replicas: 2 16 | selector: 17 | matchLabels: 18 | app.kubernetes.io/name: snapshot-controller 19 | # The snapshot controller won't be marked as ready if the v1 CRDs are unavailable. 20 | # The flag --retry-crd-interval-max is used to determine how long the controller 21 | # will wait for the CRDs to become available before exiting. The default is 30 seconds 22 | # so minReadySeconds should be set slightly higher than the flag value. 23 | minReadySeconds: 35 24 | strategy: 25 | rollingUpdate: 26 | maxSurge: 0 27 | maxUnavailable: 1 28 | type: RollingUpdate 29 | template: 30 | metadata: 31 | labels: 32 | app.kubernetes.io/name: snapshot-controller 33 | spec: 34 | serviceAccountName: simplyblock-snapshot-controller 35 | containers: 36 | - name: snapshot-controller 37 | image: registry.k8s.io/sig-storage/snapshot-controller:v8.2.0 38 | args: 39 | - "--v=5" 40 | - "--leader-election=true" 41 | imagePullPolicy: IfNotPresent 42 | 43 | {{- end -}} 44 | -------------------------------------------------------------------------------- /charts/spdk-csi/latest/spdk-csi/templates/snapshotclass.yaml: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | # Copyright (c) Arm Limited and Contributors 3 | # Copyright (c) Intel Corporation 4 | 5 | # Snapshot API version compatibility: 6 | # v1beta1: 7 | # v1.17 =< k8s < v1.20 8 | # 2.x =< snapshot-controller < v4.x 9 | # We recommend to use {sidecar, controller, crds} of same version 10 | {{- if .Values.snapshotclass.create -}} 11 | --- 12 | apiVersion: snapshot.storage.k8s.io/v1 13 | kind: VolumeSnapshotClass 14 | metadata: 15 | name: simplyblock-csi-snapshotclass 16 | annotations: 17 | helm.sh/hook: post-install,post-upgrade 18 | driver: csi.simplyblock.io 19 | deletionPolicy: Delete 20 | 21 | {{- end -}} 22 | -------------------------------------------------------------------------------- /charts/spdk-csi/latest/spdk-csi/templates/storage-node.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.storagenode.create -}} 2 | --- 3 | apiVersion: v1 4 | kind: ServiceAccount 5 | metadata: 6 | name: simplyblock-storage-node-sa 7 | 8 | --- 9 | kind: ClusterRole 10 | apiVersion: rbac.authorization.k8s.io/v1 11 | metadata: 12 | name: simplyblock-storage-node-role 13 | rules: 14 | - apiGroups: [""] 15 | resources: ["pods", "namespaces", "pods/exec"] 16 | verbs: ["list", "get", "create", "delete"] 17 | - apiGroups: ["apps"] 18 | resources: ["deployments"] 19 | verbs: ["create", "delete"] 20 | 21 | --- 22 | kind: ClusterRoleBinding 23 | apiVersion: rbac.authorization.k8s.io/v1 24 | metadata: 25 | name: simplyblock-pods-list-sn 26 | subjects: 27 | - kind: ServiceAccount 28 | name: simplyblock-storage-node-sa 29 | namespace: {{ .Release.Namespace }} 30 | roleRef: 31 | kind: ClusterRole 32 | name: simplyblock-storage-node-role 33 | apiGroup: rbac.authorization.k8s.io 34 | 35 | {{- range .Values.storagenode.daemonsets }} 36 | --- 37 | apiVersion: apps/v1 38 | kind: DaemonSet 39 | metadata: 40 | name: {{ .name }} 41 | spec: 42 | selector: 43 | matchLabels: 44 | app: {{ .appLabel }} 45 | template: 46 | metadata: 47 | labels: 48 | app: {{ .appLabel }} 49 | spec: 50 | serviceAccountName: simplyblock-storage-node-sa 51 | nodeSelector: 52 | {{ .nodeSelector.key }}: {{ .nodeSelector.value }} 53 | volumes: 54 | - name: dev-vol 55 | hostPath: 56 | path: /dev 57 | - name: etc-simplyblock 58 | hostPath: 59 | path: /var/simplyblock 60 | hostNetwork: true 61 | {{- if .tolerations.create }} 62 | tolerations: 63 | - operator: {{ .tolerations.operator }} 64 | {{- if .tolerations.effect }} 65 | effect: {{ .tolerations.effect }} 66 | {{- end }} 67 | {{- if .tolerations.key }} 68 | key: {{ .tolerations.key }} 69 | {{- end }} 70 | {{- if .tolerations.value }} 71 | value: {{ .tolerations.value }} 72 | {{- end }} 73 | {{- end }} 74 | initContainers: 75 | - name: s-node-api-config-generator 76 | image: "{{ $.Values.image.simplyblock.repository }}:{{ $.Values.image.simplyblock.tag }}" 77 | imagePullPolicy: "{{ $.Values.image.simplyblock.pullPolicy }}" 78 | command: 79 | - "python" 80 | - "simplyblock_web/node_configure.py" 81 | - "--max-lvol={{ $.Values.storagenode.maxLvol }}" 82 | - "--max-size={{ $.Values.storagenode.maxProv }}" 83 | {{- if $.Values.storagenode.pciAllowed }} 84 | - "--pci-allowed={{ $.Values.storagenode.pciAllowed }}" 85 | {{- end }} 86 | {{- if $.Values.storagenode.pciBlocked }} 87 | - "--pci-blocked={{ $.Values.storagenode.pciBlocked }}" 88 | {{- end }} 89 | {{- if $.Values.storagenode.socketsToUse }} 90 | - "--sockets-to-use={{ $.Values.storagenode.socketsToUse }}" 91 | {{- end }} 92 | {{- if $.Values.storagenode.nodesPerSocket }} 93 | - "--nodes-per-socket={{ $.Values.storagenode.nodesPerSocket }}" 94 | {{- end }} 95 | volumeMounts: 96 | - name: etc-simplyblock 97 | mountPath: /etc/simplyblock 98 | securityContext: 99 | privileged: true 100 | containers: 101 | - name: s-node-api-container 102 | image: "{{ $.Values.image.simplyblock.repository }}:{{ $.Values.image.simplyblock.tag }}" 103 | imagePullPolicy: "{{ $.Values.image.simplyblock.pullPolicy }}" 104 | command: ["python", "simplyblock_web/node_webapp.py", "storage_node_k8s"] 105 | env: 106 | - name: HOSTNAME 107 | valueFrom: 108 | fieldRef: 109 | fieldPath: spec.nodeName 110 | - name: CPUMASK 111 | value: "{{ $.Values.storagenode.cpuMask }}" 112 | securityContext: 113 | privileged: true 114 | volumeMounts: 115 | - name: dev-vol 116 | mountPath: /dev 117 | - name: etc-simplyblock 118 | mountPath: /etc/simplyblock 119 | livenessProbe: 120 | httpGet: 121 | path: /snode/get_firewall 122 | port: 5000 123 | scheme: HTTP 124 | initialDelaySeconds: 30 125 | periodSeconds: 60 126 | timeoutSeconds: 5 127 | failureThreshold: 3 128 | readinessProbe: 129 | httpGet: 130 | path: /snode/get_firewall 131 | port: 5000 132 | scheme: HTTP 133 | initialDelaySeconds: 30 134 | periodSeconds: 60 135 | timeoutSeconds: 5 136 | failureThreshold: 3 137 | 138 | {{- end }} 139 | 140 | {{- end -}} 141 | -------------------------------------------------------------------------------- /charts/spdk-csi/latest/spdk-csi/templates/storageclass.yaml: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | # Copyright (c) Arm Limited and Contributors 3 | # Copyright (c) Intel Corporation 4 | 5 | {{- if .Values.storageclass.create -}} 6 | --- 7 | apiVersion: storage.k8s.io/v1 8 | kind: StorageClass 9 | metadata: 10 | name: simplyblock-csi-sc 11 | provisioner: csi.simplyblock.io 12 | parameters: 13 | csi.storage.k8s.io/fstype: ext4 14 | pool_name: {{ .Values.logicalVolume.pool_name }} 15 | qos_rw_iops: "{{ .Values.logicalVolume.qos_rw_iops }}" 16 | qos_rw_mbytes: "{{ .Values.logicalVolume.qos_rw_mbytes }}" 17 | qos_r_mbytes: "{{ .Values.logicalVolume.qos_r_mbytes }}" 18 | qos_w_mbytes: "{{ .Values.logicalVolume.qos_w_mbytes }}" 19 | compression: "{{ .Values.logicalVolume.compression }}" 20 | encryption: "{{ .Values.logicalVolume.encryption }}" 21 | distr_ndcs: "{{ .Values.logicalVolume.distr_ndcs }}" 22 | distr_npcs: "{{ .Values.logicalVolume.distr_npcs }}" 23 | lvol_priority_class: "{{ .Values.logicalVolume.lvol_priority_class }}" 24 | reclaimPolicy: Delete 25 | volumeBindingMode: Immediate 26 | allowVolumeExpansion: true 27 | 28 | --- 29 | apiVersion: storage.k8s.io/v1 30 | kind: StorageClass 31 | metadata: 32 | name: simplyblock-csi-sc-cache 33 | provisioner: csi.simplyblock.io 34 | parameters: 35 | csi.storage.k8s.io/fstype: ext4 36 | type: cache 37 | pool_name: "{{ .Values.logicalVolume.pool_name }}" 38 | qos_rw_iops: "{{ .Values.logicalVolume.qos_rw_iops }}" 39 | qos_rw_mbytes: "{{ .Values.logicalVolume.qos_rw_mbytes }}" 40 | qos_r_mbytes: "{{ .Values.logicalVolume.qos_r_mbytes }}" 41 | qos_w_mbytes: "{{ .Values.logicalVolume.qos_w_mbytes }}" 42 | compression: "{{ .Values.logicalVolume.compression }}" 43 | encryption: "{{ .Values.logicalVolume.encryption }}" 44 | distr_ndcs: "{{ .Values.logicalVolume.distr_ndcs }}" 45 | distr_npcs: "{{ .Values.logicalVolume.distr_npcs }}" 46 | lvol_priority_class: "{{ .Values.logicalVolume.lvol_priority_class }}" 47 | reclaimPolicy: Delete 48 | volumeBindingMode: Immediate 49 | allowVolumeExpansion: true 50 | 51 | {{- end -}} 52 | -------------------------------------------------------------------------------- /charts/spdk-csi/latest/spdk-csi/values.yaml: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | # Copyright (c) Arm Limited and Contributors 3 | # Copyright (c) Intel Corporation 4 | --- 5 | driverName: csi.simplyblock.io 6 | 7 | image: 8 | csi: 9 | repository: simplyblock/spdkcsi 10 | tag: v0.1.1 11 | pullPolicy: Always 12 | csiProvisioner: 13 | repository: registry.k8s.io/sig-storage/csi-provisioner 14 | tag: v4.0.1 15 | pullPolicy: Always 16 | csiAttacher: 17 | repository: gcr.io/k8s-staging-sig-storage/csi-attacher 18 | tag: v4.5.1 19 | pullPolicy: Always 20 | nodeDriverRegistrar: 21 | repository: registry.k8s.io/sig-storage/csi-node-driver-registrar 22 | tag: v2.10.1 23 | pullPolicy: Always 24 | csiSnapshotter: 25 | repository: registry.k8s.io/sig-storage/csi-snapshotter 26 | tag: v8.2.0 27 | pullPolicy: Always 28 | csiResizer: 29 | repository: gcr.io/k8s-staging-sig-storage/csi-resizer 30 | tag: v1.10.1 31 | pullPolicy: Always 32 | csiHealthMonitor: 33 | repository: gcr.io/k8s-staging-sig-storage/csi-external-health-monitor-controller 34 | tag: v0.11.0 35 | pullPolicy: Always 36 | simplyblock: 37 | repository: simplyblock/simplyblock 38 | tag: "R25.5-Hotfix" 39 | pullPolicy: Always 40 | serviceAccount: 41 | # Specifies whether a serviceAccount should be created 42 | create: true 43 | 44 | rbac: 45 | # Specifies whether RBAC resources should be created 46 | create: true 47 | 48 | controller: 49 | replicas: 1 50 | 51 | storageclass: 52 | create: true 53 | 54 | snapshotclass: 55 | create: true 56 | 57 | snapshotcontroller: 58 | create: true 59 | 60 | externallyManagedConfigmap: 61 | # Specifies whether a externallyManagedConfigmap should be created 62 | create: true 63 | 64 | externallyManagedSecret: 65 | # Specifies whether a externallyManagedSecret should be created 66 | create: true 67 | 68 | spdkdev: 69 | # Specifies whether a spdkdev should be created 70 | create: false 71 | 72 | # Configuration for the CSI to connect to the cluster 73 | csiConfig: 74 | simplybk: 75 | uuid: 963c9d0a-4506-43c3-a722-0b7c8b157038 76 | ip: https://o5ls1ykzbb.execute-api.eu-central-1.amazonaws.com 77 | 78 | # Configuration for the csiSecret 79 | csiSecret: 80 | simplybk: 81 | secret: 2BAbQTPEDi4o73VHymg2 82 | simplybkPvc: 83 | crypto_key1: N2IzNjk1MjY4ZTJhNjYxMWEyNWFjNGIxZWUxNWYyN2Y5YmY2ZWE5NzgzZGFkYTY2YTRhNzMwZWJmMDQ5MmJmZA== 84 | crypto_key2: Nzg1MDU2MzZjODEzM2Q5YmU0MmUzNDdmODI3ODViODFhODc5Y2Q4MTMzMDQ2ZjhmYzBiMzZmMTdiMDc4YWQwYw== 85 | 86 | logicalVolume: 87 | pool_name: testing1 88 | qos_rw_iops: "0" 89 | qos_rw_mbytes: "0" 90 | qos_r_mbytes: "0" 91 | qos_w_mbytes: "0" 92 | max_size: "0" 93 | compression: "False" 94 | encryption: "False" 95 | distr_ndcs: "1" 96 | distr_npcs: "1" 97 | lvol_priority_class: "0" 98 | 99 | benchmarks: 0 100 | 101 | # FIXME: this will not work if there are group of nodes with different AMI types like: AL2, AL2023 102 | # AL2_x86_64: eth0 103 | # AL2023_x86_64_STANDARD: ens5 104 | 105 | node: 106 | tolerations: 107 | create: false 108 | operator: Exists 109 | effect: 110 | key: 111 | value: 112 | 113 | cachingnode: 114 | create: false 115 | nodeSelector: 116 | key: type 117 | value: simplyblock-cache 118 | ifname: eth0 119 | cpuMask: 120 | spdkMem: 121 | spdkImage: 122 | multipathing: true 123 | tolerations: 124 | create: false 125 | operator: Exists 126 | effect: 127 | key: 128 | value: 129 | 130 | storagenode: 131 | create: false 132 | ifname: eth0 133 | cpuMask: 134 | spdkImage: 135 | maxLvol: 10 136 | maxSnap: 10 137 | maxProv: 150g 138 | jmPercent: 3 139 | numPartitions: 0 140 | disableHAJM: false 141 | enableTestDevice: false 142 | dataNics: 143 | pciAllowed: 144 | pciBlocked: 145 | socketsToUse: 146 | nodesPerSocket: 147 | daemonsets: 148 | - name: simplyblock-storage-node-ds 149 | appLabel: storage-node 150 | nodeSelector: 151 | key: type 152 | value: simplyblock-storage-plane 153 | tolerations: 154 | create: false 155 | operator: Exists 156 | effect: 157 | key: 158 | value: 159 | - name: simplyblock-storage-node-ds-restart 160 | appLabel: storage-node-restart 161 | nodeSelector: 162 | key: type 163 | value: simplyblock-storage-plane-restart 164 | tolerations: 165 | create: false 166 | operator: Exists 167 | effect: 168 | key: 169 | value: 170 | -------------------------------------------------------------------------------- /charts/spdk-csi/v0.3.0/spdk-csi-v0.3.0.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/simplyblock-io/simplyblock-csi/5f1784f36c6badb115f32dccda18a5083fb6e411/charts/spdk-csi/v0.3.0/spdk-csi-v0.3.0.tgz -------------------------------------------------------------------------------- /charts/spdk-csi/v0.3.1/spdk-csi-v0.3.1.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/simplyblock-io/simplyblock-csi/5f1784f36c6badb115f32dccda18a5083fb6e411/charts/spdk-csi/v0.3.1/spdk-csi-v0.3.1.tgz -------------------------------------------------------------------------------- /charts/spdk-csi/v0.3.10/spdk-csi-v0.3.10.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/simplyblock-io/simplyblock-csi/5f1784f36c6badb115f32dccda18a5083fb6e411/charts/spdk-csi/v0.3.10/spdk-csi-v0.3.10.tgz -------------------------------------------------------------------------------- /charts/spdk-csi/v0.3.11/spdk-csi-v0.3.11.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/simplyblock-io/simplyblock-csi/5f1784f36c6badb115f32dccda18a5083fb6e411/charts/spdk-csi/v0.3.11/spdk-csi-v0.3.11.tgz -------------------------------------------------------------------------------- /charts/spdk-csi/v0.3.12/spdk-csi-v0.3.12.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/simplyblock-io/simplyblock-csi/5f1784f36c6badb115f32dccda18a5083fb6e411/charts/spdk-csi/v0.3.12/spdk-csi-v0.3.12.tgz -------------------------------------------------------------------------------- /charts/spdk-csi/v0.3.13/spdk-csi-v0.3.13.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/simplyblock-io/simplyblock-csi/5f1784f36c6badb115f32dccda18a5083fb6e411/charts/spdk-csi/v0.3.13/spdk-csi-v0.3.13.tgz -------------------------------------------------------------------------------- /charts/spdk-csi/v0.3.14/spdk-csi-v0.3.14.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/simplyblock-io/simplyblock-csi/5f1784f36c6badb115f32dccda18a5083fb6e411/charts/spdk-csi/v0.3.14/spdk-csi-v0.3.14.tgz -------------------------------------------------------------------------------- /charts/spdk-csi/v0.3.15/spdk-csi-v0.3.15.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/simplyblock-io/simplyblock-csi/5f1784f36c6badb115f32dccda18a5083fb6e411/charts/spdk-csi/v0.3.15/spdk-csi-v0.3.15.tgz -------------------------------------------------------------------------------- /charts/spdk-csi/v0.3.16/spdk-csi-v0.3.16.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/simplyblock-io/simplyblock-csi/5f1784f36c6badb115f32dccda18a5083fb6e411/charts/spdk-csi/v0.3.16/spdk-csi-v0.3.16.tgz -------------------------------------------------------------------------------- /charts/spdk-csi/v0.3.17/spdk-csi-v0.3.17.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/simplyblock-io/simplyblock-csi/5f1784f36c6badb115f32dccda18a5083fb6e411/charts/spdk-csi/v0.3.17/spdk-csi-v0.3.17.tgz -------------------------------------------------------------------------------- /charts/spdk-csi/v0.3.18/spdk-csi-v0.3.18.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/simplyblock-io/simplyblock-csi/5f1784f36c6badb115f32dccda18a5083fb6e411/charts/spdk-csi/v0.3.18/spdk-csi-v0.3.18.tgz -------------------------------------------------------------------------------- /charts/spdk-csi/v0.3.2/spdk-csi-v0.3.2.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/simplyblock-io/simplyblock-csi/5f1784f36c6badb115f32dccda18a5083fb6e411/charts/spdk-csi/v0.3.2/spdk-csi-v0.3.2.tgz -------------------------------------------------------------------------------- /charts/spdk-csi/v0.3.3/spdk-csi-v0.3.3.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/simplyblock-io/simplyblock-csi/5f1784f36c6badb115f32dccda18a5083fb6e411/charts/spdk-csi/v0.3.3/spdk-csi-v0.3.3.tgz -------------------------------------------------------------------------------- /charts/spdk-csi/v0.3.4/spdk-csi-v0.3.4.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/simplyblock-io/simplyblock-csi/5f1784f36c6badb115f32dccda18a5083fb6e411/charts/spdk-csi/v0.3.4/spdk-csi-v0.3.4.tgz -------------------------------------------------------------------------------- /charts/spdk-csi/v0.3.5/spdk-csi-v0.3.5.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/simplyblock-io/simplyblock-csi/5f1784f36c6badb115f32dccda18a5083fb6e411/charts/spdk-csi/v0.3.5/spdk-csi-v0.3.5.tgz -------------------------------------------------------------------------------- /charts/spdk-csi/v0.3.6/spdk-csi-v0.3.6.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/simplyblock-io/simplyblock-csi/5f1784f36c6badb115f32dccda18a5083fb6e411/charts/spdk-csi/v0.3.6/spdk-csi-v0.3.6.tgz -------------------------------------------------------------------------------- /charts/spdk-csi/v0.3.7/spdk-csi-v0.3.7.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/simplyblock-io/simplyblock-csi/5f1784f36c6badb115f32dccda18a5083fb6e411/charts/spdk-csi/v0.3.7/spdk-csi-v0.3.7.tgz -------------------------------------------------------------------------------- /charts/spdk-csi/v0.3.8/spdk-csi-v0.3.8.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/simplyblock-io/simplyblock-csi/5f1784f36c6badb115f32dccda18a5083fb6e411/charts/spdk-csi/v0.3.8/spdk-csi-v0.3.8.tgz -------------------------------------------------------------------------------- /charts/spdk-csi/v0.3.9/spdk-csi-v0.3.9.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/simplyblock-io/simplyblock-csi/5f1784f36c6badb115f32dccda18a5083fb6e411/charts/spdk-csi/v0.3.9/spdk-csi-v0.3.9.tgz -------------------------------------------------------------------------------- /cmd/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) Arm Limited and Contributors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "flag" 21 | "os" 22 | 23 | "k8s.io/klog" 24 | 25 | "github.com/spdk/spdk-csi/pkg/spdk" 26 | "github.com/spdk/spdk-csi/pkg/util" 27 | ) 28 | 29 | const ( 30 | driverName = "csi.simplyblock.io" 31 | driverVersion = "0.1.0" 32 | ) 33 | 34 | var conf = util.Config{ 35 | DriverVersion: driverVersion, 36 | } 37 | 38 | func init() { 39 | flag.StringVar(&conf.DriverName, "drivername", driverName, "Name of the driver") 40 | flag.StringVar(&conf.Endpoint, "endpoint", "unix://tmp/spdkcsi.sock", "CSI endpoint") 41 | flag.StringVar(&conf.NodeID, "nodeid", "", "node id") 42 | flag.BoolVar(&conf.IsControllerServer, "controller", false, "Start controller server") 43 | flag.BoolVar(&conf.IsNodeServer, "node", false, "Start node server") 44 | 45 | klog.InitFlags(nil) 46 | if err := flag.Set("logtostderr", "true"); err != nil { 47 | klog.Exitf("failed to set logtostderr flag: %v", err) 48 | } 49 | flag.Parse() 50 | } 51 | 52 | func main() { 53 | klog.Infof("Starting SPDK-CSI driver: %v version: %v", conf.DriverName, driverVersion) 54 | 55 | spdk.Run(&conf) 56 | 57 | os.Exit(0) 58 | } 59 | -------------------------------------------------------------------------------- /deploy/image/Dockerfile: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | # Copyright (c) Arm Limited and Contributors 3 | # Copyright (c) Intel Corporation 4 | # 5 | # XXX: pin alpine to 3.8 with e2fsprogs-1.44 6 | # e2fsprogs-1.45+ crashes my test vm when running mkfs.ext4 7 | FROM alpine:3.18 8 | LABEL maintainers="SPDK-CSI Authors" 9 | LABEL description="SPDK-CSI Plugin" 10 | 11 | COPY spdkcsi /usr/local/bin/spdkcsi 12 | 13 | RUN apk update && \ 14 | apk add nvme-cli open-iscsi e2fsprogs xfsprogs blkid xfsprogs-extra e2fsprogs-extra util-linux 15 | 16 | ENTRYPOINT ["/usr/local/bin/spdkcsi"] 17 | 18 | -------------------------------------------------------------------------------- /deploy/kubernetes/caching-node.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | name: simplyblock-caching-node-sa 6 | 7 | --- 8 | kind: ClusterRole 9 | apiVersion: rbac.authorization.k8s.io/v1 10 | metadata: 11 | name: simplyblock-caching-node-role 12 | rules: 13 | - apiGroups: [""] 14 | resources: ["pods"] 15 | verbs: ["list"] 16 | - apiGroups: ["apps"] 17 | resources: ["deployments"] 18 | verbs: ["create", "delete"] 19 | 20 | --- 21 | kind: ClusterRoleBinding 22 | apiVersion: rbac.authorization.k8s.io/v1 23 | metadata: 24 | name: simplyblock-pods-list 25 | subjects: 26 | - kind: ServiceAccount 27 | name: simplyblock-caching-node-sa 28 | namespace: default 29 | roleRef: 30 | kind: ClusterRole 31 | name: simplyblock-caching-node-role 32 | apiGroup: rbac.authorization.k8s.io 33 | 34 | --- 35 | apiVersion: apps/v1 36 | kind: DaemonSet 37 | metadata: 38 | name: simplyblock-caching-node-ds 39 | spec: 40 | selector: 41 | matchLabels: 42 | app: caching-node 43 | template: 44 | metadata: 45 | labels: 46 | app: caching-node 47 | spec: 48 | serviceAccountName: simplyblock-caching-node-sa 49 | nodeSelector: 50 | type: simplyblock-cache 51 | volumes: 52 | - name: dev-vol 53 | hostPath: 54 | path: /dev 55 | hostNetwork: true 56 | containers: 57 | - name: c-node-api-container 58 | image: simplyblock/simplyblock:dev 59 | imagePullPolicy: "Always" 60 | command: ["python", "simplyblock_web/caching_node_app_k8s.py"] 61 | env: 62 | - name: HOSTNAME 63 | valueFrom: 64 | fieldRef: 65 | fieldPath: spec.nodeName 66 | securityContext: 67 | privileged: true 68 | volumeMounts: 69 | - name: dev-vol 70 | mountPath: /dev 71 | -------------------------------------------------------------------------------- /deploy/kubernetes/config-map.yaml: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | # Copyright (c) Arm Limited and Contributors 3 | # Copyright (c) Intel Corporation 4 | --- 5 | apiVersion: v1 6 | kind: ConfigMap 7 | metadata: 8 | name: simplyblock-csi-cm 9 | data: 10 | config.json: |- 11 | { 12 | "simplybk": { 13 | "uuid": "37bd2c95-5c26-4a97-bd74-88f22c3d283b", 14 | "ip": "3.237.89.96" 15 | } 16 | } 17 | 18 | --- 19 | apiVersion: v1 20 | kind: ConfigMap 21 | metadata: 22 | name: simplyblock-caching-node-restart-script-cm 23 | data: 24 | restart_script.py: | 25 | import json, os, requests 26 | 27 | secret = json.loads(os.getenv("SPDKCSI_SECRET")) 28 | cluster_secret = secret['simplybk']['secret'] 29 | 30 | cluster_config = json.loads(os.getenv("CLUSTER_CONFIG")) 31 | cluster_uuid = cluster_config['simplybk']['uuid'] 32 | cluster_ip = cluster_config['simplybk']['ip'] 33 | hostname = os.getenv('HOSTNAME') 34 | hostname = hostname.split(".")[0] 35 | 36 | url = f'{cluster_ip}/cachingnode/recreate/{hostname}' 37 | headers = { 38 | 'Authorization': f'{cluster_uuid} {cluster_secret}' 39 | } 40 | 41 | print(f"making GET request to: {url}") 42 | response = requests.get(url, headers=headers) 43 | 44 | print("Response Text:", response.text) 45 | print("Response Code:", response.status_code) 46 | -------------------------------------------------------------------------------- /deploy/kubernetes/controller.yaml: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | # Copyright (c) Arm Limited and Contributors 3 | # Copyright (c) Intel Corporation 4 | --- 5 | kind: StatefulSet 6 | apiVersion: apps/v1 7 | metadata: 8 | name: simplyblock-csi-controller 9 | spec: 10 | serviceName: simplyblock-csi-controller 11 | replicas: 1 12 | selector: 13 | matchLabels: 14 | app: csi-controller 15 | template: 16 | metadata: 17 | labels: 18 | app: csi-controller 19 | spec: 20 | serviceAccountName: simplyblock-csi-controller-sa 21 | hostNetwork: true 22 | containers: 23 | - name: csi-provisioner 24 | image: registry.k8s.io/sig-storage/csi-provisioner:v4.0.1 25 | imagePullPolicy: "Always" 26 | args: 27 | - "--v=5" 28 | - "--csi-address=unix:///csi/csi-provisioner.sock" 29 | - "--timeout=30s" 30 | - "--retry-interval-start=500ms" 31 | - "--leader-election=false" 32 | - "--extra-create-metadata=true" 33 | - "--feature-gates=Topology=true" 34 | volumeMounts: 35 | - name: socket-dir 36 | mountPath: /csi 37 | - name: csi-snapshotter 38 | image: registry.k8s.io/sig-storage/csi-snapshotter:v8.2.0 39 | args: 40 | - "--csi-address=unix:///csi/csi-provisioner.sock" 41 | - "--v=5" 42 | - "--timeout=150s" 43 | - "--leader-election=false" 44 | imagePullPolicy: "Always" 45 | securityContext: 46 | privileged: true 47 | volumeMounts: 48 | - name: socket-dir 49 | mountPath: /csi 50 | - name: csi-attacher 51 | image: gcr.io/k8s-staging-sig-storage/csi-attacher:v4.5.1 52 | imagePullPolicy: "Always" 53 | args: 54 | - "--v=5" 55 | - "--csi-address=unix:///csi/csi-provisioner.sock" 56 | - "--leader-election=false" 57 | volumeMounts: 58 | - name: socket-dir 59 | mountPath: /csi 60 | - name: csi-resizer 61 | image: gcr.io/k8s-staging-sig-storage/csi-resizer:v1.10.1 62 | imagePullPolicy: "Always" 63 | args: 64 | - "--v=5" 65 | - "--csi-address=unix:///csi/csi-provisioner.sock" 66 | - "--leader-election=false" 67 | volumeMounts: 68 | - name: socket-dir 69 | mountPath: /csi 70 | - name: csi-health-monitor 71 | image: gcr.io/k8s-staging-sig-storage/csi-external-health-monitor-controller:v0.11.0 72 | imagePullPolicy: "Always" 73 | args: 74 | - "--v=5" 75 | - "--csi-address=unix:///csi/csi-provisioner.sock" 76 | #- "--leader-election" 77 | - "--http-endpoint=:8080" 78 | volumeMounts: 79 | - name: socket-dir 80 | mountPath: /csi 81 | ports: 82 | - containerPort: 8080 83 | name: http-endpoint 84 | protocol: TCP 85 | - name: csi-controller 86 | image: simplyblock/spdkcsi:latest 87 | imagePullPolicy: "Always" 88 | args: 89 | - "--v=5" 90 | - "--endpoint=unix:///csi/csi-provisioner.sock" 91 | - "--nodeid=$(NODE_ID)" 92 | - "--controller" 93 | env: 94 | - name: NODE_ID 95 | valueFrom: 96 | fieldRef: 97 | fieldPath: spec.nodeName 98 | volumeMounts: 99 | - name: socket-dir 100 | mountPath: /csi 101 | - name: csi-config 102 | mountPath: /etc/spdkcsi-config/ 103 | readOnly: true 104 | - name: csi-secret 105 | mountPath: /etc/spdkcsi-secret/ 106 | readOnly: true 107 | volumes: 108 | - name: socket-dir 109 | emptyDir: 110 | medium: "Memory" 111 | - name: csi-config 112 | configMap: 113 | name: simplyblock-csi-cm 114 | - name: csi-secret 115 | secret: 116 | secretName: simplyblock-csi-secret 117 | -------------------------------------------------------------------------------- /deploy/kubernetes/deploy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -ex 4 | 5 | CLUSTER_ID='cf50c029-212f-4c01-8e80-fcd3947ff7c3' 6 | MGMT_IP='44.203.108.107' 7 | CLUSTER_SECRET='ziVjCH713s4sjZPTZK30' 8 | 9 | # list in creation order 10 | files=(driver config-map nodeserver-config-map secret controller-rbac node-rbac controller node storageclass caching-node rbac-snapshot-controller setup-snapshot-controller snapshot.storage.k8s.io_volumesnapshotclasses snapshot.storage.k8s.io_volumesnapshotcontents snapshot.storage.k8s.io_volumesnapshots) 11 | 12 | if [ "$1" = "teardown" ]; then 13 | # delete in reverse order 14 | for ((i = ${#files[@]} - 1; i >= 0; i--)); do 15 | echo "=== kubectl delete -f ${files[i]}.yaml" 16 | kubectl delete -f "${files[i]}.yaml" 17 | done 18 | exit 0 19 | else 20 | for ((i = 0; i <= ${#files[@]} - 1; i++)); do 21 | echo "=== kubectl apply -f ${files[i]}.yaml" 22 | kubectl apply -f "${files[i]}.yaml" 23 | done 24 | fi 25 | 26 | echo "" 27 | echo "Deploying Caching node..." 28 | 29 | output=$(kubectl get nodes -l type=cache | wc -l) 30 | 31 | if [ $output -lt 2 ]; then 32 | echo "No caching nodes found. Exiting..." 33 | exit 0 34 | fi 35 | 36 | 37 | ## check if the caching nodes has required huge pages 38 | echo "-- caching nodes --" 39 | kubectl get nodes -l type=cache 40 | 41 | 42 | kubectl apply -f caching-node.yaml 43 | kubectl wait --timeout=3m --for=condition=ready pod -l app=caching-node 44 | 45 | for node in $(kubectl get pods -l app=caching-node -owide | awk 'NR>1 {print $(NF-3)}'); do 46 | echo "adding caching node: $node" 47 | 48 | curl --location "http://${MGMT_IP}/cachingnode/" \ 49 | --header "Content-Type: application/json" \ 50 | --header "Authorization: ${CLUSTER_ID} ${CLUSTER_SECRET}" \ 51 | --data '{ 52 | "cluster_id": "'"${CLUSTER_ID}"'", 53 | "node_ip": "'"${node}:5000"'", 54 | "iface_name": "eth0", 55 | "spdk_mem": "2g" 56 | } 57 | ' 58 | done 59 | -------------------------------------------------------------------------------- /deploy/kubernetes/driver.yaml: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | # Copyright (c) Arm Limited and Contributors 3 | # Copyright (c) Intel Corporation 4 | --- 5 | apiVersion: storage.k8s.io/v1 6 | kind: CSIDriver 7 | metadata: 8 | name: csi.simplyblock.io 9 | spec: 10 | attachRequired: true 11 | volumeLifecycleModes: 12 | - Persistent 13 | -------------------------------------------------------------------------------- /deploy/kubernetes/job.yaml: -------------------------------------------------------------------------------- 1 | kind: ClusterRoleBinding 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | metadata: 4 | name: simplyblock-pod-listing-role-binding 5 | subjects: 6 | - kind: ServiceAccount 7 | name: simplyblock-caching-node-service-account 8 | namespace: default 9 | roleRef: 10 | kind: ClusterRole 11 | name: simplyblock-pod-listing-role 12 | apiGroup: rbac.authorization.k8s.io 13 | 14 | --- 15 | kind: ClusterRole 16 | apiVersion: rbac.authorization.k8s.io/v1 17 | metadata: 18 | name: simplyblock-pod-listing-role 19 | rules: 20 | - apiGroups: [""] 21 | resources: ["pods"] 22 | verbs: ["list"] 23 | 24 | --- 25 | apiVersion: v1 26 | kind: ServiceAccount 27 | metadata: 28 | name: simplyblock-caching-node-service-account 29 | --- 30 | apiVersion: batch/v1 31 | kind: Job 32 | metadata: 33 | name: add-caching-node-job 34 | spec: 35 | template: 36 | spec: 37 | serviceAccountName: simplyblock-caching-node-service-account 38 | containers: 39 | - name: add-caching-node 40 | image: manoharbrm/add-caching-node-script:latest 41 | env: 42 | - name: SPDKCSI_SECRET 43 | valueFrom: 44 | secretKeyRef: 45 | name: simplyblock-csi-secret 46 | key: secret.json 47 | - name: CLUSTER_CONFIG 48 | valueFrom: 49 | configMapKeyRef: 50 | name: simplyblock-csi-cm 51 | key: config.json 52 | restartPolicy: Never 53 | backoffLimit: 4 54 | -------------------------------------------------------------------------------- /deploy/kubernetes/mysql-pvc.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: PersistentVolumeClaim 4 | metadata: 5 | name: mysql-pv-claim 6 | labels: 7 | app: wordpress 8 | spec: 9 | storageClassName: simplyblock-csi-sc 10 | accessModes: 11 | - ReadWriteOnce 12 | resources: 13 | requests: 14 | storage: 10Gi 15 | --- 16 | apiVersion: v1 17 | kind: Service 18 | metadata: 19 | name: wordpress-mysql 20 | labels: 21 | app: wordpress 22 | spec: 23 | ports: 24 | - port: 3306 25 | selector: 26 | app: wordpress 27 | tier: mysql 28 | clusterIP: None 29 | --- 30 | apiVersion: apps/v1 31 | kind: Deployment 32 | metadata: 33 | name: wordpress-mysql 34 | labels: 35 | app: wordpress 36 | spec: 37 | selector: 38 | matchLabels: 39 | app: wordpress 40 | tier: mysql 41 | strategy: 42 | type: Recreate 43 | template: 44 | metadata: 45 | labels: 46 | app: wordpress 47 | tier: mysql 48 | spec: 49 | containers: 50 | - image: mysql 51 | name: mysql 52 | env: 53 | - name: MYSQL_ROOT_PASSWORD 54 | value: mysql 55 | ports: 56 | - containerPort: 3306 57 | name: mysql 58 | volumeMounts: 59 | - name: mysql-persistent-storage 60 | mountPath: /var/lib/mysql 61 | volumes: 62 | - name: mysql-persistent-storage 63 | persistentVolumeClaim: 64 | claimName: mysql-pv-claim 65 | -------------------------------------------------------------------------------- /deploy/kubernetes/node-rbac.yaml: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | # Copyright (c) Arm Limited and Contributors 3 | # Copyright (c) Intel Corporation 4 | --- 5 | apiVersion: v1 6 | kind: ServiceAccount 7 | metadata: 8 | name: simplyblock-csi-node-sa 9 | -------------------------------------------------------------------------------- /deploy/kubernetes/node.yaml: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | # Copyright (c) Arm Limited and Contributors 3 | # Copyright (c) Intel Corporation 4 | --- 5 | kind: DaemonSet 6 | apiVersion: apps/v1 7 | metadata: 8 | name: simplyblock-csi-node 9 | spec: 10 | selector: 11 | matchLabels: 12 | app: csi-node 13 | template: 14 | metadata: 15 | labels: 16 | app: csi-node 17 | spec: 18 | serviceAccountName: simplyblock-csi-node-sa 19 | hostNetwork: true 20 | containers: 21 | - name: csi-registrar 22 | securityContext: 23 | privileged: true 24 | image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.10.1 25 | imagePullPolicy: "IfNotPresent" 26 | args: 27 | - "--v=5" 28 | - "--csi-address=unix:///csi/csi.sock" 29 | - "--kubelet-registration-path=/var/lib/kubelet/plugins/csi.simplyblock.io/csi.sock" 30 | - "--health-port=9809" 31 | ports: 32 | - containerPort: 9809 33 | name: healthz 34 | livenessProbe: 35 | httpGet: 36 | path: /healthz 37 | port: healthz 38 | initialDelaySeconds: 20 39 | timeoutSeconds: 10 40 | volumeMounts: 41 | - name: socket-dir 42 | mountPath: /csi 43 | - name: registration-dir 44 | mountPath: /registration 45 | - name: csi-node 46 | securityContext: 47 | privileged: true 48 | capabilities: 49 | add: ["SYS_ADMIN", "SYS_MODULE"] 50 | allowPrivilegeEscalation: true 51 | image: simplyblock/spdkcsi:latest 52 | imagePullPolicy: "Always" 53 | args: 54 | - "--v=5" 55 | - "--endpoint=unix:///csi/csi.sock" 56 | - "--nodeid=$(NODE_ID)" 57 | - "--node" 58 | env: 59 | - name: NODE_ID 60 | valueFrom: 61 | fieldRef: 62 | fieldPath: spec.nodeName 63 | lifecycle: 64 | postStart: 65 | exec: 66 | command: 67 | [ 68 | "/bin/sh", "-c", 69 | "sudo modprobe nvme-tcp || echo failed to modprobe nvme-tcp && \ 70 | if [ ! -f /var/lib/nvme/hostid ]; then uuidgen > /var/lib/nvme/hostid; fi && \ 71 | cp /var/lib/nvme/hostid /etc/nvme/hostid && \ 72 | echo \"nqn.2014-08.org.nvmexpress:uuid:$(cat /etc/nvme/hostid)\" > /etc/nvme/hostnqn" 73 | ] 74 | volumeMounts: 75 | - name: socket-dir 76 | mountPath: /csi 77 | - name: plugin-dir 78 | mountPath: /var/lib/kubelet/plugins 79 | mountPropagation: "Bidirectional" 80 | - name: pod-dir 81 | mountPath: /var/lib/kubelet/pods 82 | mountPropagation: "Bidirectional" 83 | - name: host-dev 84 | mountPath: /dev 85 | - name: host-sys 86 | mountPath: /sys 87 | - name: csi-nodeserver-config 88 | mountPath: /etc/spdkcsi-nodeserver-config/ 89 | readOnly: true 90 | - name: csi-config 91 | mountPath: /etc/spdkcsi-config/ 92 | readOnly: true 93 | - name: csi-secret 94 | mountPath: /etc/spdkcsi-secret/ 95 | readOnly: true 96 | volumes: 97 | - name: socket-dir 98 | hostPath: 99 | path: /var/lib/kubelet/plugins/csi.simplyblock.io 100 | type: DirectoryOrCreate 101 | - name: registration-dir 102 | hostPath: 103 | path: /var/lib/kubelet/plugins_registry/ 104 | type: Directory 105 | - name: plugin-dir 106 | hostPath: 107 | path: /var/lib/kubelet/plugins 108 | type: Directory 109 | - name: pod-dir 110 | hostPath: 111 | path: /var/lib/kubelet/pods 112 | type: Directory 113 | - name: host-dev 114 | hostPath: 115 | path: /dev 116 | - name: host-sys 117 | hostPath: 118 | path: /sys 119 | - name: csi-nodeserver-config 120 | configMap: 121 | name: simplyblock-csi-nodeservercm 122 | optional: true 123 | - name: csi-config 124 | configMap: 125 | name: simplyblock-csi-cm 126 | - name: csi-secret 127 | secret: 128 | secretName: simplyblock-csi-secret 129 | 130 | -------------------------------------------------------------------------------- /deploy/kubernetes/nodeserver-config-map.yaml: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | # Copyright (c) Arm Limited and Contributors 3 | # Copyright (c) Intel Corporation 4 | --- 5 | apiVersion: v1 6 | kind: ConfigMap 7 | metadata: 8 | name: simplyblock-csi-nodeservercm 9 | data: 10 | # xpu.targetType: 11 | # - SMA: xpu-sma-nvmftcp, xpu-sma-virtioblk, xpu-sma-nvme 12 | # xpu.targetAddr: 13 | # - URL to connect the xPU node through GRPC, IPADDR:PORT 14 | # - 127.0.0.1:5114 for SMA server by default 15 | # kvmPciBridges: 16 | # - used by sma-virtioblk, sma-nvme 17 | # - based on the configuration in "deploy/spdk/sma.yaml" and qemu VM 18 | # 19 | # example: 20 | # nodeserver-config.json: |- 21 | # { 22 | # "xpuList": [ 23 | # { 24 | # "name": "xPU0", 25 | # "targetType": "xpu-sma-nvmftcp", 26 | # "targetAddr": "127.0.0.1:5114" 27 | # } 28 | # ], 29 | # "kvmPciBridges": 2 30 | # } 31 | nodeserver-config.json: |- 32 | { 33 | "xpuList": [], 34 | "kvmPciBridges": null 35 | } 36 | -------------------------------------------------------------------------------- /deploy/kubernetes/prepare-cachingnode.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ "$(id -u)" -ne 0 ]; then 4 | echo "This script requires sudo privileges. Please enter your password:" 5 | exec sudo "$0" "$@" # This re-executes the script with sudo 6 | fi 7 | 8 | echo "======= setting huge pages =======" 9 | echo "vm.nr_hugepages=2048" >>/etc/sysctl.conf 10 | sysctl -p 11 | 12 | # confirm it by running 13 | cat /proc/meminfo | grep -i hug 14 | 15 | echo "======= creating huge pages mount =======" 16 | mkdir /mnt/huge 17 | mount -t hugetlbfs -o size=2G nodev /mnt/huge 18 | echo "nodev /mnt/huge hugetlbfs size=2G 0 0" >>/etc/fstab 19 | 20 | ## TODO: detect the present of additional NVMe Volume 21 | -------------------------------------------------------------------------------- /deploy/kubernetes/rbac-snapshot-controller.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | name: simplyblock-snapshot-controller 6 | namespace: kube-system 7 | 8 | --- 9 | kind: ClusterRole 10 | apiVersion: rbac.authorization.k8s.io/v1 11 | metadata: 12 | name: simplyblock-snapshot-controller-runner 13 | rules: 14 | - apiGroups: [""] 15 | resources: ["persistentvolumes"] 16 | verbs: ["get", "list", "watch"] 17 | - apiGroups: [""] 18 | resources: ["persistentvolumeclaims"] 19 | verbs: ["get", "list", "watch", "update"] 20 | - apiGroups: [""] 21 | resources: ["events"] 22 | verbs: ["list", "watch", "create", "update", "patch"] 23 | - apiGroups: ["snapshot.storage.k8s.io"] 24 | resources: ["volumesnapshotclasses"] 25 | verbs: ["get", "list", "watch"] 26 | - apiGroups: ["snapshot.storage.k8s.io"] 27 | resources: ["volumesnapshotcontents"] 28 | verbs: ["create", "get", "list", "watch", "update", "delete", "patch"] 29 | - apiGroups: ["snapshot.storage.k8s.io"] 30 | resources: ["volumesnapshotcontents/status"] 31 | verbs: ["patch"] 32 | - apiGroups: ["snapshot.storage.k8s.io"] 33 | resources: ["volumesnapshots"] 34 | verbs: ["get", "list", "watch", "update", "patch", "delete"] 35 | - apiGroups: ["snapshot.storage.k8s.io"] 36 | resources: ["volumesnapshots/status"] 37 | verbs: ["update", "patch"] 38 | 39 | - apiGroups: ["groupsnapshot.storage.k8s.io"] 40 | resources: ["volumegroupsnapshotclasses"] 41 | verbs: ["get", "list", "watch"] 42 | - apiGroups: ["groupsnapshot.storage.k8s.io"] 43 | resources: ["volumegroupsnapshotcontents"] 44 | verbs: ["create", "get", "list", "watch", "update", "delete", "patch"] 45 | - apiGroups: ["groupsnapshot.storage.k8s.io"] 46 | resources: ["volumegroupsnapshotcontents/status"] 47 | verbs: ["patch"] 48 | - apiGroups: ["groupsnapshot.storage.k8s.io"] 49 | resources: ["volumegroupsnapshots"] 50 | verbs: ["get", "list", "watch", "update", "patch"] 51 | - apiGroups: ["groupsnapshot.storage.k8s.io"] 52 | resources: ["volumegroupsnapshots/status"] 53 | verbs: ["update", "patch"] 54 | 55 | # Enable this RBAC rule only when using distributed snapshotting, i.e. when the enable-distributed-snapshotting flag is set to true 56 | # - apiGroups: [""] 57 | # resources: ["nodes"] 58 | # verbs: ["get", "list", "watch"] 59 | --- 60 | kind: ClusterRoleBinding 61 | apiVersion: rbac.authorization.k8s.io/v1 62 | metadata: 63 | name: simplyblock-snapshot-controller-role 64 | subjects: 65 | - kind: ServiceAccount 66 | name: simplyblock-snapshot-controller 67 | namespace: kube-system 68 | roleRef: 69 | kind: ClusterRole 70 | name: simplyblock-snapshot-controller-runner 71 | apiGroup: rbac.authorization.k8s.io 72 | 73 | --- 74 | kind: Role 75 | apiVersion: rbac.authorization.k8s.io/v1 76 | metadata: 77 | name: simplyblock-snapshot-controller-leaderelection 78 | namespace: kube-system 79 | rules: 80 | - apiGroups: ["coordination.k8s.io"] 81 | resources: ["leases"] 82 | verbs: ["get", "watch", "list", "delete", "update", "create"] 83 | 84 | --- 85 | kind: RoleBinding 86 | apiVersion: rbac.authorization.k8s.io/v1 87 | metadata: 88 | name: simplyblock-snapshot-controller-leaderelection 89 | namespace: kube-system 90 | subjects: 91 | - kind: ServiceAccount 92 | name: simplyblock-snapshot-controller 93 | roleRef: 94 | kind: Role 95 | name: simplyblock-snapshot-controller-leaderelection 96 | apiGroup: rbac.authorization.k8s.io 97 | -------------------------------------------------------------------------------- /deploy/kubernetes/secret.yaml: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | # Copyright (c) Arm Limited and Contributors 3 | # Copyright (c) Intel Corporation 4 | --- 5 | apiVersion: v1 6 | kind: Secret 7 | metadata: 8 | name: simplyblock-csi-secret 9 | stringData: 10 | # Specify node specific token with item "name" matches ConfigMap. 11 | # { 12 | # "name": "spdk-testnode", 13 | # "username": "myuser", 14 | # "password": "mypass" 15 | # } 16 | secret.json: |- 17 | { 18 | "simplybk": { 19 | "secret": "RhOaHtnL7cr0kPXk7WD7" 20 | } 21 | } 22 | --- 23 | apiVersion: v1 24 | kind: Secret 25 | metadata: 26 | name: simplyblock-pvc-keys 27 | namespace: default 28 | data: 29 | crypto_key1: N2IzNjk1MjY4ZTJhNjYxMWEyNWFjNGIxZWUxNWYyN2Y5YmY2ZWE5NzgzZGFkYTY2YTRhNzMwZWJmMDQ5MmJmZA== 30 | crypto_key2: Nzg1MDU2MzZjODEzM2Q5YmU0MmUzNDdmODI3ODViODFhODc5Y2Q4MTMzMDQ2ZjhmYzBiMzZmMTdiMDc4YWQwYw== 31 | -------------------------------------------------------------------------------- /deploy/kubernetes/setup-snapshot-controller.yaml: -------------------------------------------------------------------------------- 1 | # This YAML file shows how to deploy the snapshot controller 2 | 3 | # The snapshot controller implements the control loop for CSI snapshot functionality. 4 | # It should be installed as part of the base Kubernetes distribution in an appropriate 5 | # namespace for components implementing base system functionality. For installing with 6 | # Vanilla Kubernetes, kube-system makes sense for the namespace. 7 | --- 8 | kind: Deployment 9 | apiVersion: apps/v1 10 | metadata: 11 | name: simplyblock-snapshot-controller 12 | namespace: kube-system 13 | spec: 14 | replicas: 2 15 | selector: 16 | matchLabels: 17 | app.kubernetes.io/name: snapshot-controller 18 | # The snapshot controller won't be marked as ready if the v1 CRDs are unavailable. 19 | # The flag --retry-crd-interval-max is used to determine how long the controller 20 | # will wait for the CRDs to become available before exiting. The default is 30 seconds 21 | # so minReadySeconds should be set slightly higher than the flag value. 22 | minReadySeconds: 35 23 | strategy: 24 | rollingUpdate: 25 | maxSurge: 0 26 | maxUnavailable: 1 27 | type: RollingUpdate 28 | template: 29 | metadata: 30 | labels: 31 | app.kubernetes.io/name: snapshot-controller 32 | spec: 33 | serviceAccountName: simplyblock-snapshot-controller 34 | containers: 35 | - name: snapshot-controller 36 | image: registry.k8s.io/sig-storage/snapshot-controller:v8.2.0 37 | args: 38 | - "--v=5" 39 | - "--leader-election=true" 40 | imagePullPolicy: IfNotPresent 41 | -------------------------------------------------------------------------------- /deploy/kubernetes/snapshotclass.yaml: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | # Copyright (c) Arm Limited and Contributors 3 | # Copyright (c) Intel Corporation 4 | 5 | # Snapshot API version compatibility: 6 | # v1beta1: 7 | # v1.17 =< k8s < v1.20 8 | # 2.x =< snapshot-controller < v4.x 9 | # We recommend to use {sidecar, controller, crds} of same version 10 | --- 11 | apiVersion: snapshot.storage.k8s.io/v1 12 | kind: VolumeSnapshotClass 13 | metadata: 14 | name: simplyblock-csi-snapshotclass 15 | driver: csi.simplyblock.io 16 | deletionPolicy: Delete 17 | -------------------------------------------------------------------------------- /deploy/kubernetes/storageclass.yaml: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | # Copyright (c) Arm Limited and Contributors 3 | # Copyright (c) Intel Corporation 4 | --- 5 | apiVersion: storage.k8s.io/v1 6 | kind: StorageClass 7 | metadata: 8 | name: simplyblock-csi-sc 9 | provisioner: csi.simplyblock.io 10 | parameters: 11 | csi.storage.k8s.io/fstype: ext4 12 | pool_name: testing1 13 | qos_rw_iops: "0" 14 | qos_rw_mbytes: "0" 15 | qos_r_mbytes: "0" 16 | qos_w_mbytes: "0" 17 | max_size: "0" 18 | compression: "False" 19 | encryption: "False" 20 | distr_ndcs: "1" 21 | distr_npcs: "1" 22 | lvol_priority_class: "0" 23 | reclaimPolicy: Delete 24 | volumeBindingMode: Immediate 25 | allowVolumeExpansion: true 26 | 27 | --- 28 | apiVersion: storage.k8s.io/v1 29 | kind: StorageClass 30 | metadata: 31 | name: simplyblock-csi-sc-cache 32 | provisioner: csi.simplyblock.io 33 | parameters: 34 | csi.storage.k8s.io/fstype: ext4 35 | pool_name: testing1 36 | type: cache 37 | qos_rw_iops: "0" 38 | qos_rw_mbytes: "0" 39 | qos_r_mbytes: "0" 40 | qos_w_mbytes: "0" 41 | max_size: "0" 42 | compression: "False" 43 | encryption: "False" 44 | distr_ndcs: "1" 45 | distr_npcs: "1" 46 | lvol_priority_class: "0" 47 | reclaimPolicy: Delete 48 | volumeBindingMode: Immediate 49 | allowVolumeExpansion: true 50 | -------------------------------------------------------------------------------- /deploy/kubernetes/testclone.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: PersistentVolumeClaim 4 | metadata: 5 | name: spdkcsi-pvc-clone 6 | spec: 7 | storageClassName: simplyblock-csi-sc 8 | dataSource: 9 | name: spdkcsi-pvc 10 | kind: PersistentVolumeClaim 11 | apiGroup: "" 12 | accessModes: 13 | - ReadWriteOnce 14 | resources: 15 | requests: 16 | storage: 1Gi 17 | --- 18 | kind: Pod 19 | apiVersion: v1 20 | metadata: 21 | name: spdkcsi-test-clone 22 | spec: 23 | containers: 24 | - name: alpine 25 | image: alpine:3 26 | imagePullPolicy: "IfNotPresent" 27 | command: ["sleep", "365d"] 28 | volumeMounts: 29 | - mountPath: "/spdkvol" 30 | name: spdk-volume 31 | volumes: 32 | - name: spdk-volume 33 | persistentVolumeClaim: 34 | claimName: spdkcsi-pvc-clone 35 | -------------------------------------------------------------------------------- /deploy/kubernetes/testpod-cache.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | kind: PersistentVolumeClaim 3 | apiVersion: v1 4 | metadata: 5 | name: spdkcsi-cache-pvc 6 | annotations: 7 | simplybk/secret-name: simplyblock-pvc-keys 8 | simplybk/secret-namespace: default 9 | spec: 10 | accessModes: 11 | - ReadWriteOnce 12 | resources: 13 | requests: 14 | storage: 10Gi 15 | storageClassName: spdkcsi-sc-cache 16 | 17 | --- 18 | kind: Pod 19 | apiVersion: v1 20 | metadata: 21 | name: spdkcsi-cache-test 22 | spec: 23 | nodeSelector: 24 | type: cache 25 | containers: 26 | - name: alpine 27 | image: alpine:3 28 | imagePullPolicy: "IfNotPresent" 29 | command: ["sleep", "365d"] 30 | volumeMounts: 31 | - mountPath: "/spdkvol" 32 | name: spdk-volume 33 | volumes: 34 | - name: spdk-volume 35 | persistentVolumeClaim: 36 | claimName: spdkcsi-cache-pvc 37 | -------------------------------------------------------------------------------- /deploy/kubernetes/testpod.yaml: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | # Copyright (c) Arm Limited and Contributors 3 | # Copyright (c) Intel Corporation 4 | --- 5 | kind: PersistentVolumeClaim 6 | apiVersion: v1 7 | metadata: 8 | name: spdkcsi-pvc 9 | annotations: 10 | simplybk/secret-name: simplyblock-pvc-keys 11 | simplybk/secret-namespace: default 12 | spec: 13 | accessModes: 14 | - ReadWriteOnce 15 | resources: 16 | requests: 17 | storage: 10Gi 18 | storageClassName: spdkcsi-sc 19 | 20 | --- 21 | kind: Pod 22 | apiVersion: v1 23 | metadata: 24 | name: spdkcsi-test 25 | spec: 26 | containers: 27 | - name: alpine 28 | image: alpine:3 29 | imagePullPolicy: "IfNotPresent" 30 | command: ["sleep", "365d"] 31 | volumeMounts: 32 | - mountPath: "/spdkvol" 33 | name: spdk-volume 34 | volumes: 35 | - name: spdk-volume 36 | persistentVolumeClaim: 37 | claimName: spdkcsi-pvc 38 | -------------------------------------------------------------------------------- /deploy/kubernetes/testresize.yaml: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | # Copyright (c) Arm Limited and Contributors 3 | # Copyright (c) Intel Corporation 4 | --- 5 | kind: PersistentVolumeClaim 6 | apiVersion: v1 7 | metadata: 8 | name: spdkcsi-pvc 9 | spec: 10 | accessModes: 11 | - ReadWriteOnce 12 | resources: 13 | requests: 14 | storage: 456Mi 15 | storageClassName: spdkcsi-sc 16 | -------------------------------------------------------------------------------- /deploy/kubernetes/testrestore.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: PersistentVolumeClaim 4 | metadata: 5 | name: spdkcsi-pvc-restore 6 | spec: 7 | storageClassName: spdkcsi-sc 8 | dataSource: 9 | name: spdk-snapshot 10 | kind: VolumeSnapshot 11 | apiGroup: snapshot.storage.k8s.io 12 | accessModes: 13 | - ReadWriteOnce 14 | resources: 15 | requests: 16 | storage: 512Mi 17 | --- 18 | kind: Pod 19 | apiVersion: v1 20 | metadata: 21 | name: spdkcsi-test-restore 22 | spec: 23 | containers: 24 | - name: alpine 25 | image: alpine:3 26 | imagePullPolicy: "IfNotPresent" 27 | command: ["sleep", "365d"] 28 | volumeMounts: 29 | - mountPath: "/spdkvol" 30 | name: spdk-volume 31 | volumes: 32 | - name: spdk-volume 33 | persistentVolumeClaim: 34 | claimName: spdkcsi-pvc-restore 35 | -------------------------------------------------------------------------------- /deploy/kubernetes/testsnapshot.yaml: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | # Copyright (c) Arm Limited and Contributors 3 | # Copyright (c) Intel Corporation 4 | 5 | # Snapshot API version compatibility matrix: 6 | # v1betav1: 7 | # v1.17 =< k8s < v1.20 8 | # 2.x =< snapshot-controller < v4.x 9 | # v1: 10 | # k8s >= v1.20 11 | # snapshot-controller >= v4.x 12 | # We recommend to use {sidecar, controller, crds} of same version 13 | --- 14 | apiVersion: snapshot.storage.k8s.io/v1 15 | kind: VolumeSnapshot 16 | metadata: 17 | name: spdk-snapshot 18 | spec: 19 | volumeSnapshotClassName: simplyblock-csi-snapshotclass 20 | source: 21 | persistentVolumeClaimName: spdkcsi-pvc 22 | -------------------------------------------------------------------------------- /deploy/spdk/Dockerfile: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | # Copyright (c) Arm Limited and Contributors 3 | # Copyright (c) Intel Corporation 4 | FROM fedora:37 5 | 6 | ARG TAG=v23.01 7 | ARG ARCH=native 8 | 9 | WORKDIR /root 10 | RUN dnf install -y git diffutils procps-ng pip kmod 11 | RUN git clone https://github.com/spdk/spdk --branch ${TAG} --depth 1 && \ 12 | cd spdk && git submodule update --init --depth 1 && scripts/pkgdep.sh --rdma 13 | 14 | RUN pip3 install grpcio-tools==1.51.3 protobuf==4.22.1 15 | RUN cd spdk && \ 16 | ./configure --disable-tests --with-vhost --with-virtio --with-sma --with-vfio-user \ 17 | --with-rdma --target-arch=${ARCH} && \ 18 | make 19 | 20 | COPY sma.yaml . 21 | -------------------------------------------------------------------------------- /deploy/spdk/README.md: -------------------------------------------------------------------------------- 1 | # About 2 | 3 | To test SPDKCSI, we need working SPDK environments. 4 | This document explains how to launch SPDK storage nodes with JsonRPC HTTP proxy, 5 | and SPDK xPU nodes with SMA server on localhost for function tests. 6 | 7 | ```bash 8 | # build spdk container image, which could be used for SPDK storage node and xPU node 9 | cd deploy/spdk 10 | sudo docker build -t spdkdev . 11 | ``` 12 | 13 | ## Step by step to launch a SPDK storage node 14 | 15 | ```bash 16 | # allocate 2G hugepages (1024*2M) 17 | sudo sh -c 'echo 1024 > /proc/sys/vm/nr_hugepages' 18 | 19 | # start spdk target 20 | sudo docker run -it --rm --name spdkdev --privileged --net host \ 21 | -v /dev/hugepages:/dev/hugepages -v /dev/shm:/dev/shm spdkdev /root/spdk/build/bin/spdk_tgt 22 | # run below commands in another console 23 | 24 | # create 1G malloc bdev 25 | sudo docker exec -it spdkdev /root/spdk/scripts/rpc.py bdev_malloc_create -b Malloc0 1024 4096 26 | 27 | # create lvstore 28 | sudo docker exec -it spdkdev /root/spdk/scripts/rpc.py bdev_lvol_create_lvstore Malloc0 lvs0 29 | 30 | # start jsonrpc http proxy on 127.0.0.1:9009 31 | sudo docker exec -it spdkdev /root/spdk/scripts/rpc_http_proxy.py 127.0.0.1 9009 spdkcsiuser spdkcsipass 32 | ``` 33 | 34 | ## Single command to launch a SPDK storage node 35 | 36 | Combine above steps to a single command can be convenient. But it's harder to debug if error happens. 37 | 38 | ```bash 39 | sudo docker run -it --rm --privileged --net host \ 40 | -v /dev/hugepages:/dev/hugepages -v /dev/shm:/dev/shm -v /proc:/proc \ 41 | spdkdev sh -c 'echo 1024 > /proc/sys/vm/nr_hugepages && \ 42 | /root/spdk/build/bin/spdk_tgt > /tmp/spdk-tgt.log 2>&1 & \ 43 | echo wait 5s... && sleep 5s && cd /root/spdk/scripts && \ 44 | ./rpc.py bdev_malloc_create -b Malloc0 1024 4096 && \ 45 | ./rpc.py bdev_lvol_create_lvstore Malloc0 lvs0 && \ 46 | ./rpc_http_proxy.py 127.0.0.1 9009 spdkcsiuser spdkcsipass' 47 | ``` 48 | 49 | ## Step by step to launch a SPDK xPU node with SMA 50 | 51 | ```bash 52 | # start spdkdev_sma container 53 | sudo docker run -it --rm --name spdkdev_sma --privileged --net host -v /dev/hugepages:/dev/hugepages \ 54 | -v /dev/shm:/dev/shm -v /var/tmp:/var/tmp -v /lib/modules:/lib/modules spdkdev 55 | 56 | # run below commands in another console 57 | # start spdk target 58 | sudo docker exec -id spdkdev_sma sh -c \ 59 | "/root/spdk/scripts/setup.sh; /root/spdk/build/bin/spdk_tgt -S /var/tmp -m 0x3 &" 60 | 61 | # start sma server 62 | sudo docker exec -id spdkdev_sma sh -c "/root/spdk/scripts/sma.py --config /root/sma.yaml" 63 | ``` 64 | 65 | ## Step by step to launch a SPDK xPU node with OPI 66 | 67 | ```bash 68 | # start spdkdev_opi container 69 | sudo docker run -it --rm --name spdkdev_opi --privileged --net host -v /dev/hugepages:/dev/hugepages \ 70 | -v /dev/shm:/dev/shm -v /var/tmp:/var/tmp -v /lib/modules:/lib/modules spdkdev 71 | 72 | # run below commands in another console 73 | # start spdk target 74 | sudo docker exec -id spdkdev_opi sh -c \ 75 | "/root/spdk/scripts/setup.sh; /root/spdk/build/bin/spdk_tgt -S /var/tmp -m 0x3 &" 76 | 77 | # create VFIOUSER NVMf transport 78 | sudo docker exec -i spdkdev_opi /root/spdk/scripts/rpc.py nvmf_create_transport -t VFIOUSER 79 | 80 | # start opi-spdk-bridge container 81 | sudo docker run -id --name opi-spdk-bridge --net host -v /var/tmp/:/var/tmp/ "${OPIIMAGE}" /opi-spdk-bridge \ 82 | -port=50051 -kvm=true -qmp_addr=127.0.0.1:9090 -spdk_addr=/var/tmp/spdk.sock -ctrlr_dir=/var/tmp \ 83 | -tcp_trid=127.0.0.1:4420 -buses=pci.spdk.0:pci.spdk.1 84 | ``` 85 | -------------------------------------------------------------------------------- /deploy/spdk/sma.yaml: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | # Copyright (c) Arm Limited and Contributors 3 | # Copyright (c) Intel Corporation 4 | --- 5 | address: 'localhost' 6 | socket: '/var/tmp/spdk.sock' 7 | port: 5114 8 | devices: 9 | - name: 'nvmf_tcp' 10 | - name: 'vfiouser' 11 | params: 12 | buses: 13 | - name: 'pci.spdk.0' 14 | count: 32 15 | - name: 'pci.spdk.1' 16 | count: 32 17 | qmp_addr: 127.0.0.1 18 | qmp_port: 9090 19 | - name: 'vhost_blk' 20 | params: 21 | buses: 22 | - name: 'pci.spdk.0' 23 | count: 32 24 | - name: 'pci.spdk.1' 25 | count: 32 26 | qmp_addr: 127.0.0.1 27 | qmp_port: 9090 28 | -------------------------------------------------------------------------------- /docs/caching-nodes.md: -------------------------------------------------------------------------------- 1 | ### Caching nodes volume provisioning 2 | 3 | We now also have the concept of caching-nodes. These will be Kubernetes nodes which reside within the kubernetes compute cluster as normal worker nodes and can have any PODs deployed. However, they have an NVMe disk locally attached. If there are multiple NVMe disks attached, it will use the first one. 4 | 5 | 6 | ### Preparing nodes 7 | Caching nodes are a special kind of node that works as a cache with a local NVMe disk. 8 | 9 | #### Step 0: Networking & tools 10 | 11 | Make sure that the Kubernetes worker nodes to be used for cache has access to the simplyblock storage cluster. If you are using terraform to deploy the cluster. Please attach `container-instance-sg` security group to all the instances. 12 | 13 | #### Step1: Install nvme cli tools and nbd 14 | 15 | To attach NVMe device to the host machine, the CSI driver uses [nvme-cli]([url](https://github.com/linux-nvme/nvme-cli)). So lets install that 16 | ``` 17 | sudo yum install -y nvme-cli 18 | sudo modprobe nvme-tcp 19 | sudo modprobe nbd 20 | ``` 21 | 22 | #### Step1: Setup hugepages 23 | 24 | Before you prepare the caching nodes, please decide the amount of huge pages that you would like to allocate for simplyblock and set those hugepages accordingly. 25 | It is recommended to use a minimum of 1 GiB + 0.5% of the size of the local SSD, which you want to use as a cache. For example, if your local SSD has a size of 1.9 TiB, and you want to use it entirely as a write-through cache, you need to assign 10.5 GiB of RAM. If you only want to utilize 1 TiB (52.9% of the SSD), you assign 6 GiB of RAM and the cache will be automatically resized to fit the available (assigned) memory. 26 | 27 | >[!IMPORTANT] 28 | >One huge page contains 2 MiB of memory. A value of e.g. 4096 therefore is equal to 8 GiB of huge page memory. 29 | 30 | ``` 31 | sudo sysctl -w vm.nr_hugepages=4096 32 | ``` 33 | 34 | confirm the hugepage changes by running 35 | cat /proc/meminfo | grep -i hug 36 | 37 | 38 | and restart kubelet 39 | ``` 40 | sudo systemctl restart kubelet 41 | ``` 42 | 43 | conform if huge pages are added to the cluster or not. 44 | ``` 45 | kubectl describe node ip-10-0-2-184.us-east-2.compute.internal | grep hugepages-2Mi 46 | ``` 47 | this output should show 8GB. This worker node can allocate 8GB of hugepages to pods which is required in case of SPDK pods. 48 | 49 | #### Step2: Mount the SSD to be used for caching 50 | If the instance comes with a default NVMe disk, it can be used. Or an additional EBS or SSD volume can be mounted. the disks can be viewed by running: 51 | 52 | ``` 53 | sudo yum install pciutils 54 | lspci 55 | ``` 56 | 57 | 58 | #### Step3: Tag the kubernetes nodes 59 | 60 | After the nodes are prepared, label the kubernetes nodes 61 | ``` 62 | kubectl label nodes ip-10-0-4-118.us-east-2.compute.internal ip-10-0-4-176.us-east-2.compute.internal type=simplyblock-cache 63 | ``` 64 | Now the nodes are ready to deploy caching nodes. 65 | 66 | ### StorageClass 67 | 68 | If the user wants to create a PVC that uses NVMe cache, a new storage class can be used with additional volume parameter as `type: cache`. 69 | 70 | 71 | ### Usage and Implementation 72 | 73 | During dynamic volume provisioning, nodeSelector should be provided on pod, deployment, daemonset, statefulset. So that such pods are scheduled only on the nodes that has the `simplyblock-cache` label on it. 74 | 75 | As shown below 76 | ``` 77 | spec: 78 | nodeSelector: 79 | type: simplyblock-cache 80 | ``` 81 | 82 | On the controller server, when a new volume is requested, we create a `lvol` . This steps is exactly same as the current implementation. 83 | 84 | On the node driver, during the volume mount, the following steps happens. 85 | 1. Get the caching node ID of the current node 86 | 2. Connect caching node with lvol. This will create a new NVMe device on the host machine. This device will be used to mount into pod. 87 | -------------------------------------------------------------------------------- /docs/csi-debug.md: -------------------------------------------------------------------------------- 1 | ### case#1: volume mount/unmount failed 2 | 3 | #### simplyblock CSI driver requires nvme-cli to be installed. 4 | 5 | If the kubernetes worker doesn't have nvme-cli installed, you might get error like this during the volume mount stage. 6 | ``` 7 | Warning FailedMount 17s (x7 over 49s) kubelet MountVolume.MountDevice failed for volume "pvc-ac6f4696-18a8-4e98-a0d4-28237eef5ad9" : rpc error: code = Internal desc = exit status 254 8 | ``` 9 | 10 | This because the CSI driver internally nvme-cli to connect to the remote nvme volume. The CLI can be installed by running 11 | ``` 12 | sudo yum install -y nvme-cli; 13 | sudo modprobe nvme-tcp 14 | ``` 15 | 16 | #### the worker nodes should be able to contact simplyblock storage node 17 | 18 | Since SPDK uses remote nvme connection, make sure that there is network connectivity between the simplyblock storage node and the kubernetes worker nodes. 19 | 20 | If not, we get errors like this during VolumeStage step. 21 | 22 | ``` 23 | Warning FailedMount 2s (x8 over 88s) kubelet MountVolume.MountDevice failed for volume "pvc-d191cb36-c542-4d1a-806d-28c1999a50c6" : rpc error: code = Internal desc = exit status 146 24 | ``` 25 | 26 | #### check the events of pvc or pod 27 | 28 | For any other types of failures in general, you could check the events of the Pod or PVC. 29 | 30 | ```console 31 | kubectl describe pod test-pod 32 | kubectl describe pvc test-pvc 33 | ``` 34 | 35 | For a more detailed analysis during the `CreateVolume` stage, you could refer to the controller. 36 | 37 | ``` 38 | kubectl -n spdk-csi logs -f spdkcsi-controller-0 -c spdkcsi-controller 39 | ``` 40 | 41 | Or you could refer to the node driver logs to get more info during the volume mount stage 42 | 43 | ``` 44 | kubectl -n spdk-csi logs -f spdkcsi-node-8vh5n -c spdkcsi-node 45 | ``` 46 | -------------------------------------------------------------------------------- /docs/encrypted-volumes.md: -------------------------------------------------------------------------------- 1 | ## How to create encrypted LVol 2 | 3 | Simplyblock logical volumes supports encryption at rest by leveraging [crypro bdev](https://spdk.io/doc/bdev.html) module using SPDK Software Accel framework. 4 | 5 | The framework internally uses AES_XTS cipher which is used to sequence arbitrary length of block data. This type of cipher is standardly used in backup software. 6 | 7 | AES_XTS cipher requires 2 keys of length either 16 bytes of 32 bytes. The hex value of the key can be generated using the command. 8 | 9 | for example to generate a 32 byte key 10 | ``` 11 | openssl rand -hex 32 12 | ``` 13 | encode the generated 32 byte key 14 | ``` 15 | echo -n '7b3695268e2a6611a25ac4b1ee15f27f9bf6ea9783dada66a4a730ebf0492bfd' | base64 16 | 17 | echo -n '78505636c8133d9be42e347f82785b81a879cd8133046f8fc0b36f17b078ad0c' | base64 18 | ``` 19 | After the keys are generated, an encrypted pvc can created by passing the generated keys `crypto_key1` and `crypto_key2` secret data 20 | ``` 21 | apiVersion: v1 22 | kind: Secret 23 | metadata: 24 | name: simplyblock-pvc-keys 25 | namespace: default 26 | data: 27 | crypto_key1: N2IzNjk1MjY4ZTJhNjYxMWEyNWFjNGIxZWUxNWYyN2Y5YmY2ZWE5NzgzZGFkYTY2YTRhNzMwZWJmMDQ5MmJmZA== 28 | crypto_key2: Nzg1MDU2MzZjODEzM2Q5YmU0MmUzNDdmODI3ODViODFhODc5Y2Q4MTMzMDQ2ZjhmYzBiMzZmMTdiMDc4YWQwYw== 29 | ``` 30 | Then set the encryption parameter of the storageclass to `True` 31 | ``` 32 | apiVersion: storage.k8s.io/v1 33 | kind: StorageClass 34 | metadata: 35 | name: spdkcsi-sc 36 | provisioner: csi.simplyblock.io 37 | parameters: 38 | ... 39 | encryption: "True" 40 | ``` 41 | Finally set pvc annotation which will dynamically retrieve the secret data `crypto_key1` and `crypto_key2` that is required for encrypted pvc 42 | ``` 43 | kind: PersistentVolumeClaim 44 | apiVersion: v1 45 | metadata: 46 | name: spdkcsi-pvc 47 | annotations: 48 | simplybk/secret-name: simplyblock-pvc-keys 49 | simplybk/secret-namespace: default 50 | spec: 51 | ... 52 | storageClassName: spdkcsi-sc 53 | ``` 54 | -------------------------------------------------------------------------------- /docs/install-simplyblock-csi-driver.md: -------------------------------------------------------------------------------- 1 | ### Prepare SPDK storage node 2 | 3 | Follow [deploy/spdk/README](deploy/spdk/README.md) to deploy SPDK storage service on localhost. 4 | 5 | ### Deploy SPDKCSI services 6 | 7 | 1. Launch Minikube test cluster. 8 | ```bash 9 | $ cd scripts 10 | $ sudo ./minikube.sh up 11 | 12 | # Create kubectl shortcut (assume kubectl version 1.25.0) 13 | $ sudo ln -s /var/lib/minikube/binaries/v1.25.0/kubectl /usr/local/bin/kubectl 14 | 15 | # Wait for Kubernetes ready 16 | $ kubectl get pods --all-namespaces 17 | NAMESPACE NAME READY STATUS RESTARTS AGE 18 | kube-system coredns-6955765f44-dlb88 1/1 Running 0 81s 19 | ...... ...... 20 | kube-system kube-apiserver-spdkcsi-dev 1/1 Running 0 67s 21 | ...... ...... 22 | ``` 23 | 24 | 2. Install snapshot controller and CRD 25 | 26 | ```bash 27 | SNAPSHOT_VERSION="v3.0.3" ./scripts/install-snapshot.sh install 28 | 29 | # Check status 30 | $ kubectl get pod snapshot-controller-0 31 | NAME READY STATUS RESTARTS AGE 32 | snapshot-controller-0 1/1 Running 0 6m14s 33 | ``` 34 | 35 | 3. Deploy SPDK-CSI services 36 | ```bash 37 | $ cd deploy/kubernetes 38 | $ ./deploy.sh 39 | 40 | # Check status 41 | $ kubectl get pods 42 | NAME READY STATUS RESTARTS AGE 43 | spdkcsi-controller-0 3/3 Running 0 3m16s 44 | spdkcsi-node-lzvg5 2/2 Running 0 3m16s 45 | ``` 46 | 47 | 4. Deploy test pod 48 | ```bash 49 | $ cd deploy/kubernetes 50 | $ kubectl apply -f testpod.yaml 51 | 52 | # Check status 53 | $ kubectl get pv 54 | NAME CAPACITY ... STORAGECLASS REASON AGE 55 | persistentvolume/pvc-... 256Mi ... spdkcsi-sc 43s 56 | 57 | $ kubectl get pvc 58 | NAME ... CAPACITY ACCESS MODES STORAGECLASS AGE 59 | persistentvolumeclaim/spdkcsi-pvc ... 256Mi RWO spdkcsi-sc 44s 60 | 61 | $ kubectl get pods 62 | NAME READY STATUS RESTARTS AGE 63 | spdkcsi-test 1/1 Running 0 1m31s 64 | 65 | # Check attached spdk volume in test pod 66 | $ kubectl exec spdkcsi-test mount | grep spdkcsi 67 | /dev/disk/by-id/nvme-..._spdkcsi-sn on /spdkvol type ext4 (rw,relatime) 68 | ``` 69 | 70 | 5. Deploy PVC snapshot 71 | ```bash 72 | # Create snapshot of the bound PVC 73 | $ cd deploy/kubernetes 74 | $ kubectl apply -f testsnapshot.yaml 75 | 76 | # Get details about the snapshot 77 | $ kubectl get volumesnapshot spdk-snapshot 78 | NAME READYTOUSE SOURCEPVC ... SNAPSHOTCLASS AGE 79 | spdk-snapshot false spdkcsi-pvc ... csi-spdk-snapclass 29s 80 | 81 | # Get details about the volumesnapshotcontent 82 | kubectl get volumesnapshotcontent 83 | $ kubectl get volumesnapshotcontent 84 | NAME ... READYTOUSE RESTORESIZE DELETIONPOLICY DRIVER VOLUMESNAPSHOTCLASS VOLUMESNAPSHOT AGE 85 | snapcontent-... true 268435456 Delete csi.simplyblock.io csi-spdk-snapclass spdk-snapshot 29s 86 | ``` 87 | 88 | ### Teardown 89 | 90 | 1. Delete PVC snapshot 91 | ```bash 92 | cd deploy/kubernetes 93 | kubectl delete -f testsnapshot.yaml 94 | ``` 95 | 96 | 2. Delete test pod 97 | ```bash 98 | $ cd deploy/kubernetes 99 | $ kubectl delete -f testpod.yaml 100 | ``` 101 | 102 | 3. Delete SPDK-CSI services 103 | ```bash 104 | $ cd deploy/kubernetes 105 | $ ./deploy.sh teardown 106 | ``` 107 | 108 | 4. Delete snapshot controller and CRD 109 | ```bash 110 | SNAPSHOT_VERSION="v3.0.3" ./scripts/install-snapshot.sh cleanup 111 | ``` 112 | 113 | 5. Teardown Kubernetes test cluster 114 | ```bash 115 | $ cd scripts 116 | $ sudo ./minikube.sh clean 117 | ``` 118 | -------------------------------------------------------------------------------- /docs/static-pvc.md: -------------------------------------------------------------------------------- 1 | # Static PVC with SPDK-CSI 2 | 3 | This document outlines how to create static PV and static PVC from existing SPDK logical volume. 4 | 5 | ## Create static PV 6 | 7 | ### NVMe over Fabrics Target 8 | 9 | Refer to the following [SPDK documents](https://spdk.io/doc/nvmf.html) to create and publish SPDK logical volume manually. 10 | 11 | To create the static PV you need to know the `model`, `nqn`, `lvol`, `targetAddr`, 12 | `targetPort`, and `targetType` name of the SPDK logical volume. 13 | 14 | ```yaml 15 | apiVersion: v1 16 | kind: PersistentVolume 17 | metadata: 18 | annotations: 19 | pv.kubernetes.io/provisioned-by: csi.simplyblock.io 20 | finalizers: 21 | - kubernetes.io/pv-protection 22 | name: pv-static 23 | spec: 24 | accessModes: 25 | - ReadWriteOnce 26 | capacity: 27 | storage: 256Mi 28 | csi: 29 | driver: csi.simplyblock.io 30 | fsType: ext4 31 | volumeAttributes: 32 | # MODEL_NUMBER, set by the `nvmf_create_subsystem` method 33 | model: aa481c21-26f8-4056-87fa-cd306f69a71e 34 | # Subsystem NQN (ASCII), set by the `nvmf_create_subsystem` method 35 | nqn: nqn.2020-04.io.spdk.csi:uuid:aa481c21-26f8-4056-87fa-cd306f69a71e 36 | # The listen address to an NVMe-oF subsystemset, set by the `nvmf_subsystem_add_listener` method 37 | targetAddr: 127.0.0.1 38 | targetPort: "4420" 39 | # transport type, TCP or RDMA 40 | targetType: TCP 41 | # volumeHandle should be same as lvol store name(uuid) 42 | volumeHandle: aa481c21-26f8-4056-87fa-cd306f69a71e 43 | persistentVolumeReclaimPolicy: Retain 44 | storageClassName: spdkcsi-sc 45 | volumeMode: Filesystem 46 | ``` 47 | 48 | ```bash 49 | $ kubectl create -f pv-static.yaml 50 | persistentvolume/pv-static created 51 | ``` 52 | 53 | ### iSCSI Target 54 | 55 | Refer to the following [SPDK documents](https://spdk.io/doc/iscsi.html) to create and publish SPDK logical volume manually. 56 | 57 | To create the static PV you need to know the `lvol`, `targetAddr`and `targetPort` of the SPDK logical volume. 58 | 59 | ```yaml 60 | apiVersion: v1 61 | kind: PersistentVolume 62 | metadata: 63 | annotations: 64 | pv.kubernetes.io/provisioned-by: csi.simplyblock.io 65 | name: pv-static 66 | spec: 67 | accessModes: 68 | - ReadWriteOnce 69 | capacity: 70 | storage: 256Mi 71 | csi: 72 | driver: csi.simplyblock.io 73 | fsType: ext4 74 | volumeAttributes: 75 | # number Initiator group tag, the default value is `iqn.2016-06.io.spdk:`+ `volumeHandle` 76 | iqn: iqn.2016-06.io.spdk:c0cd9559-cd6e-43b6-98af-45196e41655f 77 | # iSCSI transport address, set by the `iscsi_create_portal_group` method 78 | targetAddr: 127.0.0.1 79 | targetPort: "3260" 80 | targetType: iscsi 81 | # volumeHandle should be same as lvol store name(uuid) 82 | volumeHandle: c0cd9559-cd6e-43b6-98af-45196e41655f 83 | persistentVolumeReclaimPolicy: Retain 84 | storageClassName: spdkcsi-sc 85 | volumeMode: Filesystem 86 | ``` 87 | 88 | ```bash 89 | $ kubectl create -f pv-static.yaml 90 | persistentvolume/pv-static created 91 | ``` 92 | 93 | **Note:** SPDK-CSI does not supports logical volume deletion for static PV. 94 | `persistentVolumeReclaimPolicy` in PV spec must be set to `Retain` to avoid PV delete attempt in csi-provisioner. 95 | 96 | ## Create static PVC 97 | 98 | To create the static PVC you need to know the PV name which is created above. 99 | 100 | ```yaml 101 | kind: PersistentVolumeClaim 102 | apiVersion: v1 103 | metadata: 104 | name: pvc-static 105 | spec: 106 | accessModes: 107 | - ReadWriteOnce 108 | resources: 109 | requests: 110 | storage: 256Mi 111 | # As a functional test, volumeName is same as PV name 112 | volumeName: pv-static 113 | storageClassName: spdkcsi-sc 114 | ``` 115 | 116 | ```bash 117 | $ kubectl create -f pvc-static.yaml 118 | persistentvolumeclaim/pvc-static created 119 | ``` 120 | 121 | ## Create test Pod 122 | 123 | ```yaml 124 | kind: Pod 125 | apiVersion: v1 126 | metadata: 127 | name: spdkcsi-test 128 | spec: 129 | containers: 130 | - name: alpine 131 | image: alpine:3 132 | imagePullPolicy: "IfNotPresent" 133 | command: ["sleep", "365d"] 134 | volumeMounts: 135 | - mountPath: "/spdkvol" 136 | name: spdk-volume 137 | volumes: 138 | - name: spdk-volume 139 | persistentVolumeClaim: 140 | claimName: pvc-static 141 | ``` 142 | 143 | ```bash 144 | $ kubectl create -f spdkcsi-test.yaml 145 | pod/spdkcsi-test created 146 | ``` 147 | -------------------------------------------------------------------------------- /docs/storage-nodes.md: -------------------------------------------------------------------------------- 1 | ### Storage nodes volume provisioning 2 | 3 | Apart from a disaggregated storage cluster deployment, storage-plane pods can now also be deployed onto k8s workers and they may-coexist with any compute workload (storage consumers). 4 | Depending on the type of the storage node, it has to come with either at least one locally attached nvme drive or ebs block storage volumes are auto-attached in the during the deployment (aws only). 5 | 6 | ### Preparing nodes 7 | 8 | #### Step 0: Networking & tools 9 | 10 | Make sure that the Kubernetes worker nodes running storage-plane pods have nvme-oF access to each other and - if needed - external storage nodes in the simplyblock cluster. They also need connectivity to/from the simplyblock control plane. If you are using terraform to deploy the cluster. Please attach `container-instance-sg` security group to all the instances. 11 | 12 | #### Step1: Install nvme cli tools and nbd 13 | 14 | To attach NVMe device to the host machine, the CSI driver uses [nvme-cli]([url](https://github.com/linux-nvme/nvme-cli)). So lets install that 15 | ``` 16 | sudo yum install -y nvme-cli 17 | sudo modprobe nvme-tcp 18 | sudo modprobe nbd 19 | ``` 20 | 21 | #### Step1: Setup hugepages 22 | 23 | Simplyblock uses huge page memory. It is necessary to reserve an amount of huge page memory early on. 24 | The simplyblock storage plane pod allocates huge page memory from the reserved pool when the pod is added or restarted. 25 | The amount reserved is based on parameters provided to the storage node add, such as the maximum amount of logical volumes and snapshots and the max. provisioning size of the node (see helm chart parameters). 26 | The minimum amount to reserve is 2 GiB, but try to reserve at least 25% of the node's total RAM. 27 | It is fine to reserve more than needed, as Simplyblock will allocate only the amount required from that pool and the rest can be used by the system. 28 | 29 | >[!IMPORTANT] 30 | >One huge page is 2 MiB. So e.g. a value of 4096 reserves 8 GiB of huge page memory. 31 | 32 | ``` 33 | sudo sysctl -w vm.nr_hugepages=4096 34 | ``` 35 | 36 | confirm the hugepage changes by running 37 | cat /proc/meminfo | grep -i hug 38 | 39 | 40 | and restart kubelet 41 | ``` 42 | sudo systemctl restart kubelet 43 | ``` 44 | 45 | conform if huge pages are added to the cluster or not. 46 | ``` 47 | kubectl describe node ip-10-0-2-184.us-east-2.compute.internal | grep hugepages-2Mi 48 | ``` 49 | this output should show 8GB. This worker node can allocate 8GB of hugepages to pods which is required in case of SPDK pods. 50 | 51 | #### Step2: Mount the SSD or EBS to be used by the storage node 52 | If the instance comes with a default NVMe disk, it can be used with minimum of 2 partitions and 2 device where one is used for Journal manager and the other storage node. Or 2 additional EBS one for Journal Manager and the other for the Storage. the disks can be viewed by running: 53 | 54 | ``` 55 | sudo yum install pciutils 56 | lspci 57 | ``` 58 | 59 | 60 | #### Step3: Tag the kubernetes nodes 61 | 62 | After the nodes are prepared, label the kubernetes nodes 63 | ``` 64 | kubectl label nodes ip-10-0-4-118.us-east-2.compute.internal ip-10-0-4-176.us-east-2.compute.internal type=simplyblock-storage-plane 65 | ``` 66 | Now the nodes are ready to deploy storage nodes. 67 | -------------------------------------------------------------------------------- /docs/support-ports.md: -------------------------------------------------------------------------------- 1 | # Supported Port for eks or ks3 for caching-node 2 | 3 | | Port | Protocol | Description 4 | | -------------- | ------------- | ------------- 5 | | 6443 | TCP | Kubernetes API server. Required for communication between the Kubernetes control plane and the nodes in the cluster. 6 | | 22 | TCP | SSH access to the instances. Necessary for administrative access and management. 7 | | 8080 | TCP | SPDK Proxy for the storage node. Facilitates communication between the storage nodes and the management node. 8 | | 2375 | TCP | Docker Engine API. Allows the management node to communicate with Docker engines running on other nodes. 9 | | - | ICMP | Allows ICMP Echo requests. Used for ping operations to check the availability and responsiveness of management nodes. 10 | | 5000 | TCP | Caching node. Enables communication with caching services running on the node. 11 | 12 | 13 | # Supported Port for eks or ks3 for storage-node 14 | 15 | | Port | Protocol | Description 16 | | -------------- | ------------- | ------------- 17 | | 6443 | TCP | Kubernetes API server. Required for communication between the Kubernetes control plane and the nodes in the cluster. 18 | | 22 | TCP | SSH access to the instances. Necessary for administrative access and management. 19 | | 8080 | TCP | SPDK Proxy for the storage node. Facilitates communication between the storage nodes and the management node. 20 | | 2375 | TCP | Docker Engine API. Allows the management node to communicate with Docker engines running on other nodes. 21 | | - | ICMP | Allows ICMP Echo requests. Used for ping operations to check the availability and responsiveness of management nodes. 22 | | 5000 | TCP | Storage node. Enables communication with storage-node services running on the node. 23 | | 4420 | TCP | Storage node logical volume (lvol) connection - this port must be open (1) btw. all of the workers hosting storage plane pods (2) from all workers with pods connecting to storage to any workers hosting storage plane pods and any external storage nodes. 24 | | 53 | UDP | DNS resolution from worker nodes. Necessary for resolving internal DNS queries within the cluster. 25 | | 10250-10255 | TCP | Kubernetes node communication. Used for kubelet API communication between the nodes. 26 | | 1025-65535 | UDP | Ephemeral ports for UDP traffic. Required for certain network protocols and services. 27 | -------------------------------------------------------------------------------- /e2e/clone.go: -------------------------------------------------------------------------------- 1 | package e2e 2 | 3 | import ( 4 | "time" 5 | 6 | ginkgo "github.com/onsi/ginkgo/v2" 7 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 8 | "k8s.io/kubernetes/test/e2e/framework" 9 | ) 10 | 11 | var _ = ginkgo.Describe("SPDKCSI-CLONE", func() { 12 | f := framework.NewDefaultFramework("spdkcsi") 13 | 14 | ginkgo.Context("Test SPDK CSI Volume Clone", func() { 15 | ginkgo.It("Test SPDK CSI Clone", func() { 16 | testPodLabel := metav1.ListOptions{ 17 | LabelSelector: "app=spdkcsi-pvc", 18 | } 19 | testPodName := "spdkcsi-test" 20 | cloneTestPodName := "spdkcsi-test-clone" 21 | persistData := []string{"Data that needs to be stored"} 22 | persistDataPath := []string{"/spdkvol/test"} 23 | 24 | ginkgo.By("create source pvc and write data", func() { 25 | deployPVC() 26 | deployTestPod() 27 | defer deleteTestPod() 28 | 29 | err := waitForTestPodReady(f.ClientSet, 3*time.Minute, testPodName) 30 | if err != nil { 31 | ginkgo.Fail(err.Error()) 32 | } 33 | writeDataToPod(f, &testPodLabel, persistData[0], persistDataPath[0]) 34 | }) 35 | 36 | ginkgo.By("create clone and check data persistency", func() { 37 | deployClone() 38 | defer deleteClone() 39 | defer deletePVC() 40 | 41 | err := waitForTestPodReady(f.ClientSet, 3*time.Minute, cloneTestPodName) 42 | if err != nil { 43 | ginkgo.Fail(err.Error()) 44 | } 45 | err = compareDataInPod(f, &testPodLabel, persistData, persistDataPath) 46 | if err != nil { 47 | ginkgo.Fail(err.Error()) 48 | } 49 | }) 50 | }) 51 | }) 52 | }) 53 | -------------------------------------------------------------------------------- /e2e/controlplane.go: -------------------------------------------------------------------------------- 1 | package e2e 2 | 3 | import ( 4 | "fmt" 5 | "math/rand" 6 | "time" 7 | 8 | ginko "github.com/onsi/ginkgo/v2" 9 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 10 | "k8s.io/kubernetes/test/e2e/framework" 11 | ) 12 | 13 | const ( 14 | Size5GB = 5 * 1024 * 1024 * 1024 15 | StorageclassName = "spdk-fio-hostid" 16 | configMapname = "fio-config" 17 | pvcName1 = "spdk-fio-pvc1" 18 | podName1 = "spdk-fio-pod1" 19 | pvcName2 = "spdk-fio-pvc2" 20 | podName2 = "spdk-fio-pod2" 21 | ) 22 | 23 | // TODO: use ctx.Before and ctx.After to cleanup the resources 24 | var _ = ginko.Describe("CSI Driver tests", func() { 25 | f := framework.NewDefaultFramework("spdkcsi") 26 | ginko.Context("Control Plane: delete second lvol while the first lvol has IO running", func() { 27 | ginko.It("if a node has lvol with IO running, adding and deleting an new lvol from the same node should work", func() { 28 | ginko.By("run a pod with fio and add and delete an lvol from the same node", func() { 29 | c := f.ClientSet 30 | r := rand.New(rand.NewSource(time.Now().UnixNano())) 31 | n, _ := numberOfNodes(c) 32 | randomValue := r.Intn(n) 33 | sn, _, err := getStorageNode(c, randomValue) 34 | if err != nil { 35 | fmt.Fprintf(ginko.GinkgoWriter, "Error when getStorageNode %s \n", err.Error()) 36 | ginko.Fail(err.Error()) 37 | } 38 | 39 | fmt.Fprintln(ginko.GinkgoWriter, "creating pvc on storage node: ", sn) 40 | err = createstorageClassWithHostID(c, StorageclassName, sn) 41 | if err != nil { 42 | fmt.Fprintf(ginko.GinkgoWriter, "error when creating storage class: %s \n", err.Error()) 43 | } 44 | 45 | err = createFioConfigMap(c, nameSpace, configMapname) 46 | if err != nil { 47 | fmt.Fprintf(ginko.GinkgoWriter, "Error when creating Configmap: %s \n", err.Error()) 48 | } 49 | 50 | fmt.Fprintf(ginko.GinkgoWriter, "creating pvc: %s \n", pvcName1) 51 | err = createPVC(c, nameSpace, pvcName1, StorageclassName, Size5GB) 52 | if err != nil { 53 | ginko.Fail(err.Error()) 54 | } 55 | 56 | fmt.Fprintf(ginko.GinkgoWriter, "creating pod: %s \n", podName1) 57 | err = createFioWorkloadPod(c, nameSpace, podName1, configMapname, pvcName1) 58 | if err != nil { 59 | ginko.Fail(err.Error()) 60 | } 61 | 62 | // delete the pod, pvc, storageclass and configmap at the end 63 | defer func() { 64 | err2 := c.CoreV1().ConfigMaps(nameSpace).Delete(ctx, configMapname, metav1.DeleteOptions{}) 65 | if err2 != nil { 66 | fmt.Fprintln(ginko.GinkgoWriter, "Failed to delete configMap: ", err.Error()) 67 | ginko.Fail(err2.Error()) 68 | } 69 | err2 = c.CoreV1().PersistentVolumeClaims(nameSpace).Delete(ctx, pvcName1, metav1.DeleteOptions{}) 70 | if err2 != nil { 71 | fmt.Fprintln(ginko.GinkgoWriter, "failed to delete PVC", err.Error()) 72 | } 73 | err2 = c.CoreV1().Pods(nameSpace).Delete(ctx, podName1, metav1.DeleteOptions{}) 74 | if err2 != nil { 75 | fmt.Fprintln(ginko.GinkgoWriter, "Failed to delete pod", err.Error()) 76 | } 77 | err2 = c.StorageV1().StorageClasses().Delete(ctx, StorageclassName, metav1.DeleteOptions{}) 78 | if err2 != nil { 79 | fmt.Fprintln(ginko.GinkgoWriter, "Failed to delete storageclass", err.Error()) 80 | } 81 | }() 82 | 83 | fmt.Fprintf(ginko.GinkgoWriter, "creating pvc %s on storage node: %s \n", pvcName2, sn) 84 | err = createPVC(c, nameSpace, pvcName2, StorageclassName, Size5GB) 85 | if err != nil { 86 | ginko.Fail(err.Error()) 87 | } 88 | 89 | fmt.Fprintf(ginko.GinkgoWriter, "creating pod: %s \n", podName2) 90 | err = createSimplePod(c, nameSpace, podName2, pvcName2) 91 | if err != nil { 92 | ginko.Fail(err.Error()) 93 | } 94 | 95 | fmt.Fprintf(ginko.GinkgoWriter, "deleting pod: %s \n", podName2) 96 | err = c.CoreV1().Pods(nameSpace).Delete(ctx, podName2, metav1.DeleteOptions{}) 97 | if err != nil { 98 | ginko.Fail(err.Error()) 99 | } 100 | 101 | fmt.Fprintf(ginko.GinkgoWriter, "deleting pvc: %s \n", pvcName2) 102 | err = c.CoreV1().PersistentVolumeClaims(nameSpace).Delete(ctx, pvcName2, metav1.DeleteOptions{}) 103 | if err != nil { 104 | ginko.Fail(err.Error()) 105 | } 106 | 107 | // delete the pvc and pod at the end 108 | // doing this just in case of any failure 109 | defer func() { 110 | err2 := c.CoreV1().PersistentVolumeClaims(nameSpace).Delete(ctx, pvcName2, metav1.DeleteOptions{}) 111 | if err2 != nil { 112 | fmt.Fprintln(ginko.GinkgoWriter, "failed to delete PVC", err.Error()) 113 | } 114 | err = c.CoreV1().PersistentVolumeClaims(nameSpace).Delete(ctx, pvcName2, metav1.DeleteOptions{}) 115 | if err != nil { 116 | fmt.Fprintln(ginko.GinkgoWriter, "failed to pod", err.Error()) 117 | } 118 | }() 119 | 120 | fmt.Fprintf(ginko.GinkgoWriter, "waiting for 10 secs") 121 | time.Sleep(10 * time.Second) 122 | // fio should not stop on 123 | pod, err := c.CoreV1().Pods(nameSpace).Get(ctx, podName1, metav1.GetOptions{}) 124 | if err != nil { 125 | ginko.Fail(err.Error()) 126 | } 127 | 128 | if pod.Status.Phase != "Running" { 129 | ginko.Fail("pod is not running") 130 | } else { 131 | fmt.Fprintf(ginko.GinkgoWriter, "pod is still running") 132 | } 133 | }) 134 | }) 135 | }) 136 | }) 137 | -------------------------------------------------------------------------------- /e2e/e2e.go: -------------------------------------------------------------------------------- 1 | package e2e 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "os" 7 | "path/filepath" 8 | "testing" 9 | 10 | ginkgo "github.com/onsi/ginkgo/v2" 11 | "k8s.io/klog" 12 | "k8s.io/kubernetes/test/e2e/framework" 13 | "k8s.io/kubernetes/test/e2e/framework/config" 14 | "k8s.io/kubernetes/test/e2e/storage/podlogs" 15 | ) 16 | 17 | func init() { 18 | klog.SetOutput(ginkgo.GinkgoWriter) 19 | 20 | if os.Getenv("KUBECONFIG") == "" { 21 | kubeConfigPath := filepath.Join(os.Getenv("HOME"), ".kube", "config") 22 | os.Setenv("KUBECONFIG", kubeConfigPath) 23 | } 24 | 25 | config.CopyFlags(config.Flags, flag.CommandLine) 26 | framework.RegisterCommonFlags(flag.CommandLine) 27 | framework.RegisterClusterFlags(flag.CommandLine) 28 | 29 | testing.Init() 30 | flag.Parse() 31 | framework.AfterReadingAllFlags(&framework.TestContext) 32 | } 33 | 34 | var _ = ginkgo.SynchronizedBeforeSuite(func() []byte { 35 | ginkgo.By("Watching for pod logs in 'default' namespace") 36 | cs, err := framework.LoadClientset() 37 | framework.ExpectNoError(err, "create client set") 38 | err = podlogs.CopyAllLogs(context.Background(), cs, nameSpace, podlogs.LogOutput{ 39 | LogWriter: ginkgo.GinkgoWriter, 40 | StatusWriter: ginkgo.GinkgoWriter, 41 | }) 42 | framework.ExpectNoError(err, "set watch on namespace :%s", nameSpace) 43 | 44 | return []byte{} 45 | }, func(_ []byte) {}) 46 | 47 | var _ = ginkgo.SynchronizedAfterSuite(func() { 48 | framework.RunCleanupActions() 49 | }, func() {}) 50 | -------------------------------------------------------------------------------- /e2e/e2e_test.go: -------------------------------------------------------------------------------- 1 | package e2e 2 | 3 | import ( 4 | "testing" 5 | 6 | ginkgo "github.com/onsi/ginkgo/v2" 7 | "github.com/onsi/gomega" 8 | ) 9 | 10 | func TestE2E(t *testing.T) { 11 | gomega.RegisterFailHandler(ginkgo.Fail) 12 | ginkgo.RunSpecs(t, "SPDKCSI E2E suite") 13 | } 14 | -------------------------------------------------------------------------------- /e2e/nvmeof.go: -------------------------------------------------------------------------------- 1 | package e2e 2 | 3 | import ( 4 | "time" 5 | 6 | ginkgo "github.com/onsi/ginkgo/v2" 7 | "k8s.io/kubernetes/test/e2e/framework" 8 | ) 9 | 10 | var _ = ginkgo.Describe("SPDKCSI-NVMEOF", func() { 11 | f := framework.NewDefaultFramework("spdkcsi") 12 | 13 | ginkgo.Context("Test SPDK CSI Dynamic Volume Provisioning", func() { 14 | ginkgo.It("CSI driver components should function properly", func() { 15 | ginkgo.By("checking controller statefulset is running", func() { 16 | err := waitForControllerReady(f.ClientSet, 4*time.Minute) 17 | if err != nil { 18 | ginkgo.Fail(err.Error()) 19 | } 20 | }) 21 | 22 | ginkgo.By("checking node daemonset is running", func() { 23 | err := waitForNodeServerReady(f.ClientSet, 3*time.Minute) 24 | if err != nil { 25 | ginkgo.Fail(err.Error()) 26 | } 27 | }) 28 | }) 29 | 30 | ginkgo.It("Test the flow for Dynamic volume provisioning", func() { 31 | testPodName := "spdkcsi-test" 32 | ginkgo.By("creating a PVC and verify dynamic PV", func() { 33 | deployPVC() 34 | defer deletePVC() 35 | err := verifyDynamicPVCreation(f.ClientSet, "spdkcsi-pvc", 5*time.Minute) 36 | if err != nil { 37 | ginkgo.Fail(err.Error()) 38 | } 39 | }) 40 | 41 | ginkgo.By("creating a PVC and binding it to a pod", func() { 42 | deployPVC() 43 | deployTestPod() 44 | defer deletePVCAndTestPod() 45 | err := waitForTestPodReady(f.ClientSet, 3*time.Minute, testPodName) 46 | if err != nil { 47 | ginkgo.Fail(err.Error()) 48 | } 49 | }) 50 | }) 51 | 52 | ginkgo.It("Test multiple PVCs", func() { 53 | multiTestPodName := "spdkcsi-test-multi" 54 | ginkgo.By("create multiple pvcs and a pod with multiple pvcs attached, and check data persistence after the pod is removed and recreated", func() { 55 | deployMultiPvcs() 56 | deployTestPodWithMultiPvcs() 57 | defer func() { 58 | deleteMultiPvcsAndTestPodWithMultiPvcs() 59 | if err := waitForTestPodGone(f.ClientSet, multiTestPodName); err != nil { 60 | ginkgo.Fail(err.Error()) 61 | } 62 | for _, pvcName := range []string{"spdkcsi-pvc1", "spdkcsi-pvc2", "spdkcsi-pvc3"} { 63 | if err := waitForPvcGone(f.ClientSet, pvcName); err != nil { 64 | ginkgo.Fail(err.Error()) 65 | } 66 | } 67 | }() 68 | err := waitForTestPodReady(f.ClientSet, 3*time.Minute, multiTestPodName) 69 | if err != nil { 70 | ginkgo.Fail(err.Error()) 71 | } 72 | 73 | err = checkDataPersistForMultiPvcs(f) 74 | if err != nil { 75 | ginkgo.Fail(err.Error()) 76 | } 77 | }) 78 | }) 79 | }) 80 | }) 81 | 82 | var _ = ginkgo.Describe("CACHE-NODE-TEST", func() { 83 | f := framework.NewDefaultFramework("spdkcsi") 84 | 85 | ginkgo.Context("Test caching node SPDK CSI Dynamic Volume Provisioning", func() { 86 | 87 | ginkgo.It("Test the flow for Caching nodes", func() { 88 | testPodName := "spdkcsi-test" 89 | ginkgo.By("creating a caching PVC and bind it to a pod", func() { 90 | deployCachePVC() 91 | deployCacheTestPod() 92 | defer deleteCachePVCAndCacheTestPod() 93 | err := waitForCacheTestPodReady(f.ClientSet, 3*time.Minute) 94 | if err != nil { 95 | ginkgo.Fail(err.Error()) 96 | } 97 | }) 98 | 99 | ginkgo.By("check data persistency after the pod is removed and recreated", func() { 100 | deployPVC() 101 | deployTestPod() 102 | defer deletePVCAndTestPod() 103 | 104 | err := waitForTestPodReady(f.ClientSet, 3*time.Minute, testPodName) 105 | if err != nil { 106 | ginkgo.Fail(err.Error()) 107 | } 108 | 109 | err = checkDataPersist(f) 110 | if err != nil { 111 | ginkgo.Fail(err.Error()) 112 | } 113 | }) 114 | }) 115 | }) 116 | }) 117 | -------------------------------------------------------------------------------- /e2e/restartnode.go: -------------------------------------------------------------------------------- 1 | package e2e 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "os" 7 | "time" 8 | 9 | "math/rand" 10 | 11 | ginkgo "github.com/onsi/ginkgo/v2" 12 | "gopkg.in/yaml.v3" 13 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 14 | "k8s.io/kubernetes/test/e2e/framework" 15 | ) 16 | 17 | type storageNode struct { 18 | UUID string `json:"uuid"` 19 | NodeIP string `json:"node_ip"` 20 | } 21 | 22 | var _ = ginkgo.Describe("SPDKCSI-NodeRestart", func() { 23 | f := framework.NewDefaultFramework("spdkcsi") 24 | 25 | ginkgo.Context("Test SPDK CSI node restart", func() { 26 | ginkgo.It("Test SPDK CSI node restart", func() { 27 | testPodLabel := metav1.ListOptions{ 28 | LabelSelector: "app=spdkcsi-pvc", 29 | } 30 | 31 | testPodName := "spdkcsi-test" 32 | snap1TestPodName := "spdkcsi-test-snapshot1" 33 | persistData := []string{"Data that needs to be stored"} 34 | persistDataPath := []string{"/spdkvol/test"} 35 | c := f.ClientSet 36 | n, err := numberOfNodes(c) 37 | if err != nil { 38 | ginkgo.Fail(err.Error()) 39 | } 40 | r := rand.New(rand.NewSource(time.Now().UnixNano())) 41 | randomValue := r.Intn(n) 42 | 43 | ginkgo.By("check pvc write, clone, snapshot before node restart", func() { 44 | 45 | _, storageNodeID, _ := getStorageNode(c, randomValue) 46 | 47 | data, _ := os.ReadFile(pvcPath) 48 | var yamlContent map[string]interface{} 49 | err := yaml.Unmarshal(data, &yamlContent) 50 | if err != nil { 51 | ginkgo.Fail(err.Error()) 52 | } 53 | 54 | annotations, _ := yamlContent["metadata"].(map[string]interface{})["annotations"].(map[string]interface{}) 55 | annotations["simplybk/host-id"] = storageNodeID 56 | updatedData, err := yaml.Marshal(yamlContent) 57 | if err != nil { 58 | ginkgo.Fail(err.Error()) 59 | } 60 | 61 | err = os.WriteFile(pvcPath, updatedData, 0644) 62 | if err != nil { 63 | ginkgo.Fail(err.Error()) 64 | } 65 | //write data to pvc and check 66 | deployPVC() 67 | deployTestPod() 68 | 69 | err = waitForTestPodReady(f.ClientSet, 3*time.Minute, testPodName) 70 | if err != nil { 71 | ginkgo.Fail(err.Error()) 72 | } 73 | writeDataToPod(f, &testPodLabel, persistData[0], persistDataPath[0]) 74 | 75 | deleteTestPod() 76 | 77 | // snapshot and check 78 | deploySnapshot() 79 | 80 | err = waitForTestPodReady(f.ClientSet, 3*time.Minute, snap1TestPodName) 81 | if err != nil { 82 | ginkgo.Fail(err.Error()) 83 | } 84 | err = compareDataInPod(f, &testPodLabel, persistData, persistDataPath) 85 | if err != nil { 86 | ginkgo.Fail(err.Error()) 87 | } 88 | 89 | // clone and check 90 | deployClone() 91 | 92 | err = waitForTestPodReady(f.ClientSet, 3*time.Minute, "spdkcsi-test-clone") 93 | if err != nil { 94 | ginkgo.Fail(err.Error()) 95 | } 96 | err = compareDataInPod(f, &testPodLabel, persistData, persistDataPath) 97 | if err != nil { 98 | ginkgo.Fail(err.Error()) 99 | } 100 | 101 | }) 102 | 103 | ginkgo.By("restarting the storage node", func() { 104 | pvc, _ := c.CoreV1().PersistentVolumeClaims(nameSpace).Get(context.TODO(), "spdkcsi-pvc", metav1.GetOptions{}) 105 | storageNodeID := pvc.Annotations["simplybk/host-id"] 106 | fmt.Println() 107 | err := restartStorageNode(c, storageNodeID) 108 | if err != nil { 109 | ginkgo.Fail(err.Error()) 110 | } 111 | deleteClone() 112 | deleteSnapshot() 113 | deletePVC() 114 | 115 | }) 116 | 117 | ginkgo.By("check pvc write, clone, snapshot after node restart", func() { 118 | deployPVC() 119 | deployTestPod() 120 | 121 | err := waitForTestPodReady(f.ClientSet, 3*time.Minute, testPodName) 122 | if err != nil { 123 | ginkgo.Fail(err.Error()) 124 | } 125 | writeDataToPod(f, &testPodLabel, persistData[0], persistDataPath[0]) 126 | 127 | deleteTestPod() 128 | 129 | // check snapshot 130 | deploySnapshot() 131 | 132 | err = waitForTestPodReady(f.ClientSet, 3*time.Minute, snap1TestPodName) 133 | if err != nil { 134 | ginkgo.Fail(err.Error()) 135 | } 136 | err = compareDataInPod(f, &testPodLabel, persistData, persistDataPath) 137 | if err != nil { 138 | ginkgo.Fail(err.Error()) 139 | } 140 | 141 | deleteSnapshot() 142 | 143 | //check clone 144 | 145 | deployClone() 146 | 147 | err = waitForTestPodReady(f.ClientSet, 3*time.Minute, "spdkcsi-test-clone") 148 | if err != nil { 149 | ginkgo.Fail(err.Error()) 150 | } 151 | err = compareDataInPod(f, &testPodLabel, persistData, persistDataPath) 152 | if err != nil { 153 | ginkgo.Fail(err.Error()) 154 | } 155 | 156 | deleteClone() 157 | deletePVC() 158 | }) 159 | }) 160 | }) 161 | }) 162 | -------------------------------------------------------------------------------- /e2e/snapshot.go: -------------------------------------------------------------------------------- 1 | package e2e 2 | 3 | import ( 4 | "time" 5 | 6 | ginkgo "github.com/onsi/ginkgo/v2" 7 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 8 | "k8s.io/kubernetes/test/e2e/framework" 9 | ) 10 | 11 | var _ = ginkgo.Describe("SPDKCSI-SNAPSHOT", func() { 12 | f := framework.NewDefaultFramework("spdkcsi") 13 | 14 | ginkgo.Context("Test SPDK CSI Snapshot", func() { 15 | ginkgo.It("Test SPDK CSI Snapshot", func() { 16 | testPodLabel := metav1.ListOptions{ 17 | LabelSelector: "app=spdkcsi-pvc", 18 | } 19 | testPodName := "spdkcsi-test" 20 | snap1TestPodName := "spdkcsi-test-snapshot1" 21 | snap2TestPodName := "spdkcsi-test-snapshot2" 22 | persistData := []string{"Data that needs to be stored"} 23 | persistDataPath := []string{"/spdkvol/test"} 24 | persistData2 := []string{"Data that needs to be stored", "Second data that needs to be stored"} 25 | persistDataPath2 := []string{"/spdkvol/test", "/spdkvol/test2"} 26 | 27 | ginkgo.By("create source pvc and write data", func() { 28 | deployPVC() 29 | deployTestPod() 30 | defer deleteTestPod() 31 | // do not delete pvc here, since we need it for snapshot 32 | 33 | err := waitForTestPodReady(f.ClientSet, 3*time.Minute, testPodName) 34 | if err != nil { 35 | ginkgo.Fail(err.Error()) 36 | } 37 | // write data to source pvc 38 | writeDataToPod(f, &testPodLabel, persistData[0], persistDataPath[0]) 39 | }) 40 | 41 | ginkgo.By("create snapshot1 and check data persistency", func() { 42 | deploySnapshot() 43 | defer deleteSnapshot() 44 | 45 | err := waitForTestPodReady(f.ClientSet, 3*time.Minute, snap1TestPodName) 46 | if err != nil { 47 | ginkgo.Fail(err.Error()) 48 | } 49 | err = compareDataInPod(f, &testPodLabel, persistData, persistDataPath) 50 | if err != nil { 51 | ginkgo.Fail(err.Error()) 52 | } 53 | }) 54 | 55 | ginkgo.By("write second data to the same PVC", func() { 56 | deployTestPod() 57 | defer deleteTestPod() 58 | 59 | err := waitForTestPodReady(f.ClientSet, 3*time.Minute, testPodName) 60 | if err != nil { 61 | ginkgo.Fail(err.Error()) 62 | } 63 | // write second data to source pvc 64 | writeDataToPod(f, &testPodLabel, persistData2[1], persistDataPath2[1]) 65 | }) 66 | 67 | ginkgo.By("create snapshot2 and check second data persistency", func() { 68 | deploySnapshot2() 69 | defer deleteSnapshot2() 70 | defer deletePVC() 71 | 72 | err := waitForTestPodReady(f.ClientSet, 3*time.Minute, snap2TestPodName) 73 | if err != nil { 74 | ginkgo.Fail(err.Error()) 75 | } 76 | err = compareDataInPod(f, &testPodLabel, persistData2, persistDataPath2) 77 | if err != nil { 78 | ginkgo.Fail(err.Error()) 79 | } 80 | }) 81 | }) 82 | }) 83 | }) 84 | -------------------------------------------------------------------------------- /e2e/templates/multi-pvc.yaml: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | # Copyright (c) Arm Limited and Contributors 3 | # Copyright (c) Intel Corporation 4 | --- 5 | apiVersion: v1 6 | kind: PersistentVolumeClaim 7 | metadata: 8 | name: spdkcsi-pvc1 9 | annotations: 10 | simplybk/secret-name: simplyblock-pvc-keys 11 | simplybk/secret-namespace: spdk-csi 12 | spec: 13 | accessModes: 14 | - ReadWriteOnce 15 | resources: 16 | requests: 17 | storage: 128Mi 18 | storageClassName: simplyblock-csi-sc 19 | 20 | --- 21 | apiVersion: v1 22 | kind: PersistentVolumeClaim 23 | metadata: 24 | name: spdkcsi-pvc2 25 | annotations: 26 | simplybk/secret-name: simplyblock-pvc-keys 27 | simplybk/secret-namespace: spdk-csi 28 | spec: 29 | accessModes: 30 | - ReadWriteOnce 31 | resources: 32 | requests: 33 | storage: 128Mi 34 | storageClassName: simplyblock-csi-sc 35 | 36 | --- 37 | apiVersion: v1 38 | kind: PersistentVolumeClaim 39 | metadata: 40 | name: spdkcsi-pvc3 41 | annotations: 42 | simplybk/secret-name: simplyblock-pvc-keys 43 | simplybk/secret-namespace: spdk-csi 44 | spec: 45 | accessModes: 46 | - ReadWriteOnce 47 | resources: 48 | requests: 49 | storage: 128Mi 50 | storageClassName: simplyblock-csi-sc 51 | -------------------------------------------------------------------------------- /e2e/templates/pvc-cache.yaml: -------------------------------------------------------------------------------- 1 | kind: PersistentVolumeClaim 2 | apiVersion: v1 3 | metadata: 4 | name: spdkcsi-cache-pvc 5 | annotations: 6 | simplybk/secret-name: simplyblock-pvc-keys 7 | simplybk/secret-namespace: spdk-csi 8 | spec: 9 | accessModes: 10 | - ReadWriteOnce 11 | resources: 12 | requests: 13 | storage: 256Mi 14 | storageClassName: simplyblock-csi-sc-cache 15 | -------------------------------------------------------------------------------- /e2e/templates/pvc.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: PersistentVolumeClaim 3 | metadata: 4 | annotations: 5 | simplybk/secret-name: simplyblock-pvc-keys 6 | simplybk/secret-namespace: spdk-csi 7 | name: spdkcsi-pvc 8 | spec: 9 | accessModes: 10 | - ReadWriteOnce 11 | resources: 12 | requests: 13 | storage: 256Mi 14 | storageClassName: simplyblock-csi-sc 15 | -------------------------------------------------------------------------------- /e2e/templates/testpod-cache.yaml: -------------------------------------------------------------------------------- 1 | kind: Pod 2 | apiVersion: v1 3 | metadata: 4 | name: spdkcsi-cache-test 5 | spec: 6 | nodeSelector: 7 | type: simplyblock-cache 8 | containers: 9 | - name: alpine 10 | image: alpine:3 11 | imagePullPolicy: "IfNotPresent" 12 | command: ["sleep", "365d"] 13 | volumeMounts: 14 | - mountPath: "/spdkvol" 15 | name: spdk-volume 16 | volumes: 17 | - name: spdk-volume 18 | persistentVolumeClaim: 19 | claimName: spdkcsi-cache-pvc 20 | -------------------------------------------------------------------------------- /e2e/templates/testpod-clone.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: PersistentVolumeClaim 4 | metadata: 5 | name: spdkcsi-pvc-clone 6 | spec: 7 | storageClassName: simplyblock-csi-sc 8 | dataSource: 9 | name: spdkcsi-pvc 10 | kind: PersistentVolumeClaim 11 | apiGroup: "" 12 | accessModes: 13 | - ReadWriteOnce 14 | resources: 15 | requests: 16 | storage: 256Mi 17 | --- 18 | kind: Pod 19 | apiVersion: v1 20 | metadata: 21 | name: spdkcsi-test-clone 22 | labels: 23 | app: spdkcsi-pvc 24 | spec: 25 | containers: 26 | - name: alpine 27 | image: alpine:3 28 | imagePullPolicy: "IfNotPresent" 29 | command: ["sleep", "365d"] 30 | volumeMounts: 31 | - mountPath: "/spdkvol" 32 | name: spdk-volume 33 | volumes: 34 | - name: spdk-volume 35 | persistentVolumeClaim: 36 | claimName: spdkcsi-pvc-clone 37 | -------------------------------------------------------------------------------- /e2e/templates/testpod-multi-pvc.yaml: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | # Copyright (c) Arm Limited and Contributors 3 | # Copyright (c) Intel Corporation 4 | --- 5 | apiVersion: v1 6 | kind: Pod 7 | metadata: 8 | name: spdkcsi-test-multi 9 | labels: 10 | app: spdkcsi-pvc 11 | spec: 12 | containers: 13 | - name: alpine 14 | image: alpine:3 15 | imagePullPolicy: "IfNotPresent" 16 | command: ["sleep", "365d"] 17 | volumeMounts: 18 | - mountPath: "/spdkvol1" 19 | name: spdk-volume1 20 | - mountPath: "/spdkvol2" 21 | name: spdk-volume2 22 | - mountPath: "/spdkvol3" 23 | name: spdk-volume3 24 | volumes: 25 | - name: spdk-volume1 26 | persistentVolumeClaim: 27 | claimName: spdkcsi-pvc1 28 | - name: spdk-volume2 29 | persistentVolumeClaim: 30 | claimName: spdkcsi-pvc2 31 | - name: spdk-volume3 32 | persistentVolumeClaim: 33 | claimName: spdkcsi-pvc3 34 | -------------------------------------------------------------------------------- /e2e/templates/testpod-snapshot.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: snapshot.storage.k8s.io/v1 3 | kind: VolumeSnapshot 4 | metadata: 5 | name: spdk-snapshot 6 | spec: 7 | volumeSnapshotClassName: simplyblock-csi-snapshotclass 8 | source: 9 | persistentVolumeClaimName: spdkcsi-pvc 10 | 11 | --- 12 | apiVersion: v1 13 | kind: PersistentVolumeClaim 14 | metadata: 15 | name: spdkcsi-pvc-restore 16 | spec: 17 | storageClassName: simplyblock-csi-sc 18 | dataSource: 19 | name: spdk-snapshot 20 | kind: VolumeSnapshot 21 | apiGroup: snapshot.storage.k8s.io 22 | accessModes: 23 | - ReadWriteOnce 24 | resources: 25 | requests: 26 | storage: 256Mi 27 | --- 28 | kind: Pod 29 | apiVersion: v1 30 | metadata: 31 | name: spdkcsi-test-snapshot1 32 | labels: 33 | app: spdkcsi-pvc 34 | spec: 35 | containers: 36 | - name: alpine 37 | image: alpine:3 38 | imagePullPolicy: "IfNotPresent" 39 | command: ["sleep", "365d"] 40 | volumeMounts: 41 | - mountPath: "/spdkvol" 42 | name: spdk-volume 43 | volumes: 44 | - name: spdk-volume 45 | persistentVolumeClaim: 46 | claimName: spdkcsi-pvc-restore 47 | -------------------------------------------------------------------------------- /e2e/templates/testpod-snapshot2.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: snapshot.storage.k8s.io/v1 3 | kind: VolumeSnapshot 4 | metadata: 5 | name: spdk-snapshot2 6 | spec: 7 | volumeSnapshotClassName: simplyblock-csi-snapshotclass 8 | source: 9 | persistentVolumeClaimName: spdkcsi-pvc 10 | 11 | --- 12 | apiVersion: v1 13 | kind: PersistentVolumeClaim 14 | metadata: 15 | name: spdkcsi-pvc-restore2 16 | spec: 17 | storageClassName: simplyblock-csi-sc 18 | dataSource: 19 | name: spdk-snapshot2 20 | kind: VolumeSnapshot 21 | apiGroup: snapshot.storage.k8s.io 22 | accessModes: 23 | - ReadWriteOnce 24 | resources: 25 | requests: 26 | storage: 256Mi 27 | --- 28 | kind: Pod 29 | apiVersion: v1 30 | metadata: 31 | name: spdkcsi-test-snapshot2 32 | labels: 33 | app: spdkcsi-pvc 34 | spec: 35 | containers: 36 | - name: alpine 37 | image: alpine:3 38 | imagePullPolicy: "IfNotPresent" 39 | command: ["sleep", "365d"] 40 | volumeMounts: 41 | - mountPath: "/spdkvol" 42 | name: spdk-volume 43 | volumes: 44 | - name: spdk-volume 45 | persistentVolumeClaim: 46 | claimName: spdkcsi-pvc-restore2 47 | -------------------------------------------------------------------------------- /e2e/templates/testpod.yaml: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | # Copyright (c) Arm Limited and Contributors 3 | # Copyright (c) Intel Corporation 4 | --- 5 | apiVersion: v1 6 | kind: Pod 7 | metadata: 8 | name: spdkcsi-test 9 | labels: 10 | app: spdkcsi-pvc 11 | spec: 12 | containers: 13 | - name: alpine 14 | image: alpine:3 15 | imagePullPolicy: "IfNotPresent" 16 | command: ["sleep", "365d"] 17 | volumeMounts: 18 | - mountPath: "/spdkvol" 19 | name: spdk-volume 20 | volumes: 21 | - name: spdk-volume 22 | persistentVolumeClaim: 23 | claimName: spdkcsi-pvc 24 | -------------------------------------------------------------------------------- /pkg/csi-common/controllerserver-default.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | //nolint:revive // csi spec 18 | package csicommon 19 | 20 | import ( 21 | "context" 22 | 23 | "github.com/container-storage-interface/spec/lib/go/csi" 24 | "google.golang.org/grpc/codes" 25 | "google.golang.org/grpc/status" 26 | "k8s.io/klog" 27 | ) 28 | 29 | type DefaultControllerServer struct { 30 | Driver *CSIDriver 31 | } 32 | 33 | func (cs *DefaultControllerServer) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error) { 34 | return nil, status.Error(codes.Unimplemented, "") 35 | } 36 | 37 | func (cs *DefaultControllerServer) DeleteVolume(ctx context.Context, req *csi.DeleteVolumeRequest) (*csi.DeleteVolumeResponse, error) { 38 | return nil, status.Error(codes.Unimplemented, "") 39 | } 40 | 41 | func (cs *DefaultControllerServer) ControllerPublishVolume(ctx context.Context, req *csi.ControllerPublishVolumeRequest) (*csi.ControllerPublishVolumeResponse, error) { 42 | return nil, status.Error(codes.Unimplemented, "") 43 | } 44 | 45 | func (cs *DefaultControllerServer) ControllerUnpublishVolume(ctx context.Context, req *csi.ControllerUnpublishVolumeRequest) (*csi.ControllerUnpublishVolumeResponse, error) { 46 | return nil, status.Error(codes.Unimplemented, "") 47 | } 48 | 49 | func (cs *DefaultControllerServer) ValidateVolumeCapabilities(ctx context.Context, req *csi.ValidateVolumeCapabilitiesRequest) (*csi.ValidateVolumeCapabilitiesResponse, error) { 50 | return nil, status.Error(codes.Unimplemented, "") 51 | } 52 | 53 | func (cs *DefaultControllerServer) ListVolumes(ctx context.Context, req *csi.ListVolumesRequest) (*csi.ListVolumesResponse, error) { 54 | return nil, status.Error(codes.Unimplemented, "") 55 | } 56 | 57 | func (cs *DefaultControllerServer) GetCapacity(ctx context.Context, req *csi.GetCapacityRequest) (*csi.GetCapacityResponse, error) { 58 | return nil, status.Error(codes.Unimplemented, "") 59 | } 60 | 61 | func (cs *DefaultControllerServer) ControllerGetVolume(context.Context, *csi.ControllerGetVolumeRequest) (*csi.ControllerGetVolumeResponse, error) { 62 | return nil, status.Error(codes.Unimplemented, "") 63 | } 64 | 65 | // Default supports all capabilities 66 | func (cs *DefaultControllerServer) ControllerGetCapabilities(ctx context.Context, req *csi.ControllerGetCapabilitiesRequest) (*csi.ControllerGetCapabilitiesResponse, error) { 67 | klog.V(5).Infof("Using default ControllerGetCapabilities") 68 | 69 | return &csi.ControllerGetCapabilitiesResponse{ 70 | Capabilities: cs.Driver.cap, 71 | }, nil 72 | } 73 | 74 | func (cs *DefaultControllerServer) CreateSnapshot(ctx context.Context, req *csi.CreateSnapshotRequest) (*csi.CreateSnapshotResponse, error) { 75 | return nil, status.Error(codes.Unimplemented, "") 76 | } 77 | 78 | func (cs *DefaultControllerServer) DeleteSnapshot(ctx context.Context, req *csi.DeleteSnapshotRequest) (*csi.DeleteSnapshotResponse, error) { 79 | return nil, status.Error(codes.Unimplemented, "") 80 | } 81 | 82 | func (cs *DefaultControllerServer) ListSnapshots(ctx context.Context, req *csi.ListSnapshotsRequest) (*csi.ListSnapshotsResponse, error) { 83 | return nil, status.Error(codes.Unimplemented, "") 84 | } 85 | 86 | func (cs *DefaultControllerServer) ControllerExpandVolume(ctx context.Context, req *csi.ControllerExpandVolumeRequest) (*csi.ControllerExpandVolumeResponse, error) { 87 | return nil, status.Error(codes.Unimplemented, "") 88 | } 89 | -------------------------------------------------------------------------------- /pkg/csi-common/driver.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package csicommon 18 | 19 | import ( 20 | "github.com/container-storage-interface/spec/lib/go/csi" 21 | "google.golang.org/grpc/codes" 22 | "google.golang.org/grpc/status" 23 | "k8s.io/klog" 24 | ) 25 | 26 | type CSIDriver struct { 27 | name string 28 | nodeID string 29 | version string 30 | cap []*csi.ControllerServiceCapability 31 | vc []*csi.VolumeCapability_AccessMode 32 | } 33 | 34 | // NewCSIDriver creates a driver object. Assumes vendor version is equal to driver version & 35 | // does not support optional driver plugin info manifest field. Refer to CSI spec for more details. 36 | func NewCSIDriver(name, v, nodeID string) *CSIDriver { 37 | if name == "" { 38 | klog.Errorf("Driver name missing") 39 | return nil 40 | } 41 | 42 | if nodeID == "" { 43 | klog.Errorf("NodeID missing") 44 | return nil 45 | } 46 | if v == "" { 47 | klog.Errorf("Version argument missing") 48 | return nil 49 | } 50 | 51 | driver := CSIDriver{ 52 | name: name, 53 | version: v, 54 | nodeID: nodeID, 55 | } 56 | 57 | return &driver 58 | } 59 | 60 | func (d *CSIDriver) ValidateControllerServiceRequest(c csi.ControllerServiceCapability_RPC_Type) error { 61 | if c == csi.ControllerServiceCapability_RPC_UNKNOWN { 62 | return nil 63 | } 64 | 65 | for _, cap := range d.cap { 66 | if c == cap.GetRpc().GetType() { 67 | return nil 68 | } 69 | } 70 | return status.Error(codes.InvalidArgument, c.String()) 71 | } 72 | 73 | func (d *CSIDriver) AddControllerServiceCapabilities(cl []csi.ControllerServiceCapability_RPC_Type) { 74 | var csc []*csi.ControllerServiceCapability 75 | 76 | for _, c := range cl { 77 | klog.Infof("Enabling controller service capability: %v", c.String()) 78 | csc = append(csc, NewControllerServiceCapability(c)) 79 | } 80 | 81 | d.cap = csc 82 | } 83 | 84 | func (d *CSIDriver) AddVolumeCapabilityAccessModes(vc []csi.VolumeCapability_AccessMode_Mode) []*csi.VolumeCapability_AccessMode { 85 | var vca []*csi.VolumeCapability_AccessMode 86 | for _, c := range vc { 87 | klog.Infof("Enabling volume access mode: %v", c.String()) 88 | vca = append(vca, NewVolumeCapabilityAccessMode(c)) 89 | } 90 | d.vc = vca 91 | return vca 92 | } 93 | 94 | func (d *CSIDriver) GetVolumeCapabilityAccessModes() []*csi.VolumeCapability_AccessMode { 95 | return d.vc 96 | } 97 | -------------------------------------------------------------------------------- /pkg/csi-common/identityserver-default.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | //nolint:revive // csi spec 18 | package csicommon 19 | 20 | import ( 21 | "context" 22 | 23 | "github.com/container-storage-interface/spec/lib/go/csi" 24 | "google.golang.org/grpc/codes" 25 | "google.golang.org/grpc/status" 26 | "k8s.io/klog" 27 | ) 28 | 29 | type DefaultIdentityServer struct { 30 | Driver *CSIDriver 31 | } 32 | 33 | func (ids *DefaultIdentityServer) GetPluginInfo(ctx context.Context, req *csi.GetPluginInfoRequest) (*csi.GetPluginInfoResponse, error) { 34 | klog.V(5).Infof("Using default GetPluginInfo") 35 | 36 | if ids.Driver.name == "" { 37 | return nil, status.Error(codes.Unavailable, "Driver name not configured") 38 | } 39 | 40 | if ids.Driver.version == "" { 41 | return nil, status.Error(codes.Unavailable, "Driver is missing version") 42 | } 43 | 44 | return &csi.GetPluginInfoResponse{ 45 | Name: ids.Driver.name, 46 | VendorVersion: ids.Driver.version, 47 | }, nil 48 | } 49 | 50 | func (ids *DefaultIdentityServer) Probe(ctx context.Context, req *csi.ProbeRequest) (*csi.ProbeResponse, error) { 51 | return &csi.ProbeResponse{}, nil 52 | } 53 | 54 | func (ids *DefaultIdentityServer) GetPluginCapabilities(ctx context.Context, req *csi.GetPluginCapabilitiesRequest) (*csi.GetPluginCapabilitiesResponse, error) { 55 | klog.V(5).Infof("Using default capabilities") 56 | return &csi.GetPluginCapabilitiesResponse{ 57 | Capabilities: []*csi.PluginCapability{ 58 | { 59 | Type: &csi.PluginCapability_Service_{ 60 | Service: &csi.PluginCapability_Service{ 61 | Type: csi.PluginCapability_Service_CONTROLLER_SERVICE, 62 | }, 63 | }, 64 | }, 65 | }, 66 | }, nil 67 | } 68 | -------------------------------------------------------------------------------- /pkg/csi-common/nodeserver-default.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | //nolint:revive // csi spec 18 | package csicommon 19 | 20 | import ( 21 | "context" 22 | 23 | "github.com/container-storage-interface/spec/lib/go/csi" 24 | "google.golang.org/grpc/codes" 25 | "google.golang.org/grpc/status" 26 | "k8s.io/klog" 27 | ) 28 | 29 | type DefaultNodeServer struct { 30 | Driver *CSIDriver 31 | } 32 | 33 | func (ns *DefaultNodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) { 34 | return nil, status.Error(codes.Unimplemented, "") 35 | } 36 | 37 | func (ns *DefaultNodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpublishVolumeRequest) (*csi.NodeUnpublishVolumeResponse, error) { 38 | return nil, status.Error(codes.Unimplemented, "") 39 | } 40 | 41 | func (ns *DefaultNodeServer) NodeGetInfo(ctx context.Context, req *csi.NodeGetInfoRequest) (*csi.NodeGetInfoResponse, error) { 42 | klog.V(5).Infof("Using default NodeGetInfo") 43 | 44 | return &csi.NodeGetInfoResponse{ 45 | NodeId: ns.Driver.nodeID, 46 | }, nil 47 | } 48 | 49 | func (ns *DefaultNodeServer) NodeGetCapabilities(ctx context.Context, req *csi.NodeGetCapabilitiesRequest) (*csi.NodeGetCapabilitiesResponse, error) { 50 | klog.V(5).Infof("Using default NodeGetCapabilities") 51 | 52 | return &csi.NodeGetCapabilitiesResponse{ 53 | Capabilities: []*csi.NodeServiceCapability{ 54 | { 55 | Type: &csi.NodeServiceCapability_Rpc{ 56 | Rpc: &csi.NodeServiceCapability_RPC{ 57 | Type: csi.NodeServiceCapability_RPC_UNKNOWN, 58 | }, 59 | }, 60 | }, 61 | }, 62 | }, nil 63 | } 64 | 65 | func (ns *DefaultNodeServer) NodeGetVolumeStats(ctx context.Context, in *csi.NodeGetVolumeStatsRequest) (*csi.NodeGetVolumeStatsResponse, error) { 66 | return nil, status.Error(codes.Unimplemented, "") 67 | } 68 | 69 | func (ns *DefaultNodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRequest) (*csi.NodeStageVolumeResponse, error) { 70 | return nil, status.Error(codes.Unimplemented, "") 71 | } 72 | 73 | func (ns *DefaultNodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstageVolumeRequest) (*csi.NodeUnstageVolumeResponse, error) { 74 | return nil, status.Error(codes.Unimplemented, "") 75 | } 76 | 77 | func (ns *DefaultNodeServer) NodeExpandVolume(ctx context.Context, req *csi.NodeExpandVolumeRequest) (*csi.NodeExpandVolumeResponse, error) { 78 | return nil, status.Error(codes.Unimplemented, "") 79 | } 80 | -------------------------------------------------------------------------------- /pkg/csi-common/server.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package csicommon 18 | 19 | import ( 20 | "net" 21 | "os" 22 | "sync" 23 | 24 | "github.com/container-storage-interface/spec/lib/go/csi" 25 | "google.golang.org/grpc" 26 | "k8s.io/klog" 27 | ) 28 | 29 | type NonBlockingGRPCServer interface { 30 | Start(endpoint string, ids csi.IdentityServer, cs csi.ControllerServer, ns csi.NodeServer) 31 | Wait() 32 | Stop() 33 | ForceStop() 34 | } 35 | 36 | func NewNonBlockingGRPCServer() NonBlockingGRPCServer { 37 | return &nonBlockingGRPCServer{} 38 | } 39 | 40 | type nonBlockingGRPCServer struct { 41 | wg sync.WaitGroup 42 | server *grpc.Server 43 | } 44 | 45 | func (s *nonBlockingGRPCServer) Start(endpoint string, ids csi.IdentityServer, cs csi.ControllerServer, ns csi.NodeServer) { 46 | s.wg.Add(1) 47 | 48 | go s.serve(endpoint, ids, cs, ns) 49 | } 50 | 51 | func (s *nonBlockingGRPCServer) Wait() { 52 | s.wg.Wait() 53 | } 54 | 55 | func (s *nonBlockingGRPCServer) Stop() { 56 | s.server.GracefulStop() 57 | } 58 | 59 | func (s *nonBlockingGRPCServer) ForceStop() { 60 | s.server.Stop() 61 | } 62 | 63 | func (s *nonBlockingGRPCServer) serve(endpoint string, ids csi.IdentityServer, cs csi.ControllerServer, ns csi.NodeServer) { 64 | var err error 65 | 66 | proto, addr, err := parseEndpoint(endpoint) 67 | if err != nil { 68 | klog.Fatal(err.Error()) 69 | } 70 | 71 | if proto == "unix" { 72 | addr = "/" + addr 73 | if err = os.Remove(addr); err != nil && !os.IsNotExist(err) { 74 | klog.Fatalf("Failed to remove %s, error: %s", addr, err.Error()) 75 | } 76 | } 77 | 78 | listener, err := net.Listen(proto, addr) 79 | if err != nil { 80 | klog.Fatalf("Failed to listen: %v", err) 81 | } 82 | 83 | opts := []grpc.ServerOption{ 84 | grpc.UnaryInterceptor(logGRPC), 85 | } 86 | server := grpc.NewServer(opts...) 87 | s.server = server 88 | 89 | if ids != nil { 90 | csi.RegisterIdentityServer(server, ids) 91 | } 92 | if cs != nil { 93 | csi.RegisterControllerServer(server, cs) 94 | } 95 | if ns != nil { 96 | csi.RegisterNodeServer(server, ns) 97 | } 98 | 99 | klog.Infof("Listening for connections on address: %#v", listener.Addr()) 100 | 101 | err = server.Serve(listener) 102 | if err != nil { 103 | klog.Fatalf("Failed to start GRPC server: %v", err) 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /pkg/csi-common/utils.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package csicommon 18 | 19 | import ( 20 | "context" 21 | "fmt" 22 | "strings" 23 | 24 | "github.com/container-storage-interface/spec/lib/go/csi" 25 | "github.com/kubernetes-csi/csi-lib-utils/protosanitizer" 26 | "google.golang.org/grpc" 27 | "k8s.io/klog" 28 | ) 29 | 30 | func parseEndpoint(ep string) (proto, addr string, _ error) { 31 | if strings.HasPrefix(strings.ToLower(ep), "unix://") || strings.HasPrefix(strings.ToLower(ep), "tcp://") { 32 | s := strings.SplitN(ep, "://", 2) 33 | if s[1] != "" { 34 | return s[0], s[1], nil 35 | } 36 | } 37 | return "", "", fmt.Errorf("invalid endpoint: %v", ep) 38 | } 39 | 40 | func NewVolumeCapabilityAccessMode(mode csi.VolumeCapability_AccessMode_Mode) *csi.VolumeCapability_AccessMode { 41 | return &csi.VolumeCapability_AccessMode{Mode: mode} 42 | } 43 | 44 | func NewDefaultNodeServer(d *CSIDriver) *DefaultNodeServer { 45 | return &DefaultNodeServer{ 46 | Driver: d, 47 | } 48 | } 49 | 50 | func NewDefaultIdentityServer(d *CSIDriver) *DefaultIdentityServer { 51 | return &DefaultIdentityServer{ 52 | Driver: d, 53 | } 54 | } 55 | 56 | func NewDefaultControllerServer(d *CSIDriver) *DefaultControllerServer { 57 | return &DefaultControllerServer{ 58 | Driver: d, 59 | } 60 | } 61 | 62 | func NewControllerServiceCapability(captype csi.ControllerServiceCapability_RPC_Type) *csi.ControllerServiceCapability { 63 | return &csi.ControllerServiceCapability{ 64 | Type: &csi.ControllerServiceCapability_Rpc{ 65 | Rpc: &csi.ControllerServiceCapability_RPC{ 66 | Type: captype, 67 | }, 68 | }, 69 | } 70 | } 71 | 72 | func logGRPC(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { 73 | klog.V(3).Infof("GRPC call: %s", info.FullMethod) 74 | klog.V(5).Infof("GRPC request: %s", protosanitizer.StripSecrets(req)) 75 | resp, err := handler(ctx, req) 76 | if err != nil { 77 | klog.Errorf("GRPC error: %v", err) 78 | } else { 79 | klog.V(5).Infof("GRPC response: %s", protosanitizer.StripSecrets(resp)) 80 | } 81 | return resp, err 82 | } 83 | -------------------------------------------------------------------------------- /pkg/spdk/driver.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) Arm Limited and Contributors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package spdk 18 | 19 | import ( 20 | "github.com/container-storage-interface/spec/lib/go/csi" 21 | "k8s.io/klog" 22 | 23 | csicommon "github.com/spdk/spdk-csi/pkg/csi-common" 24 | "github.com/spdk/spdk-csi/pkg/util" 25 | ) 26 | 27 | func Run(conf *util.Config) { 28 | var ( 29 | cd *csicommon.CSIDriver 30 | ids *identityServer 31 | cs *controllerServer 32 | ns *nodeServer 33 | 34 | controllerCaps = []csi.ControllerServiceCapability_RPC_Type{ 35 | csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME, 36 | csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT, 37 | csi.ControllerServiceCapability_RPC_LIST_SNAPSHOTS, 38 | csi.ControllerServiceCapability_RPC_EXPAND_VOLUME, 39 | csi.ControllerServiceCapability_RPC_CLONE_VOLUME, 40 | csi.ControllerServiceCapability_RPC_GET_VOLUME, 41 | // csi.ControllerServiceCapability_RPC_LIST_VOLUMES, 42 | csi.ControllerServiceCapability_RPC_VOLUME_CONDITION, 43 | } 44 | volumeModes = []csi.VolumeCapability_AccessMode_Mode{ 45 | csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER, 46 | } 47 | ) 48 | 49 | cd = csicommon.NewCSIDriver(conf.DriverName, conf.DriverVersion, conf.NodeID) 50 | if cd == nil { 51 | klog.Fatalln("Failed to initialize CSI Driver.") 52 | } 53 | if conf.IsControllerServer { 54 | cd.AddControllerServiceCapabilities(controllerCaps) 55 | cd.AddVolumeCapabilityAccessModes(volumeModes) 56 | } 57 | 58 | ids = newIdentityServer(cd) 59 | 60 | if conf.IsNodeServer { 61 | var err error 62 | ns, err = newNodeServer(cd) 63 | if err != nil { 64 | klog.Fatalf("failed to create node server: %s", err) 65 | } 66 | } 67 | 68 | if conf.IsControllerServer { 69 | var err error 70 | cs, err = newControllerServer(cd) 71 | if err != nil { 72 | klog.Fatalf("failed to create controller server: %s", err) 73 | } 74 | } 75 | 76 | s := csicommon.NewNonBlockingGRPCServer() 77 | s.Start(conf.Endpoint, ids, cs, ns) 78 | s.Wait() 79 | } 80 | -------------------------------------------------------------------------------- /pkg/spdk/identityserver.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) Arm Limited and Contributors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package spdk 18 | 19 | import ( 20 | "context" 21 | 22 | "github.com/container-storage-interface/spec/lib/go/csi" 23 | 24 | csicommon "github.com/spdk/spdk-csi/pkg/csi-common" 25 | ) 26 | 27 | type identityServer struct { 28 | *csicommon.DefaultIdentityServer 29 | } 30 | 31 | func newIdentityServer(d *csicommon.CSIDriver) *identityServer { 32 | return &identityServer{ 33 | DefaultIdentityServer: csicommon.NewDefaultIdentityServer(d), 34 | } 35 | } 36 | 37 | func (ids *identityServer) GetPluginCapabilities(_ context.Context, _ *csi.GetPluginCapabilitiesRequest) (*csi.GetPluginCapabilitiesResponse, error) { 38 | return &csi.GetPluginCapabilitiesResponse{ 39 | Capabilities: []*csi.PluginCapability{ 40 | { 41 | Type: &csi.PluginCapability_Service_{ 42 | Service: &csi.PluginCapability_Service{ 43 | Type: csi.PluginCapability_Service_CONTROLLER_SERVICE, 44 | }, 45 | }, 46 | }, 47 | { 48 | Type: &csi.PluginCapability_VolumeExpansion_{ 49 | VolumeExpansion: &csi.PluginCapability_VolumeExpansion{ 50 | Type: csi.PluginCapability_VolumeExpansion_ONLINE, 51 | }, 52 | }, 53 | }, 54 | { 55 | Type: &csi.PluginCapability_VolumeExpansion_{ 56 | VolumeExpansion: &csi.PluginCapability_VolumeExpansion{ 57 | Type: csi.PluginCapability_VolumeExpansion_OFFLINE, 58 | }, 59 | }, 60 | }, 61 | }, 62 | }, nil 63 | } 64 | -------------------------------------------------------------------------------- /pkg/util/config.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) Arm Limited and Contributors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package util 18 | 19 | const ( 20 | cfgRPCTimeoutSeconds = 60 21 | ) 22 | 23 | // Config stores parsed command line parameters 24 | type Config struct { 25 | DriverName string 26 | DriverVersion string 27 | Endpoint string 28 | NodeID string 29 | 30 | IsControllerServer bool 31 | IsNodeServer bool 32 | } 33 | -------------------------------------------------------------------------------- /pkg/util/idlocker.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) Arm Limited and Contributors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package util 18 | 19 | import ( 20 | "sync" 21 | ) 22 | 23 | // VolumeLocks simple locks that can be acquired by volumeID 24 | type VolumeLocks struct { 25 | mutexes sync.Map 26 | } 27 | 28 | // NewVolumeLocks returns new VolumeLocks. 29 | func NewVolumeLocks() *VolumeLocks { 30 | return &VolumeLocks{} 31 | } 32 | 33 | // Lock obtain the lock corresponding to the volumeID 34 | func (vl *VolumeLocks) Lock(volumeID string) func() { 35 | value, _ := vl.mutexes.LoadOrStore(volumeID, &sync.Mutex{}) 36 | mtx, _ := value.(*sync.Mutex) //nolint:errcheck // will not fail to convert 37 | mtx.Lock() 38 | return func() { mtx.Unlock() } 39 | } 40 | -------------------------------------------------------------------------------- /pkg/util/idlocker_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) Arm Limited and Contributors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package util 18 | 19 | import ( 20 | "testing" 21 | "time" 22 | ) 23 | 24 | func TestIDLocker(t *testing.T) { 25 | t.Parallel() 26 | fakeID := "fake-id" 27 | locks := NewVolumeLocks() 28 | // acquire lock for fake-id 29 | start := time.Now() 30 | unlock := locks.Lock(fakeID) 31 | go func() { 32 | time.Sleep(1 * time.Second) 33 | unlock() 34 | }() 35 | unlock2 := locks.Lock(fakeID) 36 | unlock2() 37 | duration := time.Since(start) 38 | // time.Sleep() might not be very accurate, 39 | // using 900 milliseconds to check the test would be more stable. 40 | if duration.Milliseconds() < 900 { 41 | t.Errorf("lock was required before it was released") 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /pkg/util/initiator_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) Arm Limited and Contributors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // whitebox test of some functions in initiator.go 18 | package util 19 | 20 | import ( 21 | "testing" 22 | "time" 23 | ) 24 | 25 | func TestExecWithTimeoutPositive(t *testing.T) { 26 | elapsed, err := runExecWithTimeout([]string{"true"}, 10) 27 | if err != nil { 28 | t.Fatal("should succeed") 29 | } 30 | if elapsed > 3 { 31 | t.Fatal("timeout error") 32 | } 33 | } 34 | 35 | func TestExecWithTimeoutNegative(t *testing.T) { 36 | elapsed, err := runExecWithTimeout([]string{"false"}, 10) 37 | if err == nil { 38 | t.Fatal("should fail") 39 | } 40 | if elapsed > 3 { 41 | t.Fatal("timeout error") 42 | } 43 | } 44 | 45 | func TestExecWithTimeoutTimeout(t *testing.T) { 46 | elapsed, err := runExecWithTimeout([]string{"sleep", "10"}, 1) 47 | if err == nil { 48 | t.Fatal("should fail") 49 | } 50 | if elapsed > 3 { 51 | t.Fatal("timeout error") 52 | } 53 | } 54 | 55 | func runExecWithTimeout(cmdLine []string, timeout int) (int, error) { 56 | start := time.Now() 57 | err := execWithTimeout(cmdLine, timeout) 58 | elapsed := int(time.Since(start) / time.Second) 59 | return elapsed, err 60 | } 61 | -------------------------------------------------------------------------------- /pkg/util/nvmf_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) Arm Limited and Contributors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package util 18 | 19 | import "testing" 20 | 21 | func TestNewNVMf(t *testing.T) { 22 | clusterID := "clusterID" 23 | clusterIP := "clusterIP" 24 | clusterSecret := "clusterSecret" 25 | 26 | node := NewNVMf(clusterID, clusterIP, clusterSecret) 27 | 28 | if node == nil { 29 | t.Fatal("NewNVMf returned nil") 30 | } 31 | 32 | if node.clusterID != clusterID { 33 | t.Errorf("Expected clusterID %s, but got %s", clusterID, node.clusterID) 34 | } 35 | 36 | if node.clusterIP != clusterIP { 37 | t.Errorf("Expected clusterIP %s, but got %s", clusterIP, node.clusterIP) 38 | } 39 | 40 | if node.clusterSecret != clusterSecret { 41 | t.Errorf("Expected clusterSecret %s, but got %s", clusterSecret, node.clusterSecret) 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /pkg/util/util_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) Arm Limited and Contributors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // blackbox test of util package 18 | package util_test 19 | 20 | import ( 21 | "os" 22 | "sync" 23 | "sync/atomic" 24 | "testing" 25 | 26 | "github.com/spdk/spdk-csi/pkg/util" 27 | ) 28 | 29 | func TestTryLockSequential(t *testing.T) { 30 | var tryLock util.TryLock 31 | 32 | // acquire lock 33 | if !tryLock.Lock() { 34 | t.Fatalf("failed to acquire lock") 35 | } 36 | // acquire a locked lock should fail 37 | if tryLock.Lock() { 38 | t.Fatalf("acquired a locked lock") 39 | } 40 | // acquire a released lock should succeed 41 | tryLock.Unlock() 42 | if !tryLock.Lock() { 43 | t.Fatal("failed to acquire a release lock") 44 | } 45 | } 46 | 47 | func TestTryLockConcurrent(t *testing.T) { 48 | var tryLock util.TryLock 49 | var wg sync.WaitGroup 50 | var lockCount int32 51 | const taskCount = 50 52 | 53 | // only one task should acquire the lock 54 | for i := 0; i < taskCount; i++ { 55 | wg.Add(1) 56 | go func() { 57 | if tryLock.Lock() { 58 | atomic.AddInt32(&lockCount, 1) 59 | } 60 | wg.Done() 61 | }() 62 | } 63 | wg.Wait() 64 | 65 | if lockCount != 1 { 66 | t.Fatal("concurrency test failed") 67 | } 68 | } 69 | 70 | func TestVolumeContext(t *testing.T) { 71 | volumeContextFileName := "volumeContext.json" 72 | 73 | dir, err := os.MkdirTemp("", "test") 74 | if err != nil { 75 | t.Fatal(err) 76 | } 77 | defer os.RemoveAll(dir) 78 | 79 | volumeContext := map[string]string{ 80 | "key1": "value1", 81 | "key2": "value2", 82 | } 83 | 84 | err = util.StashVolumeContext(volumeContext, dir) 85 | if err != nil { 86 | t.Fatalf("StashVolumeContext returned error: %v", err) 87 | } 88 | 89 | returnedContext, err := util.LookupVolumeContext(dir) 90 | if err != nil { 91 | t.Fatalf("LookupVolumeContext returned error: %v", err) 92 | } 93 | 94 | if volumeContext["key1"] != returnedContext["key1"] || volumeContext["key2"] != returnedContext["key2"] { 95 | t.Fatalf("LookupVolumeContext returned unexpected value: got %v, want %v", returnedContext, volumeContext) 96 | } 97 | 98 | err = util.CleanUpVolumeContext(dir) 99 | if err != nil { 100 | t.Fatalf("CleanUpVolumeContext returned error: %v", err) 101 | } 102 | 103 | _, err = os.Stat(dir + "/" + volumeContextFileName) 104 | if !os.IsNotExist(err) { 105 | t.Fatalf("CleanUpVolumeContext failed to cleanup volume context stash") 106 | } 107 | } 108 | -------------------------------------------------------------------------------- /scripts/ci/env: -------------------------------------------------------------------------------- 1 | GOVERSION=1.19 2 | KUBE_VERSION=v1.25.0 3 | MINIKUBE_VERSION=v1.27.0 4 | CRITOOLS_VERSION=v1.25.0 5 | CRIDOCKERD_VERSION=0.2.5 6 | CNIPLUGIN_VERSION=v1.1.1 7 | SPDKIMAGE=spdkdev:latest 8 | 9 | # FIXME(avalluri): Using 'main' image instead of release image as the latest (v0.1.1) does support KVM. 10 | # Move to release v0.1.2+ image once it is available. 11 | OPIIMAGE=opiproject/opi-spdk-bridge@sha256:885f751030d83e6d13ca4efbdf1c1f5f41a8803f0bd9ff78d9b2922d6170532c 12 | 13 | JSONRPC_IP=127.0.0.1 14 | JSONRPC_PORT=9009 15 | JSONRPC_USER=spdkcsiuser 16 | JSONRPC_PASS=spdkcsipass 17 | 18 | DOCKER_MIRROR=mirror.gcr.io 19 | 20 | # source code root directory 21 | ROOTDIR="${DIR}/../.." 22 | WORKERDIR="${WORKERDIR:-/tmp/qemu-vm}" 23 | -------------------------------------------------------------------------------- /scripts/ci/mdl_rules.rb: -------------------------------------------------------------------------------- 1 | all 2 | rule 'MD013', :line_length => 120 3 | rule 'MD029', :style => "ordered" 4 | exclude_rule 'MD031' 5 | -------------------------------------------------------------------------------- /scripts/ci/prepare.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) 4 | 5 | # shellcheck source=scripts/ci/env 6 | source "${DIR}/env" 7 | # shellcheck source=scripts/ci/common.sh 8 | source "${DIR}/common.sh" 9 | 10 | trap cleanup ERR 11 | 12 | PROMPT_FLAG=${PROMPT_FLAG:-true} 13 | 14 | if [[ $(id -u) != "0" ]]; then 15 | echo "Go away user, come back as root." 16 | exit 1 17 | fi 18 | 19 | # Prepare VM for running xPU tests on amd64 hosts. 20 | # To avoid this invoke the script with -x (no vm) 21 | if [ "${ARCH}" = amd64 ]; then 22 | PREPARE_VM=yes 23 | else 24 | PREPARE_VM=no 25 | fi 26 | 27 | while getopts 'yu:p:x' optchar; do 28 | case "$optchar" in 29 | y) 30 | PROMPT_FLAG=false 31 | ;; 32 | u) 33 | DOCKERHUB_USER="$OPTARG" 34 | ;; 35 | p) 36 | DOCKERHUB_SECRET="$OPTARG" 37 | ;; 38 | x) 39 | PREPARE_VM=no 40 | ;; 41 | *) 42 | echo "$0: invalid argument '$optchar'" 43 | exit 1 44 | ;; 45 | esac 46 | done 47 | 48 | if $PROMPT_FLAG; then 49 | echo "This script is meant to run on CI nodes." 50 | echo "It will install packages and docker images on current host." 51 | echo "Make sure you understand what it does before going on." 52 | read -r -p "Do you want to continue (yes/no)? " yn 53 | case "${yn}" in 54 | y|Y|yes|Yes|YES) :;; 55 | *) exit 0;; 56 | esac 57 | fi 58 | set -x 59 | #export_proxy 60 | check_os 61 | allocate_hugepages 2048 62 | install_packages_"${distro}" 63 | install_golang 64 | #docker_login 65 | # shellcheck disable=SC2119 66 | #build_spdkimage "--force" 67 | build_spdkcsi 68 | #build_test_binary 69 | 70 | vm= 71 | if [ "${PREPARE_VM}" = yes ]; then 72 | allocate_hugepages 10240 73 | vm_build 74 | vm_start 75 | vm="vm" 76 | distro="fedora" 77 | vm "install_golang; install_docker" 78 | vm_copy_spdkcsi_image "--force" 79 | vm_copy_test_binary 80 | fi 81 | 82 | $vm "configure_system_${distro}" 83 | $vm "setup_cri_dockerd" 84 | $vm "setup_cni_networking" 85 | $vm "stop_host_iscsid" 86 | $vm "docker_login" 87 | 88 | # workaround minikube permissions issues when running as root in ci(-like) env 89 | $vm sysctl fs.protected_regular=0 90 | $vm prepare_k8s_cluster 91 | 92 | #prepare_spdk_storage 93 | #prepare_xpu_node 94 | sudo ln -s /var/lib/minikube/binaries/v1.25.0/kubectl /usr/local/bin/kubectl 95 | echo "End of test environment setup!" 96 | -------------------------------------------------------------------------------- /scripts/ci/spdkcsi-ci.yaml: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | # Copyright (c) Arm Limited and Contributors 3 | # Copyright (c) Intel Corporation 4 | --- 5 | # spdkcsi ci jenkins job 6 | - job: 7 | name: spdkcsi-ci 8 | description: CI for SPDK-CSI project 9 | node: master 10 | project-type: matrix 11 | concurrent: false 12 | disabled: false 13 | axes: 14 | - axis: 15 | name: nodes 16 | type: slave 17 | values: 18 | - spdkcsi-ci-x86 19 | - spdkcsi-ci-arm 20 | builders: 21 | - shell: |- 22 | #!/bin/bash -e 23 | git fetch origin ${GERRIT_REFSPEC} 24 | git checkout FETCH_HEAD 25 | scripts/ci/test.sh 26 | scm: 27 | - git: 28 | branches: 29 | - '*/master' 30 | url: 'https://review.spdk.io/gerrit/spdk/spdk-csi' 31 | wipe-workspace: true 32 | triggers: 33 | - gerrit: 34 | # server-name configured in "gerrit trigger" plugin 35 | server-name: spdkcsi-gerrit 36 | projects: 37 | - branches: 38 | - branch-compare-type: ANT 39 | branch-pattern: '**' 40 | project-compare-type: ANT 41 | project-pattern: spdk/spdk-csi 42 | trigger-on: 43 | - patchset-created-event: 44 | exclude-trivial-rebase: false 45 | exclude-no-code-change: false 46 | - draft-published-event 47 | - comment-added-contains-event: 48 | # yamllint disable-line rule:line-length 49 | comment-contains-value: '(?i)^(Patch Set [0-9]+:)?( [\w\\+-]*)*(\n\n)?\s*(retrigger)' 50 | wrappers: 51 | - workspace-cleanup 52 | - timeout: 53 | timeout: 30 54 | abort: true 55 | -------------------------------------------------------------------------------- /scripts/ci/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | # build and test spdkcsi, can be invoked manually or by jenkins 4 | 5 | DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) 6 | 7 | # shellcheck source=scripts/ci/env 8 | source "${DIR}/env" 9 | # shellcheck source=scripts/ci/common.sh 10 | source "${DIR}/common.sh" 11 | 12 | Usage() { 13 | usage="Usage: $0 [option] 14 | Run all SPDK-CSI tests 15 | Options: 16 | -h, --help display this help and exit 17 | -x Exclude running xPU tests." 18 | echo "$usage" >&2 19 | } 20 | 21 | echo "Running script as user: $(whoami)" 22 | 23 | # Default run all xPU tests on amd64 hosts. 24 | # Invoke this script with -x if to exclude xPU tets 25 | if [ "${ARCH}" = amd64 ]; then 26 | RUN_XPU_TESTS=yes 27 | else 28 | RUN_XPU_TESTS=no 29 | fi 30 | 31 | for arg in "$@" ; do 32 | case "$arg" in 33 | -h|--help) Usage ; exit ;; 34 | -x) RUN_XPU_TESTS=no ;; 35 | *) echo "Ignoring unknown argument: $arg" >&2 ;; 36 | esac 37 | shift 38 | done 39 | 40 | trap on_exit EXIT ERR 41 | 42 | unit_test 43 | set -x 44 | perm="$(id -u):$(id -g)" 45 | if [ "${RUN_XPU_TESTS}" = yes ]; then 46 | echo "Running E2E tests in VM" 47 | # ./scripts/ci/prepare.sh is run with sudo user. Where 48 | # the ssh key generated is owned by root, hence make it 49 | # accessible by the current user 50 | sudo chown "$perm" "${WORKERDIR}"/id_rsa 51 | 52 | vm e2e_test "-xpu=true" 53 | vm "make -C \${ROOTDIR} helm-test HELM_SKIP_SPDKDEV_CHECK=1" 54 | vm_stop 55 | else 56 | # As minikube is installed by the root user, it is necessary to copy the 57 | # authentication information to a regular user 58 | sudo cp -r /root/.kube /root/.minikube "${HOME}" 59 | sudo chown -R "$perm" "${HOME}"/.kube "${HOME}"/.minikube 60 | sed -i "s#/root/#$HOME/#g" "${HOME}"/.kube/config 61 | e2e_test "-xpu=false" 62 | helm_test 63 | fi 64 | set +x 65 | exit 0 66 | -------------------------------------------------------------------------------- /scripts/config-apply.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | LABEL_KEY="type" 4 | LABEL_VALUE="simplyblock-storage-plane" 5 | ISOLATE_CORES="true" 6 | NAMESPACE="${1:-simplyblk}" 7 | NODES=$(kubectl get nodes -l "${LABEL_KEY}=${LABEL_VALUE}" -o jsonpath='{.items[*].metadata.name}') 8 | 9 | for NODE in $NODES; do 10 | NODE_IP=$(kubectl get node $NODE -o jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}') 11 | SANITIZED_NODE=$(echo "$NODE" | tr '.' '-') 12 | JOB_NAME="apply-config-${SANITIZED_NODE}" 13 | 14 | cat < 10 | MAX_PROV= 11 | 12 | NODES=$(kubectl get nodes -l "${LABEL_KEY}=${LABEL_VALUE}" -o jsonpath='{.items[*].metadata.name}') 13 | 14 | for NODE in $NODES; do 15 | SANITIZED_NODE=$(echo "$NODE" | tr '.' '-') 16 | JOB_NAME="simplyblock-upgrade-${SANITIZED_NODE}" 17 | 18 | cat < kube_version 1 -> 1 (Major) 48 | # v1.17.2 -> kube_version 2 -> 17 (Minor) 49 | function kube_version() { 50 | echo "${KUBE_VERSION}" | sed 's/^v//' | cut -d'.' -f"${1}" 51 | } 52 | 53 | if ! get_kube_version=$(kubectl version) || 54 | [[ -z "${get_kube_version}" ]]; then 55 | echo "could not get Kubernetes server version" 56 | echo "hint: check if you have specified the right host or port" 57 | exit 1 58 | fi 59 | 60 | KUBE_VERSION=$(echo "${get_kube_version}" | grep "^Server Version" | cut -d' ' -f3) 61 | KUBE_MAJOR=$(kube_version 1) 62 | KUBE_MINOR=$(kube_version 2) 63 | 64 | # skip snapshot operation if kube version is less than 1.17.0 65 | if [[ "${KUBE_MAJOR}" -lt 1 ]] || [[ "${KUBE_MAJOR}" -eq 1 && "${KUBE_MINOR}" -lt 17 ]]; then 66 | echo "skipping: Kubernetes server version is < 1.17.0" 67 | exit 1 68 | fi 69 | 70 | case "${1:-}" in 71 | install) 72 | install_snapshot_crds "$2" 73 | ;; 74 | delete) 75 | delete_snapshot_crds "$2" 76 | ;; 77 | *) 78 | echo "usage:" >&2 79 | echo " $0 install [namespace]" >&2 80 | echo " $0 delete-crd [namespace]" >&2 81 | ;; 82 | esac 83 | 84 | -------------------------------------------------------------------------------- /scripts/minikube.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | KUBE_VERSION=${KUBE_VERSION:-v1.25.0} 4 | MINIKUBE_VERSION=${MINIKUBE_VERSION:-v1.27.0} 5 | MINIKUBE_DRIVER=${MINIKUBE_DRIVER:-none} 6 | MINIKUBE_ARCH=amd64 7 | if [ "$(uname -m)" = "aarch64" ]; then 8 | MINIKUBE_ARCH=arm64 9 | fi 10 | 11 | function install_minikube() { 12 | if hash minikube 2> /dev/null; then 13 | version=$(minikube version | awk '{print $3; exit;}') 14 | if [[ "${version}" != "${MINIKUBE_VERSION}" ]]; then 15 | echo "installed minikube doesn't match requested version ${MINIKUBE_VERSION}" 16 | echo "please remove minikube ${version} first" 17 | exit 1 18 | fi 19 | echo "minikube-${version} already installed" 20 | return 21 | fi 22 | 23 | echo "=== downloading minikube-${MINIKUBE_VERSION}" 24 | curl -Lo /usr/local/bin/minikube https://storage.googleapis.com/minikube/releases/"${MINIKUBE_VERSION}"/minikube-linux-"${MINIKUBE_ARCH}" && chmod +x /usr/local/bin/minikube 25 | } 26 | 27 | case "$1" in 28 | up) 29 | install_minikube 30 | echo "=== starting minikube with kubeadm bootstrapper" 31 | CHANGE_MINIKUBE_NONE_USER=true minikube start -b kubeadm --kubernetes-version="${KUBE_VERSION}" --vm-driver="${MINIKUBE_DRIVER}" --alsologtostderr -v=5 32 | ;; 33 | down) 34 | minikube stop 35 | ;; 36 | clean) 37 | minikube delete 38 | ;; 39 | *) 40 | echo "$0 [up|down|clean]" 41 | ;; 42 | esac 43 | -------------------------------------------------------------------------------- /scripts/verify-helm-yamllint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | helmPath=charts/spdk-csi 4 | LOG=/tmp/yamllint.log 5 | 6 | yamllint -s -c scripts/yamllint.yml -f parsable $helmPath/*.yaml 7 | yamllint -s -c scripts/yamllint.yml -f parsable $helmPath/templates/*.yaml | grep -v -e "line too long" -e "too many spaces inside braces" -e "missing document start" -e "syntax error" > $LOG 8 | 9 | linecount=$(wc -l < $LOG) 10 | if [ "$linecount" -gt 0 ]; then 11 | cat $LOG 12 | exit 1 13 | fi 14 | -------------------------------------------------------------------------------- /scripts/yamllint.yml: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | # Copyright (c) Arm Limited and Contributors 3 | # Copyright (c) Intel Corporation 4 | --- 5 | extends: default 6 | 7 | rules: 8 | indentation: 9 | spaces: 2 10 | indent-sequences: false 11 | check-multi-line-strings: false 12 | -------------------------------------------------------------------------------- /test/sanity/README.md: -------------------------------------------------------------------------------- 1 | ## Sanity Tests 2 | Testing the SMB CSI driver using the [`sanity`](https://github.com/kubernetes-csi/csi-test/tree/master/pkg/sanity) package test suite. 3 | 4 | ### Run sanity tests 5 | ``` 6 | make sanity-test 7 | ``` 8 | -------------------------------------------------------------------------------- /test/sanity/params.yaml: -------------------------------------------------------------------------------- 1 | source: //127.0.0.1/share -------------------------------------------------------------------------------- /test/sanity/run-test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2019 The Kubernetes Authors. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | set -eo pipefail 18 | 19 | function cleanup { 20 | echo 'pkill -f smbplugin' 21 | if [ -z "$GITHUB_ACTIONS" ] 22 | then 23 | # if not running on github actions, do not use sudo 24 | pkill -f smbplugin 25 | else 26 | # if running on github actions, use sudo 27 | sudo pkill -f smbplugin 28 | fi 29 | echo 'Deleting CSI sanity test binary' 30 | rm -rf csi-test 31 | echo 'Uninstalling samba server on localhost' 32 | docker rm samba -f 33 | } 34 | 35 | trap cleanup EXIT 36 | 37 | function install_csi_sanity_bin { 38 | echo 'Installing CSI sanity test binary...' 39 | mkdir -p $GOPATH/src/github.com/kubernetes-csi 40 | pushd $GOPATH/src/github.com/kubernetes-csi 41 | export GO111MODULE=off 42 | git clone https://github.com/kubernetes-csi/csi-test.git -b v5.0.0 43 | pushd csi-test/cmd/csi-sanity 44 | make install 45 | popd 46 | popd 47 | } 48 | 49 | function provision_simplyblock_cluster { 50 | echo 'Running samba server on localhost' 51 | docker run -e PERMISSIONS=0777 -p 445:445 --name samba -d andyzhangx/samba:win-fix -s "share;/smbshare/;yes;no;no;all;none" -u "sanity;sanitytestpassword" -p 52 | } 53 | 54 | provision_simplyblock_cluster 55 | 56 | if [[ -z "$(command -v csi-sanity)" ]]; then 57 | install_csi_sanity_bin 58 | fi 59 | 60 | readonly endpoint='unix:///tmp/csi.sock' 61 | nodeid='CSINode' 62 | if [[ "$#" -gt 0 ]] && [[ -n "$1" ]]; then 63 | nodeid="$1" 64 | fi 65 | 66 | ARCH=$(uname -p) 67 | if [[ "${ARCH}" == "x86_64" || ${ARCH} == "unknown" ]]; then 68 | ARCH="amd64" 69 | fi 70 | 71 | if [ -z "$GITHUB_ACTIONS" ] 72 | then 73 | # if not running on github actions, do not use sudo 74 | _output/${ARCH}/smbplugin --endpoint "$endpoint" --nodeid "$nodeid" -v=5 & 75 | else 76 | # if running on github actions, use sudo 77 | sudo _output/${ARCH}/smbplugin --endpoint "$endpoint" --nodeid "$nodeid" -v=5 & 78 | fi 79 | 80 | # sleep a while waiting for azurefileplugin start up 81 | sleep 1 82 | 83 | echo 'Begin to run sanity test...' 84 | CSI_SANITY_BIN=$GOPATH/bin/csi-sanity 85 | skipTests='create a volume with already existing name and different capacity|should fail when requesting to create a volume with already existing name and different capacity|should fail when the requested volume does not exist' 86 | if [ -z "$GITHUB_ACTIONS" ] 87 | then 88 | # if not running on github actions, do not use sudo 89 | "$CSI_SANITY_BIN" --ginkgo.v --csi.secrets="$(pwd)/test/sanity/secrets.yaml" --csi.testvolumeparameters="$(pwd)/test/sanity/params.yaml" --csi.endpoint="$endpoint" --ginkgo.skip="$skipTests" 90 | else 91 | # if running on github actions, use sudo 92 | sudo "$CSI_SANITY_BIN" --ginkgo.v --csi.secrets="$(pwd)/test/sanity/secrets.yaml" --csi.testvolumeparameters="$(pwd)/test/sanity/params.yaml" --csi.endpoint="$endpoint" --ginkgo.skip="$skipTests" 93 | fi 94 | -------------------------------------------------------------------------------- /test/sanity/secrets.yaml: -------------------------------------------------------------------------------- 1 | NodeStageVolumeSecret: 2 | username: sanity 3 | password: sanitytestpassword 4 | CreateVolumeSecret: 5 | username: sanity 6 | password: sanitytestpassword 7 | --------------------------------------------------------------------------------