├── .dockerignore ├── .github └── workflows │ ├── codeql.yml │ ├── lint.yaml │ └── publish.yml ├── .gitignore ├── Dockerfile ├── Dockerfile.UBI ├── Jenkinsfile ├── LICENSE ├── Makefile ├── PROJECT ├── README.md ├── SECURITY.md ├── build └── Dockerfile ├── cmd ├── manager ├── pmem-csi-driver │ ├── main.go │ └── main_test.go └── pmem-csi-operator │ ├── main.go │ └── main_test.go ├── conf.json ├── conf.py ├── deploy ├── common │ ├── pmem-app-block-volume.yaml │ ├── pmem-app-ephemeral.yaml │ ├── pmem-app-generic-ephemeral.yaml │ ├── pmem-app-late-binding.yaml │ ├── pmem-app.yaml │ ├── pmem-csi.intel.com_v1beta1_pmemcsideployment_cr.yaml │ ├── pmem-kata-app-ephemeral.yaml │ ├── pmem-kata-app.yaml │ ├── pmem-kata-pvc.yaml │ ├── pmem-pvc-block-volume.yaml │ ├── pmem-pvc-late-binding.yaml │ ├── pmem-pvc.yaml │ ├── pmem-storageclass-default.yaml │ ├── pmem-storageclass-ext4-fileio.yaml │ ├── pmem-storageclass-ext4-kata.yaml │ ├── pmem-storageclass-ext4.yaml │ ├── pmem-storageclass-late-binding.yaml │ ├── pmem-storageclass-xfs-fileio.yaml │ ├── pmem-storageclass-xfs-kata.yaml │ └── pmem-storageclass-xfs.yaml ├── crd │ └── pmem-csi.intel.com_pmemcsideployments.yaml ├── kubernetes-1.21 │ ├── direct │ │ ├── kustomization.yaml │ │ ├── pmem-csi.yaml │ │ └── testing │ │ │ ├── kustomization.yaml │ │ │ └── pmem-csi.yaml │ ├── lvm │ │ ├── kustomization.yaml │ │ ├── pmem-csi.yaml │ │ └── testing │ │ │ ├── kustomization.yaml │ │ │ └── pmem-csi.yaml │ ├── pmem-csi-direct-testing.yaml │ ├── pmem-csi-direct.yaml │ ├── pmem-csi-lvm-testing.yaml │ └── pmem-csi-lvm.yaml ├── kubernetes-1.22 │ ├── direct │ │ ├── kustomization.yaml │ │ ├── pmem-csi.yaml │ │ └── testing │ │ │ ├── kustomization.yaml │ │ │ └── pmem-csi.yaml │ ├── lvm │ │ ├── kustomization.yaml │ │ ├── pmem-csi.yaml │ │ └── testing │ │ │ ├── kustomization.yaml │ │ │ └── pmem-csi.yaml │ ├── pmem-csi-direct-testing.yaml │ ├── pmem-csi-direct.yaml │ ├── pmem-csi-lvm-testing.yaml │ └── pmem-csi-lvm.yaml ├── kubernetes-1.23 │ ├── direct │ │ ├── kustomization.yaml │ │ ├── pmem-csi.yaml │ │ └── testing │ │ │ ├── kustomization.yaml │ │ │ └── pmem-csi.yaml │ ├── lvm │ │ ├── kustomization.yaml │ │ ├── pmem-csi.yaml │ │ └── testing │ │ │ ├── kustomization.yaml │ │ │ └── pmem-csi.yaml │ ├── pmem-csi-direct-testing.yaml │ ├── pmem-csi-direct.yaml │ ├── pmem-csi-lvm-testing.yaml │ └── pmem-csi-lvm.yaml ├── kubernetes-1.24 │ ├── direct │ │ ├── kustomization.yaml │ │ ├── pmem-csi.yaml │ │ └── testing │ │ │ ├── kustomization.yaml │ │ │ └── pmem-csi.yaml │ ├── lvm │ │ ├── kustomization.yaml │ │ ├── pmem-csi.yaml │ │ └── testing │ │ │ ├── kustomization.yaml │ │ │ └── pmem-csi.yaml │ ├── pmem-csi-direct-testing.yaml │ ├── pmem-csi-direct.yaml │ ├── pmem-csi-lvm-testing.yaml │ └── pmem-csi-lvm.yaml ├── kubernetes-1.25 │ ├── direct │ │ ├── kustomization.yaml │ │ ├── pmem-csi.yaml │ │ └── testing │ │ │ ├── kustomization.yaml │ │ │ └── pmem-csi.yaml │ ├── lvm │ │ ├── kustomization.yaml │ │ ├── pmem-csi.yaml │ │ └── testing │ │ │ ├── kustomization.yaml │ │ │ └── pmem-csi.yaml │ ├── pmem-csi-direct-testing.yaml │ ├── pmem-csi-direct.yaml │ ├── pmem-csi-lvm-testing.yaml │ └── pmem-csi-lvm.yaml ├── kustomize │ ├── driver │ │ ├── README.md │ │ ├── kustomization.yaml │ │ └── pmem-csi.yaml │ ├── kubernetes-base-direct-coverage │ │ └── kustomization.yaml │ ├── kubernetes-base-direct-testing-coverage │ │ └── kustomization.yaml │ ├── kubernetes-base-direct-testing │ │ └── kustomization.yaml │ ├── kubernetes-base-direct │ │ ├── README.md │ │ └── kustomization.yaml │ ├── kubernetes-base-fake │ │ ├── fake-device-mode-patch.yaml │ │ └── kustomization.yaml │ ├── kubernetes-base-lvm-coverage │ │ └── kustomization.yaml │ ├── kubernetes-base-lvm-testing-coverage │ │ └── kustomization.yaml │ ├── kubernetes-base-lvm-testing │ │ └── kustomization.yaml │ ├── kubernetes-base-lvm │ │ ├── README.md │ │ └── kustomization.yaml │ ├── kubernetes-no-metrics │ │ ├── README.md │ │ ├── driverinfo-beta.yaml │ │ ├── kustomization.yaml │ │ └── rbac │ │ │ ├── README.md │ │ │ └── kustomization.yaml │ ├── kubernetes-with-metrics │ │ └── kustomization.yaml │ ├── memcached │ │ ├── ephemeral │ │ │ ├── kustomization.yaml │ │ │ └── memcached-ephemeral.yaml │ │ └── persistent │ │ │ ├── kustomization.yaml │ │ │ └── memcached-persistent.yaml │ ├── metrics-server │ │ └── kustomization.yaml │ ├── olm-catalog │ │ ├── bases │ │ │ └── pmem-csi-operator.clusterserviceversion.yaml │ │ └── kustomization.yaml │ ├── operator │ │ ├── README.md │ │ ├── events-rbac.yaml │ │ ├── kustomization.yaml │ │ ├── kustomizeconfig.yaml │ │ ├── namespace.yaml │ │ ├── operator-events-namespace-patch.yaml │ │ └── operator.yaml │ ├── patches │ │ ├── controller-role-patch.yaml │ │ ├── delete-csi-provisioner-service-account-patch.yaml │ │ ├── direct-patch.yaml │ │ ├── external-provisioner-connection-timeout-patch.yaml │ │ ├── lvm-patch.yaml │ │ ├── metrics-controller.yaml │ │ ├── metrics-node.yaml │ │ ├── storageclass-late-binding-patch.yaml │ │ └── usage-fileio.yaml │ ├── scheduler │ │ └── kustomization.yaml │ ├── storageclass-ext4-fileio │ │ └── kustomization.yaml │ ├── storageclass-ext4 │ │ ├── ext4.yaml │ │ └── kustomization.yaml │ ├── storageclass-late-binding │ │ └── kustomization.yaml │ ├── storageclass-xfs-fileio │ │ └── kustomization.yaml │ ├── storageclass-xfs │ │ ├── kustomization.yaml │ │ └── xfs.yaml │ ├── storageclass │ │ ├── kustomization.yaml │ │ └── pmem-storageclass.yaml │ ├── testing │ │ ├── README.md │ │ ├── controller-coverage-patch.yaml │ │ ├── controller-verbosity-patch.yaml │ │ ├── kustomization.yaml │ │ ├── node-coverage-patch.yaml │ │ ├── node-verbosity-patch.yaml │ │ └── socat.yaml │ ├── vpa-for-pmem-csi │ │ ├── kustomization.yaml │ │ ├── vpa-controller.yaml │ │ └── vpa-node.yaml │ └── webhook │ │ ├── kustomization.yaml │ │ ├── webhook-service.yaml │ │ └── webhook.yaml ├── operator │ └── pmem-csi-operator.yaml ├── prometheus.yaml └── yamls.go ├── docs ├── DEVELOPMENT.md ├── autotest.md ├── design.md ├── diagrams │ ├── pmem-csi-communication-diagram.dia │ ├── pmem-csi.dia │ └── sequence.wsd ├── html │ ├── index.html │ └── index2.html ├── images │ ├── communication │ │ └── pmem-csi-communication-diagram.png │ ├── devicemodes │ │ ├── pmem-csi-direct.png │ │ └── pmem-csi-lvm.png │ └── sequence │ │ └── pmem-csi-persistent-sequence-diagram.png ├── install.md ├── js │ └── copybutton.js ├── requirements.txt ├── screencast-pmem.sh ├── sphinx_ext │ └── linkcheck2.py ├── static │ └── override.css └── substitutions.txt ├── examples ├── gce.md ├── memcached.md ├── readme.rst └── redis-operator.md ├── go.mod ├── go.sum ├── hack ├── bump-image-versions.sh ├── copy-modules-license.sh ├── list-direct-imports.sh ├── merge-release.sh ├── python3-fake-debian-package ├── set-version.sh ├── setup-va.sh ├── stress-driver.sh ├── tooling │ └── dependencies.go ├── verify-generated.sh ├── verify-go-version.sh └── verify-vendor.sh ├── index.rst ├── make.bat ├── operator └── operator.make ├── pkg ├── apis │ ├── addtoscheme_pmemcsi_v1beta1.go │ ├── apis.go │ └── pmemcsi │ │ ├── group.go │ │ └── v1beta1 │ │ ├── deployment_types.go │ │ ├── deployment_types_test.go │ │ ├── doc.go │ │ ├── register.go │ │ └── zz_generated.deepcopy.go ├── coverage │ └── coverage.go ├── deployments │ ├── doc.go │ ├── load.go │ └── load_test.go ├── errors │ └── errors.go ├── exec │ ├── exec.go │ └── exec_test.go ├── grpc-server │ └── server.go ├── imagefile │ ├── imagefile.go │ ├── imagefile_test.go │ └── test │ │ ├── imagefile_test.go │ │ └── imagefiletest.go ├── k8sutil │ ├── client.go │ ├── doc.go │ └── namespace.go ├── logger │ ├── capacity.go │ ├── flag.go │ ├── kubernetes.go │ └── logger.go ├── math │ └── math.go ├── ndctl │ ├── bus.go │ ├── dimm.go │ ├── fake │ │ ├── bus.go │ │ ├── dimm.go │ │ ├── mapping.go │ │ ├── namespace.go │ │ ├── ndctl.go │ │ └── region.go │ ├── mapping.go │ ├── marshal.go │ ├── namespace.go │ ├── ndctl.go │ └── region.go ├── pmem-common │ ├── termination.go │ ├── tracing.go │ └── vgname.go ├── pmem-csi-driver │ ├── controllerserver-default.go │ ├── controllerserver-node.go │ ├── identityserver.go │ ├── main.go │ ├── nodeserver.go │ ├── parameters │ │ ├── parameters.go │ │ └── parameters_test.go │ ├── pmem-csi-driver.go │ ├── pmem-csi-driver_test.go │ ├── rescheduler.go │ └── rescheduler_test.go ├── pmem-csi-operator │ ├── controller │ │ ├── controller.go │ │ └── deployment │ │ │ ├── controller_driver.go │ │ │ ├── deployment_controller.go │ │ │ ├── deployment_controller_test.go │ │ │ └── testcases │ │ │ └── testcases.go │ ├── main.go │ └── metrics │ │ └── metrics.go ├── pmem-device-manager │ ├── convert.go │ ├── convert_test.go │ ├── metrics.go │ ├── pmd-fake.go │ ├── pmd-lvm.go │ ├── pmd-manager.go │ ├── pmd-manager_test.go │ ├── pmd-ndctl.go │ └── pmd-util.go ├── pmem-grpc │ └── grpc.go ├── pmem-state │ ├── pmem-state.go │ └── pmem-state_test.go ├── types │ └── types.go ├── version │ ├── version.go │ └── version_test.go ├── volumepathhandler │ ├── volume_path_handler.go │ └── volume_path_handler_linux.go └── xfs │ ├── xfs.go │ └── xfs_test.go ├── runtime-deps.csv ├── setup.py ├── test ├── check-imagefile.sh ├── cmd │ ├── pmem-access-hugepages │ │ └── main.go │ ├── pmem-dax-check │ │ └── main.go │ └── watch-pvs │ │ └── watch-pvs.go ├── delete-deployment.sh ├── e2e │ ├── deploy │ │ ├── cluster.go │ │ ├── deploy.go │ │ ├── doc.go │ │ ├── operator.go │ │ ├── pmem.go │ │ └── volumeleaks.go │ ├── driver │ │ ├── driver.go │ │ └── pattern.go │ ├── e2e.go │ ├── e2e_test.go │ ├── ephemeral │ │ └── ephemeral.go │ ├── filesource_test.go │ ├── gotests │ │ └── gotests.go │ ├── image │ │ └── image.go │ ├── imagefile │ │ └── imagefilee2e.go │ ├── metrics │ │ └── metrics.go │ ├── operator │ │ ├── deployment_api.go │ │ └── validate │ │ │ └── validate.go │ ├── pod │ │ ├── dial.go │ │ ├── exec.go │ │ └── logs.go │ ├── storage │ │ ├── conversion.go │ │ ├── csi_volumes.go │ │ ├── dax │ │ │ └── dax.go │ │ ├── immediatebinding.go │ │ ├── pmem_csi.go │ │ ├── provisioning.go │ │ ├── sanity.go │ │ └── wait.go │ ├── tls │ │ ├── nmap-ssl-enum-ciphers.patch │ │ └── tls.go │ └── versionskew │ │ └── versionskew.go ├── pull-images.sh ├── setup-clear-govm.sh ├── setup-deployment.sh ├── setup-fedora-govm.sh ├── setup-kata-containers.sh ├── setup-kubernetes.sh ├── start-kubernetes.sh ├── start-operator.sh ├── start-stop-olm.sh ├── start-stop-openshift.sh ├── start-stop.make ├── stop-operator.sh ├── test-config.d │ ├── .gitignore │ └── README.md ├── test-config.sh ├── test-config │ └── config.go └── test.make ├── third-party ├── README.md └── go-fibmap │ ├── .gitignore │ ├── LICENSE │ ├── README.md │ └── fibmap.go └── tox.ini /.dockerignore: -------------------------------------------------------------------------------- 1 | * 2 | !*/*.make 3 | !Dockerfile 4 | !LICENSE 5 | !Makefile 6 | !cmd/ 7 | !pkg/ 8 | !test/ 9 | !third-party/ 10 | !hack/ 11 | !vendor/ 12 | !go.mod 13 | !go.sum 14 | !operator/ 15 | !deploy/ 16 | -------------------------------------------------------------------------------- /.github/workflows/codeql.yml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | # You may wish to alter this file to override the set of languages analyzed, 5 | # or to provide custom queries or build logic. 6 | # 7 | # ******** NOTE ******** 8 | # We have attempted to detect the languages in your repository. Please check 9 | # the `language` matrix defined below to confirm you have the correct set of 10 | # supported CodeQL languages. 11 | # 12 | name: "CodeQL" 13 | 14 | on: 15 | push: 16 | branches: [ "devel" ] 17 | pull_request: 18 | branches: [ "devel" ] 19 | schedule: 20 | - cron: '45 2 * * 0' 21 | 22 | # Declare default permissions as read only. 23 | permissions: read-all 24 | 25 | jobs: 26 | analyze: 27 | name: Analyze 28 | # Runner size impacts CodeQL analysis time. To learn more, please see: 29 | # - https://gh.io/recommended-hardware-resources-for-running-codeql 30 | # - https://gh.io/supported-runners-and-hardware-resources 31 | # - https://gh.io/using-larger-runners 32 | # Consider using larger runners for possible analysis time improvements. 33 | runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }} 34 | timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }} 35 | permissions: 36 | # required for all workflows 37 | security-events: write 38 | 39 | # only required for workflows in private repositories 40 | actions: read 41 | contents: read 42 | 43 | strategy: 44 | fail-fast: false 45 | matrix: 46 | language: [ 'go', 'python' ] 47 | # CodeQL supports [ 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift' ] 48 | # Use only 'java-kotlin' to analyze code written in Java, Kotlin or both 49 | # Use only 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both 50 | # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support 51 | 52 | steps: 53 | - name: Checkout repository 54 | uses: actions/checkout@v4 55 | 56 | # Initializes the CodeQL tools for scanning. 57 | - name: Initialize CodeQL 58 | uses: github/codeql-action/init@v3 59 | with: 60 | languages: ${{ matrix.language }} 61 | # If you wish to specify custom queries, you can do so here or in a config file. 62 | # By default, queries listed here will override any specified in a config file. 63 | # Prefix the list here with "+" to use these queries and those in the config file. 64 | 65 | # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs 66 | # queries: security-extended,security-and-quality 67 | 68 | 69 | # Autobuild attempts to build any compiled languages (C/C++, C#, Go, Java, or Swift). 70 | # If this step fails, then you should remove it and run the build manually (see below) 71 | - name: Autobuild 72 | uses: github/codeql-action/autobuild@v3 73 | 74 | # ℹ️ Command-line programs to run using the OS shell. 75 | # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun 76 | 77 | # If the Autobuild fails above, remove it and uncomment the following three lines. 78 | # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. 79 | 80 | # - run: | 81 | # echo "Run, Build Application using script" 82 | # ./location_of_script_within_repo/buildscript.sh 83 | 84 | - name: Perform CodeQL Analysis 85 | uses: github/codeql-action/analyze@v3 86 | with: 87 | category: "/language:${{matrix.language}}" 88 | -------------------------------------------------------------------------------- /.github/workflows/lint.yaml: -------------------------------------------------------------------------------- 1 | name: golangci-lint 2 | on: 3 | push: 4 | tags: 5 | - "v*" 6 | branches: 7 | - "*" 8 | pull_request: 9 | 10 | # Declare default permissions as read only. 11 | permissions: read-all 12 | 13 | jobs: 14 | golangci: 15 | name: lint 16 | runs-on: ubuntu-latest 17 | steps: 18 | - uses: actions/checkout@v2 19 | - name: golangci-lint 20 | uses: golangci/golangci-lint-action@v2 21 | with: 22 | # version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version 23 | version: latest 24 | 25 | # show only new issues if it's a pull request. The default value is `false`. 26 | # We still have issues in the existing code, but at least should not make it worse. 27 | only-new-issues: true 28 | 29 | args: --timeout=20m 30 | -------------------------------------------------------------------------------- /.github/workflows/publish.yml: -------------------------------------------------------------------------------- 1 | name: Publish 2 | 3 | on: 4 | push: 5 | branches: 6 | - devel 7 | - release-0.7 8 | - release-0.8 9 | - release-0.9 10 | - release-1.0 11 | - release-1.1 12 | 13 | 14 | # Declare default permissions as read only. 15 | permissions: read-all 16 | 17 | jobs: 18 | build: 19 | 20 | runs-on: ubuntu-latest 21 | 22 | steps: 23 | - name: Install dependencies 24 | run: | 25 | sudo apt-get update 26 | sudo apt-get install -y python3-venv git 27 | - uses: actions/checkout@v2 28 | with: 29 | ref: devel 30 | - name: Set up shared doc directory 31 | run: | 32 | mkdir $HOME/output 33 | touch $HOME/output/.nojekyll 34 | # https://intel.github.io/pmem-csi/ redirects to the latest stable release. 35 | echo "" >"$HOME/output/index.html" 36 | # "latest" used to be what "devel" is now, i.e. the documentation for the "devel" 37 | # branch. Now it points to the latest stable release. 38 | ln -s 1.0 "$HOME/output/latest" 39 | - name: Build latest 40 | run: | 41 | GITHUB_SHA=$(git rev-parse HEAD) 42 | export GITHUB_SHA 43 | rm -rf _work/venv 44 | make vhtml 45 | mv _output/html $HOME/output/devel 46 | - uses: actions/checkout@v2 47 | with: 48 | ref: release-0.7 49 | - name: Build release-0.7 50 | run: | 51 | GITHUB_SHA=$(git rev-parse HEAD) 52 | export GITHUB_SHA 53 | rm -rf _work/venv 54 | make vhtml 55 | mv _output/html $HOME/output/0.7 56 | - uses: actions/checkout@v2 57 | with: 58 | ref: release-0.8 59 | - name: Build release-0.8 60 | run: | 61 | GITHUB_SHA=$(git rev-parse HEAD) 62 | export GITHUB_SHA 63 | rm -rf _work/venv 64 | make vhtml 65 | mv _output/html $HOME/output/0.8 66 | - uses: actions/checkout@v2 67 | with: 68 | ref: release-0.9 69 | - name: Build release-0.9 70 | run: | 71 | GITHUB_SHA=$(git rev-parse HEAD) 72 | export GITHUB_SHA 73 | rm -rf _work/venv 74 | make vhtml 75 | mv _output/html $HOME/output/0.9 76 | - uses: actions/checkout@v2 77 | with: 78 | ref: release-1.0 79 | - name: Build release-1.0 80 | run: | 81 | GITHUB_SHA=$(git rev-parse HEAD) 82 | export GITHUB_SHA 83 | rm -rf _work/venv 84 | make vhtml 85 | mv _output/html $HOME/output/1.0 86 | - uses: actions/checkout@v2 87 | with: 88 | ref: release-1.1 89 | - name: Build release-1.1 90 | run: | 91 | GITHUB_SHA=$(git rev-parse HEAD) 92 | export GITHUB_SHA 93 | rm -rf _work/venv 94 | make vhtml 95 | mv _output/html $HOME/output/1.1 96 | - name: Deploy the docs 97 | run: | 98 | cd $HOME/output 99 | git init 100 | git config --global user.name "${GITHUB_ACTOR}" 101 | git config --global user.email "${GITHUB_ACTOR}@github.com" 102 | git add . 103 | git commit -m "latest html output" 104 | git push -f https://${GITHUB_ACTOR}:${{secrets.ACCESS_TOKEN}}@github.com/${GITHUB_REPOSITORY}.git HEAD:gh-pages 105 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /vendor 2 | /_output 3 | /_work 4 | /.tox 5 | Manifest 6 | /_build 7 | /deploy/olm-bundle 8 | /deploy/kustomize/olm-catalog/crd 9 | 10 | *.pyc -------------------------------------------------------------------------------- /PROJECT: -------------------------------------------------------------------------------- 1 | # Project configuration file used by controller-gen and operator-sdk tools 2 | { 3 | # Kubebuilder configuration version 4 | version: "3", 5 | domain: "pmem-csi.intel.com", 6 | repo: "github.com/intel/pmem-csi/pkg/pmem-csi-operator", 7 | projectName: "pmem-csi-operator" 8 | } 9 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | Intel is committed to rapidly addressing security vulnerabilities affecting our customers and providing clear guidance on the solution, impact, severity and mitigation. 3 | 4 | ## Reporting a Vulnerability 5 | Please report any security vulnerabilities in this project utilizing the guidelines [here](https://www.intel.com/content/www/us/en/security-center/vulnerability-handling-guidelines.html). 6 | -------------------------------------------------------------------------------- /build/Dockerfile: -------------------------------------------------------------------------------- 1 | ../Dockerfile -------------------------------------------------------------------------------- /cmd/manager: -------------------------------------------------------------------------------- 1 | ./pmem-csi-operator/ -------------------------------------------------------------------------------- /cmd/pmem-csi-driver/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 The Kubernetes Authors. 3 | Copyright 2018 Intel Coporation. 4 | 5 | SPDX-License-Identifier: Apache-2.0 6 | */ 7 | 8 | package main 9 | 10 | import ( 11 | "os" 12 | 13 | pmemcsidriver "github.com/intel/pmem-csi/pkg/pmem-csi-driver" 14 | ) 15 | 16 | func main() { 17 | os.Exit(pmemcsidriver.Main()) 18 | } 19 | -------------------------------------------------------------------------------- /cmd/pmem-csi-driver/main_test.go: -------------------------------------------------------------------------------- 1 | package main_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/intel/pmem-csi/pkg/coverage" 7 | pmemcsidriver "github.com/intel/pmem-csi/pkg/pmem-csi-driver" 8 | ) 9 | 10 | func TestMain(t *testing.T) { 11 | coverage.Run(pmemcsidriver.Main) 12 | } 13 | -------------------------------------------------------------------------------- /cmd/pmem-csi-operator/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | SPDX-License-Identifier: Apache-2.0 5 | */ 6 | 7 | package main 8 | 9 | import ( 10 | "os" 11 | 12 | pmemoperator "github.com/intel/pmem-csi/pkg/pmem-csi-operator" 13 | ) 14 | 15 | func main() { 16 | os.Exit(pmemoperator.Main()) 17 | } 18 | -------------------------------------------------------------------------------- /cmd/pmem-csi-operator/main_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | SPDX-License-Identifier: Apache-2.0 5 | */ 6 | 7 | package main_test 8 | 9 | import ( 10 | "testing" 11 | 12 | "github.com/intel/pmem-csi/pkg/coverage" 13 | pmemoperator "github.com/intel/pmem-csi/pkg/pmem-csi-operator" 14 | ) 15 | 16 | func TestMain(t *testing.T) { 17 | coverage.Run(pmemoperator.Main) 18 | } 19 | -------------------------------------------------------------------------------- /conf.json: -------------------------------------------------------------------------------- 1 | { 2 | "author": "", 3 | "copyright": "2021, Intel Corporation", 4 | "exclude_patterns": [ 5 | "_output", 6 | "Thumbs.db", 7 | ".DS_Store", 8 | ".tox", 9 | "_work", 10 | "deploy/kustomize", 11 | "third-party", 12 | "test/test-config.d" 13 | ], 14 | "extensions": [ 15 | "linkcheck2", 16 | "recommonmark", 17 | "sphinx_markdown_tables", 18 | "sphinx_md", 19 | "sphinx_copybutton" 20 | ], 21 | "linkcheck_ignore": [ 22 | "https://help.github.com/en/articles/creating-a-personal-access-token-for-the-command-line", 23 | "https://help.github.com/articles/using-pull-requests/", 24 | "https://intel.github.io/pmem-csi/1.1/", 25 | ".*cloudnative-k8sci.southcentralus.cloudapp.azure.com.*" 26 | ], 27 | "linkcheck_anchors_ignore": [ 28 | "^!", 29 | "^L[0-9]+-L[0-9]+$", 30 | "^discussion_r[0-9]+$" 31 | ], 32 | "tls_verify": false, 33 | "copybutton_prompt_text": "$ ", 34 | "html_theme": "sphinx_rtd_theme", 35 | "project": "PMEM-CSI", 36 | "templates_path": [ 37 | "_templates" 38 | ], 39 | "html_copy_source": false, 40 | "rst_epilog": ".. include:: /docs/substitutions.txt", 41 | "source_suffix": { 42 | ".rst": "restructuredtext", 43 | ".md": "markdown" 44 | }, 45 | "html_static_path":["docs/static"] 46 | } 47 | -------------------------------------------------------------------------------- /conf.py: -------------------------------------------------------------------------------- 1 | 2 | import json 3 | import os 4 | import sys 5 | import subprocess 6 | from os import getenv 7 | #support for modified code block 8 | from pygments.lexers.shell import BashSessionLexer 9 | from sphinx.highlighting import lexers 10 | 11 | sys.path.append(os.path.abspath("./docs/sphinx_ext")) 12 | 13 | ############# 14 | # 15 | # Add a special lexer to add a class to console lexer 16 | # 17 | ############# 18 | 19 | def setup(app): 20 | app.add_css_file("override.css") 21 | 22 | class copyAllConsole (BashSessionLexer): 23 | name = 'ShellSession' 24 | 25 | lexers['ShellSession'] = copyAllConsole(startinLine=True) 26 | 27 | # Get settings from conf.json 28 | 29 | with open('conf.json') as jsonFile: 30 | conf = json.load(jsonFile) 31 | 32 | for item in conf: 33 | globals()[item] = (conf[item]) 34 | 35 | # Dynamically determine the major version based on the branch name: 36 | # vx.y for release-x.y, "devel" for everything else 37 | branch = subprocess.check_output("git rev-parse --abbrev-ref HEAD".split(), encoding="utf-8") 38 | if branch.startswith("release-"): 39 | version = "v" + branch[len("release-"):].strip() 40 | else: 41 | version = "devel" 42 | release = version 43 | 44 | sphinx_md_useGitHubURL = True 45 | baseBranch = "devel" 46 | commitSHA = getenv('GITHUB_SHA') 47 | githubBaseURL = 'https://github.com/' + (getenv('GITHUB_REPOSITORY') or 'intel/pmem-csi') + '/' 48 | githubFileURL = githubBaseURL + "blob/" 49 | githubDirURL = githubBaseURL + "tree/" 50 | if commitSHA: 51 | githubFileURL = githubFileURL + commitSHA + "/" 52 | githubDirURL = githubDirURL + commitSHA + "/" 53 | else: 54 | githubFileURL = githubFileURL + baseBranch + "/" 55 | githubDirURL = githubDirURL + baseBranch + "/" 56 | sphinx_md_githubFileURL = githubFileURL 57 | sphinx_md_githubDirURL = githubDirURL 58 | -------------------------------------------------------------------------------- /deploy/common/pmem-app-block-volume.yaml: -------------------------------------------------------------------------------- 1 | kind: Pod 2 | apiVersion: v1 3 | metadata: 4 | name: my-csi-app 5 | spec: 6 | initContainers: 7 | # This init container is a workaround for https://github.com/kubernetes/kubernetes/issues/85624. 8 | - name: store-device 9 | image: ubuntu 10 | command: 11 | - "sh" 12 | - "-c" 13 | - "(echo '#!/bin/sh' && stat -c 'mknod /dev-xpmem b 0x%t 0x%T' /dev-xpmem) >/data/create-dev.sh && chmod a+x /data/create-dev.sh" 14 | volumeMounts: 15 | - name: data 16 | mountPath: /data 17 | volumeDevices: 18 | - name: my-csi-device 19 | devicePath: /dev-xpmem 20 | containers: 21 | - name: my-frontend 22 | image: ubuntu 23 | securityContext: 24 | privileged: True 25 | command: 26 | - "sh" 27 | - "-c" 28 | # mkfs.ext4 may fail here if the volume was already formatted before, so we ignore the return code. 29 | - "if [ ! -e /dev-xpmem ]; then /data/create-dev.sh; fi && mkfs.ext4 -b 4096 /dev-xpmem; mkdir -p /mnt && mount -odax /dev-xpmem /mnt && mount | grep /mnt | grep dax && sleep 100000" 30 | volumeMounts: 31 | - name: data 32 | mountPath: /data 33 | volumes: 34 | - name: my-csi-device 35 | persistentVolumeClaim: 36 | claimName: pmem-csi-pvc-block-volume 37 | - name: data 38 | emptyDir: 39 | -------------------------------------------------------------------------------- /deploy/common/pmem-app-ephemeral.yaml: -------------------------------------------------------------------------------- 1 | # This is an example Pod definition to demonstrates 2 | # how to use inline PMEM csi volumes. 3 | kind: Pod 4 | apiVersion: v1 5 | metadata: 6 | name: my-csi-app-inline-volume 7 | spec: 8 | containers: 9 | - name: my-frontend 10 | image: intel/pmem-csi-driver-test:canary 11 | command: [ "sleep", "100000" ] 12 | volumeMounts: 13 | - mountPath: "/data" 14 | name: my-csi-volume 15 | volumes: 16 | - name: my-csi-volume 17 | csi: 18 | driver: pmem-csi.intel.com 19 | fsType: "xfs" 20 | volumeAttributes: 21 | size: "2Gi" 22 | 23 | -------------------------------------------------------------------------------- /deploy/common/pmem-app-generic-ephemeral.yaml: -------------------------------------------------------------------------------- 1 | # This example Pod definition demonstrates 2 | # how to use generic ephemeral inline volumes 3 | # with a PMEM-CSI storage class. 4 | kind: Pod 5 | apiVersion: v1 6 | metadata: 7 | name: my-csi-app-inline-volume 8 | spec: 9 | containers: 10 | - name: my-frontend 11 | image: intel/pmem-csi-driver-test:canary 12 | command: [ "sleep", "100000" ] 13 | volumeMounts: 14 | - mountPath: "/data" 15 | name: my-csi-volume 16 | volumes: 17 | - name: my-csi-volume 18 | ephemeral: 19 | volumeClaimTemplate: 20 | spec: 21 | accessModes: 22 | - ReadWriteOnce 23 | resources: 24 | requests: 25 | storage: 4Gi 26 | storageClassName: pmem-csi-sc-late-binding 27 | -------------------------------------------------------------------------------- /deploy/common/pmem-app-late-binding.yaml: -------------------------------------------------------------------------------- 1 | kind: Pod 2 | apiVersion: v1 3 | metadata: 4 | name: my-csi-app 5 | spec: 6 | containers: 7 | - name: my-frontend 8 | image: intel/pmem-csi-driver-test:canary 9 | command: [ "sleep", "100000" ] 10 | volumeMounts: 11 | - mountPath: "/data" 12 | name: my-csi-volume 13 | volumes: 14 | - name: my-csi-volume 15 | persistentVolumeClaim: 16 | claimName: pmem-csi-pvc-late-binding 17 | -------------------------------------------------------------------------------- /deploy/common/pmem-app.yaml: -------------------------------------------------------------------------------- 1 | kind: Pod 2 | apiVersion: v1 3 | metadata: 4 | name: my-csi-app-1 5 | spec: 6 | containers: 7 | - name: my-frontend 8 | image: intel/pmem-csi-driver-test:canary 9 | command: [ "sleep", "100000" ] 10 | volumeMounts: 11 | - mountPath: "/data" 12 | name: my-csi-volume 13 | volumes: 14 | - name: my-csi-volume 15 | persistentVolumeClaim: 16 | claimName: pmem-csi-pvc-ext4 17 | --- 18 | kind: Pod 19 | apiVersion: v1 20 | metadata: 21 | name: my-csi-app-2 22 | spec: 23 | containers: 24 | - name: my-frontend 25 | image: intel/pmem-csi-driver-test:canary 26 | command: [ "sleep", "100000" ] 27 | volumeMounts: 28 | - mountPath: "/data" 29 | name: my-csi-volume 30 | volumes: 31 | - name: my-csi-volume 32 | persistentVolumeClaim: 33 | claimName: pmem-csi-pvc-xfs 34 | -------------------------------------------------------------------------------- /deploy/common/pmem-csi.intel.com_v1beta1_pmemcsideployment_cr.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: pmem-csi.intel.com/v1beta1 2 | kind: PmemCSIDeployment 3 | metadata: 4 | name: pmem-csi.intel.com 5 | spec: 6 | deviceMode: "lvm" 7 | nodeSelector: 8 | # When using Node Feature Discovery (NFD): 9 | feature.node.kubernetes.io/memory-nv.dax: "true" 10 | # When using manual node labeling with that label: 11 | # storage: pmem 12 | 13 | -------------------------------------------------------------------------------- /deploy/common/pmem-kata-app-ephemeral.yaml: -------------------------------------------------------------------------------- 1 | kind: Pod 2 | apiVersion: v1 3 | metadata: 4 | name: my-csi-app-inline-volume 5 | labels: 6 | io.katacontainers.config.hypervisor.memory_offset: "2147483648" # 2Gi, must be at least as large as the PMEM volume 7 | spec: 8 | # see https://github.com/kata-containers/packaging/tree/1.11.0-rc0/kata-deploy#run-a-sample-workload 9 | runtimeClassName: kata-qemu 10 | nodeSelector: 11 | katacontainers.io/kata-runtime: "true" 12 | containers: 13 | - name: my-frontend 14 | image: intel/pmem-csi-driver-test:canary 15 | command: [ "sleep", "100000" ] 16 | volumeMounts: 17 | - mountPath: "/data" 18 | name: my-csi-volume 19 | volumes: 20 | - name: my-csi-volume 21 | csi: 22 | driver: pmem-csi.intel.com 23 | fsType: "xfs" 24 | volumeAttributes: 25 | size: "2Gi" 26 | kataContainers: "true" 27 | -------------------------------------------------------------------------------- /deploy/common/pmem-kata-app.yaml: -------------------------------------------------------------------------------- 1 | kind: Pod 2 | apiVersion: v1 3 | metadata: 4 | name: my-csi-kata-app 5 | annotations: 6 | io.katacontainers.config.hypervisor.memory_offset: "2147483648" # 2Gi, must be at least as large as the PMEM volume 7 | spec: 8 | # see https://github.com/kata-containers/packaging/tree/1.11.0-rc0/kata-deploy#run-a-sample-workload 9 | runtimeClassName: kata-qemu 10 | nodeSelector: 11 | katacontainers.io/kata-runtime: "true" 12 | containers: 13 | - name: my-frontend 14 | image: intel/pmem-csi-driver-test:canary 15 | command: [ "sleep", "100000" ] 16 | volumeMounts: 17 | - mountPath: "/data" 18 | name: my-csi-volume 19 | volumes: 20 | - name: my-csi-volume 21 | persistentVolumeClaim: 22 | claimName: pmem-csi-pvc-kata # see pmem-kata-pvc.yaml 23 | -------------------------------------------------------------------------------- /deploy/common/pmem-kata-pvc.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: PersistentVolumeClaim 3 | metadata: 4 | name: pmem-csi-pvc-kata 5 | spec: 6 | accessModes: 7 | - ReadWriteOnce 8 | resources: 9 | requests: 10 | storage: 4Gi 11 | storageClassName: pmem-csi-sc-ext4-kata # defined in pmem-storageclass-ext4-kata.yaml 12 | -------------------------------------------------------------------------------- /deploy/common/pmem-pvc-block-volume.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: PersistentVolumeClaim 3 | metadata: 4 | name: pmem-csi-pvc-block-volume 5 | spec: 6 | volumeMode: Block 7 | accessModes: 8 | - ReadWriteOnce 9 | resources: 10 | requests: 11 | storage: 8Gi 12 | storageClassName: pmem-csi-sc-ext4 # defined in deploy/kubernetes-{VERSION}/pmem-storageclass-ext4.yaml 13 | -------------------------------------------------------------------------------- /deploy/common/pmem-pvc-late-binding.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: PersistentVolumeClaim 3 | metadata: 4 | name: pmem-csi-pvc-late-binding 5 | spec: 6 | accessModes: 7 | - ReadWriteOnce 8 | resources: 9 | requests: 10 | storage: 4Gi 11 | storageClassName: pmem-csi-sc-late-binding # defined in pmem-storageclass-late-binding.yaml 12 | -------------------------------------------------------------------------------- /deploy/common/pmem-pvc.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: PersistentVolumeClaim 3 | metadata: 4 | name: pmem-csi-pvc-ext4 5 | spec: 6 | accessModes: 7 | - ReadWriteOnce 8 | resources: 9 | requests: 10 | storage: 4Gi 11 | storageClassName: pmem-csi-sc-ext4 # defined in pmem-storageclass-ext4.yaml 12 | --- 13 | apiVersion: v1 14 | kind: PersistentVolumeClaim 15 | metadata: 16 | name: pmem-csi-pvc-xfs 17 | spec: 18 | accessModes: 19 | - ReadWriteOnce 20 | resources: 21 | requests: 22 | storage: 4Gi 23 | storageClassName: pmem-csi-sc-xfs # defined in pmem-storageclass-xfs.yaml 24 | -------------------------------------------------------------------------------- /deploy/common/pmem-storageclass-default.yaml: -------------------------------------------------------------------------------- 1 | # Generated with "make kustomize", do not edit! 2 | 3 | apiVersion: storage.k8s.io/v1 4 | kind: StorageClass 5 | metadata: 6 | name: pmem-csi-sc 7 | provisioner: pmem-csi.intel.com 8 | -------------------------------------------------------------------------------- /deploy/common/pmem-storageclass-ext4-fileio.yaml: -------------------------------------------------------------------------------- 1 | # Generated with "make kustomize", do not edit! 2 | 3 | apiVersion: storage.k8s.io/v1 4 | kind: StorageClass 5 | metadata: 6 | name: pmem-csi-sc-ext4-fileio 7 | parameters: 8 | csi.storage.k8s.io/fstype: ext4 9 | eraseafter: "true" 10 | usage: FileIO 11 | provisioner: pmem-csi.intel.com 12 | -------------------------------------------------------------------------------- /deploy/common/pmem-storageclass-ext4-kata.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: storage.k8s.io/v1 2 | kind: StorageClass 3 | metadata: 4 | name: pmem-csi-sc-ext4-kata 5 | parameters: 6 | csi.storage.k8s.io/fstype: ext4 7 | eraseafter: "true" 8 | kataContainers: "true" 9 | provisioner: pmem-csi.intel.com 10 | reclaimPolicy: Delete 11 | # Kata Containers might not be available on all nodes, wait for pod scheduling 12 | # and then create volume on the chosen node(s). 13 | volumeBindingMode: WaitForFirstConsumer 14 | -------------------------------------------------------------------------------- /deploy/common/pmem-storageclass-ext4.yaml: -------------------------------------------------------------------------------- 1 | # Generated with "make kustomize", do not edit! 2 | 3 | apiVersion: storage.k8s.io/v1 4 | kind: StorageClass 5 | metadata: 6 | name: pmem-csi-sc-ext4 7 | parameters: 8 | csi.storage.k8s.io/fstype: ext4 9 | eraseafter: "true" 10 | provisioner: pmem-csi.intel.com 11 | -------------------------------------------------------------------------------- /deploy/common/pmem-storageclass-late-binding.yaml: -------------------------------------------------------------------------------- 1 | # Generated with "make kustomize", do not edit! 2 | 3 | apiVersion: storage.k8s.io/v1 4 | kind: StorageClass 5 | metadata: 6 | name: pmem-csi-sc-late-binding 7 | provisioner: pmem-csi.intel.com 8 | volumeBindingMode: WaitForFirstConsumer 9 | -------------------------------------------------------------------------------- /deploy/common/pmem-storageclass-xfs-fileio.yaml: -------------------------------------------------------------------------------- 1 | # Generated with "make kustomize", do not edit! 2 | 3 | apiVersion: storage.k8s.io/v1 4 | kind: StorageClass 5 | metadata: 6 | name: pmem-csi-sc-xfs-fileio 7 | parameters: 8 | csi.storage.k8s.io/fstype: xfs 9 | eraseafter: "false" 10 | usage: FileIO 11 | provisioner: pmem-csi.intel.com 12 | -------------------------------------------------------------------------------- /deploy/common/pmem-storageclass-xfs-kata.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: storage.k8s.io/v1 2 | kind: StorageClass 3 | metadata: 4 | name: pmem-csi-sc-xfs-kata 5 | parameters: 6 | csi.storage.k8s.io/fstype: xfs 7 | eraseafter: "true" 8 | kataContainers: "true" 9 | provisioner: pmem-csi.intel.com 10 | reclaimPolicy: Delete 11 | # Kata Containers might not be available on all nodes, wait for pod scheduling 12 | # and then create volume on the chosen node(s). 13 | volumeBindingMode: WaitForFirstConsumer 14 | -------------------------------------------------------------------------------- /deploy/common/pmem-storageclass-xfs.yaml: -------------------------------------------------------------------------------- 1 | # Generated with "make kustomize", do not edit! 2 | 3 | apiVersion: storage.k8s.io/v1 4 | kind: StorageClass 5 | metadata: 6 | name: pmem-csi-sc-xfs 7 | parameters: 8 | csi.storage.k8s.io/fstype: xfs 9 | eraseafter: "false" 10 | provisioner: pmem-csi.intel.com 11 | -------------------------------------------------------------------------------- /deploy/kubernetes-1.21/direct/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: [ pmem-csi.yaml ] 2 | -------------------------------------------------------------------------------- /deploy/kubernetes-1.21/direct/testing/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: [ pmem-csi.yaml ] 2 | -------------------------------------------------------------------------------- /deploy/kubernetes-1.21/lvm/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: [ pmem-csi.yaml ] 2 | -------------------------------------------------------------------------------- /deploy/kubernetes-1.21/lvm/testing/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: [ pmem-csi.yaml ] 2 | -------------------------------------------------------------------------------- /deploy/kubernetes-1.22/direct/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: [ pmem-csi.yaml ] 2 | -------------------------------------------------------------------------------- /deploy/kubernetes-1.22/direct/testing/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: [ pmem-csi.yaml ] 2 | -------------------------------------------------------------------------------- /deploy/kubernetes-1.22/lvm/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: [ pmem-csi.yaml ] 2 | -------------------------------------------------------------------------------- /deploy/kubernetes-1.22/lvm/testing/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: [ pmem-csi.yaml ] 2 | -------------------------------------------------------------------------------- /deploy/kubernetes-1.23/direct/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: [ pmem-csi.yaml ] 2 | -------------------------------------------------------------------------------- /deploy/kubernetes-1.23/direct/testing/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: [ pmem-csi.yaml ] 2 | -------------------------------------------------------------------------------- /deploy/kubernetes-1.23/lvm/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: [ pmem-csi.yaml ] 2 | -------------------------------------------------------------------------------- /deploy/kubernetes-1.23/lvm/testing/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: [ pmem-csi.yaml ] 2 | -------------------------------------------------------------------------------- /deploy/kubernetes-1.24/direct/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: [ pmem-csi.yaml ] 2 | -------------------------------------------------------------------------------- /deploy/kubernetes-1.24/direct/testing/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: [ pmem-csi.yaml ] 2 | -------------------------------------------------------------------------------- /deploy/kubernetes-1.24/lvm/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: [ pmem-csi.yaml ] 2 | -------------------------------------------------------------------------------- /deploy/kubernetes-1.24/lvm/testing/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: [ pmem-csi.yaml ] 2 | -------------------------------------------------------------------------------- /deploy/kubernetes-1.25/direct/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: [ pmem-csi.yaml ] 2 | -------------------------------------------------------------------------------- /deploy/kubernetes-1.25/direct/testing/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: [ pmem-csi.yaml ] 2 | -------------------------------------------------------------------------------- /deploy/kubernetes-1.25/lvm/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: [ pmem-csi.yaml ] 2 | -------------------------------------------------------------------------------- /deploy/kubernetes-1.25/lvm/testing/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: [ pmem-csi.yaml ] 2 | -------------------------------------------------------------------------------- /deploy/kustomize/driver/README.md: -------------------------------------------------------------------------------- 1 | # Driver 2 | 3 | The common parts for a PMEM-CSI driver deployment. Image versions and 4 | additional parameters for LVM vs. direct mode will be added in 5 | overlays. 6 | -------------------------------------------------------------------------------- /deploy/kustomize/driver/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - pmem-csi.yaml 3 | -------------------------------------------------------------------------------- /deploy/kustomize/kubernetes-base-direct-coverage/kustomization.yaml: -------------------------------------------------------------------------------- 1 | bases: 2 | - ../kubernetes-base-direct/ 3 | 4 | patchesJson6902: 5 | - target: 6 | group: apps 7 | version: v1 8 | kind: Deployment 9 | name: pmem-csi-intel-com-controller 10 | path: ../testing/controller-coverage-patch.yaml 11 | 12 | - target: 13 | group: apps 14 | version: v1 15 | kind: DaemonSet 16 | name: pmem-csi-intel-com-node 17 | path: ../testing/node-coverage-patch.yaml 18 | 19 | images: 20 | - name: intel/pmem-csi-driver 21 | newName: intel/pmem-csi-driver-test 22 | -------------------------------------------------------------------------------- /deploy/kustomize/kubernetes-base-direct-testing-coverage/kustomization.yaml: -------------------------------------------------------------------------------- 1 | bases: 2 | - ../kubernetes-base-direct-testing/ 3 | 4 | patchesJson6902: 5 | - target: 6 | group: apps 7 | version: v1 8 | kind: Deployment 9 | name: pmem-csi-intel-com-controller 10 | path: ../testing/controller-coverage-patch.yaml 11 | 12 | - target: 13 | group: apps 14 | version: v1 15 | kind: DaemonSet 16 | name: pmem-csi-intel-com-node 17 | path: ../testing/node-coverage-patch.yaml 18 | 19 | images: 20 | - name: intel/pmem-csi-driver 21 | newName: intel/pmem-csi-driver-test 22 | -------------------------------------------------------------------------------- /deploy/kustomize/kubernetes-base-direct-testing/kustomization.yaml: -------------------------------------------------------------------------------- 1 | bases: 2 | - ../kubernetes-base-direct/ 3 | - ../testing/ 4 | 5 | commonLabels: 6 | pmem-csi.intel.com/deployment: direct-testing 7 | 8 | patchesJson6902: 9 | - target: 10 | group: apps 11 | version: v1 12 | kind: Deployment 13 | name: pmem-csi-intel-com-controller 14 | path: ../testing/controller-verbosity-patch.yaml 15 | 16 | - target: 17 | group: apps 18 | version: v1 19 | kind: DaemonSet 20 | name: pmem-csi-intel-com-node 21 | path: ../testing/node-verbosity-patch.yaml 22 | -------------------------------------------------------------------------------- /deploy/kustomize/kubernetes-base-direct/README.md: -------------------------------------------------------------------------------- 1 | # Kubernetes v1.16 Direct Mode specific changes 2 | 3 | This overlay configures the driver to use direct mode. 4 | -------------------------------------------------------------------------------- /deploy/kustomize/kubernetes-base-direct/kustomization.yaml: -------------------------------------------------------------------------------- 1 | # Turns the base deployment with metrics into a deployment for direct mode. 2 | 3 | bases: 4 | - ../kubernetes-with-metrics 5 | 6 | commonLabels: 7 | pmem-csi.intel.com/deployment: direct-production 8 | 9 | patchesJson6902: 10 | - target: 11 | group: apps 12 | version: v1 13 | kind: DaemonSet 14 | name: pmem-csi-intel-com-node 15 | path: ../patches/direct-patch.yaml 16 | -------------------------------------------------------------------------------- /deploy/kustomize/kubernetes-base-fake/fake-device-mode-patch.yaml: -------------------------------------------------------------------------------- 1 | - op: add 2 | path: /spec/template/spec/containers/0/command/- 3 | value: "-deviceManager=fake" 4 | -------------------------------------------------------------------------------- /deploy/kustomize/kubernetes-base-fake/kustomization.yaml: -------------------------------------------------------------------------------- 1 | bases: 2 | - ../kubernetes-base-direct 3 | 4 | commonLabels: 5 | pmem-csi.intel.com/deployment: fake-production 6 | 7 | patchesJson6902: 8 | - target: 9 | group: apps 10 | version: v1 11 | kind: DaemonSet 12 | name: pmem-csi-intel-com-node 13 | path: fake-device-mode-patch.yaml 14 | -------------------------------------------------------------------------------- /deploy/kustomize/kubernetes-base-lvm-coverage/kustomization.yaml: -------------------------------------------------------------------------------- 1 | bases: 2 | - ../kubernetes-base-lvm/ 3 | 4 | patchesJson6902: 5 | - target: 6 | group: apps 7 | version: v1 8 | kind: Deployment 9 | name: pmem-csi-intel-com-controller 10 | path: ../testing/controller-coverage-patch.yaml 11 | 12 | - target: 13 | group: apps 14 | version: v1 15 | kind: DaemonSet 16 | name: pmem-csi-intel-com-node 17 | path: ../testing/node-coverage-patch.yaml 18 | 19 | images: 20 | - name: intel/pmem-csi-driver 21 | newName: intel/pmem-csi-driver-test 22 | -------------------------------------------------------------------------------- /deploy/kustomize/kubernetes-base-lvm-testing-coverage/kustomization.yaml: -------------------------------------------------------------------------------- 1 | bases: 2 | - ../kubernetes-base-lvm-testing/ 3 | 4 | patchesJson6902: 5 | - target: 6 | group: apps 7 | version: v1 8 | kind: Deployment 9 | name: pmem-csi-intel-com-controller 10 | path: ../testing/controller-coverage-patch.yaml 11 | 12 | - target: 13 | group: apps 14 | version: v1 15 | kind: DaemonSet 16 | name: pmem-csi-intel-com-node 17 | path: ../testing/node-coverage-patch.yaml 18 | 19 | images: 20 | - name: intel/pmem-csi-driver 21 | newName: intel/pmem-csi-driver-test 22 | -------------------------------------------------------------------------------- /deploy/kustomize/kubernetes-base-lvm-testing/kustomization.yaml: -------------------------------------------------------------------------------- 1 | bases: 2 | - ../kubernetes-base-lvm/ 3 | - ../testing/ 4 | 5 | commonLabels: 6 | pmem-csi.intel.com/deployment: lvm-testing 7 | 8 | patchesJson6902: 9 | - target: 10 | group: apps 11 | version: v1 12 | kind: Deployment 13 | name: pmem-csi-intel-com-controller 14 | path: ../testing/controller-verbosity-patch.yaml 15 | 16 | - target: 17 | group: apps 18 | version: v1 19 | kind: DaemonSet 20 | name: pmem-csi-intel-com-node 21 | path: ../testing/node-verbosity-patch.yaml 22 | -------------------------------------------------------------------------------- /deploy/kustomize/kubernetes-base-lvm/README.md: -------------------------------------------------------------------------------- 1 | # Kubernetes v1.16 LVM Mode specific changes 2 | 3 | This overlay configures the driver to use LVM mode. 4 | -------------------------------------------------------------------------------- /deploy/kustomize/kubernetes-base-lvm/kustomization.yaml: -------------------------------------------------------------------------------- 1 | # Turns the base deployment with metrics support into a deployment for LVM mode. 2 | 3 | bases: 4 | - ../kubernetes-with-metrics 5 | 6 | commonLabels: 7 | pmem-csi.intel.com/deployment: lvm-production 8 | 9 | patchesJson6902: 10 | - target: 11 | group: apps 12 | version: v1 13 | kind: DaemonSet 14 | name: pmem-csi-intel-com-node 15 | path: ../patches/lvm-patch.yaml 16 | -------------------------------------------------------------------------------- /deploy/kustomize/kubernetes-no-metrics/README.md: -------------------------------------------------------------------------------- 1 | # Kubernetes v1.16 specific changes 2 | 3 | This overlay adds the actual version numbers of the sidecars and the 4 | corresponding RBAC rules such that the base deployment matches 5 | https://github.com/kubernetes-csi/csi-driver-host-path/tree/master/deploy/kubernetes-1.16 6 | -------------------------------------------------------------------------------- /deploy/kustomize/kubernetes-no-metrics/driverinfo-beta.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: storage.k8s.io/v1 2 | kind: CSIDriver 3 | metadata: 4 | name: pmem-csi.intel.com 5 | spec: 6 | attachRequired: false 7 | podInfoOnMount: true 8 | storageCapacity: true # beta in 1.21, GA in 1.23 9 | volumeLifecycleModes: 10 | - Persistent 11 | - Ephemeral 12 | -------------------------------------------------------------------------------- /deploy/kustomize/kubernetes-no-metrics/kustomization.yaml: -------------------------------------------------------------------------------- 1 | bases: 2 | - ../driver 3 | - rbac 4 | 5 | resources: 6 | - driverinfo-beta.yaml 7 | 8 | # The RBAC files must match the image versions. 9 | # The sidecar versions must be kept in sync with the 10 | # operator defaults in pkg/apis/pmemcsi/v1beta1/deployment_types.go! 11 | # hack/bump-image-versions.sh can be used to update both. 12 | images: 13 | - name: k8s.gcr.io/sig-storage/csi-provisioner 14 | newTag: v2.2.2 15 | - name: k8s.gcr.io/sig-storage/csi-node-driver-registrar 16 | newTag: v2.2.0 17 | 18 | # Keep all the driver objects in 'pmem-csi' namespace 19 | namespace: pmem-csi 20 | -------------------------------------------------------------------------------- /deploy/kustomize/kubernetes-no-metrics/rbac/README.md: -------------------------------------------------------------------------------- 1 | # Kubernetes v1.16 RBAC specific changes 2 | 3 | These are the necessary RBAC rules for the sidecar containers that we 4 | use for Kubernetes 1.16. 5 | -------------------------------------------------------------------------------- /deploy/kustomize/kubernetes-no-metrics/rbac/kustomization.yaml: -------------------------------------------------------------------------------- 1 | namePrefix: pmem-csi-intel-com- 2 | 3 | resources: 4 | - https://github.com/kubernetes-csi/external-provisioner/raw/v3.2.1/deploy/kubernetes/rbac.yaml 5 | 6 | # We define our own service account. 7 | patchesStrategicMerge: 8 | - ../../patches/delete-csi-provisioner-service-account-patch.yaml 9 | 10 | # We use the upstream [Cluster]RoleBinding and just replace 11 | # the account name. 12 | patchesJson6902: 13 | - target: 14 | group: rbac.authorization.k8s.io 15 | version: v1 16 | kind: ClusterRoleBinding 17 | name: csi-provisioner-role 18 | path: ../../patches/controller-role-patch.yaml 19 | - target: 20 | group: rbac.authorization.k8s.io 21 | version: v1 22 | kind: RoleBinding 23 | name: csi-provisioner-role-cfg 24 | path: ../../patches/controller-role-patch.yaml 25 | - target: 26 | group: rbac.authorization.k8s.io 27 | version: v1 28 | kind: ClusterRole 29 | name: external-provisioner-runner 30 | patch: |- 31 | # PMEM-CSI uses "skip attach" and thus external-provisioner doesn't 32 | # need to watch VolumeAttachment objects as there never should be 33 | # any that are relevant for it. We can avoid the overhead (in case 34 | # other drivers cause the creation of such objects) and potential bugs 35 | # in Kubernetes (during downgrade testing, "skip attach" did not work 36 | # properly, potentially because the CSIDriver object wasn't known yet 37 | # directly after installing the driver). 38 | # 39 | # There is currently no command line option to remove the watch, 40 | # but we can disable it indirectly by removing the RBAC permissions. 41 | # However, that then triggers some runtime warning. 42 | # 43 | # We know that the "volumeattachments" resource is listed as last element. 44 | - op: remove 45 | path: /rules/8 46 | -------------------------------------------------------------------------------- /deploy/kustomize/kubernetes-with-metrics/kustomization.yaml: -------------------------------------------------------------------------------- 1 | # Enables metrics support in the base deployment. 2 | 3 | bases: 4 | - ../kubernetes-no-metrics 5 | 6 | patchesJson6902: 7 | - target: 8 | group: apps 9 | version: v1 10 | kind: DaemonSet 11 | name: pmem-csi-intel-com-node 12 | path: ../patches/metrics-node.yaml 13 | 14 | - target: 15 | group: apps 16 | version: v1 17 | kind: Deployment 18 | name: pmem-csi-intel-com-controller 19 | path: ../patches/metrics-controller.yaml 20 | -------------------------------------------------------------------------------- /deploy/kustomize/memcached/ephemeral/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - memcached-ephemeral.yaml 3 | -------------------------------------------------------------------------------- /deploy/kustomize/memcached/ephemeral/memcached-ephemeral.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: pmem-memcached 5 | namespace: default 6 | spec: 7 | ports: 8 | - name: db 9 | port: 11211 10 | protocol: TCP 11 | targetPort: db 12 | selector: 13 | app.kubernetes.io/kind: memcached 14 | app.kubernetes.io/name: pmem-memcached 15 | type: ClusterIP 16 | --- 17 | apiVersion: apps/v1 18 | kind: Deployment 19 | metadata: 20 | name: pmem-memcached 21 | namespace: default 22 | spec: 23 | replicas: 1 24 | selector: 25 | matchLabels: 26 | app.kubernetes.io/kind: memcached 27 | app.kubernetes.io/name: pmem-memcached 28 | strategy: 29 | rollingUpdate: 30 | maxSurge: 25% 31 | maxUnavailable: 25% 32 | type: RollingUpdate 33 | template: 34 | metadata: 35 | labels: 36 | app.kubernetes.io/kind: memcached 37 | app.kubernetes.io/name: pmem-memcached 38 | spec: 39 | initContainers: 40 | - name: data-volume-init 41 | # This first creates a file that is as large as root can make it (larger 42 | # than a normal user because of reserved blocks). Size is rounded down to 43 | # MiB because that is what memcached expects and 50 MiB are substracted 44 | # for filesystem overhead. 45 | # 46 | # Then it changes the ownership of the data volume so that the memcached user 47 | # can read and write files there. The exact UID varies between 48 | # memcached image versions (therefore we cannot use Kubernetes to change 49 | # the ownership for us as that relies on a numeric value), but the "memcache" 50 | # name is the same, so by running in the same image as memcached we can set 51 | # the owner by name. 52 | command: 53 | - sh 54 | - -c 55 | - | 56 | fallocate --length=$(( $(stat --file-system --format='%b * %s / 1024 / 1024 - 50' /data) ))MiB /data/memcached-memory-file && 57 | chown -R memcache /data 58 | image: memcached:1.5.22 59 | securityContext: 60 | privileged: true 61 | runAsUser: 0 62 | volumeMounts: 63 | - mountPath: /data 64 | name: data-volume 65 | containers: 66 | - name: memcached 67 | image: memcached:1.5.22 68 | command: 69 | - sh 70 | - -c 71 | - exec memcached --memory-limit=$(( $(stat --format='%s / 1024 / 1024' /data/memcached-memory-file) )) --memory-file=/data/memcached-memory-file $(MEMCACHED_ARGS) 72 | ports: 73 | - containerPort: 11211 74 | name: db 75 | protocol: TCP 76 | volumeMounts: 77 | - mountPath: /data 78 | name: data-volume 79 | env: 80 | - name: MEMCACHED_ARGS 81 | value: 82 | volumes: 83 | - name: data-volume 84 | csi: 85 | driver: pmem-csi.intel.com 86 | volumeAttributes: 87 | size: 200Mi 88 | -------------------------------------------------------------------------------- /deploy/kustomize/memcached/persistent/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - memcached-persistent.yaml 3 | -------------------------------------------------------------------------------- /deploy/kustomize/memcached/persistent/memcached-persistent.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: pmem-memcached 5 | namespace: default 6 | spec: 7 | ports: 8 | - name: db 9 | port: 11211 10 | protocol: TCP 11 | targetPort: db 12 | selector: 13 | app.kubernetes.io/kind: memcached 14 | app.kubernetes.io/name: pmem-memcached 15 | type: ClusterIP 16 | --- 17 | apiVersion: apps/v1 18 | kind: StatefulSet 19 | metadata: 20 | name: pmem-memcached 21 | namespace: default 22 | spec: 23 | replicas: 1 24 | selector: 25 | matchLabels: 26 | app.kubernetes.io/kind: memcached 27 | app.kubernetes.io/name: pmem-memcached 28 | serviceName: pmem-memcached 29 | template: 30 | metadata: 31 | labels: 32 | app.kubernetes.io/kind: memcached 33 | app.kubernetes.io/name: pmem-memcached 34 | spec: 35 | initContainers: 36 | - name: data-volume-init 37 | # This first creates a file that is as large as root can make it (larger 38 | # than a normal user because of reserved blocks). Size is rounded down to 39 | # MiB because that is what memcached expects and 50 MiB are substracted 40 | # for filesystem overhead and the memcached state files. 41 | # 42 | # Then it changes the ownership of the data volume so that the memcached user 43 | # can read and write files there. The exact UID varies between 44 | # memcached image versions (therefore we cannot use Kubernetes to change 45 | # the ownership for us as that relies on a numeric value), but the "memcache" 46 | # name is the same, so by running in the same image as memcached we can set 47 | # the owner by name. 48 | command: 49 | - sh 50 | - -c 51 | - | 52 | fallocate --length=$(( $(stat --file-system --format='%b * %s / 1024 / 1024 - 50' /data) ))MiB /data/memcached-memory-file && 53 | chown -R memcache /data 54 | image: memcached:1.5.22 55 | securityContext: 56 | privileged: true 57 | runAsUser: 0 58 | volumeMounts: 59 | - mountPath: /data 60 | name: memcached-data-volume 61 | containers: 62 | - name: memcached 63 | image: memcached:1.5.22 64 | command: 65 | - sh 66 | - -c 67 | - | 68 | set -x 69 | memcached --memory-limit=$(( $(stat --format='%s / 1024 / 1024' /data/memcached-memory-file) )) --memory-file=/data/memcached-memory-file $(MEMCACHED_ARGS) & 70 | pid=$$! 71 | trap 'kill -USR1 $$pid; wait $$pid' TERM 72 | wait $$pid 73 | ports: 74 | - containerPort: 11211 75 | name: db 76 | protocol: TCP 77 | volumeMounts: 78 | - mountPath: /data 79 | name: memcached-data-volume 80 | env: 81 | - name: MEMCACHED_ARGS 82 | value: 83 | terminationGracePeriodSeconds: 30 84 | volumeClaimTemplates: 85 | - metadata: 86 | name: memcached-data-volume 87 | spec: 88 | accessModes: [ "ReadWriteOnce" ] 89 | storageClassName: pmem-csi-sc-ext4 90 | resources: 91 | requests: 92 | storage: 200Mi 93 | -------------------------------------------------------------------------------- /deploy/kustomize/metrics-server/kustomization.yaml: -------------------------------------------------------------------------------- 1 | bases: 2 | - https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml 3 | 4 | # Inline patches are supported by recent kustomize, but not necessarily kubectl. 5 | patchesJSON6902: 6 | - target: 7 | group: apps 8 | version: v1 9 | kind: Deployment 10 | name: metrics-server 11 | namespace: kube-system 12 | patch: |- 13 | - op: add 14 | path: /spec/template/spec/containers/0/args/- 15 | # Needed because of https://github.com/kubernetes/kubeadm/issues/2028 16 | value: --kubelet-insecure-tls 17 | 18 | -------------------------------------------------------------------------------- /deploy/kustomize/olm-catalog/kustomization.yaml: -------------------------------------------------------------------------------- 1 | bases: 2 | - ../operator 3 | 4 | resources: 5 | - ../../common/pmem-csi.intel.com_v1beta1_pmemcsideployment_cr.yaml 6 | - ../../crd/pmem-csi.intel.com_pmemcsideployments.yaml 7 | 8 | images: 9 | - name: intel/pmem-csi-driver 10 | # this version will be replaced during make operator-generate-catalog with the actual version number 11 | newTag: vX.Y.Z 12 | 13 | -------------------------------------------------------------------------------- /deploy/kustomize/operator/README.md: -------------------------------------------------------------------------------- 1 | # Operator 2 | 3 | The common parts for a PMEM-CSI operator deployment. Additional layers 4 | could be added to customize the base operator deployment. 5 | -------------------------------------------------------------------------------- /deploy/kustomize/operator/events-rbac.yaml: -------------------------------------------------------------------------------- 1 | # 2 | # RBAC rules required for the operator to generate 3 | # deployment events. 4 | # Events are namespace scoped. PMEM-CSI Deployment CR is 5 | # cluster-scoped. Events on cluster-scoped objects are 6 | # generated in the default namespace. 7 | # 8 | # Hence, these RBAC rules are required for the operator 9 | # to generate events in default namespace even if the 10 | # operator deployed in different namespace. 11 | # 12 | apiVersion: rbac.authorization.k8s.io/v1 13 | kind: Role 14 | metadata: 15 | name: pmem-csi-operator-event 16 | rules: 17 | - apiGroups: 18 | - "" 19 | resources: 20 | - events 21 | verbs: 22 | - '*' 23 | --- 24 | apiVersion: rbac.authorization.k8s.io/v1 25 | kind: RoleBinding 26 | metadata: 27 | name: pmem-csi-operator-event 28 | roleRef: 29 | apiGroup: rbac.authorization.k8s.io 30 | kind: Role 31 | name: pmem-csi-operator-event 32 | subjects: 33 | - kind: ServiceAccount 34 | name: pmem-csi-operator 35 | namespace: $(SERVICE_ACCOUNT_NAMESPACE) # replaced by kustomize with service account namespace 36 | -------------------------------------------------------------------------------- /deploy/kustomize/operator/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - namespace.yaml 3 | - operator.yaml 4 | - events-rbac.yaml 5 | 6 | # Deploy the operator in a dedicated namespace 7 | namespace: pmem-csi 8 | 9 | patchesJSON6902: 10 | - target: 11 | group: rbac.authorization.k8s.io 12 | version: v1 13 | kind: Role 14 | name: pmem-csi-operator-event 15 | path: operator-events-namespace-patch.yaml 16 | - target: 17 | group: rbac.authorization.k8s.io 18 | version: v1 19 | kind: RoleBinding 20 | name: pmem-csi-operator-event 21 | path: operator-events-namespace-patch.yaml 22 | 23 | vars: 24 | - name: SERVICE_ACCOUNT_NAMESPACE 25 | objref: 26 | apiVersion: v1 27 | kind: ServiceAccount 28 | name: pmem-csi-operator 29 | fieldRef: 30 | fieldPath: metadata.namespace 31 | 32 | configurations: 33 | - kustomizeconfig.yaml 34 | -------------------------------------------------------------------------------- /deploy/kustomize/operator/kustomizeconfig.yaml: -------------------------------------------------------------------------------- 1 | varReference: 2 | - kind: RoleBinding 3 | group: rbac.authorization.k8s.io 4 | path: subjects/namespace 5 | -------------------------------------------------------------------------------- /deploy/kustomize/operator/namespace.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: pmem-csi 5 | -------------------------------------------------------------------------------- /deploy/kustomize/operator/operator-events-namespace-patch.yaml: -------------------------------------------------------------------------------- 1 | # replace an object's namespace to default 2 | # this supposed to patch the objects whose namespace 3 | # is explicitly in default namespace even if the operator 4 | # deployed in non-default namespace. 5 | - op: replace 6 | path: /metadata/namespace 7 | value: default 8 | -------------------------------------------------------------------------------- /deploy/kustomize/patches/controller-role-patch.yaml: -------------------------------------------------------------------------------- 1 | # Applies to a [Cluster]RoleBinding and sets the name of the first target to "controller". 2 | - op: replace 3 | path: /subjects/0/name 4 | value: pmem-csi-intel-com-controller 5 | -------------------------------------------------------------------------------- /deploy/kustomize/patches/delete-csi-provisioner-service-account-patch.yaml: -------------------------------------------------------------------------------- 1 | # We define our own service account and use this patch to remove 2 | # the pre-defined one from the upstream rbac.yaml files. 3 | apiVersion: v1 4 | kind: ServiceAccount 5 | metadata: 6 | name: csi-provisioner 7 | namespace: default 8 | $patch: delete 9 | -------------------------------------------------------------------------------- /deploy/kustomize/patches/direct-patch.yaml: -------------------------------------------------------------------------------- 1 | # Select direct mode. The PMEM-CSI driver must be in container #0. 2 | - op: add 3 | path: /spec/template/spec/containers/0/command/1 4 | value: "-deviceManager=direct" 5 | 6 | -------------------------------------------------------------------------------- /deploy/kustomize/patches/external-provisioner-connection-timeout-patch.yaml: -------------------------------------------------------------------------------- 1 | # Add -connection-timeout to external-controller in second container. 2 | - op: add 3 | path: /spec/template/spec/containers/1/args/0 4 | value: "--connection-timeout=5m" 5 | -------------------------------------------------------------------------------- /deploy/kustomize/patches/lvm-patch.yaml: -------------------------------------------------------------------------------- 1 | # Select LVM mode. The PMEM-CSI driver must be in container #0. 2 | - op: add 3 | path: /spec/template/spec/containers/0/command/1 4 | value: "-deviceManager=lvm" 5 | -------------------------------------------------------------------------------- /deploy/kustomize/patches/metrics-controller.yaml: -------------------------------------------------------------------------------- 1 | # This JSON patch adds the necessary annotation, port definitions and 2 | # arguments to the PMEM-CSI controller pod. 3 | 4 | # PMEM-CSI: 5 | - op: add 6 | path: /spec/template/metadata/annotations 7 | value: 8 | pmem-csi.intel.com/scrape: containers 9 | - op: add 10 | path: /spec/template/spec/containers/0/ports 11 | value: 12 | - name: metrics 13 | containerPort: 10010 14 | - op: add 15 | path: /spec/template/spec/containers/0/command/- 16 | value: -metricsListen=:10010 17 | - op: add 18 | path: /spec/template/spec/containers/0/livenessProbe 19 | value: 20 | # If the PMEM-CSI driver is able to serve metrics, 21 | # then it is alive. 22 | httpGet: 23 | scheme: HTTP 24 | path: /metrics/simple 25 | port: metrics 26 | # Allow it to for a total duration of one minute. 27 | # This is conservative because the probe is new. 28 | failureThreshold: 6 29 | periodSeconds: 10 30 | successThreshold: 1 31 | timeoutSeconds: 5 32 | - op: add 33 | path: /spec/template/spec/containers/0/startupProbe 34 | value: 35 | httpGet: 36 | scheme: HTTP 37 | path: /metrics/simple 38 | port: metrics 39 | # Check more frequently while the container starts up 40 | # to get it into a ready state quickly. 41 | failureThreshold: 60 42 | periodSeconds: 1 43 | successThreshold: 1 44 | timeoutSeconds: 5 45 | -------------------------------------------------------------------------------- /deploy/kustomize/patches/metrics-node.yaml: -------------------------------------------------------------------------------- 1 | # This JSON patch adds the necessary annotation, port definitions and 2 | # arguments to the PMEM-CSI node pod. 3 | 4 | # PMEM-CSI: 5 | - op: add 6 | path: /spec/template/metadata/annotations 7 | value: 8 | pmem-csi.intel.com/scrape: containers 9 | - op: add 10 | path: /spec/template/spec/containers/0/ports 11 | value: 12 | - name: metrics 13 | containerPort: 10010 14 | - op: add 15 | path: /spec/template/spec/containers/0/command/- 16 | value: -metricsListen=:10010 17 | - op: add 18 | path: /spec/template/spec/containers/0/livenessProbe 19 | value: 20 | # If the PMEM-CSI driver is able to serve metrics, 21 | # then it is alive, for some definition of "alive". 22 | # 23 | # In particular this does *not covers capacity 24 | # checking, because that needs to take a lock 25 | # which can take an unpredictable amount of time 26 | # when there is an operation in progress like 27 | # scrubbing a volume. 28 | httpGet: 29 | scheme: HTTP 30 | path: /metrics/simple 31 | port: metrics 32 | # Allow it to for a total duration of one minute. 33 | # This is conservative because the probe is new. 34 | failureThreshold: 6 35 | periodSeconds: 10 36 | successThreshold: 1 37 | timeoutSeconds: 5 38 | - op: add 39 | path: /spec/template/spec/containers/0/startupProbe 40 | value: 41 | httpGet: 42 | scheme: HTTP 43 | path: /metrics/simple 44 | port: metrics 45 | # Startup may be slower when LVM needs to be set up first. 46 | # Check more frequently to get it into a ready state quickly. 47 | failureThreshold: 300 48 | periodSeconds: 1 49 | successThreshold: 1 50 | timeoutSeconds: 5 51 | 52 | # TODO: node-driver-registrar once it has metrics support. 53 | 54 | # external-provisioner: 55 | - op: add 56 | path: /spec/template/metadata/annotations 57 | value: 58 | pmem-csi.intel.com/scrape: containers 59 | - op: add 60 | path: /spec/template/spec/containers/2/ports 61 | value: 62 | - name: metrics 63 | containerPort: 10011 64 | - op: add 65 | path: /spec/template/spec/containers/2/args/- 66 | value: --metrics-address=:10011 67 | - op: add 68 | path: /spec/template/spec/containers/2/livenessProbe 69 | value: 70 | # If the provisioner is able to serve metrics, 71 | # then it is alive. 72 | httpGet: 73 | scheme: HTTP 74 | path: /metrics 75 | port: metrics 76 | # Same settings as for driver because the provisioner 77 | # waits for the driver. 78 | failureThreshold: 6 79 | periodSeconds: 10 80 | successThreshold: 1 81 | timeoutSeconds: 5 82 | - op: add 83 | path: /spec/template/spec/containers/2/startupProbe 84 | value: 85 | httpGet: 86 | scheme: HTTP 87 | path: /metrics 88 | port: metrics 89 | failureThreshold: 300 90 | periodSeconds: 1 91 | successThreshold: 1 92 | timeoutSeconds: 5 93 | -------------------------------------------------------------------------------- /deploy/kustomize/patches/storageclass-late-binding-patch.yaml: -------------------------------------------------------------------------------- 1 | - op: replace 2 | path: /volumeBindingMode 3 | value: WaitForFirstConsumer 4 | -------------------------------------------------------------------------------- /deploy/kustomize/patches/usage-fileio.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: storage.k8s.io/v1 2 | kind: StorageClass 3 | metadata: 4 | name: pmem-csi-sc 5 | parameters: 6 | usage: FileIO 7 | -------------------------------------------------------------------------------- /deploy/kustomize/scheduler/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - scheduler-service.yaml 3 | -------------------------------------------------------------------------------- /deploy/kustomize/storageclass-ext4-fileio/kustomization.yaml: -------------------------------------------------------------------------------- 1 | nameSuffix: -fileio 2 | 3 | bases: 4 | - ../storageclass-ext4 5 | 6 | patchesStrategicMerge: 7 | - ../patches/usage-fileio.yaml 8 | -------------------------------------------------------------------------------- /deploy/kustomize/storageclass-ext4/ext4.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: storage.k8s.io/v1 2 | kind: StorageClass 3 | metadata: 4 | name: pmem-csi-sc 5 | parameters: 6 | csi.storage.k8s.io/fstype: ext4 7 | eraseafter: "true" # the default 8 | -------------------------------------------------------------------------------- /deploy/kustomize/storageclass-ext4/kustomization.yaml: -------------------------------------------------------------------------------- 1 | nameSuffix: -ext4 2 | 3 | bases: 4 | - ../storageclass 5 | 6 | patchesStrategicMerge: 7 | - ext4.yaml 8 | -------------------------------------------------------------------------------- /deploy/kustomize/storageclass-late-binding/kustomization.yaml: -------------------------------------------------------------------------------- 1 | nameSuffix: -late-binding 2 | 3 | bases: 4 | - ../storageclass 5 | 6 | patchesJson6902: 7 | - target: 8 | group: storage.k8s.io 9 | version: v1 10 | kind: StorageClass 11 | name: pmem-csi-sc 12 | path: ../patches/storageclass-late-binding-patch.yaml 13 | 14 | -------------------------------------------------------------------------------- /deploy/kustomize/storageclass-xfs-fileio/kustomization.yaml: -------------------------------------------------------------------------------- 1 | nameSuffix: -fileio 2 | 3 | bases: 4 | - ../storageclass-xfs 5 | 6 | patchesStrategicMerge: 7 | - ../patches/usage-fileio.yaml 8 | -------------------------------------------------------------------------------- /deploy/kustomize/storageclass-xfs/kustomization.yaml: -------------------------------------------------------------------------------- 1 | nameSuffix: -xfs 2 | 3 | bases: 4 | - ../storageclass 5 | 6 | patchesStrategicMerge: 7 | - xfs.yaml 8 | -------------------------------------------------------------------------------- /deploy/kustomize/storageclass-xfs/xfs.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: storage.k8s.io/v1 2 | kind: StorageClass 3 | metadata: 4 | name: pmem-csi-sc 5 | parameters: 6 | csi.storage.k8s.io/fstype: xfs 7 | eraseafter: "false" # beware of the security implications 8 | -------------------------------------------------------------------------------- /deploy/kustomize/storageclass/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - pmem-storageclass.yaml 3 | -------------------------------------------------------------------------------- /deploy/kustomize/storageclass/pmem-storageclass.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: storage.k8s.io/v1 2 | kind: StorageClass 3 | metadata: 4 | name: pmem-csi-sc 5 | provisioner: pmem-csi.intel.com 6 | -------------------------------------------------------------------------------- /deploy/kustomize/testing/README.md: -------------------------------------------------------------------------------- 1 | # Testing 2 | 3 | This mixin for a regular production deployment of PMEM-CSI adds port 4 | forwarding to the outside world. 5 | 6 | The pmem-csi-intel-com-node-testing DaemonSet forwards 7 | /var/lib/kubelet/plugins/pmem-csi.intel.com/csi.sock on all nodes, 8 | using the fixed port 9735 (arbitrarily chosen). The advantage of this 9 | approach is that: 10 | - all nodes can be checked 11 | - simple deployment (no dynamic creation of services) 12 | - normal TCP connections from outside clients (compared to a solution 13 | like "kubectl exec" with stdin/out forwarding into a socat container 14 | on a node) 15 | The fixed port of course is the disadvantage. 16 | -------------------------------------------------------------------------------- /deploy/kustomize/testing/controller-coverage-patch.yaml: -------------------------------------------------------------------------------- 1 | # Container #0 is expected to be pmem-driver. 2 | - op: add 3 | path: /spec/template/spec/containers/0/command/- 4 | value: -coverprofile=/var/lib/pmem-csi-coverage/pmem-csi-driver-controller-*.out 5 | - op: add 6 | path: /spec/template/spec/containers/0/volumeMounts 7 | value: 8 | - mountPath: /var/lib/pmem-csi-coverage 9 | name: coverage-dir 10 | - op: add 11 | path: /spec/template/spec/volumes 12 | value: 13 | - name: coverage-dir 14 | hostPath: 15 | path: /var/lib/pmem-csi-coverage 16 | type: DirectoryOrCreate 17 | -------------------------------------------------------------------------------- /deploy/kustomize/testing/controller-verbosity-patch.yaml: -------------------------------------------------------------------------------- 1 | # Raise log verbosity level to 5 in first container. 2 | - op: add 3 | path: /spec/template/spec/containers/0/command/- 4 | value: "-v=5" 5 | -------------------------------------------------------------------------------- /deploy/kustomize/testing/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - socat.yaml 3 | 4 | # Keep all the driver objects in 'pmem-csi' namespace 5 | namespace: pmem-csi 6 | -------------------------------------------------------------------------------- /deploy/kustomize/testing/node-coverage-patch.yaml: -------------------------------------------------------------------------------- 1 | # Container #0 is expected to be pmem-driver. 2 | - op: add 3 | path: /spec/template/spec/containers/0/command/- 4 | value: -coverprofile=/var/lib/pmem-csi-coverage/pmem-csi-driver-node-*.out 5 | - op: add 6 | path: /spec/template/spec/containers/0/volumeMounts/- 7 | value: 8 | mountPath: /var/lib/pmem-csi-coverage 9 | name: coverage-dir 10 | - op: add 11 | path: /spec/template/spec/volumes/- 12 | value: 13 | name: coverage-dir 14 | hostPath: 15 | path: /var/lib/pmem-csi-coverage 16 | type: DirectoryOrCreate 17 | -------------------------------------------------------------------------------- /deploy/kustomize/testing/node-verbosity-patch.yaml: -------------------------------------------------------------------------------- 1 | # Raise log verbosity level to 5 in the DaemonSet. 2 | - op: add 3 | path: /spec/template/spec/containers/0/command/- 4 | value: "-v=5" 5 | 6 | - op: add 7 | path: /spec/template/spec/containers/1/args/- 8 | value: "-v=5" 9 | 10 | - op: add 11 | path: /spec/template/spec/containers/2/args/- 12 | value: "-v=5" 13 | -------------------------------------------------------------------------------- /deploy/kustomize/testing/socat.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: DaemonSet 3 | metadata: 4 | name: pmem-csi-intel-com-node-testing 5 | namespace: default 6 | labels: 7 | app.kubernetes.io/name: pmem-csi-node-testing 8 | app.kubernetes.io/part-of: pmem-csi 9 | app.kubernetes.io/component: node-testing 10 | app.kubernetes.io/instance: pmem-csi.intel.com 11 | spec: 12 | selector: 13 | matchLabels: 14 | app.kubernetes.io/name: pmem-csi-node-testing 15 | app.kubernetes.io/instance: pmem-csi.intel.com 16 | template: 17 | metadata: 18 | labels: 19 | app.kubernetes.io/name: pmem-csi-node-testing 20 | app.kubernetes.io/part-of: pmem-csi 21 | app.kubernetes.io/component: node-testing 22 | app.kubernetes.io/instance: pmem-csi.intel.com 23 | pmem-csi.intel.com/webhook: ignore 24 | spec: 25 | # Same settings as for PMEM-CSI driver. 26 | tolerations: 27 | - effect: NoSchedule 28 | operator: Exists 29 | - effect: NoExecute 30 | operator: Exists 31 | priorityClassName: system-node-critical 32 | nodeSelector: 33 | storage: pmem 34 | containers: 35 | - name: socat 36 | image: alpine/socat:1.0.3 37 | args: 38 | - -s 39 | - tcp-listen:9735,fork,reuseaddr 40 | - unix-connect:/csi/csi.sock 41 | securityContext: 42 | privileged: true 43 | volumeMounts: 44 | - name: socket-dir 45 | mountPath: /csi 46 | # mountpoint-dir and staging-dir are needed for creating directories during sanity testing. 47 | - name: mountpoint-dir 48 | mountPath: /var/lib/kubelet/pods 49 | mountPropagation: Bidirectional 50 | - name: staging-dir 51 | mountPath: /var/lib/kubelet/plugins/kubernetes.io/csi/pv # preliminary (?), https://github.com/kubernetes-csi/docs/issues/130 52 | mountPropagation: Bidirectional 53 | ports: 54 | - name: csi-socket 55 | containerPort: 9735 56 | volumes: 57 | - hostPath: 58 | path: /var/lib/kubelet/plugins/pmem-csi.intel.com 59 | type: DirectoryOrCreate 60 | name: socket-dir 61 | - hostPath: 62 | path: /var/lib/kubelet/plugins/kubernetes.io/csi/pv 63 | type: DirectoryOrCreate 64 | name: staging-dir 65 | - hostPath: 66 | path: /var/lib/kubelet/pods 67 | type: DirectoryOrCreate 68 | name: mountpoint-dir 69 | -------------------------------------------------------------------------------- /deploy/kustomize/vpa-for-pmem-csi/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - vpa-controller.yaml 3 | - vpa-node.yaml 4 | 5 | namespace: pmem-csi 6 | -------------------------------------------------------------------------------- /deploy/kustomize/vpa-for-pmem-csi/vpa-controller.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: autoscaling.k8s.io/v1 2 | kind: VerticalPodAutoscaler 3 | metadata: 4 | name: pmem-csi-intel-com-controller 5 | spec: 6 | targetRef: 7 | apiVersion: "apps/v1" 8 | kind: Deployment 9 | name: pmem-csi-intel-com-controller 10 | updatePolicy: 11 | updateMode: "Off" 12 | -------------------------------------------------------------------------------- /deploy/kustomize/vpa-for-pmem-csi/vpa-node.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: autoscaling.k8s.io/v1 2 | kind: VerticalPodAutoscaler 3 | metadata: 4 | name: pmem-csi-intel-com-node 5 | spec: 6 | targetRef: 7 | apiVersion: "apps/v1" 8 | kind: DaemonSet 9 | name: pmem-csi-intel-com-node 10 | updatePolicy: 11 | updateMode: "Off" 12 | -------------------------------------------------------------------------------- /deploy/kustomize/webhook/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - webhook.yaml 3 | - webhook-service.yaml 4 | -------------------------------------------------------------------------------- /deploy/kustomize/webhook/webhook-service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: pmem-csi-intel-com-webhook 5 | namespace: pmem-csi 6 | spec: 7 | selector: 8 | app.kubernetes.io/name: pmem-csi-controller 9 | app.kubernetes.io/instance: pmem-csi.intel.com 10 | ports: 11 | - targetPort: 8000 12 | port: 443 13 | -------------------------------------------------------------------------------- /deploy/kustomize/webhook/webhook.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: admissionregistration.k8s.io/v1 2 | kind: MutatingWebhookConfiguration 3 | metadata: 4 | name: pmem-csi-intel-com-hook 5 | webhooks: 6 | - name: pod-hook.pmem-csi.intel.com 7 | namespaceSelector: 8 | matchExpressions: 9 | - key: pmem-csi.intel.com/webhook 10 | operator: NotIn 11 | values: ["ignore"] 12 | objectSelector: 13 | matchExpressions: 14 | - key: pmem-csi.intel.com/webhook 15 | operator: NotIn 16 | values: ["ignore"] 17 | # For pods with PMEM volumes using late binding, not activating the 18 | # scheduler extension is okayish (but see https://github.com/kubernetes-csi/external-provisioner/issues/544). 19 | # It's worse for CSI ephemeral volumes because there is no recovery. 20 | # Nonetheless, failing pod scheduling entirely when PMEM-CSI is down 21 | # seems worse, so we let the scheduler continue despite failures. 22 | failurePolicy: Ignore 23 | sideEffects: None 24 | admissionReviewVersions: ["v1"] 25 | clientConfig: 26 | service: 27 | name: pmem-csi-intel-com-webhook 28 | namespace: pmem-csi 29 | path: /pod/mutate 30 | caBundle: 31 | rules: 32 | - operations: ["CREATE"] 33 | apiGroups: [""] 34 | apiVersions: ["v1"] 35 | resources: ["pods"] 36 | -------------------------------------------------------------------------------- /deploy/prometheus.yaml: -------------------------------------------------------------------------------- 1 | extraScrapeConfigs: | 2 | # Example scrape config for PMEM-CSI containers. 3 | # 4 | # The relabeling allows the actual container scrape endpoints to be configured in the 5 | # Kubernetes deployment via the "pmem-csi.intel.com/scrape" annotations and the "metrics" 6 | # container ports. 7 | 8 | - job_name: 'pmem-csi-containers' 9 | 10 | kubernetes_sd_configs: 11 | - role: pod 12 | 13 | relabel_configs: 14 | # Only scrape containers which have the "pmem-csi.intel.com/scrape" annotation.... 15 | - source_labels: [__meta_kubernetes_pod_annotation_pmem_csi_intel_com_scrape] 16 | action: keep 17 | regex: containers 18 | # ... and a "metrics" port. 19 | - source_labels: [__meta_kubernetes_pod_container_port_name] 20 | action: keep 21 | regex: metrics 22 | # Overwrite default port (if any) with the container port number. 23 | - source_labels: [__address__, __meta_kubernetes_pod_container_port_number] 24 | action: replace 25 | regex: ([^:]+)(?::\d+)?;(\d+) 26 | replacement: $1:$2 27 | target_label: __address__ 28 | # Add certain information from Kubernetes (namespace, pod and container names, node) 29 | # as label to each sample. The node name is useful for daemon sets where 30 | # pods typically have no meaningful name. 31 | - source_labels: [__meta_kubernetes_namespace] 32 | action: replace 33 | target_label: kubernetes_namespace 34 | - source_labels: [__meta_kubernetes_pod_name] 35 | action: replace 36 | target_label: kubernetes_pod_name 37 | - source_labels: [__meta_kubernetes_pod_container_name] 38 | action: replace 39 | target_label: kubernetes_pod_container_name 40 | - source_labels: [__meta_kubernetes_pod_node_name] 41 | action: replace 42 | target_label: kubernetes_pod_node_name 43 | -------------------------------------------------------------------------------- /deploy/yamls.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 Intel Corporation. 3 | 4 | SPDX-License-Identifier: Apache-2.0 5 | */ 6 | 7 | package deploy 8 | 9 | import ( 10 | "embed" 11 | "fmt" 12 | "path" 13 | "regexp" 14 | 15 | api "github.com/intel/pmem-csi/pkg/apis/pmemcsi/v1beta1" 16 | "github.com/intel/pmem-csi/pkg/version" 17 | ) 18 | 19 | //go:embed kubernetes-*/pmem-csi-*.yaml 20 | var assets embed.FS 21 | 22 | // YamlFile contains all objects of a certain deployment. 23 | type YamlFile struct { 24 | // Name is the unique string which identifies the deployment. 25 | Name string 26 | 27 | // Kubernetes is the . version the deployment 28 | // was written for. 29 | Kubernetes version.Version 30 | 31 | // Flavor is a variant of the normal deployment for the Kubernetes version. 32 | // Empty or a string leading with a hyphen. 33 | Flavor string 34 | 35 | // DeviceMode defines in which mode the deployed driver will 36 | // operate. 37 | DeviceMode api.DeviceMode 38 | } 39 | 40 | var yamls []YamlFile 41 | 42 | var re = regexp.MustCompile(`^kubernetes-([0-9\.]*)/pmem-csi-(lvm|direct)(.*).yaml$`) 43 | 44 | func init() { 45 | deployDir, err := assets.ReadDir(".") 46 | if err != nil { 47 | panic(err) 48 | } 49 | for _, item := range deployDir { 50 | if !item.IsDir() { 51 | continue 52 | } 53 | kubernetesDir, err := assets.ReadDir(item.Name()) 54 | if err != nil { 55 | panic(err) 56 | } 57 | for _, item2 := range kubernetesDir { 58 | name := path.Join(item.Name(), item2.Name()) 59 | parts := re.FindStringSubmatch(name) 60 | if parts == nil { 61 | continue 62 | } 63 | kubernetes, err := version.Parse(parts[1]) 64 | if err != nil { 65 | panic(fmt.Sprintf("unexpected version in %s: %v", name, err)) 66 | } 67 | yamls = append(yamls, YamlFile{ 68 | Name: name, 69 | Kubernetes: kubernetes, 70 | Flavor: parts[3], 71 | DeviceMode: api.DeviceMode(parts[2]), 72 | }) 73 | } 74 | } 75 | } 76 | 77 | // ListAll returns information about all embedded YAML files. 78 | func ListAll() []YamlFile { 79 | return yamls 80 | } 81 | 82 | // Asset returns the content of an embedded file. 83 | // The path must be relative to the "deploy" dir. 84 | func Asset(path string) ([]byte, error) { 85 | return assets.ReadFile(path) 86 | } 87 | -------------------------------------------------------------------------------- /docs/diagrams/pmem-csi-communication-diagram.dia: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intel/pmem-csi/1f1f9c2a357a94a998c4b44e270440d2de053e5f/docs/diagrams/pmem-csi-communication-diagram.dia -------------------------------------------------------------------------------- /docs/diagrams/pmem-csi.dia: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intel/pmem-csi/1f1f9c2a357a94a998c4b44e270440d2de053e5f/docs/diagrams/pmem-csi.dia -------------------------------------------------------------------------------- /docs/html/index.html: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/html/index2.html: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/images/communication/pmem-csi-communication-diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intel/pmem-csi/1f1f9c2a357a94a998c4b44e270440d2de053e5f/docs/images/communication/pmem-csi-communication-diagram.png -------------------------------------------------------------------------------- /docs/images/devicemodes/pmem-csi-direct.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intel/pmem-csi/1f1f9c2a357a94a998c4b44e270440d2de053e5f/docs/images/devicemodes/pmem-csi-direct.png -------------------------------------------------------------------------------- /docs/images/devicemodes/pmem-csi-lvm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intel/pmem-csi/1f1f9c2a357a94a998c4b44e270440d2de053e5f/docs/images/devicemodes/pmem-csi-lvm.png -------------------------------------------------------------------------------- /docs/images/sequence/pmem-csi-persistent-sequence-diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intel/pmem-csi/1f1f9c2a357a94a998c4b44e270440d2de053e5f/docs/images/sequence/pmem-csi-persistent-sequence-diagram.png -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | sphinx == 4.5.0 2 | sphinx_rtd_theme == 1.0.0 3 | recommonmark == 0.7.1 4 | sphinx-markdown-tables == 0.0.15 5 | sphinx-copybutton == 0.2.11 6 | sphinx-md == 0.0.3 7 | markdown == 3.3.7 8 | 9 | # A working combination: 10 | # Jinja2-3.1.2 MarkupSafe-2.1.1 Pygments-2.12.0 alabaster-0.7.12 babel-2.10.1 beautifulsoup4-4.11.1 bs4-0.0.1 certifi-2022.5.18.1 charset-normalizer-2.0.12 commonmark-0.9.1 docutils-0.17.1 idna-3.3 imagesize-1.3.0 importlib-metadata-4.11.4 markdown-3.3.7 packaging-21.3 pyparsing-3.0.9 pytz-2022.1 recommonmark-0.7.1 requests-2.27.1 snowballstemmer-2.2.0 soupsieve-2.3.2.post1 sphinx-4.5.0 sphinx-copybutton-0.2.11 sphinx-markdown-tables-0.0.15 sphinx-md-0.0.3 sphinx-rtd-theme-1.0.0 sphinxcontrib-applehelp-1.0.2 sphinxcontrib-devhelp-1.0.2 sphinxcontrib-htmlhelp-2.0.0 sphinxcontrib-jsmath-1.0.1 sphinxcontrib-qthelp-1.0.3 sphinxcontrib-serializinghtml-1.1.5 typing-extensions-4.2.0 urllib3-1.26.9 zipp-3.8.0 11 | -------------------------------------------------------------------------------- /docs/static/override.css: -------------------------------------------------------------------------------- 1 | .rst-content a.reference code.literal { 2 | color: #2980b9; 3 | } 4 | 5 | /* 6 | * We want tables to fit the screen width by wrapping text, not via scrolling. 7 | * Solution from https://github.com/readthedocs/sphinx_rtd_theme/issues/117#issuecomment-41606147 8 | */ 9 | 10 | /* override table width restrictions */ 11 | .wy-table-responsive table td, .wy-table-responsive table th { 12 | /* !important prevents the common CSS stylesheets from 13 | overriding this as on RTD they are loaded after this stylesheet */ 14 | white-space: normal !important; 15 | } 16 | 17 | .wy-table-responsive { 18 | overflow: visible !important; 19 | } 20 | -------------------------------------------------------------------------------- /docs/substitutions.txt: -------------------------------------------------------------------------------- 1 | .. |PR| replace:: Project Name -------------------------------------------------------------------------------- /examples/readme.rst: -------------------------------------------------------------------------------- 1 | Application examples 2 | #################### 3 | 4 | `Redis-pmem operator `__ 5 | Deploy a Redis cluster through the redis-operator using QEMU-emulated persistent memory devices 6 | 7 | `memcached with PMEM `__ 8 | Deploy memcached with PMEM as replacement for DRAM. 9 | 10 | `Google Cloud Engine `__ 11 | Install Kubernetes and PMEM-CSI on Google Cloud machines. 12 | 13 | .. toctree:: 14 | :hidden: 15 | 16 | redis-operator.md 17 | memcached.md 18 | gce.md 19 | 20 | -------------------------------------------------------------------------------- /hack/bump-image-versions.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Copyright 2021 The Kubernetes Authors. 4 | # Copyright 2021 Intel Corporation. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | # This script will determine the latest release of all sidecars and 19 | # update to that version in *all* deployment files. Do not commit 20 | # everything! Sometimes sidecars must be locked on an older release 21 | # for older Kubernetes releases. 22 | 23 | set -e 24 | 25 | # List of images under https://console.cloud.google.com/gcr/images/k8s-staging-sig-storage/GLOBAL 26 | images=" 27 | csi-attacher 28 | csi-node-driver-registrar 29 | csi-provisioner 30 | csi-resizer 31 | csi-snapshotter 32 | csi-external-health-monitor-agent 33 | csi-external-health-monitor-controller 34 | livenessprobe 35 | " 36 | 37 | for image in $images; do 38 | latest=$(gcloud container images list-tags k8s.gcr.io/sig-storage/$image --format='get(tags)' --filter='tags~^v AND NOT tags~v2020 AND NOT tags~-rc' --sort-by=tags | tail -n 1) 39 | 40 | sed -i -e "s;\(image: k8s.gcr.io/sig-storage/$image:\).*;\1$latest;" $(find deploy -type f) 41 | sed -i -e "s;\(Image = \"k8s.gcr.io/sig-storage/$image:\).*\";\1$latest\";" $(find pkg/apis/pmemcsi -type f) 42 | # Match the line with the image and the next line, then replace newTag in those two lines. 43 | sed -i -e "\\;name: k8s.gcr.io/sig-storage/$image;,+1 s/newTag: .*/newTag: $latest/" deploy/kustomize/*/kustomization.yaml 44 | done 45 | -------------------------------------------------------------------------------- /hack/copy-modules-license.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Copyright 2019 Intel Corporation. 4 | # 5 | # SPDX-License-Identifier: Apache-2.0 6 | # 7 | # Copy the licenses of ".Deps" modules for a package to a target directory. 8 | # The "vendor" directory must exist and must be up-to-date. 9 | 10 | set -o errexit 11 | set -o nounset 12 | set -o pipefail 13 | 14 | if [ $# -lt 2 ] || [ "$1" = "?" ] || [ "$1" = "--help" ]; then 15 | echo "Usage: $0 ..." >&2 16 | exit 1 17 | fi 18 | 19 | licensedir=$1 20 | shift 21 | 22 | if [ ! -d "$licensedir" ] || [ ! -w "$licensedir" ]; then 23 | echo "Error: cannot use $licensedir as the license target directory" 24 | exit 1 25 | fi 26 | 27 | LICENSE_FILES=$(find vendor |grep -e LICENSE -e NOTICE|cut -d / -f 2-) 28 | PACKAGE_DEPS=$(go list -f '{{ join .Deps "\n" }}' "$@" |grep "\.") 29 | 30 | pushd vendor > /dev/null 31 | 32 | for lic in $LICENSE_FILES; do 33 | # Copy the license if its repository path is found in package .Deps 34 | if echo "$PACKAGE_DEPS" | grep -q "^$(dirname $lic)"; then 35 | cp -t "$licensedir" --parents $lic 36 | fi 37 | done 38 | 39 | popd > /dev/null 40 | 41 | # Same for forked third-party code. 42 | LICENSE_FILES=$(find third-party | grep -e LICENSE -e NOTICE | cut -d / -f 2-) 43 | 44 | pushd third-party > /dev/null 45 | 46 | for lic in $LICENSE_FILES; do 47 | if echo "$PACKAGE_DEPS" | grep -q "^github.com/intel/pmem-csi/third-party/$(dirname $lic)"; then 48 | cp -t "$licensedir" --parents $lic 49 | fi 50 | done 51 | 52 | popd > /dev/null 53 | -------------------------------------------------------------------------------- /hack/list-direct-imports.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # Copyright 2020 Intel Corporation. 4 | # 5 | # SPDX-License-Identifier: Apache-2.0 6 | # 7 | # Lists all direct dependencies of code in a certain module. 8 | # 9 | # Usage: ... 10 | # Example: github.com/intel/pmem-csi ./cmd/pmem-csi-driver 11 | 12 | PACKAGE="$1" 13 | shift 14 | 15 | : ${GO:=go} 16 | 17 | # Normalize package so that they start with the module. 18 | IMPORTS=$(env $GO list "$@") 19 | 20 | # Repeatedly expand the list of imports that come from the package(s) 21 | # until we find no new ones, then print the direct imports of those. 22 | while true; do 23 | NEW_IMPORTS=$( (echo "$IMPORTS"; env $GO list -f '{{join .Imports "\n"}}' $(echo "$IMPORTS" | grep "^$PACKAGE") ) | sort -u) 24 | if [ "$NEW_IMPORTS" != "$IMPORTS" ]; then 25 | IMPORTS="$NEW_IMPORTS" 26 | else 27 | break 28 | fi 29 | done 30 | 31 | env $GO list -f '{{join .Imports "\n"}}' $(echo "$IMPORTS" | grep "^$PACKAGE") | sort -u 32 | -------------------------------------------------------------------------------- /hack/merge-release.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh -ex 2 | # 3 | # Copyright 2019 Intel Corporation. 4 | # 5 | # SPDX-License-Identifier: Apache-2.0 6 | # 7 | # Invoke this script will generate a merge commit of the latest 8 | # release branch into the current branch, without actually changing 9 | # any file. 10 | # 11 | # This is valid on the devel branch because our development process 12 | # ensures that bug fixes and features are always applied on the devel 13 | # branch first. 14 | # 15 | # The purpose is to teach "git describe --tags" that the "devel" 16 | # branch is more recent than the latest tagged release from the 17 | # release branch. 18 | 19 | git fetch origin 20 | head_tree=$(git show --no-patch --pretty=format:%T HEAD) 21 | head_commit=$(git rev-parse HEAD) 22 | latest_release=$(git branch -r | grep 'origin/release-[0-9]*\.[0-9]*' | sort -n | tail -n 1) 23 | release_commit=$(git rev-parse ${latest_release}) 24 | new_commit=$(git commit-tree ${head_tree} -p ${head_commit} -p ${release_commit} -F -) < (version with v prefix, e.g. v0.9.0)" >&2 13 | exit 1 14 | fi 15 | 16 | old_version=$1 17 | new_version=$2 18 | 19 | set -x 20 | 21 | sed -i -e "s;\(IMAGE_VERSION?*=\|intel/pmem-[^ ]*:\)[^ ^;/]*;\1${new_version};g" $(git grep -l 'IMAGE_VERSION?*=\|intel/pmem-[^ ]*:' Makefile test/*.sh deploy) 22 | sed -i -e "s;/pmem-csi-driver\([^ ]*\):[^ {}]*;/pmem-csi-driver\1:${new_version};g" test/test-config.sh docs/*.md 23 | sed -i -e "s;\(TEST_PMEM_IMAGE_TAG:=\)[^}]*;\1${new_version};g" test/test-config.sh 24 | sed -i -e "s;github.com/intel/pmem-csi/raw/[^/]*/;github.com/intel/pmem-csi/raw/${new_version}/;g" docs/*.md 25 | sed -i -e "s;LABEL version=\".*\";LABEL version=\"${new_version}\";" Dockerfile.UBI 26 | -------------------------------------------------------------------------------- /hack/setup-va.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # Copyright 2020 Intel Corporation 4 | 5 | set -ex 6 | 7 | if ! [ -d _work/autoscaler ]; then 8 | git clone https://github.com/kubernetes/autoscaler _work/autoscaler 9 | fi 10 | cd _work/autoscaler 11 | git fetch origin 12 | git checkout vertical-pod-autoscaler-0.9.0 13 | 14 | cd vertical-pod-autoscaler 15 | hack/vpa-up.sh 16 | -------------------------------------------------------------------------------- /hack/stress-driver.sh: -------------------------------------------------------------------------------- 1 | # !/bin/bash 2 | 3 | # Utility script to stress the driver by creating and deleting the volumes. It 4 | # runs by default 100 runs, each run it creates and deletes 10(5 xfs and 5 ext) 5 | # volumes of each 4GB size. Hence, it requires approximately 40 GB of PMEM memory 6 | # 7 | # The main intention of this script is to determine the default resource requirements 8 | # to be set for the driver deployed via the operator. 9 | # This is expected to run by setting up the metrics-server and VirtualPodAutoscaler 10 | # describe in the 'Performance and resource measurements' section in the 11 | # developer documentation(DEPLOYMENT.md). 12 | # 13 | # NOTE: This script is *not* expected to run on real clusters where the user 14 | # has exisitng PV/PVCs. 15 | 16 | ROUNDS=${ROUNDS:-100} 17 | VOL_COUNT=${VOL_COUNT:-5} # 5 ext4 + 5 xfs = 10 * 4Gi ~= 40Gi 18 | 19 | for i in $(seq 1 1 $ROUNDS) ; do 20 | echo "Round #$i:" 21 | echo "Creating volumes..." 22 | for j in $(seq 1 1 $VOL_COUNT) ; do 23 | sed -e "s;\(.*name:\)\(.*\);\1\2-$j;g" < deploy/common/pmem-pvc.yaml | kubectl create -f - 24 | done 25 | echo "Deleting all pvc..." 26 | while [ "$(kubectl get pv --no-headers | wc -l)" -ne "0" ]; do 27 | kubectl delete pvc --all 28 | kubectl delete pv --all 29 | done 30 | done 31 | -------------------------------------------------------------------------------- /hack/tooling/dependencies.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 Intel Coporation. 3 | 4 | SPDX-License-Identifier: Apache-2.0 5 | */ 6 | 7 | // Package tooling contains dependencies for some of the code that 8 | // we only need at build time. It ensures that "go mod tidy" doesn't remove 9 | // those dependencies from the top-level go.mod. 10 | package tooling 11 | 12 | import ( 13 | _ "github.com/go-bindata/go-bindata" 14 | ) 15 | -------------------------------------------------------------------------------- /hack/verify-generated.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Copyright 2019 The Kubernetes Authors. 4 | # 5 | # Derived from https://github.com/kubernetes-csi/csi-release-tools/blob/c8a1c4af933311a7e63765cd2b64ca45a0fb7dba/verify-vendor.sh 6 | 7 | # JOB_NAME and CHANGE_ID are standard Jenkins env vars, 8 | # GIT_BRANCH comes from https://wiki.jenkins.io/display/JENKINS/Git+Plugin. 9 | if [ "${JOB_NAME}" ] && 10 | ( ! [ "${CHANGE_ID}" ] || 11 | # We consider Dockerfile here because that is where we define the Go version 12 | # used by the CI, which may have an effect. 13 | [ "$( (git diff "${GIT_BRANCH}..HEAD" -- pkg/apis Dockerfile; 14 | git diff "${}..HEAD" | grep -e '^@@.*@@ import (' -e '^[+-]import') | 15 | wc -l)" -eq 0 ] ); then 16 | echo "Skipping generated code check because the CI job does not affect it." 17 | elif ! (set -x; make generate); then 18 | echo "ERROR: code generation failed." 19 | exit 1 20 | elif [ "$(git status --porcelain -- $(find . -name '*generated*' | grep -v -e ^./vendor -e ^./hack) | wc -l)" -gt 0 ]; then 21 | echo "ERROR: generated files *not* up-to-date, they did get modified by 'make generate':"; 22 | git diff -- $(find . -name '*generated*' | grep -v -e ^./vendor -e ^./hack) 23 | exit 1 24 | else 25 | echo "Generated code is up-to-date." 26 | fi 27 | -------------------------------------------------------------------------------- /hack/verify-go-version.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright 2019 The Kubernetes Authors. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | GO="$1" 18 | 19 | if [ ! "$GO" ]; then 20 | echo >&2 "usage: $0 " 21 | exit 1 22 | fi 23 | 24 | die () { 25 | echo "ERROR: $*" 26 | exit 1 27 | } 28 | 29 | version=$("$GO" version) || die "determining version of $GO failed" 30 | # shellcheck disable=SC2001 31 | majorminor=$(echo "$version" | sed -e 's/.*go\([0-9]*\)\.\([0-9]*\).*/\1.\2/') 32 | # shellcheck disable=SC2001 33 | expected=$(grep "^ARG GO_VERSION" Dockerfile | sed -e 's/.*GO_VERSION *= *"*\([0-9]*\)\.\([0-9]*\).*/\1.\2/') 34 | 35 | if [ "$majorminor" != "$expected" ]; then 36 | cat >&2 < 19 | -------------------------------------------------------------------------------- /make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=. 11 | set BUILDDIR=_output 12 | 13 | if "%1" == "" goto help 14 | 15 | if "%1" == "html" goto html 16 | 17 | %SPHINXBUILD% >NUL 2>NUL 18 | if errorlevel 9009 ( 19 | echo. 20 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 21 | echo.installed, then set the SPHINXBUILD environment variable to point 22 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 23 | echo.may add the Sphinx directory to PATH. 24 | echo. 25 | echo.If you don't have Sphinx installed, grab it from 26 | echo.http://sphinx-doc.org/ 27 | exit /b 1 28 | ) 29 | 30 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 31 | goto end 32 | 33 | :html 34 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 35 | copy docs\html\index.html %BUILDDIR%\html\index.html 36 | copy docs\js\copybutton.js %BUILDDIR%\html\_static\copybutton.js 37 | goto end 38 | 39 | :help 40 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 41 | 42 | :end 43 | popd 44 | -------------------------------------------------------------------------------- /pkg/apis/addtoscheme_pmemcsi_v1beta1.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | SPDX-License-Identifier: Apache-2.0 5 | */ 6 | 7 | package apis 8 | 9 | import ( 10 | "github.com/intel/pmem-csi/pkg/apis/pmemcsi/v1beta1" 11 | ) 12 | 13 | func init() { 14 | // Register the types with the Scheme so the components can map objects to GroupVersionKinds and back 15 | AddToSchemes = append(AddToSchemes, v1beta1.SchemeBuilder.AddToScheme) 16 | } 17 | -------------------------------------------------------------------------------- /pkg/apis/apis.go: -------------------------------------------------------------------------------- 1 | package apis 2 | 3 | import ( 4 | "k8s.io/apimachinery/pkg/runtime" 5 | ) 6 | 7 | // AddToSchemes may be used to add all resources defined in the project to a Scheme 8 | var AddToSchemes runtime.SchemeBuilder 9 | 10 | // AddToScheme adds all Resources to the Scheme 11 | func AddToScheme(s *runtime.Scheme) error { 12 | return AddToSchemes.AddToScheme(s) 13 | } 14 | -------------------------------------------------------------------------------- /pkg/apis/pmemcsi/group.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | SPDX-License-Identifier: Apache-2.0 5 | */ 6 | 7 | // Package pmemcsi contains pmemcsi API versions. 8 | // 9 | // This file ensures Go source parsers acknowledge the pmemcsi package 10 | // and any child packages. It can be removed if any other Go source files are 11 | // added to this package. 12 | package pmemcsi 13 | -------------------------------------------------------------------------------- /pkg/apis/pmemcsi/v1beta1/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | SPDX-License-Identifier: Apache-2.0 5 | */ 6 | 7 | // Package v1beta1 contains API Schema definitions for the pmem-csi v1beta1 API group 8 | // +groupName=pmem-csi.intel.com 9 | package v1beta1 10 | -------------------------------------------------------------------------------- /pkg/apis/pmemcsi/v1beta1/register.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | SPDX-License-Identifier: Apache-2.0 5 | */ 6 | 7 | // NOTE: Boilerplate only. Ignore this file. 8 | 9 | package v1beta1 10 | 11 | import ( 12 | "k8s.io/apimachinery/pkg/runtime/schema" 13 | "sigs.k8s.io/controller-runtime/pkg/scheme" 14 | ) 15 | 16 | var ( 17 | // SchemeGroupVersion is group version used to register these objects 18 | SchemeGroupVersion = schema.GroupVersion{Group: "pmem-csi.intel.com", Version: "v1beta1"} 19 | 20 | // SchemeBuilder is used to add go types to the GroupVersionKind scheme 21 | SchemeBuilder = &scheme.Builder{GroupVersion: SchemeGroupVersion} 22 | ) 23 | -------------------------------------------------------------------------------- /pkg/coverage/coverage.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 Intel Corporation 3 | 4 | SPDX-License-Identifier: Apache-2.0 5 | */ 6 | 7 | // Package coverage adds one command line flag similar to 8 | // -test.coverprofile. The difference is that the file is created 9 | // during startup with ioutil.Tmpfile (i.e. * gets replaced with 10 | // a unique string) and then passed to a restarted binary 11 | // as -test.coverprofile. 12 | 13 | package coverage 14 | 15 | import ( 16 | "flag" 17 | "io/ioutil" 18 | "os" 19 | "path/filepath" 20 | "syscall" 21 | 22 | "k8s.io/klog/v2" 23 | ) 24 | 25 | var coverage = flag.String("coverprofile", "", "write a coverage profile to a unique file (* in name replaced with random string, otherwise appended)") 26 | 27 | // Run re-execs the program with -test.coverprofile if -coverprofile 28 | // was used, otherwise it executes the program's main function. 29 | func Run(main func() int) { 30 | if *coverage != "" { 31 | abspath, err := filepath.Abs(*coverage) 32 | if err != nil { 33 | klog.Fatalf("cover profile %q: %s", *coverage, err) 34 | } 35 | f, err := ioutil.TempFile(filepath.Dir(abspath), filepath.Base(abspath)) 36 | if err != nil { 37 | klog.Fatalf("temporary cover profile %q: %s", abspath, err) 38 | } 39 | // Overwrite -coverprofile with empty string during next program run to 40 | // avoid endless recursion. Instead use -test.coverprofile with the new 41 | // name. 42 | args := os.Args 43 | args = append(args, "-coverprofile=") 44 | args = append(args, "-test.coverprofile="+f.Name()) 45 | if err := syscall.Exec(args[0], args, os.Environ()); err != nil { 46 | klog.Fatalf("re-exec %v: %s", args, err) 47 | } 48 | } 49 | main() 50 | } 51 | -------------------------------------------------------------------------------- /pkg/deployments/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 Intel Corporation. 3 | 4 | SPDX-License-Identifier: Apache-2.0 5 | */ 6 | 7 | // The deployments package reads the reference YAML files and 8 | // optionally modifies them for deployment. 9 | package deployments 10 | -------------------------------------------------------------------------------- /pkg/deployments/load_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 Intel Corporation. 3 | 4 | SPDX-License-Identifier: Apache-2.0 5 | */ 6 | 7 | package deployments_test 8 | 9 | import ( 10 | "fmt" 11 | "testing" 12 | 13 | "github.com/stretchr/testify/assert" 14 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 15 | 16 | "github.com/intel/pmem-csi/deploy" 17 | api "github.com/intel/pmem-csi/pkg/apis/pmemcsi/v1beta1" 18 | "github.com/intel/pmem-csi/pkg/deployments" 19 | "github.com/intel/pmem-csi/pkg/version" 20 | ) 21 | 22 | func TestLoadObjects(t *testing.T) { 23 | _, err := deployments.LoadObjects(version.NewVersion(1, 0), api.DeviceModeDirect) 24 | assert.Error(t, err, "load yaml for unsupported version") 25 | 26 | yamls := deploy.ListAll() 27 | assert.NotEmpty(t, yamls, "should have builtin yaml deployments") 28 | 29 | for _, testCase := range yamls { 30 | t.Run(testCase.Name, func(t *testing.T) { 31 | objects, err := deployments.LoadObjects(testCase.Kubernetes, testCase.DeviceMode) 32 | if assert.NoError(t, err, "load yaml") { 33 | assert.NotEmpty(t, objects, "have objects") 34 | 35 | // Check that all objects have the right label. 36 | expectedDeployment := fmt.Sprintf("%s-production", testCase.DeviceMode) 37 | for _, obj := range objects { 38 | labels := obj.GetLabels() 39 | if assert.Contains(t, labels, "pmem-csi.intel.com/deployment", "object %v should have deployment label", obj) { 40 | deployment := labels["pmem-csi.intel.com/deployment"] 41 | assert.Equal(t, expectedDeployment, deployment, "deployment label") 42 | } 43 | } 44 | } 45 | // Check customizing namespace and name. More customization tests 46 | // currently run as part of test/e2e/operator API testing, with 47 | // the code in controller_driver.go serving as reference. 48 | namespace := "kube-system" 49 | deployment := api.PmemCSIDeployment{ 50 | ObjectMeta: metav1.ObjectMeta{ 51 | Name: "pmem-csi.example.org", 52 | }, 53 | } 54 | objects, err = deployments.LoadAndCustomizeObjects(testCase.Kubernetes, testCase.DeviceMode, namespace, deployment) 55 | if assert.NoError(t, err, "load and customize yaml") { 56 | assert.NotEmpty(t, objects, "have customized objects") 57 | 58 | for _, obj := range objects { 59 | if obj.GetKind() == "CSIDriver" { 60 | assert.Equal(t, deployment.GetName(), obj.GetName(), "CSIDriver name") 61 | } else { 62 | assert.Contains(t, obj.GetName(), deployment.GetHyphenedName(), "other object name") 63 | } 64 | switch obj.GetKind() { 65 | case "CSIDriver", "ClusterRole", "ClusterRoleBinding": 66 | assert.Equal(t, "", obj.GetNamespace(), "non-namespaced %s namespace", obj.GetName()) 67 | default: 68 | assert.Equal(t, namespace, obj.GetNamespace(), "non-namespaced %s namespace", obj.GetName()) 69 | } 70 | } 71 | } 72 | }) 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /pkg/errors/errors.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 Intel Coporation. 3 | 4 | SPDX-License-Identifier: Apache-2.0 5 | */ 6 | 7 | // Package errors contain some well-defined errors that may have to be 8 | // passed up from low-level layers in the PMEM-CSI stack up to the 9 | // gRPC interface. 10 | // 11 | // These errors must be wrapped (for example, with %w) so that the 12 | // upper layers can use errors.Is to recognize these special errors if 13 | // needed. 14 | package errors 15 | 16 | import ( 17 | "errors" 18 | ) 19 | 20 | var ( 21 | // DeviceExists device with given id already exists 22 | DeviceExists = errors.New("device exists") 23 | 24 | // ErrDeviceNotFound device does not exists 25 | DeviceNotFound = errors.New("device not found") 26 | 27 | // ErrDeviceInUse device is in use 28 | DeviceInUse = errors.New("device in use") 29 | 30 | // ErrNotEnoughSpace no space to create the device 31 | NotEnoughSpace = errors.New("not enough space") 32 | ) 33 | -------------------------------------------------------------------------------- /pkg/exec/exec.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 Intel Corporation. 3 | 4 | SPDX-License-Identifier: Apache-2.0 5 | */ 6 | 7 | package exec 8 | 9 | import ( 10 | "bufio" 11 | "bytes" 12 | "context" 13 | "fmt" 14 | "io" 15 | "os/exec" 16 | "sync" 17 | 18 | "k8s.io/klog/v2" 19 | ) 20 | 21 | // RunCommand executes the command with logging through klog, with 22 | // output processed line-by-line with the command path as prefix. It 23 | // returns the combined output and, if there was a problem, includes 24 | // that output and the command in the error. 25 | func RunCommand(ctx context.Context, cmd string, args ...string) (string, error) { 26 | return Run(ctx, exec.Command(cmd, args...)) 27 | } 28 | 29 | // Run does the same as RunCommand but takes a pre-populated 30 | // cmd. Stdout and stderr are ignored and replaced with the output 31 | // handling described for RunCommand. 32 | func Run(ctx context.Context, cmd *exec.Cmd) (string, error) { 33 | logger := klog.FromContext(ctx).WithValues("command", cmd.Path) 34 | logger.V(4).Info("Starting command", "args", cmd.Args) 35 | 36 | r, w := io.Pipe() 37 | r2, w2 := io.Pipe() 38 | cmd.Stdout = w 39 | cmd.Stderr = w2 40 | var stdout, both bytes.Buffer 41 | var wg sync.WaitGroup 42 | wg.Add(2) 43 | // Collect stdout and stderr separately. Storing in the 44 | // combined buffer is a bit racy, but we need to know which 45 | // output is stdout. 46 | go dumpOutput(klog.NewContext(ctx, logger.WithName("stdout")), &wg, r, []io.Writer{&stdout, &both}) 47 | go dumpOutput(klog.NewContext(ctx, logger.WithName("stderr")), &wg, r2, []io.Writer{&both}) 48 | err := cmd.Run() 49 | w.Close() 50 | w2.Close() 51 | wg.Wait() 52 | logger.V(4).Info("Command terminated", "stdout-len", stdout.Len(), "combined-len", both.Len(), "error", err) 53 | 54 | switch { 55 | case err != nil && both.Len() > 0: 56 | err = fmt.Errorf("%q: command failed: %v\nCombined stderr/stdout output: %s", cmd, err, both.String()) 57 | case err != nil: 58 | err = fmt.Errorf("%q: command failed with no output: %v", cmd, err) 59 | } 60 | return stdout.String(), err 61 | } 62 | 63 | func dumpOutput(ctx context.Context, wg *sync.WaitGroup, in io.Reader, out []io.Writer) { 64 | logger := klog.FromContext(ctx) 65 | defer wg.Done() 66 | scanner := bufio.NewScanner(in) 67 | for scanner.Scan() { 68 | for _, o := range out { 69 | _, _ = o.Write(scanner.Bytes()) 70 | _, _ = o.Write([]byte("\n")) 71 | } 72 | logger.V(5).Info(scanner.Text()) 73 | } 74 | } 75 | 76 | // CmdResult always returns an informative description of what command ran and what the 77 | // outcome (stdout+stderr, exit code if any) was. Logging is left entirely to the caller. 78 | func CmdResult(cmd string, args ...string) string { 79 | c := exec.Command(cmd, args...) 80 | output, err := c.CombinedOutput() 81 | result := fmt.Sprintf("%q:", c) 82 | switch { 83 | case err != nil && len(output) == 0: 84 | result += fmt.Sprintf(" command failed with no output: %v", err) 85 | case err != nil: 86 | result += fmt.Sprintf(" command failed: %v\nOutput:\n%s", err, string(output)) 87 | case len(output) > 0: 88 | result += fmt.Sprintf("\n%s", output) 89 | default: 90 | result += " no output" 91 | } 92 | return result 93 | } 94 | -------------------------------------------------------------------------------- /pkg/grpc-server/server.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 The Kubernetes Authors. 3 | 4 | SPDX-License-Identifier: Apache-2.0 5 | */ 6 | 7 | package pmemcsidriver 8 | 9 | import ( 10 | "context" 11 | "crypto/tls" 12 | "fmt" 13 | "sync" 14 | 15 | "github.com/kubernetes-csi/csi-lib-utils/metrics" 16 | "google.golang.org/grpc" 17 | "k8s.io/klog/v2" 18 | 19 | pmemgrpc "github.com/intel/pmem-csi/pkg/pmem-grpc" 20 | ) 21 | 22 | type Service interface { 23 | // RegisterService will be called by NonBlockingGRPCServer whenever 24 | // its about to start a grpc server on an endpoint. 25 | RegisterService(s *grpc.Server) 26 | } 27 | 28 | // NonBlocking server 29 | type NonBlockingGRPCServer struct { 30 | wg sync.WaitGroup 31 | servers []*grpc.Server 32 | } 33 | 34 | func NewNonBlockingGRPCServer() *NonBlockingGRPCServer { 35 | return &NonBlockingGRPCServer{} 36 | } 37 | 38 | func (s *NonBlockingGRPCServer) Start(ctx context.Context, endpoint, errorPrefix string, tlsConfig *tls.Config, csiMetricsManager metrics.CSIMetricsManager, services ...Service) error { 39 | if endpoint == "" { 40 | return fmt.Errorf("endpoint cannot be empty") 41 | } 42 | rpcServer, l, err := pmemgrpc.NewServer(endpoint, errorPrefix, tlsConfig, csiMetricsManager) 43 | if err != nil { 44 | return nil 45 | } 46 | for _, service := range services { 47 | service.RegisterService(rpcServer) 48 | } 49 | s.servers = append(s.servers, rpcServer) 50 | 51 | logger := klog.FromContext(ctx).WithName("GRPC-server").WithValues("endpoint", endpoint) 52 | s.wg.Add(1) 53 | go func() { 54 | defer s.wg.Done() 55 | logger.V(3).Info("Listening for connections") 56 | if err := rpcServer.Serve(l); err != nil { 57 | logger.Error(err, "Listen failure") 58 | } 59 | logger.V(3).Info("Stopped") 60 | }() 61 | 62 | return nil 63 | } 64 | 65 | func (s *NonBlockingGRPCServer) Wait() { 66 | s.wg.Wait() 67 | } 68 | 69 | func (s *NonBlockingGRPCServer) Stop() { 70 | for _, s := range s.servers { 71 | s.GracefulStop() 72 | } 73 | } 74 | 75 | func (s *NonBlockingGRPCServer) ForceStop() { 76 | for _, s := range s.servers { 77 | s.Stop() 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /pkg/imagefile/test/imagefile_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | Copyright (c) 2017-2019 Intel Corporation 4 | 5 | SPDX-License-Identifier: Apache-2.0 6 | 7 | */ 8 | 9 | // Package test contains a black-box test for the imagefile package. 10 | // It is a separate package to allow importing it into the E2E suite 11 | // where we can use the setup files for the current cluster to 12 | // actually use the image file inside a VM. 13 | package test 14 | 15 | import ( 16 | "testing" 17 | ) 18 | 19 | type TWrapper testing.T 20 | 21 | func (t *TWrapper) Outer(name string, cb func(t TInterface)) { 22 | (*testing.T)(t).Run(name, func(t *testing.T) { 23 | cb((*TWrapper)(t)) 24 | }) 25 | } 26 | 27 | func (t *TWrapper) Inner(name string, cb func(t TInterface)) { 28 | t.Outer(name, cb) 29 | } 30 | 31 | func (t *TWrapper) Parallel() {} 32 | 33 | // TestImageFile is used by "go test". 34 | func TestImageFile(t *testing.T) { 35 | ImageFile((*TWrapper)(t)) 36 | } 37 | -------------------------------------------------------------------------------- /pkg/k8sutil/client.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 Intel Coporation. 3 | 4 | SPDX-License-Identifier: Apache-2.0 5 | */ 6 | 7 | package k8sutil 8 | 9 | import ( 10 | "context" 11 | "fmt" 12 | "os" 13 | "regexp" 14 | "strconv" 15 | 16 | "github.com/intel/pmem-csi/pkg/version" 17 | apiclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" 18 | apierrors "k8s.io/apimachinery/pkg/api/errors" 19 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 20 | "k8s.io/client-go/discovery" 21 | "k8s.io/client-go/kubernetes" 22 | "k8s.io/client-go/rest" 23 | "k8s.io/client-go/tools/clientcmd" 24 | ) 25 | 26 | // NewClient connects to an API server either through KUBECONFIG (if set) or 27 | // through the in-cluster env variables. 28 | func NewClient(qps float64, burst int) (kubernetes.Interface, error) { 29 | var config *rest.Config 30 | var err error 31 | 32 | if kubeconfig := os.Getenv("KUBECONFIG"); kubeconfig != "" { 33 | config, err = clientcmd.BuildConfigFromFlags("" /* master */, kubeconfig) 34 | } else { 35 | config, err = rest.InClusterConfig() 36 | } 37 | if err != nil { 38 | return nil, fmt.Errorf("create Kubernetes REST config: %v", err) 39 | } 40 | config.QPS = float32(qps) 41 | config.Burst = burst 42 | client, err := kubernetes.NewForConfig(config) 43 | if err != nil { 44 | return nil, fmt.Errorf("create Kubernetes client: %v", err) 45 | } 46 | return client, nil 47 | } 48 | 49 | // GetKubernetesVersion returns kubernetes server version 50 | func GetKubernetesVersion(cfg *rest.Config) (*version.Version, error) { 51 | client, err := discovery.NewDiscoveryClientForConfig(cfg) 52 | if err != nil { 53 | return nil, err 54 | } 55 | ver, err := client.ServerVersion() 56 | if err != nil { 57 | return nil, err 58 | } 59 | 60 | // Suppress all non digits, version might contain special charcters like, + 61 | reg, _ := regexp.Compile("[^0-9]+") 62 | major, err := strconv.Atoi(reg.ReplaceAllString(ver.Major, "")) 63 | if err != nil { 64 | return nil, fmt.Errorf("failed to parse Kubernetes major version %q: %v", ver.Major, err) 65 | } 66 | minor, err := strconv.Atoi(reg.ReplaceAllString(ver.Minor, "")) 67 | if err != nil { 68 | return nil, fmt.Errorf("failed to parse Kubernetes minor version %q: %v", ver.Minor, err) 69 | } 70 | 71 | v := version.NewVersion(uint(major), uint(minor)) 72 | return &v, nil 73 | } 74 | 75 | // IsOpenShift determines whether the cluster is based on OpenShift. 76 | func IsOpenShift(cfg *rest.Config) (bool, error) { 77 | client, err := apiclient.NewForConfig(cfg) 78 | if err != nil { 79 | return false, err 80 | } 81 | // For our purposed we run on OpenShift if the scheduler operator is installed. 82 | if _, err := client.ApiextensionsV1().CustomResourceDefinitions().Get(context.Background(), "schedulers.config.openshift.io", metav1.GetOptions{}); err != nil { 83 | if apierrors.IsNotFound(err) { 84 | return false, nil 85 | } 86 | return false, fmt.Errorf("check for OpenShift CRD: %v", err) 87 | } 88 | return true, nil 89 | } 90 | -------------------------------------------------------------------------------- /pkg/k8sutil/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 Intel Coporation. 3 | 4 | SPDX-License-Identifier: Apache-2.0 5 | */ 6 | 7 | // The package k8sutil contains some helper code which simplifies 8 | // interaction with a Kubernetes cluster. 9 | package k8sutil 10 | -------------------------------------------------------------------------------- /pkg/k8sutil/namespace.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 Intel Coporation. 3 | 4 | SPDX-License-Identifier: Apache-2.0 5 | */ 6 | 7 | package k8sutil 8 | 9 | import ( 10 | "context" 11 | "io/ioutil" 12 | "os" 13 | 14 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 15 | "k8s.io/klog/v2" 16 | ) 17 | 18 | const ( 19 | namespaceEnvVar = "WATCH_NAMESPACE" 20 | namespaceFile = "/var/run/secrets/kubernetes.io/serviceaccount/namespace" 21 | defaultOperatorNamespace = metav1.NamespaceSystem 22 | ) 23 | 24 | // GetNamespace returns the namespace of the operator pod 25 | // defaults to "kube-system" 26 | func GetNamespace(ctx context.Context) string { 27 | logger := klog.FromContext(ctx).WithValues("namespace-file", namespaceFile) 28 | ns := os.Getenv(namespaceEnvVar) 29 | if ns == "" { 30 | // If environment variable not set, give it a try to fetch it from 31 | // mounted filesystem by Kubernetes 32 | data, err := ioutil.ReadFile(namespaceFile) 33 | if err != nil { 34 | logger.Info("Could not read namespace from secret, using fallback "+defaultOperatorNamespace, 35 | "error", err, 36 | ) 37 | } else { 38 | ns = string(data) 39 | logger.V(3).Info("Retrieved namespace from secret", "namespace", ns) 40 | } 41 | } 42 | 43 | if ns == "" { 44 | ns = defaultOperatorNamespace 45 | } 46 | 47 | return ns 48 | } 49 | -------------------------------------------------------------------------------- /pkg/logger/capacity.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 Intel Coporation. 3 | 4 | SPDX-License-Identifier: Apache-2.0 5 | */ 6 | 7 | package logger 8 | 9 | import ( 10 | "k8s.io/apimachinery/pkg/api/resource" 11 | ) 12 | 13 | // CapacityRef returns an object that pretty-prints the given number of bytes. 14 | func CapacityRef(size int64) *resource.Quantity { 15 | return resource.NewQuantity(size, resource.BinarySI) 16 | } 17 | -------------------------------------------------------------------------------- /pkg/logger/flag.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 Intel Coporation. 3 | 4 | SPDX-License-Identifier: Apache-2.0 5 | */ 6 | 7 | package logger 8 | 9 | import ( 10 | "flag" 11 | 12 | "k8s.io/component-base/featuregate" 13 | logsapi "k8s.io/component-base/logs/api/v1" 14 | _ "k8s.io/component-base/logs/json/register" 15 | ) 16 | 17 | func NewFlag() *Options { 18 | f := &Options{ 19 | LoggingConfiguration: *logsapi.NewLoggingConfiguration(), 20 | } 21 | flag.Var(f, "logging-format", "determines log output format, 'text' and 'json' are supported") 22 | return f 23 | } 24 | 25 | // Options is a wrapper around Options which makes 26 | // it usable with flags.Var. 27 | type Options struct { 28 | logsapi.LoggingConfiguration 29 | } 30 | 31 | func (f *Options) Set(value string) error { 32 | f.Format = value 33 | 34 | // We want contextual logging to be enabled. 35 | featureGate := featuregate.NewFeatureGate() 36 | logsapi.AddFeatureGates(featureGate) 37 | featureGate.SetFromMap(map[string]bool{ 38 | string(logsapi.ContextualLogging): true, 39 | }) 40 | 41 | return logsapi.ValidateAndApply(&f.LoggingConfiguration, featureGate) 42 | } 43 | 44 | func (f *Options) String() string { 45 | return f.Format 46 | } 47 | -------------------------------------------------------------------------------- /pkg/logger/kubernetes.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 Intel Coporation. 3 | 4 | SPDX-License-Identifier: Apache-2.0 5 | */ 6 | 7 | package logger 8 | 9 | import ( 10 | "fmt" 11 | 12 | "k8s.io/klog/v2" 13 | ) 14 | 15 | var ( 16 | KObj = klog.KObj 17 | KRef = klog.KRef 18 | ) 19 | 20 | // ObjectRef references a kubernetes object 21 | type ObjectRefWithType struct { 22 | klog.ObjectRef 23 | Type string `json:"type,omitempty"` 24 | } 25 | 26 | func (ref ObjectRefWithType) String() string { 27 | return fmt.Sprintf("%s <%s>", ref.ObjectRef.String(), ref.Type) 28 | } 29 | 30 | // KObj returns ObjectRefWithType from ObjectMeta 31 | func KObjWithType(obj klog.KMetadata) ObjectRefWithType { 32 | return ObjectRefWithType{ 33 | ObjectRef: klog.KObj(obj), 34 | Type: fmt.Sprintf("%T", obj), 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /pkg/logger/logger.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 Intel Coporation. 3 | 4 | SPDX-License-Identifier: Apache-2.0 5 | */ 6 | 7 | // Package logger defines an interface for adding a structured logger 8 | // to a context and retrieving it again. The fallback when a context 9 | // doesn't have a logger is the global klog logger. 10 | // 11 | // This uses the same context key as logr and thus is compatible 12 | // with code that uses that interface. The difference is that 13 | // the Get function here never returns nil. 14 | // 15 | // Also contains an extension of klog.KObj which includes the 16 | // type of the object in the log output. 17 | package logger 18 | 19 | import ( 20 | "context" 21 | 22 | "k8s.io/klog/v2" 23 | ) 24 | 25 | // WithName updates the logger in the context and returns the new logger and context. 26 | func WithName(ctx context.Context, name string) (context.Context, klog.Logger) { 27 | logger := klog.FromContext(ctx).WithName(name) 28 | ctx = klog.NewContext(ctx, logger) 29 | return ctx, logger 30 | } 31 | -------------------------------------------------------------------------------- /pkg/math/math.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 Intel Coporation. 3 | 4 | SPDX-License-Identifier: Apache-2.0 5 | */ 6 | 7 | // Package math contains some arithmetic helper functions. 8 | // 9 | // Origin of the GCD and LCM is the Google Playground, 10 | // with simplifications by Intel. 11 | package math 12 | 13 | // GCD returns the greatest common divisor, using the Euclidian algorithm. 14 | func GCD(a, b uint64) uint64 { 15 | for b != 0 { 16 | t := b 17 | b = a % b 18 | a = t 19 | } 20 | return a 21 | } 22 | 23 | // LCM returns the least common multiple. 24 | func LCM(a, b uint64) uint64 { 25 | return a * b / GCD(a, b) 26 | } 27 | -------------------------------------------------------------------------------- /pkg/ndctl/bus.go: -------------------------------------------------------------------------------- 1 | package ndctl 2 | 3 | //#cgo pkg-config: libndctl 4 | //#define ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0])) 5 | //#include 6 | //#include 7 | import "C" 8 | 9 | // Bus is a go wrapper for ndctl_bus. 10 | type Bus interface { 11 | // Provider returns the bus provider. 12 | Provider() string 13 | // DeviceName returns the bus device name. 14 | DeviceName() string 15 | // Dimms returns the dimms provided by the bus. 16 | Dimms() []Dimm 17 | // ActiveRegions returns all active regions in the bus. 18 | ActiveRegions() []Region 19 | // AllRegions returns all regions in the bus including disabled regions. 20 | AllRegions() []Region 21 | // GetRegionByPhysicalAddress finds a region by physical address. 22 | GetRegionByPhysicalAddress(address uint64) Region 23 | } 24 | 25 | type bus = C.struct_ndctl_bus 26 | 27 | var _ Bus = &bus{} 28 | 29 | func (b *bus) Provider() string { 30 | return C.GoString(C.ndctl_bus_get_provider(b)) 31 | } 32 | 33 | func (b *bus) DeviceName() string { 34 | return C.GoString(C.ndctl_bus_get_devname(b)) 35 | } 36 | 37 | func (b *bus) Dimms() []Dimm { 38 | var dimms []Dimm 39 | for nddimm := C.ndctl_dimm_get_first(b); nddimm != nil; nddimm = C.ndctl_dimm_get_next(nddimm) { 40 | dimms = append(dimms, nddimm) 41 | } 42 | return dimms 43 | } 44 | 45 | func (b *bus) ActiveRegions() []Region { 46 | return b.regions(true) 47 | } 48 | 49 | func (b *bus) AllRegions() []Region { 50 | return b.regions(false) 51 | } 52 | 53 | func (b *bus) GetRegionByPhysicalAddress(address uint64) Region { 54 | ndr := C.ndctl_bus_get_region_by_physical_address(b, C.ulonglong(address)) 55 | return ndr 56 | } 57 | 58 | // Strings formats all relevant attributes as JSON. 59 | func (b *bus) String() string { 60 | return marshal(map[string]interface{}{ 61 | "provider": b.Provider(), 62 | "dev": b.DeviceName(), 63 | "regions": b.ActiveRegions(), 64 | "dimms": b.Dimms(), 65 | }) 66 | } 67 | 68 | func (b *bus) regions(onlyActive bool) []Region { 69 | var regions []Region 70 | for ndr := C.ndctl_region_get_first(b); ndr != nil; ndr = C.ndctl_region_get_next(ndr) { 71 | if !onlyActive || int(C.ndctl_region_is_enabled(ndr)) == 1 { 72 | regions = append(regions, ndr) 73 | } 74 | } 75 | 76 | return regions 77 | } 78 | -------------------------------------------------------------------------------- /pkg/ndctl/dimm.go: -------------------------------------------------------------------------------- 1 | package ndctl 2 | 3 | //#cgo pkg-config: libndctl 4 | //#define ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0])) 5 | //#include 6 | //#include 7 | import "C" 8 | 9 | // Dimm is a go wrapper for ndctl_dimm. 10 | type Dimm interface { 11 | // Enabled returns if the dimm is enabled. 12 | Enabled() bool 13 | // Active returns if the the device is active. 14 | Active() bool 15 | // ID returns the dimm's unique identifier string. 16 | ID() string 17 | // PhysicalID returns the dimm's physical id number. 18 | PhysicalID() int 19 | // DeviceName returns the dimm's device name. 20 | DeviceName() string 21 | // Handle returns the dimm's handle. 22 | Handle() int16 23 | } 24 | 25 | type dimm = C.struct_ndctl_dimm 26 | 27 | var _ Dimm = &dimm{} 28 | 29 | func (d *dimm) Enabled() bool { 30 | return C.ndctl_dimm_is_enabled(d) == 1 31 | } 32 | 33 | func (d *dimm) Active() bool { 34 | return C.ndctl_dimm_is_active(d) == 1 35 | } 36 | 37 | func (d *dimm) ID() string { 38 | return C.GoString(C.ndctl_dimm_get_unique_id(d)) 39 | } 40 | 41 | func (d *dimm) PhysicalID() int { 42 | return int(C.ndctl_dimm_get_phys_id(d)) 43 | } 44 | 45 | func (d *dimm) DeviceName() string { 46 | return C.GoString(C.ndctl_dimm_get_devname(d)) 47 | } 48 | 49 | func (d *dimm) Handle() int16 { 50 | return int16(C.ndctl_dimm_get_handle(d)) 51 | } 52 | 53 | // Strings formats all relevant attributes as JSON. 54 | func (d *dimm) String() string { 55 | return marshal(map[string]interface{}{ 56 | "id": d.ID(), 57 | "dev": d.DeviceName(), 58 | "handle": d.Handle(), 59 | "phys_id": d.PhysicalID(), 60 | "enabled": d.Enabled(), 61 | }) 62 | } 63 | -------------------------------------------------------------------------------- /pkg/ndctl/fake/bus.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Intel Corporation. 3 | 4 | SPDX-License-Identifier: Apache-2.0 5 | */ 6 | 7 | package fake 8 | 9 | import ( 10 | "github.com/intel/pmem-csi/pkg/ndctl" 11 | ) 12 | 13 | type Bus struct { 14 | Provider_ string 15 | DeviceName_ string 16 | Dimms_ []ndctl.Dimm 17 | Regions_ []ndctl.Region 18 | } 19 | 20 | var _ ndctl.Bus = &Bus{} 21 | 22 | func (b *Bus) Provider() string { 23 | return b.Provider_ 24 | } 25 | 26 | func (b *Bus) DeviceName() string { 27 | return b.DeviceName_ 28 | } 29 | 30 | func (b *Bus) Dimms() []ndctl.Dimm { 31 | return b.Dimms_ 32 | } 33 | 34 | func (b *Bus) ActiveRegions() []ndctl.Region { 35 | return b.regions(true) 36 | } 37 | 38 | func (b *Bus) AllRegions() []ndctl.Region { 39 | return b.regions(false) 40 | } 41 | 42 | func (b *Bus) GetRegionByPhysicalAddress(address uint64) ndctl.Region { 43 | return nil 44 | } 45 | 46 | func (b *Bus) regions(onlyActive bool) []ndctl.Region { 47 | var regions []ndctl.Region 48 | for _, region := range b.Regions_ { 49 | if !onlyActive || region.Enabled() { 50 | regions = append(regions, region) 51 | } 52 | } 53 | return regions 54 | } 55 | -------------------------------------------------------------------------------- /pkg/ndctl/fake/dimm.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Intel Corporation. 3 | 4 | SPDX-License-Identifier: Apache-2.0 5 | */ 6 | 7 | package fake 8 | 9 | import ( 10 | "github.com/intel/pmem-csi/pkg/ndctl" 11 | ) 12 | 13 | type Dimm struct { 14 | Enabled_ bool 15 | Active_ bool 16 | ID_ string 17 | PhysicalID_ int 18 | DeviceName_ string 19 | Handle_ int16 20 | } 21 | 22 | var _ ndctl.Dimm = &Dimm{} 23 | 24 | func (d *Dimm) Enabled() bool { 25 | return d.Enabled_ 26 | } 27 | 28 | func (d *Dimm) Active() bool { 29 | return d.Active_ 30 | } 31 | 32 | func (d *Dimm) ID() string { 33 | return d.ID_ 34 | } 35 | 36 | func (d *Dimm) PhysicalID() int { 37 | return d.PhysicalID_ 38 | } 39 | 40 | func (d *Dimm) DeviceName() string { 41 | return d.DeviceName_ 42 | } 43 | 44 | func (d *Dimm) Handle() int16 { 45 | return d.Handle_ 46 | } 47 | -------------------------------------------------------------------------------- /pkg/ndctl/fake/mapping.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Intel Corporation. 3 | 4 | SPDX-License-Identifier: Apache-2.0 5 | */ 6 | package fake 7 | 8 | import ( 9 | "github.com/intel/pmem-csi/pkg/ndctl" 10 | ) 11 | 12 | type Mapping struct { 13 | Offset_ uint64 14 | Length_ uint64 15 | Position_ int 16 | 17 | Region_ ndctl.Region 18 | Dimm_ ndctl.Dimm 19 | } 20 | 21 | var _ ndctl.Mapping = &Mapping{} 22 | 23 | func (m *Mapping) Offset() uint64 { 24 | return m.Offset_ 25 | } 26 | 27 | func (m *Mapping) Length() uint64 { 28 | return m.Length_ 29 | } 30 | 31 | func (m *Mapping) Position() int { 32 | return m.Position_ 33 | } 34 | 35 | func (m *Mapping) Region() ndctl.Region { 36 | return m.Region_ 37 | } 38 | 39 | func (m *Mapping) Dimm() ndctl.Dimm { 40 | return m.Dimm_ 41 | } 42 | -------------------------------------------------------------------------------- /pkg/ndctl/fake/namespace.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Intel Corporation. 3 | 4 | SPDX-License-Identifier: Apache-2.0 5 | */ 6 | 7 | package fake 8 | 9 | import ( 10 | "github.com/google/uuid" 11 | "github.com/intel/pmem-csi/pkg/ndctl" 12 | ) 13 | 14 | type Namespace struct { 15 | ID_ uint 16 | Name_ string 17 | DeviceName_ string 18 | BlockDeviceName_ string 19 | Size_ uint64 20 | Overhead_ uint64 21 | Mode_ ndctl.NamespaceMode 22 | Type_ ndctl.NamespaceType 23 | Enabled_ bool 24 | Active_ bool 25 | UUID_ uuid.UUID 26 | Location_ ndctl.MapLocation 27 | 28 | Region_ ndctl.Region 29 | } 30 | 31 | var _ ndctl.Namespace = &Namespace{} 32 | 33 | func (ns *Namespace) ID() uint { 34 | return ns.ID_ 35 | } 36 | 37 | func (ns *Namespace) Name() string { 38 | return ns.Name_ 39 | } 40 | 41 | func (ns *Namespace) DeviceName() string { 42 | return ns.DeviceName_ 43 | } 44 | 45 | func (ns *Namespace) BlockDeviceName() string { 46 | return ns.BlockDeviceName_ 47 | } 48 | 49 | func (ns *Namespace) Size() uint64 { 50 | return ns.Size_ 51 | } 52 | 53 | func (ns *Namespace) RawSize() uint64 { 54 | return ns.Size_ + ns.Overhead_ 55 | } 56 | 57 | func (ns *Namespace) Mode() ndctl.NamespaceMode { 58 | return ns.Mode_ 59 | } 60 | 61 | func (ns *Namespace) Type() ndctl.NamespaceType { 62 | return ns.Type_ 63 | } 64 | 65 | func (ns *Namespace) Enabled() bool { 66 | return ns.Enabled_ 67 | } 68 | 69 | func (ns *Namespace) Active() bool { 70 | return ns.Active_ 71 | } 72 | 73 | func (ns *Namespace) UUID() uuid.UUID { 74 | return ns.UUID_ 75 | } 76 | 77 | func (ns *Namespace) Location() ndctl.MapLocation { 78 | return ns.Location_ 79 | } 80 | 81 | func (ns *Namespace) Region() ndctl.Region { 82 | return ns.Region_ 83 | } 84 | 85 | func (ns *Namespace) SetAltName(name string) error { 86 | return nil 87 | } 88 | 89 | func (ns *Namespace) SetSize(size uint64) error { 90 | ns.Size_ = size 91 | return nil 92 | } 93 | 94 | func (ns *Namespace) SetUUID(uid uuid.UUID) error { 95 | ns.UUID_ = uid 96 | return nil 97 | } 98 | 99 | func (ns *Namespace) SetSectorSize(sectorSize uint64) error { 100 | return nil 101 | } 102 | 103 | func (ns *Namespace) SetEnforceMode(mode ndctl.NamespaceMode) error { 104 | ns.Mode_ = mode 105 | return nil 106 | } 107 | 108 | func (ns *Namespace) Enable() error { 109 | ns.Enabled_ = true 110 | return nil 111 | } 112 | 113 | func (ns *Namespace) Disable() error { 114 | ns.Enabled_ = false 115 | return nil 116 | } 117 | 118 | func (ns *Namespace) SetRawMode(raw bool) error { 119 | return nil 120 | } 121 | 122 | func (ns *Namespace) SetPfnSeed(loc ndctl.MapLocation, align uint64) error { 123 | return nil 124 | } 125 | -------------------------------------------------------------------------------- /pkg/ndctl/fake/ndctl.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Intel Corporation. 3 | 4 | SPDX-License-Identifier: Apache-2.0 5 | */ 6 | 7 | package fake 8 | 9 | import ( 10 | "github.com/intel/pmem-csi/pkg/ndctl" 11 | ) 12 | 13 | const ( 14 | kib uint64 = 1024 15 | kib4 uint64 = kib * 4 16 | mib uint64 = kib * 1024 17 | mib2 uint64 = mib * 2 18 | ) 19 | 20 | type Context struct { 21 | Buses []ndctl.Bus 22 | } 23 | 24 | var _ ndctl.Context = &Context{} 25 | 26 | // NewContext initializes the cross-references between 27 | // items. 28 | func NewContext(ctx *Context) *Context { 29 | for _, bus := range ctx.Buses { 30 | bus, ok := bus.(*Bus) 31 | if !ok { 32 | continue 33 | } 34 | for _, region := range bus.Regions_ { 35 | region, ok := region.(*Region) 36 | if !ok { 37 | continue 38 | } 39 | for _, mapping := range region.Mappings_ { 40 | if mapping, ok := mapping.(*Mapping); ok { 41 | mapping.Region_ = region 42 | } 43 | } 44 | for _, namespace := range region.Namespaces_ { 45 | if namespace, ok := namespace.(*Namespace); ok { 46 | namespace.Region_ = region 47 | } 48 | } 49 | region.Bus_ = bus 50 | } 51 | } 52 | 53 | return ctx 54 | } 55 | 56 | func (ctx *Context) Free() { 57 | } 58 | 59 | func (ctx *Context) GetBuses() []ndctl.Bus { 60 | return ctx.Buses 61 | } 62 | -------------------------------------------------------------------------------- /pkg/ndctl/mapping.go: -------------------------------------------------------------------------------- 1 | package ndctl 2 | 3 | //#cgo pkg-config: libndctl 4 | //#include 5 | import "C" 6 | 7 | // Mapping is a go wrapper for ndctl_mapping. 8 | type Mapping interface { 9 | // Offset returns the offset within the region. 10 | Offset() uint64 11 | // Length returns the mapping's length. 12 | Length() uint64 13 | // Position returns the mapping's position. 14 | Position() int 15 | // Region gets the associated region. 16 | Region() Region 17 | // Dimm gets the associated dimm. 18 | Dimm() Dimm 19 | } 20 | 21 | type mapping = C.struct_ndctl_mapping 22 | 23 | var _ Mapping = &mapping{} 24 | 25 | func (m *mapping) Offset() uint64 { 26 | return uint64(C.ndctl_mapping_get_offset(m)) 27 | } 28 | 29 | func (m *mapping) Length() uint64 { 30 | return uint64(C.ndctl_mapping_get_length(m)) 31 | } 32 | 33 | func (m *mapping) Position() int { 34 | return int(C.ndctl_mapping_get_position(m)) 35 | } 36 | 37 | func (m *mapping) Region() Region { 38 | return C.ndctl_mapping_get_region(m) 39 | } 40 | 41 | func (m *mapping) Dimm() Dimm { 42 | return C.ndctl_mapping_get_dimm(m) 43 | } 44 | 45 | // Strings formats all relevant attributes as JSON. 46 | func (m *mapping) String() string { 47 | return marshal(map[string]interface{}{ 48 | "dimm": m.Dimm().DeviceName(), 49 | "offset": m.Offset(), 50 | "length": m.Length(), 51 | "position": m.Position(), 52 | }) 53 | } 54 | -------------------------------------------------------------------------------- /pkg/ndctl/marshal.go: -------------------------------------------------------------------------------- 1 | package ndctl 2 | 3 | import "encoding/json" 4 | 5 | func marshal(item interface{}) string { 6 | data, err := json.Marshal(item) 7 | if err != nil { 8 | return err.Error() 9 | } 10 | return string(data) 11 | } 12 | -------------------------------------------------------------------------------- /pkg/pmem-common/termination.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 Intel Coporation. 3 | 4 | SPDX-License-Identifier: Apache-2.0 5 | */ 6 | 7 | package pmemcommon 8 | 9 | import ( 10 | "fmt" 11 | "io/ioutil" 12 | "os" 13 | ) 14 | 15 | func ExitError(msg string, e error) { 16 | str := msg + ": " + e.Error() 17 | fmt.Println(str) 18 | terminationMsgPath := os.Getenv("TERMINATION_LOG_PATH") 19 | if terminationMsgPath != "" { 20 | err := ioutil.WriteFile(terminationMsgPath, []byte(str), os.FileMode(0644)) 21 | if err != nil { 22 | fmt.Println("Can not create termination log file:" + terminationMsgPath) 23 | } 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /pkg/pmem-common/tracing.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 The Kubernetes Authors. 3 | Copyright 2018 Intel Corporation. 4 | 5 | SPDX-License-Identifier: Apache-2.0 6 | */ 7 | 8 | package pmemcommon 9 | 10 | import ( 11 | "github.com/kubernetes-csi/csi-lib-utils/protosanitizer" 12 | "golang.org/x/net/context" 13 | "google.golang.org/grpc" 14 | "k8s.io/klog/v2" 15 | ) 16 | 17 | // LogGRPCServer logs the server-side call information via klog. 18 | func LogGRPCServer(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { 19 | logger := klog.FromContext(ctx) 20 | values := []interface{}{"full-method", info.FullMethod} 21 | if logger.V(5).Enabled() { 22 | values = append(values, "request", protosanitizer.StripSecrets(req)) 23 | } 24 | logger.V(3).Info("Processing gRPC call", values...) 25 | resp, err := handler(ctx, req) 26 | if err != nil { 27 | logger.Error(err, "gRPC call failed") 28 | } else { 29 | logger.V(5).Info("Completed gRPC call", "response", protosanitizer.StripSecrets(resp)) 30 | } 31 | return resp, err 32 | } 33 | 34 | // LogGRPCClient does the same as LogGRPCServer, only on the client side. 35 | func LogGRPCClient(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { 36 | logger := klog.FromContext(ctx) 37 | values := []interface{}{"full-method", method} 38 | if logger.V(5).Enabled() { 39 | values = append(values, "request", protosanitizer.StripSecrets(req)) 40 | } 41 | logger.V(3).Info("Invoking gRPC call", values...) 42 | err := invoker(ctx, method, req, reply, cc, opts...) 43 | if err != nil { 44 | logger.Error(err, "Received gRPC error") 45 | } else { 46 | logger.V(5).Info("Received gRPC response", "response", protosanitizer.StripSecrets(reply)) 47 | } 48 | return err 49 | } 50 | -------------------------------------------------------------------------------- /pkg/pmem-common/vgname.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 Intel Coporation. 3 | 4 | SPDX-License-Identifier: Apache-2.0 5 | */ 6 | 7 | package pmemcommon 8 | 9 | import ( 10 | "github.com/intel/pmem-csi/pkg/ndctl" 11 | ) 12 | 13 | func VgName(bus ndctl.Bus, region ndctl.Region) string { 14 | // Hard-coded string to indicate all namespaces are in "FSDAX" mode. 15 | nsmode := "fsdax" 16 | // This is present to avoid API break: names used to indicate nsmode 17 | // before the sector-mode support was dropped. 18 | return bus.DeviceName() + region.DeviceName() + nsmode 19 | } 20 | -------------------------------------------------------------------------------- /pkg/pmem-csi-driver/controllerserver-default.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 The Kubernetes Authors. 3 | 4 | SPDX-License-Identifier: Apache-2.0 5 | */ 6 | 7 | package pmemcsidriver 8 | 9 | import ( 10 | "github.com/container-storage-interface/spec/lib/go/csi" 11 | "golang.org/x/net/context" 12 | "google.golang.org/grpc/codes" 13 | "google.golang.org/grpc/status" 14 | ) 15 | 16 | type DefaultControllerServer struct { 17 | serviceCaps []*csi.ControllerServiceCapability 18 | } 19 | 20 | func NewDefaultControllerServer(caps []csi.ControllerServiceCapability_RPC_Type) *DefaultControllerServer { 21 | 22 | serviceCaps := []*csi.ControllerServiceCapability{} 23 | for _, cap := range caps { 24 | serviceCaps = append(serviceCaps, &csi.ControllerServiceCapability{ 25 | Type: &csi.ControllerServiceCapability_Rpc{ 26 | Rpc: &csi.ControllerServiceCapability_RPC{ 27 | Type: cap, 28 | }, 29 | }, 30 | }) 31 | } 32 | 33 | return &DefaultControllerServer{ 34 | serviceCaps: serviceCaps, 35 | } 36 | } 37 | 38 | func (cs *DefaultControllerServer) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error) { 39 | return nil, status.Error(codes.Unimplemented, "") 40 | } 41 | 42 | func (cs *DefaultControllerServer) DeleteVolume(ctx context.Context, req *csi.DeleteVolumeRequest) (*csi.DeleteVolumeResponse, error) { 43 | return nil, status.Error(codes.Unimplemented, "") 44 | } 45 | 46 | func (cs *DefaultControllerServer) ControllerPublishVolume(ctx context.Context, req *csi.ControllerPublishVolumeRequest) (*csi.ControllerPublishVolumeResponse, error) { 47 | return nil, status.Error(codes.Unimplemented, "") 48 | } 49 | 50 | func (cs *DefaultControllerServer) ControllerUnpublishVolume(ctx context.Context, req *csi.ControllerUnpublishVolumeRequest) (*csi.ControllerUnpublishVolumeResponse, error) { 51 | return nil, status.Error(codes.Unimplemented, "") 52 | } 53 | 54 | func (cs *DefaultControllerServer) ListVolumes(ctx context.Context, req *csi.ListVolumesRequest) (*csi.ListVolumesResponse, error) { 55 | return nil, status.Error(codes.Unimplemented, "") 56 | } 57 | 58 | func (cs *DefaultControllerServer) GetCapacity(ctx context.Context, req *csi.GetCapacityRequest) (*csi.GetCapacityResponse, error) { 59 | return nil, status.Error(codes.Unimplemented, "") 60 | } 61 | 62 | // ControllerGetCapabilities implements the default GRPC callout. 63 | // Default supports all capabilities 64 | func (cs *DefaultControllerServer) ControllerGetCapabilities(ctx context.Context, req *csi.ControllerGetCapabilitiesRequest) (*csi.ControllerGetCapabilitiesResponse, error) { 65 | return &csi.ControllerGetCapabilitiesResponse{ 66 | Capabilities: cs.serviceCaps, 67 | }, nil 68 | } 69 | 70 | func (cs *DefaultControllerServer) CreateSnapshot(ctx context.Context, req *csi.CreateSnapshotRequest) (*csi.CreateSnapshotResponse, error) { 71 | return nil, status.Error(codes.Unimplemented, "") 72 | } 73 | 74 | func (cs *DefaultControllerServer) DeleteSnapshot(ctx context.Context, req *csi.DeleteSnapshotRequest) (*csi.DeleteSnapshotResponse, error) { 75 | return nil, status.Error(codes.Unimplemented, "") 76 | } 77 | 78 | func (cs *DefaultControllerServer) ListSnapshots(ctx context.Context, req *csi.ListSnapshotsRequest) (*csi.ListSnapshotsResponse, error) { 79 | return nil, status.Error(codes.Unimplemented, "") 80 | } 81 | 82 | func (cs *DefaultControllerServer) ValidateControllerServiceRequest(c csi.ControllerServiceCapability_RPC_Type) error { 83 | if c == csi.ControllerServiceCapability_RPC_UNKNOWN { 84 | return nil 85 | } 86 | 87 | for _, cap := range cs.serviceCaps { 88 | if c == cap.GetRpc().GetType() { 89 | return nil 90 | } 91 | } 92 | return status.Error(codes.InvalidArgument, string(c)) 93 | } 94 | -------------------------------------------------------------------------------- /pkg/pmem-csi-driver/identityserver.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 The Kubernetes Authors. 3 | 4 | SPDX-License-Identifier: Apache-2.0 5 | */ 6 | 7 | package pmemcsidriver 8 | 9 | import ( 10 | csi "github.com/container-storage-interface/spec/lib/go/csi" 11 | grpcserver "github.com/intel/pmem-csi/pkg/grpc-server" 12 | "golang.org/x/net/context" 13 | "google.golang.org/grpc" 14 | ) 15 | 16 | type identityServer struct { 17 | name string 18 | version string 19 | pluginCaps []*csi.PluginCapability 20 | } 21 | 22 | var _ grpcserver.Service = &identityServer{} 23 | 24 | func NewIdentityServer(name, version string) *identityServer { 25 | return &identityServer{ 26 | name: name, 27 | version: version, 28 | pluginCaps: []*csi.PluginCapability{ 29 | { 30 | Type: &csi.PluginCapability_Service_{ 31 | Service: &csi.PluginCapability_Service{ 32 | Type: csi.PluginCapability_Service_CONTROLLER_SERVICE, 33 | }, 34 | }, 35 | }, 36 | { 37 | Type: &csi.PluginCapability_Service_{ 38 | Service: &csi.PluginCapability_Service{ 39 | Type: csi.PluginCapability_Service_VOLUME_ACCESSIBILITY_CONSTRAINTS, 40 | }, 41 | }, 42 | }, 43 | }, 44 | } 45 | } 46 | 47 | func (ids *identityServer) RegisterService(rpcServer *grpc.Server) { 48 | csi.RegisterIdentityServer(rpcServer, ids) 49 | } 50 | 51 | func (ids *identityServer) GetPluginInfo(ctx context.Context, req *csi.GetPluginInfoRequest) (*csi.GetPluginInfoResponse, error) { 52 | return &csi.GetPluginInfoResponse{ 53 | Name: ids.name, 54 | VendorVersion: ids.version, 55 | }, nil 56 | } 57 | 58 | func (ids *identityServer) Probe(ctx context.Context, req *csi.ProbeRequest) (*csi.ProbeResponse, error) { 59 | return &csi.ProbeResponse{}, nil 60 | } 61 | 62 | func (ids *identityServer) GetPluginCapabilities(ctx context.Context, req *csi.GetPluginCapabilitiesRequest) (*csi.GetPluginCapabilitiesResponse, error) { 63 | 64 | return &csi.GetPluginCapabilitiesResponse{ 65 | Capabilities: ids.pluginCaps, 66 | }, nil 67 | } 68 | -------------------------------------------------------------------------------- /pkg/pmem-csi-driver/pmem-csi-driver_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019,2020 Intel Corporation 3 | 4 | SPDX-License-Identifier: Apache-2.0 5 | */ 6 | 7 | package pmemcsidriver 8 | 9 | import ( 10 | "bytes" 11 | "context" 12 | "fmt" 13 | "io/ioutil" 14 | "net/http" 15 | "testing" 16 | 17 | "github.com/stretchr/testify/assert" 18 | "github.com/stretchr/testify/require" 19 | ) 20 | 21 | func TestMetrics(t *testing.T) { 22 | cases := map[string]struct { 23 | path string 24 | response http.Response 25 | }{ 26 | "version": { 27 | response: http.Response{ 28 | StatusCode: 200, 29 | Body: ioutil.NopCloser(bytes.NewBufferString(`# HELP build_info A metric with a constant '1' value labeled by version. 30 | # TYPE build_info gauge 31 | build_info{version="foo-bar-test"} 1 32 | `)), 33 | }, 34 | }, 35 | "not found": { 36 | path: "/invalid", 37 | response: http.Response{ 38 | StatusCode: 404, 39 | }, 40 | }, 41 | } 42 | for n, c := range cases { 43 | t.Run(n, func(t *testing.T) { 44 | path := "/metrics2" 45 | pmemd, err := GetCSIDriver(Config{ 46 | Mode: Controller, 47 | DriverName: "pmem-csi", 48 | NodeID: "testnode", 49 | Endpoint: "unused", 50 | Version: "foo-bar-test", 51 | metricsPath: path, 52 | metricsListen: "127.0.0.1:", // port allocated dynamically 53 | }) 54 | require.NoError(t, err, "get PMEM-CSI driver") 55 | 56 | ctx, cancel := context.WithCancel(context.Background()) 57 | defer cancel() 58 | addr, err := pmemd.startMetrics(ctx, cancel) 59 | require.NoError(t, err, "start server") 60 | 61 | tr := &http.Transport{} 62 | defer tr.CloseIdleConnections() 63 | client := &http.Client{ 64 | Transport: tr, 65 | } 66 | url := fmt.Sprintf("http://%s%s%s", addr, path, c.path) 67 | resp, err := client.Get(url) 68 | checkResponse(t, &c.response, resp, err, n) 69 | }) 70 | } 71 | } 72 | 73 | func checkResponse(t *testing.T, expected, actual *http.Response, err error, what string) { 74 | if assert.NoError(t, err, what) { 75 | defer actual.Body.Close() 76 | 77 | assert.Equal(t, expected.StatusCode, actual.StatusCode, what) 78 | actualBody, err := ioutil.ReadAll(actual.Body) 79 | if assert.NoError(t, err, "read actual body") && 80 | expected.Body != nil { 81 | expectedBody, err := ioutil.ReadAll(expected.Body) 82 | if assert.NoError(t, err, "read expected body") { 83 | // Substring search because the full body contains a lot of additional metrics data. 84 | assert.Contains(t, string(actualBody), string(expectedBody), "body") 85 | } 86 | } 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /pkg/pmem-csi-operator/controller/controller.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | SPDX-License-Identifier: Apache-2.0 5 | */ 6 | 7 | package controller 8 | 9 | import ( 10 | "context" 11 | 12 | "github.com/intel/pmem-csi/pkg/version" 13 | v1 "k8s.io/client-go/kubernetes/typed/core/v1" 14 | "k8s.io/client-go/rest" 15 | "sigs.k8s.io/controller-runtime/pkg/manager" 16 | ) 17 | 18 | // ControllerOptions type defintions for options to be passed to reconcile controller 19 | type ControllerOptions struct { 20 | // K8sVersion represents version of the running Kubernetes cluster/API server 21 | K8sVersion version.Version 22 | // Namespace to use for namespace-scoped sub-resources created by the controller 23 | Namespace string 24 | // DriverImage to use as default image for driver deployment 25 | DriverImage string 26 | // Config kubernetes config used 27 | Config *rest.Config 28 | // EventClient events client to use for recording events 29 | EventsClient v1.EventInterface 30 | } 31 | 32 | // AddToManagerFuncs is a list of functions to add all Controllers to the Manager 33 | var AddToManagerFuncs []func(context.Context, manager.Manager, ControllerOptions) error 34 | 35 | // AddToManager adds all Controllers to the Manager 36 | func AddToManager(ctx context.Context, m manager.Manager, opts ControllerOptions) error { 37 | for _, f := range AddToManagerFuncs { 38 | if err := f(ctx, m, opts); err != nil { 39 | return err 40 | } 41 | } 42 | return nil 43 | } 44 | -------------------------------------------------------------------------------- /pkg/pmem-device-manager/metrics.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 Intel Corporation. 3 | 4 | SPDX-License-Identifier: Apache-2.0 5 | */ 6 | 7 | package pmdmanager 8 | 9 | import ( 10 | "context" 11 | 12 | "k8s.io/klog/v2" 13 | 14 | "github.com/prometheus/client_golang/prometheus" 15 | ) 16 | 17 | var ( 18 | pmemMaxDesc = prometheus.NewDesc( 19 | "pmem_amount_max_volume_size", 20 | "The size of the largest PMEM volume that can be created.", 21 | nil, nil, 22 | ) 23 | pmemAvailableDesc = prometheus.NewDesc( 24 | "pmem_amount_available", 25 | "Remaining amount of PMEM on the host that can be used for new volumes.", 26 | nil, nil, 27 | ) 28 | pmemManagedDesc = prometheus.NewDesc( 29 | "pmem_amount_managed", 30 | "Amount of PMEM on the host that is managed by PMEM-CSI.", 31 | nil, nil, 32 | ) 33 | pmemTotalDesc = prometheus.NewDesc( 34 | "pmem_amount_total", 35 | "Total amount of PMEM on the host.", 36 | nil, nil, 37 | ) 38 | ) 39 | 40 | // NodeLabel is a label used for Prometheus which identifies the 41 | // node that the controller talks to. 42 | const NodeLabel = "node" 43 | 44 | // CapacityCollector is a wrapper around a PMEM device manager which 45 | // takes GetCapacity values and turns them into metrics data. 46 | type CapacityCollector struct { 47 | PmemDeviceCapacity 48 | } 49 | 50 | // MustRegister adds the collector to the registry, using labels to tag each sample with node and driver name. 51 | func (cc CapacityCollector) MustRegister(reg prometheus.Registerer, nodeName, driverName string) { 52 | labels := prometheus.Labels{ 53 | NodeLabel: nodeName, 54 | "driver_name": driverName, // same label name as in csi-lib-utils for CSI gRPC calls 55 | } 56 | prometheus.WrapRegistererWith(labels, reg).MustRegister(cc) 57 | } 58 | 59 | // Describe implements prometheus.Collector.Describe. 60 | func (cc CapacityCollector) Describe(ch chan<- *prometheus.Desc) { 61 | prometheus.DescribeByCollect(cc, ch) 62 | } 63 | 64 | // Collect implements prometheus.Collector.Collect. 65 | func (cc CapacityCollector) Collect(ch chan<- prometheus.Metric) { 66 | ctx := context.TODO() // would be nicer to get it from caller 67 | logger := klog.FromContext(ctx).WithName("Prometheus Collect") 68 | klog.NewContext(ctx, logger) 69 | 70 | capacity, err := cc.GetCapacity(ctx) 71 | if err != nil { 72 | return 73 | } 74 | ch <- prometheus.MustNewConstMetric( 75 | pmemMaxDesc, 76 | prometheus.GaugeValue, 77 | float64(capacity.MaxVolumeSize), 78 | ) 79 | ch <- prometheus.MustNewConstMetric( 80 | pmemAvailableDesc, 81 | prometheus.GaugeValue, 82 | float64(capacity.Available), 83 | ) 84 | ch <- prometheus.MustNewConstMetric( 85 | pmemManagedDesc, 86 | prometheus.GaugeValue, 87 | float64(capacity.Managed), 88 | ) 89 | ch <- prometheus.MustNewConstMetric( 90 | pmemTotalDesc, 91 | prometheus.GaugeValue, 92 | float64(capacity.Total), 93 | ) 94 | } 95 | 96 | var _ prometheus.Collector = CapacityCollector{} 97 | -------------------------------------------------------------------------------- /pkg/pmem-device-manager/pmd-fake.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 Intel Corporation. 3 | 4 | SPDX-License-Identifier: Apache-2.0 5 | */ 6 | 7 | package pmdmanager 8 | 9 | import ( 10 | "context" 11 | "fmt" 12 | "sync" 13 | 14 | api "github.com/intel/pmem-csi/pkg/apis/pmemcsi/v1beta1" 15 | "github.com/intel/pmem-csi/pkg/pmem-csi-driver/parameters" 16 | 17 | pmemerr "github.com/intel/pmem-csi/pkg/errors" 18 | ) 19 | 20 | type fakeDM struct { 21 | capacity uint64 22 | mutex sync.Mutex 23 | 24 | devices map[string]*PmemDeviceInfo 25 | } 26 | 27 | var _ PmemDeviceManager = &fakeDM{} 28 | 29 | const totalCapacity uint64 = 1024 * 1024 * 1024 * 1024 30 | 31 | // NewFake instantiates a fake PMEM device manager. The overall capacity 32 | // is hard-coded as 1TB. Usable capacity can be configured via the 33 | // percentage. Space is assumed to be contiguous with no fragmentation 34 | // issues. 35 | func newFake(pmemPercentage uint) (PmemDeviceManager, error) { 36 | if pmemPercentage > 100 { 37 | return nil, fmt.Errorf("invalid pmemPercentage '%d'. Value must be 0..100", pmemPercentage) 38 | } 39 | 40 | return &fakeDM{ 41 | capacity: uint64(pmemPercentage) * totalCapacity / 100, 42 | devices: map[string]*PmemDeviceInfo{}, 43 | }, nil 44 | } 45 | 46 | func (dm *fakeDM) GetMode() api.DeviceMode { 47 | return api.DeviceModeFake 48 | } 49 | 50 | func (dm *fakeDM) GetCapacity(ctx context.Context) (capacity Capacity, err error) { 51 | dm.mutex.Lock() 52 | defer dm.mutex.Unlock() 53 | 54 | return dm.getCapacity(), nil 55 | } 56 | 57 | func (dm *fakeDM) getCapacity() Capacity { 58 | remaining := dm.capacity 59 | for _, dev := range dm.devices { 60 | remaining -= dev.Size 61 | } 62 | return Capacity{ 63 | Available: remaining, 64 | MaxVolumeSize: remaining, 65 | Managed: dm.capacity, 66 | Total: totalCapacity, 67 | } 68 | } 69 | 70 | func (dm *fakeDM) CreateDevice(ctx context.Context, volumeId string, size uint64, usage parameters.Usage) (uint64, error) { 71 | dm.mutex.Lock() 72 | defer dm.mutex.Unlock() 73 | 74 | _, ok := dm.devices[volumeId] 75 | if ok { 76 | return 0, pmemerr.DeviceExists 77 | } 78 | 79 | if size > dm.getCapacity().MaxVolumeSize { 80 | return 0, pmemerr.NotEnoughSpace 81 | } 82 | 83 | dm.devices[volumeId] = &PmemDeviceInfo{ 84 | VolumeId: volumeId, 85 | Size: size, 86 | Path: FakeDevicePathPrefix + volumeId, 87 | } 88 | return size, nil 89 | } 90 | 91 | func (dm *fakeDM) DeleteDevice(ctx context.Context, volumeId string, flush bool) error { 92 | dm.mutex.Lock() 93 | defer dm.mutex.Unlock() 94 | 95 | // Remove device, whether it exists or not. 96 | delete(dm.devices, volumeId) 97 | 98 | return nil 99 | } 100 | 101 | func (dm *fakeDM) ListDevices(ctx context.Context) ([]*PmemDeviceInfo, error) { 102 | dm.mutex.Lock() 103 | defer dm.mutex.Unlock() 104 | 105 | devices := []*PmemDeviceInfo{} 106 | for _, dev := range dm.devices { 107 | devices = append(devices, dev) 108 | } 109 | 110 | return devices, nil 111 | } 112 | 113 | func (dm *fakeDM) GetDevice(ctx context.Context, volumeId string) (*PmemDeviceInfo, error) { 114 | dm.mutex.Lock() 115 | defer dm.mutex.Unlock() 116 | 117 | dev, ok := dm.devices[volumeId] 118 | if !ok { 119 | return nil, pmemerr.DeviceNotFound 120 | } 121 | return dev, nil 122 | } 123 | -------------------------------------------------------------------------------- /pkg/pmem-device-manager/pmd-util.go: -------------------------------------------------------------------------------- 1 | package pmdmanager 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "os" 7 | "strconv" 8 | "time" 9 | 10 | "k8s.io/klog/v2" 11 | 12 | pmemerr "github.com/intel/pmem-csi/pkg/errors" 13 | pmemexec "github.com/intel/pmem-csi/pkg/exec" 14 | "golang.org/x/sys/unix" 15 | ) 16 | 17 | const ( 18 | retryStatTimeout time.Duration = 100 * time.Millisecond 19 | ) 20 | 21 | func clearDevice(ctx context.Context, dev *PmemDeviceInfo, flush bool) error { 22 | logger := klog.FromContext(ctx).WithName("clearDevice").WithValues("device", dev.Path) 23 | ctx = klog.NewContext(ctx, logger) 24 | logger.V(4).Info("Starting", "flush", flush) 25 | 26 | // by default, clear 4 kbytes to avoid recognizing file system by next volume seeing data area 27 | var blocks uint64 = 4 28 | if flush { 29 | // clear all data if "erase all" asked specifically 30 | blocks = 0 31 | } 32 | 33 | // erase data on block device. 34 | // zero number of blocks causes overwriting whole device with random data. 35 | // nonzero number of blocks clears blocks*1024 bytes. 36 | // Before action, check that dev.Path exists and is device 37 | fileinfo, err := os.Stat(dev.Path) 38 | if err != nil { 39 | return fmt.Errorf("clear device: %v", err) 40 | } 41 | 42 | // Check if device 43 | if (fileinfo.Mode() & os.ModeDevice) == 0 { 44 | return fmt.Errorf("%s is not device", dev.Path) 45 | } 46 | 47 | fd, err := unix.Open(dev.Path, unix.O_RDONLY|unix.O_EXCL|unix.O_CLOEXEC, 0) 48 | defer unix.Close(fd) 49 | 50 | if err != nil { 51 | return fmt.Errorf("failed to clear device %q: %w", dev.Path, pmemerr.DeviceInUse) 52 | } 53 | 54 | if blocks == 0 { 55 | logger.V(5).Info("Wiping entire device") 56 | // shred would write n times using random data, followed by optional write of zeroes. 57 | // For faster operation, and because we consider zeroing enough for 58 | // reasonable clearing in case of a memory device, we force zero iterations 59 | // with random data, followed by one pass writing zeroes. 60 | if _, err := pmemexec.RunCommand(ctx, "shred", "-n", "0", "-z", dev.Path); err != nil { 61 | return fmt.Errorf("device shred failure: %v", err.Error()) 62 | } 63 | } else { 64 | logger.V(5).Info("Zeroing blocks at start of device", "blocks", blocks, "dev-size", dev.Size) 65 | of := "of=" + dev.Path 66 | // guard against writing more than volume size 67 | if blocks*1024 > dev.Size { 68 | blocks = dev.Size / 1024 69 | } 70 | count := "count=" + strconv.FormatUint(blocks, 10) 71 | if _, err := pmemexec.RunCommand(ctx, "dd", "if=/dev/zero", of, "bs=1024", count); err != nil { 72 | return fmt.Errorf("device zeroing failure: %v", err.Error()) 73 | } 74 | } 75 | return nil 76 | } 77 | 78 | func waitDeviceAppears(ctx context.Context, dev *PmemDeviceInfo) error { 79 | logger := klog.FromContext(ctx).WithName("waitDeviceAppears").WithValues("device", dev.Path) 80 | for i := 0; i < 10; i++ { 81 | if _, err := os.Stat(dev.Path); err == nil { 82 | return nil 83 | } 84 | 85 | logger.V(2).Info("Device does not exist, sleep and retry", 86 | "attempt", i, 87 | "timeout", retryStatTimeout, 88 | ) 89 | time.Sleep(retryStatTimeout) 90 | } 91 | return fmt.Errorf("%s: device not ready", dev.Path) 92 | } 93 | -------------------------------------------------------------------------------- /pkg/types/types.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Intel Corporation. 3 | 4 | SPDX-License-Identifier: Apache-2.0 5 | */ 6 | 7 | // Package types contains some type definitions that are used in 8 | // various places. 9 | package types 10 | 11 | import ( 12 | "bytes" 13 | "encoding/json" 14 | "strings" 15 | ) 16 | 17 | // NodeSelector is a set of unique keys and their values. 18 | type NodeSelector map[string]string 19 | 20 | // Set converts a JSON representation into a NodeSelector. 21 | func (n *NodeSelector) Set(value string) error { 22 | // Decoding into a plain map yields better error messages: 23 | // "cannot unmarshal string into Go value of type types.NodeSelector" 24 | // vs. 25 | // "cannot unmarshal string into Go value of type map[string]string" 26 | var m map[string]string 27 | if err := json.NewDecoder(bytes.NewBufferString(value)).Decode(&m); err != nil { 28 | return err 29 | } 30 | *n = m 31 | return nil 32 | } 33 | 34 | // String converts into the JSON representation expected by Set. 35 | func (n *NodeSelector) String() string { 36 | var value bytes.Buffer 37 | if err := json.NewEncoder(&value).Encode(n); err != nil { 38 | panic(err) 39 | } 40 | return strings.TrimSpace(value.String()) 41 | } 42 | 43 | // MatchesLabels returns true if all key/value pairs in the selector 44 | // are set in the labels. 45 | func (n *NodeSelector) MatchesLabels(labels map[string]string) bool { 46 | for key, value := range *n { 47 | if labels[key] != value { 48 | return false 49 | } 50 | } 51 | return true 52 | } 53 | -------------------------------------------------------------------------------- /pkg/version/version.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | SPDX-License-Identifier: Apache-2.0 5 | */ 6 | package version 7 | 8 | import ( 9 | "errors" 10 | "fmt" 11 | "strconv" 12 | "strings" 13 | ) 14 | 15 | // Version type definition for handling simple version comparision 16 | type Version struct { 17 | major, minor uint 18 | } 19 | 20 | // NewVersion creates a new version object for given 21 | // major and minor version values 22 | func NewVersion(major, minor uint) Version { 23 | return Version{ 24 | major: major, 25 | minor: minor, 26 | } 27 | } 28 | 29 | // Parse creates a new version with major/minor from the given string, which must 30 | // have the format .. Error texts do not include the version string 31 | // itself. 32 | func Parse(version string) (Version, error) { 33 | parts := strings.Split(version, ".") 34 | if len(parts) < 2 { 35 | return Version{}, errors.New("must have . format") 36 | } 37 | major, err := strconv.ParseUint(parts[0], 10, 32) 38 | if err != nil { 39 | return Version{}, fmt.Errorf("major version %q: %v", parts[0], err) 40 | } 41 | minor, err := strconv.ParseUint(parts[1], 10, 32) 42 | if err != nil { 43 | return Version{}, fmt.Errorf("minor version %q: %v", parts[1], err) 44 | } 45 | return NewVersion(uint(major), uint(minor)), nil 46 | } 47 | 48 | func (v Version) String() string { 49 | return fmt.Sprintf("%d.%d", v.major, v.minor) 50 | } 51 | 52 | // Major returns major version of v 53 | func (v Version) Major() uint { 54 | return v.major 55 | } 56 | 57 | // Minor returns minor version of v 58 | func (v Version) Minor() uint { 59 | return v.minor 60 | } 61 | 62 | // Compare compares v with given otherVersion 63 | // Returns 64 | // - 0 if two versions are same 65 | // - >0 if v is greater otherVersion 66 | // - <0 if v is less than otherVersion 67 | func (v Version) Compare(major, minor uint) int { 68 | d := int(v.major - major) 69 | if d == 0 { 70 | d = int(v.minor - minor) 71 | } 72 | 73 | return d 74 | } 75 | 76 | // CompareVersion is the same as Compare with a Version as argument. 77 | func (v Version) CompareVersion(other Version) int { 78 | return v.Compare(other.major, other.minor) 79 | } 80 | -------------------------------------------------------------------------------- /pkg/version/version_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | SPDX-License-Identifier: Apache-2.0 5 | */ 6 | package version_test 7 | 8 | import ( 9 | "testing" 10 | 11 | "github.com/intel/pmem-csi/pkg/version" 12 | "github.com/stretchr/testify/assert" 13 | ) 14 | 15 | func TestVersion(t *testing.T) { 16 | t.Run("new version", func(t *testing.T) { 17 | major, minor := uint(1), uint(6) 18 | v := version.NewVersion(major, minor) 19 | assert.Equal(t, "1.6", v.String(), "mismatched version string") 20 | assert.Equal(t, major, v.Major(), "mismatched major number") 21 | assert.Equal(t, minor, v.Minor(), "mismatched minor number") 22 | 23 | v = version.NewVersion(1, 160) 24 | assert.Equal(t, "1.160", v.String(), "mismatched version string") 25 | 26 | v = version.NewVersion(0, 6) 27 | assert.Equal(t, "0.6", v.String(), "mismatched version string") 28 | }) 29 | 30 | t.Run("version comparison", func(t *testing.T) { 31 | v := version.NewVersion(1, 10) 32 | assert.Greater(t, v.Compare(1, 5), 0, "comparision: 1.10 must be greater than 1.5") 33 | assert.Equal(t, v.Compare(1, 10), 0, "comparision: must be equal") 34 | assert.Less(t, v.Compare(1, 12), 0, "comparison: 1.10 must be less than 1.12") 35 | 36 | v = version.NewVersion(101, 1000) 37 | assert.Equal(t, v.Compare(101, 1000), 0, "comparision: must be equal") 38 | assert.Greater(t, v.Compare(10, 11000), 0, "comparision: 101.1000 must be greater than 10.11000") 39 | assert.Less(t, v.Compare(1011, 0), 0, "comparision: 101.1000 must be less than 1011.0") 40 | assert.Greater(t, v.Compare(1, 1011000), 0, "comparision: 101.1000 must be less than 1.1011000") 41 | }) 42 | 43 | invalid := []string{ 44 | "foo", 45 | "1", 46 | "a.b", 47 | "1.a", 48 | "a.2", 49 | "-1.0", 50 | "0.-1", 51 | "1000000000000000000000000.0", 52 | } 53 | valid := map[string]version.Version{ 54 | "10.1": version.NewVersion(10, 1), 55 | "2.10.1": version.NewVersion(2, 10), 56 | } 57 | 58 | t.Run("parsing", func(t *testing.T) { 59 | for _, str := range invalid { 60 | t.Run(str, func(t *testing.T) { 61 | _, err := version.Parse(str) 62 | assert.Error(t, err) 63 | }) 64 | } 65 | for str, expected := range valid { 66 | t.Run(str, func(t *testing.T) { 67 | actual, err := version.Parse(str) 68 | if assert.NoError(t, err) { 69 | assert.Equal(t, expected, actual) 70 | } 71 | }) 72 | } 73 | }) 74 | } 75 | -------------------------------------------------------------------------------- /pkg/xfs/xfs.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022 Intel Corporation 3 | 4 | SPDX-License-Identifier: Apache-2.0 5 | */ 6 | 7 | package xfs 8 | 9 | // #include 10 | // #include 11 | // #include 12 | // #include 13 | // 14 | // char *getxattr(int fd, struct fsxattr *arg) { 15 | // return ioctl(fd, FS_IOC_FSGETXATTR, arg) == 0 ? 0 : strerror(errno); 16 | // } 17 | // 18 | // char *setxattr(int fd, struct fsxattr *arg) { 19 | // return ioctl(fd, FS_IOC_FSSETXATTR, arg) == 0 ? 0 : strerror(errno); 20 | // } 21 | import "C" 22 | 23 | import ( 24 | "fmt" 25 | "os" 26 | ) 27 | 28 | // ConfigureFS must be called after mkfs.xfs for the mounted 29 | // XFS filesystem to prepare the volume for usage as fsdax. 30 | // It is idempotent. 31 | func ConfigureFS(path string) error { 32 | // Operate on root directory. 33 | file, err := os.Open(path) 34 | if err != nil { 35 | return fmt.Errorf("open %q: %v", path, err) 36 | } 37 | defer file.Close() 38 | fd := C.int(file.Fd()) 39 | 40 | // Get extended attributes. 41 | var attr C.struct_fsxattr 42 | if errnostr := C.getxattr(fd, &attr); errnostr != nil { 43 | return fmt.Errorf("FS_IOC_FSGETXATTR for %q: %v", path, C.GoString(errnostr)) 44 | } 45 | 46 | // Set extsize to 2m to enable hugepages in combination with 47 | // fsdax. This is equivalent to the "xfs_io -c 'extsize 2m'" invocation 48 | // mentioned in https://nvdimm.wiki.kernel.org/2mib_fs_dax 49 | attr.fsx_xflags |= C.FS_XFLAG_EXTSZINHERIT 50 | attr.fsx_extsize = 2 * 1024 * 1024 51 | if errnostr := C.setxattr(fd, &attr); errnostr != nil { 52 | return fmt.Errorf("FS_IOC_FSSETXATTR for %q: %v", path, C.GoString(errnostr)) 53 | } 54 | 55 | return nil 56 | } 57 | -------------------------------------------------------------------------------- /pkg/xfs/xfs_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022 Intel Corporation 3 | 4 | SPDX-License-Identifier: Apache-2.0 5 | */ 6 | 7 | package xfs 8 | 9 | import ( 10 | "testing" 11 | ) 12 | 13 | func Test_ConfigureFS(t *testing.T) { 14 | // This is assumed to be backed by tmpfs and thus doesn't support xattr. 15 | tmp := t.TempDir() 16 | err := ConfigureFS(tmp) 17 | if err == nil { 18 | t.Fatal("did not get expected error") 19 | } 20 | t.Logf("got expected error: %v", err) 21 | } 22 | -------------------------------------------------------------------------------- /runtime-deps.csv: -------------------------------------------------------------------------------- 1 | github.com/beorn7/perks/quantile,MIT 2 | github.com/blang/semver/v4,MIT 3 | github.com/cespare/xxhash/v2,MIT 4 | github.com/container-storage-interface/spec/lib/go/csi,Apache-2.0 5 | github.com/davecgh/go-spew/spew,ISC 6 | github.com/emicklei/go-restful/v3,MIT 7 | github.com/evanphx/json-patch/v5,BSD-3-Clause 8 | github.com/fsnotify/fsnotify,BSD-3-Clause 9 | github.com/go-logr/logr,Apache-2.0 10 | github.com/go-logr/stdr,Apache-2.0 11 | github.com/go-logr/zapr,Apache-2.0 12 | github.com/go-openapi/jsonpointer,Apache-2.0 13 | github.com/go-openapi/jsonreference,Apache-2.0 14 | github.com/go-openapi/swag,Apache-2.0 15 | github.com/gogo/protobuf,BSD-3-Clause 16 | github.com/golang/groupcache/lru,Apache-2.0 17 | github.com/golang/protobuf,BSD-3-Clause 18 | github.com/google/gnostic-models,Apache-2.0 19 | github.com/google/go-cmp/cmp,BSD-3-Clause 20 | github.com/google/gofuzz,Apache-2.0 21 | github.com/google/uuid,BSD-3-Clause 22 | github.com/imdario/mergo,BSD-3-Clause 23 | github.com/josharian/intern,MIT 24 | github.com/json-iterator/go,MIT 25 | github.com/kubernetes-csi/csi-lib-utils,Apache-2.0 26 | github.com/mailru/easyjson,MIT 27 | github.com/matttproud/golang_protobuf_extensions/pbutil,Apache-2.0 28 | github.com/miekg/dns,BSD-3-Clause 29 | github.com/modern-go/concurrent,Apache-2.0 30 | github.com/modern-go/reflect2,Apache-2.0 31 | github.com/munnerz/goautoneg,BSD-3-Clause 32 | github.com/pkg/errors,BSD-2-Clause 33 | github.com/prometheus/client_golang/prometheus,Apache-2.0 34 | github.com/prometheus/client_model/go,Apache-2.0 35 | github.com/prometheus/common,Apache-2.0 36 | github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg,BSD-3-Clause 37 | github.com/prometheus/procfs,Apache-2.0 38 | github.com/spf13/cobra,Apache-2.0 39 | github.com/spf13/pflag,BSD-3-Clause 40 | go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc,Apache-2.0 41 | go.opentelemetry.io/otel,Apache-2.0 42 | go.opentelemetry.io/otel/metric,Apache-2.0 43 | go.opentelemetry.io/otel/trace,Apache-2.0 44 | go.uber.org/multierr,MIT 45 | go.uber.org/zap,MIT 46 | golang.org/x/exp/maps,BSD-3-Clause 47 | golang.org/x/net,BSD-3-Clause 48 | golang.org/x/oauth2,BSD-3-Clause 49 | golang.org/x/sys/unix,BSD-3-Clause 50 | golang.org/x/term,BSD-3-Clause 51 | golang.org/x/text,BSD-3-Clause 52 | golang.org/x/time/rate,BSD-3-Clause 53 | gomodules.xyz/jsonpatch/v2,Apache-2.0 54 | google.golang.org/genproto/googleapis/rpc/status,Apache-2.0 55 | google.golang.org/grpc,Apache-2.0 56 | google.golang.org/protobuf,BSD-3-Clause 57 | gopkg.in/inf.v0,BSD-3-Clause 58 | gopkg.in/yaml.v2,Apache-2.0 59 | gopkg.in/yaml.v3,MIT 60 | k8s.io/api,Apache-2.0 61 | k8s.io/apiextensions-apiserver/pkg,Apache-2.0 62 | k8s.io/apimachinery/pkg,Apache-2.0 63 | k8s.io/apimachinery/third_party/forked/golang,BSD-3-Clause 64 | k8s.io/client-go,Apache-2.0 65 | k8s.io/component-base,Apache-2.0 66 | k8s.io/klog/v2,Apache-2.0 67 | k8s.io/kube-openapi/pkg,Apache-2.0 68 | k8s.io/kube-openapi/pkg/internal/third_party/go-json-experiment/json,BSD-3-Clause 69 | k8s.io/kube-openapi/pkg/validation/spec,Apache-2.0 70 | k8s.io/kubectl/pkg/scheme,Apache-2.0 71 | k8s.io/utils,Apache-2.0 72 | k8s.io/utils/internal/third_party/forked/golang/net,BSD-3-Clause 73 | sigs.k8s.io/controller-runtime/pkg,Apache-2.0 74 | sigs.k8s.io/json,Apache-2.0 75 | sigs.k8s.io/sig-storage-lib-external-provisioner/v6,Apache-2.0 76 | sigs.k8s.io/structured-merge-diff/v4,Apache-2.0 77 | sigs.k8s.io/yaml,Apache-2.0 78 | sigs.k8s.io/yaml/goyaml.v2,Apache-2.0 79 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from distutils.core import setup 4 | 5 | setup(name='Sphinx GUI Utility', 6 | version='0.1', 7 | description='Build Sphinx docs from a GUI', 8 | author='Kevin Putnam', 9 | author_email='kevin.putnam@intel.com', 10 | url='https://github.com/intel/pmem-csi', 11 | ) 12 | -------------------------------------------------------------------------------- /test/cmd/pmem-access-hugepages/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 The Kubernetes Authors. 3 | Copyright 2018 Intel Coporation. 4 | 5 | SPDX-License-Identifier: Apache-2.0 6 | */ 7 | 8 | /* 9 | This helper program is meant to cause a page fault event for 10 | dax-mounted hugepage related testing. This program runs in a pod. 11 | It creates a file and accesses data in it so that kernel gets a page fault, 12 | while worker host has enabled tracing showing paging events. 13 | This program prints out inode number and page address of data accessed, 14 | (catenated together without whitespace) which is then used by e2e testing 15 | code to verify that traced event was related to the action made here. 16 | */ 17 | 18 | package main 19 | 20 | import ( 21 | "fmt" 22 | "os" 23 | "syscall" 24 | "unsafe" 25 | ) 26 | 27 | func handleError(err error) { 28 | if err != nil { 29 | fmt.Println(err) 30 | os.Exit(1) 31 | } 32 | } 33 | 34 | func main() { 35 | const fname = "/mnt/hugepagedata" 36 | var stat syscall.Stat_t 37 | const size = 2*1024*1024 + 4 38 | runPid := os.Getpid() 39 | // Create the file 40 | map_file, err := os.Create(fname) 41 | handleError(err) 42 | _, err = map_file.Seek(int64(size-1), 0) 43 | handleError(err) 44 | _, err = map_file.Write([]byte(" ")) 45 | handleError(err) 46 | // Get inode number of the file 47 | err = syscall.Stat(fname, &stat) 48 | handleError(err) 49 | 50 | mmap, err := syscall.Mmap(int(map_file.Fd()), 0, size, syscall.PROT_READ|syscall.PROT_WRITE, syscall.MAP_SHARED) 51 | handleError(err) 52 | mapped := (*[size]byte)(unsafe.Pointer(&mmap[0])) 53 | for i := 1; i < size; i++ { 54 | mapped[i] = byte(runPid) 55 | } 56 | 57 | err = syscall.Munmap(mmap) 58 | handleError(err) 59 | err = map_file.Close() 60 | handleError(err) 61 | fmt.Printf("0x%x%p", stat.Ino, mapped) 62 | } 63 | -------------------------------------------------------------------------------- /test/cmd/pmem-dax-check/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 The Kubernetes Authors. 3 | Copyright 2018 Intel Coporation. 4 | 5 | SPDX-License-Identifier: Apache-2.0 6 | */ 7 | 8 | package main 9 | 10 | import ( 11 | "flag" 12 | "fmt" 13 | "os" 14 | "os/signal" 15 | "syscall" 16 | ) 17 | 18 | func main() { 19 | wait := flag.Bool("wait", false, "wait for CTRL-C while memory is mapped") 20 | size := flag.Int("size", 4096, "size of the mapped region") 21 | flag.Parse() 22 | if flag.NArg() != 1 { 23 | fmt.Fprintf(os.Stderr, "need exactly one file name as parameter\n") 24 | os.Exit(2) 25 | } 26 | 27 | code, err := run(flag.Arg(0), *size, *wait) 28 | if err != nil { 29 | fmt.Fprintf(os.Stdout, "%v\n", err) 30 | } 31 | os.Exit(code) 32 | } 33 | 34 | func run(filename string, size int, wait bool) (int, error) { 35 | prot := syscall.PROT_READ 36 | created := false 37 | 38 | // Open the file. 39 | file, err := os.OpenFile(filename, os.O_RDONLY, 0) 40 | if err != nil { 41 | if !os.IsNotExist(err) { 42 | return 2, err 43 | } 44 | file, err = os.Create(filename) 45 | if err != nil { 46 | return 2, err 47 | } 48 | defer os.Remove(filename) 49 | created = true 50 | } 51 | defer file.Close() 52 | 53 | info, err := os.Stat(filename) 54 | if err != nil { 55 | return 2, err 56 | } 57 | if info.Mode().IsRegular() && info.Size() < int64(size) { 58 | if err := syscall.Ftruncate(int(file.Fd()), int64(size)); err != nil { 59 | return 2, fmt.Errorf("enlarge %q to %d bytes: %v", filename, size, err) 60 | } 61 | } 62 | 63 | // Some flags are currently (Go 1.13.6) not available. This value is from 64 | // /usr/include/x86_64-linux-gnu/bits/mman.h and /usr/include/asm-generic/mman.h. 65 | const MAP_SYNC = 0x80000 66 | const MAP_SHARED_VALIDATE = 0x03 67 | 68 | // Try to mmap in different ways to determine whether that works in principle 69 | // (it should) and whether MAP_SYNC works (may or may not). 70 | data, err := syscall.Mmap(int(file.Fd()), 0, size, prot, MAP_SHARED_VALIDATE) 71 | if err != nil { 72 | return 2, fmt.Errorf("%s: mmap without MAP_SYNC failed: %v", filename, err) 73 | } 74 | if err := syscall.Munmap(data); err != nil { 75 | return 2, fmt.Errorf("%s: mmunmap failed: %v", filename, err) 76 | } 77 | 78 | data, err = syscall.Mmap(int(file.Fd()), 0, size, prot, MAP_SHARED_VALIDATE|MAP_SYNC) 79 | if err != nil { 80 | return 1, fmt.Errorf("%s: does not support MAP_SYNC: %v", filename, err) 81 | } 82 | 83 | // Check for non-zero bytes. This has the intended side effect that all pages 84 | // are really touched instead of just being mapped. 85 | zeroed := true 86 | for i := 0; i < size; i++ { 87 | if data[i] != 0 { 88 | zeroed = false 89 | } 90 | } 91 | 92 | if !zeroed && created { 93 | return 2, fmt.Errorf("%s: MAP_SYNC succeeded, but the newly created file contained garbage", filename) 94 | } 95 | 96 | if wait { 97 | fmt.Printf("map(MAP_SYNC) succeeded. Continue with CTRL-C.\n") 98 | exitSignal := make(chan os.Signal, 2) 99 | signal.Notify(exitSignal, syscall.SIGINT, syscall.SIGTERM) 100 | <-exitSignal 101 | } 102 | 103 | if err := syscall.Munmap(data); err != nil { 104 | return 2, fmt.Errorf("%s: mmunmap failed: %v", filename, err) 105 | } 106 | return 0, fmt.Errorf("%s: supports MAP_SYNC, content is zeroed: %v", filename, zeroed) 107 | } 108 | -------------------------------------------------------------------------------- /test/cmd/watch-pvs/watch-pvs.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 The Kubernetes Authors. 3 | Copyright 2020 Intel Coporation. 4 | 5 | SPDX-License-Identifier: Apache-2.0 6 | */ 7 | 8 | package main 9 | 10 | import ( 11 | "context" 12 | "os" 13 | "time" 14 | 15 | "github.com/google/go-cmp/cmp" 16 | "k8s.io/client-go/informers" 17 | "k8s.io/client-go/kubernetes" 18 | "k8s.io/client-go/tools/cache" 19 | "k8s.io/client-go/tools/clientcmd" 20 | "k8s.io/klog/v2" 21 | "sigs.k8s.io/yaml" 22 | ) 23 | 24 | func toYAML(obj interface{}) string { 25 | out, err := yaml.Marshal(obj) 26 | if err != nil { 27 | klog.Fatalf("marshal %+q: %v", obj, err) 28 | } 29 | return string(out) 30 | } 31 | 32 | func main() { 33 | ctx := context.Background() 34 | 35 | // get the KUBECONFIG from env if specified (useful for local/debug cluster) 36 | kubeconfigEnv := os.Getenv("KUBECONFIG") 37 | config, err := clientcmd.BuildConfigFromFlags("", kubeconfigEnv) 38 | if err != nil { 39 | klog.Fatalf("Failed to create config from KUBECONFIG=%s: %v", kubeconfigEnv, err) 40 | } 41 | 42 | clientset, err := kubernetes.NewForConfig(config) 43 | if err != nil { 44 | klog.Fatalf("Failed to create client: %v", err) 45 | } 46 | 47 | factory := informers.NewSharedInformerFactory(clientset, time.Hour) 48 | claimInformer := factory.Core().V1().PersistentVolumeClaims().Informer() 49 | volumeInformer := factory.Core().V1().PersistentVolumes().Informer() 50 | 51 | claimHandler := cache.ResourceEventHandlerFuncs{ 52 | AddFunc: func(obj interface{}) { 53 | klog.Infof("PVC added:\n%s\n", toYAML(obj)) 54 | }, 55 | UpdateFunc: func(oldObj, newObj interface{}) { 56 | klog.Infof("PVC updated:\n%s\n%s\n", 57 | toYAML(newObj), 58 | cmp.Diff(oldObj, newObj), 59 | ) 60 | }, 61 | DeleteFunc: func(obj interface{}) { 62 | klog.Infof("PVC deleted:\n%s\n", toYAML(obj)) 63 | }, 64 | } 65 | claimInformer.AddEventHandlerWithResyncPeriod(claimHandler, time.Hour) 66 | 67 | volumeHandler := cache.ResourceEventHandlerFuncs{ 68 | AddFunc: func(obj interface{}) { 69 | klog.Infof("PV added:\n%s\n", toYAML(obj)) 70 | }, 71 | UpdateFunc: func(oldObj, newObj interface{}) { 72 | klog.Infof("PV updated:\n%s\n%s\n", 73 | toYAML(newObj), 74 | cmp.Diff(oldObj, newObj), 75 | ) 76 | }, 77 | DeleteFunc: func(obj interface{}) { 78 | klog.Infof("PV deleted:\n%s\n", toYAML(obj)) 79 | }, 80 | } 81 | volumeInformer.AddEventHandlerWithResyncPeriod(volumeHandler, time.Hour) 82 | 83 | factory.Start(ctx.Done()) 84 | <-ctx.Done() 85 | } 86 | -------------------------------------------------------------------------------- /test/delete-deployment.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | TEST_DIRECTORY=${TEST_DIRECTORY:-$(dirname $(readlink -f $0))} 4 | source ${TEST_CONFIG:-${TEST_DIRECTORY}/test-config.sh} 5 | 6 | CLUSTER=${CLUSTER:-pmem-govm} 7 | REPO_DIRECTORY="${REPO_DIRECTORY:-$(dirname $(dirname $(readlink -f $0)))}" 8 | CLUSTER_DIRECTORY="${CLUSTER_DIRECTORY:-${REPO_DIRECTORY}/_work/${CLUSTER}}" 9 | SSH="${CLUSTER_DIRECTORY}/ssh.0" 10 | KUBECTL="${SSH} kubectl" # Always use the kubectl installed in the cluster. 11 | 12 | kinds=" 13 | deployments 14 | replicasets 15 | statefulsets 16 | daemonsets 17 | 18 | clusterrolebindings 19 | clusterroles 20 | crd 21 | csidrivers 22 | mutatingwebhookconfigurations 23 | pods 24 | rolebindings 25 | roles 26 | serviceaccounts 27 | services 28 | storageclasses 29 | " 30 | for dep in $(${KUBECTL} get pmemcsideployments -o 'jsonpath={.items[*].metadata.name}'); do 31 | ${KUBECTL} delete pmemcsideployments/$dep 32 | done 33 | for kind in $kinds; do 34 | echo -n "$kind: " 35 | ${KUBECTL} delete --all-namespaces -l pmem-csi.intel.com/deployment $kind 36 | ${KUBECTL} delete --all-namespaces -l app.kubernetes.io/part-of=pmem-csi $kind 37 | done 38 | -------------------------------------------------------------------------------- /test/e2e/deploy/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 Intel Corporation. 3 | 4 | SPDX-License-Identifier: Apache-2.0 5 | */ 6 | 7 | // Package deploy contains code which provides information about the 8 | // cluster and deploying PMEM-CSI inside it. 9 | package deploy 10 | -------------------------------------------------------------------------------- /test/e2e/deploy/pmem.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package deploy 18 | 19 | import ( 20 | "context" 21 | "fmt" 22 | "os" 23 | "strings" 24 | 25 | apierrors "k8s.io/apimachinery/pkg/util/errors" 26 | 27 | pmemexec "github.com/intel/pmem-csi/pkg/exec" 28 | ) 29 | 30 | func ResetPMEM(ctx context.Context, node string) error { 31 | var errs []error 32 | 33 | sshcmd := fmt.Sprintf("%s/_work/%s/ssh.%s", os.Getenv("REPO_ROOT"), os.Getenv("CLUSTER"), node) 34 | 35 | if err := resetLVM(ctx, sshcmd, "vgs", "vgremove"); err != nil { 36 | errs = append(errs, fmt.Errorf("LVM volume groups: %v", err)) 37 | } 38 | 39 | if err := resetLVM(ctx, sshcmd, "pvs", "pvremove"); err != nil { 40 | errs = append(errs, fmt.Errorf("LVM physical volumes: %v", err)) 41 | } 42 | 43 | if _, err := pmemexec.RunCommand(ctx, sshcmd, "sudo ndctl destroy-namespace --force all"); err != nil { 44 | errs = append(errs, fmt.Errorf("erasing namespaces failed: %v", err)) 45 | } 46 | 47 | return apierrors.NewAggregate(errs) 48 | } 49 | 50 | func resetLVM(ctx context.Context, sshCmd, listCmd, rmCmd string) error { 51 | out, err := pmemexec.RunCommand(ctx, sshCmd, "sudo "+listCmd+" --noheadings --options name") 52 | if err != nil { 53 | return fmt.Errorf("listing failed: %v", err) 54 | } else { 55 | for _, item := range strings.Split(string(out), "\n") { 56 | switch listCmd { 57 | case "vgs": 58 | if !strings.HasPrefix(item, "ndbus") { 59 | continue 60 | } 61 | case "pvs": 62 | if !strings.HasPrefix(item, "/dev/pmem") { 63 | continue 64 | } 65 | } 66 | 67 | if _, err := pmemexec.RunCommand(ctx, sshCmd, "sudo "+rmCmd+" -f "+item); err != nil { 68 | return fmt.Errorf("removal failed: %v", err) 69 | } 70 | } 71 | } 72 | return nil 73 | } 74 | -------------------------------------------------------------------------------- /test/e2e/driver/pattern.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package driver 18 | 19 | import ( 20 | "encoding/json" 21 | "fmt" 22 | "strings" 23 | 24 | v1 "k8s.io/api/core/v1" 25 | storageframework "k8s.io/kubernetes/test/e2e/storage/framework" 26 | ) 27 | 28 | // StorageClassParameters can be used in combination with DynamicDriver to implement test patterns 29 | // that encode additional parameters in the test pattern name. This is a workaround for the 30 | // fixed content of the original test pattern struct. 31 | type StorageClassParameters struct { 32 | FSType string 33 | Parameters map[string]string 34 | } 35 | 36 | func (scp *StorageClassParameters) Encode() (string, error) { 37 | data, err := json.Marshal(scp) 38 | return string(data), err 39 | } 40 | 41 | func (scp *StorageClassParameters) MustEncode() string { 42 | data, err := scp.Encode() 43 | if err != nil { 44 | panic(err) 45 | } 46 | return data 47 | } 48 | 49 | func (scp *StorageClassParameters) Decode(parameters string) error { 50 | return json.Unmarshal([]byte(parameters), scp) 51 | } 52 | 53 | func EncodeTestPatternName(volType storageframework.TestVolType, volMode v1.PersistentVolumeMode, scp StorageClassParameters) string { 54 | return fmt.Sprintf("%s %s %s", volType, volMode, scp.MustEncode()) 55 | } 56 | 57 | func DecodeTestPatternName(name string) (volType storageframework.TestVolType, volMode v1.PersistentVolumeMode, scp *StorageClassParameters, err error) { 58 | parts := strings.SplitN(name, " ", 3) 59 | if len(parts) != 3 { 60 | err = fmt.Errorf("not of format ' {}': %s", name) 61 | return 62 | } 63 | scp = &StorageClassParameters{} 64 | volType = storageframework.TestVolType(parts[0]) 65 | volMode = v1.PersistentVolumeMode(parts[1]) 66 | err = scp.Decode(parts[2]) 67 | return 68 | } 69 | -------------------------------------------------------------------------------- /test/e2e/e2e_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2015 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package e2e 18 | 19 | import ( 20 | "flag" 21 | "os" 22 | "testing" 23 | 24 | "k8s.io/klog/v2" 25 | "k8s.io/kubernetes/test/e2e/framework" 26 | "k8s.io/kubernetes/test/e2e/framework/config" 27 | "k8s.io/kubernetes/test/e2e/framework/testfiles" 28 | 29 | "github.com/onsi/ginkgo/v2" 30 | 31 | // test sources 32 | _ "github.com/intel/pmem-csi/test/e2e/gotests" 33 | _ "github.com/intel/pmem-csi/test/e2e/image" 34 | _ "github.com/intel/pmem-csi/test/e2e/imagefile" 35 | _ "github.com/intel/pmem-csi/test/e2e/metrics" 36 | _ "github.com/intel/pmem-csi/test/e2e/operator" 37 | _ "github.com/intel/pmem-csi/test/e2e/storage" 38 | _ "github.com/intel/pmem-csi/test/e2e/tls" 39 | _ "github.com/intel/pmem-csi/test/e2e/versionskew" 40 | 41 | "github.com/intel/pmem-csi/test/e2e/deploy" 42 | ) 43 | 44 | func TestMain(m *testing.M) { 45 | klog.SetOutput(ginkgo.GinkgoWriter) 46 | 47 | deploy.DefineTests() 48 | 49 | config.CopyFlags(config.Flags, flag.CommandLine) 50 | framework.RegisterCommonFlags(flag.CommandLine) 51 | framework.RegisterClusterFlags(flag.CommandLine) 52 | flag.Parse() 53 | 54 | // Register framework flags, then handle flags. 55 | framework.AfterReadingAllFlags(&framework.TestContext) 56 | 57 | // We need extra files at runtime. 58 | repoRoot := os.Getenv("REPO_ROOT") 59 | if repoRoot != "" { 60 | testfiles.AddFileSource(RootFileSource{Root: repoRoot}) 61 | } 62 | 63 | // Now run the test suite. 64 | os.Exit(m.Run()) 65 | } 66 | 67 | func TestE2E(t *testing.T) { 68 | RunE2ETests(t) 69 | } 70 | -------------------------------------------------------------------------------- /test/e2e/ephemeral/ephemeral.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 Intel Corporation. 3 | 4 | SPDX-License-Identifier: Apache-2.0 5 | */ 6 | 7 | package ephemeral 8 | 9 | import ( 10 | "fmt" 11 | "os" 12 | 13 | "k8s.io/kubernetes/test/e2e/framework" 14 | ) 15 | 16 | var Supported = func() bool { 17 | k8sVersion := os.Getenv("TEST_KUBERNETES_VERSION") 18 | 19 | if k8sVersion == "" { 20 | // No K8S version set, default enable ephemeral tests 21 | framework.Logf("No Kubernetes version set! Providing (via TEST_KUBERNETES_VERSION environment) the right Kubernetes version might affect the test suites to be run.") 22 | return true 23 | } 24 | var major, minor int 25 | if _, err := fmt.Sscanf(k8sVersion, "%d.%d", &major, &minor); err != nil { 26 | framework.Logf("Failed to parse 'TEST_KUBERNETES_VERSION=%s': %s. Enabling ephemeral volume tests.", k8sVersion, err.Error()) 27 | // Allow ephemeral tests 28 | return true 29 | } 30 | if (major <= 0) || (major == 1 && minor <= 14) { 31 | // Kubernetes version <= 1.14 does not support ephemeral volumes. 32 | framework.Logf("Provided Kubernetes version '%s' does not support ephemeral volumes. Tests involving ephemeral volumes are disabled.", k8sVersion) 33 | return false 34 | } 35 | 36 | return true 37 | }() 38 | -------------------------------------------------------------------------------- /test/e2e/filesource_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package e2e 18 | 19 | import ( 20 | "fmt" 21 | "io/ioutil" 22 | "os" 23 | "path" 24 | "path/filepath" 25 | ) 26 | 27 | // TODO: make framework/testfiles support absolute paths 28 | 29 | // RootFileSource looks for files relative to a root directory. 30 | type RootFileSource struct { 31 | Root string 32 | } 33 | 34 | // ReadTestFile looks for the file relative to the configured 35 | // root directory. 36 | func (r RootFileSource) ReadTestFile(filePath string) ([]byte, error) { 37 | var fullPath string 38 | if path.IsAbs(filePath) { 39 | fullPath = filePath 40 | } else { 41 | fullPath = filepath.Join(r.Root, filePath) 42 | } 43 | data, err := ioutil.ReadFile(fullPath) 44 | if os.IsNotExist(err) { 45 | // Not an error (yet), some other provider may have the file. 46 | return nil, nil 47 | } 48 | return data, err 49 | } 50 | 51 | // DescribeFiles explains that it looks for files inside a certain 52 | // root directory. 53 | func (r RootFileSource) DescribeFiles() string { 54 | description := fmt.Sprintf("Test files are expected in %q", r.Root) 55 | if !path.IsAbs(r.Root) { 56 | // The default in test_context.go is the relative path 57 | // ../../, which doesn't really help locating the 58 | // actual location. Therefore we add also the absolute 59 | // path if necessary. 60 | abs, err := filepath.Abs(r.Root) 61 | if err == nil { 62 | description += fmt.Sprintf(" = %q", abs) 63 | } 64 | } 65 | description += "." 66 | return description 67 | } 68 | -------------------------------------------------------------------------------- /test/e2e/gotests/gotests.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 Intel Corporation. 3 | 4 | SPDX-License-Identifier: Apache-2.0 5 | */ 6 | 7 | package gotests 8 | 9 | import ( 10 | "context" 11 | "fmt" 12 | "os" 13 | "os/exec" 14 | "strings" 15 | 16 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 17 | "k8s.io/apimachinery/pkg/labels" 18 | "k8s.io/kubernetes/test/e2e/framework" 19 | 20 | . "github.com/onsi/ginkgo/v2" 21 | . "github.com/onsi/gomega" 22 | 23 | "github.com/intel/pmem-csi/test/e2e/deploy" 24 | "github.com/intel/pmem-csi/test/e2e/pod" 25 | ) 26 | 27 | // We are using direct mode here because it needs to do less work 28 | // during startup. 29 | var _ = deploy.Describe("direct-testing", "direct-testing-gotests", "", func(d *deploy.Deployment) { 30 | f := framework.NewDefaultFramework("gotests") 31 | f.SkipNamespaceCreation = true 32 | 33 | // Register one test for each package. 34 | for _, pkg := range strings.Split(os.Getenv("TEST_PKGS"), " ") { 35 | pkg := pkg 36 | It(pkg, func() { runGoTest(f, pkg) }) 37 | } 38 | }) 39 | 40 | // runGoTest builds and copies the Go test binary into the PMEM-CSI 41 | // driver container and executes it there. This way it runs in exactly 42 | // the same environment as the driver (the distro's kernel, our 43 | // container user space), which may or may not expose bugs that are 44 | // not found when running those tests on the build host. 45 | func runGoTest(f *framework.Framework, pkg string) { 46 | root := os.Getenv("REPO_ROOT") 47 | var err error 48 | 49 | build := exec.Command("/bin/sh", "-c", os.Getenv("TEST_CMD")+" --cover -covermode=atomic -c -o _work/test.test "+pkg) 50 | build.Stdout = GinkgoWriter 51 | build.Stderr = GinkgoWriter 52 | build.Dir = root 53 | By("Compiling with: " + strings.Join(build.Args, " ")) 54 | err = build.Run() 55 | framework.ExpectNoError(err, "compile test program for %s", pkg) 56 | 57 | label := labels.SelectorFromSet(labels.Set(map[string]string{"app.kubernetes.io/name": "pmem-csi-node"})) 58 | pods, err := f.ClientSet.CoreV1().Pods("default").List(context.Background(), metav1.ListOptions{LabelSelector: label.String()}) 59 | framework.ExpectNoError(err, "list PMEM-CSI pods") 60 | Expect(pods.Items).NotTo(BeEmpty(), "have PMEM-CSI pods") 61 | pmem := pods.Items[0] 62 | 63 | coverDir := "/var/lib/pmem-csi-coverage" 64 | coverFile := strings.ReplaceAll(pkg, "/", "-") 65 | if strings.HasPrefix(coverFile, ".") { 66 | coverFile = coverFile[1:] 67 | } 68 | 69 | By(fmt.Sprintf("Running in PMEM-CSI pod %s", pmem.Name)) 70 | pod.RunInPod(f, root, 71 | []string{"_work/test.test"}, 72 | ""+ 73 | "if [ -d "+coverDir+" ]; then "+ 74 | " if /tmp/test.test -h 2>&1 | grep -q ginkgo; then "+ 75 | " TEST_WORK=_work REPO_ROOT=. /tmp/test.test -test.coverprofile="+coverDir+"/gotest"+coverFile+".out -ginkgo.v; "+ 76 | " else "+ 77 | " TEST_WORK=_work REPO_ROOT=. /tmp/test.test -test.coverprofile="+coverDir+"/gotest"+coverFile+".out;"+ 78 | " fi;"+ 79 | "else "+ 80 | " if /tmp/test.test -h 2>&1 | grep -q ginkgo; then "+ 81 | " TEST_WORK=_work REPO_ROOT=. /tmp/test.test -ginkgo.v; "+ 82 | " else "+ 83 | " TEST_WORK=_work REPO_ROOT=. /tmp/test.test;"+ 84 | " fi;"+ 85 | "fi", 86 | pmem.Namespace, pmem.Name, "pmem-driver") 87 | } 88 | -------------------------------------------------------------------------------- /test/e2e/image/image.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 Intel Corporation. 3 | 4 | SPDX-License-Identifier: Apache-2.0 5 | */ 6 | 7 | package image 8 | 9 | import ( 10 | v1 "k8s.io/api/core/v1" 11 | "k8s.io/apimachinery/pkg/labels" 12 | "k8s.io/kubernetes/test/e2e/framework" 13 | 14 | . "github.com/onsi/ginkgo/v2" 15 | 16 | "github.com/intel/pmem-csi/test/e2e/deploy" 17 | "github.com/intel/pmem-csi/test/e2e/pod" 18 | ) 19 | 20 | // We are using direct mode here because it needs to do less work 21 | // during startup. All we care about is that we get pods running our 22 | // image. 23 | var _ = deploy.Describe("direct-production", "direct-production-image", "", func(d *deploy.Deployment) { 24 | f := framework.NewDefaultFramework("image") 25 | f.SkipNamespaceCreation = true 26 | 27 | var nodeDriver *v1.Pod 28 | 29 | BeforeEach(func() { 30 | cluster, err := deploy.NewCluster(f.ClientSet, f.DynamicClient, f.ClientConfig()) 31 | framework.ExpectNoError(err, "create cluster") 32 | nodeDriver = cluster.WaitForAppInstance(labels.Set{"app.kubernetes.io/name": "pmem-csi-node"}, 33 | "", /* no IP, any of the pods is fine */ 34 | d.Namespace, 35 | ) 36 | }) 37 | 38 | Context("ipmctl", func() { 39 | It("can run", func() { 40 | // "ipmctl version" would be nicer, but 41 | // doesn't work at the moment 42 | // (https://github.com/intel/ipmctl/issues/172). 43 | pod.RunInPod(f, "/", nil, "ipmctl help", nodeDriver.Namespace, nodeDriver.Name, "pmem-driver") 44 | }) 45 | }) 46 | }) 47 | -------------------------------------------------------------------------------- /test/e2e/imagefile/imagefilee2e.go: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | Copyright (c) 2017-2019 Intel Corporation 4 | 5 | SPDX-License-Identifier: Apache-2.0 6 | 7 | */ 8 | 9 | package imagefilee2e 10 | 11 | import ( 12 | "fmt" 13 | 14 | "github.com/onsi/ginkgo/v2" 15 | 16 | "github.com/intel/pmem-csi/pkg/imagefile/test" 17 | "github.com/intel/pmem-csi/test/e2e/deploy" 18 | ) 19 | 20 | type tImplementation struct { 21 | ginkgo.GinkgoTInterface 22 | } 23 | 24 | func (t *tImplementation) Outer(name string, cb func(t test.TInterface)) { 25 | ginkgo.Context(name, func() { 26 | cb(t) 27 | }) 28 | } 29 | 30 | func (t *tImplementation) Inner(name string, cb func(t test.TInterface)) { 31 | ginkgo.It(name, func() { 32 | // This code now can call GinkgoT and pass some actual implementation 33 | // of the interface. 34 | cb(&tImplementation{ginkgo.GinkgoT()}) 35 | }) 36 | } 37 | 38 | // Only necessary because of https://github.com/onsi/ginkgo/issues/659 39 | func (t *tImplementation) Skipf(format string, args ...interface{}) { 40 | ginkgo.Skip(fmt.Sprintf(format, args...)) 41 | } 42 | 43 | var _ = deploy.Describe("", "imagefile", "", func(d *deploy.Deployment) { 44 | // Our Outer and Inner implementation do not need a valid pointer. 45 | test.ImageFile((*tImplementation)(nil)) 46 | }) 47 | -------------------------------------------------------------------------------- /test/e2e/pod/exec.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 Intel Corporation. 3 | 4 | SPDX-License-Identifier: Apache-2.0 5 | */ 6 | 7 | package pod 8 | 9 | import ( 10 | "bytes" 11 | "fmt" 12 | "io/ioutil" 13 | "os" 14 | "path" 15 | "strings" 16 | 17 | "k8s.io/kubernetes/test/e2e/framework" 18 | e2epod "k8s.io/kubernetes/test/e2e/framework/pod" 19 | 20 | . "github.com/onsi/ginkgo/v2" 21 | ) 22 | 23 | // RunInPod optionally transfers some files into /tmp (source file 24 | // relative to root dir, with same base file name, without path, with 25 | // same x bit) and executes a shell command. Any error is treated as 26 | // test failure. 27 | // 28 | // Relies on dd in the target container. 29 | func RunInPod(f *framework.Framework, rootdir string, files []string, command string, namespace, pod, container string) (string, string) { 30 | parts := []string{"cd /tmp"} 31 | var input hiddenContent 32 | for _, file := range files { 33 | base := path.Base(file) 34 | full := path.Join(rootdir, file) 35 | data, err := ioutil.ReadFile(full) 36 | framework.ExpectNoError(err, "read input file %q", full) 37 | input.Write(data) 38 | // Somehow count=1 bs= resulted in truncated data transfers. 39 | // This works, but has higher overhead. 40 | parts = append(parts, fmt.Sprintf("dd of=%s count=%d bs=1 status=none", base, len(data))) 41 | stat, err := os.Stat(full) 42 | framework.ExpectNoError(err) 43 | if stat.Mode().Perm()&0111 != 0 { 44 | parts = append(parts, fmt.Sprintf("chmod a+x %s", base)) 45 | } 46 | } 47 | parts = append(parts, command) 48 | options := e2epod.ExecOptions{ 49 | Command: []string{ 50 | "/bin/sh", 51 | "-c", 52 | strings.Join(parts, " && "), 53 | }, 54 | Namespace: namespace, 55 | PodName: pod, 56 | ContainerName: container, 57 | Stdin: &input, 58 | CaptureStdout: true, 59 | CaptureStderr: true, 60 | } 61 | stdout, stderr, err := e2epod.ExecWithOptions(f, options) 62 | framework.ExpectNoError(err, "command failed in namespace %s, pod/container %s/%s:\nstderr:\n%s\nstdout:%s\n", 63 | namespace, pod, container, stderr, stdout) 64 | fmt.Fprintf(GinkgoWriter, "stderr:\n%s\nstdout:\n%s\n", 65 | stderr, stdout) 66 | 67 | return stdout, stderr 68 | } 69 | 70 | // hiddentContent is the same as bytes.Buffer, but doesn't print its entire content in String(). 71 | type hiddenContent struct { 72 | bytes.Buffer 73 | } 74 | 75 | func (h *hiddenContent) String() string { 76 | return fmt.Sprintf("<%d bytes>", h.Len()) 77 | } 78 | -------------------------------------------------------------------------------- /test/e2e/pod/logs.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Intel Corporation. 3 | 4 | SPDX-License-Identifier: Apache-2.0 5 | */ 6 | 7 | package pod 8 | 9 | import ( 10 | "context" 11 | "fmt" 12 | "time" 13 | 14 | "k8s.io/client-go/kubernetes" 15 | e2epod "k8s.io/kubernetes/test/e2e/framework/pod" 16 | ) 17 | 18 | // Retrieves a container log, with retry in case of errors. 19 | // It is okay if the pod does not exist yet. 20 | func Logs(ctx context.Context, client kubernetes.Interface, namespace, pod, container string) (output string, err error) { 21 | for { 22 | output, err = e2epod.GetPodLogs(ctx, client, namespace, pod, container) 23 | if err == nil { 24 | return 25 | } 26 | if ctx.Err() != nil { 27 | return "", fmt.Errorf("waiting for pod log of container %s in pod %s/%s: %v: %v", container, namespace, pod, ctx.Err(), err) 28 | } 29 | time.Sleep(5 * time.Second) 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /test/e2e/storage/wait.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2014 The Kubernetes Authors. 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | 14 | package storage 15 | 16 | import ( 17 | "math" 18 | "time" 19 | 20 | "k8s.io/apimachinery/pkg/util/wait" 21 | "k8s.io/utils/clock" 22 | ) 23 | 24 | // BackoffManager manages backoff with a particular scheme based on its underlying implementation. It provides 25 | // an interface to return a timer for backoff, and caller shall backoff until Timer.C returns. If the second Backoff() 26 | // is called before the timer from the first Backoff() call finishes, the first timer will NOT be drained. 27 | // The BackoffManager is supposed to be called in a single-threaded environment. 28 | type BackoffManager interface { 29 | Backoff() clock.Timer 30 | } 31 | 32 | type exponentialBackoffManagerImpl struct { 33 | backoff *wait.Backoff 34 | backoffTimer clock.Timer 35 | lastBackoffStart time.Time 36 | initialBackoff time.Duration 37 | backoffResetDuration time.Duration 38 | clock clock.Clock 39 | } 40 | 41 | // NewExponentialBackoffManager returns a manager for managing exponential backoff. Each backoff is jittered and 42 | // backoff will not exceed the given max. If the backoff is not called within resetDuration, the backoff is reset. 43 | // This backoff manager is used to reduce load during upstream unhealthiness. 44 | func NewExponentialBackoffManager(initBackoff, maxBackoff, resetDuration time.Duration, backoffFactor, jitter float64, c clock.Clock) BackoffManager { 45 | return &exponentialBackoffManagerImpl{ 46 | backoff: &wait.Backoff{ 47 | Duration: initBackoff, 48 | Factor: backoffFactor, 49 | Jitter: jitter, 50 | 51 | // the current impl of wait.Backoff returns Backoff.Duration once steps are used up, which is not 52 | // what we ideally need here, we set it to max int and assume we will never use up the steps 53 | Steps: math.MaxInt32, 54 | Cap: maxBackoff, 55 | }, 56 | backoffTimer: c.NewTimer(0), 57 | initialBackoff: initBackoff, 58 | lastBackoffStart: c.Now(), 59 | backoffResetDuration: resetDuration, 60 | clock: c, 61 | } 62 | } 63 | 64 | func (b *exponentialBackoffManagerImpl) getNextBackoff() time.Duration { 65 | if b.clock.Now().Sub(b.lastBackoffStart) > b.backoffResetDuration { 66 | b.backoff.Steps = math.MaxInt32 67 | b.backoff.Duration = b.initialBackoff 68 | } 69 | b.lastBackoffStart = b.clock.Now() 70 | return b.backoff.Step() 71 | } 72 | 73 | // Backoff implements BackoffManager.Backoff, it returns a timer so caller can block on the timer for backoff. 74 | func (b *exponentialBackoffManagerImpl) Backoff() clock.Timer { 75 | b.backoffTimer.Reset(b.getNextBackoff()) 76 | return b.backoffTimer 77 | } 78 | -------------------------------------------------------------------------------- /test/e2e/tls/nmap-ssl-enum-ciphers.patch: -------------------------------------------------------------------------------- 1 | Without this patch, nmap skips cipers using ECDHE. 2 | https://github.com/nmap/nmap/issues/1187#issuecomment-587031079 3 | 4 | diff -c /usr/share/nmap/scripts/ssl-enum-ciphers.nse.orig /usr/share/nmap/scripts/ssl-enum-ciphers.nse 5 | *** /usr/share/nmap/scripts/ssl-enum-ciphers.nse.orig 2020-03-06 09:41:59.657722734 +0000 6 | --- /usr/share/nmap/scripts/ssl-enum-ciphers.nse 2020-03-06 09:43:24.982459541 +0000 7 | *************** 8 | *** 516,521 **** 9 | --- 516,523 ---- 10 | return { 11 | -- Claim to support common elliptic curves 12 | ["elliptic_curves"] = tls.EXTENSION_HELPERS["elliptic_curves"](tls.DEFAULT_ELLIPTIC_CURVES), 13 | + -- Claim to support every EC point format 14 | + ["ec_point_formats"] = tls.EXTENSION_HELPERS["ec_point_formats"](sorted_keys(tls.EC_POINT_FORMATS)), 15 | -- Enable SNI if a server name is available 16 | ["server_name"] = tlsname and tls.EXTENSION_HELPERS["server_name"](tlsname), 17 | } 18 | -------------------------------------------------------------------------------- /test/pull-images.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -o errexit 4 | 5 | TEST_DIRECTORY=${TEST_DIRECTORY:-$(dirname $(readlink -f $0))} 6 | source ${TEST_CONFIG:-${TEST_DIRECTORY}/test-config.sh} 7 | 8 | CLUSTER=${CLUSTER:-pmem-govm} 9 | REPO_DIRECTORY="${REPO_DIRECTORY:-$(dirname $(dirname $(readlink -f $0)))}" 10 | CLUSTER_DIRECTORY="${CLUSTER_DIRECTORY:-${REPO_DIRECTORY}/_work/${CLUSTER}}" 11 | SSH="${CLUSTER_DIRECTORY}/ssh.0" 12 | KUBECTL="${SSH} kubectl" # Always use the kubectl installed in the cluster. 13 | 14 | $KUBECTL apply -f - <&1 | wc -l) -gt 1 ]; then 24 | # Hack for https://github.com/kata-containers/kata-containers/issues/2088: 25 | # wait some more until hopefully all nodes are configured, then fix the 26 | # configuration so that memory_offset is enabled. 27 | sleep 30 28 | 29 | echo "Kata Containers runtime available on:" 30 | ${KUBECTL} get nodes -l katacontainers.io/kata-runtime=true 31 | 32 | echo "Updating /opt/kata/share/defaults/kata-containers/configuration-qemu.toml on each node" 33 | for pod in $(${KUBECTL} get pods -n kube-system -l name=kata-deploy -o 'jsonpath={.items[*].metadata.name}'); do 34 | ${KUBECTL} exec -t -n kube-system $pod -- /bin/sh </dev/null || true); \ 27 | else \ 28 | rm -rf _work/$(CLUSTER); \ 29 | fi; \ 30 | else \ 31 | echo "Cluster $(CLUSTER) was already removed."; \ 32 | fi 33 | 34 | # Clean shutdown of all VMs, then waits for reboot of cluster. 35 | restart: 36 | @ if [ -f _work/$(CLUSTER)/stop.sh ]; then \ 37 | _work/$(CLUSTER)/restart.sh; \ 38 | else \ 39 | echo "Cluster $(CLUSTER) is not running, it cannot be restarted."; \ 40 | exit 1; \ 41 | fi 42 | 43 | start_test_vm: _work/bin/govm 44 | PATH="$(PWD)/_work/bin:$$PATH" NODES=$(NODE) INIT_CLUSTER=false test/start-kubernetes.sh 45 | -------------------------------------------------------------------------------- /test/stop-operator.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Script to delete a running operator deployment. 4 | # 5 | set -o errexit 6 | set -o pipefail 7 | 8 | TEST_DIRECTORY=${TEST_DIRECTORY:-$(dirname "$(readlink -f "$0")")} 9 | source "${TEST_CONFIG:-${TEST_DIRECTORY}/test-config.sh}" 10 | 11 | REPO_DIRECTORY="${REPO_DIRECTORY:-$(dirname "${TEST_DIRECTORY}")}" 12 | CLUSTER=${CLUSTER:-pmem-govm} 13 | CLUSTER_DIRECTORY="${CLUSTER_DIRECTORY:-${REPO_DIRECTORY}/_work/${CLUSTER}}" 14 | SSH="${CLUSTER_DIRECTORY}/ssh.0" 15 | KUBECTL="${SSH} kubectl" # Always use the kubectl installed in the cluster. 16 | 17 | keep_crd=false 18 | keep_namespace=false 19 | 20 | function delete_olm_operator() { 21 | BINDIR=${REPO_DIRECTORY}/_work/bin 22 | 23 | namespace="" 24 | if [ "${TEST_OPERATOR_NAMESPACE}" != "" ]; then 25 | namespace="--namespace ${TEST_OPERATOR_NAMESPACE}" 26 | fi 27 | 28 | echo "Cleaning up the operator deployment using OLM" 29 | output=$(${BINDIR}/operator-sdk cleanup pmem-csi-operator $namespace 2>&1) 30 | if [ $? -ne 0 ] ; then 31 | echo "Failed to delete the operator: $output" 32 | exit 1 33 | fi 34 | 35 | timeout=180 # 3 minutes 36 | SECONDS=0 37 | echo "Waiting for all the operator bundle objects gets deleted..." 38 | while true ; do 39 | output=$(${KUBECTL} get subscriptions,catalogsource,installplans,csvs,po $namespace 2>&1 | grep -v "No rsources found" || true) 40 | if ! echo $output | grep -q -E 'pmem-csi-(operator|bundle)' ; then 41 | echo "Done" 42 | return 43 | fi 44 | if [ $SECONDS -gt $timeout ]; then 45 | echo "Remove objects timedout: $output" 46 | exit 1 47 | fi 48 | sleep 1 49 | done 50 | } 51 | 52 | function delete_operator() { 53 | tmpdir=$(mktemp -d) 54 | trap "rm -rf $tmpdir" SIGTERM SIGINT EXIT 55 | 56 | cp "${REPO_DIRECTORY}/deploy/operator/pmem-csi-operator.yaml" ${tmpdir}/pmem-csi-operator.yaml 57 | cat > ${tmpdir}/kustomization.yaml < ${tmpdir}/patch.yaml <&1 | grep -v NotFound || true 84 | 85 | if ! $keep_crd ; then 86 | echo "Deleting CRD..." 87 | ${KUBECTL} delete crd/pmemcsideployments.pmem-csi.intel.com 2>&1 | grep -v NotFound || true 88 | fi 89 | } 90 | 91 | function Usage() { 92 | echo "Usage: 93 | $0 [-olm] [-keep-namespace] [-keep-crd]" 94 | exit 95 | } 96 | 97 | deploy_method=yaml 98 | 99 | for arg in $@; do 100 | case $arg in 101 | "-olm") deploy_method=olm ;; 102 | "-keep-crd") keep_crd=true ;; 103 | "-keep-namespace") keep_namespace=true ;; 104 | "-h") Usage ;; 105 | *) echo "Ignoring unknown argument: $arg" 106 | Usage ;; 107 | esac 108 | done 109 | 110 | case $deploy_method in 111 | yaml) 112 | delete_operator;; 113 | olm) 114 | delete_olm_operator;; 115 | *) 116 | echo >&2 "Unknown deploy method!!!" 117 | exit 1 ;; 118 | esac 119 | -------------------------------------------------------------------------------- /test/test-config.d/.gitignore: -------------------------------------------------------------------------------- 1 | /*.sh 2 | -------------------------------------------------------------------------------- /test/test-config.d/README.md: -------------------------------------------------------------------------------- 1 | # Configure test-config.d 2 | 3 | This directory can be used to override the default configuration from 4 | [test-config.sh](../test-config.sh). Just create one or more files 5 | ending in `.sh`, set variables in those files and then those variables 6 | will override the defaults. 7 | 8 | The files are sourced in a bash shell and thus can use arbitrary shell 9 | constructs like `if/else/fi`. 10 | 11 | To set defaults that can still be overridden by environment variables, 12 | use this idiom: 13 | 14 | ``` shell 15 | : ${TEST_DEVICEMODE:=direct} 16 | ``` 17 | -------------------------------------------------------------------------------- /test/test-config/config.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Intel Corporation. 3 | 4 | SPDX-License-Identifier: Apache-2.0 5 | */ 6 | 7 | // Package testconfig reads config strings from test-config.sh or the environment. 8 | package testconfig 9 | 10 | import ( 11 | "fmt" 12 | "os" 13 | "os/exec" 14 | "strings" 15 | ) 16 | 17 | // Get returns the test config value, an empty string if not set, or an error. 18 | func Get(name string) (string, error) { 19 | root := os.Getenv("REPO_ROOT") 20 | if root == "" { 21 | // The current directory may or may not work as fallback. 22 | root = "." 23 | } 24 | config := fmt.Sprintf("%s/test/test-config.sh", root) 25 | cmd := exec.Command("/bin/sh", "-c", fmt.Sprintf(`. '%s' && echo "$%s"`, config, name)) 26 | value, err := cmd.CombinedOutput() 27 | if err != nil { 28 | return "", fmt.Errorf("read %s from %s: %v (%s)", name, config, err, string(value)) 29 | } 30 | return strings.TrimRight(string(value), "\n"), nil 31 | } 32 | 33 | // GetOrFail will panic when Get returns an error. 34 | func GetOrFail(name string) string { 35 | value, err := Get(name) 36 | if err != nil { 37 | panic(err) 38 | } 39 | return value 40 | } 41 | 42 | // GetNodeLabelOrFail is a convenience function which returns TEST_PMEM_NODE_LABEL as string pair. 43 | func GetNodeLabelOrFail() (name, value string) { 44 | nodeLabel := GetOrFail("TEST_PMEM_NODE_LABEL") 45 | parts := strings.SplitN(nodeLabel, "=", 2) 46 | if len(parts) < 2 { 47 | panic(fmt.Sprintf("expected label=value: TEST_PMEM_NODE_LABEL=%q", nodeLabel)) 48 | } 49 | return parts[0], parts[1] 50 | } 51 | -------------------------------------------------------------------------------- /third-party/README.md: -------------------------------------------------------------------------------- 1 | This directory contains forks of upstream code under various licenses. 2 | Code might have to be forked because additional changes had to be made 3 | or upstream is inactive and we need to maintain the code ourselves. 4 | -------------------------------------------------------------------------------- /third-party/go-fibmap/.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | 10 | # Architecture specific extensions/prefixes 11 | *.[568vq] 12 | [568vq].out 13 | 14 | *.cgo1.go 15 | *.cgo2.c 16 | _cgo_defun.c 17 | _cgo_gotypes.go 18 | _cgo_export.* 19 | 20 | _testmain.go 21 | 22 | *.exe 23 | *.test 24 | -------------------------------------------------------------------------------- /third-party/go-fibmap/LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 Andreas Klauer 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /third-party/go-fibmap/README.md: -------------------------------------------------------------------------------- 1 | fibmap 2 | ====== 3 | 4 | Golang Linux ioctl FIBMAP, FIEMAP, SEEK_DATA/HOLE, PUNCH_HOLE, ... 5 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py3-{mylinux,mywindows} 3 | 4 | [testenv] 5 | platform = mylinux: linux 6 | mywindows: win32 7 | whitelist_externals = make.bat 8 | /usr/bin/make 9 | deps = -rdocs/requirements.txt 10 | commands = 11 | mylinux: make {posargs} 12 | mywindows: make.bat {posargs} 13 | 14 | --------------------------------------------------------------------------------