├── .gitattributes ├── .github └── workflows │ ├── publish.yml │ └── pull.yml ├── .gitignore ├── .travis.yml ├── CONTRIBUTING.md ├── LICENSE ├── Makefile ├── OWNERS ├── README.md ├── SECURITY_CONTACTS ├── book ├── book.toml ├── custom.js └── src │ ├── README.md │ ├── SUMMARY.md │ ├── _book │ ├── Users.md │ ├── example │ │ ├── csi-app.yaml │ │ ├── csi-pvc.yaml │ │ └── csi-setup.yaml │ ├── gitbook │ │ ├── fonts │ │ │ └── fontawesome │ │ │ │ ├── FontAwesome.otf │ │ │ │ ├── fontawesome-webfont.eot │ │ │ │ ├── fontawesome-webfont.svg │ │ │ │ ├── fontawesome-webfont.ttf │ │ │ │ ├── fontawesome-webfont.woff │ │ │ │ └── fontawesome-webfont.woff2 │ │ ├── gitbook-plugin-fontsettings │ │ │ ├── fontsettings.js │ │ │ └── website.css │ │ ├── gitbook-plugin-highlight │ │ │ ├── ebook.css │ │ │ └── website.css │ │ ├── gitbook-plugin-lunr │ │ │ ├── lunr.min.js │ │ │ └── search-lunr.js │ │ ├── gitbook-plugin-search │ │ │ ├── lunr.min.js │ │ │ ├── search-engine.js │ │ │ ├── search.css │ │ │ └── search.js │ │ ├── gitbook-plugin-sharing │ │ │ └── buttons.js │ │ ├── gitbook.js │ │ ├── images │ │ │ ├── apple-touch-icon-precomposed-152.png │ │ │ └── favicon.ico │ │ ├── style.css │ │ └── theme.js │ ├── images │ │ ├── kubelet.png │ │ └── sidecar-container.png │ └── search_index.json │ ├── api.md │ ├── api │ └── volume-snapshot.md │ ├── changed-block-tracking.md │ ├── cluster-driver-registrar.md │ ├── cross-namespace-data-sources.md │ ├── csi-driver-object.md │ ├── csi-node-object.md │ ├── csi-objects.md │ ├── csi-proxy.md │ ├── csi-windows.md │ ├── deploying.md │ ├── developing.md │ ├── drivers.md │ ├── ephemeral-local-volumes.md │ ├── example.md │ ├── example │ ├── snapshot │ │ ├── csi-hostpath-snapshotter.yaml │ │ ├── csi-restore.yaml │ │ ├── csi-snapshot.yaml │ │ └── csi-snapshotclass.yaml │ └── usage │ │ ├── csi-app.yaml │ │ ├── csi-pvc.yaml │ │ └── csi-storageclass.yaml │ ├── external-attacher.md │ ├── external-health-monitor-agent.md │ ├── external-health-monitor-controller.md │ ├── external-provisioner.md │ ├── external-resizer.md │ ├── external-snapshot-metadata.md │ ├── external-snapshotter.md │ ├── features.md │ ├── functional-testing.md │ ├── group-snapshot-restore-feature.md │ ├── group-snapshot.md │ ├── images │ ├── kubelet.png │ └── sidecar-container.png │ ├── introduction.md │ ├── kubernetes-changelog.md │ ├── kubernetes-cluster-controllers.md │ ├── livenessprobe.md │ ├── node-driver-registrar.md │ ├── pod-info.md │ ├── prevent-volume-mode-conversion.md │ ├── project-policies.md │ ├── raw-block.md │ ├── secrets-and-credentials-storage-class.md │ ├── secrets-and-credentials-volume-group-snapshot-class.md │ ├── secrets-and-credentials-volume-snapshot-class.md │ ├── secrets-and-credentials.md │ ├── sidecar-containers.md │ ├── skip-attach.md │ ├── snapshot-controller.md │ ├── snapshot-restore-feature.md │ ├── snapshot-validation-webhook.md │ ├── storage-capacity-tracking.md │ ├── support-fsgroup.md │ ├── testing-drivers.md │ ├── token-requests.md │ ├── topology.md │ ├── troubleshooting.md │ ├── unit-testing.md │ ├── volume-attributes-class.md │ ├── volume-cloning.md │ ├── volume-datasources.md │ ├── volume-expansion.md │ ├── volume-health-monitor.md │ └── volume-limits.md ├── code-of-conduct.md └── hack ├── gen-api.json ├── gen-api.sh └── update-example.sh /.gitattributes: -------------------------------------------------------------------------------- 1 | docs/* linguist-generated=true 2 | -------------------------------------------------------------------------------- /.github/workflows/publish.yml: -------------------------------------------------------------------------------- 1 | name: publish 2 | 3 | # Controls when the workflow will run 4 | on: 5 | # Triggers the workflow on push or pull request events but only for the master branch 6 | push: 7 | branches: [ master ] 8 | 9 | # Allows you to run this workflow manually from the Actions tab 10 | workflow_dispatch: 11 | 12 | # A workflow run is made up of one or more jobs that can run sequentially or in parallel 13 | jobs: 14 | # This workflow contains a single job called "build" 15 | build: 16 | # The type of runner that the job will run on 17 | runs-on: ubuntu-latest 18 | 19 | # Steps represent a sequence of tasks that will be executed as part of the job 20 | steps: 21 | # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it 22 | - uses: actions/checkout@v2 23 | 24 | # The same command that users can use locally is used to build documentation. 25 | - name: Build documentation 26 | run: make 27 | 28 | # Publish the result by overwriting the "gh-pages" branch with the new content. 29 | - name: Deploy 30 | uses: JamesIves/github-pages-deploy-action@4.1.4 31 | with: 32 | branch: gh-pages 33 | folder: docs 34 | git-config-name: GitHub Actions 35 | git-config-email: github-actions@kubernetes-csi 36 | # This secret was created in the repo's settings under "Secrets". 37 | # It contains an SSH private key created specifically for this job. 38 | # The corresponding public key was added to the repo's deploy keys. 39 | ssh-key: ${{ secrets.DEPLOY_KEY }} 40 | -------------------------------------------------------------------------------- /.github/workflows/pull.yml: -------------------------------------------------------------------------------- 1 | name: pull 2 | 3 | # Controls when the workflow will run 4 | on: 5 | pull_request: 6 | branches: [ master ] 7 | 8 | # Allows you to run this workflow manually from the Actions tab 9 | workflow_dispatch: 10 | 11 | # A workflow run is made up of one or more jobs that can run sequentially or in parallel 12 | jobs: 13 | # This workflow contains a single job called "build" 14 | build: 15 | # The type of runner that the job will run on 16 | runs-on: ubuntu-latest 17 | 18 | # Steps represent a sequence of tasks that will be executed as part of the job 19 | steps: 20 | # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it 21 | - uses: actions/checkout@v2 22 | 23 | # The same command that users can use locally is used to build documentation. 24 | - name: Build documentation 25 | run: make 26 | 27 | # Publish generated site as artifact. Unfortunately viewing it requires 28 | # downloading a .zip and uncompressing that (https://github.com/actions/upload-artifact/issues/14#issuecomment-620728238) 29 | - uses: actions/upload-artifact@v4 30 | with: 31 | name: html-docs 32 | path: docs 33 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # The example references the latest .yaml file directly in the source repository, 2 | # therefore we don't need the copy in the generated documentation. 3 | docs/example 4 | 5 | # vim 6 | *.swp 7 | 8 | /mdbook 9 | /mdbook*.tar.gz 10 | /docs 11 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: generic 2 | 3 | script: bash ./hack/deploy.sh 4 | env: 5 | global: 6 | - ENCRYPTION_LABEL: "4d61ca75e316" # from travis encrypt-file deploy_key, see https://gist.github.com/domenic/ec8b0fc8ab45f39403dd#get-encrypted-credentials 7 | - COMMIT_AUTHOR_EMAIL: "travis-ci@kubernetes-csi" 8 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing Guidelines 2 | 3 | Welcome to Kubernetes. We are excited about the prospect of you joining our [community](https://github.com/kubernetes/community)! The Kubernetes community abides by the CNCF [code of conduct](code-of-conduct.md). Here is an excerpt: 4 | 5 | _As contributors and maintainers of this project, and in the interest of fostering an open and welcoming community, we pledge to respect all people who contribute through reporting issues, posting feature requests, updating documentation, submitting pull requests or patches, and other activities._ 6 | 7 | ## Getting Started 8 | 9 | ### Adding documentation 10 | Please feel free to send PRs for the documentation in the `book` directory. If you fork and edit on your system, please generate a new output. 11 | 12 | We have full documentation on how to get started contributing here: 13 | 14 | - [Contributor License Agreement](https://git.k8s.io/community/CLA.md) Kubernetes projects require that you sign a Contributor License Agreement (CLA) before we can accept your pull requests 15 | - [Kubernetes Contributor Guide](https://git.k8s.io/community/contributors/guide) - Main contributor documentation, or you can just jump directly to the [contributing section](https://git.k8s.io/community/contributors/guide#contributing) 16 | - [Contributor Cheat Sheet](https://git.k8s.io/community/contributors/guide/contributor-cheatsheet) - Common resources for existing developers 17 | 18 | ## Mentorship 19 | 20 | - [Mentoring Initiatives](https://git.k8s.io/community/mentoring) - We have a diverse set of mentorship programs available that are always looking for volunteers! 21 | 22 | ## Contact Information 23 | 24 | - [Slack channel](https://kubernetes.slack.com/messages/sig-storage) 25 | - [Mailing list](https://groups.google.com/forum/#!forum/kubernetes-sig-storage) 26 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | ROOT := $(shell pwd) 2 | 3 | # The default is to download the official release binary 4 | # for x86_64-unknown-linux-gnu. If that default is unsuitable, 5 | # build your own binary and copy or symlink it here, or 6 | # override the variable with "make MDBOOK_BINARY=...". 7 | MDBOOK_BINARY = $(ROOT)/mdbook 8 | 9 | # The architecture can be changed. Only Linux .tar.gz files 10 | # are currently supported, though. See https://github.com/rust-lang-nursery/mdBook/releases 11 | # for available architectures. 12 | ifeq ($(shell uname),Darwin) 13 | MDBOOK_ARCH = x86_64-apple-darwin 14 | else 15 | MDBOOK_ARCH = x86_64-unknown-linux-gnu 16 | endif 17 | 18 | # The mdbook version. 19 | MDBOOK_RELEASE = v0.4.5 20 | 21 | # Download URL for mdbook and resulting file. 22 | MDBOOK_FILE = mdbook-$(MDBOOK_RELEASE)-$(MDBOOK_ARCH).tar.gz 23 | MDBOOK_URL = https://github.com/rust-lang-nursery/mdBook/releases/download/$(MDBOOK_RELEASE)/$(MDBOOK_FILE) 24 | 25 | # As an extra sanity check, the hash of the downloaded file must match before it is used. 26 | ifeq ($(shell uname),Darwin) 27 | MDBOOK_SHA1 = 9656d6dedb7a56a30aeba8214186702e7824f18d 28 | else 29 | MDBOOK_SHA1 = dd51a3bc1d41092446b710c2f4b69054dc2ea666 30 | endif 31 | 32 | all: $(MDBOOK_BINARY) 33 | cd book && $(MDBOOK_BINARY) build 34 | 35 | clean: 36 | rm -rf docs mdbook-*.tar.gz 37 | 38 | clobber: clean 39 | rm -f mdbook 40 | 41 | # Start mdbook as web server. 42 | MDBOOK_HOSTNAME ?= localhost 43 | MDBOOK_PORT = 3000 44 | serve: 45 | cd book && $(MDBOOK_BINARY) serve --hostname $(MDBOOK_HOSTNAME) --port $(MDBOOK_PORT) 46 | 47 | $(MDBOOK_BINARY): $(MDBOOK_FILE) 48 | if [ "`sha1sum < $(MDBOOK_FILE) | sed -e 's/ *-$$//'`" != $(MDBOOK_SHA1) ]; then \ 49 | echo "ERROR: hash mismatch, check downloaded file $(MDBOOK_FILE) and/or update MDBOOK_SHA1"; \ 50 | exit 1; \ 51 | fi 52 | tar xf $(MDBOOK_FILE) 53 | touch $@ 54 | 55 | $(MDBOOK_FILE): 56 | curl -L -O $(MDBOOK_URL) 57 | 58 | .PHONY: all clean clobber serve 59 | -------------------------------------------------------------------------------- /OWNERS: -------------------------------------------------------------------------------- 1 | approvers: 2 | - saad-ali 3 | - lpabon 4 | - msau42 5 | - xing-yang 6 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Kubernetes-CSI Documentation 2 | 3 | This repository contains documentation capturing how to develop and deploy a [Container Storage Interface](https://github.com/container-storage-interface/spec/blob/master/spec.md) (CSI) driver on Kubernetes. 4 | 5 | To access the documentation go to: [kubernetes-csi.github.io/docs](https://kubernetes-csi.github.io/docs/) 6 | 7 | To make changes to the documentation, modify the files in [book/src](https://github.com/kubernetes-csi/docs/tree/master/book/src) and submit a new PR. 8 | 9 | Once the PR is reviewed and merged, the CI will automatically generate (using [mdbook](https://github.com/rust-lang-nursery/mdBook)) the HTML to serve and check it in to [src/_book](https://github.com/kubernetes-csi/docs/tree/master/book/src/_book). 10 | 11 | To update the [CRD API documentation](book/src/api.md), run: 12 | 13 | ```sh 14 | ./hack/gen-api.sh 15 | ``` 16 | 17 | The script uses the [gen-crd-api-reference-docs](https://github.com/ahmetb/gen-crd-api-reference-docs/) tool to generate a markdown document for the [VolumeSnapshot CRD API](https://github.com/kubernetes-csi/external-snapshotter/tree/master/client/apis/volumesnapshot/v1). See the script for more information and supported configuration. 18 | 19 | ## Community, discussion, contribution, and support 20 | 21 | Learn how to engage with the Kubernetes community on the [community page](https://kubernetes.io/community/). 22 | 23 | You can reach the maintainers of this project at: 24 | 25 | - [Slack channel](https://kubernetes.slack.com/messages/sig-storage) 26 | - [Mailing list](https://groups.google.com/forum/#!forum/kubernetes-sig-storage) 27 | 28 | ### Code of conduct 29 | 30 | Participation in the Kubernetes community is governed by the [Kubernetes Code of Conduct](code-of-conduct.md). 31 | 32 | ## To start editing on localhost 33 | 34 | ```bash 35 | $ git clone git@github.com:kubernetes-csi/docs.git 36 | $ cd docs 37 | $ make $(pwd)/mdbook 38 | $ make serve 39 | ``` 40 | 41 | Access to http:localhost:3000 and you can view a book on localhost! 42 | -------------------------------------------------------------------------------- /SECURITY_CONTACTS: -------------------------------------------------------------------------------- 1 | # Defined below are the security contacts for this repo. 2 | # 3 | # They are the contact point for the Product Security Team to reach out 4 | # to for triaging and handling of incoming issues. 5 | # 6 | # The below names agree to abide by the 7 | # [Embargo Policy](https://github.com/kubernetes/sig-release/blob/master/security-release-process-documentation/security-release-process.md#embargo-policy) 8 | # and will be removed and replaced if they violate that agreement. 9 | # 10 | # DO NOT REPORT SECURITY VULNERABILITIES DIRECTLY TO THESE NAMES, FOLLOW THE 11 | # INSTRUCTIONS AT https://kubernetes.io/security/ 12 | 13 | saad-ali 14 | -------------------------------------------------------------------------------- /book/book.toml: -------------------------------------------------------------------------------- 1 | [book] 2 | title = "Kubernetes CSI Developer Documentation" 3 | description = "This site documents how to develop and deploy a Container Storage Interface (CSI) driver on Kubernetes." 4 | authors = ["The Kubernetes Authors"] 5 | 6 | [output.html] 7 | additional-js = ["custom.js"] 8 | 9 | [build] 10 | build-dir = "../docs" 11 | -------------------------------------------------------------------------------- /book/custom.js: -------------------------------------------------------------------------------- 1 | document.querySelectorAll(".production_drivers a").forEach(function(externalLink){ 2 | externalLink.addEventListener("click", function(e){ 3 | if (confirm("You are being redirected to an external website.\nThe Kubernetes community does not validate external CSI drivers and they should be used at your own risk.") == true) { 4 | return true; 5 | } else { 6 | e.preventDefault(); // cancel the event 7 | return false; 8 | } 9 | }); 10 | }); 11 | -------------------------------------------------------------------------------- /book/src/README.md: -------------------------------------------------------------------------------- 1 | # Kubernetes CSI Documentation 2 | 3 | Any changes to the files in this repo will result in changes to https://kubernetes-csi.github.io/docs/ -------------------------------------------------------------------------------- /book/src/SUMMARY.md: -------------------------------------------------------------------------------- 1 | # Kubernetes/CSI Book 2 | 3 | - [Introduction](introduction.md) 4 | - [Developing a CSI Driver for Kubernetes](developing.md) 5 | - [Versioning, Support, Compatibility Policies](project-policies.md) 6 | - [Kubernetes Changelog](kubernetes-changelog.md) 7 | - [Kubernetes Cluster Controllers](kubernetes-cluster-controllers.md) 8 | - [Snapshot Controller](snapshot-controller.md) 9 | - [Snapshot Validation Webhook](snapshot-validation-webhook.md) 10 | - [CSI Proxy](csi-proxy.md) 11 | - [Sidecar Containers](sidecar-containers.md) 12 | - [external-attacher](external-attacher.md) 13 | - [external-provisioner](external-provisioner.md) 14 | - [external-resizer](external-resizer.md) 15 | - [external-snapshotter](external-snapshotter.md) 16 | - [livenessprobe](livenessprobe.md) 17 | - [node-driver-registrar](node-driver-registrar.md) 18 | - [cluster-driver-registrar](cluster-driver-registrar.md) (deprecated) 19 | - [external-health-monitor-controller](external-health-monitor-controller.md) 20 | - [external-health-monitor-agent](external-health-monitor-agent.md) 21 | - [external-snapshot-metadata](external-snapshot-metadata.md) 22 | - [CSI objects](csi-objects.md) 23 | - [CSIDriver Object](csi-driver-object.md) 24 | - [CSINode Object](csi-node-object.md) 25 | - [Features](features.md) 26 | - [Secrets & Credentials](secrets-and-credentials.md) 27 | - [StorageClass Secrets](secrets-and-credentials-storage-class.md) 28 | - [VolumeSnapshotClass Secrets](secrets-and-credentials-volume-snapshot-class.md) 29 | - [VolumeGroupSnapshotClass Secrets](secrets-and-credentials-volume-group-snapshot-class.md) 30 | - [Topology](topology.md) 31 | - [Raw Block Volume](raw-block.md) 32 | - [Skip Attach](skip-attach.md) 33 | - [Pod Info on Mount](pod-info.md) 34 | - [Volume expansion](volume-expansion.md) 35 | - [Data Sources](volume-datasources.md) 36 | - [Cloning](volume-cloning.md) 37 | - [Volume Snapshot & Restore](snapshot-restore-feature.md) 38 | - [Volume Group Snapshot & Restore](group-snapshot-restore-feature.md) 39 | - [Ephemeral Local Volumes](ephemeral-local-volumes.md) 40 | - [Volume Limits](volume-limits.md) 41 | - [Storage Capacity Tracking](storage-capacity-tracking.md) 42 | - [Volume Health Monitoring](volume-health-monitor.md) 43 | - [Token Requests](token-requests.md) 44 | - [FSGroup Support](support-fsgroup.md) 45 | - [CSI Windows](csi-windows.md) 46 | - [Volume Mode Conversion](prevent-volume-mode-conversion.md) 47 | - [Cross-Namespace Data Sources](cross-namespace-data-sources.md) 48 | - [Changed Block Tracking](changed-block-tracking.md) 49 | - [Volume Attributes Class](volume-attributes-class.md) 50 | - [Deploying a CSI Driver on Kubernetes](deploying.md) 51 | - [Example](example.md) 52 | - [Driver Testing](testing-drivers.md) 53 | - [Unit Testing](unit-testing.md) 54 | - [Functional Testing](functional-testing.md) 55 | - [Drivers](drivers.md) 56 | - [API Reference](api.md) 57 | - [Volume Snapshot](api/volume-snapshot.md) 58 | - [Troubleshooting](troubleshooting.md) 59 | -------------------------------------------------------------------------------- /book/src/_book/Users.md: -------------------------------------------------------------------------------- 1 | # Users 2 | This section describes to users how to deploy, use, and manage a CSI driver. -------------------------------------------------------------------------------- /book/src/_book/example/csi-app.yaml: -------------------------------------------------------------------------------- 1 | kind: Pod 2 | apiVersion: v1 3 | metadata: 4 | name: my-csi-app 5 | spec: 6 | containers: 7 | - name: my-frontend 8 | image: busybox 9 | volumeMounts: 10 | - mountPath: "/data" 11 | name: my-csi-volume 12 | command: [ "sleep", "1000000" ] 13 | volumes: 14 | - name: my-csi-volume 15 | persistentVolumeClaim: 16 | claimName: csi-pvc # defined in csi-pvs.yaml -------------------------------------------------------------------------------- /book/src/_book/example/csi-pvc.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: PersistentVolumeClaim 3 | metadata: 4 | name: csi-pvc 5 | spec: 6 | accessModes: 7 | - ReadWriteOnce 8 | resources: 9 | requests: 10 | storage: 1Gi 11 | storageClassName: csi-hostpath-sc # defined in csi-setup.yaml -------------------------------------------------------------------------------- /book/src/_book/example/csi-setup.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: storage.k8s.io/v1 2 | kind: StorageClass 3 | metadata: 4 | name: csi-hostpath-sc 5 | provisioner: csi-hostpath 6 | reclaimPolicy: Delete 7 | volumeBindingMode: Immediate 8 | --- 9 | apiVersion: v1 10 | kind: ServiceAccount 11 | metadata: 12 | name: csi-service-account 13 | namespace: default 14 | --- 15 | apiVersion: rbac.authorization.k8s.io/v1 16 | kind: ClusterRole 17 | metadata: 18 | name: csi-cluster-role 19 | rules: 20 | - apiGroups: 21 | - "" 22 | resources: 23 | - persistentvolumes 24 | verbs: 25 | - create 26 | - delete 27 | - get 28 | - list 29 | - watch 30 | - update 31 | - apiGroups: 32 | - "" 33 | resources: 34 | - secrets 35 | verbs: 36 | - get 37 | - list 38 | - apiGroups: 39 | - "" 40 | resources: 41 | - persistentvolumeclaims 42 | verbs: 43 | - get 44 | - list 45 | - watch 46 | - update 47 | - apiGroups: 48 | - "" 49 | resources: 50 | - nodes 51 | verbs: 52 | - get 53 | - list 54 | - watch 55 | - update 56 | - apiGroups: 57 | - storage.k8s.io 58 | resources: 59 | - volumeattachments 60 | verbs: 61 | - get 62 | - list 63 | - watch 64 | - update 65 | - apiGroups: 66 | - storage.k8s.io 67 | resources: 68 | - storageclasses 69 | verbs: 70 | - get 71 | - list 72 | - watch 73 | - apiGroups: 74 | - "" 75 | resources: 76 | - events 77 | verbs: 78 | - list 79 | - watch 80 | - create 81 | - update 82 | - patch 83 | --- 84 | apiVersion: rbac.authorization.k8s.io/v1 85 | kind: ClusterRoleBinding 86 | metadata: 87 | name: csi-role-binding 88 | roleRef: 89 | apiGroup: rbac.authorization.k8s.io 90 | kind: ClusterRole 91 | name: csi-cluster-role 92 | subjects: 93 | - kind: ServiceAccount 94 | name: csi-service-account 95 | namespace: default 96 | --- 97 | apiVersion: v1 98 | kind: Pod 99 | metadata: 100 | labels: 101 | app: hostpath-driver 102 | name: csi-pod 103 | namespace: default 104 | spec: 105 | serviceAccount: csi-service-account 106 | containers: 107 | - name: external-provisioner 108 | args: 109 | - --v=5 110 | - --provisioner=csi-hostpath 111 | - --csi-address=/csi/csi.sock 112 | image: quay.io/k8scsi/csi-provisioner:v0.2.0 113 | imagePullPolicy: Always 114 | volumeMounts: 115 | - mountPath: /csi 116 | name: socket-dir 117 | - name: driver-registrar 118 | args: 119 | - --v=5 120 | - --csi-address=/csi/csi.sock 121 | env: 122 | - name: KUBE_NODE_NAME 123 | valueFrom: 124 | fieldRef: 125 | apiVersion: v1 126 | fieldPath: spec.nodeName 127 | image: quay.io/k8scsi/driver-registrar:v0.2.0 128 | imagePullPolicy: Always 129 | volumeMounts: 130 | - mountPath: /csi 131 | name: socket-dir 132 | - name: external-attacher 133 | args: 134 | - --v=5 135 | - --csi-address=/csi/csi.sock 136 | image: quay.io/k8scsi/csi-attacher:v0.2.0 137 | imagePullPolicy: Always 138 | volumeMounts: 139 | - mountPath: /csi 140 | name: socket-dir 141 | - name: hostpath-driver 142 | args: 143 | - --v=5 144 | - --endpoint=$(CSI_ENDPOINT) 145 | - --nodeid=$(KUBE_NODE_NAME) 146 | env: 147 | - name: CSI_ENDPOINT 148 | value: unix:///csi/csi.sock 149 | - name: KUBE_NODE_NAME 150 | valueFrom: 151 | fieldRef: 152 | apiVersion: v1 153 | fieldPath: spec.nodeName 154 | image: quay.io/k8scsi/hostpathplugin:v0.2.0 155 | imagePullPolicy: Always 156 | securityContext: 157 | privileged: true 158 | volumeMounts: 159 | - mountPath: /csi 160 | name: socket-dir 161 | - mountPath: /var/lib/kubelet/pods 162 | mountPropagation: Bidirectional 163 | name: mountpoint-dir 164 | volumes: 165 | - hostPath: 166 | path: /var/lib/kubelet/plugins/csi-hostpath 167 | type: DirectoryOrCreate 168 | name: socket-dir 169 | - hostPath: 170 | path: /var/lib/kubelet/pods 171 | type: DirectoryOrCreate 172 | name: mountpoint-dir -------------------------------------------------------------------------------- /book/src/_book/gitbook/fonts/fontawesome/FontAwesome.otf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kubernetes-csi/docs/5041993eaa6e214c22591b8a1168fdc8548b6b38/book/src/_book/gitbook/fonts/fontawesome/FontAwesome.otf -------------------------------------------------------------------------------- /book/src/_book/gitbook/fonts/fontawesome/fontawesome-webfont.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kubernetes-csi/docs/5041993eaa6e214c22591b8a1168fdc8548b6b38/book/src/_book/gitbook/fonts/fontawesome/fontawesome-webfont.eot -------------------------------------------------------------------------------- /book/src/_book/gitbook/fonts/fontawesome/fontawesome-webfont.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kubernetes-csi/docs/5041993eaa6e214c22591b8a1168fdc8548b6b38/book/src/_book/gitbook/fonts/fontawesome/fontawesome-webfont.ttf -------------------------------------------------------------------------------- /book/src/_book/gitbook/fonts/fontawesome/fontawesome-webfont.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kubernetes-csi/docs/5041993eaa6e214c22591b8a1168fdc8548b6b38/book/src/_book/gitbook/fonts/fontawesome/fontawesome-webfont.woff -------------------------------------------------------------------------------- /book/src/_book/gitbook/fonts/fontawesome/fontawesome-webfont.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kubernetes-csi/docs/5041993eaa6e214c22591b8a1168fdc8548b6b38/book/src/_book/gitbook/fonts/fontawesome/fontawesome-webfont.woff2 -------------------------------------------------------------------------------- /book/src/_book/gitbook/gitbook-plugin-fontsettings/fontsettings.js: -------------------------------------------------------------------------------- 1 | require(['gitbook', 'jquery'], function(gitbook, $) { 2 | // Configuration 3 | var MAX_SIZE = 4, 4 | MIN_SIZE = 0, 5 | BUTTON_ID; 6 | 7 | // Current fontsettings state 8 | var fontState; 9 | 10 | // Default themes 11 | var THEMES = [ 12 | { 13 | config: 'white', 14 | text: 'White', 15 | id: 0 16 | }, 17 | { 18 | config: 'sepia', 19 | text: 'Sepia', 20 | id: 1 21 | }, 22 | { 23 | config: 'night', 24 | text: 'Night', 25 | id: 2 26 | } 27 | ]; 28 | 29 | // Default font families 30 | var FAMILIES = [ 31 | { 32 | config: 'serif', 33 | text: 'Serif', 34 | id: 0 35 | }, 36 | { 37 | config: 'sans', 38 | text: 'Sans', 39 | id: 1 40 | } 41 | ]; 42 | 43 | // Return configured themes 44 | function getThemes() { 45 | return THEMES; 46 | } 47 | 48 | // Modify configured themes 49 | function setThemes(themes) { 50 | THEMES = themes; 51 | updateButtons(); 52 | } 53 | 54 | // Return configured font families 55 | function getFamilies() { 56 | return FAMILIES; 57 | } 58 | 59 | // Modify configured font families 60 | function setFamilies(families) { 61 | FAMILIES = families; 62 | updateButtons(); 63 | } 64 | 65 | // Save current font settings 66 | function saveFontSettings() { 67 | gitbook.storage.set('fontState', fontState); 68 | update(); 69 | } 70 | 71 | // Increase font size 72 | function enlargeFontSize(e) { 73 | e.preventDefault(); 74 | if (fontState.size >= MAX_SIZE) return; 75 | 76 | fontState.size++; 77 | saveFontSettings(); 78 | } 79 | 80 | // Decrease font size 81 | function reduceFontSize(e) { 82 | e.preventDefault(); 83 | if (fontState.size <= MIN_SIZE) return; 84 | 85 | fontState.size--; 86 | saveFontSettings(); 87 | } 88 | 89 | // Change font family 90 | function changeFontFamily(configName, e) { 91 | if (e && e instanceof Event) { 92 | e.preventDefault(); 93 | } 94 | 95 | var familyId = getFontFamilyId(configName); 96 | fontState.family = familyId; 97 | saveFontSettings(); 98 | } 99 | 100 | // Change type of color theme 101 | function changeColorTheme(configName, e) { 102 | if (e && e instanceof Event) { 103 | e.preventDefault(); 104 | } 105 | 106 | var $book = gitbook.state.$book; 107 | 108 | // Remove currently applied color theme 109 | if (fontState.theme !== 0) 110 | $book.removeClass('color-theme-'+fontState.theme); 111 | 112 | // Set new color theme 113 | var themeId = getThemeId(configName); 114 | fontState.theme = themeId; 115 | if (fontState.theme !== 0) 116 | $book.addClass('color-theme-'+fontState.theme); 117 | 118 | saveFontSettings(); 119 | } 120 | 121 | // Return the correct id for a font-family config key 122 | // Default to first font-family 123 | function getFontFamilyId(configName) { 124 | // Search for plugin configured font family 125 | var configFamily = $.grep(FAMILIES, function(family) { 126 | return family.config == configName; 127 | })[0]; 128 | // Fallback to default font family 129 | return (!!configFamily)? configFamily.id : 0; 130 | } 131 | 132 | // Return the correct id for a theme config key 133 | // Default to first theme 134 | function getThemeId(configName) { 135 | // Search for plugin configured theme 136 | var configTheme = $.grep(THEMES, function(theme) { 137 | return theme.config == configName; 138 | })[0]; 139 | // Fallback to default theme 140 | return (!!configTheme)? configTheme.id : 0; 141 | } 142 | 143 | function update() { 144 | var $book = gitbook.state.$book; 145 | 146 | $('.font-settings .font-family-list li').removeClass('active'); 147 | $('.font-settings .font-family-list li:nth-child('+(fontState.family+1)+')').addClass('active'); 148 | 149 | $book[0].className = $book[0].className.replace(/\bfont-\S+/g, ''); 150 | $book.addClass('font-size-'+fontState.size); 151 | $book.addClass('font-family-'+fontState.family); 152 | 153 | if(fontState.theme !== 0) { 154 | $book[0].className = $book[0].className.replace(/\bcolor-theme-\S+/g, ''); 155 | $book.addClass('color-theme-'+fontState.theme); 156 | } 157 | } 158 | 159 | function init(config) { 160 | // Search for plugin configured font family 161 | var configFamily = getFontFamilyId(config.family), 162 | configTheme = getThemeId(config.theme); 163 | 164 | // Instantiate font state object 165 | fontState = gitbook.storage.get('fontState', { 166 | size: config.size || 2, 167 | family: configFamily, 168 | theme: configTheme 169 | }); 170 | 171 | update(); 172 | } 173 | 174 | function updateButtons() { 175 | // Remove existing fontsettings buttons 176 | if (!!BUTTON_ID) { 177 | gitbook.toolbar.removeButton(BUTTON_ID); 178 | } 179 | 180 | // Create buttons in toolbar 181 | BUTTON_ID = gitbook.toolbar.createButton({ 182 | icon: 'fa fa-font', 183 | label: 'Font Settings', 184 | className: 'font-settings', 185 | dropdown: [ 186 | [ 187 | { 188 | text: 'A', 189 | className: 'font-reduce', 190 | onClick: reduceFontSize 191 | }, 192 | { 193 | text: 'A', 194 | className: 'font-enlarge', 195 | onClick: enlargeFontSize 196 | } 197 | ], 198 | $.map(FAMILIES, function(family) { 199 | family.onClick = function(e) { 200 | return changeFontFamily(family.config, e); 201 | }; 202 | 203 | return family; 204 | }), 205 | $.map(THEMES, function(theme) { 206 | theme.onClick = function(e) { 207 | return changeColorTheme(theme.config, e); 208 | }; 209 | 210 | return theme; 211 | }) 212 | ] 213 | }); 214 | } 215 | 216 | // Init configuration at start 217 | gitbook.events.bind('start', function(e, config) { 218 | var opts = config.fontsettings; 219 | 220 | // Generate buttons at start 221 | updateButtons(); 222 | 223 | // Init current settings 224 | init(opts); 225 | }); 226 | 227 | // Expose API 228 | gitbook.fontsettings = { 229 | enlargeFontSize: enlargeFontSize, 230 | reduceFontSize: reduceFontSize, 231 | setTheme: changeColorTheme, 232 | setFamily: changeFontFamily, 233 | getThemes: getThemes, 234 | setThemes: setThemes, 235 | getFamilies: getFamilies, 236 | setFamilies: setFamilies 237 | }; 238 | }); 239 | 240 | 241 | -------------------------------------------------------------------------------- /book/src/_book/gitbook/gitbook-plugin-highlight/ebook.css: -------------------------------------------------------------------------------- 1 | pre, 2 | code { 3 | /* http://jmblog.github.io/color-themes-for-highlightjs */ 4 | /* Tomorrow Comment */ 5 | /* Tomorrow Red */ 6 | /* Tomorrow Orange */ 7 | /* Tomorrow Yellow */ 8 | /* Tomorrow Green */ 9 | /* Tomorrow Aqua */ 10 | /* Tomorrow Blue */ 11 | /* Tomorrow Purple */ 12 | } 13 | pre .hljs-comment, 14 | code .hljs-comment, 15 | pre .hljs-title, 16 | code .hljs-title { 17 | color: #8e908c; 18 | } 19 | pre .hljs-variable, 20 | code .hljs-variable, 21 | pre .hljs-attribute, 22 | code .hljs-attribute, 23 | pre .hljs-tag, 24 | code .hljs-tag, 25 | pre .hljs-regexp, 26 | code .hljs-regexp, 27 | pre .hljs-deletion, 28 | code .hljs-deletion, 29 | pre .ruby .hljs-constant, 30 | code .ruby .hljs-constant, 31 | pre .xml .hljs-tag .hljs-title, 32 | code .xml .hljs-tag .hljs-title, 33 | pre .xml .hljs-pi, 34 | code .xml .hljs-pi, 35 | pre .xml .hljs-doctype, 36 | code .xml .hljs-doctype, 37 | pre .html .hljs-doctype, 38 | code .html .hljs-doctype, 39 | pre .css .hljs-id, 40 | code .css .hljs-id, 41 | pre .css .hljs-class, 42 | code .css .hljs-class, 43 | pre .css .hljs-pseudo, 44 | code .css .hljs-pseudo { 45 | color: #c82829; 46 | } 47 | pre .hljs-number, 48 | code .hljs-number, 49 | pre .hljs-preprocessor, 50 | code .hljs-preprocessor, 51 | pre .hljs-pragma, 52 | code .hljs-pragma, 53 | pre .hljs-built_in, 54 | code .hljs-built_in, 55 | pre .hljs-literal, 56 | code .hljs-literal, 57 | pre .hljs-params, 58 | code .hljs-params, 59 | pre .hljs-constant, 60 | code .hljs-constant { 61 | color: #f5871f; 62 | } 63 | pre .ruby .hljs-class .hljs-title, 64 | code .ruby .hljs-class .hljs-title, 65 | pre .css .hljs-rules .hljs-attribute, 66 | code .css .hljs-rules .hljs-attribute { 67 | color: #eab700; 68 | } 69 | pre .hljs-string, 70 | code .hljs-string, 71 | pre .hljs-value, 72 | code .hljs-value, 73 | pre .hljs-inheritance, 74 | code .hljs-inheritance, 75 | pre .hljs-header, 76 | code .hljs-header, 77 | pre .hljs-addition, 78 | code .hljs-addition, 79 | pre .ruby .hljs-symbol, 80 | code .ruby .hljs-symbol, 81 | pre .xml .hljs-cdata, 82 | code .xml .hljs-cdata { 83 | color: #718c00; 84 | } 85 | pre .css .hljs-hexcolor, 86 | code .css .hljs-hexcolor { 87 | color: #3e999f; 88 | } 89 | pre .hljs-function, 90 | code .hljs-function, 91 | pre .python .hljs-decorator, 92 | code .python .hljs-decorator, 93 | pre .python .hljs-title, 94 | code .python .hljs-title, 95 | pre .ruby .hljs-function .hljs-title, 96 | code .ruby .hljs-function .hljs-title, 97 | pre .ruby .hljs-title .hljs-keyword, 98 | code .ruby .hljs-title .hljs-keyword, 99 | pre .perl .hljs-sub, 100 | code .perl .hljs-sub, 101 | pre .javascript .hljs-title, 102 | code .javascript .hljs-title, 103 | pre .coffeescript .hljs-title, 104 | code .coffeescript .hljs-title { 105 | color: #4271ae; 106 | } 107 | pre .hljs-keyword, 108 | code .hljs-keyword, 109 | pre .javascript .hljs-function, 110 | code .javascript .hljs-function { 111 | color: #8959a8; 112 | } 113 | pre .hljs, 114 | code .hljs { 115 | display: block; 116 | background: white; 117 | color: #4d4d4c; 118 | padding: 0.5em; 119 | } 120 | pre .coffeescript .javascript, 121 | code .coffeescript .javascript, 122 | pre .javascript .xml, 123 | code .javascript .xml, 124 | pre .tex .hljs-formula, 125 | code .tex .hljs-formula, 126 | pre .xml .javascript, 127 | code .xml .javascript, 128 | pre .xml .vbscript, 129 | code .xml .vbscript, 130 | pre .xml .css, 131 | code .xml .css, 132 | pre .xml .hljs-cdata, 133 | code .xml .hljs-cdata { 134 | opacity: 0.5; 135 | } 136 | -------------------------------------------------------------------------------- /book/src/_book/gitbook/gitbook-plugin-lunr/search-lunr.js: -------------------------------------------------------------------------------- 1 | require([ 2 | 'gitbook', 3 | 'jquery' 4 | ], function(gitbook, $) { 5 | // Define global search engine 6 | function LunrSearchEngine() { 7 | this.index = null; 8 | this.store = {}; 9 | this.name = 'LunrSearchEngine'; 10 | } 11 | 12 | // Initialize lunr by fetching the search index 13 | LunrSearchEngine.prototype.init = function() { 14 | var that = this; 15 | var d = $.Deferred(); 16 | 17 | $.getJSON(gitbook.state.basePath+'/search_index.json') 18 | .then(function(data) { 19 | // eslint-disable-next-line no-undef 20 | that.index = lunr.Index.load(data.index); 21 | that.store = data.store; 22 | d.resolve(); 23 | }); 24 | 25 | return d.promise(); 26 | }; 27 | 28 | // Search for a term and return results 29 | LunrSearchEngine.prototype.search = function(q, offset, length) { 30 | var that = this; 31 | var results = []; 32 | 33 | if (this.index) { 34 | results = $.map(this.index.search(q), function(result) { 35 | var doc = that.store[result.ref]; 36 | 37 | return { 38 | title: doc.title, 39 | url: doc.url, 40 | body: doc.summary || doc.body 41 | }; 42 | }); 43 | } 44 | 45 | return $.Deferred().resolve({ 46 | query: q, 47 | results: results.slice(0, length), 48 | count: results.length 49 | }).promise(); 50 | }; 51 | 52 | // Set gitbook research 53 | gitbook.events.bind('start', function(e, config) { 54 | var engine = gitbook.search.getEngine(); 55 | if (!engine) { 56 | gitbook.search.setEngine(LunrSearchEngine, config); 57 | } 58 | }); 59 | }); 60 | -------------------------------------------------------------------------------- /book/src/_book/gitbook/gitbook-plugin-search/search-engine.js: -------------------------------------------------------------------------------- 1 | require([ 2 | 'gitbook', 3 | 'jquery' 4 | ], function(gitbook, $) { 5 | // Global search objects 6 | var engine = null; 7 | var initialized = false; 8 | 9 | // Set a new search engine 10 | function setEngine(Engine, config) { 11 | initialized = false; 12 | engine = new Engine(config); 13 | 14 | init(config); 15 | } 16 | 17 | // Initialize search engine with config 18 | function init(config) { 19 | if (!engine) throw new Error('No engine set for research. Set an engine using gitbook.research.setEngine(Engine).'); 20 | 21 | return engine.init(config) 22 | .then(function() { 23 | initialized = true; 24 | gitbook.events.trigger('search.ready'); 25 | }); 26 | } 27 | 28 | // Launch search for query q 29 | function query(q, offset, length) { 30 | if (!initialized) throw new Error('Search has not been initialized'); 31 | return engine.search(q, offset, length); 32 | } 33 | 34 | // Get stats about search 35 | function getEngine() { 36 | return engine? engine.name : null; 37 | } 38 | 39 | function isInitialized() { 40 | return initialized; 41 | } 42 | 43 | // Initialize gitbook.search 44 | gitbook.search = { 45 | setEngine: setEngine, 46 | getEngine: getEngine, 47 | query: query, 48 | isInitialized: isInitialized 49 | }; 50 | }); -------------------------------------------------------------------------------- /book/src/_book/gitbook/gitbook-plugin-search/search.css: -------------------------------------------------------------------------------- 1 | /* 2 | This CSS only styled the search results section, not the search input 3 | It defines the basic interraction to hide content when displaying results, etc 4 | */ 5 | #book-search-results .search-results { 6 | display: none; 7 | } 8 | #book-search-results .search-results ul.search-results-list { 9 | list-style-type: none; 10 | padding-left: 0; 11 | } 12 | #book-search-results .search-results ul.search-results-list li { 13 | margin-bottom: 1.5rem; 14 | padding-bottom: 0.5rem; 15 | /* Highlight results */ 16 | } 17 | #book-search-results .search-results ul.search-results-list li p em { 18 | background-color: rgba(255, 220, 0, 0.4); 19 | font-style: normal; 20 | } 21 | #book-search-results .search-results .no-results { 22 | display: none; 23 | } 24 | #book-search-results.open .search-results { 25 | display: block; 26 | } 27 | #book-search-results.open .search-noresults { 28 | display: none; 29 | } 30 | #book-search-results.no-results .search-results .has-results { 31 | display: none; 32 | } 33 | #book-search-results.no-results .search-results .no-results { 34 | display: block; 35 | } 36 | -------------------------------------------------------------------------------- /book/src/_book/gitbook/gitbook-plugin-search/search.js: -------------------------------------------------------------------------------- 1 | require([ 2 | 'gitbook', 3 | 'jquery' 4 | ], function(gitbook, $) { 5 | var MAX_RESULTS = 15; 6 | var MAX_DESCRIPTION_SIZE = 500; 7 | 8 | var usePushState = (typeof history.pushState !== 'undefined'); 9 | 10 | // DOM Elements 11 | var $body = $('body'); 12 | var $bookSearchResults; 13 | var $searchInput; 14 | var $searchList; 15 | var $searchTitle; 16 | var $searchResultsCount; 17 | var $searchQuery; 18 | 19 | // Throttle search 20 | function throttle(fn, wait) { 21 | var timeout; 22 | 23 | return function() { 24 | var ctx = this, args = arguments; 25 | if (!timeout) { 26 | timeout = setTimeout(function() { 27 | timeout = null; 28 | fn.apply(ctx, args); 29 | }, wait); 30 | } 31 | }; 32 | } 33 | 34 | function displayResults(res) { 35 | $bookSearchResults.addClass('open'); 36 | 37 | var noResults = res.count == 0; 38 | $bookSearchResults.toggleClass('no-results', noResults); 39 | 40 | // Clear old results 41 | $searchList.empty(); 42 | 43 | // Display title for research 44 | $searchResultsCount.text(res.count); 45 | $searchQuery.text(res.query); 46 | 47 | // Create an
  • element for each result 48 | res.results.forEach(function(res) { 49 | var $li = $('
  • ', { 50 | 'class': 'search-results-item' 51 | }); 52 | 53 | var $title = $('

    '); 54 | 55 | var $link = $('', { 56 | 'href': gitbook.state.basePath + '/' + res.url, 57 | 'text': res.title 58 | }); 59 | 60 | var content = res.body.trim(); 61 | if (content.length > MAX_DESCRIPTION_SIZE) { 62 | content = content.slice(0, MAX_DESCRIPTION_SIZE).trim()+'...'; 63 | } 64 | var $content = $('

    ').html(content); 65 | 66 | $link.appendTo($title); 67 | $title.appendTo($li); 68 | $content.appendTo($li); 69 | $li.appendTo($searchList); 70 | }); 71 | } 72 | 73 | function launchSearch(q) { 74 | // Add class for loading 75 | $body.addClass('with-search'); 76 | $body.addClass('search-loading'); 77 | 78 | // Launch search query 79 | throttle(gitbook.search.query(q, 0, MAX_RESULTS) 80 | .then(function(results) { 81 | displayResults(results); 82 | }) 83 | .always(function() { 84 | $body.removeClass('search-loading'); 85 | }), 1000); 86 | } 87 | 88 | function closeSearch() { 89 | $body.removeClass('with-search'); 90 | $bookSearchResults.removeClass('open'); 91 | } 92 | 93 | function launchSearchFromQueryString() { 94 | var q = getParameterByName('q'); 95 | if (q && q.length > 0) { 96 | // Update search input 97 | $searchInput.val(q); 98 | 99 | // Launch search 100 | launchSearch(q); 101 | } 102 | } 103 | 104 | function bindSearch() { 105 | // Bind DOM 106 | $searchInput = $('#book-search-input input'); 107 | $bookSearchResults = $('#book-search-results'); 108 | $searchList = $bookSearchResults.find('.search-results-list'); 109 | $searchTitle = $bookSearchResults.find('.search-results-title'); 110 | $searchResultsCount = $searchTitle.find('.search-results-count'); 111 | $searchQuery = $searchTitle.find('.search-query'); 112 | 113 | // Launch query based on input content 114 | function handleUpdate() { 115 | var q = $searchInput.val(); 116 | 117 | if (q.length == 0) { 118 | closeSearch(); 119 | } 120 | else { 121 | launchSearch(q); 122 | } 123 | } 124 | 125 | // Detect true content change in search input 126 | // Workaround for IE < 9 127 | var propertyChangeUnbound = false; 128 | $searchInput.on('propertychange', function(e) { 129 | if (e.originalEvent.propertyName == 'value') { 130 | handleUpdate(); 131 | } 132 | }); 133 | 134 | // HTML5 (IE9 & others) 135 | $searchInput.on('input', function(e) { 136 | // Unbind propertychange event for IE9+ 137 | if (!propertyChangeUnbound) { 138 | $(this).unbind('propertychange'); 139 | propertyChangeUnbound = true; 140 | } 141 | 142 | handleUpdate(); 143 | }); 144 | 145 | // Push to history on blur 146 | $searchInput.on('blur', function(e) { 147 | // Update history state 148 | if (usePushState) { 149 | var uri = updateQueryString('q', $(this).val()); 150 | history.pushState({ path: uri }, null, uri); 151 | } 152 | }); 153 | } 154 | 155 | gitbook.events.on('page.change', function() { 156 | bindSearch(); 157 | closeSearch(); 158 | 159 | // Launch search based on query parameter 160 | if (gitbook.search.isInitialized()) { 161 | launchSearchFromQueryString(); 162 | } 163 | }); 164 | 165 | gitbook.events.on('search.ready', function() { 166 | bindSearch(); 167 | 168 | // Launch search from query param at start 169 | launchSearchFromQueryString(); 170 | }); 171 | 172 | function getParameterByName(name) { 173 | var url = window.location.href; 174 | name = name.replace(/[\[\]]/g, '\\$&'); 175 | var regex = new RegExp('[?&]' + name + '(=([^&#]*)|&|#|$)', 'i'), 176 | results = regex.exec(url); 177 | if (!results) return null; 178 | if (!results[2]) return ''; 179 | return decodeURIComponent(results[2].replace(/\+/g, ' ')); 180 | } 181 | 182 | function updateQueryString(key, value) { 183 | value = encodeURIComponent(value); 184 | 185 | var url = window.location.href; 186 | var re = new RegExp('([?&])' + key + '=.*?(&|#|$)(.*)', 'gi'), 187 | hash; 188 | 189 | if (re.test(url)) { 190 | if (typeof value !== 'undefined' && value !== null) 191 | return url.replace(re, '$1' + key + '=' + value + '$2$3'); 192 | else { 193 | hash = url.split('#'); 194 | url = hash[0].replace(re, '$1$3').replace(/(&|\?)$/, ''); 195 | if (typeof hash[1] !== 'undefined' && hash[1] !== null) 196 | url += '#' + hash[1]; 197 | return url; 198 | } 199 | } 200 | else { 201 | if (typeof value !== 'undefined' && value !== null) { 202 | var separator = url.indexOf('?') !== -1 ? '&' : '?'; 203 | hash = url.split('#'); 204 | url = hash[0] + separator + key + '=' + value; 205 | if (typeof hash[1] !== 'undefined' && hash[1] !== null) 206 | url += '#' + hash[1]; 207 | return url; 208 | } 209 | else 210 | return url; 211 | } 212 | } 213 | }); 214 | -------------------------------------------------------------------------------- /book/src/_book/gitbook/gitbook-plugin-sharing/buttons.js: -------------------------------------------------------------------------------- 1 | require(['gitbook', 'jquery'], function(gitbook, $) { 2 | var SITES = { 3 | 'facebook': { 4 | 'label': 'Facebook', 5 | 'icon': 'fa fa-facebook', 6 | 'onClick': function(e) { 7 | e.preventDefault(); 8 | window.open('http://www.facebook.com/sharer/sharer.php?s=100&p[url]='+encodeURIComponent(location.href)); 9 | } 10 | }, 11 | 'twitter': { 12 | 'label': 'Twitter', 13 | 'icon': 'fa fa-twitter', 14 | 'onClick': function(e) { 15 | e.preventDefault(); 16 | window.open('http://twitter.com/home?status='+encodeURIComponent(document.title+' '+location.href)); 17 | } 18 | }, 19 | 'google': { 20 | 'label': 'Google+', 21 | 'icon': 'fa fa-google-plus', 22 | 'onClick': function(e) { 23 | e.preventDefault(); 24 | window.open('https://plus.google.com/share?url='+encodeURIComponent(location.href)); 25 | } 26 | }, 27 | 'weibo': { 28 | 'label': 'Weibo', 29 | 'icon': 'fa fa-weibo', 30 | 'onClick': function(e) { 31 | e.preventDefault(); 32 | window.open('http://service.weibo.com/share/share.php?content=utf-8&url='+encodeURIComponent(location.href)+'&title='+encodeURIComponent(document.title)); 33 | } 34 | }, 35 | 'instapaper': { 36 | 'label': 'Instapaper', 37 | 'icon': 'fa fa-instapaper', 38 | 'onClick': function(e) { 39 | e.preventDefault(); 40 | window.open('http://www.instapaper.com/text?u='+encodeURIComponent(location.href)); 41 | } 42 | }, 43 | 'vk': { 44 | 'label': 'VK', 45 | 'icon': 'fa fa-vk', 46 | 'onClick': function(e) { 47 | e.preventDefault(); 48 | window.open('http://vkontakte.ru/share.php?url='+encodeURIComponent(location.href)); 49 | } 50 | } 51 | }; 52 | 53 | 54 | 55 | gitbook.events.bind('start', function(e, config) { 56 | var opts = config.sharing; 57 | 58 | // Create dropdown menu 59 | var menu = $.map(opts.all, function(id) { 60 | var site = SITES[id]; 61 | 62 | return { 63 | text: site.label, 64 | onClick: site.onClick 65 | }; 66 | }); 67 | 68 | // Create main button with dropdown 69 | if (menu.length > 0) { 70 | gitbook.toolbar.createButton({ 71 | icon: 'fa fa-share-alt', 72 | label: 'Share', 73 | position: 'right', 74 | dropdown: [menu] 75 | }); 76 | } 77 | 78 | // Direct actions to share 79 | $.each(SITES, function(sideId, site) { 80 | if (!opts[sideId]) return; 81 | 82 | gitbook.toolbar.createButton({ 83 | icon: site.icon, 84 | label: site.text, 85 | position: 'right', 86 | onClick: site.onClick 87 | }); 88 | }); 89 | }); 90 | }); 91 | -------------------------------------------------------------------------------- /book/src/_book/gitbook/images/apple-touch-icon-precomposed-152.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kubernetes-csi/docs/5041993eaa6e214c22591b8a1168fdc8548b6b38/book/src/_book/gitbook/images/apple-touch-icon-precomposed-152.png -------------------------------------------------------------------------------- /book/src/_book/gitbook/images/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kubernetes-csi/docs/5041993eaa6e214c22591b8a1168fdc8548b6b38/book/src/_book/gitbook/images/favicon.ico -------------------------------------------------------------------------------- /book/src/_book/images/kubelet.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kubernetes-csi/docs/5041993eaa6e214c22591b8a1168fdc8548b6b38/book/src/_book/images/kubelet.png -------------------------------------------------------------------------------- /book/src/_book/images/sidecar-container.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kubernetes-csi/docs/5041993eaa6e214c22591b8a1168fdc8548b6b38/book/src/_book/images/sidecar-container.png -------------------------------------------------------------------------------- /book/src/api.md: -------------------------------------------------------------------------------- 1 | # API Reference 2 | 3 | The following is the list of CSI APIs: 4 | 5 | * [Volume Snapshot](api/volume-snapshot.md) 6 | -------------------------------------------------------------------------------- /book/src/changed-block-tracking.md: -------------------------------------------------------------------------------- 1 | # Changed Block Tracking 2 | 3 | ## Status 4 | 5 | Status | Min K8s Version | Max K8s Version | Min CSI Version | Max CSI Version | 6 | -------|-----------------|-----------------|-----------------|-----------------| 7 | Alpha | 1.33 | - | [v1.10.0](https://github.com/container-storage-interface/spec/releases/tag/v1.10.0) | - 8 | 9 | 10 | ## Overview 11 | 12 | This optional feature provides a secure mechanism to obtain metadata 13 | on the allocated blocks of a CSI VolumeSnapshot, or the changed blocks between two arbitrary pairs of CSI VolumeSnapshot objects of the same PersistentVolume. 14 | See [Kubernetes Enhancement Proposal 3314](https://github.com/kubernetes/enhancements/tree/master/keps/sig-storage/3314-csi-changed-block-tracking) 15 | for full details. 16 | 17 | A CSI driver must advertise its support for this feature by specifying the 18 | `SNAPSHOT_METADATA_SERVICE` capability in the response of the 19 | [GetPluginCapabilities](https://github.com/container-storage-interface/spec/blob/master/spec.md#getplugincapabilities) 20 | RPC (part of the 21 | [CSI Identity Service](https://github.com/container-storage-interface/spec/blob/master/spec.md#identity-service-rpc)), 22 | and must implement the 23 | [CSI SnapshotMetadata Service](https://github.com/container-storage-interface/spec/blob/master/spec.md#snapshot-metadata-service-rpcs). 24 | 25 | CSI drivers that implement this feature should deploy the 26 | [external-snapshot-metadata](./external-snapshot-metadata.md) 27 | sidecar and create the associated 28 | [Snapshot Metadata Service CR](https://github.com/kubernetes/enhancements/tree/master/keps/sig-storage/3314-csi-changed-block-tracking#snapshot-metadata-service-custom-resource). 29 | 30 | The `external-snapshot-metadata` sidecar implements the server side of the 31 | [Kubernetes SnapshotMetadata Service API](https://github.com/kubernetes/enhancements/tree/master/keps/sig-storage/3314-csi-changed-block-tracking#the-kubernetes-snapshotmetadata-service-api). 32 | Backup applications obtain VolumeSnapshot metadata directly from the sidecar 33 | through this API, bypassing the Kubernetes API server for the most part. 34 | Backup application developers should refer to the [Usage](external-snapshot-metadata.md#usage) 35 | and [Resources](external-snapshot-metadata.md#resources) 36 | sections of the sidecar documentation for details. 37 | 38 | -------------------------------------------------------------------------------- /book/src/cluster-driver-registrar.md: -------------------------------------------------------------------------------- 1 | # CSI cluster-driver-registrar 2 | 3 | ## Deprecated 4 | 5 | This sidecar container was not updated since Kubernetes 1.13. As of Kubernetes 6 | 1.16, this side car container is officially deprecated. 7 | 8 | The purpose of this side car container was to automatically register 9 | a _CSIDriver_ object containing information about the driver with Kubernetes. 10 | Without this side car, developers and CSI driver vendors will now have to add 11 | a CSIDriver object in their installation manifest or any tool that installs 12 | their CSI driver. 13 | 14 | Please see [CSIDriver](csi-driver-object.md) for more information. 15 | 16 | ## Status and Releases 17 | 18 | **Git Repository:** [https://github.com/kubernetes-csi/cluster-driver-registrar](https://github.com/kubernetes-csi/cluster-driver-registrar) 19 | 20 | **Status:** Alpha 21 | 22 | Latest stable release | Branch | Compatible with CSI Version | Container Image | Min k8s Version | Max k8s version 23 | --|--|--|--|--|-- 24 | [cluster-driver-registrar v1.0.1](https://github.com/kubernetes-csi/cluster-driver-registrar/releases/tag/v1.0.1) | [release-1.0](https://github.com/kubernetes-csi/cluster-driver-registrar/tree/release-1.0) | [v1.0.0](https://github.com/container-storage-interface/spec/releases/tag/v1.0.0) | quay.io/k8scsi/csi-cluster-driver-registrar:v1.0.1 | v1.13 | - 25 | [driver-registrar v0.4.2](https://github.com/kubernetes-csi/driver-registrar/releases/tag/v0.4.2) | [release-0.4](https://github.com/kubernetes-csi/driver-registrar/tree/release-0.4) | [v0.3.0](https://github.com/container-storage-interface/spec/releases/tag/v0.3.0) | quay.io/k8scsi/driver-registrar:v0.4.2 | v1.10 | - 26 | 27 | ## Description 28 | 29 | The CSI `cluster-driver-registrar` is a sidecar container that registers a CSI Driver with a Kubernetes cluster by creating a [CSIDriver Object](csi-driver-object.md) which enables the driver to customize how Kubernetes interacts with it. 30 | 31 | ## Usage 32 | 33 | CSI drivers that use one of the following Kubernetes features should use this sidecar container: 34 | 35 | * [Skip Attach](skip-attach.md) 36 | * For drivers that don't support [`ControllerPublishVolume`](https://github.com/container-storage-interface/spec/blob/master/spec.md#controllerpublishvolume), this indicates to Kubernetes to skip the attach operation and eliminates the need to deploy the `external-attacher` sidecar. 37 | * [Pod Info on Mount](pod-info.md) 38 | * This causes Kubernetes to pass metadata such as Pod name and namespace to the `NodePublishVolume` call. 39 | 40 | If you are not using one of these features, this sidecar container (and the creation of the [CSIDriver Object](csi-driver-object.md)) is not required. However, it is still recommended, because the [CSIDriver Object](csi-driver-object.md) makes it easier for users to easily discover the CSI drivers installed on their clusters. 41 | 42 | For detailed information (binary parameters, etc.), see the README of the relevant branch. 43 | 44 | ## Deployment 45 | 46 | The CSI `cluster-driver-registrar` is deployed as a controller. See [deployment section](deploying.md) for more details. 47 | -------------------------------------------------------------------------------- /book/src/cross-namespace-data-sources.md: -------------------------------------------------------------------------------- 1 | # Cross-namespace storage data sources 2 | 3 | ## Status 4 | 5 | Status | Min K8s Version | Max K8s Version | external-provisioner Version 6 | -------|-----------------|-----------------|----------------------------- 7 | Alpha | 1.26 | - | 3.4.0+ 8 | 9 | ## Overview 10 | By default, a `VolumeSnapshot` is a namespace-scoped resource while a `VolumeSnapshotContent` is a cluster-scope resource. 11 | Consequently, you can not restore a snapshot from a different namespace than the source. 12 | 13 | With that feature enabled, you can specify a `namespace` attribute in the [`dataSourceRef`](snapshot-restore-feature.md). Once Kubernetes checks that access is OK, the new PersistentVolume can populate its data from the storage source specified in another 14 | namespace. 15 | 16 | > See the [Kubernetes Enhancement Proposal](https://github.com/kubernetes/enhancements/tree/master/keps/sig-storage/3294-provision-volumes-from-cross-namespace-snapshots) 17 | > for more details on the background, design and discussions. 18 | 19 | ## Usage 20 | 21 | To enable this feature, cluster administrators must: 22 | * Install a CRD for `ReferenceGrants` supplied by the [gateway API project](https://github.com/kubernetes-sigs/gateway-api) 23 | * Enable the `AnyVolumeDataSource` and `CrossNamespaceVolumeDataSource` [feature gates](https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/) for the kube-apiserver and kube-controller-manager 24 | * Install a CRD for the specific `VolumeSnapShot` controller 25 | * Start the CSI Provisioner controller with the argument `--feature-gates=CrossNamespaceVolumeDataSource=true` 26 | * Grant the CSI Provisioner with **get**, **list**, and **watch** permissions for `referencegrants` (API group `gateway.networking.k8s.io`) 27 | * Install the CSI driver 28 | 29 | > For more information about how to use the feature, visit the [Kubernetes blog](https://kubernetes.io/blog/2023/01/02/cross-namespace-data-sources-alpha/) page. 30 | -------------------------------------------------------------------------------- /book/src/csi-node-object.md: -------------------------------------------------------------------------------- 1 | # CSINode Object 2 | 3 | ## Status 4 | 5 | Status | Min K8s Version | Max K8s Version 6 | --|--|-- 7 | Alpha | 1.12 | 1.13 8 | Beta | 1.14 | 1.16 9 | GA | 1.17 | - 10 | 11 | 12 | ## What is the CSINode object? 13 | 14 | CSI drivers generate node specific information. Instead of storing this in the Kubernetes `Node` API Object, a new CSI specific Kubernetes `CSINode` object was created. 15 | 16 | It serves the following purposes: 17 | 18 | 1. Mapping Kubernetes node name to CSI Node name, 19 | * The CSI `GetNodeInfo` call returns the name by which the storage system refers to a node. Kubernetes must use this name in future `ControllerPublishVolume` calls. Therefore, when a new CSI driver is registered, Kubernetes stores the storage system node ID in the `CSINode` object for future reference. 20 | 2. Driver availability 21 | * A way for kubelet to communicate to the kube-controller-manager and kubernetes scheduler whether the driver is available (registered) on the node or not. 22 | 3. Volume topology 23 | * The CSI `GetNodeInfo` call returns a set of keys/values labels identifying the topology of that node. Kubernetes uses this information to do topology-aware provisioning (see [PVC Volume Binding Modes](https://kubernetes.io/docs/concepts/storage/storage-classes/#volume-binding-mode) for more details). It stores the key/values as labels on the Kubernetes node object. In order to recall which `Node` label keys belong to a specific CSI driver, the kubelet stores the keys in the `CSINode` object for future reference. 24 | 25 | ## What fields does the CSINode object have? 26 | 27 | Here is an example of a v1 `CSINode` object: 28 | 29 | ```YAML 30 | apiVersion: storage.k8s.io/v1 31 | kind: CSINode 32 | metadata: 33 | name: node1 34 | spec: 35 | drivers: 36 | - name: mycsidriver.example.com 37 | nodeID: storageNodeID1 38 | topologyKeys: ['mycsidriver.example.com/regions', "mycsidriver.example.com/zones"] 39 | ``` 40 | 41 | What the fields mean: 42 | 43 | - `drivers` - list of CSI drivers running on the node and their properties. 44 | - `name` - the CSI driver that this object refers to. 45 | - `nodeID` - the assigned identifier for the node as determined by the driver. 46 | - `topologyKeys` - A list of topology keys assigned to the node as supported by the driver. 47 | 48 | ## What creates the CSINode object? 49 | 50 | CSI drivers do not need to create the `CSINode` object directly. Kubelet manages the object when a CSI driver registers through the kubelet plugin registration mechanism. The [node-driver-registrar](node-driver-registrar.md) sidecar container helps with this registration. 51 | 52 | ## Changes from Alpha to Beta 53 | 54 | ### CRD to Built in Type 55 | The alpha object was called `CSINodeInfo`, whereas the beta object is called 56 | `CSINode`. The alpha `CSINodeInfo` object was also defined as a [Custom Resource Definition](https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions/#create-a-customresourcedefinition) (CRD). As part of the promotion to beta the object has been moved to the built-in Kubernetes API. 57 | 58 | In the move from alpha to beta, the API Group for this object changed from `csi.storage.k8s.io/v1alpha1` to `storage.k8s.io/v1beta1`. 59 | 60 | There is no automatic update of existing CRDs and their CRs during Kubernetes update to the new build-in type. 61 | 62 | ### Enabling CSINodeInfo on Kubernetes 63 | In Kubernetes v1.12 and v1.13, because the feature was alpha, it was disabled by default. To enable the use of `CSINodeInfo` on these versions, do the following: 64 | 65 | 1. Ensure the feature gate is enabled with `--feature-gates=CSINodeInfo=true` 66 | 2. Either ensure the `CSIDriver` CRD is automatically installed via the [Kubernetes Storage CRD addon](https://github.com/kubernetes/kubernetes/tree/release-1.13/cluster/addons/storage-crds) OR manually install the `CSINodeInfo` CRD on the Kubernetes cluster with the following command: 67 | 68 | ``` 69 | $> kubectl create -f https://raw.githubusercontent.com/kubernetes/csi-api/master/pkg/crd/manifests/csinodeinfo.yaml 70 | ``` 71 | Kubernetes v1.14+, uses the same Kubernetes feature flag, but because the feature is beta, it is enabled by default. And since the API type (as of beta) is built in to the Kubernetes API, installation of the CRD is no longer required. 72 | -------------------------------------------------------------------------------- /book/src/csi-objects.md: -------------------------------------------------------------------------------- 1 | # CSI objects 2 | 3 | The Kubernetes API contains the following CSI specific objects: 4 | 5 | * [CSIDriver Object](csi-driver-object.md) 6 | * [CSINode Object](csi-node-object.md) 7 | 8 | The schema definition for the objects can be found in the [Kubernetes API reference](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#-strong-config-and-storage-apis-strong-) 9 | -------------------------------------------------------------------------------- /book/src/csi-proxy.md: -------------------------------------------------------------------------------- 1 | # CSI Proxy 2 | 3 | ## Status and Releases 4 | 5 | **Git Repository:** [https://github.com/kubernetes-csi/csi-proxy](https://github.com/kubernetes-csi/csi-proxy) 6 | 7 | **Status:** V1 starting with v1.0.0 8 | 9 | Status | Min K8s Version | Max K8s Version 10 | --|--|-- 11 | v0.1.0 | 1.18 | - 12 | v0.2.0+ | 1.18 | - 13 | v1.0.0+ | 1.18 | - 14 | 15 | ## Description 16 | 17 | CSI Proxy is a binary that exposes a set of gRPC APIs around storage operations over named pipes in Windows. A container, such as CSI node plugins, can mount the named pipes depending on operations it wants to exercise on the host and invoke the APIs. 18 | 19 | Each named pipe will support a specific version of an API (e.g. v1alpha1, v2beta1) that targets a specific area of storage (e.g. disk, volume, file, SMB, iSCSI). For example, `\\.\pipe\csi-proxy-filesystem-v1alpha1`, `\\.\pipe\csi-proxy-disk-v1beta1`. Any release of csi-proxy.exe binary will strive to maintain backward compatibility across as many prior stable versions of an API group as possible. Please see details in this [CSI Windows support KEP](https://github.com/kubernetes/enhancements/tree/master/keps/sig-windows/1122-windows-csi-support) 20 | 21 | ## Usage 22 | 23 | Run csi-proxy.exe binary directly on a Windows node. The command line options are: 24 | 25 | * `-kubelet-path`: This is the prefix path of the kubelet directory in the host file system (the default value is set to `C:\var\lib\kubelet`) 26 | 27 | * `-windows-service`: Configure as a Windows Service 28 | 29 | * `-log_file`: If non-empty, use this log file. (Note: must set `logtostdrr`=false if setting -log_file) 30 | 31 | 32 | Note that `-kubelet-pod-path` and `-kubelet-csi-plugins-path` were used in prior 1.0.0 versions, and they are now replaced by new parameter `-kubelet-path` 33 | 34 | For detailed information (binary parameters, etc.), see the README of the relevant branch. 35 | 36 | ## Deployment 37 | 38 | It the responsibility of the Kubernetes distribution or cluster admin to install csi-proxy. Directly run csi-proxy.exe binary or run it as a Windows Service on Kubernetes nodes. 39 | For example, 40 | 41 | ``` 42 | $flags = "-windows-service -log_file=\etc\kubernetes\logs\csi-proxy.log -logtostderr=false" 43 | sc.exe create csiproxy binPath= "${env:NODE_DIR}\csi-proxy.exe $flags" 44 | sc.exe failure csiproxy reset= 0 actions= restart/10000 45 | sc.exe start csiproxy 46 | ``` 47 | -------------------------------------------------------------------------------- /book/src/csi-windows.md: -------------------------------------------------------------------------------- 1 | # CSI Windows Support 2 | 3 | ## Status 4 | 5 | Status | Min K8s Version | Min CSI proxy Version | Min Node Driver Registrar Version 6 | --|--|--|-- 7 | GA | 1.19 | 1.0.0 | 1.3.0 8 | Beta | 1.19 | 0.2.0 | 1.3.0 9 | Alpha | 1.18 | 0.1.0 | 1.3.0 10 | 11 | 12 | ## Overview 13 | 14 | CSI drivers (e.g. AzureDisk, GCE PD, etc.) are recommended to be deployed as containers. CSI driver’s node plugin typically runs on every worker node in the cluster (as a DaemonSet). Node plugin containers need to run with elevated privileges to perform storage related operations. However, Windows was not supporting privileged containers (Note: privileged containers a.k.a Host process is introduced as alpha feature in Kubernetes 1.22 very recently). To solve this problem, [CSI Proxy](https://github.com/kubernetes-csi/csi-proxy) is a binary that runs on the Windows host and executes a set of privileged storage operations on Windows nodes on behalf of containers in a CSI Node plugin daemonset. This enables multiple CSI Node plugins to execute privileged storage operations on Windows nodes without having to ship a custom privileged operation proxy. 15 | 16 | Please note that CSI controller level operations/sidecars are not supported on Windows. 17 | 18 | ## How to use the CSI Proxy for Windows? 19 | See how to install CSI Proxy in the [Deployment chapter](csi-proxy.md#Deployment). 20 | 21 | For CSI driver authors, import CSI proxy client under github.com/kubernetes-csi/csi-proxy/client. There are six client API groups including disk, filesystem, iscsi, smb, system, volume. See [link](https://github.com/kubernetes-csi/csi-proxy/tree/master/client/groups) for details. 22 | As an example, please check how GCE PD Driver import disk, volume and filesystem client API groups [here](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/blob/release-1.2/pkg/mount-manager/safe-mounter_windows.go#L28) 23 | 24 | The Daemonset specification of a CSI node plugin for Windows can mount the desired named pipes from CSI Proxy based on the version of the API groups that the node-plugin needs to execute. 25 | 26 | 27 | The following Daemonset YAML shows how to mount various API groups from CSI Proxy into a CSI Node plugin: 28 | 29 | ``` 30 | kind: DaemonSet 31 | apiVersion: apps/v1 32 | metadata: 33 | name: csi-storage-node-win 34 | spec: 35 | selector: 36 | matchLabels: 37 | app: csi-driver-win 38 | template: 39 | metadata: 40 | labels: 41 | app: csi-driver-win 42 | spec: 43 | serviceAccountName: csi-node-sa 44 | nodeSelector: 45 | kubernetes.io/os: windows 46 | containers: 47 | - name: csi-driver-registrar 48 | image: registry.k8s.io/sig-storage/csi-node-driver-registrar 49 | args: 50 | - "--v=5" 51 | - "--csi-address=unix://C:\\csi\\csi.sock" 52 | - "--kubelet-registration-path=C:\\kubelet\\plugins\\plugin.csi\\csi.sock" 53 | volumeMounts: 54 | - name: plugin-dir 55 | mountPath: C:\csi 56 | - name: registration-dir 57 | mountPath: C:\registration 58 | - name: csi-driver 59 | image: registry.k8s.io/sig-storage/csi-driver:win-v1 60 | args: 61 | - "--v=5" 62 | - "--endpoint=unix:/csi/csi.sock" 63 | volumeMounts: 64 | - name: kubelet-dir 65 | mountPath: C:\var\lib\kubelet 66 | - name: plugin-dir 67 | mountPath: C:\csi 68 | - name: csi-proxy-disk-pipe 69 | mountPath: \\.\pipe\csi-proxy-disk-v1 70 | - name: csi-proxy-volume-pipe 71 | mountPath: \\.\pipe\csi-proxy-volume-v1 72 | - name: csi-proxy-filesystem-pipe 73 | mountPath: \\.\pipe\csi-proxy-filesystem-v1 74 | volumes: 75 | - name: csi-proxy-disk-pipe 76 | hostPath: 77 | path: \\.\pipe\csi-proxy-disk-v1 78 | type: "" 79 | - name: csi-proxy-volume-pipe 80 | hostPath: 81 | path: \\.\pipe\csi-proxy-volume-v1 82 | type: "" 83 | - name: csi-proxy-filesystem-pipe 84 | hostPath: 85 | path: \\.\pipe\csi-proxy-filesystem-v1 86 | type: "" 87 | - name: registration-dir 88 | hostPath: 89 | path: C:\var\lib\kubelet\plugins_registry\ 90 | type: Directory 91 | - name: kubelet-dir 92 | hostPath: 93 | path: C:\var\lib\kubelet\ 94 | type: Directory 95 | - name: plugin-dir 96 | hostPath: 97 | path: C:\var\lib\kubelet\plugins\csi.org.io\ 98 | type: DirectoryOrCreate 99 | ``` 100 | 101 | -------------------------------------------------------------------------------- /book/src/deploying.md: -------------------------------------------------------------------------------- 1 | # Deploying CSI Driver on Kubernetes 2 | 3 | This page describes to CSI driver developers how to deploy their driver onto a Kubernetes cluster. 4 | 5 | ## Overview 6 | 7 | A CSI driver is typically deployed in Kubernetes as two components: 8 | a controller component and a per-node component. 9 | 10 | ### Controller Plugin 11 | 12 | The controller component can be deployed as a Deployment or StatefulSet on 13 | any node in the cluster. It consists of the CSI driver that implements the 14 | CSI Controller service and one or more 15 | [sidecar containers](sidecar-containers.md). These controller 16 | sidecar containers typically interact with Kubernetes objects and make calls 17 | to the driver's CSI Controller service. 18 | 19 | It generally does not need direct access to the host and can perform all its 20 | operations through the Kubernetes API and external control plane services. 21 | Multiple copies of the controller component can be deployed for HA, however 22 | it is recommended to use leader election to ensure there is only one active 23 | controller at a time. 24 | 25 | Controller sidecars include the external-provisioner, external-attacher, 26 | external-snapshotter, and external-resizer. Including a sidecar in the 27 | deployment may be optional. See each sidecar's page for more details. 28 | 29 | #### Communication with Sidecars 30 | [![sidecar-container](images/sidecar-container.png)](https://docs.google.com/a/greatdanedata.com/drawings/d/1JExJ_98dt0NAsJ7iI0_9loeTn2rbLeEcpOMEvKrF-9w/edit?usp=sharing) 31 | 32 | Sidecar containers manage Kubernetes events and make the appropriate 33 | calls to the CSI driver. The calls are made by sharing a UNIX domain socket 34 | through an emptyDir volume between the sidecars and CSI Driver. 35 | 36 | #### RBAC Rules 37 | 38 | Most controller sidecars interact with Kubernetes objects and therefore need 39 | to set RBAC policies. Each sidecar repository contains example RBAC 40 | configurations. 41 | 42 | ### Node Plugin 43 | 44 | The node component should be deployed on every node in the cluster through a 45 | DaemonSet. It consists of the CSI driver that implements the CSI Node service and the 46 | [node-driver-registrar](node-driver-registrar) sidecar container. 47 | 48 | #### Communication with Kubelet 49 | 50 | [![kubelet](images/kubelet.png)](https://docs.google.com/a/greatdanedata.com/drawings/d/1NXaVNDh3mSDhog7Q3Y9eELyEF24F8Z-Kk0ujR3pyOes/edit?usp=sharing) 51 | 52 | The Kubernetes kubelet runs on every node and is responsible for making the CSI 53 | Node service calls. These calls mount and unmount the storage volume from the 54 | storage system, making it available to the Pod to consume. Kubelet makes calls 55 | to the CSI driver through a UNIX domain socket shared on the host via a HostPath 56 | volume. There is also a second UNIX domain socket that the node-driver-registrar 57 | uses to register the CSI driver to kubelet. 58 | 59 | #### Driver Volume Mounts 60 | The node plugin needs direct access to the host for making block devices and/or 61 | filesystem mounts available to the Kubernetes kubelet. 62 | 63 | The mount point used by the CSI driver must be set to _Bidirectional_ to allow Kubelet 64 | on the host to see mounts created by the CSI driver container. See the example below: 65 | 66 | ```yaml 67 | containers: 68 | - name: my-csi-driver 69 | ... 70 | volumeMounts: 71 | - name: socket-dir 72 | mountPath: /csi 73 | - name: mountpoint-dir 74 | mountPath: /var/lib/kubelet/pods 75 | mountPropagation: "Bidirectional" 76 | - name: node-driver-registrar 77 | ... 78 | volumeMounts: 79 | - name: registration-dir 80 | mountPath: /registration 81 | volumes: 82 | # This volume is where the socket for kubelet->driver communication is done 83 | - name: socket-dir 84 | hostPath: 85 | path: /var/lib/kubelet/plugins/ 86 | type: DirectoryOrCreate 87 | # This volume is where the driver mounts volumes 88 | - name: mountpoint-dir 89 | hostPath: 90 | path: /var/lib/kubelet/pods 91 | type: Directory 92 | # This volume is where the node-driver-registrar registers the plugin 93 | # with kubelet 94 | - name: registration-dir 95 | hostPath: 96 | path: /var/lib/kubelet/plugins_registry 97 | type: Directory 98 | ``` 99 | 100 | 101 | ## Deploying 102 | Deploying a CSI driver onto Kubernetes is highlighted in detail in [_Recommended Mechanism for Deploying CSI Drivers on Kubernetes_](https://github.com/kubernetes/design-proposals-archive/blob/main/storage/container-storage-interface.md#recommended-mechanism-for-deploying-csi-drivers-on-kubernetes). 103 | 104 | ## Enable privileged Pods 105 | 106 | To use CSI drivers, your Kubernetes cluster must allow privileged pods (i.e. `--allow-privileged` flag must be set to `true` for both the API server and the kubelet). This is the default in some environments (e.g. GCE, GKE, `kubeadm`). 107 | 108 | Ensure your API server are started with the privileged flag: 109 | 110 | ```shell 111 | $ ./kube-apiserver ... --allow-privileged=true ... 112 | ``` 113 | 114 | ```shell 115 | $ ./kubelet ... --allow-privileged=true ... 116 | ``` 117 | 118 | > Note: Starting from Kubernetes 1.13.0, --allow-privileged is true for kubelet. It'll be deprecated in future kubernetes releases. 119 | 120 | ## Enabling mount propagation 121 | Another feature that CSI depends on is mount propagation. It allows the sharing of volumes mounted by one container with other containers in the same pod, or even to other pods on the same node. For mount propagation to work, the Docker daemon for the cluster must allow shared mounts. See the [mount propagation docs][mount-propagation-docs] to find out how to enable this feature for your cluster. [This page][docker-shared-mount] explains how to check if shared mounts are enabled and how to configure Docker for shared mounts. 122 | 123 | [mount-propagation-docs]: https://kubernetes.io/docs/concepts/storage/volumes/#mount-propagation 124 | [docker-shared-mount]: https://kubernetes.io/docs/concepts/storage/volumes/#configuration 125 | 126 | ### Examples 127 | 128 | - Simple deployment example using a single pod for all components: see the [hostpath example](example.md). 129 | - Full deployment example using a _DaemonSet_ for the node plugin and _StatefulSet_ for the controller plugin: TODO 130 | 131 | ## More information 132 | For more information, please read [_CSI Volume Plugins in Kubernetes Design Doc_](https://github.com/kubernetes/design-proposals-archive/blob/main/storage/container-storage-interface.md). 133 | -------------------------------------------------------------------------------- /book/src/developing.md: -------------------------------------------------------------------------------- 1 | # Developing CSI Driver for Kubernetes 2 | 3 | ## Remain Informed 4 | 5 | All developers of CSI drivers should join https://groups.google.com/forum/#!forum/container-storage-interface-drivers-announce to remain informed about changes to CSI or Kubernetes that may affect existing CSI drivers. 6 | 7 | ## Overview 8 | 9 | The first step to creating a CSI driver is writing an application implementing the [gRPC](https://grpc.io/docs/) services described in the [CSI specification](https://github.com/container-storage-interface/spec/blob/master/spec.md#rpc-interface) 10 | 11 | At a minimum, CSI drivers must implement the following CSI services: 12 | 13 | * CSI `Identity` service 14 | * Enables callers (Kubernetes components and CSI sidecar containers) to identify the driver and what optional functionality it supports. 15 | * CSI `Node` service 16 | * Only `NodePublishVolume`, `NodeUnpublishVolume`, and `NodeGetCapabilities` are required. 17 | * Required methods enable callers to make a volume available at a specified path and discover what optional functionality the driver supports. 18 | 19 | All CSI services may be implemented in the same CSI driver application. The CSI driver application should be containerized to make it easy to deploy on Kubernetes. Once containerized, the CSI driver can be paired with CSI [Sidecar Containers](sidecar-containers.md) and deployed in node and/or controller mode as appropriate. 20 | 21 | ## Capabilities 22 | 23 | If your driver supports additional features, CSI "capabilities" can be used to advertise the optional methods/services it supports, for example: 24 | 25 | * `CONTROLLER_SERVICE` (`PluginCapability`) 26 | * The entire CSI `Controller` service is optional. This capability indicates the driver implement one or more of the methods in the CSI `Controller` service. 27 | * `VOLUME_ACCESSIBILITY_CONSTRAINTS` (`PluginCapability`) 28 | * This capability indicates the volumes for this driver may not be equally accessible from all nodes in the cluster, and that the driver will return additional topology related information that Kubernetes can use to schedule workloads more intelligently or influence where a volume will be provisioned. 29 | * `VolumeExpansion` (`PluginCapability`) 30 | * This capability indicates the driver supports resizing (expanding) volumes after creation. 31 | * `CREATE_DELETE_VOLUME` (`ControllerServiceCapability`) 32 | * This capability indicates the driver supports dynamic volume provisioning and deleting. 33 | * `PUBLISH_UNPUBLISH_VOLUME` (`ControllerServiceCapability`) 34 | * This capability indicates the driver implements `ControllerPublishVolume` and `ControllerUnpublishVolume` -- operations that correspond to the Kubernetes volume attach/detach operations. This may, for example, result in a "volume attach" operation against the Google Cloud control plane to attach the specified volume to the specified node for the Google Cloud PD CSI Driver. 35 | * `CREATE_DELETE_SNAPSHOT` (`ControllerServiceCapability`) 36 | * This capability indicates the driver supports provisioning volume snapshots and the ability to provision new volumes using those snapshots. 37 | * `CLONE_VOLUME` (`ControllerServiceCapability`) 38 | * This capability indicates the driver supports cloning of volumes. 39 | * `STAGE_UNSTAGE_VOLUME` (`NodeServiceCapability`) 40 | * This capability indicates the driver implements `NodeStageVolume` and `NodeUnstageVolume` -- operations that correspond to the Kubernetes volume device mount/unmount operations. This may, for example, be used to create a global (per node) volume mount of a block storage device. 41 | 42 | This is an partial list, please see the [CSI spec](https://github.com/container-storage-interface/spec/blob/master/spec.md) for a complete list of capabilities. 43 | Also see the [Features](features.md) section to understand how a feature integrates with Kubernetes. 44 | -------------------------------------------------------------------------------- /book/src/example.md: -------------------------------------------------------------------------------- 1 | # Example 2 | 3 | The [Hostpath CSI driver](https://github.com/kubernetes-csi/csi-driver-host-path) is a simple sample driver that provisions a directory on the host. It can be used as an example to get started writing a driver, however it is not meant for production use. 4 | The [deployment example](https://github.com/kubernetes-csi/csi-driver-host-path#deployment) shows how to deploy and use that driver in Kubernetes. 5 | 6 | The example deployment uses the original RBAC rule files that are maintained together with sidecar apps and deploys into the default namespace. A real production should copy the RBAC files and customize them as explained in the comments of those files. 7 | 8 | The Hostpath CSI driver only supports the [Changed Block Tracking](./changed-block-tracking.md) feature for 9 | PersistentVolumes with `Block` VolumeMode. 10 | The support is not enabled by default, but requires additional configuration when deploying the driver. 11 | When enabled, the 12 | [external-snapshot-metadata](./external-snapshot-metadata.md) 13 | sidecar will also be deployed, and the additional RBAC policies needed will get created. 14 | See the [docs/example-snapshot-metadata.md](https://github.com/kubernetes-csi/csi-driver-host-path/blob/master/docs/example-snapshot-metadata.md) 15 | document for details. 16 | 17 | If you encounter any problems, please check the [Troubleshooting page](troubleshooting.html). 18 | -------------------------------------------------------------------------------- /book/src/example/snapshot/csi-hostpath-snapshotter.yaml: -------------------------------------------------------------------------------- 1 | kind: Service 2 | apiVersion: v1 3 | metadata: 4 | name: csi-hostpath-snapshotter 5 | labels: 6 | app: csi-hostpath-snapshotter 7 | spec: 8 | selector: 9 | app: csi-hostpath-snapshotter 10 | ports: 11 | - name: dummy 12 | port: 12345 13 | 14 | --- 15 | kind: StatefulSet 16 | apiVersion: apps/v1 17 | metadata: 18 | name: csi-hostpath-snapshotter 19 | spec: 20 | serviceName: "csi-hostpath-snapshotter" 21 | replicas: 1 22 | selector: 23 | matchLabels: 24 | app: csi-hostpath-snapshotter 25 | template: 26 | metadata: 27 | labels: 28 | app: csi-hostpath-snapshotter 29 | spec: 30 | serviceAccount: csi-snapshotter 31 | containers: 32 | - name: csi-snapshotter 33 | image: quay.io/k8scsi/csi-snapshotter:v0.4.1 34 | args: 35 | - "--csi-address=$(ADDRESS)" 36 | - "--connection-timeout=15s" 37 | env: 38 | - name: ADDRESS 39 | value: /csi/csi.sock 40 | imagePullPolicy: Always 41 | volumeMounts: 42 | - mountPath: /csi 43 | name: socket-dir 44 | volumes: 45 | - hostPath: 46 | path: /var/lib/kubelet/plugins/csi-hostpath 47 | type: DirectoryOrCreate 48 | name: socket-dir 49 | -------------------------------------------------------------------------------- /book/src/example/snapshot/csi-restore.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: PersistentVolumeClaim 3 | metadata: 4 | name: hpvc-restore 5 | spec: 6 | storageClassName: csi-hostpath-sc 7 | dataSource: 8 | name: new-snapshot-demo 9 | kind: VolumeSnapshot 10 | apiGroup: snapshot.storage.k8s.io 11 | accessModes: 12 | - ReadWriteOnce 13 | resources: 14 | requests: 15 | storage: 1Gi 16 | -------------------------------------------------------------------------------- /book/src/example/snapshot/csi-snapshot.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: snapshot.storage.k8s.io/v1alpha1 2 | kind: VolumeSnapshot 3 | metadata: 4 | name: new-snapshot-demo 5 | spec: 6 | snapshotClassName: csi-hostpath-snapclass 7 | source: 8 | name: csi-pvc 9 | kind: PersistentVolumeClaim 10 | -------------------------------------------------------------------------------- /book/src/example/snapshot/csi-snapshotclass.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: snapshot.storage.k8s.io/v1alpha1 2 | kind: VolumeSnapshotClass 3 | metadata: 4 | name: csi-hostpath-snapclass 5 | snapshotter: csi-hostpath 6 | -------------------------------------------------------------------------------- /book/src/example/usage/csi-app.yaml: -------------------------------------------------------------------------------- 1 | kind: Pod 2 | apiVersion: v1 3 | metadata: 4 | name: my-csi-app 5 | spec: 6 | containers: 7 | - name: my-frontend 8 | image: busybox 9 | volumeMounts: 10 | - mountPath: "/data" 11 | name: my-csi-volume 12 | command: [ "sleep", "1000000" ] 13 | volumes: 14 | - name: my-csi-volume 15 | persistentVolumeClaim: 16 | claimName: csi-pvc # defined in csi-pvs.yaml -------------------------------------------------------------------------------- /book/src/example/usage/csi-pvc.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: PersistentVolumeClaim 3 | metadata: 4 | name: csi-pvc 5 | spec: 6 | accessModes: 7 | - ReadWriteOnce 8 | resources: 9 | requests: 10 | storage: 1Gi 11 | storageClassName: csi-hostpath-sc # defined in csi-setup.yaml -------------------------------------------------------------------------------- /book/src/example/usage/csi-storageclass.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: storage.k8s.io/v1 2 | kind: StorageClass 3 | metadata: 4 | name: csi-hostpath-sc 5 | provisioner: csi-hostpath 6 | reclaimPolicy: Delete 7 | volumeBindingMode: Immediate 8 | -------------------------------------------------------------------------------- /book/src/external-health-monitor-agent.md: -------------------------------------------------------------------------------- 1 | # CSI external-health-monitor-agent 2 | 3 | ## Status and Releases 4 | 5 | **Git Repository:** [https://github.com/kubernetes-csi/external-health-monitor](https://github.com/kubernetes-csi/external-health-monitor) 6 | 7 | **Status:** Deprecated 8 | 9 | ### Unsupported Versions 10 | Latest stable release | Branch | Min CSI Version | Max CSI Version | Container Image 11 | --|--|--|--|-- 12 | [external-health-monitor-agent v0.2.0](https://github.com/kubernetes-csi/external-health-monitor/releases/tag/v0.2.0) | [release-0.2](https://github.com/kubernetes-csi/external-health-monitor/tree/release-0.2) | [v1.3.0](https://github.com/container-storage-interface/spec/releases/tag/v1.3.0) | - | registry.k8s.io/sig-storage/csi-external-health-monitor-agent:v0.2.0 13 | 14 | ## Description 15 | 16 | *Note*: This sidecar has been deprecated and replaced with the CSIVolumeHealth feature 17 | in Kubernetes. 18 | 19 | The CSI `external-health-monitor-agent` is a sidecar container that is deployed together with the CSI node driver, similar to how the CSI `node-driver-registrar` sidecar is deployed. It calls the CSI node RPC `NodeGetVolumeStats` to check the health condition of the CSI volumes and report events on `Pod` if the condition of a volume is `abnormal`. 20 | 21 | ## Usage 22 | 23 | CSI drivers that support `VOLUME_CONDITION` and `NODE_GET_VOLUME_STATS` node capabilities should use this sidecar container. 24 | 25 | For detailed information (binary parameters, RBAC rules, etc.), see [https://github.com/kubernetes-csi/external-health-monitor/blob/master/README.md](https://github.com/kubernetes-csi/external-health-monitor/blob/master/README.md). 26 | 27 | ## Deployment 28 | 29 | The CSI `external-health-monitor-agent` is deployed as a DaemonSet. See [https://github.com/kubernetes-csi/external-health-monitor/blob/master/README.md](https://github.com/kubernetes-csi/external-health-monitor/blob/master/README.md) for more details. 30 | -------------------------------------------------------------------------------- /book/src/external-health-monitor-controller.md: -------------------------------------------------------------------------------- 1 | # CSI external-health-monitor-controller 2 | 3 | ## Status and Releases 4 | 5 | **Git Repository:** [https://github.com/kubernetes-csi/external-health-monitor](https://github.com/kubernetes-csi/external-health-monitor) 6 | 7 | **Status:** Alpha 8 | 9 | ### Supported Versions 10 | Latest stable release | Branch | Min CSI Version | Max CSI Version | Container Image 11 | --|--|--|--|-- 12 | [external-health-monitor-controller v0.14.0](https://github.com/kubernetes-csi/external-health-monitor/releases/tag/v0.14.0) | [release-0.14](https://github.com/kubernetes-csi/external-health-monitor/tree/release-0.14) | [v1.3.0](https://github.com/container-storage-interface/spec/releases/tag/v1.3.0) | - | registry.k8s.io/sig-storage/csi-external-health-monitor-controller:v0.10.0 13 | [external-health-monitor-controller v0.13.0](https://github.com/kubernetes-csi/external-health-monitor/releases/tag/v0.13.0) | [release-0.13](https://github.com/kubernetes-csi/external-health-monitor/tree/release-0.13) | [v1.3.0](https://github.com/container-storage-interface/spec/releases/tag/v1.3.0) | - | registry.k8s.io/sig-storage/csi-external-health-monitor-controller:v0.9.0 14 | [external-health-monitor-controller v0.12.1](https://github.com/kubernetes-csi/external-health-monitor/releases/tag/v0.12.1) | [release-0.12](https://github.com/kubernetes-csi/external-health-monitor/tree/release-0.12) | [v1.3.0](https://github.com/container-storage-interface/spec/releases/tag/v1.3.0) | - | registry.k8s.io/sig-storage/csi-external-health-monitor-controller:v0.8.0 15 | 16 | ### Unsupported Versions 17 |

    18 | 19 | List of previous versions 20 | 21 | Latest stable release | Branch | Min CSI Version | Max CSI Version | Container Image 22 | --|--|--|--|-- 23 | [external-health-monitor-controller v0.10.0](https://github.com/kubernetes-csi/external-health-monitor/releases/tag/v0.10.0) | [release-0.8](https://github.com/kubernetes-csi/external-health-monitor/tree/release-0.8) | [v1.3.0](https://github.com/container-storage-interface/spec/releases/tag/v1.3.0) | - | registry.k8s.io/sig-storage/csi-external-health-monitor-controller:v0.10.0 24 | [external-health-monitor-controller v0.9.0](https://github.com/kubernetes-csi/external-health-monitor/releases/tag/v0.9.0) | [release-0.8](https://github.com/kubernetes-csi/external-health-monitor/tree/release-0.8) | [v1.3.0](https://github.com/container-storage-interface/spec/releases/tag/v1.3.0) | - | registry.k8s.io/sig-storage/csi-external-health-monitor-controller:v0.9.0 25 | [external-health-monitor-controller v0.8.0](https://github.com/kubernetes-csi/external-health-monitor/releases/tag/v0.8.0) | [release-0.8](https://github.com/kubernetes-csi/external-health-monitor/tree/release-0.8) | [v1.3.0](https://github.com/container-storage-interface/spec/releases/tag/v1.3.0) | - | registry.k8s.io/sig-storage/csi-external-health-monitor-controller:v0.8.0 26 | [external-health-monitor-controller v0.7.0](https://github.com/kubernetes-csi/external-health-monitor/releases/tag/v0.7.0) | [release-0.7](https://github.com/kubernetes-csi/external-health-monitor/tree/release-0.7) | [v1.3.0](https://github.com/container-storage-interface/spec/releases/tag/v1.3.0) | - | registry.k8s.io/sig-storage/csi-external-health-monitor-controller:v0.7.0 27 | [external-health-monitor-controller v0.6.0](https://github.com/kubernetes-csi/external-health-monitor/releases/tag/v0.6.0) | [release-0.6](https://github.com/kubernetes-csi/external-health-monitor/tree/release-0.6) | [v1.3.0](https://github.com/container-storage-interface/spec/releases/tag/v1.3.0) | - | registry.k8s.io/sig-storage/csi-external-health-monitor-controller:v0.6.0 28 | [external-health-monitor-controller v0.4.0](https://github.com/kubernetes-csi/external-health-monitor/releases/tag/v0.4.0) | [release-0.4](https://github.com/kubernetes-csi/external-health-monitor/tree/release-0.4) | [v1.3.0](https://github.com/container-storage-interface/spec/releases/tag/v1.3.0) | - | registry.k8s.io/sig-storage/csi-external-health-monitor-controller:v0.4.0 29 | [external-health-monitor-controller v0.3.0](https://github.com/kubernetes-csi/external-health-monitor/releases/tag/v0.3.0) | [release-0.3](https://github.com/kubernetes-csi/external-health-monitor/tree/release-0.3) | [v1.3.0](https://github.com/container-storage-interface/spec/releases/tag/v1.3.0) | - | registry.k8s.io/sig-storage/csi-external-health-monitor-controller:v0.3.0 30 | [external-health-monitor-controller v0.2.0](https://github.com/kubernetes-csi/external-health-monitor/releases/tag/v0.2.0) | [release-0.2](https://github.com/kubernetes-csi/external-health-monitor/tree/release-0.2) | [v1.3.0](https://github.com/container-storage-interface/spec/releases/tag/v1.3.0) | - | registry.k8s.io/sig-storage/csi-external-health-monitor-controller:v0.2.0 31 | 32 |
    33 | 34 | ## Description 35 | 36 | The CSI `external-health-monitor-controller` is a sidecar container that is deployed together with the CSI controller driver, similar to how the CSI `external-provisioner` sidecar is deployed. It calls the CSI controller RPC `ListVolumes` or `ControllerGetVolume` to check the health condition of the CSI volumes and report events on `PersistentVolumeClaim` if the condition of a volume is `abnormal`. 37 | 38 | The CSI `external-health-monitor-controller` also watches for node failure events. This component can be enabled by setting the `enable-node-watcher` flag to `true`. This will only have effects on local PVs now. When a node failure event is detected, an event will be reported on the PVC to indicate that pods using this PVC are on a failed node. 39 | 40 | ## Usage 41 | 42 | CSI drivers that support `VOLUME_CONDITION` and `LIST_VOLUMES` or `VOLUME_CONDITION` and `GET_VOLUME` controller capabilities should use this sidecar container. 43 | 44 | For detailed information (binary parameters, RBAC rules, etc.), see [https://github.com/kubernetes-csi/external-health-monitor/blob/master/README.md](https://github.com/kubernetes-csi/external-health-monitor/blob/master/README.md). 45 | 46 | ## Deployment 47 | 48 | The CSI `external-health-monitor-controller` is deployed as a controller. See [https://github.com/kubernetes-csi/external-health-monitor/blob/master/README.md](https://github.com/kubernetes-csi/external-health-monitor/blob/master/README.md) for more details. 49 | -------------------------------------------------------------------------------- /book/src/external-resizer.md: -------------------------------------------------------------------------------- 1 | # CSI external-resizer 2 | 3 | ## Status and Releases 4 | 5 | **Git Repository:** [https://github.com/kubernetes-csi/external-resizer](https://github.com/kubernetes-csi/external-resizer) 6 | 7 | **Status:** Beta starting with v0.3.0 8 | 9 | ### Supported Versions 10 | 11 | Latest stable release | Branch | Min CSI Version | Max CSI Version | Container Image | [Min K8s Version](project-policies.md#minimum-version) | [Max K8s Version](project-policies.md#maximum-version) | [Recommended K8s Version](project-policies.md#recommended-version) | 12 | --|--|--|--|--|--|--|-- 13 | [external-resizer v1.13.0](https://github.com/kubernetes-csi/external-resizer/releases/tag/v1.13.0) | [release-1.9](https://github.com/kubernetes-csi/external-resizer/tree/release-1.13) |[v1.5.0](https://github.com/container-storage-interface/spec/releases/tag/v1.5.0) | - | registry.k8s.io/sig-storage/csi-resizer:v1.13.0 | v1.16 | - | v1.32 14 | [external-resizer v1.12.0](https://github.com/kubernetes-csi/external-resizer/releases/tag/v1.12.0) | [release-1.8](https://github.com/kubernetes-csi/external-resizer/tree/release-1.12) |[v1.5.0](https://github.com/container-storage-interface/spec/releases/tag/v1.5.0) | - | registry.k8s.io/sig-storage/csi-resizer:v1.12.0 | v1.16 | - | v1.21 15 | [external-resizer v1.11.0](https://github.com/kubernetes-csi/external-resizer/releases/tag/v1.11.0) | [release-1.7](https://github.com/kubernetes-csi/external-resizer/tree/release-1.11) |[v1.5.0](https://github.com/container-storage-interface/spec/releases/tag/v1.5.0) | - | registry.k8s.io/sig-storage/csi-resizer:v1.7.0 | v1.16 | - | v1.29 16 | 17 | 18 | ### Unsupported Versions 19 |
    20 | 21 | List of previous versions 22 | 23 | Latest stable release | Branch | Min CSI Version | Max CSI Version | Container Image | [Min K8s Version](project-policies.md#minimum-version) | [Max K8s Version](project-policies.md#maximum-version) | [Recommended K8s Version](project-policies.md#recommended-version) | 24 | --|--|--|--|--|--|--|-- 25 | [external-resizer v1.9.0](https://github.com/kubernetes-csi/external-resizer/tree/v1.9.0) | [release-1.9](https://github.com/kubernetes-csi/external-resizer/tree/release-1.8) |[v1.5.0](https://github.com/container-storage-interface/spec/releases/tag/v1.5.0) | - | registry.k8s.io/sig-storage/csi-resizer:v1.9.0 | v1.16 | - | v1.28 26 | [external-resizer v1.8.0](https://github.com/kubernetes-csi/external-resizer/tree/v1.8.0) | [release-1.8](https://github.com/kubernetes-csi/external-resizer/tree/release-1.8) |[v1.5.0](https://github.com/container-storage-interface/spec/releases/tag/v1.5.0) | - | registry.k8s.io/sig-storage/csi-resizer:v1.8.0 | v1.16 | - | v1.23 27 | [external-resizer v1.7.0](https://github.com/kubernetes-csi/external-resizer/tree/v1.7.0) | [release-1.7](https://github.com/kubernetes-csi/external-resizer/tree/release-1.7) |[v1.5.0](https://github.com/container-storage-interface/spec/releases/tag/v1.5.0) | - | registry.k8s.io/sig-storage/csi-resizer:v1.7.0 | v1.16 | - | v1.23 28 | [external-resizer v1.6.0](https://github.com/kubernetes-csi/external-resizer/tree/v1.6.0) | [release-1.6](https://github.com/kubernetes-csi/external-resizer/tree/release-1.6) |[v1.5.0](https://github.com/container-storage-interface/spec/releases/tag/v1.5.0) | - | registry.k8s.io/sig-storage/csi-resizer:v1.6.0 | v1.16 | - | v1.23 29 | [external-resizer v1.5.0](https://github.com/kubernetes-csi/external-resizer/tree/v1.5.0) | [release-1.5](https://github.com/kubernetes-csi/external-resizer/tree/release-1.5) |[v1.5.0](https://github.com/container-storage-interface/spec/releases/tag/v1.5.0) | - | registry.k8s.io/sig-storage/csi-resizer:v1.5.0 | v1.16 | - | v1.23 30 | [external-resizer v1.4.0](https://github.com/kubernetes-csi/external-resizer/tree/v1.4.0) | [release-1.4](https://github.com/kubernetes-csi/external-resizer/tree/release-1.4) |[v1.5.0](https://github.com/container-storage-interface/spec/releases/tag/v1.5.0) | - | registry.k8s.io/sig-storage/csi-resizer:v1.4.0 | v1.16 | - | v1.23 31 | [external-resizer v1.3.0](https://github.com/kubernetes-csi/external-resizer/tree/v1.3.0) | [release-1.3](https://github.com/kubernetes-csi/external-resizer/tree/release-1.3) |[v1.5.0](https://github.com/container-storage-interface/spec/releases/tag/v1.5.0) | - | registry.k8s.io/sig-storage/csi-resizer:v1.3.0 | v1.16 | - | v1.22 32 | [external-resizer v1.2.0](https://github.com/kubernetes-csi/external-resizer/tree/v1.2.0) | [release-1.2](https://github.com/kubernetes-csi/external-resizer/tree/release-1.2) |[v1.2.0](https://github.com/container-storage-interface/spec/releases/tag/v1.2.0) | - | registry.k8s.io/sig-storage/csi-resizer:v1.2.0 | v1.16 | - | v1.21 33 | [external-resizer v1.1.0](https://github.com/kubernetes-csi/external-resizer/tree/v1.1.0) | [release-1.1](https://github.com/kubernetes-csi/external-resizer/tree/release-1.1) |[v1.2.0](https://github.com/container-storage-interface/spec/releases/tag/v1.2.0) | - | registry.k8s.io/sig-storage/csi-resizer:v1.1.0 | v1.16 | - | v1.16 34 | [external-resizer v0.5.0](https://github.com/kubernetes-csi/external-resizer/tree/v0.5.0) | [release-0.5](https://github.com/kubernetes-csi/external-resizer/tree/release-0.5) |[v1.2.0](https://github.com/container-storage-interface/spec/releases/tag/v1.2.0) | - | quay.io/k8scsi/csi-resizer:v0.5.0 | v1.15 | - | v1.16 35 | [external-resizer v0.2.0](https://github.com/kubernetes-csi/external-resizer/tree/v0.2.0) | [release-0.2](https://github.com/kubernetes-csi/external-resizer/tree/release-0.2) |[v1.1.0](https://github.com/container-storage-interface/spec/releases/tag/v1.1.0) | - | quay.io/k8scsi/csi-resizer:v0.2.0 | v1.15 | - | v1.15 36 | [external-resizer v0.1.0](https://github.com/kubernetes-csi/external-resizer/tree/v0.1.0) | [release-0.1](https://github.com/kubernetes-csi/external-resizer/tree/release-0.1) |[v1.1.0](https://github.com/container-storage-interface/spec/releases/tag/v1.1.0) | - | quay.io/k8scsi/csi-resizer:v0.1.0 | v1.14 | v1.14 | v1.14 37 | [external-resizer v1.0.1](https://github.com/kubernetes-csi/external-resizer/tree/v1.0.1) | [release-1.0](https://github.com/kubernetes-csi/external-resizer/tree/release-1.0) |[v1.2.0](https://github.com/container-storage-interface/spec/releases/tag/v1.2.0) | - | quay.io/k8scsi/csi-resizer:v1.0.1 | v1.16 | - | v1.16 38 | 39 |
    40 | 41 | ## Description 42 | 43 | The CSI `external-resizer` is a sidecar container that watches the Kubernetes API server for `PersistentVolumeClaim` object edits and 44 | triggers `ControllerExpandVolume` operations against a CSI endpoint if user requested more storage on `PersistentVolumeClaim` object. 45 | 46 | ## Usage 47 | 48 | CSI drivers that support Kubernetes volume expansion should use this sidecar container, and advertise the CSI `VolumeExpansion` plugin capability. 49 | 50 | ## Deployment 51 | 52 | The CSI `external-resizer` is deployed as a controller. See [deployment section](deploying.md) for more details. 53 | -------------------------------------------------------------------------------- /book/src/external-snapshot-metadata.md: -------------------------------------------------------------------------------- 1 | # CSI external-snapshot-metadata 2 | 3 | ## Status and Releases 4 | 5 | **Git Repository:** [https://github.com/kubernetes-csi/external-snapshot-metadata](https://github.com/kubernetes-csi/external-snapshot-metadata) 6 | 7 | ### Supported Versions 8 | 9 | Latest stable release | Branch | Min CSI Version | Max CSI Version | Container Image | [Min K8s Version](project-policies.md#minimum-version) | [Max K8s Version](project-policies.md#maximum-version) | [Recommended K8s Version](project-policies.md#recommended-version) | 10 | --|--|--|--|--|--|--|-- 11 | v0.1.0 | [v0.1.0](https://github.com/kubernetes-csi/external-snapshot-metadata/releases/tag/v0.1.0) | [v1.10.0](https://github.com/container-storage-interface/spec/releases/tag/v1.10.0) | - | registry.k8s.io/sig-storage/csi-snapshot-metadata:v0.1.0 | v1.33 | - | v1.33 12 | 13 | 14 | ## Alpha 15 | 16 | ### Description 17 | This sidecar securely serves snapshot metadata to Kubernetes clients through the 18 | [Kubernetes SnapshotMetadata Service API](https://github.com/kubernetes/enhancements/tree/master/keps/sig-storage/3314-csi-changed-block-tracking#the-kubernetes-snapshotmetadata-service-api). 19 | This API is similar to the 20 | [CSI SnapshotMetadata Service](https://github.com/container-storage-interface/spec/blob/master/spec.md#snapshot-metadata-service-rpcs) 21 | but is designed to be used by Kuberetes authenticated and authorized backup applications. 22 | Its [protobuf specification](https://github.com/kubernetes-csi/external-snapshot-metadata/tree/main/proto/schema.proto) 23 | is available in the sidecar repository. 24 | 25 | The sidecar authenticates and authorizes each Kubernetes backup application request made through the 26 | Kubernetes SnapshotMetadata Service API. 27 | It then acts as a proxy as it fetches the desired metadata from the CSI driver and 28 | streams it directly to the requesting application with no load on the Kubernetes API server. 29 | 30 | See ["The External Snapshot Metadata Sidecar"](https://github.com/kubernetes/enhancements/tree/master/keps/sig-storage/3314-csi-changed-block-tracking#the-external-snapshot-metadata-sidecar) 31 | section in the CSI Changed Block Tracking KEP for additional details on the sidecar. 32 | 33 | ### Usage 34 | Backup applications, identified by their authorized ServiceAccount objects, 35 | directly communicate with the sidecar using the 36 | [Kubernetes SnapshotMetadata Service API](https://github.com/kubernetes/enhancements/tree/master/keps/sig-storage/3314-csi-changed-block-tracking#the-kubernetes-snapshotmetadata-service-api). 37 | The authorization needed is described in the 38 | ["Risks and Mitigations"](https://github.com/kubernetes/enhancements/tree/master/keps/sig-storage/3314-csi-changed-block-tracking#risks-and-mitigations) 39 | section of the CSI Changed Block Tracking KEP. 40 | In particular, this requires the ability to use the Kubernetes 41 | [TokenRequest](https://kubernetes.io/docs/reference/kubernetes-api/authentication-resources/token-request-v1/) 42 | API and to access the objects required to use the API. 43 | 44 | The availability of this optional feature is advertised to backup applications by the presence of a 45 | [Snapshot Metadata Service CR](https://github.com/kubernetes/enhancements/tree/master/keps/sig-storage/3314-csi-changed-block-tracking#snapshot-metadata-service-custom-resource) 46 | that is named for the CSI driver that provisioned the PersistentVolume and VolumeSnapshot objects involved. 47 | The CR contains a gRPC service endpoint and CA certificate, and an audience string for authentication. 48 | A backup application must use the Kubernetes 49 | [TokenRequest](https://kubernetes.io/docs/reference/kubernetes-api/authentication-resources/token-request-v1/) 50 | API with the audience string to obtain a Kubernetes authentication token for use in the 51 | Kubernetes SnapshotMetadata Service API call. 52 | The backup application should establish trust for the CA certificate before making the gRPC call 53 | to the service endpoint. 54 | 55 | The Kubernetes SnapshotMetadata Service API uses a gRPC stream to return VolumeSnapshot metadata 56 | to the backup application. Metadata can be lengthy, so the API supports 57 | restarting an interrupted metadata request from an intermediate point in case of failure. 58 | The [Resources](external-snapshot-metadata.md#resources) section below describes the 59 | programming artifacts available to support backup application use of this API. 60 | 61 | ### Deployment 62 | The CSI `external-snapshot-metadata` sidecar should be deployed by 63 | CSI drivers that support the 64 | [Changed Block Tracking](./changed-block-tracking.md) feature. 65 | The sidecar must be deployed in the same pod as the CSI driver and 66 | will communicate with its gRPC [CSI SnapshotMetadata Service](https://github.com/container-storage-interface/spec/blob/master/spec.md#snapshot-metadata-service-rpcs) 67 | and [CSI Identity Service](https://github.com/container-storage-interface/spec/blob/master/spec.md#identity-service-rpc) 68 | over a UNIX domain socket. 69 | 70 | The sidecar should be configured to run under the authority of its 71 | CSI driver ServiceAccount, which must be authorized as described in the 72 | ["Risks and Mitigations"](https://github.com/kubernetes/enhancements/tree/master/keps/sig-storage/3314-csi-changed-block-tracking#risks-and-mitigations) 73 | section of the CSI Changed Block Tracking KEP. 74 | In particular, this requires the ability to use the Kubernetes 75 | [TokenReview](https://kubernetes.io/docs/reference/kubernetes-api/authentication-resources/token-review-v1/) 76 | and 77 | [SubjectAccessReview](https://kubernetes.io/docs/reference/kubernetes-api/authorization-resources/subject-access-review-v1/) 78 | APIs. 79 | 80 | A Service object must be created to expose the endpoints of the 81 | [Kubernetes SnapshotMetadata Service API](https://github.com/kubernetes/enhancements/tree/master/keps/sig-storage/3314-csi-changed-block-tracking#the-kubernetes-snapshotmetadata-service-api) 82 | gRPC server implemented by the sidecar. 83 | 84 | A [SnapshotMetadataService CR](https://github.com/kubernetes/enhancements/tree/master/keps/sig-storage/3314-csi-changed-block-tracking#snapshot-metadata-service-custom-resource), 85 | named for the CSI driver, must be created to advertise the 86 | availability of this optional feature to Kubernetes backup application clients. 87 | The CR contains the CA certificate and Service endpoint address 88 | of the sidecar and the audience string needed for the client 89 | authentication token. 90 | 91 | ### Resources 92 | 93 | The [external-snapshot-metadata repository](https://github.com/kubernetes-csi/external-snapshot-metadata) contains 94 | the [protobuf specification](https://github.com/kubernetes-csi/external-snapshot-metadata/tree/main/proto/schema.proto) 95 | of the 96 | [Kubernetes SnapshotMetadata Service API](https://github.com/kubernetes/enhancements/tree/master/keps/sig-storage/3314-csi-changed-block-tracking#the-kubernetes-snapshotmetadata-service-api). 97 | 98 | In addition, the repository has a number of useful artifacts to support Go language programs: 99 | 100 | - The 101 | [Go client interface](https://github.com/kubernetes-csi/external-snapshot-metadata/tree/main/client) 102 | for the Kubernetes SnapshotMetadata Service API. 103 | 104 | - The 105 | [pkg/iterator](https://github.com/kubernetes-csi/external-snapshot-metadata/tree/main/pkg/iterator) 106 | client helper package. 107 | This may be used by backup applications instead of working directly with the 108 | Kubernetes SnapshotMetadata Service gRPC client interfaces. 109 | The 110 | [snapshot-metadata-lister](https://github.com/kubernetes-csi/external-snapshot-metadata/tree/main/examples/snapshot-metadata-lister) 111 | example Go command illustrates the use of this helper package. 112 | 113 | - Go language 114 | [CSI SnapshotMetadataService API client mocks](https://github.com/kubernetes-csi/external-snapshot-metadata/tree/main/pkg/csiclientmocks). 115 | 116 | The sample [Hostpath CSI driver](example.md) has been extended to support the 117 | [Changed Block Tracking](./changed-block-tracking.md) feature 118 | and provides an illustration on how to deploy a CSI driver. 119 | -------------------------------------------------------------------------------- /book/src/features.md: -------------------------------------------------------------------------------- 1 | # Features 2 | 3 | The Kubernetes implementation of CSI has multiple sub-features. This section describes these sub-features, their status (although support for CSI in Kubernetes is GA/stable, support of sub-features moves independently so sub-features maybe alpha or beta), and how to integrate them in to your CSI Driver. 4 | -------------------------------------------------------------------------------- /book/src/functional-testing.md: -------------------------------------------------------------------------------- 1 | # Functional Testing 2 | 3 | Drivers should be functionally "end-to-end" tested while deployed in a Kubernetes cluster. Previously, how to do this and what tests to run was left up to driver authors. Now, a standard set of Kubernetes CSI end-to-end tests can be imported and run by third party CSI drivers. This documentation specifies how to do so. 4 | 5 | The CSI community is also looking in to establishing an official "CSI Conformance Suite" to recognize "officially certified CSI drivers". This documentation will be updated with more information once that process has been defined. 6 | 7 | # Kubernetes End to End Testing for CSI Storage Plugins 8 | 9 | Currently, [csi-sanity](https://github.com/kubernetes-csi/csi-test/tree/master/cmd/csi-sanity) exists to help test compliance with the CSI spec, but e2e testing of plugins is needed as well to provide plugin authors and users validation that their plugin is integrated well with specific versions of Kubernetes. 10 | 11 | ## Setting up End to End tests for your CSI Plugin 12 | 13 | ### Prerequisites: 14 | * A Kubernetes v1.13+ Cluster 15 | * [Kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/#install-kubectl) 16 | 17 | There are two ways to run end-to-end tests for your CSI Plugin 18 | 1) use [Kubernetes E2E Tests](https://github.com/kubernetes/kubernetes/tree/master/test/e2e/storage/external), by providing a DriverDefinition YAML file via a parameter. 19 | * **Note**: In some cases you would not be able to use this method, in running e2e tests by just providing a YAML file defining your CSI plugin. For example the [NFS CSI plugin](https://github.com/kubernetes-csi/csi-driver-nfs) currently does not support dynamic provisioning, so we would want to skip those and run only pre-provisioned tests. For such cases, you would need to write your own testdriver, which is discussed below. 20 | 21 | 2) import the in-tree storage tests and run them using `go test`. 22 | 23 | This doc will cover how to run the E2E tests using the second method. 24 | 25 | ## Importing the E2E test suite as a library 26 | 27 | In-tree storage e2e tests could be used to test CSI storage plugins. Your repo should be setup similar to how the [NFS CSI plugin](https://github.com/kubernetes-csi/csi-driver-nfs) is setup, where the testfiles are in a `test` directory and the main test file is in the `cmd` directory. 28 | 29 | To be able to import Kubernetes in-tree storage tests, the CSI plugin would need to use **Kubernetes v1.14+** (add to plugin's GoPkg.toml, since pluggable E2E tests become available in v1.14). CSI plugin authors would also be required to implement a [testdriver](https://github.com/kubernetes/kubernetes/blob/6644db9914379a4a7b3d3487b41b2010f226e4dc/test/e2e/storage/testsuites/testdriver.go#L31) for their CSI plugin. The testdriver provides required functionality that would help setup testcases for a particular plugin. 30 | 31 | For any testdriver these functions would be required (Since it implements the [TestDriver Interface](https://github.com/kubernetes/kubernetes/blob/6644db9914379a4a7b3d3487b41b2010f226e4dc/test/e2e/storage/testsuites/testdriver.go#L31)): 32 | - `GetDriverInfo() *testsuites.DriverInfo` 33 | - `SkipUnsupportedTest(pattern testpatterns.TestPattern)` 34 | - `PrepareTest(f *framework.Framework) (*testsuites.PerTestConfig, func())` 35 | 36 | The `PrepareTest` method is where you would write code to setup your CSI plugin, and it would be called before each test case. It is recommended that you don't deploy your plugin in this method, and rather deploy it manually before running your tests. 37 | 38 | `GetDriverInfo` will return a `DriverInfo` object that has all of the plugin's capabilities and required information. This object helps tests find the deployed plugin, and also decides which tests should run (depending on the plugin's capabilities). 39 | 40 | Here are examples of the NFS and Hostpath DriverInfo objects: 41 | 42 | ``` 43 | testsuites.DriverInfo{ 44 | Name: "csi-nfsplugin", 45 | MaxFileSize: testpatterns.FileSizeLarge, 46 | SupportedFsType: sets.NewString( 47 | "", // Default fsType 48 | ), 49 | Capabilities: map[testsuites.Capability]bool{ 50 | testsuites.CapPersistence: true, 51 | testsuites.CapExec: true, 52 | }, 53 | } 54 | ``` 55 | 56 | ``` 57 | testsuites.DriverInfo{ 58 | Name: "csi-hostpath", 59 | FeatureTag: "", 60 | MaxFileSize: testpatterns.FileSizeMedium, 61 | SupportedFsType: sets.NewString( 62 | "", // Default fsType 63 | ), 64 | Capabilities: map[testsuites.Capability]bool{ 65 | testsuites.CapPersistence: true, 66 | }, 67 | } 68 | ``` 69 | 70 | You would define something similar for your CSI plugin. 71 | 72 | `SkipUnsupportedTest` simply skips any tests that you define there. 73 | 74 | Depending on your plugin's specs, you would implement other interfaces defined [here](https://github.com/kubernetes/kubernetes/blob/6644db9914379a4a7b3d3487b41b2010f226e4dc/test/e2e/storage/testsuites/testdriver.go#L61). For example the [NFS testdriver](https://github.com/kubernetes-csi/csi-driver-nfs/blob/193faa0f2aa92a3be0855764a1126ff3cdcd3e77/test/nfs-testdriver.go#L66) also implements PreprovisionedVolumeTestDriver and PreprovisionedPVTestDriver interfaces, to enable pre-provisioned tests. 75 | 76 | After implementing the testdriver for your CSI plugin, you would create a `csi-volumes.go` file, where the implemented testdriver is used to run in-tree storage testsuites, [similar to how the NFS CSI plugin does so](https://github.com/kubernetes-csi/csi-driver-nfs/blob/193faa0f2aa92a3be0855764a1126ff3cdcd3e77/test/csi-volumes.go#L37). This is where you would define which testsuites you would want to run for your plugin. All available in-tree testsuites can be found [here](https://github.com/kubernetes/kubernetes/tree/master/test/e2e/storage/testsuites). 77 | 78 | Finally, importing the `test` package into your [main test file](https://github.com/kubernetes-csi/csi-driver-nfs/blob/193faa0f2aa92a3be0855764a1126ff3cdcd3e77/cmd/tests/nfs-e2e.go#L18) will [initialize the testsuites to run the E2E tests](https://github.com/kubernetes-csi/csi-driver-nfs/blob/193faa0f2aa92a3be0855764a1126ff3cdcd3e77/test/csi-volumes.go#L37). 79 | 80 | The NFS plugin creates a binary to run E2E tests, but you could use `go test` instead to run E2E tests using a command like this: 81 | ``` 82 | go test -v
    -ginkgo.v -ginkgo.progress --kubeconfig= -timeout=0 83 | ``` 84 | 85 | -------------------------------------------------------------------------------- /book/src/group-snapshot-restore-feature.md: -------------------------------------------------------------------------------- 1 | # Volume Group Snapshot Feature 2 | 3 | ## Status 4 | 5 | Status | Min K8s Version | Max K8s Version | snapshot-controller Version | snapshot-validation-webhook Version | CSI external-snapshotter sidecar Version | external-provisioner Version 6 | --|--|--|--|--|--|-- 7 | Alpha | 1.27 | 1.32 | 7.0+ | 7.0+ | 7.0+ | 4.0+ 8 | Beta | 1.32 | - | 8.2+ | 8.2+ | 8.2+ | 5.1+ 9 | 10 | **IMPORTANT**: The validation logic for VolumeGroupSnapshots and VolumeGroupSnapshotContents has been replaced by CEL validation rules. The validating webhook is now only being used for VolumeGroupSnapshotClasses to ensure that there's at most one default class per CSI Driver. The validation webhook is deprecated and will be removed in the next release 11 | 12 | ## Overview 13 | 14 | Some storage systems provide the ability to create a crash consistent snapshot of 15 | multiple volumes. A group snapshot represents “copies” from multiple volumes that 16 | are taken at the same point-in-time. A group snapshot can be used either to rehydrate 17 | new volumes (pre-populated with the snapshot data) or to restore existing volumes to 18 | a previous state (represented by the snapshots). 19 | 20 | Kubernetes CSI currently enables CSI Drivers to expose the following 21 | functionality via the Kubernetes API: 22 | 23 | 1. Creation and deletion of volume group snapshots via [Kubernetes native 24 | API](https://github.com/kubernetes-csi/external-snapshotter/tree/master/client/apis/volumegroupsnapshot/v1alpha1). 25 | 2. Creation of new volumes pre-populated with the data from a snapshot that is 26 | part of the volume group snapshot via Kubernetes [dynamic volume 27 | provisioning](https://kubernetes.io/docs/concepts/storage/dynamic-provisioning/). 28 | 29 | ## Implementing Volume Group Snapshot Functionality in your CSI Driver 30 | 31 | To implement the volume group snapshot feature, a CSI driver MUST: 32 | 33 | * Implement a new group controller service. 34 | * Implement group controller RPCs: `CreateVolumeGroupSnapshot`, `DeleteVolumeGroupSnapshot`, and `GetVolumeGroupSnapshot`. 35 | * Add group controller capability `CREATE_DELETE_GET_VOLUME_GROUP_SNAPSHOT`. 36 | 37 | For details, see the [CSI spec](https://github.com/container-storage-interface/spec/blob/master/spec.md). 38 | 39 | ## Sidecar Deployment 40 | 41 | The Kubernetes CSI development team maintains the 42 | [external-snapshotter](external-snapshotter.md) Kubernetes CSI [Sidecar 43 | Containers](sidecar-containers.md). This sidecar container implements the logic 44 | for watching the Kubernetes API objects and issuing the appropriate CSI volume 45 | group snapshot calls against a CSI endpoint. For more details, see 46 | [external-snapshotter documentation](external-snapshotter.md). 47 | 48 | ## Volume Group Snapshot APIs 49 | 50 | With the introduction of Volume Group Snapshot, the user can create and delete a 51 | group snapshot using Kubernetes APIs. 52 | 53 | The schema definition for the custom resources (CRs) can be found 54 | [here](https://github.com/kubernetes-csi/external-snapshotter/tree/master/client/config/crd). 55 | The CRDs should be installed by the Kubernetes distributions. 56 | 57 | There are 3 APIs: 58 | 59 | `VolumeGroupSnapshot` 60 | : Created by a Kubernetes user (or perhaps by your own automation) to request 61 | creation of a volume group snapshot for multiple volumes. 62 | It contains information about the volume group snapshot operation such as the 63 | timestamp when the volume group snapshot was taken and whether it is ready to use. 64 | The creation and deletion of this object represents a desire to create or delete a 65 | cluster resource (a group snapshot). 66 | 67 | `VolumeGroupSnapshotContent` 68 | : Created by the snapshot controller for a dynamically created VolumeGroupSnapshot. 69 | It contains information about the volume group snapshot including the volume group 70 | snapshot ID. 71 | This object represents a provisioned resource on the cluster (a group snapshot). 72 | The VolumeGroupSnapshotContent object binds to the VolumeGroupSnapshot for which it 73 | was created with a one-to-one mapping. 74 | 75 | `VolumeGroupSnapshotClass` 76 | : Created by cluster administrators to describe how volume group snapshots should be 77 | created. including the driver information, the deletion policy, etc. 78 | 79 | ## Controller 80 | 81 | * The controller logic for volume group snapshot is added to the snapshot 82 | controller and the CSI external-snapshotter sidecar. 83 | 84 | The snapshot controller is deployed by the Kubernetes distributions and is 85 | responsible for watching the VolumeGroupSnapshot CRD objects and manges the 86 | creation and deletion lifecycle of volume group snapshots. 87 | 88 | The CSI external-snapshotter sidecar watches Kubernetes 89 | VolumeGroupSnapshotContent CRD objects and triggers 90 | CreateVolumeGroupSnapshot/DeleteVolumeGroupSnapshot against a CSI endpoint. 91 | 92 | ## Snapshot Validation Webhook 93 | 94 | The validating webhook server is updated to validate volume group snapshot 95 | objects. This SHOULD be installed by the Kubernetes distros along with the 96 | snapshot-controller, not end users. It SHOULD be installed in all Kubernetes 97 | clusters that has the volume group snapshot feature enabled. See [Snapshot 98 | Validation Webhook](snapshot-validation-webhook.md) for more details on how to 99 | use the webhook. 100 | 101 | ## Kubernetes Cluster Setup 102 | 103 | See the Deployment section of [Snapshot Controller](snapshot-controller.md) on 104 | how to set up the snapshot controller and CRDs. 105 | 106 | See the Deployment section of [Snapshot Validation 107 | Webhook](snapshot-validation-webhook.md) for more details on how to use the 108 | webhook. 109 | 110 | ## Test Volume Group Snapshot Feature 111 | 112 | To test volume group snapshot version, use the following [example yaml files](https://github.com/kubernetes-csi/external-snapshotter/tree/master/examples/kubernetes). 113 | 114 | Create a _StorageClass_: 115 | ``` 116 | kubectl create -f storageclass.yaml 117 | ``` 118 | 119 | Create _PVCs_: 120 | ``` 121 | # This will create a PVC named hpvc 122 | kubectl create -f pvc.yaml 123 | 124 | # This will create a PVC named hpvc-2 125 | sed "s/hpvc/hpvc-2/" pvc.yaml | kubectl create -f - 126 | ``` 127 | 128 | Add a label to _PVC_: 129 | ``` 130 | kubectl label pvc hpvc hpvc-2 app.kubernetes.io/name=postgresql 131 | ``` 132 | 133 | Create a _VolumeGroupSnapshotClass_: 134 | ``` 135 | kubectl create -f groupsnapshotclass-v1beta1.yaml 136 | ``` 137 | 138 | Create a _VolumeGroupSnapshot_: 139 | ``` 140 | kubectl create -f groupsnapshot-v1beta1.yaml 141 | ``` 142 | 143 | Once the _VolumeGroupSnapshot_ is ready, list the _VolumeSnapshot_ whose owner is the _VolumeGroupSnapshot_: 144 | 145 | ``` 146 | kubectl get volumegroupsnapshot 147 | NAME READYTOUSE VOLUMEGROUPSNAPSHOTCLASS VOLUMEGROUPSNAPSHOTCONTENT CREATIONTIME AGE 148 | new-groupsnapshot-demo true csi-hostpath-groupsnapclass groupsnapcontent-fb7a1c20-54d3-444c-a604-b3ff0f4a8801 4m57s 5m26s 149 | ``` 150 | 151 | ``` 152 | kubectl get volumesnapshot -o=jsonpath='{range .items[?(@.metadata.ownerReferences[0].name=="new-groupsnapshot-demo")]}{.metadata.name}{"\n"}{end}' 153 | 154 | snapshot-4dc1c53a29538b36e85003503a4bcac5dbde4cff59e81f1e3bb80b6c18c3fd03 155 | snapshot-fbfe59eff570171765df664280910c3bf1a4d56e233a5364cd8cb0152a35965b 156 | ``` 157 | 158 | Create a _PVC_ from a _VolumeSnapshot_ that is part of the group snapshot: 159 | ``` 160 | # In the command below, the volume snapshot name should be chosen from 161 | # the ones listed in the output of the previous command 162 | sed 's/new-snapshot-demo-v1/snapshot-4dc1c53a29538b36e85003503a4bcac5dbde4cff59e81f1e3bb80b6c18c3fd03/' restore.yaml | kubectl create -f - 163 | ``` 164 | 165 | ## Examples 166 | 167 | See the [Drivers](drivers.md) for a list of CSI drivers that implement the group snapshot feature. 168 | -------------------------------------------------------------------------------- /book/src/group-snapshot.md: -------------------------------------------------------------------------------- 1 | # Volume Group Snapshot & Restore 2 | -------------------------------------------------------------------------------- /book/src/images/kubelet.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kubernetes-csi/docs/5041993eaa6e214c22591b8a1168fdc8548b6b38/book/src/images/kubelet.png -------------------------------------------------------------------------------- /book/src/images/sidecar-container.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kubernetes-csi/docs/5041993eaa6e214c22591b8a1168fdc8548b6b38/book/src/images/sidecar-container.png -------------------------------------------------------------------------------- /book/src/introduction.md: -------------------------------------------------------------------------------- 1 | # Introduction 2 | 3 | ## Kubernetes Container Storage Interface (CSI) Documentation 4 | 5 | This site documents how to develop, deploy, and test a [Container Storage Interface](https://github.com/container-storage-interface/spec/blob/master/spec.md) (CSI) driver on Kubernetes. 6 | 7 | The [Container Storage Interface](https://github.com/container-storage-interface/spec/blob/master/spec.md) (CSI) is a standard for exposing arbitrary block and file storage systems to containerized workloads on Container Orchestration Systems (COs) like Kubernetes. Using CSI third-party storage providers can write and deploy plugins exposing new storage systems in Kubernetes without ever having to touch the core Kubernetes code. 8 | 9 | The target audience for this site is third-party developers interested in developing CSI drivers for Kubernetes. 10 | 11 | Kubernetes users interested in how to deploy or manage an existing CSI driver on Kubernetes should look at the documentation provided by the author of the CSI driver. 12 | 13 | Kubernetes users interested in how to use a CSI driver should look at [kubernetes.io documentation](https://kubernetes.io/docs/concepts/storage/volumes/#csi). 14 | 15 | ## Kubernetes Releases 16 | | Kubernetes | CSI Spec Compatibility | Status | 17 | | ---------- | -------- | ------ | 18 | | v1.9 | [v0.1.0](https://github.com/container-storage-interface/spec/releases/tag/v0.1.0) | Alpha | 19 | | v1.10 | [v0.2.0](https://github.com/container-storage-interface/spec/releases/tag/v0.2.0) | Beta | 20 | | v1.11 | [v0.3.0](https://github.com/container-storage-interface/spec/releases/tag/v0.3.0) | Beta | 21 | | v1.13 | [v0.3.0](https://github.com/container-storage-interface/spec/releases/tag/v0.3.0), [v1.0.0](https://github.com/container-storage-interface/spec/releases/tag/v1.0.0) | GA | 22 | 23 | # Development and Deployment 24 | 25 | ## Minimum Requirements (for Developing and Deploying a CSI driver for Kubernetes) 26 | 27 | Kubernetes is as minimally prescriptive about packaging and deployment of a CSI Volume Driver as possible. 28 | 29 | The only requirements are around how Kubernetes (master and node) components find and communicate with a CSI driver. 30 | 31 | Specifically, the following is dictated by Kubernetes regarding CSI: 32 | 33 | * Kubelet to CSI Driver Communication 34 | * Kubelet directly issues CSI calls (like `NodeStageVolume`, `NodePublishVolume`, etc.) to CSI drivers via a Unix Domain Socket to mount and unmount volumes. 35 | * Kubelet discovers CSI drivers (and the Unix Domain Socket to use to interact with a CSI driver) via the [kubelet plugin registration mechanism](https://github.com/kubernetes/kubernetes/blob/master/pkg/kubelet/pluginmanager/pluginwatcher/README.md). 36 | * Therefore, all CSI drivers deployed on Kubernetes MUST register themselves using the kubelet plugin registration mechanism on each supported node. 37 | * Master to CSI Driver Communication 38 | * Kubernetes master components do not communicate directly (via a Unix Domain Socket or otherwise) with CSI drivers. 39 | * Kubernetes master components interact only with the Kubernetes API. 40 | * Therefore, CSI drivers that require operations that depend on the Kubernetes API (like volume create, volume attach, volume snapshot, etc.) MUST watch the Kubernetes API and trigger the appropriate CSI operations against it. 41 | 42 | Because these requirements are minimally prescriptive, CSI driver developers are free to implement and deploy their drivers as they see fit. 43 | 44 | _That said, to ease development and deployment, the mechanism described below is recommended._ 45 | 46 | ## Recommended Mechanism (for Developing and Deploying a CSI driver for Kubernetes) 47 | 48 | The Kubernetes development team has established a "Recommended Mechanism" for developing, deploying, and testing CSI Drivers on Kubernetes. 49 | It aims to reduce boilerplate code and simplify the overall process for CSI Driver developers. 50 | 51 | This "Recommended Mechanism" makes use of the following components: 52 | 53 | * Kubernetes CSI [Sidecar Containers](sidecar-containers.md) 54 | * Kubernetes CSI [objects](csi-objects.md) 55 | * CSI [Driver Testing](testing-drivers.md) tools 56 | 57 | To implement a CSI driver using this mechanism, a CSI driver developer should: 58 | 59 | 1. Create a containerized application implementing the _Identity_, _Node_, and optionally the _Controller_ services described in the [CSI specification](https://github.com/container-storage-interface/spec/blob/master/spec.md#rpc-interface) (the CSI driver container). 60 | * See [Developing CSI Driver](developing.md) for more information. 61 | 2. Unit test it using csi-sanity. 62 | * See [Driver - Unit Testing](unit-testing.md) for more information. 63 | 4. Define Kubernetes API YAML files that deploy the CSI driver container along with appropriate sidecar containers. 64 | * See [Deploying in Kubernetes](deploying.md) for more information. 65 | 5. Deploy the driver on a Kubernetes cluster and run end-to-end functional tests on it. 66 | * See [Driver - Functional Testing](functional-testing.md) 67 | 68 | ## Reference Links 69 | 70 | * [Design Doc](https://github.com/kubernetes/design-proposals-archive/blob/main/storage/container-storage-interface.md) -------------------------------------------------------------------------------- /book/src/kubernetes-cluster-controllers.md: -------------------------------------------------------------------------------- 1 | # Kubernetes Cluster Controllers 2 | 3 | The Kubernetes cluster controllers are responsible for managing snapshot objects and operations across multiple CSI drivers, so they should be bundled and deployed by the Kubernetes distributors as part of their Kubernetes cluster management process (independent of any CSI Driver). 4 | 5 | The Kubernetes development team maintains the following Kubernetes cluster controllers: 6 | 7 | * [snapshot-controller](snapshot-controller.md) 8 | -------------------------------------------------------------------------------- /book/src/livenessprobe.md: -------------------------------------------------------------------------------- 1 | # CSI livenessprobe 2 | 3 | ## Status and Releases 4 | 5 | **Git Repository:** [https://github.com/kubernetes-csi/livenessprobe](https://github.com/kubernetes-csi/livenessprobe) 6 | 7 | **Status:** GA/Stable 8 | 9 | ### Supported Versions 10 | 11 | Latest stable release | Branch | Min CSI Version | Max CSI Version | Container Image | [Min K8s Version](project-policies.md#minimum-version) | [Max K8s Version](project-policies.md#maximum-version) | 12 | --|--|--|--|--|--|-- 13 | [livenessprobe v2.15.0](https://github.com/kubernetes-csi/livenessprobe/releases/tag/v2.15.0) | [release-2.15](https://github.com/kubernetes-csi/livenessprobe/tree/release-2.15) | [v1.0.0](https://github.com/container-storage-interface/spec/releases/tag/v1.0.0) |-| registry.k8s.io/sig-storage/livenessprobe:v2.15.0 | v1.13 | - 14 | [livenessprobe v2.14.0](https://github.com/kubernetes-csi/livenessprobe/releases/tag/v2.14.0) | [release-2.14](https://github.com/kubernetes-csi/livenessprobe/tree/release-2.14) | [v1.0.0](https://github.com/container-storage-interface/spec/releases/tag/v1.0.0) |-| registry.k8s.io/sig-storage/livenessprobe:v2.14.0 | v1.13 | - 15 | [livenessprobe v2.13.1](https://github.com/kubernetes-csi/livenessprobe/releases/tag/v2.13.1) | [release-2.13](https://github.com/kubernetes-csi/livenessprobe/tree/release-2.10) | [v1.0.0](https://github.com/container-storage-interface/spec/releases/tag/v1.0.0) |-| registry.k8s.io/sig-storage/livenessprobe:v2.13.1 | v1.13 | - 16 | 17 | 18 | 19 | ### Unsupported Versions 20 | 21 |
    22 | 23 | List of previous versions 24 | 25 | Latest stable release | Branch | Min CSI Version | Max CSI Version | Container Image | [Min K8s Version](project-policies.md#minimum-version) | [Max K8s Version](project-policies.md#maximum-version) | 26 | --|--|--|--|--|--|-- 27 | [livenessprobe v2.12.0](https://github.com/kubernetes-csi/livenessprobe/releases/tag/v2.12.0) | [release-2.12](https://github.com/kubernetes-csi/livenessprobe/tree/release-2.12) | [v1.0.0](https://github.com/container-storage-interface/spec/releases/tag/v1.0.0) |-| registry.k8s.io/sig-storage/livenessprobe:v2.12.0 | v1.13 | - 28 | [livenessprobe v2.11.0](https://github.com/kubernetes-csi/livenessprobe/releases/tag/v2.11.0) | [release-2.11](https://github.com/kubernetes-csi/livenessprobe/tree/release-2.11) | [v1.0.0](https://github.com/container-storage-interface/spec/releases/tag/v1.0.0) |-| registry.k8s.io/sig-storage/livenessprobe:v2.11.0 | v1.13 | - 29 | [livenessprobe v2.10.0](https://github.com/kubernetes-csi/livenessprobe/releases/tag/v2.10.0) | [release-2.10](https://github.com/kubernetes-csi/livenessprobe/tree/release-2.10) | [v1.0.0](https://github.com/container-storage-interface/spec/releases/tag/v1.0.0) |-| registry.k8s.io/sig-storage/livenessprobe:v2.10.0 | v1.13 | - 30 | [livenessprobe v2.9.0](https://github.com/kubernetes-csi/livenessprobe/releases/tag/v2.9.0) | [release-2.9](https://github.com/kubernetes-csi/livenessprobe/tree/release-2.9) | [v1.0.0](https://github.com/container-storage-interface/spec/releases/tag/v1.0.0) |-| registry.k8s.io/sig-storage/livenessprobe:v2.9.0 | v1.13 | - 31 | [livenessprobe v2.8.0](https://github.com/kubernetes-csi/livenessprobe/releases/tag/v2.8.0) | [release-2.8](https://github.com/kubernetes-csi/livenessprobe/tree/release-2.8) | [v1.0.0](https://github.com/container-storage-interface/spec/releases/tag/v1.0.0) |-| registry.k8s.io/sig-storage/livenessprobe:v2.8.0 | v1.13 | - 32 | [livenessprobe v2.7.0](https://github.com/kubernetes-csi/livenessprobe/releases/tag/v2.7.0) | [release-2.7](https://github.com/kubernetes-csi/livenessprobe/tree/release-2.7) | [v1.0.0](https://github.com/container-storage-interface/spec/releases/tag/v1.0.0) |-| registry.k8s.io/sig-storage/livenessprobe:v2.7.0 | v1.13 | - 33 | [livenessprobe v2.6.0](https://github.com/kubernetes-csi/livenessprobe/releases/tag/v2.6.0) | [release-2.6](https://github.com/kubernetes-csi/livenessprobe/tree/release-2.6) | [v1.0.0](https://github.com/container-storage-interface/spec/releases/tag/v1.0.0) |-| registry.k8s.io/sig-storage/livenessprobe:v2.6.0 | v1.13 | - 34 | [livenessprobe v2.5.0](https://github.com/kubernetes-csi/livenessprobe/releases/tag/v2.5.0) | [release-2.5](https://github.com/kubernetes-csi/livenessprobe/tree/release-2.5) | [v1.0.0](https://github.com/container-storage-interface/spec/releases/tag/v1.0.0) |-| registry.k8s.io/sig-storage/livenessprobe:v2.5.0 | v1.13 | - 35 | [livenessprobe v2.4.0](https://github.com/kubernetes-csi/livenessprobe/releases/tag/v2.4.0) | [release-2.4](https://github.com/kubernetes-csi/livenessprobe/tree/release-2.4) | [v1.0.0](https://github.com/container-storage-interface/spec/releases/tag/v1.0.0) |-| registry.k8s.io/sig-storage/livenessprobe:v2.4.0 | v1.13 | - 36 | [livenessprobe v2.3.0](https://github.com/kubernetes-csi/livenessprobe/releases/tag/v2.3.0) | [release-2.3](https://github.com/kubernetes-csi/livenessprobe/tree/release-2.3) | [v1.0.0](https://github.com/container-storage-interface/spec/releases/tag/v1.0.0) |-| registry.k8s.io/sig-storage/livenessprobe:v2.3.0 | v1.13 | - 37 | [livenessprobe v2.2.0](https://github.com/kubernetes-csi/livenessprobe/releases/tag/v2.2.0) | [release-2.2](https://github.com/kubernetes-csi/livenessprobe/tree/release-2.2) | [v1.0.0](https://github.com/container-storage-interface/spec/releases/tag/v1.0.0) |-| registry.k8s.io/sig-storage/livenessprobe:v2.2.0 | v1.13 | - 38 | [livenessprobe v2.1.0](https://github.com/kubernetes-csi/livenessprobe/releases/tag/v2.1.0) | [release-2.1](https://github.com/kubernetes-csi/livenessprobe/tree/release-2.1) | [v1.0.0](https://github.com/container-storage-interface/spec/releases/tag/v1.0.0) |-| registry.k8s.io/sig-storage/livenessprobe:v2.1.0 | v1.13 | - 39 | [livenessprobe v2.0.0](https://github.com/kubernetes-csi/livenessprobe/releases/tag/v2.0.0) | [release-2.0](https://github.com/kubernetes-csi/livenessprobe/tree/release-2.0) | [v1.0.0](https://github.com/container-storage-interface/spec/releases/tag/v1.0.0) |-| quay.io/k8scsi/livenessprobe:v2.0.0 | v1.13 | - 40 | [livenessprobe v1.1.0](https://github.com/kubernetes-csi/livenessprobe/releases/tag/v1.1.0) | [release-1.1](https://github.com/kubernetes-csi/livenessprobe/tree/release-1.1) | [v1.0.0](https://github.com/container-storage-interface/spec/releases/tag/v1.0.0) | -|quay.io/k8scsi/livenessprobe:v1.1.0 | v1.13 | - 41 | Unsupported. | No 0.x branch. | [v0.3.0](https://github.com/container-storage-interface/spec/releases/tag/v0.3.0) |[v0.3.0](https://github.com/container-storage-interface/spec/releases/tag/v0.3.0)| quay.io/k8scsi/livenessprobe:v0.4.1 | v1.10 | v1.16 42 | 43 |
    44 | 45 | ## Description 46 | 47 | The CSI `livenessprobe` is a sidecar container that monitors the health of the CSI driver and reports it to Kubernetes via the [Liveness Probe mechanism](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/). This enables Kubernetes to automatically detect issues with the driver and restart the pod to try and fix the issue. 48 | 49 | ## Usage 50 | 51 | All CSI drivers should use the liveness probe to improve the availability of the driver while deployed on Kubernetes. 52 | 53 | For detailed information (binary parameters, RBAC rules, etc.), see [https://github.com/kubernetes-csi/livenessprobe/blob/master/README.md](https://github.com/kubernetes-csi/livenessprobe/blob/master/README.md). 54 | 55 | ## Deployment 56 | 57 | The CSI `livenessprobe` is deployed as part of controller and node deployments. See [deployment section](deploying.md) for more details. 58 | -------------------------------------------------------------------------------- /book/src/node-driver-registrar.md: -------------------------------------------------------------------------------- 1 | # CSI node-driver-registrar 2 | 3 | ## Status and Releases 4 | 5 | **Git Repository:** [https://github.com/kubernetes-csi/node-driver-registrar](https://github.com/kubernetes-csi/node-driver-registrar) 6 | 7 | **Status:** GA/Stable 8 | 9 | ### Supported Versions 10 | 11 | Latest stable release | Branch | Min CSI Version | Max CSI Version | Container Image | [Min K8s Version](project-policies.md#minimum-version) | [Max K8s Version](project-policies.md#maximum-version) | [Recommended K8s Version](project-policies.md#recommended-version) | 12 | --|--|--|--|--|--|--|-- 13 | [node-driver-registrar v2.13.0](https://github.com/kubernetes-csi/node-driver-registrar/releases/tag/v2.13.0) | [release-2.13](https://github.com/kubernetes-csi/node-driver-registrar/tree/release-2.13) | [v1.0.0](https://github.com/container-storage-interface/spec/releases/tag/v1.0.0) | - | registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.13.0 | v1.13 | - | 1.25 14 | [node-driver-registrar v2.12.0](https://github.com/kubernetes-csi/node-driver-registrar/releases/tag/v2.12.0) | [release-2.12](https://github.com/kubernetes-csi/node-driver-registrar/tree/release-2.12) | [v1.0.0](https://github.com/container-storage-interface/spec/releases/tag/v1.0.0) | - | registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.12.0 | v1.13 | - | 1.25 15 | [node-driver-registrar v2.11.1](https://github.com/kubernetes-csi/node-driver-registrar/releases/tag/v2.11.1) | [release-2.7](https://github.com/kubernetes-csi/node-driver-registrar/tree/release-2.11) | [v1.0.0](https://github.com/container-storage-interface/spec/releases/tag/v1.0.0) | - | registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.11.1 | v1.13 | - | 1.25 16 | 17 | ### Unsupported Versions 18 | 19 |
    20 | 21 | List of previous versions 22 | 23 | Latest stable release | Branch | Min CSI Version | Max CSI Version | Container Image | [Min K8s Version](project-policies.md#minimum-version) | [Max K8s Version](project-policies.md#maximum-version) | [Recommended K8s Version](project-policies.md#recommended-version) | 24 | --|--|--|--|--|--|--|-- 25 | [node-driver-registrar v2.9.0](https://github.com/kubernetes-csi/node-driver-registrar/releases/tag/v2.9.0) | [release-2.8](https://github.com/kubernetes-csi/node-driver-registrar/tree/release-2.9) | [v1.0.0](https://github.com/container-storage-interface/spec/releases/tag/v1.0.0) | - | registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.9.0 | v1.13 | - | 1.25 26 | [node-driver-registrar v2.8.0](https://github.com/kubernetes-csi/node-driver-registrar/releases/tag/v2.8.0) | [release-2.8](https://github.com/kubernetes-csi/node-driver-registrar/tree/release-2.8) | [v1.0.0](https://github.com/container-storage-interface/spec/releases/tag/v1.0.0) | - | registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.8.0 | v1.13 | - 27 | [node-driver-registrar v2.7.0](https://github.com/kubernetes-csi/node-driver-registrar/releases/tag/v2.7.0) | [release-2.7](https://github.com/kubernetes-csi/node-driver-registrar/tree/release-2.7) | [v1.0.0](https://github.com/container-storage-interface/spec/releases/tag/v1.0.0) | - | registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.7.0 | v1.13 | - 28 | [node-driver-registrar v2.6.3](https://github.com/kubernetes-csi/node-driver-registrar/releases/tag/v2.6.3) | [release-2.6](https://github.com/kubernetes-csi/node-driver-registrar/tree/release-2.6) | [v1.0.0](https://github.com/container-storage-interface/spec/releases/tag/v1.0.0) | - | registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.6.3 | v1.13 | - 29 | [node-driver-registrar v2.5.1](https://github.com/kubernetes-csi/node-driver-registrar/releases/tag/v2.5.1) | [release-2.5](https://github.com/kubernetes-csi/node-driver-registrar/tree/release-2.5) | [v1.0.0](https://github.com/container-storage-interface/spec/releases/tag/v1.0.0) | - | registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1 | v1.13 | - 30 | [node-driver-registrar v2.4.0](https://github.com/kubernetes-csi/node-driver-registrar/releases/tag/v2.4.0) | [release-2.4](https://github.com/kubernetes-csi/node-driver-registrar/tree/release-2.4) | [v1.0.0](https://github.com/container-storage-interface/spec/releases/tag/v1.0.0) | - | registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.4.0 | v1.13 | - 31 | [node-driver-registrar v2.3.0](https://github.com/kubernetes-csi/node-driver-registrar/releases/tag/v2.3.0) | [release-2.3](https://github.com/kubernetes-csi/node-driver-registrar/tree/release-2.3) | [v1.0.0](https://github.com/container-storage-interface/spec/releases/tag/v1.0.0) | - | registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.3.0 | v1.13 | - 32 | [node-driver-registrar v2.2.0](https://github.com/kubernetes-csi/node-driver-registrar/releases/tag/v2.2.0) | [release-2.2](https://github.com/kubernetes-csi/node-driver-registrar/tree/release-2.2) | [v1.0.0](https://github.com/container-storage-interface/spec/releases/tag/v1.0.0) | - | registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.2.0 | v1.13 | - 33 | [node-driver-registrar v2.1.0](https://github.com/kubernetes-csi/node-driver-registrar/releases/tag/v2.1.0) | [release-2.1](https://github.com/kubernetes-csi/node-driver-registrar/tree/release-2.1) | [v1.0.0](https://github.com/container-storage-interface/spec/releases/tag/v1.0.0) | - | registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.1.0 | v1.13 | - 34 | [node-driver-registrar v2.0.0](https://github.com/kubernetes-csi/node-driver-registrar/releases/tag/v2.0.0) | [release-2.0](https://github.com/kubernetes-csi/node-driver-registrar/tree/release-2.0) | [v1.0.0](https://github.com/container-storage-interface/spec/releases/tag/v1.0.0) | - | registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.0.0 | v1.13 | - 35 | [node-driver-registrar v1.2.0](https://github.com/kubernetes-csi/node-driver-registrar/releases/tag/v1.2.0) | [release-1.2](https://github.com/kubernetes-csi/node-driver-registrar/tree/release-1.2) | [v1.0.0](https://github.com/container-storage-interface/spec/releases/tag/v1.0.0) | - | quay.io/k8scsi/csi-node-driver-registrar:v1.2.0 | v1.13 | - 36 | [driver-registrar v0.4.2](https://github.com/kubernetes-csi/driver-registrar/releases/tag/v0.4.2) | [release-0.4](https://github.com/kubernetes-csi/driver-registrar/tree/release-0.4) | [v0.3.0](https://github.com/container-storage-interface/spec/releases/tag/v0.3.0) | [v0.3.0](https://github.com/container-storage-interface/spec/releases/tag/v0.3.0) | quay.io/k8scsi/driver-registrar:v0.4.2 | v1.10 | v1.16 37 | 38 |
    39 | 40 | ## Description 41 | 42 | The CSI `node-driver-registrar` is a sidecar container that fetches driver information (using `NodeGetInfo`) from a CSI endpoint and registers it with the kubelet on that node using the [kubelet plugin registration mechanism](https://github.com/kubernetes/kubernetes/blob/master/pkg/kubelet/pluginmanager/pluginwatcher/README.md). 43 | 44 | ## Usage 45 | 46 | Kubelet directly issues CSI `NodeGetInfo`, `NodeStageVolume`, and `NodePublishVolume` calls against CSI drivers. It uses the [kubelet plugin registration mechanism](https://github.com/kubernetes/kubernetes/blob/master/pkg/kubelet/pluginmanager/pluginwatcher/README.md) to discover the unix domain socket to talk to the CSI driver. Therefore, all CSI drivers should use this sidecar container to register themselves with kubelet. 47 | 48 | For detailed information (binary parameters, etc.), see the README of the relevant branch. 49 | 50 | ## Deployment 51 | 52 | The CSI `node-driver-registrar` is deployed per node. See [deployment section](deploying.md) for more details. 53 | -------------------------------------------------------------------------------- /book/src/pod-info.md: -------------------------------------------------------------------------------- 1 | # Pod Info on Mount 2 | 3 | # Status 4 | 5 | Status | Min K8s Version | Max K8s Version | cluster-driver-registrar Version 6 | --|--|--|-- 7 | Alpha | 1.12 | 1.12 | 0.4 8 | Alpha | 1.13 | 1.13 | 1.0 9 | Beta | 1.14 | 1.17 | n/a 10 | GA | 1.18 | - | n/a 11 | 12 | # Overview 13 | 14 | CSI avoids encoding Kubernetes specific information in to the specification, since it aims to support multiple orchestration systems (beyond just Kubernetes). 15 | 16 | This can be problematic because some CSI drivers require information about the workload (e.g. which pod is referencing this volume), and CSI does not provide this information natively to drivers. 17 | 18 | # Pod Info on Mount with CSI Driver Object 19 | 20 | The [CSIDriver Object](csi-driver-object.md) enables CSI Drivers to specify how Kubernetes should interact with it. 21 | 22 | Specifically the `podInfoOnMount` field instructs Kubernetes that the CSI driver requires additional pod information (like podName, podUID, etc.) during mount operations. 23 | 24 | For example, the existence of the following object would cause Kubernetes to add pod information at mount time to the `NodePublishVolumeRequest.volume_context` map. 25 | 26 | ``` 27 | apiVersion: storage.k8s.io/v1 28 | kind: CSIDriver 29 | metadata: 30 | name: testcsidriver.example.com 31 | spec: 32 | podInfoOnMount: true 33 | ``` 34 | 35 | If the `podInfoOnMount` field is set to `true`, during mount, Kubelet will add the following key/values to the `volume_context` field in the CSI `NodePublishVolumeRequest`: 36 | 37 | 38 | * `csi.storage.k8s.io/pod.name: {pod.Name}` 39 | * `csi.storage.k8s.io/pod.namespace: {pod.Namespace}` 40 | * `csi.storage.k8s.io/pod.uid: {pod.UID}` 41 | * `csi.storage.k8s.io/serviceAccount.name: {pod.Spec.ServiceAccountName}` 42 | 43 | The CSIDriver object should be manually included in the driver manifests. 44 | 45 | Previously, the [cluster-driver-registrar](cluster-driver-registrar.md) sidecar container could be used to create the object. Once the flags to this container are configured correctly, it will automatically create a [CSIDriver Object](csi-driver-object.md) when it starts with the correct fields set. 46 | 47 | ## Alpha Functionality 48 | In alpha, this feature was enabled by setting the `podInfoOnMountVersion` field in the `CSIDriver` Object CRD to `v1`. 49 | 50 | ``` 51 | apiVersion: csi.storage.k8s.io/v1alpha1 52 | kind: CSIDriver 53 | metadata: 54 | name: testcsidriver.example.com 55 | spec: 56 | podInfoOnMountVersion: v1 57 | ``` 58 | -------------------------------------------------------------------------------- /book/src/prevent-volume-mode-conversion.md: -------------------------------------------------------------------------------- 1 | # Prevent unauthorised volume mode conversion 2 | 3 | ## Status 4 | 5 | Status | Min K8s Version | Max K8s Version | external-snapshotter Version | external-provisioner Version 6 | --|--|--|--|-- 7 | Alpha | 1.24 | - | 6.0.1+ | 3.2.1+ 8 | Beta | 1.28 | - | 7.0.0+ | 4.0.0+ 9 | GA | 1.30 | - | 8.0.1+ | 5.0.1+ 10 | 11 | ## Overview 12 | 13 | Malicious users can populate the `spec.volumeMode` field of a `PersistentVolumeClaim` 14 | with a [Volume Mode](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#volume-mode) 15 | that differs from the original volume's mode to potentially exploit an as-yet-unknown 16 | vulnerability in the host operating system. 17 | This feature allows cluster administrators to prevent unauthorized users from converting 18 | the mode of a volume when a `PersistentVolumeClaim` is being created from an existing 19 | `VolumeSnapshot` instance. 20 | 21 | > See the [Kubernetes Enhancement Proposal](https://github.com/kubernetes/enhancements/tree/master/keps/sig-storage/3141-prevent-volume-mode-conversion) 22 | > for more details on the background, design and discussions. 23 | 24 | ## Usage 25 | 26 | This feature is enabled by default and moved to GA with the Kubernetes 1.30 release. 27 | To use this feature, cluster administrators must: 28 | 29 | * Create `VolumeSnapshot` APIs with a minimum version of [`v8.0.1`](https://github.com/kubernetes-csi/external-snapshotter/releases/tag/v8.0.1). 30 | * Use `snapshot-controller` and `snapshot-validation-webhook` with a minimum version of `v8.0.1`. 31 | * Use `external-provisioner` with a minimum version of [`v5.0.1`](https://github.com/kubernetes-csi/external-provisioner/releases/tag/v5.0.1). 32 | 33 | > For more information about how to use the feature, visit the [Kubernetes blog](https://kubernetes.io/blog/2024/04/30/prevent-unauthorized-volume-mode-conversion-ga/) page. -------------------------------------------------------------------------------- /book/src/project-policies.md: -------------------------------------------------------------------------------- 1 | # Versioning, Support, and Kubernetes Compatibility 2 | 3 | ## Versioning 4 | 5 | Each Kubernetes CSI component version is expressed as *x.y.z*, where *x* is the 6 | major version, *y* is the minor version, and *z* is the patch version, 7 | following [Semantic Versioning](https://semver.org/). 8 | 9 | Patch version releases only contain bug fixes that do not break any backwards 10 | compatibility. 11 | 12 | Minor version releases may contain new functionality that do not 13 | break any backwards compatibility (except for alpha features). 14 | 15 | Major version releases may contain new functionality or fixes that may break 16 | backwards compatibility with previous major releases. Changes that require a 17 | major version increase include: removing or changing API, flags, or behavior, new 18 | RBAC requirements that are not opt-in, new Kubernetes minimum version 19 | requirements. 20 | 21 | A litmus test for not breaking compatibility is to replace the image of a 22 | component in an existing deployment without changing that deployment in any 23 | other way. 24 | 25 | To minimize the number of branches we need to support, we do not have a general 26 | policy for releasing new minor versions on older majors. We will make exceptions 27 | for work related to meeting production readiness requirements. Only the previous 28 | major version will be eligible for these exceptions, so long as the time between 29 | the previous major version and the current major version is under six months. 30 | For example, if "X.0.0" and "X+1.0.0" were released under six months apart, 31 | "X.0.0" would be eligible for new minor releases. 32 | 33 | ## Support 34 | 35 | The Kubernetes CSI project follows the broader Kubernetes project on support. 36 | Every minor release branch will be supported with patch releases on an as-needed 37 | basis for at least 1 year, 38 | starting with the first release of that minor version. In addition, the minor 39 | release branch will be supported for at least 3 months after the next minor 40 | version is released, to allow time to integrate with the latest release. 41 | 42 | ### Alpha Features 43 | 44 | Alpha features are subject to break or be 45 | removed across Kubernetes and CSI component releases. There is no guarantee alpha 46 | features will continue to function if upgrading the Kubernetes cluster or 47 | upgrading a CSI sidecar or controller. 48 | 49 | ## Kubernetes Compatibility 50 | 51 | Each release of a CSI component has a minimum, maximum and recommended Kubernetes version 52 | that it is compatible with. 53 | 54 | ### Minimum Version 55 | 56 | Minimum version specifies the lowest Kubernetes version where the component will 57 | function with the most basic functionality, and no additional features added later. 58 | Generally, this aligns with the Kubernetes version where that CSI spec version was added. 59 | 60 | ### Maximum Version 61 | 62 | The maximum Kubernetes version specifies the last working Kubernetes version for 63 | all beta and GA features that the component supports. This generally aligns with one 64 | Kubernetes release before support for the CSI spec version was removed or if a particular 65 | Kubernetes API or feature was removed. 66 | 67 | ### Recommended Version 68 | 69 | Note that any new features added may have 70 | dependencies on Kubernetes versions greater than the minimum Kubernetes version. 71 | The recommended Kubernetes version specifies the lowest Kubernetes version 72 | needed where all its supported features will function correctly. Trying to use a 73 | new sidecar feature on a Kubernetes cluster below the recommended Kubernetes 74 | version may fail to function correctly. For that reason, it is encouraged to 75 | stay as close to the recommended Kubernetes version as possible. 76 | 77 | For more details on which features are supported with which Kubernetes versions 78 | and their corresponding CSI components, please see each feature's individual page. 79 | -------------------------------------------------------------------------------- /book/src/raw-block.md: -------------------------------------------------------------------------------- 1 | # Raw Block Volume Feature 2 | 3 | ## Status 4 | 5 | Status | Min K8s Version | Max K8s Version | external-provisioner Version | external-attacher Version 6 | --|--|--|--|-- 7 | Alpha | 1.11 | 1.13 | 0.4 | 0.4 8 | Alpha | 1.13 | 1.13 | 1.0 | 1.0 9 | Beta | 1.14 | 1.17 | 1.1+ | 1.1+ 10 | GA | 1.18 | - | 1.1+ | 1.1+ 11 | 12 | ## Overview 13 | 14 | This page documents how to implement raw block volume support to a CSI Driver. 15 | 16 | A *block volume* is a volume that will appear as a block device inside the container. 17 | A *mounted (file) volume* is volume that will be mounted using a specified file system and appear as a directory inside the container. 18 | 19 | The [CSI spec](https://github.com/container-storage-interface/spec/blob/master/spec.md) supports both block and mounted (file) volumes. 20 | 21 | ## Implementing Raw Block Volume Support in Your CSI Driver 22 | 23 | CSI doesn't provide a capability query for block volumes, so COs will simply pass through requests for 24 | block volume creation to CSI plugins, and plugins are allowed to fail with the `InvalidArgument` GRPC 25 | error code if they don't support block volumes. Kubernetes doesn't make any assumptions about which CSI 26 | plugins support blocks and which don't, so users have to know if any given storage class is capable of 27 | creating block volumes. 28 | 29 | The difference between a request for a mounted (file) volume and a block volume is the `VolumeCapabilities` 30 | field of the request. Note that this field is an array and the created volume must support ALL of the 31 | capabilities requested, or else return an error. If the `AccessType` method of a `VolumeCapability` is 32 | `VolumeCapability_Block`, then the capability is requesting a raw block volume. Unlike mount volumes, block 33 | volumes don't have any specific capabilities that need to be validated, although access modes still 34 | apply. 35 | 36 | Block volumes are much more likely to support multi-node flavors of `VolumeCapability_AccessMode_Mode` 37 | than mount volumes, because there's no file system state stored on the node side that creates any technical 38 | impediments to multi-attaching block volumes. While there may still be good reasons to prevent 39 | multi-attaching block volumes, and there may be implementations that are not capable of supporting 40 | multi-attach, you should think carefully about what makes sense for your driver. 41 | 42 | CSI plugins that support both mount and block volumes must be sure to check the capabilities of all CSI RPC 43 | requests and ensure that the capability of the request matches the capability of the volume, to avoid trying 44 | to do file-system-related things to block volumes and block-related things to file system volumes. The 45 | following RPCs specify capabilities that must be validated: 46 | * `CreateVolume()` (multiple capabilities) 47 | * `ControllerPublishVolume()` 48 | * `ValidateVolumeCapabilities()` (multiple capabilities) 49 | * `GetCapacity()` (see below) 50 | * `NodeStageVolume()` 51 | * `NodePublishVolume()` 52 | 53 | Also, CSI plugins that implement the optional `GetCapacity()` RPC should note that that RPC includes 54 | capabilities too, and if the capacity for mount volumes is not the same as the capacity for block 55 | volumes, that needs to be handled in the implementation of that RPC. 56 | 57 | Q: Can CSI plugins support only block volumes and not mount volumes? 58 | A: Yes! This is just the reverse case of supporting mount volumes only. Plugins may return `InvalidArgument` 59 | for any creation request with an `AccessType` of `VolumeCapability_Mount`. 60 | 61 | ## Differences Between Block and Mount Volumes 62 | 63 | The main difference between block volumes and mount volumes is the expected result of the `NodePublish()`. 64 | For mount volumes, the CO expects the result to be a mounted directory, at `TargetPath`. For block volumes, 65 | the CO expects there to be a device file at `TargetPath`. The device file can be a bind-mounted device from 66 | the hosts `/dev` file system, or it can be a device node created at that location using `mknod()`. 67 | 68 | It's desirable but not required to expose an unfiltered device node. For example, CSI plugins based on 69 | technologies that implement SCSI protocols should expect that pods consuming the block volumes they create 70 | may want to send SCSI commands to the device. This is something that should "just work" by default (subject 71 | to container capabilities) so CSI plugins should avoid anything that would break this kind of use case. The 72 | only hard requirement is that the device implements block reading/writing however. 73 | 74 | For plugins with the `RPC_STAGE_UNSTAGE_VOLUME` capability, the CO doesn't care exactly what is placed at 75 | the `StagingTargetPath`, but it's worth noting that some CSI RPCs are allowed to pass the plugin either 76 | a staging path or a publish path, so it's important to think carefully about how `NodeStageVolume()` is 77 | implemented, knowing that either path could get used by the CO to refer to the volume later on. This is 78 | made more challenging because the CSI spec says that `StagingTargetPath` is always a directory even for 79 | block volumes. 80 | 81 | ## Sidecar Deployment 82 | 83 | The raw block feature requires the 84 | [external-provisioner](external-provisioner.md) and 85 | [external-attacher](external-attacher.md) sidecars to be deployed. 86 | 87 | ## Kubernetes Cluster Setup 88 | 89 | The `BlockVolume` and `CSIBlockVolume` feature gates need to be enabled on 90 | all Kubernetes masters and nodes. 91 | 92 | ``` 93 | --feature-gates=BlockVolume=true,CSIBlockVolume=true... 94 | ``` 95 | 96 | * TODO: detail how Kubernetes API raw block fields get mapped to CSI methods/fields. 97 | -------------------------------------------------------------------------------- /book/src/secrets-and-credentials-volume-group-snapshot-class.md: -------------------------------------------------------------------------------- 1 | # VolumeGroupSnapshotClass Secrets 2 | 3 | The CSI [external-snapshotter](external-snapshotter.md) sidecar container facilitates the handling of secrets for the following operations: 4 | * `CreateGroupSnapshotRequest` 5 | * `DeleteGroupSnapshotRequest` 6 | * `GetGroupSnapshotRequest` 7 | 8 | CSI `external-snapshotter` v8.1.0+ supports the following keys in `VolumeGroupSnapshotClass.parameters`: 9 | 10 | * `csi.storage.k8s.io/group-snapshotter-secret-name` 11 | * `csi.storage.k8s.io/group-snapshotter-secret-namespace` 12 | 13 | With CSI `external-snapshotter` v8.2.0 the following additional `VolumeGroupSnapshotClass.parameters` were added specifically for `GetGroupSnapshotRequest` operations: 14 | 15 | * `csi.storage.k8s.io/group-snapshotter-get-secret-name` 16 | * `csi.storage.k8s.io/group-snapshotter-get-secret-namespace` 17 | 18 | Cluster admins can populate the secret fields for the operations listed above with data from Kubernetes `Secret` objects by specifying these keys in the `VolumeGroupSnapshotClass` object. 19 | 20 | ## Operations 21 | Details for each secret supported by the external-snapshotter can be found below. 22 | 23 | ### Create/Delete VolumeGroupSnapshot Secret 24 | 25 | CSI `external-snapshotter` v8.1.0+ looks for the following keys in `VolumeGroupSnapshotClass.parameters` for `CreateGroupSnapshotRequest` and `DeleteGroupSnapshotRequest` operations: 26 | 27 | * `csi.storage.k8s.io/group-snapshotter-secret-name` 28 | * `csi.storage.k8s.io/group-snapshotter-secret-namespace` 29 | 30 | The values of both of these parameters, together, refer to the name and namespace of a `Secret` object in the Kubernetes API. 31 | 32 | If specified, the CSI `external-snapshotter` will attempt to fetch the secret before creation and deletion. 33 | 34 | If the secret is retrieved successfully, the snapshotter passes it to the CSI driver in the `CreateGroupSnapshotRequest.secrets` or `DeleteGroupSnapshotRequest.secrets` field. 35 | 36 | If no such secret exists in the Kubernetes API, or the snapshotter is unable to fetch it, the create operation will fail. 37 | 38 | Note, however, that the delete operation will continue even if the secret is not found (because, for example, the entire namespace containing the secret was deleted). In this case, if the driver requires a secret for deletion, then the volume group snapshot and related snapshots need to be manually cleaned up. 39 | 40 | The values of these parameters may be "templates". The `external-snapshotter` will automatically resolve templates at volume group snapshot create time, as detailed below: 41 | 42 | * `csi.storage.k8s.io/group-snapshotter-secret-name` 43 | * `${volumegroupsnapshotcontent.name}` 44 | * Replaced with name of the `VolumeGroupSnapshotContent` object being created. 45 | * `${volumegroupsnapshot.namespace}` 46 | * Replaced with namespace of the `VolumeGroupSnapshot` object that triggered creation. 47 | * `${volumegroupsnapshot.name}` 48 | * Replaced with the name of the `VolumeGroupSnapshot` object that triggered creation. 49 | * `csi.storage.k8s.io/group-snapshotter-secret-namespace` 50 | * `${volumegroupsnapshotcontent.name}` 51 | * Replaced with name of the `VolumeGroupSnapshotContent` object being created. 52 | * `${volumegroupsnapshot.namespace}` 53 | * Replaced with namespace of the `VolumeGroupSnapshot` object that triggered creation. 54 | 55 | ### Get VolumeGroupSnapshot Secret 56 | 57 | CSI `external-snapshotter` v8.2.0+ looks for the following keys in `VolumeGroupSnapshotClass.parameters` for `GetGroupSnapshotRequest` operations: 58 | 59 | * `csi.storage.k8s.io/group-snapshotter-get-secret-name` 60 | * `csi.storage.k8s.io/group-snapshotter-get-secret-namespace` 61 | 62 | The values of both of these parameters, together, refer to the name and namespace of a `Secret` object in the Kubernetes API. 63 | 64 | If specified, the CSI `external-snapshotter` will attempt to fetch the secret before creation and deletion. 65 | 66 | If the secret is retrieved successfully, the snapshotter passes it to the CSI driver in the `GetGroupSnapshotRequest.secrets` field. 67 | 68 | If no such secret exists in the Kubernetes API, or the snapshotter is unable to fetch it, the create operation will fail. 69 | 70 | The values of these parameters may be "templates". The `external-snapshotter` will automatically resolve templates at volume group snapshot create time, as detailed below: 71 | 72 | * `csi.storage.k8s.io/group-snapshotter-secret-name` 73 | * `${volumegroupsnapshotcontent.name}` 74 | * Replaced with name of the `VolumeGroupSnapshotContent` object being created. 75 | * `${volumegroupsnapshot.namespace}` 76 | * Replaced with namespace of the `VolumeGroupSnapshot` object that triggered creation. 77 | * `${volumegroupsnapshot.name}` 78 | * Replaced with the name of the `VolumeGroupSnapshot` object that triggered creation. 79 | * `csi.storage.k8s.io/group-snapshotter-secret-namespace` 80 | * `${volumegroupsnapshotcontent.name}` 81 | * Replaced with name of the `VolumeGroupSnapshotContent` object being created. 82 | * `${volumegroupsnapshot.namespace}` 83 | * Replaced with namespace of the `VolumeGroupSnapshot` object that triggered creation. 84 | -------------------------------------------------------------------------------- /book/src/secrets-and-credentials-volume-snapshot-class.md: -------------------------------------------------------------------------------- 1 | # VolumeSnapshotClass Secrets 2 | 3 | The CSI [external-snapshotter](external-snapshotter.md) sidecar container facilitates the handling of secrets for the following operations: 4 | * `CreateSnapshotRequest` 5 | * `DeleteSnapshotRequest` 6 | 7 | CSI `external-snapshotter` v1.0.1+ supports the following keys in `VolumeSnapshotClass.parameters`: 8 | 9 | * `csi.storage.k8s.io/snapshotter-secret-name` 10 | * `csi.storage.k8s.io/snapshotter-secret-namespace` 11 | 12 | Cluster admins can populate the secret fields for the operations listed above with data from Kubernetes `Secret` objects by specifying these keys in the `VolumeSnapshotClass` object. 13 | 14 | ## Operations 15 | Details for each secret supported by the external-snapshotter can be found below. 16 | 17 | ### Create/Delete VolumeSnapshot Secret 18 | 19 | CSI `external-snapshotter` v1.0.1+ looks for the following keys in `VolumeSnapshotClass.parameters`: 20 | 21 | * `csi.storage.k8s.io/snapshotter-secret-name` 22 | * `csi.storage.k8s.io/snapshotter-secret-namespace` 23 | 24 | The values of both of these parameters, together, refer to the name and namespace of a `Secret` object in the Kubernetes API. 25 | 26 | If specified, the CSI `external-snapshotter` will attempt to fetch the secret before creation and deletion. 27 | 28 | If the secret is retrieved successfully, the snapshotter passes it to the CSI driver in the `CreateSnapshotRequest.secrets` or `DeleteSnapshotRequest.secrets` field. 29 | 30 | If no such secret exists in the Kubernetes API, or the snapshotter is unable to fetch it, the create operation will fail. 31 | 32 | Note, however, that the delete operation will continue even if the secret is not found (because, for example, the entire namespace containing the secret was deleted). In this case, if the driver requires a secret for deletion, then the volume and PV may need to be manually cleaned up. 33 | 34 | The values of these parameters may be "templates". The `external-snapshotter` will automatically resolve templates at snapshot create time, as detailed below: 35 | 36 | * `csi.storage.k8s.io/snapshotter-secret-name` 37 | * `${volumesnapshotcontent.name}` 38 | * Replaced with name of the `VolumeSnapshotContent` object being created. 39 | * `${volumesnapshot.namespace}` 40 | * Replaced with namespace of the `VolumeSnapshot` object that triggered creation. 41 | * `${volumesnapshot.name}` 42 | * Replaced with the name of the `VolumeSnapshot` object that triggered creation. 43 | * `csi.storage.k8s.io/snapshotter-secret-namespace` 44 | * `${volumesnapshotcontent.name}` 45 | * Replaced with name of the `VolumeSnapshotContent` object being created. 46 | * `${volumesnapshot.namespace}` 47 | * Replaced with namespace of the `VolumeSnapshot` object that triggered creation. 48 | -------------------------------------------------------------------------------- /book/src/secrets-and-credentials.md: -------------------------------------------------------------------------------- 1 | # Secrets and Credentials 2 | 3 | Some drivers may require a secret in order to complete operations. 4 | 5 | ## CSI Driver Secrets 6 | 7 | If a CSI Driver requires secrets for a backend (a service account, for example), and this secret is required at the "per driver" granularity (not different "per CSI operation" or "per volume"), then the secret SHOULD be injected directly in to CSI driver pods via [standard Kubernetes secret distribution mechanisms](https://kubernetes.io/docs/tasks/inject-data-application/distribute-credentials-secure/) during deployment. 8 | 9 | ## CSI Operation Secrets 10 | 11 | If a CSI Driver requires secrets "per CSI operation" or "per volume" or "per storage pool", the CSI spec allows secrets to be passed in for various CSI operations (including `CreateVolumeRequest`, `ControllerPublishVolumeRequest`, and more). 12 | 13 | Cluster admins can populate such secrets by creating Kubernetes `Secret` objects and specifying the keys in the `StorageClass` or `SnapshotClass` objects. 14 | 15 | The CSI sidecar containers facilitate the handling of secrets between Kubernetes and the CSI Driver. For more details see: 16 | * [StorageClass Secrets](secrets-and-credentials-storage-class.md) 17 | * [VolumeSnapshotClass Secrets](secrets-and-credentials-volume-snapshot-class.md) 18 | 19 | ### Secret RBAC Rules 20 | 21 | For reducing RBAC permissions as much as possible, secret rules are disabled in each sidecar repository by default. 22 | 23 | Please add or update RBAC rules if secret is expected to use. 24 | 25 | To set proper secret permission, uncomment related lines defined in `rbac.yaml` (e.g. [external-provisioner/deploy/kubernetes/rbac.yaml](https://github.com/kubernetes-csi/external-provisioner/blob/22bb6401d2484ee3ca18a23d75c3864c774e5f32/deploy/kubernetes/rbac.yaml#L24)) 26 | 27 | ## Handling Sensitive Information 28 | 29 | CSI Drivers that accept secrets SHOULD handle this data carefully. It may contain sensitive information and MUST be treated as such (e.g. not logged). 30 | 31 | To make it easier to handle secret fields (e.g. strip them from CSI protos when logging), the CSI spec defines a decorator (`csi_secret`) on all fields containing sensitive information. Any fields decorated with `csi_secret` MUST be treated as if they contain sensitive information (e.g. not logged, etc.). 32 | 33 | The Kubernetes CSI development team also provides a GO lang package called `protosanitizer` that CSI driver developers may be used to remove values for all fields in a gRPC messages decorated with `csi_secret`. The library can be found in [kubernetes-csi/csi-lib-utils/protosanitizer](https://github.com/kubernetes-csi/csi-lib-utils/tree/master/protosanitizer). The Kubernetes CSI [Sidecar Containers](sidecar-containers.md) and sample drivers use this library to ensure no sensitive information is logged. 34 | -------------------------------------------------------------------------------- /book/src/sidecar-containers.md: -------------------------------------------------------------------------------- 1 | # Kubernetes CSI Sidecar Containers 2 | 3 | Kubernetes CSI Sidecar Containers are a set of standard containers that aim to 4 | simplify the development and deployment of CSI Drivers on Kubernetes. 5 | 6 | These containers contain common logic to watch the Kubernetes API, trigger 7 | appropriate operations against the “CSI volume driver” container, and update the 8 | Kubernetes API as appropriate. 9 | 10 | The containers are intended to be bundled with third-party CSI driver containers and deployed together as pods. 11 | 12 | The containers are developed and maintained by the Kubernetes Storage community. 13 | 14 | Use of the containers is strictly optional, but highly recommended. 15 | 16 | Benefits of these sidecar containers include: 17 | 18 | * Reduction of "boilerplate" code. 19 | * CSI Driver developers do not have to worry about complicated, "Kubernetes specific" code. 20 | * Separation of concerns. 21 | * Code that interacts with the Kubernetes API is isolated from (and in a different container than) the code that implements the CSI interface. 22 | 23 | The Kubernetes development team maintains the following Kubernetes CSI Sidecar Containers: 24 | 25 | * [external-provisioner](external-provisioner.md) 26 | * [external-attacher](external-attacher.md) 27 | * [external-snapshotter](external-snapshotter.md) 28 | * [external-resizer](external-resizer.md) 29 | * [node-driver-registrar](node-driver-registrar.md) 30 | * [cluster-driver-registrar](cluster-driver-registrar.md) (deprecated) 31 | * [livenessprobe](livenessprobe.md) 32 | -------------------------------------------------------------------------------- /book/src/skip-attach.md: -------------------------------------------------------------------------------- 1 | # Skip Kubernetes Attach and Detach 2 | 3 | ## Status 4 | 5 | Status | Min K8s Version | Max K8s Version | cluster-driver-registrar Version 6 | --|--|--|-- 7 | Alpha | 1.12 | 1.12 | 0.4 8 | Alpha | 1.13 | 1.13 | 1.0 9 | Beta | 1.14 | 1.17 | n/a 10 | GA | 1.18 | - | n/a 11 | 12 | # Overview 13 | 14 | Volume drivers, like NFS, for example, have no concept of an attach (`ControllerPublishVolume`). However, Kubernetes always executes `Attach` and `Detach` operations even if the CSI driver does not implement an attach operation (i.e. even if the CSI Driver does not implement a `ControllerPublishVolume` call). 15 | 16 | This was problematic because it meant *all* CSI drivers had to handle Kubernetes attachment. CSI Drivers that did not implement the `PUBLISH_UNPUBLISH_VOLUME` controller capability could work around this by deploying an [external-attacher](external-attacher.md) and the `external-attacher` would responds to Kubernetes attach operations and simply do a noop (because the CSI driver did not advertise the `PUBLISH_UNPUBLISH_VOLUME` controller capability). 17 | 18 | Although the workaround works, it adds an unnecessary operation (round-trip) in the preparation of a volume for a container, and requires CSI Drivers to deploy an unnecessary sidecar container (`external-attacher`). 19 | 20 | # Skip Attach with CSI Driver Object 21 | 22 | The [CSIDriver Object](csi-driver-object.md) enables CSI Drivers to specify how Kubernetes should interact with it. 23 | 24 | Specifically the `attachRequired` field instructs Kubernetes to skip any attach operation altogether. 25 | 26 | For example, the existence of the following object would cause Kubernetes to skip attach operations for all CSI Driver `testcsidriver.example.com` volumes. 27 | 28 | ``` 29 | apiVersion: storage.k8s.io/v1 30 | kind: CSIDriver 31 | metadata: 32 | name: testcsidriver.example.com 33 | spec: 34 | attachRequired: false 35 | ``` 36 | 37 | CSIDriver object should be manually included in the driver deployment manifests. 38 | 39 | Previously, the [cluster-driver-registrar](cluster-driver-registrar.md) sidecar container could be deployed to automatically create the object. Once the flags to this container are configured correctly, it will automatically create a [CSIDriver Object](csi-driver-object.md) when it starts with the correct fields set. 40 | 41 | ## Alpha Functionality 42 | In alpha, this feature was enabled via the [CSIDriver Object](csi-driver-object.md) CRD. 43 | 44 | ``` 45 | apiVersion: csi.storage.k8s.io/v1alpha1 46 | kind: CSIDriver 47 | metadata: 48 | .... 49 | ``` 50 | -------------------------------------------------------------------------------- /book/src/snapshot-controller.md: -------------------------------------------------------------------------------- 1 | # Snapshot Controller 2 | 3 | ## Status and Releases 4 | 5 | **Git Repository:** [https://github.com/kubernetes-csi/external-snapshotter](https://github.com/kubernetes-csi/external-snapshotter) 6 | 7 | **Status:** GA v4.0.0+ 8 | 9 | When Volume Snapshot is promoted to Beta in Kubernetes 1.17, the CSI external-snapshotter sidecar controller is split into two controllers: a snapshot-controller and a CSI external-snapshotter sidecar. See the following table for snapshot-controller release information. 10 | 11 | ### Supported Versions 12 | 13 | Latest stable release | Branch | Min CSI Version | Max CSI Version | Container Image | [Min K8s Version](project-policies.md#minimum-version) | [Max K8s Version](project-policies.md#maximum-version) | [Recommended K8s Version](project-policies.md#recommended-version) 14 | --|--|--|--|--|--|--|-- 15 | [external-snapshotter v8.2.0](https://github.com/kubernetes-csi/external-snapshotter/releases/tag/v8.2.0) | [release-8.2](https://github.com/kubernetes-csi/external-snapshotter/tree/release-8.2) | [v1.11.0](https://github.com/container-storage-interface/spec/releases/tag/v1.11.0) | - | registry.k8s.io/sig-storage/snapshot-controller:v8.2.0 | v1.25 | - | v1.25 16 | [external-snapshotter v8.1.0](https://github.com/kubernetes-csi/external-snapshotter/releases/tag/v8.1.0) | [release-8.1](https://github.com/kubernetes-csi/external-snapshotter/tree/release-8.1) | [v1.11.0](https://github.com/container-storage-interface/spec/releases/tag/v1.11.0) | - | registry.k8s.io/sig-storage/snapshot-controller:v8.1.0 | v1.25 | - | v1.25 17 | [external-snapshotter v8.0.2](https://github.com/kubernetes-csi/external-snapshotter/releases/tag/v8.0.2) | [release-8.0](https://github.com/kubernetes-csi/external-snapshotter/tree/release-8.0) | [v1.9.0](https://github.com/container-storage-interface/spec/releases/tag/v1.9.0) | - | registry.k8s.io/sig-storage/snapshot-controller:v8.0.2 | v1.25 | - | v1.25 18 | 19 | ### Unsupported Versions 20 |
    21 | 22 | List of previous versions 23 | 24 | Latest stable release | Branch | Min CSI Version | Max CSI Version | Container Image | [Min K8s Version](project-policies.md#minimum-version) | [Max K8s Version](project-policies.md#maximum-version) | [Recommended K8s Version](project-policies.md#recommended-version) 25 | --|--|--|--|--|--|--|-- 26 | [external-snapshotter v6.3.0](https://github.com/kubernetes-csi/external-snapshotter/releases/tag/v6.3.0) | [release-6.2](https://github.com/kubernetes-csi/external-snapshotter/tree/release-6.3) | [v1.0.0](https://github.com/container-storage-interface/spec/releases/tag/v1.0.0) | - | registry.k8s.io/sig-storage/snapshot-controller:v6.2.1 | v1.20 | - | v1.24 27 | [external-snapshotter v6.2.2](https://github.com/kubernetes-csi/external-snapshotter/releases/tag/v6.2.2) | [release-6.2](https://github.com/kubernetes-csi/external-snapshotter/tree/release-6.2) | [v1.0.0](https://github.com/container-storage-interface/spec/releases/tag/v1.0.0) | - | registry.k8s.io/sig-storage/snapshot-controller:v6.2.1 | v1.20 | - | v1.24 28 | [external-snapshotter v6.1.0](https://github.com/kubernetes-csi/external-snapshotter/releases/tag/v6.1.0) | [release-6.1](https://github.com/kubernetes-csi/external-snapshotter/tree/release-6.1) | [v1.0.0](https://github.com/container-storage-interface/spec/releases/tag/v1.0.0) | - | registry.k8s.io/sig-storage/snapshot-controller:v6.1.0 | v1.20 | - | v1.24 29 | [external-snapshotter v6.0.1](https://github.com/kubernetes-csi/external-snapshotter/releases/tag/v6.0.1) | [release-6.0](https://github.com/kubernetes-csi/external-snapshotter/tree/release-6.0) | [v1.0.0](https://github.com/container-storage-interface/spec/releases/tag/v1.0.0) | - | registry.k8s.io/sig-storage/snapshot-controller:v6.0.1 | v1.20 | - | v1.24 30 | [external-snapshotter v5.0.1](https://github.com/kubernetes-csi/external-snapshotter/releases/tag/v5.0.1) | [release-5.0](https://github.com/kubernetes-csi/external-snapshotter/tree/release-5.0) | [v1.0.0](https://github.com/container-storage-interface/spec/releases/tag/v1.0.0) | - | registry.k8s.io/sig-storage/snapshot-controller:v5.0.1 | v1.20 | - | v1.22 31 | [external-snapshotter v4.2.1](https://github.com/kubernetes-csi/external-snapshotter/releases/tag/v4.2.1) | [release-4.2](https://github.com/kubernetes-csi/external-snapshotter/tree/release-4.2) | [v1.0.0](https://github.com/container-storage-interface/spec/releases/tag/v1.0.0) | - | registry.k8s.io/sig-storage/snapshot-controller:v4.2.1 | v1.20 | - | v1.22 32 | [external-snapshotter v4.1.1](https://github.com/kubernetes-csi/external-snapshotter/releases/tag/v4.1.1) | [release-4.1](https://github.com/kubernetes-csi/external-snapshotter/tree/release-4.1) | [v1.0.0](https://github.com/container-storage-interface/spec/releases/tag/v1.0.0) | - | registry.k8s.io/sig-storage/snapshot-controller:v4.1.1 | v1.20 | - | v1.20 33 | [external-snapshotter v4.0.1](https://github.com/kubernetes-csi/external-snapshotter/releases/tag/v4.0.1) | [release-4.0](https://github.com/kubernetes-csi/external-snapshotter/tree/release-4.0) | [v1.0.0](https://github.com/container-storage-interface/spec/releases/tag/v1.0.0) | - | registry.k8s.io/sig-storage/snapshot-controller:v4.0.1 | v1.20 | - | v1.20 34 | [external-snapshotter v3.0.3 (beta)](https://github.com/kubernetes-csi/external-snapshotter/releases/tag/v3.0.3) | [release-3.0](https://github.com/kubernetes-csi/external-snapshotter/tree/release-3.0) | [v1.0.0](https://github.com/container-storage-interface/spec/releases/tag/v1.0.0) | - | registry.k8s.io/sig-storage/snapshot-controller:v3.0.3 | v1.17 | - | v1.17 35 | [external-snapshotter v2.1.4 (beta)](https://github.com/kubernetes-csi/external-snapshotter/releases/tag/v2.1.4) | [release-2.1](https://github.com/kubernetes-csi/external-snapshotter/tree/release-2.1) | [v1.0.0](https://github.com/container-storage-interface/spec/releases/tag/v1.0.0) | - | registry.k8s.io/sig-storage/snapshot-controller:v2.1.4 | v1.17 | - | v1.17 36 | 37 |
    38 | 39 | For more information on the CSI external-snapshotter sidecar, see [this external-snapshotter page](external-snapshotter.md). 40 | 41 | ## Description 42 | 43 | The snapshot controller will be watching the Kubernetes API server for `VolumeSnapshot` and `VolumeSnapshotContent` CRD objects. The CSI `external-snapshotter` sidecar only watches the Kubernetes API server for `VolumeSnapshotContent` CRD objects. The snapshot controller will be creating the `VolumeSnapshotContent` CRD object which triggers the CSI `external-snapshotter` sidecar to create a snapshot on the storage system. 44 | 45 | The snapshot controller will be watching for `VolumeGroupSnapshot` and `VolumeGroupSnapshotContent` CRD objects when Volume Group Snapshot support is enabled via the `--enable-volume-group-snapshots` option. 46 | 47 | For detailed snapshot beta design changes, see the design doc [here](https://github.com/kubernetes/enhancements/blob/master/keps/sig-storage/177-volume-snapshot/README.md). 48 | 49 | For detailed information about volume snapshot and restore functionality, see [Volume Snapshot & Restore](snapshot-restore-feature.md). 50 | 51 | For detailed information about volume group snapshot and restore functionality, see [Volume Snapshot & Restore](group-snapshot.md). 52 | 53 | For detailed information (binary parameters, RBAC rules, etc.), see [https://github.com/kubernetes-csi/external-snapshotter/blob/release-6.2/README.md](https://github.com/kubernetes-csi/external-snapshotter/blob/release-6.2/README.md). 54 | 55 | ## Deployment 56 | 57 | Kubernetes distributors should bundle and deploy the controller and CRDs as part of their Kubernetes cluster management process (independent of any CSI Driver). 58 | 59 | If your cluster does not come pre-installed with the correct components, you may manually install these components by executing the following steps. 60 | 61 | ``` 62 | git clone https://github.com/kubernetes-csi/external-snapshotter/ 63 | cd ./external-snapshotter 64 | git checkout release-6.2 65 | kubectl kustomize client/config/crd | kubectl create -f - 66 | kubectl -n kube-system kustomize deploy/kubernetes/snapshot-controller | kubectl create -f - 67 | ``` -------------------------------------------------------------------------------- /book/src/snapshot-validation-webhook.md: -------------------------------------------------------------------------------- 1 | # Snapshot Validation Webhook 2 | 3 | ## Status and Releases 4 | 5 | **Git Repository:** [https://github.com/kubernetes-csi/external-snapshotter](https://github.com/kubernetes-csi/external-snapshotter) 6 | 7 | **Status:** GA as of 4.0.0 8 | 9 | There is a new validating webhook server which provides tightened validation on snapshot objects. This SHOULD be installed by the Kubernetes distros along with the snapshot-controller, not end users. It SHOULD be installed in all Kubernetes clusters that has the snapshot feature enabled. 10 | 11 | ### Supported Versions 12 | 13 | Latest stable release | Branch | Min CSI Version | Max CSI Version | Container Image | [Min K8s Version](project-policies.md#minimum-version) | [Max K8s Version](project-policies.md#maximum-version) | [Recommended K8s Version](project-policies.md#recommended-version) 14 | --|--|--|--|--|--|--|-- 15 | [external-snapshotter v8.1.0](https://github.com/kubernetes-csi/external-snapshotter/releases/tag/v8.1.0) | [release-8.1](https://github.com/kubernetes-csi/external-snapshotter/tree/release-8.1) | [v1.11.0](https://github.com/container-storage-interface/spec/releases/tag/v1.11.0) | - | registry.k8s.io/sig-storage/snapshot-validation-webhook:v8.1.0 | v1.25 | - | v1.25 16 | [external-snapshotter v8.0.2](https://github.com/kubernetes-csi/external-snapshotter/releases/tag/v8.0.2) | [release-8.0](https://github.com/kubernetes-csi/external-snapshotter/tree/release-8.0) | [v1.9.0](https://github.com/container-storage-interface/spec/releases/tag/v1.9.0) | - | registry.k8s.io/sig-storage/snapshot-validation-webhook:v8.0.2 | v1.25 | - | v1.25 17 | 18 | 19 | 20 | 21 | ### Unsupported Versions 22 |
    23 | 24 | List of previous versions 25 | 26 | Latest stable release | Branch | Min CSI Version | Max CSI Version | Container Image | [Min K8s Version](project-policies.md#minimum-version) | [Max K8s Version](project-policies.md#maximum-version) | [Recommended K8s Version](project-policies.md#recommended-version) 27 | --|--|--|--|--|--|--|-- 28 | [external-snapshotter v6.3.0](https://github.com/kubernetes-csi/external-snapshotter/releases/tag/v6.3.0) | [release-6.2](https://github.com/kubernetes-csi/external-snapshotter/tree/release-6.3) | [v1.0.0](https://github.com/container-storage-interface/spec/releases/tag/v1.0.0) | - | registry.k8s.io/sig-storage/napshot-validation-webhook:v6.2.1 | v1.20 | - | v1.24 29 | [external-snapshotter v6.2.2](https://github.com/kubernetes-csi/external-snapshotter/releases/tag/v6.2.2) | [release-6.2](https://github.com/kubernetes-csi/external-snapshotter/tree/release-6.2) | [v1.0.0](https://github.com/container-storage-interface/spec/releases/tag/v1.0.0) | - | registry.k8s.io/sig-storage/napshot-validation-webhook:v6.2.1 | v1.20 | - | v1.24 30 | [external-snapshotter v6.1.0](https://github.com/kubernetes-csi/external-snapshotter/releases/tag/v6.1.0) | [release-6.1](https://github.com/kubernetes-csi/external-snapshotter/tree/release-6.1) | [v1.0.0](https://github.com/container-storage-interface/spec/releases/tag/v1.0.0) | - | registry.k8s.io/sig-storage/napshot-validation-webhookr:v6.1.0 | v1.20 | - | v1.24 31 | [snapshot-validation-webhook v6.0.1](https://github.com/kubernetes-csi/external-snapshotter/releases/tag/v6.0.1) | [release-6.0](https://github.com/kubernetes-csi/external-snapshotter/tree/release-6.0) | [v1.0.0](https://github.com/container-storage-interface/spec/releases/tag/v1.0.0) | - | registry.k8s.io/sig-storage/snapshot-validation-webhook:v6.0.1 | v1.20 | - | v1.24 32 | [snapshot-validation-webhook v5.0.1](https://github.com/kubernetes-csi/external-snapshotter/releases/tag/v5.0.1) | [release-5.0](https://github.com/kubernetes-csi/external-snapshotter/tree/release-5.0) | [v1.0.0](https://github.com/container-storage-interface/spec/releases/tag/v1.0.0) | - | registry.k8s.io/sig-storage/snapshot-validation-webhook:v5.0.1 | v1.20 | - | v1.22 33 | [snapshot-validation-webhook v4.2.1](https://github.com/kubernetes-csi/external-snapshotter/releases/tag/v4.2.1) | [release-4.2](https://github.com/kubernetes-csi/external-snapshotter/tree/release-4.2) | [v1.0.0](https://github.com/container-storage-interface/spec/releases/tag/v1.0.0) | - | registry.k8s.io/sig-storage/snapshot-validation-webhook:v4.2.1 | v1.20 | - | v1.22 34 | [snapshot-validation-webhook v4.1.1](https://github.com/kubernetes-csi/external-snapshotter/releases/tag/v4.1.1) | [release-4.1](https://github.com/kubernetes-csi/external-snapshotter/tree/release-4.1) | [v1.0.0](https://github.com/container-storage-interface/spec/releases/tag/v1.0.0) | - | registry.k8s.io/sig-storage/snapshot-validation-webhook:v4.1.0 | v1.20 | - | v1.20 35 | [snapshot-validation-webhook v4.0.1](https://github.com/kubernetes-csi/external-snapshotter/releases/tag/v4.0.1) | [release-4.0](https://github.com/kubernetes-csi/external-snapshotter/tree/release-4.0) | [v1.0.0](https://github.com/container-storage-interface/spec/releases/tag/v1.0.0) | - | registry.k8s.io/sig-storage/snapshot-validation-webhook:v4.0.1 | v1.20 | - | v1.20 36 | [snapshot-validation-webhook v3.0.3](https://github.com/kubernetes-csi/external-snapshotter/releases/tag/v3.0.3) | [release-3.0](https://github.com/kubernetes-csi/external-snapshotter/tree/release-3.0) | [v1.0.0](https://github.com/container-storage-interface/spec/releases/tag/v1.0.0) | - | registry.k8s.io/sig-storage/snapshot-validation-webhook:v3.0.3 | v1.17 | - | v1.17 37 | 38 |
    39 | 40 | ## Description 41 | 42 | The snapshot validating webhook is an HTTP callback which responds to [admission requests](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/). It is part of a larger [plan](https://github.com/kubernetes/enhancements/tree/master/keps/sig-storage/1900-volume-snapshot-validation-webhook) to tighten validation for volume snapshot objects. This webhook introduces the [ratcheting validation](https://github.com/kubernetes/enhancements/tree/master/keps/sig-storage/1900-volume-snapshot-validation-webhook#backwards-compatibility) mechanism targeting the tighter validation. The cluster admin or Kubernetes distribution admin should install the webhook alongside the snapshot controllers and CRDs. 43 | 44 | > :warning: **WARNING**: Cluster admins choosing not to install the webhook server and participate in the phased release process can cause future problems when upgrading from `v1beta1` to `v1` volumesnapshot API, if there are currently persisted objects which fail the new stricter validation. Potential impacts include being unable to delete invalid snapshot objects. 45 | 46 | ## Deployment 47 | 48 | Kubernetes distributors should bundle and deploy the snapshot validation webhook along with the snapshot controller and CRDs as part of their Kubernetes cluster management process (independent of any CSI Driver). 49 | 50 | Read more about how to install the example webhook [here](https://github.com/kubernetes-csi/external-snapshotter/tree/master/deploy/kubernetes/webhook-example). 51 | -------------------------------------------------------------------------------- /book/src/storage-capacity-tracking.md: -------------------------------------------------------------------------------- 1 | # Storage Capacity Tracking 2 | 3 | ## Status 4 | 5 | Status | Min K8s Version | Max K8s Version 6 | --|--|-- 7 | Alpha | 1.19 | - 8 | 9 | ## Overview 10 | 11 | Storage capacity tracking allows the Kubernetes scheduler to make more 12 | informed choices about where to start pods which depend on unbound 13 | volumes with late binding (aka "wait for first consumer"). Without 14 | storage capacity tracking, a node is chosen without knowing whether 15 | those volumes can be made available for the node. Volume creation is 16 | attempted and if that fails, the pod has to be rescheduled, 17 | potentially landing on the same node again. With storage capacity 18 | tracking, the scheduler filters out nodes which do not have enough 19 | capacity. 20 | 21 | > For design information, see the [enhancement 22 | > proposal](https://github.com/kubernetes/enhancements/tree/master/keps/sig-storage/1472-storage-capacity-tracking). 23 | 24 | ## Usage 25 | 26 | To support rescheduling of a pod, a CSI driver deployment must: 27 | - return the `ResourceExhausted` gRPC status code in `CreateVolume` if 28 | capacity is exhausted 29 | - use external-provisioner >= 1.6.0 because older releases did not 30 | properly support rescheduling after a `ResourceExhausted` error 31 | 32 | To support storage capacity tracking, a CSI driver deployment must: 33 | - implement the `GetCapacity` call 34 | - use external-provisioner >= 2.0.0 35 | - enable producing of storage capacity objects as explained in the 36 | [external-provisioner 37 | documentation](https://github.com/kubernetes-csi/external-provisioner/tree/release-2.0#capacity-support) 38 | - enable usage of that information by setting the 39 | [`CSIDriverSpec.StorageCapacity`](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.19/#csidriverspec-v1-storage-k8s-io) 40 | field to `True` 41 | - run on a cluster where the storage capacity API is 42 | [enabled](https://kubernetes.io/docs/concepts/storage/storage-capacity/#enabling-storage-capacity-tracking) 43 | 44 | > Further information can be found in the [Kubernetes 45 | > documentation](https://kubernetes.io/docs/concepts/storage/storage-capacity/). 46 | -------------------------------------------------------------------------------- /book/src/support-fsgroup.md: -------------------------------------------------------------------------------- 1 | # CSI Driver fsGroup Support 2 | 3 | There are two features related to supporting `fsGroup` for the CSI driver: CSI volume fsGroup policy and delegating fsGroup to CSI driver. For more information about using `fsGroup` in Kubernetes, please refer to the [Kubernetes documentation on Pod security context](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/). 4 | 5 | ## CSI Volume fsGroup Policy 6 | 7 | ### Status 8 | 9 | Status | Min K8s Version | Max K8s Version 10 | --|--|-- 11 | Alpha | 1.19 | 1.19 12 | Beta | 1.20 | 1.22 13 | GA | 1.23 | - 14 | 15 | ### Overview 16 | 17 | CSI Drivers can indicate whether or not they support modifying a volume's ownership or permissions when the volume is being mounted. This can be useful if the CSI Driver does not support the operation, or wishes to re-use volumes with constantly changing permissions. 18 | 19 | > See the [design document](https://github.com/kubernetes/enhancements/tree/master/keps/sig-storage/1682-csi-driver-skip-permission) for further information. 20 | 21 | ### Example Usage 22 | When creating the CSI Driver object, `fsGroupPolicy` is defined in the driver's spec. The following shows the hostpath driver with `None` included, indicating that the volumes should not be modified when mounted: 23 | 24 | ```yaml 25 | apiVersion: storage.k8s.io/v1 26 | kind: CSIDriver 27 | metadata: 28 | name: hostpath.csi.k8s.io 29 | spec: 30 | # Supports persistent and ephemeral inline volumes. 31 | volumeLifecycleModes: 32 | - Persistent 33 | - Ephemeral 34 | # To determine at runtime which mode a volume uses, pod info and its 35 | # "csi.storage.k8s.io/ephemeral" entry are needed. 36 | podInfoOnMount: true 37 | fsGroupPolicy: None 38 | ``` 39 | 40 | #### Supported Modes 41 | 42 | * The following modes are supported: 43 | * `None`: Indicates that volumes will be mounted with no modifications, as the CSI volume driver does not support these operations. 44 | * `File`: Indicates that the CSI volume driver supports volume ownership and permission change via fsGroup, and Kubernetes may use fsGroup to change permissions and ownership of the volume to match user requested fsGroup in the pod's SecurityPolicy regardless of fstype or access mode. 45 | * `ReadWriteOnceWithFSType`: Indicates that volumes will be examined to determine if volume ownership and permissions should be modified to match the pod's security policy. 46 | Changes will only occur if the `fsType` is defined and the persistent volume's `accessModes` contains `ReadWriteOnce`. . 47 | 48 | If undefined, `fsGroupPolicy` will default to `ReadWriteOnceWithFSType`, keeping the previous behavior. 49 | 50 | #### Feature Gates 51 | To use this field, Kubernetes 1.19 binaries must start with the `CSIVolumeFSGroupPolicy` feature gate enabled: 52 | ``` 53 | --feature-gates=CSIVolumeFSGroupPolicy=true 54 | ``` 55 | This is enabled by default on 1.20 and higher. 56 | 57 | 58 | ## Delegate fsGroup to CSI Driver 59 | 60 | ### Status 61 | 62 | Status | Min K8s Version | Max K8s Version 63 | --|--|-- 64 | Alpha | 1.22 | 1.22 65 | Beta | 1.23 | - 66 | GA | 1.26 | - 67 | 68 | ### Overview 69 | 70 | For most drivers, kubelet applies the `fsGroup` specified in a Pod spec by recursively changing volume ownership during the mount process. This does not work for certain drivers. For example: 71 | * A driver requires passing `fsGroup` to mount options in order for it to take effect. 72 | * A driver needs to apply `fsGroup` at the stage step (`NodeStageVolume` in CSI; `MountDevice` in Kubernetes) instead of the mount step (`NodePublishVolume` in CSI; `SetUp/SetUpAt` in Kubernetes). 73 | 74 | This feature provides a mechanism for the driver to apply `fsGroup` instead of kubelet. Specifically, it passes `fsGroup` to the CSI driver through `NodeStageVolume` and `NodePublishVolume` calls, and the kubelet `fsGroup` logic is disabled. The driver is expected to apply the `fsGroup` within one of these calls. 75 | 76 | If this feature is enabled in Kubernetes and a volume uses a driver that supports this feature, `CSIDriver.spec.fsGroupPolicy` and `Pod.spec.securityContext.fsGroupChangePolicy` are ignored. 77 | 78 | > See the [design document](https://github.com/kubernetes/enhancements/blob/master/keps/sig-storage/2317-fsgroup-on-mount/README.md) and the description of the `VolumeCapability.MountVolume.volume_mount_group` field in the [CSI spec](https://github.com/container-storage-interface/spec/blob/master/spec.md#createvolume) for further information. 79 | 80 | ### Usage 81 | 82 | The CSI driver must implement the `VOLUME_MOUNT_GROUP` node service capability. The Pod-specified `fsGroup` will be available in `NodeStageVolumeRequest` and `NodePublishVolumeRequest` via `VolumeCapability.MountVolume.VolumeMountGroup`. 83 | 84 | #### Feature Gates 85 | To use this field, Kubernetes 1.22 binaries must start with the `DelegateFSGroupToCSIDriver` feature gate enabled: 86 | ``` 87 | --feature-gates=DelegateFSGroupToCSIDriver=true 88 | ``` 89 | This is enabled by default on 1.23 and higher. 90 | -------------------------------------------------------------------------------- /book/src/testing-drivers.md: -------------------------------------------------------------------------------- 1 | # Testing 2 | 3 | This section describes how CSI developers can test their CSI drivers. 4 | 5 | * [Unit Testing](unit-testing.md) 6 | * [Functional Testing](functional-testing.md) 7 | -------------------------------------------------------------------------------- /book/src/token-requests.md: -------------------------------------------------------------------------------- 1 | # Token Requests 2 | 3 | ## Status 4 | 5 | | Status | Min K8s Version | Max K8s Version | 6 | | ------ | --------------- | --------------- | 7 | | Alpha | 1.20 | 1.20 | 8 | | Beta | 1.21 | 1.21 | 9 | | GA | 1.22 | - | 10 | 11 | ## Overview 12 | 13 | This feature allows CSI drivers to impersonate the pods that they mount the 14 | volumes for. This improves the security posture in the mounting process where 15 | the volumes are ACL’ed on the pods’ service account without handing out 16 | unnecessary permissions to the CSI drivers’ service account. 17 | This feature is especially important for secret-handling CSI drivers, such as 18 | the [secrets-store-csi-driver](https://github.com/kubernetes-sigs/secrets-store-csi-driver). 19 | Since these tokens can be rotated and short-lived, 20 | this feature also provides a knob for CSI drivers to receive NodePublishVolume 21 | RPC calls periodically with the new token. This knob is also useful when volumes 22 | are short-lived, e.g. certificates. 23 | 24 | > See more details at the [design document](https://github.com/kubernetes/enhancements/blob/master/keps/sig-storage/1855-csi-driver-service-account-token/README.md). 25 | 26 | ## Usage 27 | 28 | This feature adds two fields in `CSIDriver` spec: 29 | 30 | ```go 31 | type CSIDriverSpec struct { 32 | ... // existing fields 33 | 34 | RequiresRepublish *bool 35 | TokenRequests []TokenRequest 36 | } 37 | 38 | type TokenRequest struct { 39 | Audience string 40 | ExpirationSeconds *int64 41 | } 42 | ``` 43 | 44 | - **`TokenRequest.Audience`**: 45 | 46 | - This is a required field. 47 | - Audiences should be distinct, otherwise the validation will fail. 48 | - If it is empty string, the audience of the token is the `APIAudiences` of kube-apiserver. 49 | one of the audiences specified. 50 | - See more about audience specification [here](https://tools.ietf.org/html/rfc7519#section-4.1.3) 51 | 52 | - **`TokenRequest.ExpirationSeconds`**: 53 | 54 | - The field is optional. 55 | - It has to be at least 10 minutes (600 seconds) and no more than `1 << 32` seconds. 56 | 57 | - **`RequiresRepublish`**: 58 | 59 | - This field is optional. 60 | - If this is true, `NodePublishVolume` will be periodically called. When used 61 | with `TokenRequest`, the token will be refreshed if it expired. 62 | `NodePublishVolume` should only change the contents rather than the 63 | mount because container will not be restarted to reflect the mount 64 | change. The period between `NodePublishVolume` is 0.1s. 65 | 66 | The token will be bounded to the pod that the CSI driver is mounting volumes for 67 | and will be set in `VolumeContext`: 68 | 69 | ```go 70 | "csi.storage.k8s.io/serviceAccount.tokens": { 71 | : { 72 | 'token': , 73 | 'expirationTimestamp': , 74 | }, 75 | ... 76 | } 77 | ``` 78 | 79 | If CSI driver doesn't find token recorded in the `volume_context`, it should return error in `NodePublishVolume` to inform Kubelet to retry. 80 | 81 | ### Example 82 | 83 | Here is an example of a `CSIDriver` object: 84 | 85 | ```yaml 86 | apiVersion: storage.k8s.io/v1 87 | kind: CSIDriver 88 | metadata: 89 | name: mycsidriver.example.com 90 | spec: 91 | tokenRequests: 92 | - audience: "gcp" 93 | - audience: "" 94 | expirationSeconds: 3600 95 | requiresRepublish: true 96 | ``` 97 | 98 | ### Feature gate 99 | 100 | Kube apiserver must start with the `CSIServiceAccountToken` feature gate enabled: 101 | 102 | ``` 103 | --feature-gates=CSIServiceAccountToken=true 104 | ``` 105 | 106 | It is enabled by default in Kubernetes 1.21 and cannot be disabled since 1.22. 107 | 108 | ### Example CSI Drivers 109 | 110 | - [secrets-store-csi-driver](https://github.com/kubernetes-sigs/secrets-store-csi-driver) 111 | - With [GCP](https://github.com/GoogleCloudPlatform/secrets-store-csi-driver-provider-gcp), 112 | the driver will pass the token to GCP provider to exchange for GCP credentials, and then request 113 | secrets from Secret Manager. 114 | - With [Vault](https://github.com/hashicorp/secrets-store-csi-driver-provider-vault), 115 | the Vault provider will send the token to Vault which will use the token in 116 | `TokenReview` request to authenticate. 117 | - With [Azure](https://github.com/Azure/secrets-store-csi-driver-provider-azure), 118 | the driver will pass the token to Azure provider to exchange for Azure credentials, and then request 119 | secrets from Key Vault. 120 | -------------------------------------------------------------------------------- /book/src/topology.md: -------------------------------------------------------------------------------- 1 | # CSI Topology Feature 2 | 3 | ## Status 4 | 5 | Status | Min K8s Version | Max K8s Version | external-provisioner Version 6 | --|--|--|-- 7 | Alpha | 1.12 | 1.12 | 0.4 8 | Alpha | 1.13 | 1.13 | 1.0 9 | Beta | 1.14 | 1.16 | 1.1-1.4 10 | GA | 1.17 | - | 1.5+ 11 | 12 | ## Overview 13 | Some storage systems expose volumes that are not equally accessible by all nodes in a Kubernetes cluster. Instead volumes may be constrained to some subset of node(s) in the cluster. The cluster may be segmented into, for example, “racks” or “regions” and “zones” or some other grouping, and a given volume may be accessible only from one of those groups. 14 | 15 | To enable orchestration systems, like Kubernetes, to work well with storage systems which expose volumes that are not equally accessible by all nodes, the [CSI spec](https://github.com/container-storage-interface/spec/blob/master/spec.md) enables: 16 | 17 | 1. Ability for a CSI Driver to opaquely specify where a particular node exists (e.g. "node A" is in "zone 1"). 18 | 2. Ability for Kubernetes (users or components) to influence where a volume is provisioned (e.g. provision new volume in either "zone 1" or "zone 2"). 19 | 3. Ability for a CSI Driver to opaquely specify where a particular volume exists (e.g. "volume X" is accessible by all nodes in "zone 1" and "zone 2"). 20 | 21 | Kubernetes and the [external-provisioner](external-provisioner.md) use these abilities to make intelligent scheduling and provisioning decisions (that Kubernetes can both influence and act on topology information for each volume), 22 | 23 | ## Implementing Topology in your CSI Driver 24 | 25 | To support topology in a CSI driver, the following must be implemented: 26 | 27 | * The `PluginCapability` must support `VOLUME_ACCESSIBILITY_CONSTRAINTS`. 28 | * The plugin must fill in `accessible_topology` in `NodeGetInfoResponse`. 29 | This information will be used to populate the Kubernetes [CSINode object](csi-node-object.md) and add the topology labels to the Node object. 30 | * During `CreateVolume`, the topology information will get passed in through `CreateVolumeRequest.accessibility_requirements`. 31 | 32 | In the StorageClass object, both `volumeBindingMode` values of `Immediate` and 33 | `WaitForFirstConsumer` are supported. 34 | 35 | * If `Immediate` is set, then the 36 | external-provisioner will pass in all available topologies in the cluster for 37 | the driver. 38 | * If `WaitForFirstConsumer` is set, then the external-provisioner will wait for 39 | the scheduler to pick a node. The topology of that selected node will then be 40 | set as the first entry in `CreateVolumeRequest.accessibility_requirements.preferred`. 41 | All remaining topologies are still included in the `requisite` and `preferred` 42 | fields to support storage systems that span across multiple topologies. 43 | 44 | ## Sidecar Deployment 45 | 46 | The topology feature requires the 47 | [external-provisioner](external-provisioner.md) sidecar with the 48 | Topology feature gate enabled: 49 | 50 | ``` 51 | --feature-gates=Topology=true 52 | ``` 53 | 54 | ## Kubernetes Cluster Setup 55 | 56 | ### Beta 57 | 58 | In the *Kubernetes cluster* the `CSINodeInfo` feature must be enabled on both Kubernetes master and nodes (refer to the [CSINode Object](csi-node-object.md) section for more info): 59 | 60 | ``` 61 | --feature-gates=CSINodeInfo=true 62 | ``` 63 | 64 | In order to fully function properly, all Kubernetes master and nodes must be on at least 65 | Kubernetes 1.14. If a selected node is on a lower version, topology is ignored and not 66 | passed to the driver during `CreateVolume`. 67 | 68 | ### Alpha 69 | 70 | The alpha feature in the external-provisioner is not compatible across 71 | Kubernetes versions. In addition, Kubernetes master and node version skew and 72 | upgrades are not supported. 73 | 74 | The `CSINodeInfo`, `VolumeScheduling`, and `KubeletPluginsWatcher` feature gates 75 | must be enabled on both Kubernetes master and nodes. 76 | 77 | The [CSINodeInfo](csi-node-object.md) CRDs also have to be manually installed in the 78 | cluster. 79 | 80 | ## Storage Internal Topology 81 | 82 | Note that a storage system may also have an "internal topology" different from (independent of) the topology of the cluster where workloads are scheduled. Meaning volumes exposed by the storage system are equally accessible by all nodes in the Kubernetes cluster, but the storage system has some internal topology that may influence, for example, the performance of a volume from a given node. 83 | 84 | CSI does not currently expose a first class mechanism to influence such storage system internal topology on provisioning. Therefore, Kubernetes can not programmatically influence such topology. However, a CSI Driver may expose the ability to specify internal storage topology during volume provisioning using an opaque parameter in the `CreateVolume` CSI call (CSI enables CSI Drivers to expose an arbitrary set of configuration options during dynamic provisioning by allowing opaque parameters to be passed from cluster admins to the storage plugins) -- this would enable cluster admins to be able to control the storage system internal topology during provisioning. 85 | -------------------------------------------------------------------------------- /book/src/troubleshooting.md: -------------------------------------------------------------------------------- 1 | # Troubleshooting 2 | 3 | # Known Issues 4 | - [[minikube-3378](https://github.com/kubernetes/minikube/issues/3378)]: Volume mount causes minikube VM to become corrupted 5 | 6 | # Common Errors 7 | 8 | ### Node plugin pod does not start with *RunContainerError* status 9 | 10 | `kubectl describe pod your-nodeplugin-pod` shows: 11 | ``` 12 | failed to start container "your-driver": Error response from daemon: 13 | linux mounts: Path /var/lib/kubelet/pods is mounted on / but it is not a shared mount 14 | ``` 15 | 16 | Your Docker host is not configured to allow shared mounts. Take a look at [this page][docker-shared-mount] for instructions to enable them. 17 | 18 | [docker-shared-mount]: https://kubernetes.io/docs/concepts/storage/volumes/#configuration 19 | 20 | 21 | ### External attacher can't find _VolumeAttachments_ 22 | 23 | If you have a Kubernetes 1.9 cluster, not being able to list _VolumeAttachment_ 24 | and the following error are due to the lack of the 25 | `storage.k8s.io/v1alpha1=true` runtime configuration: 26 | 27 | ``` 28 | $ kubectl logs csi-pod external-attacher 29 | ... 30 | I0306 16:34:50.976069 1 reflector.go:240] Listing and watching *v1alpha1.VolumeAttachment from github.com/kubernetes-csi/external-attacher/vendor/k8s.io/client-go/informers/factory.go:86 31 | 32 | E0306 16:34:50.992034 1 reflector.go:205] github.com/kubernetes-csi/external-attacher/vendor/k8s.io/client-go/informers/factory.go:86: Failed to list *v1alpha1.VolumeAttachment: the server could not find the requested resource 33 | ... 34 | ``` 35 | 36 | 37 | ### Problems with the external components 38 | 39 | The external components images are under active development. It can 40 | happen that they become incompatible with each other. If the 41 | issues above have been ruled out, [contact the sig-storage 42 | team](https://github.com/kubernetes/community/tree/master/sig-storage) and/or 43 | [run the e2e test](https://github.com/kubernetes/community/blob/master/contributors/devel/sig-testing/e2e-tests.md#local-clusters): 44 | ``` 45 | go run hack/e2e.go -- --provider=local --test --test_args="--ginkgo.focus=Feature:CSI" 46 | ``` 47 | -------------------------------------------------------------------------------- /book/src/unit-testing.md: -------------------------------------------------------------------------------- 1 | # Unit Testing 2 | 3 | The [CSI `sanity`](https://github.com/kubernetes-csi/csi-test/tree/master/pkg/sanity) package from [csi-test](https://github.com/kubernetes-csi/csi-test) can be used for unit testing your CSI driver. 4 | 5 | It contains a set of basic tests that all CSI drivers should pass (for example, `NodePublishVolume should fail when no volume id is provided`, etc.). 6 | 7 | This package can be used in two modes: 8 | 9 | * Via a Golang test framework (`sanity` package is imported as a dependency) 10 | * Via a command line against your driver binary. 11 | 12 | Read the [documentation of the `sanity` package](https://github.com/kubernetes-csi/csi-test/blob/master/pkg/sanity/README.md) for more details. 13 | -------------------------------------------------------------------------------- /book/src/volume-attributes-class.md: -------------------------------------------------------------------------------- 1 | # Volume Attributes Classes 2 | 3 | ## Status 4 | 5 | Status | Min K8s Version | Max K8s Version | external-provisioner | external-resizer 6 | -------|-----------------|-----------------|-----------------------|------------------------- 7 | Alpha | 1.29 | - | v4.0.0 | v1.10.0 8 | Beta | 1.31 | - | v5.1.0 | v1.12.0 9 | 10 | ## Overview 11 | A `VolumeAttributesClass` is a cluster-scoped resource that can be used to control and change the settings of a `PersistentVolume`. 12 | Its primary use is to adjust the class of service for a volume (e.g., bronze, silver, gold) to meet different performance, quality-of-service, or resilience requirements. 13 | 14 | > See the [Kubernetes Enhancement Proposal](https://github.com/kubernetes/enhancements/blob/master/keps/sig-storage/3751-volume-attributes-class/README.md) 15 | > for more details on the background, design and discussions. 16 | 17 | ## Usage 18 | This feature is enabled by default in the [external-provisioner](https://github.com/kubernetes-csi/external-provisioner/?tab=readme-ov-file#feature-status). 19 | 20 | To take advantage of `VolumeAttibutesClass` feature: 21 | * The CSI driver must implement the `MODIFY_VOLUME` capability. 22 | * The Kubernetes administrator must create the `VolumeAttributesClass` with the relevant `parameters` for the `driverName` 23 | * The user can then patch the `PersistentVolumeClaim` under `spec.volumeAttributesClassName` to select or update the settings he needs for the volume. 24 | 25 | > For more information about how to use the feature, visit the [Kubernetes blog](https://kubernetes.io/blog/2024/08/15/kubernetes-1-31-volume-attributes-class/) page. 26 | -------------------------------------------------------------------------------- /book/src/volume-cloning.md: -------------------------------------------------------------------------------- 1 | # Volume Cloning 2 | 3 | ## Status and Releases 4 | 5 | Status | Min k8s Version | Max k8s version | external-provisioner Version 6 | --|--|--|-- 7 | Alpha | 1.15 | 1.15 | 1.3 8 | Beta | 1.16 | 1.17 | 1.4 9 | GA | 1.18 | - | 1.6 10 | 11 | ## Overview 12 | 13 | A Clone is defined as a duplicate of an existing Kubernetes Volume. For more information on cloning in Kubernetes see the concepts doc for [Volume Cloning](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#volume-cloning). A storage provider that allows volume cloning as a create feature, may choose to implement volume cloning via a control-plan CSI RPC call. 14 | 15 | For details regarding the kubernetes API for volume cloning, please see [kubernetes concepts](https://kubernetes.io/docs/concepts/storage/volume-pvc-datasource/). 16 | 17 | ## Implementing Volume cloning functionality 18 | 19 | To implement volume cloning the CSI driver MUST: 20 | 21 | 1. Implement checks for `csi.CreateVolumeRequest.VolumeContentSource` in the plugin's `CreateVolume` function implementation. 22 | 2. Implement `CLONE_VOLUME` controller capability. 23 | 24 | It is the responsibility of the storage plugin to either implement an expansion after clone if a provision request size is greater than the source, or allow the external-resizer to handle it. In the case that the plugin does not support resize capability and it does not have the capability to create a clone that is greater in size than the specified source volume, then the provision request should result in a failure. 25 | 26 | ## Deploying volume clone functionality 27 | 28 | The Kubernetes CSI development team maintains the [external-provisioner](external-provisioner.md) which is responsible for detecting requests for a PVC DataSource and providing that information to the plugin via the `csi.CreateVolumeRequest`. It's up to the plugin to check the `csi.CreateVolumeRequest` for a `VolumeContentSource` entry in the CreateVolumeRequest object. 29 | 30 | There are no additional side-cars or add on components required. 31 | 32 | ## Enabling Cloning for CSI volumes in Kubernetes 33 | Volume cloning was promoted to Beta in version 1.16 and GA in 1.18, and as such is enabled by defult for kubernetes versions >= 1.16 34 | 35 | In Kubernetes 1.15 this feature was alpha status and required enabling the appropriate feature gate: 36 | 37 | ``` 38 | --feature-gates=VolumePVCDataSource=true 39 | ``` 40 | 41 | ## Example implementation 42 | 43 | A trivial example implementation can be found in the [csi-hostpath plugin](https://github.com/kubernetes-csi/csi-driver-host-path) in its implementation of `CreateVolume`. 44 | -------------------------------------------------------------------------------- /book/src/volume-datasources.md: -------------------------------------------------------------------------------- 1 | ## Kubernetes PVC DataSource (CSI VolumeContentSource) 2 | 3 | When creating a new PersistentVolumeClaim, the Kubernetes API provides a `PersistentVolumeClaim.DataSource` parameter. This parameter is used to specify the CSI `CreateVolumeRequest.VolumeContentSource` option for CSI Provisioners. The `VolumeContentSource` parameter instructs the CSI plugin to pre-populate the volume being provisioned with data from the specified source. 4 | 5 | ### External Provisioner Responsibilities 6 | 7 | If a `DataSource` is specified in the `CreateVolume` call to the CSI external provisioner, the external provisioner will fetch the specified resource and pass the appropriate object id to the plugin. 8 | 9 | ## Supported DataSources 10 | 11 | Currently there are two types of `PersistentVolumeClaim.DataSource` objects that are supported: 12 | 1. [VolumeSnapshot](snapshot-restore-feature.md) 13 | 2. [PersistentVolumeClaim (Cloning)](volume-cloning.md) 14 | 15 | 16 | -------------------------------------------------------------------------------- /book/src/volume-expansion.md: -------------------------------------------------------------------------------- 1 | # Volume Expansion 2 | 3 | ## Status 4 | 5 | Status | Min K8s Version | Max K8s Version | external-resizer Version 6 | --|--|--|-- 7 | Alpha | 1.14 | 1.15 | 0.2 8 | Beta | 1.16 | - | 0.3 9 | 10 | ## Overview 11 | 12 | A storage provider that allows volume expansion after creation, may choose to implement volume expansion either via a 13 | control-plane CSI RPC call or via node CSI RPC call or both as a two step process. 14 | 15 | ## Implementing Volume expansion functionality 16 | 17 | To implement volume expansion the CSI driver MUST: 18 | 19 | 1. Implement `VolumeExpansion` plugin capability. 20 | 2. Implement `EXPAND_VOLUME` controller capability or implement `EXPAND_VOLUME` node capability or both. 21 | 22 | `ControllerExpandVolume` RPC call can be made when volume is `ONLINE` or `OFFLINE` depending on `VolumeExpansion` plugin 23 | capability. Where `ONLINE` and `OFFLINE` means: 24 | 25 | 1. *ONLINE* : Volume is currently published or available on a node. 26 | 2. *OFFLINE* : Volume is currently not published or available on a node. 27 | 28 | `NodeExpandVolume` RPC call on the other hand - *always* requires volume to be published or staged on a node (and hence `ONLINE`). 29 | For block storage file systems - `NodeExpandVolume` is typically used for expanding the file system on the node, but it can be also 30 | used to perform other volume expansion related housekeeping operations on the node. 31 | 32 | For details, see the [CSI spec](https://github.com/container-storage-interface/spec/blob/master/spec.md). 33 | 34 | ## Deploying volume expansion functionality 35 | 36 | The Kubernetes CSI development team maintains [external-resizer](external-resizer.md) Kubernetes CSI [Sidecar Containers](sidecar-containers.md). 37 | This sidecar container implements the logic for watching the Kubernetes API for Persistent Volume claim edits and issuing `ControllerExpandVolume` RPC call against a CSI endpoint and updating `PersistentVolume` object to reflect new size. 38 | 39 | This sidecar is needed even if CSI driver does not have `EXPAND_VOLUME` controller capability, in this case it performs a NO-OP expansion and updates `PersistentVolume` object. `NodeExpandVolume` is always called by Kubelet on the node. 40 | 41 | For more details, see [external-resizer](external-resizer.md). 42 | 43 | 44 | ## Enabling Volume expansion for CSI volumes in Kubernetes 45 | 46 | To expand a volume if permitted by the [storage class](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#expanding-persistent-volumes-claims), users just need to edit the persistent volume claim object and request more storage. 47 | 48 | In Kubernetes 1.14 and 1.15, this feature was in alpha status and required enabling the following feature gate: 49 | 50 | ``` 51 | --feature-gates=ExpandCSIVolumes=true 52 | ``` 53 | 54 | Also in Kubernetes 1.14 and 1.15, online expansion had to be enabled explicitly: 55 | 56 | ``` 57 | --feature-gates=ExpandInUsePersistentVolumes=true 58 | ``` 59 | 60 | external-resizer and kubelet add appropriate events and conditions to persistent volume claim objects indicating progress of volume expansion operations. 61 | -------------------------------------------------------------------------------- /book/src/volume-health-monitor.md: -------------------------------------------------------------------------------- 1 | # Volume Health Monitoring Feature 2 | 3 | ## Status 4 | 5 | Status | Min K8s Version | Max K8s Version | external-health-monitor-controller Version 6 | --|--|--|-- 7 | Alpha | 1.21 | - | 0.8.0 8 | 9 | 10 | ## Overview 11 | 12 | The External Health Monitor is part of Kubernetes implementation of [Container Storage Interface (CSI)](https://github.com/container-storage-interface/spec). It was introduced as an Alpha feature in Kubernetes v1.19. In Kubernetes 1.21, a second Alpha was done due to a design change which deprecated [External Health Monitor Agent](external-health-monitor-agent). 13 | 14 | The [External Health Monitor](https://github.com/kubernetes/enhancements/tree/master/keps/sig-storage/1432-volume-health-monitor) is implemented as two components: `External Health Monitor Controller` and `Kubelet`. 15 | 16 | - External Health Monitor Controller: 17 | - The external health monitor controller will be deployed as a sidecar together with the CSI controller driver, similar to how the external-provisioner sidecar is deployed. 18 | - Trigger controller RPC to check the health condition of the CSI volumes. 19 | - The external controller sidecar will also watch for node failure events. This component can be enabled via a flag. 20 | 21 | - Kubelet: 22 | - In addition to existing volume stats collected already, Kubelet will also check volume's mounting conditions collected from the same CSI node RPC and log events to Pods if volume condition is abnormal. 23 | 24 | The Volume Health Monitoring feature need to invoke the following CSI interfaces. 25 | 26 | - External Health Monitor Controller: 27 | - ListVolumes (If both `ListVolumes` and `ControllerGetVolume` are supported, `ListVolumes` will be used) 28 | - ControllerGetVolume 29 | - Kubelet: 30 | - NodeGetVolumeStats 31 | - This feature in Kubelet is controlled by an Alpha feature gate `CSIVolumeHealth`. 32 | 33 | See [external-health-monitor-controller.md](external-health-monitor-controller.md) for more details on the CSI `external-health-monitor-controller` sidecar. 34 | -------------------------------------------------------------------------------- /book/src/volume-limits.md: -------------------------------------------------------------------------------- 1 | # Volume Limits 2 | 3 | ## Status 4 | 5 | Status | Min K8s Version | Max K8s Version 6 | --|--|-- 7 | Alpha | 1.11 | 1.11 8 | Beta | 1.12 | 1.16 9 | GA | 1.17 | - 10 | 11 | ## Overview 12 | 13 | Some storage providers may have a restriction on the number of volumes that can be used in a Node. This is common in cloud providers, but other providers might impose restriction as well. 14 | 15 | Kubernetes will respect this limit as long the CSI driver advertises it. To support volume limits in a CSI driver, the plugin must fill in `max_volumes_per_node` in `NodeGetInfoResponse`. 16 | 17 | It is recommended that CSI drivers allow for customization of volume limits. That way cluster administrators can distribute the limits of the same storage backends (e.g. iSCSI) accross different drivers, according to their individual needs. 18 | -------------------------------------------------------------------------------- /code-of-conduct.md: -------------------------------------------------------------------------------- 1 | # Kubernetes Community Code of Conduct 2 | 3 | Please refer to our [Kubernetes Community Code of Conduct](https://git.k8s.io/community/code-of-conduct.md) 4 | -------------------------------------------------------------------------------- /hack/gen-api.json: -------------------------------------------------------------------------------- 1 | { 2 | "hideMemberFields": [ 3 | "TypeMeta" 4 | ], 5 | "hideTypePatterns": [ 6 | "ParseError$", 7 | "List$" 8 | ], 9 | "externalPackages": [ 10 | { 11 | "typeMatchPrefix": "^k8s\\.io/apimachinery/pkg/apis/meta/v1\\.Duration$", 12 | "docsURLTemplate": "https://pkg.go.dev/k8s.io/apimachinery/pkg/apis/meta/v1#Duration" 13 | }, 14 | { 15 | "typeMatchPrefix": "^k8s\\.io/(api|apimachinery/pkg/apis|apimachinery/pkg/api)/", 16 | "docsURLTemplate": "https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.23/#{{lower .TypeIdentifier}}-{{arrIndex .PackageSegments -1}}-{{arrIndex .PackageSegments -2}}" 17 | }, 18 | { 19 | "typeMatchPrefix": "^k8s\\.io/api/core/v1/\\.ObjectReference$", 20 | "docsURLTemplate": "https://pkg.go.dev/k8s.io/api/core/v1#ObjectReference" 21 | } 22 | ], 23 | "typeDisplayNamePrefixOverrides": { 24 | "k8s.io/api/": "Kubernetes ", 25 | "k8s.io/apimachinery/pkg/apis/": "Kubernetes " 26 | }, 27 | "markdownDisabled": false 28 | } 29 | -------------------------------------------------------------------------------- /hack/gen-api.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh -e 2 | 3 | # This script uses the gen-crd-api-reference-docs tool to generate a markdown 4 | # document for the CSI CRD API. 5 | # 6 | # It clones the gen-crd-reference-docs and external-snapshotter repositories 7 | # into two separate temporary folders. The references of the repositories are 8 | # defaulted to master. They can be overwritten by using the: 9 | # - CSI_REF 10 | # - GEN_CRD_REF 11 | # variables. 12 | # 13 | # The tool parses the Go files in the external-snapshotter's CRD package and 14 | # writes the content to the book/api folder. The configuration of the tool can 15 | # be found in the hack/gen-api.json file. 16 | 17 | GEN_TOOL_REF=${GEN_TOOL_REF:-master} 18 | CSI_REF=${CSI_REF:-master} 19 | 20 | curr_dir=$(pwd) 21 | gen_tool_dir=$(mktemp -d) 22 | csi_dir=$(mktemp -d) 23 | # trap 'rm -rf "$gen_tool_dir" "$csi_dir"' EXIT 24 | 25 | config_path=$(find . -iname "*gen-api.json") 26 | 27 | git clone git@github.com:ahmetb/gen-crd-api-reference-docs.git "$gen_tool_dir" 28 | git clone git@github.com:kubernetes-csi/external-snapshotter.git "$csi_dir" 29 | 30 | cd "$gen_tool_dir" 31 | git checkout "$GEN_TOOL_REF" 32 | go build "$gen_tool_dir" 33 | 34 | cd "$csi_dir" 35 | git checkout "$CSI_REF" 36 | $gen_tool_dir/gen-crd-api-reference-docs \ 37 | -config="$curr_dir/$config_path" \ 38 | -api-dir="./client/apis/volumesnapshot/v1" \ 39 | -template-dir="$gen_tool_dir/template" \ 40 | -out-file="out.html" 41 | 42 | # add title to page 43 | printf "# Volume Snapshot\n\n%s" "$(cat out.html)" > "$curr_dir"/book/src/api/volume-snapshot.md 44 | -------------------------------------------------------------------------------- /hack/update-example.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh -e 2 | # 3 | # Updates book/src/Example.md with the latest revisions of all referenced files. 4 | # While doing so it prints a diff between files and a commit log. 5 | # 6 | # Updating the links to files embedded in the docs repo itself together 7 | # with the Readme.md is a bit harder: 8 | # - commit the updated .yaml files locally 9 | # - manually update the revision hash 10 | # - submit the PR *without* touching the updated .yaml files 11 | # 12 | # It's easier to update the .yaml files, submit a PR, and then 13 | # once accepted, update the example using this script. 14 | 15 | example=book/src/Example.md 16 | 17 | for url in $(grep 'kubectl create -f.*https://raw.githubusercontent.com/' $example | sed -e 's;.*\(https://raw.githubusercontent.com[^ (]*\).*;\1;'); do 18 | # split URL into pieces 19 | eval $(echo $url | sed -e 's/\]$//' -e 's;https://raw.githubusercontent.com/\([^/]*/[^/]*\)/\([^/]*\)/\(.*\);repo=\1 rev=\2 file=\3;') 20 | git fetch https://github.com/$repo refs/heads/master:refs/remotes/$repo/master 21 | echo $repo/$file 22 | set -x 23 | git log $rev..$repo/master -- $file | cat 24 | git diff $rev..$repo/master -- $file | cat 25 | set +x 26 | echo 27 | latest=$(git rev-list --max-count=1 $repo/master) 28 | sed -i \ 29 | -e "s;https://raw.githubusercontent.com/$repo/$rev/$file;https://raw.githubusercontent.com/$repo/$latest/$file;" \ 30 | -e "s;https://github.com/$repo/blob/$rev/$file;https://github.com/$repo/blob/$latest/$file;" \ 31 | $example 32 | done 33 | --------------------------------------------------------------------------------