├── .gitignore ├── README.md ├── controller.tf ├── csi_driver.tf ├── csi_rbac.tf ├── locals.tf ├── metrics.tf ├── node.tf ├── output.tf ├── storage_class.tf ├── token.tf └── variables.tf /.gitignore: -------------------------------------------------------------------------------- 1 | .vscode/ -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # terraform-kubernetes-hcloud-csi-driver 2 | A simple module to provision the [Hetzner Container Storage Interface Driver](https://github.com/hetznercloud/csi-driver/) within a Kubernetes cluster running on Hetzner Cloud. See the variables file for the available configuration options. Please note that this module **requires Kubernetes 1.15 or newer**. 3 | 4 | ### **Prerequisites** 5 | 6 | Requires cluster nodes be prior [initialized by a cloud-controller-manager](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/#taint-based-evictions). You can use the [terraform-kubernetes-hcloud-controller-manager](https://github.com/colinwilson/terraform-kubernetes-hcloud-controller-manager) module to initialize your cluster nodes. 7 | 8 | ### **Deploy Test Persistent Volume** 9 | 10 | Verify everything is working, create a persistent volume claim and a pod which uses that volume: 11 | 12 | ``` 13 | apiVersion: v1 14 | kind: PersistentVolumeClaim 15 | metadata: 16 | name: csi-pvc 17 | spec: 18 | accessModes: 19 | - ReadWriteOnce 20 | resources: 21 | requests: 22 | storage: 10Gi 23 | storageClassName: hcloud-volumes 24 | --- 25 | kind: Pod 26 | apiVersion: v1 27 | metadata: 28 | name: my-csi-app 29 | spec: 30 | containers: 31 | - name: my-frontend 32 | image: busybox 33 | volumeMounts: 34 | - mountPath: "/data" 35 | name: my-csi-volume 36 | command: [ "sleep", "1000000" ] 37 | volumes: 38 | - name: my-csi-volume 39 | persistentVolumeClaim: 40 | claimName: csi-pvc 41 | ``` 42 | 43 | Once the pod is ready, exec a shell and check that your volume is mounted at `/data`. 44 | 45 | ``` 46 | kubectl exec -it my-csi-app -- /bin/sh 47 | ``` -------------------------------------------------------------------------------- /controller.tf: -------------------------------------------------------------------------------- 1 | # Create CSI Controller StatefulSet 2 | resource "kubernetes_stateful_set" "hcloud_csi_controller" { 3 | metadata { 4 | name = "hcloud-csi-controller" 5 | namespace = "kube-system" 6 | } 7 | 8 | spec { 9 | selector { 10 | match_labels = { 11 | app = "hcloud-csi-controller" 12 | } 13 | } 14 | 15 | service_name = "hcloud-csi-controller" 16 | replicas = 1 17 | 18 | template { 19 | metadata { 20 | labels = { 21 | app = "hcloud-csi-controller" 22 | } 23 | } 24 | 25 | spec { 26 | automount_service_account_token = true # override Terraform's default false - https://github.com/kubernetes/kubernetes/issues/27973#issuecomment-462185284 27 | service_account_name = "hcloud-csi" 28 | 29 | container { 30 | name = "csi-attacher" 31 | image = var.kube_version < 1.16 ? local.IMAGE_CSI_ATTACHER_LEGACY : local.IMAGE_CSI_ATTACHER 32 | 33 | args = [ 34 | "--csi-address=/var/lib/csi/sockets/pluginproxy/csi.sock", 35 | "--v=5", 36 | ] 37 | 38 | volume_mount { 39 | name = "socket-dir" 40 | mount_path = "/var/lib/csi/sockets/pluginproxy/" 41 | } 42 | 43 | security_context { 44 | privileged = true 45 | capabilities { 46 | add = ["SYS_ADMIN"] 47 | } 48 | allow_privilege_escalation = true 49 | } 50 | } 51 | 52 | dynamic "container" { 53 | for_each = var.kube_version < 1.16 ? [] : [1] 54 | 55 | content { 56 | name = "csi-resizer" 57 | image = local.IMAGE_CSI_RESIZER 58 | 59 | args = [ 60 | "--csi-address=/var/lib/csi/sockets/pluginproxy/csi.sock", 61 | "--v=5", 62 | ] 63 | 64 | volume_mount { 65 | name = "socket-dir" 66 | mount_path = "/var/lib/csi/sockets/pluginproxy/" 67 | } 68 | 69 | security_context { 70 | privileged = true 71 | capabilities { 72 | add = ["SYS_ADMIN"] 73 | } 74 | allow_privilege_escalation = true 75 | } 76 | } 77 | } 78 | 79 | container { 80 | name = "csi-provisioner" 81 | image = var.kube_version < 1.16 ? local.IMAGE_CSI_PROVISIONER_LEGACY : local.IMAGE_CSI_PROVISIONER 82 | 83 | args = [ 84 | "--provisioner=csi.hetzner.cloud", 85 | "--csi-address=/var/lib/csi/sockets/pluginproxy/csi.sock", 86 | "--feature-gates=Topology=true", 87 | "--v=5", 88 | ] 89 | 90 | volume_mount { 91 | name = "socket-dir" 92 | mount_path = "/var/lib/csi/sockets/pluginproxy/" 93 | } 94 | 95 | security_context { 96 | privileged = true 97 | capabilities { 98 | add = ["SYS_ADMIN"] 99 | } 100 | allow_privilege_escalation = true 101 | } 102 | } 103 | 104 | container { 105 | name = "hcloud-csi-driver" 106 | image = var.kube_version < 1.16 ? local.IMAGE_HCLOUD_CSI_DRIVER_LEGACY : local.IMAGE_HCLOUD_CSI_DRIVER 107 | image_pull_policy = "Always" 108 | 109 | env { 110 | name = "CSI_ENDPOINT" 111 | value = "unix:///var/lib/csi/sockets/pluginproxy/csi.sock" 112 | } 113 | 114 | dynamic "env" { 115 | for_each = var.kube_version < 1.16 ? [] : [1] 116 | 117 | content { 118 | name = "METRICS_ENDPOINT" 119 | value = "0.0.0.0:9189" 120 | } 121 | } 122 | 123 | env { 124 | name = "HCLOUD_TOKEN" 125 | value_from { 126 | secret_key_ref { 127 | name = "hcloud-csi" 128 | key = "token" 129 | } 130 | } 131 | } 132 | env { 133 | name = "KUBE_NODE_NAME" 134 | value_from { 135 | field_ref { 136 | api_version = "v1" 137 | field_path = "spec.nodeName" 138 | } 139 | } 140 | } 141 | volume_mount { 142 | name = "socket-dir" 143 | mount_path = "/var/lib/csi/sockets/pluginproxy/" 144 | } 145 | 146 | dynamic "port" { 147 | for_each = var.kube_version < 1.16 ? [] : [1] 148 | 149 | content { 150 | container_port = 9189 151 | name = "metrics" 152 | } 153 | } 154 | 155 | dynamic "port" { 156 | for_each = var.kube_version < 1.16 ? [] : [1] 157 | 158 | content { 159 | container_port = 9808 160 | name = "healthz" 161 | protocol = "TCP" 162 | } 163 | } 164 | 165 | dynamic "liveness_probe" { 166 | for_each = var.kube_version < 1.16 ? [] : [1] 167 | 168 | content { 169 | http_get { 170 | path = "/healthz" 171 | port = "healthz" 172 | } 173 | 174 | initial_delay_seconds = 10 175 | timeout_seconds = 3 176 | period_seconds = 2 177 | } 178 | } 179 | 180 | security_context { 181 | privileged = true 182 | capabilities { 183 | add = ["SYS_ADMIN"] 184 | } 185 | allow_privilege_escalation = true 186 | } 187 | } 188 | 189 | dynamic "container" { 190 | for_each = var.kube_version < 1.16 ? [] : [1] 191 | 192 | content { 193 | name = "liveness-probe" 194 | image = local.IMAGE_LIVENESSPROBE 195 | image_pull_policy = "Always" 196 | 197 | args = [ 198 | "--csi-address=/var/lib/csi/sockets/pluginproxy/csi.sock", 199 | ] 200 | 201 | volume_mount { 202 | name = "socket-dir" 203 | mount_path = "/var/lib/csi/sockets/pluginproxy/" 204 | } 205 | } 206 | } 207 | 208 | volume { 209 | name = "socket-dir" 210 | 211 | empty_dir {} 212 | } 213 | } 214 | } 215 | } 216 | } -------------------------------------------------------------------------------- /csi_driver.tf: -------------------------------------------------------------------------------- 1 | # Create Hetzner cloud csi driver 2 | resource "kubernetes_csi_driver" "csi_driver" { 3 | #count = var.kube_version <= 1.13 ? 0 : 1 4 | 5 | metadata { 6 | name = "csi.hetzner.cloud" 7 | } 8 | 9 | spec { 10 | attach_required = true 11 | pod_info_on_mount = true 12 | volume_lifecycle_modes = var.kube_version < 1.16 ? null : ["Persistent"] 13 | } 14 | } -------------------------------------------------------------------------------- /csi_rbac.tf: -------------------------------------------------------------------------------- 1 | # Create Service Account 2 | resource "kubernetes_service_account" "hcloud_csi" { 3 | metadata { 4 | name = "hcloud-csi" 5 | namespace = "kube-system" 6 | } 7 | } 8 | 9 | # Create Cluster Role 10 | resource "kubernetes_cluster_role" "hcloud_csi" { 11 | metadata { 12 | name = "hcloud-csi" 13 | } 14 | # attacher 15 | rule { 16 | api_groups = [""] 17 | resources = ["persistentvolumes"] 18 | verbs = var.kube_version < 1.16 ? ["get", "list", "watch", "update"] : ["get", "list", "watch", "update", "patch"] 19 | } 20 | 21 | rule { 22 | api_groups = [""] 23 | resources = ["nodes"] 24 | verbs = ["get", "list", "watch"] 25 | } 26 | 27 | dynamic "rule" { 28 | for_each = var.kube_version < 1.16 ? [] : [1] 29 | 30 | content { 31 | api_groups = ["csi.storage.k8s.io"] 32 | resources = ["csinodeinfos"] 33 | verbs = ["get", "list", "watch"] 34 | } 35 | } 36 | 37 | rule { 38 | api_groups = ["storage.k8s.io"] 39 | resources = ["csinodes"] 40 | verbs = ["get", "list", "watch"] 41 | } 42 | 43 | rule { 44 | api_groups = ["storage.k8s.io"] 45 | resources = ["volumeattachments"] 46 | verbs = var.kube_version < 1.16 ? ["get", "list", "watch", "update"] : ["get", "list", "watch", "update", "patch"] 47 | } 48 | 49 | # provisioner 50 | rule { 51 | api_groups = [""] 52 | resources = ["secrets"] 53 | verbs = ["get", "list"] 54 | } 55 | 56 | rule { 57 | api_groups = [""] 58 | resources = ["persistentvolumes"] 59 | verbs = var.kube_version < 1.16 ? ["get", "list", "watch", "create", "delete"] : ["get", "list", "watch", "create", "delete", "patch"] 60 | } 61 | 62 | rule { 63 | api_groups = [""] 64 | resources = var.kube_version < 1.16 ? ["persistentvolumeclaims"] : ["persistentvolumeclaims", "persistentvolumeclaims/status"] 65 | verbs = var.kube_version < 1.16 ? ["get", "list", "watch", "update"] : ["get", "list", "watch", "update", "patch"] 66 | } 67 | 68 | rule { 69 | api_groups = ["storage.k8s.io"] 70 | resources = ["storageclasses"] 71 | verbs = ["get", "list", "watch"] 72 | } 73 | 74 | rule { 75 | api_groups = [""] 76 | resources = ["events"] 77 | verbs = ["list", "watch", "create", "update", "patch"] 78 | } 79 | 80 | rule { 81 | api_groups = ["snapshot.storage.k8s.io"] 82 | resources = ["volumesnapshots"] 83 | verbs = ["get", "list"] 84 | } 85 | 86 | rule { 87 | api_groups = ["snapshot.storage.k8s.io"] 88 | resources = ["volumesnapshotcontents"] 89 | verbs = ["get", "list"] 90 | } 91 | 92 | # node 93 | rule { 94 | api_groups = [""] 95 | resources = ["events"] 96 | verbs = ["get", "list", "watch", "create", "update", "patch"] 97 | } 98 | } 99 | 100 | 101 | # Create cluster role binding 102 | resource "kubernetes_cluster_role_binding" "hcloud_csi" { 103 | metadata { 104 | name = "hcloud-csi" 105 | } 106 | 107 | role_ref { 108 | api_group = "rbac.authorization.k8s.io" 109 | kind = "ClusterRole" 110 | name = "hcloud-csi" 111 | } 112 | 113 | subject { 114 | kind = "ServiceAccount" 115 | name = "hcloud-csi" 116 | namespace = "kube-system" 117 | } 118 | } -------------------------------------------------------------------------------- /locals.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | 3 | IMAGE_HCLOUD_CSI_DRIVER_LEGACY = "hetznercloud/hcloud-csi-driver:1.1.5" 4 | IMAGE_HCLOUD_CSI_DRIVER = "hetznercloud/hcloud-csi-driver:1.5.3" 5 | 6 | IMAGE_CSI_NODE_DRIVER_REGISTRAR_LEGACY = "quay.io/k8scsi/csi-node-driver-registrar:v1.1.0" 7 | IMAGE_CSI_NODE_DRIVER_REGISTRAR = "quay.io/k8scsi/csi-node-driver-registrar:v1.3.0" 8 | 9 | IMAGE_CSI_PROVISIONER_LEGACY = "quay.io/k8scsi/csi-provisioner:v1.2.1" 10 | IMAGE_CSI_PROVISIONER = "quay.io/k8scsi/csi-provisioner:v1.6.0" 11 | 12 | IMAGE_CSI_ATTACHER_LEGACY = "quay.io/k8scsi/csi-attacher:v1.1.1" 13 | IMAGE_CSI_ATTACHER = "quay.io/k8scsi/csi-attacher:v2.2.0" 14 | 15 | IMAGE_LIVENESSPROBE = "quay.io/k8scsi/livenessprobe:v1.1.0" 16 | 17 | IMAGE_CSI_RESIZER = "quay.io/k8scsi/csi-resizer:v0.3.0" 18 | 19 | 20 | } -------------------------------------------------------------------------------- /metrics.tf: -------------------------------------------------------------------------------- 1 | # Create Metrics Controller Service 2 | resource "kubernetes_service" "hcloud_csi_controller_metrics" { 3 | count = var.kube_version < 1.16 ? 0 : 1 4 | 5 | metadata { 6 | name = "hcloud-csi-controller-metrics" 7 | namespace = "kube-system" 8 | labels = { 9 | app = "hcloud-csi" 10 | } 11 | } 12 | 13 | spec { 14 | selector = { 15 | app = "hcloud-csi-controller" 16 | } 17 | 18 | port { 19 | name = "metrics" 20 | port = 9189 21 | target_port = "metrics" 22 | } 23 | } 24 | } 25 | 26 | # Create Metrics Node Service 27 | resource "kubernetes_service" "hcloud_csi_node_metrics" { 28 | count = var.kube_version < 1.16 ? 0 : 1 29 | 30 | metadata { 31 | name = "hcloud-csi-node-metrics" 32 | namespace = "kube-system" 33 | labels = { 34 | app = "hcloud-csi" 35 | } 36 | } 37 | 38 | spec { 39 | selector = { 40 | app = "hcloud-csi" 41 | } 42 | 43 | port { 44 | name = "metrics" 45 | port = 9189 46 | target_port = "metrics" 47 | } 48 | } 49 | } -------------------------------------------------------------------------------- /node.tf: -------------------------------------------------------------------------------- 1 | # Create DaemonSet 2 | resource "kubernetes_daemonset" "hcloud_csi_node" { 3 | metadata { 4 | name = "hcloud-csi-node" 5 | namespace = "kube-system" 6 | labels = { 7 | app = "hcloud-csi" 8 | } 9 | } 10 | 11 | spec { 12 | selector { 13 | match_labels = { 14 | app = "hcloud-csi" 15 | } 16 | } 17 | 18 | template { 19 | metadata { 20 | labels = { 21 | app = "hcloud-csi" 22 | } 23 | } 24 | 25 | spec { 26 | dynamic "toleration" { 27 | for_each = var.kube_version < 1.16 ? [] : [1] 28 | 29 | content { 30 | effect = "NoExecute" 31 | operator = "Exists" 32 | } 33 | } 34 | 35 | dynamic "toleration" { 36 | for_each = var.kube_version < 1.16 ? [] : [1] 37 | 38 | content { 39 | effect = "NoSchedule" 40 | operator = "Exists" 41 | } 42 | } 43 | 44 | dynamic "toleration" { 45 | for_each = var.kube_version < 1.16 ? [] : [1] 46 | 47 | content { 48 | key = "CriticalAddonsOnly" 49 | operator = "Exists" 50 | } 51 | } 52 | 53 | automount_service_account_token = true # override Terraform's default false - https://github.com/kubernetes/kubernetes/issues/27973#issuecomment-462185284 54 | service_account_name = "hcloud-csi" 55 | host_network = var.kube_version < 1.16 ? true : null 56 | 57 | container { 58 | name = "csi-node-driver-registrar" 59 | image = var.kube_version < 1.16 ? local.IMAGE_CSI_NODE_DRIVER_REGISTRAR_LEGACY : local.IMAGE_CSI_NODE_DRIVER_REGISTRAR 60 | 61 | args = [ 62 | "--v=5", 63 | "--csi-address=/csi/csi.sock", 64 | "--kubelet-registration-path=/var/lib/kubelet/plugins/csi.hetzner.cloud/csi.sock", 65 | ] 66 | 67 | env { 68 | name = "KUBE_NODE_NAME" 69 | value_from { 70 | field_ref { 71 | api_version = "v1" 72 | field_path = "spec.nodeName" 73 | } 74 | } 75 | } 76 | 77 | volume_mount { 78 | name = "plugin-dir" 79 | mount_path = "/csi" 80 | } 81 | 82 | volume_mount { 83 | name = "registration-dir" 84 | mount_path = "/registration" 85 | } 86 | 87 | security_context { 88 | privileged = true 89 | } 90 | 91 | } 92 | 93 | container { 94 | name = "hcloud-csi-driver" 95 | image = var.kube_version < 1.16 ? "hetznercloud/hcloud-csi-driver:1.1.5" : "hetznercloud/hcloud-csi-driver:1.5.1" 96 | image_pull_policy = "Always" 97 | 98 | env { 99 | name = "CSI_ENDPOINT" 100 | value = "unix:///csi/csi.sock" 101 | } 102 | 103 | dynamic "env" { 104 | for_each = var.kube_version < 1.16 ? [] : [1] 105 | 106 | content { 107 | name = "METRICS_ENDPOINT" 108 | value = "0.0.0.0:9189" 109 | } 110 | } 111 | 112 | env { 113 | name = "HCLOUD_TOKEN" 114 | value_from { 115 | secret_key_ref { 116 | name = "hcloud-csi" 117 | key = "token" 118 | } 119 | } 120 | } 121 | env { 122 | name = "KUBE_NODE_NAME" 123 | value_from { 124 | field_ref { 125 | api_version = "v1" 126 | field_path = "spec.nodeName" 127 | } 128 | } 129 | } 130 | volume_mount { 131 | name = "kubelet-dir" 132 | mount_path = "/var/lib/kubelet" 133 | mount_propagation = "Bidirectional" 134 | } 135 | 136 | volume_mount { 137 | name = "plugin-dir" 138 | mount_path = "/csi" 139 | } 140 | 141 | volume_mount { 142 | name = "device-dir" 143 | mount_path = "/dev" 144 | } 145 | 146 | security_context { 147 | privileged = true 148 | } 149 | 150 | dynamic "port" { 151 | for_each = var.kube_version < 1.16 ? [] : [1] 152 | 153 | content { 154 | container_port = 9189 155 | name = "metrics" 156 | } 157 | } 158 | 159 | dynamic "port" { 160 | for_each = var.kube_version < 1.16 ? [] : [1] 161 | 162 | content { 163 | container_port = 9808 164 | name = "healthz" 165 | protocol = "TCP" 166 | } 167 | } 168 | 169 | dynamic "liveness_probe" { 170 | for_each = var.kube_version < 1.16 ? [] : [1] 171 | 172 | content { 173 | failure_threshold = 5 174 | http_get { 175 | path = "/healthz" 176 | port = "healthz" 177 | } 178 | 179 | initial_delay_seconds = 10 180 | timeout_seconds = 3 181 | period_seconds = 2 182 | } 183 | } 184 | 185 | } 186 | 187 | dynamic "container" { 188 | for_each = var.kube_version < 1.16 ? [] : [1] 189 | 190 | content { 191 | name = "liveness-probe" 192 | image = local.IMAGE_LIVENESSPROBE 193 | image_pull_policy = "Always" 194 | 195 | args = [ 196 | "--csi-address=/csi/csi.sock", 197 | ] 198 | 199 | volume_mount { 200 | name = "plugin-dir" 201 | mount_path = "/csi" 202 | } 203 | } 204 | } 205 | 206 | volume { 207 | name = "kubelet-dir" 208 | host_path { 209 | path = "/var/lib/kubelet" 210 | type = "Directory" 211 | } 212 | } 213 | 214 | volume { 215 | name = "plugin-dir" 216 | host_path { 217 | path = "/var/lib/kubelet/plugins/csi.hetzner.cloud/" 218 | type = "DirectoryOrCreate" 219 | } 220 | } 221 | 222 | volume { 223 | name = "registration-dir" 224 | host_path { 225 | path = "/var/lib/kubelet/plugins_registry/" 226 | type = "Directory" 227 | } 228 | } 229 | 230 | volume { 231 | name = "device-dir" 232 | host_path { 233 | path = "/dev" 234 | type = "Directory" 235 | } 236 | } 237 | 238 | } 239 | } 240 | } 241 | } -------------------------------------------------------------------------------- /output.tf: -------------------------------------------------------------------------------- 1 | output "hcloud_csi_driver_name" { 2 | description = "The Name of the Hetzner Cloud CSI driver" 3 | value = kubernetes_csi_driver.csi_driver.metadata[0].name 4 | } -------------------------------------------------------------------------------- /storage_class.tf: -------------------------------------------------------------------------------- 1 | # Create Storage Class 2 | resource "kubernetes_storage_class" "hcloud_volumes" { 3 | metadata { 4 | name = "hcloud-volumes" 5 | annotations = { 6 | "storageclass.kubernetes.io/is-default-class" = true 7 | } 8 | } 9 | storage_provisioner = "csi.hetzner.cloud" 10 | volume_binding_mode = "WaitForFirstConsumer" 11 | allow_volume_expansion = var.kube_version < 1.16 ? null : true 12 | } -------------------------------------------------------------------------------- /token.tf: -------------------------------------------------------------------------------- 1 | # Create secret containing Hetzner Cloud API token 2 | resource "kubernetes_secret" "hcloud_token" { 3 | metadata { 4 | name = "hcloud-csi" 5 | namespace = "kube-system" 6 | } 7 | 8 | data = { 9 | token = var.hcloud_token 10 | } 11 | } -------------------------------------------------------------------------------- /variables.tf: -------------------------------------------------------------------------------- 1 | # Required configuration variables 2 | variable "hcloud_token" { 3 | description = "Hetzner Cloud API Token" 4 | } 5 | 6 | # Optional configuration 7 | variable "kube_version" { 8 | default = 1.18 9 | type = number 10 | description = "Kuberenetes Cluster Version e.g. 1.18" 11 | } --------------------------------------------------------------------------------