├── .github
    └── workflows
    │   ├── commitsar.yaml
    │   └── release-please.yml
├── .gitignore
├── CHANGELOG.md
├── README.md
├── kubeconfig.sh
├── main.tf
├── modules
    └── worker_group
    │   ├── main.tf
    │   ├── variables.tf
    │   └── versions.tf
├── outputs.tf
├── variables.tf
└── versions.tf


/.github/workflows/commitsar.yaml:
--------------------------------------------------------------------------------
 1 | ---
 2 | name: "Linters"
 3 | 
 4 | on: ["pull_request"]
 5 | 
 6 | jobs:
 7 |   validate-commits:
 8 |     runs-on: "ubuntu-latest"
 9 |     steps:
10 |       - name: "Check out code into the Go module directory"
11 |         uses: "actions/checkout@v1"
12 |       - name: "Commitsar check"
13 |         uses: "docker://aevea/commitsar"
14 | 


--------------------------------------------------------------------------------
/.github/workflows/release-please.yml:
--------------------------------------------------------------------------------
 1 | ---
 2 | on:
 3 |   push:
 4 |     branches:
 5 |       - "master"
 6 | 
 7 | name: "release-please"
 8 | 
 9 | jobs:
10 |   release-please:
11 |     runs-on: "ubuntu-latest"
12 |     steps:
13 |       - uses: "GoogleCloudPlatform/release-please-action@v2"
14 |         with:
15 |           release-type: "terraform-module"
16 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | .terraform/
2 | 


--------------------------------------------------------------------------------
/CHANGELOG.md:
--------------------------------------------------------------------------------
 1 | # Changelog
 2 | 
 3 | ### [1.0.1](https://www.github.com/camptocamp/terraform-docker-k3s/compare/v1.0.0...v1.0.1) (2021-08-25)
 4 | 
 5 | 
 6 | ### Bug Fixes
 7 | 
 8 | * set worker_groups type ([4a384f0](https://www.github.com/camptocamp/terraform-docker-k3s/commit/4a384f0a2805fadfb4450d94c434cb767037c193))
 9 | 
10 | ## [1.0.0](https://www.github.com/camptocamp/terraform-docker-k3s/compare/v0.11.1...v1.0.0) (2021-08-24)
11 | 
12 | 
13 | ### ⚠ BREAKING CHANGES
14 | 
15 | * allow to have multiple worker groups
16 | 
17 | ### Features
18 | 
19 | * allow to have multiple worker groups ([b9bd484](https://www.github.com/camptocamp/terraform-docker-k3s/commit/b9bd484d2589c3a4c33e6ac7896d9cc73b536fe8))
20 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # terraform-docker-k3s
2 | 
3 | A terraform module to create a K3s cluster on Docker. Available through the [Terraform registry](https://registry.terraform.io/modules/camptocamp/k3s/docker).
4 | Inspired by and adapted from [this docker composition](https://github.com/k3s-io/k3s/blob/master/docker-compose.yml).
5 | 


--------------------------------------------------------------------------------
/kubeconfig.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/sh
 2 | 
 3 | set -e
 4 | 
 5 | eval "$(jq -r '@sh "CONTAINER_NAME=\(.container_name) CONTAINER_IP_ADDRESS=\(.container_ip_address)"')"
 6 | 
 7 | kubeconfig=$(docker exec "$CONTAINER_NAME" sed -e "s/127.0.0.1/$CONTAINER_IP_ADDRESS/" /etc/rancher/k3s/k3s.yaml)
 8 | 
 9 | jq -n --arg kubeconfig "$kubeconfig" '{"kubeconfig":$kubeconfig}'
10 | 


--------------------------------------------------------------------------------
/main.tf:
--------------------------------------------------------------------------------
  1 | locals {
  2 |   network_name     = var.network_name == null ? docker_network.k3s.0.name : var.network_name
  3 |   cluster_endpoint = coalesce(var.cluster_endpoint, docker_container.k3s_server.ip_address)
  4 |   server_config    = var.cluster_endpoint == null ? var.server_config : concat(["--tls-san", var.cluster_endpoint], var.server_config)
  5 | }
  6 | 
  7 | # Mimics https://github.com/rancher/k3s/blob/master/docker-compose.yml
  8 | resource "docker_volume" "k3s_server" {
  9 |   name = "k3s-server-${var.cluster_name}"
 10 | }
 11 | 
 12 | resource "docker_network" "k3s" {
 13 |   count = var.network_name == null ? 1 : 0
 14 | 
 15 |   name = "k3s-${var.cluster_name}"
 16 | }
 17 | 
 18 | resource "docker_image" "registry" {
 19 |   name         = "registry:2"
 20 |   keep_locally = true
 21 | }
 22 | 
 23 | resource "docker_container" "registry_mirror" {
 24 |   for_each = var.registry_mirrors
 25 | 
 26 |   image = docker_image.registry.latest
 27 |   name  = format("registry-%s-%s", replace(each.key, ".", "-"), var.cluster_name)
 28 | 
 29 |   restart = var.restart
 30 | 
 31 |   networks_advanced {
 32 |     name = local.network_name
 33 |   }
 34 | 
 35 |   env = each.value
 36 | 
 37 |   mounts {
 38 |     target = "/var/lib/registry"
 39 |     source = "registry"
 40 |     type   = "volume"
 41 |   }
 42 | }
 43 | 
 44 | resource "local_file" "registries_yaml" {
 45 |   content  = <<EOF
 46 | ---
 47 | mirrors:
 48 | %{for key, registry_mirror in docker_container.registry_mirror~}
 49 |   ${key}:
 50 |     endpoint:
 51 |       - http://${registry_mirror.ip_address}:5000
 52 | %{endfor~}
 53 | EOF
 54 |   filename = "${path.module}/registries.yaml"
 55 | }
 56 | 
 57 | resource "docker_image" "k3s" {
 58 |   name         = "rancher/k3s:${var.k3s_version}"
 59 |   keep_locally = true
 60 | }
 61 | 
 62 | resource "docker_volume" "k3s_server_kubelet" {
 63 |   count = var.csi_support ? 1 : 0
 64 |   name  = "k3s-server-kubelet-${var.cluster_name}"
 65 | }
 66 | 
 67 | resource "docker_container" "k3s_server" {
 68 |   image = docker_image.k3s.latest
 69 |   name  = "k3s-server-${var.cluster_name}"
 70 | 
 71 |   restart = var.restart
 72 | 
 73 |   command = concat(["server"], local.server_config)
 74 | 
 75 |   privileged = true
 76 | 
 77 |   networks_advanced {
 78 |     name = local.network_name
 79 |   }
 80 | 
 81 |   env = [
 82 |     "K3S_TOKEN=${random_password.k3s_token.result}",
 83 |   ]
 84 | 
 85 |   mounts {
 86 |     target = "/run"
 87 |     type   = "tmpfs"
 88 |   }
 89 | 
 90 |   mounts {
 91 |     target = "/var/run"
 92 |     type   = "tmpfs"
 93 |   }
 94 | 
 95 |   mounts {
 96 |     target = "/etc/rancher/k3s/registries.yaml"
 97 |     source = abspath(local_file.registries_yaml.filename)
 98 |     type   = "bind"
 99 |   }
100 | 
101 |   mounts {
102 |     target = "/var/lib/rancher/k3s"
103 |     source = docker_volume.k3s_server.name
104 |     type   = "volume"
105 |   }
106 | 
107 |   dynamic "mounts" {
108 |     for_each = var.csi_support ? [1] : []
109 | 
110 |     content {
111 |       target = "/var/lib/kubelet"
112 |       source = docker_volume.k3s_server_kubelet[0].mountpoint
113 |       type   = "bind"
114 | 
115 |       bind_options {
116 |         propagation = "rshared"
117 |       }
118 |     }
119 |   }
120 | 
121 |   dynamic "ports" {
122 |     for_each = var.server_ports
123 | 
124 |     content {
125 |       internal = ports.value.internal
126 |       external = ports.value.external
127 |       ip       = ports.value.ip
128 |       protocol = ports.value.protocol
129 |     }
130 |   }
131 | }
132 | 
133 | resource "null_resource" "destroy_k3s_server" {
134 |   triggers = {
135 |     server_container_name = docker_container.k3s_server.name
136 |     hostname              = docker_container.k3s_server.hostname
137 |   }
138 | 
139 |   provisioner "local-exec" {
140 |     when    = destroy
141 |     command = "docker exec ${self.triggers.server_container_name} kubectl drain ${self.triggers.hostname} --delete-emptydir-data --disable-eviction --ignore-daemonsets --grace-period=60"
142 |   }
143 | }
144 | 
145 | module "worker_groups" {
146 |   for_each = var.worker_groups
147 | 
148 |   source = "./modules/worker_group"
149 | 
150 |   k3s_version           = var.k3s_version
151 |   containers_name       = format("%s-%s", var.cluster_name, each.key)
152 |   restart               = var.restart
153 |   network_name          = local.network_name
154 |   k3s_token             = random_password.k3s_token.result
155 |   k3s_url               = format("https://%s:6443", docker_container.k3s_server.ip_address)
156 |   registries_yaml       = abspath(local_file.registries_yaml.filename)
157 |   server_container_name = docker_container.k3s_server.name
158 | 
159 |   node_count  = each.value.node_count
160 |   node_labels = each.value.node_labels
161 |   node_taints = each.value.node_taints
162 | }
163 | 
164 | resource "random_password" "k3s_token" {
165 |   length = 16
166 | }
167 | 
168 | resource "null_resource" "wait_for_cluster" {
169 |   depends_on = [
170 |     docker_container.k3s_server,
171 |   ]
172 | 
173 |   provisioner "local-exec" {
174 |     command     = var.wait_for_cluster_cmd
175 |     interpreter = var.wait_for_cluster_interpreter
176 |     environment = {
177 |       ENDPOINT = format("https://%s:6443", local.cluster_endpoint)
178 |     }
179 |   }
180 | }
181 | 
182 | data "external" "kubeconfig" {
183 |   program = ["sh", "${path.module}/kubeconfig.sh"]
184 | 
185 |   query = {
186 |     container_name       = docker_container.k3s_server.name
187 |     container_ip_address = local.cluster_endpoint
188 |   }
189 | 
190 |   depends_on = [
191 |     null_resource.wait_for_cluster,
192 |   ]
193 | }
194 | 


--------------------------------------------------------------------------------
/modules/worker_group/main.tf:
--------------------------------------------------------------------------------
 1 | resource "docker_image" "k3s" {
 2 |   name         = "rancher/k3s:${var.k3s_version}"
 3 |   keep_locally = true
 4 | }
 5 | 
 6 | resource "docker_volume" "kubelet" {
 7 |   count = var.csi_support ? var.node_count : 0
 8 | 
 9 |   name = "k3s-agent-kubelet-${var.containers_name}-${count.index}"
10 | }
11 | 
12 | resource "docker_container" "this" {
13 |   count = var.node_count
14 | 
15 |   image = docker_image.k3s.latest
16 |   name  = "k3s-agent-${var.containers_name}-${count.index}"
17 | 
18 |   restart = var.restart
19 | 
20 |   command = flatten(
21 |     concat(
22 |       ["agent"],
23 |       [for label in var.node_labels : ["--node-label", label]],
24 |       [for taint in var.node_taints : ["--node-taint", taint]],
25 |     )
26 |   )
27 | 
28 |   privileged = true
29 | 
30 |   networks_advanced {
31 |     name = var.network_name
32 |   }
33 | 
34 |   env = [
35 |     "K3S_TOKEN=${var.k3s_token}",
36 |     "K3S_URL=${var.k3s_url}",
37 |   ]
38 | 
39 |   mounts {
40 |     target = "/run"
41 |     type   = "tmpfs"
42 |   }
43 | 
44 |   mounts {
45 |     target = "/var/run"
46 |     type   = "tmpfs"
47 |   }
48 | 
49 |   mounts {
50 |     target = "/etc/rancher/k3s/registries.yaml"
51 |     source = var.registries_yaml
52 |     type   = "bind"
53 |   }
54 | 
55 |   dynamic "mounts" {
56 |     for_each = var.csi_support ? [1] : []
57 | 
58 |     content {
59 |       target = "/var/lib/kubelet"
60 |       source = docker_volume.kubelet[count.index].mountpoint
61 |       type   = "bind"
62 | 
63 |       bind_options {
64 |         propagation = "rshared"
65 |       }
66 |     }
67 |   }
68 | }
69 | 
70 | resource "null_resource" "destroy" {
71 |   count = var.node_count
72 | 
73 |   triggers = {
74 |     server_container_name = var.server_container_name
75 |     hostname              = docker_container.this[count.index].hostname
76 |   }
77 | 
78 |   provisioner "local-exec" {
79 |     when    = destroy
80 |     command = "docker exec ${self.triggers.server_container_name} kubectl drain ${self.triggers.hostname} --delete-emptydir-data --disable-eviction --ignore-daemonsets --grace-period=60"
81 |   }
82 | }
83 | 


--------------------------------------------------------------------------------
/modules/worker_group/variables.tf:
--------------------------------------------------------------------------------
 1 | variable "k3s_version" {
 2 |   description = "The K3s version to use"
 3 |   type        = string
 4 | }
 5 | 
 6 | variable "containers_name" {
 7 |   description = "The name of the containers of this worker group"
 8 |   type        = string
 9 | }
10 | 
11 | variable "network_name" {
12 |   description = "Docker network to use. Creates a new one if null."
13 |   type        = string
14 | }
15 | 
16 | variable "k3s_token" {
17 |   description = "The K3S_TOKEN to use"
18 |   type        = string
19 | }
20 | 
21 | variable "k3s_url" {
22 |   description = "The K3S_URL to use"
23 |   type        = string
24 | }
25 | 
26 | variable "registries_yaml" {
27 |   description = "The path of the registries.yaml file to use"
28 |   type        = string
29 | }
30 | 
31 | variable "server_container_name" {
32 |   description = "The name of the K3s server's container"
33 |   type        = string
34 | }
35 | 
36 | variable "node_count" {
37 |   description = "Number of nodes to deploy"
38 |   type        = number
39 | }
40 | 
41 | variable "node_labels" {
42 |   description = "Registering and starting kubelet with set of labels"
43 |   type        = list(string)
44 |   default     = []
45 | }
46 | 
47 | variable "node_taints" {
48 |   description = "Registering kubelet with set of taints"
49 |   type        = list(string)
50 |   default     = []
51 | }
52 | 
53 | variable "restart" {
54 |   description = "Restart policy for the cluster."
55 |   type        = string
56 |   default     = "unless-stopped"
57 | }
58 | 
59 | variable "csi_support" {
60 |   description = "Container Storage Interface requires /var/lib/kubelet to be mounted with rshared propagation, that can cause some issues."
61 |   type        = bool
62 |   default     = false
63 | }
64 | 


--------------------------------------------------------------------------------
/modules/worker_group/versions.tf:
--------------------------------------------------------------------------------
 1 | terraform {
 2 |   required_providers {
 3 |     docker = {
 4 |       source  = "kreuzwerker/docker"
 5 |       version = "~> 2.8"
 6 |     }
 7 |   }
 8 | 
 9 |   required_version = ">= 0.14"
10 | }
11 | 


--------------------------------------------------------------------------------
/outputs.tf:
--------------------------------------------------------------------------------
 1 | output "cluster_endpoint" {
 2 |   value = format("https://%s:6443", local.cluster_endpoint)
 3 | }
 4 | 
 5 | output "ingress_ip_address" {
 6 |   value = docker_container.k3s_server.ip_address
 7 | }
 8 | 
 9 | output "kubeconfig" {
10 |   description = "kubectl config file contents for this K3s cluster."
11 |   value       = data.external.kubeconfig.result.kubeconfig
12 |   sensitive   = true
13 | }
14 | 


--------------------------------------------------------------------------------
/variables.tf:
--------------------------------------------------------------------------------
  1 | variable "cluster_name" {
  2 |   description = "The name of the Kubernetes cluster to create."
  3 |   type        = string
  4 | }
  5 | 
  6 | variable "k3s_version" {
  7 |   description = "The K3s version to use"
  8 |   type        = string
  9 |   default     = "v1.19.2-k3s1"
 10 | }
 11 | 
 12 | variable "server_config" {
 13 |   description = "The command line flags passed to the K3s server"
 14 |   type        = list(string)
 15 |   default     = []
 16 | }
 17 | 
 18 | variable "worker_groups" {
 19 |   description = "A map defining worker group configurations"
 20 | 
 21 |   type = map(object({
 22 |     node_count  = number
 23 |     node_labels = list(string)
 24 |     node_taints = list(string)
 25 |   }))
 26 | 
 27 |   default = {
 28 |     "default" = {
 29 |       node_count  = 2
 30 |       node_labels = []
 31 |       node_taints = []
 32 |     },
 33 |   }
 34 | }
 35 | 
 36 | variable "network_name" {
 37 |   description = "Docker network to use. Creates a new one if null."
 38 |   type        = string
 39 |   default     = null
 40 | }
 41 | 
 42 | variable "wait_for_cluster_cmd" {
 43 |   description = "Custom local-exec command to execute for determining if the eks cluster is healthy. Cluster endpoint will be available as an environment variable called ENDPOINT"
 44 |   type        = string
 45 |   default     = "for i in `seq 1 60`; do if `command -v wget > /dev/null`; then wget --no-check-certificate -O - -q $ENDPOINT/ping >/dev/null && exit 0 || true; else curl -k -s $ENDPOINT/ping >/dev/null && exit 0 || true;fi; sleep 5; done; echo TIMEOUT && exit 1"
 46 | }
 47 | 
 48 | variable "wait_for_cluster_interpreter" {
 49 |   description = "Custom local-exec command line interpreter for the command to determining if the eks cluster is healthy."
 50 |   type        = list(string)
 51 |   default     = ["/bin/sh", "-c"]
 52 | }
 53 | 
 54 | variable "csi_support" {
 55 |   description = "Container Storage Interface requires /var/lib/kubelet to be mounted with rshared propagation, that can cause some issues."
 56 |   type        = bool
 57 |   default     = false
 58 | }
 59 | 
 60 | variable "registry_mirrors" {
 61 |   default = {
 62 |     "docker.io" = [
 63 |       "REGISTRY_PROXY_REMOTEURL=https://registry-1.docker.io",
 64 |     ],
 65 |     "quay.io" = [
 66 |       "REGISTRY_PROXY_REMOTEURL=https://quay.io/repository",
 67 |       "REGISTRY_COMPATIBILITY_SCHEMA1_ENABLED=true",
 68 |     ],
 69 |     "gcr.io" = [
 70 |       "REGISTRY_PROXY_REMOTEURL=https://gcr.io",
 71 |     ],
 72 |     "us.gcr.io" = [
 73 |       "REGISTRY_PROXY_REMOTEURL=https://us.gcr.io",
 74 |     ],
 75 |   }
 76 | }
 77 | 
 78 | variable "restart" {
 79 |   description = "Restart policy for the cluster."
 80 |   default     = "unless-stopped"
 81 | }
 82 | 
 83 | variable "server_ports" {
 84 |   description = "Port mappings of the server container."
 85 |   default     = []
 86 | 
 87 |   type = set(object({
 88 |     internal = number
 89 |     external = optional(number)
 90 |     ip       = optional(string)
 91 |     protocol = optional(string)
 92 |   }))
 93 | }
 94 | 
 95 | variable "cluster_endpoint" {
 96 |   description = "The api endpoint, when empty it's the container's IP."
 97 |   type        = string
 98 |   default     = null
 99 | }
100 | 


--------------------------------------------------------------------------------
/versions.tf:
--------------------------------------------------------------------------------
 1 | terraform {
 2 |   required_providers {
 3 |     local = {
 4 |       source  = "hashicorp/local"
 5 |       version = "~> 2.0"
 6 |     }
 7 |     null = {
 8 |       source  = "hashicorp/null"
 9 |       version = "~> 3.0"
10 |     }
11 |     random = {
12 |       source  = "hashicorp/random"
13 |       version = "~> 3.0"
14 |     }
15 |     docker = {
16 |       source  = "kreuzwerker/docker"
17 |       version = "~> 2.8"
18 |     }
19 |   }
20 | 
21 |   required_version = ">= 0.14"
22 | 
23 |   experiments = [
24 |     module_variable_optional_attrs,
25 |   ]
26 | }
27 | 


--------------------------------------------------------------------------------