├── .circleci └── config.yml ├── .gitignore ├── LICENSE ├── README.md ├── files └── 10-kubeadm.conf ├── install-calico.tf ├── main.tf ├── outputs.tf ├── scripts ├── bootstrap.sh ├── copy-kubeadm-token.sh ├── master.sh └── node.sh ├── variables.tf └── versions.tf /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | jobs: 4 | shellcheck: 5 | docker: 6 | - image: nlknguyen/alpine-shellcheck:v0.4.6 7 | steps: 8 | - checkout 9 | - run: 10 | name: Check Scripts 11 | command: | 12 | find . -type f -name '*.sh' | wc -l 13 | find . -type f -name '*.sh' | xargs shellcheck --external-sources 14 | validate_terraform: 15 | docker: 16 | - image: hashicorp/terraform:0.11.7 17 | steps: 18 | - checkout 19 | - run: 20 | name: Validate Terraform Formatting 21 | command: "[ -z \"$(terraform fmt -write=false)\" ] || { terraform fmt -write=false -diff; exit 1;}" 22 | 23 | workflows: 24 | version: 2 25 | validate: 26 | jobs: 27 | - shellcheck 28 | - validate_terraform 29 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .terraform.* 2 | .terraform/ 3 | terraform.tfstate* 4 | terraform.tfvars 5 | .auto.tfvars 6 | .DS_Store 7 | secrets/ 8 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Niclas Mietz 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Terraform Kubernetes on Hetzner Cloud 2 | 3 | This repository will help to setup an opionated Kubernetes Cluster with [kubeadm](https://kubernetes.io/docs/setup/independent/create-cluster-kubeadm/) on [Hetzner Cloud](https://www.hetzner.com/cloud?country=us). 4 | 5 | ## Usage 6 | 7 | ``` 8 | $ git clone https://github.com/solidnerd/terraform-k8s-hcloud.git 9 | $ terraform init 10 | $ terraform apply 11 | ``` 12 | 13 | ## Example 14 | 15 | ``` 16 | $ terraform init 17 | $ terraform apply 18 | $ KUBECONFIG=secrets/admin.conf kubectl get nodes 19 | $ KUBECONFIG=secrets/admin.conf kubectl apply -f https://docs.projectcalico.org/archive/v3.15/manifests/calico.yaml 20 | $ KUBECONFIG=secrets/admin.conf kubectl get pods --namespace=kube-system -o wide 21 | $ KUBECONFIG=secrets/admin.conf kubectl run nginx --image=nginx 22 | $ KUBECONFIG=secrets/admin.conf kubectl expose deploy nginx --port=80 --type NodePort 23 | ``` 24 | 25 | ## Variables 26 | 27 | | Name | Default | Description | Required | 28 | |:-------------------------|:-------------|:----------------------------------------------------------------------------------|:--------:| 29 | | `hcloud_token` | `` |API Token that will be generated through your hetzner cloud project https://console.hetzner.cloud/projects | Yes | 30 | | `master_count` | `1` | Amount of masters that will be created | No | 31 | | `master_image` | `ubuntu-20.04` | Predefined Image that will be used to spin up the machines (Currently supported: ubuntu-20.04, ubuntu-18.04) | No | 32 | | `master_type` | `cx11` | Machine type for more types have a look at https://www.hetzner.de/cloud | No | 33 | | `node_count` | `1` | Amount of nodes that will be created | No | 34 | | `node_image` | `ubuntu-20.04` | Predefined Image that will be used to spin up the machines (Currently supported: ubuntu-20.04, ubuntu-18.04) | No | 35 | | `node_type` | `cx11` | Machine type for more types have a look at https://www.hetzner.de/cloud | No | 36 | | `ssh_private_key` | `~/.ssh/id_ed25519` | Private Key to access the machines | No | 37 | | `ssh_public_key` | `~/.ssh/id_ed25519.pub` | Public Key to authorized the access for the machines | No | 38 | | `docker_version` | `19.03` | Docker CE version that will be installed | No | 39 | | `kubernetes_version` | `1.18.6` | Kubernetes version that will be installed | No | 40 | | `feature_gates` | `` | Add your own Feature Gates for Kubeadm | No | 41 | | `calico_enabled` | `false` | Installs Calico Network Provider after the master comes up | No | 42 | 43 | All variables cloud be passed through `environment variables` or a `tfvars` file. 44 | 45 | An example for a `tfvars` file would be the following `terraform.tfvars` 46 | 47 | ```toml 48 | # terraform.tfvars 49 | hcloud_token = "" 50 | master_type = "cx21" 51 | master_count = 1 52 | node_type = "cx31" 53 | node_count = 2 54 | kubernetes_version = "1.18.6" 55 | docker_version = "19.03" 56 | ``` 57 | 58 | Or passing directly via Arguments 59 | 60 | ```console 61 | $ terraform apply \ 62 | -var hcloud_token="" \ 63 | -var docker_version=19.03 \ 64 | -var kubernetes_version=1.18.6 \ 65 | -var master_type=cx21 \ 66 | -var master_count=1 \ 67 | -var node_type=cx31 \ 68 | -var node_count=2 69 | ``` 70 | 71 | 72 | ## Contributing 73 | 74 | ### Bug Reports & Feature Requests 75 | 76 | Please use the [issue tracker](https://github.com/solidnerd/terraform-k8s-hcloud/issues) to report any bugs or file feature requests. 77 | 78 | 79 | **Tested with** 80 | 81 | - Terraform [v0.12.24](https://github.com/hashicorp/terraform/tree/v0.12.24) 82 | - provider.hcloud [v1.19.0](https://github.com/terraform-providers/terraform-provider-hcloud) 83 | -------------------------------------------------------------------------------- /files/10-kubeadm.conf: -------------------------------------------------------------------------------- 1 | [Service] 2 | Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf --cgroup-driver=cgroupfs" 3 | Environment="KUBELET_SYSTEM_PODS_ARGS=--pod-manifest-path=/etc/kubernetes/manifests" 4 | Environment="KUBELET_NETWORK_ARGS=--network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin" 5 | Environment="KUBELET_DNS_ARGS=--cluster-dns=10.96.0.10 --cluster-domain=cluster.local --resolv-conf=/run/systemd/resolve/resolv.conf" 6 | Environment="KUBELET_AUTHZ_ARGS=--authorization-mode=Webhook --client-ca-file=/etc/kubernetes/pki/ca.crt" 7 | Environment="KUBELET_CERTIFICATE_ARGS=--rotate-certificates=true --cert-dir=/var/lib/kubelet/pki" 8 | ExecStart= 9 | ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_SYSTEM_PODS_ARGS $KUBELET_NETWORK_ARGS $KUBELET_DNS_ARGS $KUBELET_AUTHZ_ARGS $KUBELET_CERTIFICATE_ARGS $KUBELET_EXTRA_ARGS 10 | -------------------------------------------------------------------------------- /install-calico.tf: -------------------------------------------------------------------------------- 1 | resource "null_resource" "calico" { 2 | count = var.calico_enabled ? 1 : 0 3 | 4 | connection { 5 | host = hcloud_server.master.0.ipv4_address 6 | private_key = file(var.ssh_private_key) 7 | } 8 | 9 | provisioner "remote-exec" { 10 | inline = ["kubectl apply -f https://docs.projectcalico.org/archive/v3.15/manifests/calico.yaml"] 11 | } 12 | 13 | depends_on = ["hcloud_server.master"] 14 | } 15 | 16 | -------------------------------------------------------------------------------- /main.tf: -------------------------------------------------------------------------------- 1 | provider "hcloud" { 2 | token = var.hcloud_token 3 | } 4 | 5 | resource "hcloud_ssh_key" "k8s_admin" { 6 | name = "k8s_admin" 7 | public_key = file(var.ssh_public_key) 8 | } 9 | 10 | resource "hcloud_server" "master" { 11 | count = var.master_count 12 | name = "master-${count.index + 1}" 13 | server_type = var.master_type 14 | image = var.master_image 15 | ssh_keys = [hcloud_ssh_key.k8s_admin.id] 16 | 17 | connection { 18 | host = self.ipv4_address 19 | type = "ssh" 20 | private_key = file(var.ssh_private_key) 21 | } 22 | 23 | provisioner "file" { 24 | source = "files/10-kubeadm.conf" 25 | destination = "/root/10-kubeadm.conf" 26 | } 27 | 28 | provisioner "file" { 29 | source = "scripts/bootstrap.sh" 30 | destination = "/root/bootstrap.sh" 31 | } 32 | 33 | provisioner "remote-exec" { 34 | inline = ["DOCKER_VERSION=${var.docker_version} KUBERNETES_VERSION=${var.kubernetes_version} bash /root/bootstrap.sh"] 35 | } 36 | 37 | provisioner "file" { 38 | source = "scripts/master.sh" 39 | destination = "/root/master.sh" 40 | } 41 | 42 | provisioner "remote-exec" { 43 | inline = ["FEATURE_GATES=${var.feature_gates} bash /root/master.sh"] 44 | } 45 | 46 | provisioner "local-exec" { 47 | command = "bash scripts/copy-kubeadm-token.sh" 48 | 49 | environment = { 50 | SSH_PRIVATE_KEY = var.ssh_private_key 51 | SSH_USERNAME = "root" 52 | SSH_HOST = hcloud_server.master[0].ipv4_address 53 | TARGET = "${path.module}/secrets/" 54 | } 55 | } 56 | } 57 | 58 | resource "hcloud_server" "node" { 59 | count = var.node_count 60 | name = "node-${count.index + 1}" 61 | server_type = var.node_type 62 | image = var.node_image 63 | depends_on = [hcloud_server.master] 64 | ssh_keys = [hcloud_ssh_key.k8s_admin.id] 65 | 66 | connection { 67 | host = self.ipv4_address 68 | type = "ssh" 69 | private_key = file(var.ssh_private_key) 70 | } 71 | 72 | provisioner "file" { 73 | source = "files/10-kubeadm.conf" 74 | destination = "/root/10-kubeadm.conf" 75 | } 76 | 77 | provisioner "file" { 78 | source = "scripts/bootstrap.sh" 79 | destination = "/root/bootstrap.sh" 80 | } 81 | 82 | provisioner "remote-exec" { 83 | inline = ["DOCKER_VERSION=${var.docker_version} KUBERNETES_VERSION=${var.kubernetes_version} bash /root/bootstrap.sh"] 84 | } 85 | 86 | provisioner "file" { 87 | source = "${path.module}/secrets/kubeadm_join" 88 | destination = "/tmp/kubeadm_join" 89 | 90 | connection { 91 | host = self.ipv4_address 92 | type = "ssh" 93 | user = "root" 94 | private_key = file(var.ssh_private_key) 95 | } 96 | } 97 | 98 | provisioner "file" { 99 | source = "scripts/node.sh" 100 | destination = "/root/node.sh" 101 | } 102 | 103 | provisioner "remote-exec" { 104 | inline = ["bash /root/node.sh"] 105 | } 106 | } 107 | 108 | -------------------------------------------------------------------------------- /outputs.tf: -------------------------------------------------------------------------------- 1 | output "node_ips" { 2 | value = [hcloud_server.node.*.ipv4_address] 3 | } 4 | 5 | output "master_ips" { 6 | value = [hcloud_server.master.*.ipv4_address] 7 | } 8 | 9 | -------------------------------------------------------------------------------- /scripts/bootstrap.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -eu 3 | DOCKER_VERSION=${DOCKER_VERSION:-} 4 | KUBERNETES_VERSION=${KUBERNETES_VERSION:-} 5 | 6 | 7 | waitforapt(){ 8 | while fuser /var/lib/apt/lists/lock >/dev/null 2>&1 ; do 9 | echo "Waiting for other software managers to finish..." 10 | sleep 1 11 | done 12 | } 13 | 14 | echo " 15 | Package: docker-ce 16 | Pin: version ${DOCKER_VERSION}.* 17 | Pin-Priority: 1000 18 | " > /etc/apt/preferences.d/docker-ce 19 | waitforapt 20 | apt-get -qq update 21 | apt-get -qq install -y \ 22 | apt-transport-https \ 23 | ca-certificates \ 24 | curl \ 25 | software-properties-common 26 | curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - 27 | add-apt-repository \ 28 | "deb [arch=amd64] https://download.docker.com/linux/ubuntu \ 29 | $(lsb_release -cs) \ 30 | stable" 31 | apt-get -qq update && apt-get -qq install -y docker-ce 32 | 33 | cat > /etc/docker/daemon.json </etc/apt/sources.list.d/kubernetes.list 43 | deb http://apt.kubernetes.io/ kubernetes-xenial main 44 | EOF 45 | 46 | echo " 47 | Package: kubelet 48 | Pin: version ${KUBERNETES_VERSION}-* 49 | Pin-Priority: 1000 50 | " > /etc/apt/preferences.d/kubelet 51 | 52 | echo " 53 | Package: kubeadm 54 | Pin: version ${KUBERNETES_VERSION}-* 55 | Pin-Priority: 1000 56 | " > /etc/apt/preferences.d/kubeadm 57 | 58 | waitforapt 59 | apt-get -qq update 60 | apt-get -qq install -y kubelet kubeadm 61 | 62 | mv -v /root/10-kubeadm.conf /etc/systemd/system/kubelet.service.d/10-kubeadm.conf 63 | 64 | 65 | systemctl daemon-reload 66 | systemctl restart kubelet 67 | -------------------------------------------------------------------------------- /scripts/copy-kubeadm-token.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -eu 3 | SSH_PRIVATE_KEY=${SSH_PRIVATE_KEY:-} 4 | SSH_USERNAME=${SSH_USERNAME:-} 5 | SSH_HOST=${SSH_HOST:-} 6 | 7 | TARGET=${TARGET:-} 8 | 9 | mkdir -p "${TARGET}" 10 | 11 | scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ 12 | -i "${SSH_PRIVATE_KEY}" \ 13 | "${SSH_USERNAME}@${SSH_HOST}:/tmp/kubeadm_join" \ 14 | "${TARGET}" 15 | 16 | scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ 17 | -i "${SSH_PRIVATE_KEY}" \ 18 | "${SSH_USERNAME}@${SSH_HOST}:/etc/kubernetes/admin.conf" \ 19 | "${TARGET}" 20 | -------------------------------------------------------------------------------- /scripts/master.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/bash 2 | set -eu 3 | 4 | # Initialize Cluster 5 | if [[ -n "$FEATURE_GATES" ]] 6 | then 7 | kubeadm init --pod-network-cidr=192.168.0.0/16 --feature-gates "$FEATURE_GATES" 8 | else 9 | kubeadm init --pod-network-cidr=192.168.0.0/16 10 | fi 11 | systemctl enable docker kubelet 12 | 13 | # used to join nodes to the cluster 14 | kubeadm token create --print-join-command > /tmp/kubeadm_join 15 | 16 | mkdir -p "$HOME/.kube" 17 | cp /etc/kubernetes/admin.conf "$HOME/.kube/config" 18 | -------------------------------------------------------------------------------- /scripts/node.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/bash 2 | set -eu 3 | 4 | eval "$(cat /tmp/kubeadm_join)" 5 | systemctl enable docker kubelet 6 | -------------------------------------------------------------------------------- /variables.tf: -------------------------------------------------------------------------------- 1 | variable "hcloud_token" { 2 | } 3 | 4 | variable "master_count" { 5 | } 6 | 7 | variable "master_image" { 8 | description = "Predefined Image that will be used to spin up the machines (Currently supported: ubuntu-20.04, ubuntu-18.04)" 9 | default = "ubuntu-20.04" 10 | } 11 | 12 | variable "master_type" { 13 | description = "For more types have a look at https://www.hetzner.de/cloud" 14 | default = "cx21" 15 | } 16 | 17 | variable "node_count" { 18 | } 19 | 20 | variable "node_image" { 21 | description = "Predefined Image that will be used to spin up the machines (Currently supported: ubuntu-20.04, ubuntu-18.04)" 22 | default = "ubuntu-20.04" 23 | } 24 | 25 | variable "node_type" { 26 | description = "For more types have a look at https://www.hetzner.de/cloud" 27 | default = "cx21" 28 | } 29 | 30 | variable "ssh_private_key" { 31 | description = "Private Key to access the machines" 32 | default = "~/.ssh/id_ed25519" 33 | } 34 | 35 | variable "ssh_public_key" { 36 | description = "Public Key to authorized the access for the machines" 37 | default = "~/.ssh/id_ed25519.pub" 38 | } 39 | 40 | variable "docker_version" { 41 | default = "19.03" 42 | } 43 | 44 | variable "kubernetes_version" { 45 | default = "1.18.6" 46 | } 47 | 48 | variable "feature_gates" { 49 | description = "Add Feature Gates e.g. 'DynamicKubeletConfig=true'" 50 | default = "" 51 | } 52 | 53 | variable "calico_enabled" { 54 | default = false 55 | } 56 | 57 | -------------------------------------------------------------------------------- /versions.tf: -------------------------------------------------------------------------------- 1 | 2 | terraform { 3 | required_version = ">= 0.12" 4 | } 5 | --------------------------------------------------------------------------------