├── .dockerignore ├── .github ├── dependabot.yml └── workflows │ ├── ocp-feature.yml │ ├── ocp-master.yml │ ├── okd-feature.yml │ └── okd-master.yml ├── .gitignore ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── ansible ├── ansible.cfg ├── playbooks │ ├── setup_haproxy.yml │ ├── setup_ignition.yml │ └── wait_for_ssh.yml └── site.yml ├── packer ├── config-2.2.0.ign ├── config-3.0.0.ign ├── hcloud-fcos.json └── hcloud-rhcos.json ├── renovate.json ├── terraform ├── cacert.tf ├── dns.tf ├── firewall.tf ├── image.tf ├── inventory.tf ├── loadbalancer.tf ├── main.tf ├── modules │ ├── hcloud_coreos │ │ ├── dns.tf │ │ ├── ignition.tf │ │ ├── main.tf │ │ ├── network.tf │ │ ├── output.tf │ │ ├── templates │ │ │ ├── ignition.ign │ │ │ └── resolv.conf │ │ ├── variables.tf │ │ ├── versions.tf │ │ └── volumes.tf │ └── hcloud_instance │ │ ├── main.tf │ │ ├── network.tf │ │ ├── output.tf │ │ ├── variables.tf │ │ ├── versions.tf │ │ └── volumes.tf ├── network.tf ├── output.tf ├── provider.tf ├── ssh_keys.tf ├── templates │ ├── cloud-init.tpl │ └── inventory.tpl ├── variables.tf └── versions.tf └── tests └── image.tests.yaml /.dockerignore: -------------------------------------------------------------------------------- 1 | *.swp 2 | env.sh 3 | install-config.yaml 4 | config 5 | ignition 6 | packer/*.ign 7 | terraform/.terraform 8 | terraform/.terraform* 9 | terraform/terraform.tfstate 10 | terraform/terraform.tfstate.backup 11 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: docker 4 | directory: "/" 5 | schedule: 6 | interval: daily 7 | time: "04:00" 8 | open-pull-requests-limit: 10 9 | ignore: 10 | - dependency-name: hashicorp/terraform 11 | versions: 12 | - 0.14.10 13 | - 0.14.5 14 | - 0.14.6 15 | - 0.14.7 16 | - 0.14.8 17 | - 0.14.9 18 | - 0.15.0 19 | - dependency-name: hashicorp/packer 20 | versions: 21 | - 1.7.0 22 | - 1.7.1 23 | - dependency-name: alpine 24 | versions: 25 | - 3.13.1 26 | - 3.13.2 27 | - 3.13.3 28 | - 3.13.4 29 | - dependency-name: alpine/helm 30 | versions: 31 | - 3.5.1 32 | - 3.5.2 33 | - 3.5.3 34 | -------------------------------------------------------------------------------- /.github/workflows/ocp-feature.yml: -------------------------------------------------------------------------------- 1 | name: Docker Build 2 | 3 | on: 4 | push: 5 | branches-ignore: 6 | - master 7 | 8 | jobs: 9 | build-ocp-feature: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v2 13 | - name: Fetch external dependencies 14 | run: make fetch DEPLOYMENT_TYPE=ocp OPENSHIFT_RELEASE=$(make latest_version DEPLOYMENT_TYPE=ocp) 15 | - name: Build the Docker image 16 | run: make build DEPLOYMENT_TYPE=ocp OPENSHIFT_RELEASE=$(make latest_version DEPLOYMENT_TYPE=ocp) 17 | - name: Test Docker image 18 | run: make test DEPLOYMENT_TYPE=ocp OPENSHIFT_RELEASE=$(make latest_version DEPLOYMENT_TYPE=ocp) 19 | -------------------------------------------------------------------------------- /.github/workflows/ocp-master.yml: -------------------------------------------------------------------------------- 1 | name: Docker Build 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | 8 | jobs: 9 | build-ocp: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v2 13 | - name: Fetch external dependencies 14 | run: make fetch DEPLOYMENT_TYPE=ocp OPENSHIFT_RELEASE=$(make latest_version DEPLOYMENT_TYPE=ocp) 15 | - name: Build the Docker image 16 | run: make build DEPLOYMENT_TYPE=ocp OPENSHIFT_RELEASE=$(make latest_version DEPLOYMENT_TYPE=ocp) 17 | - name: Test Docker image 18 | run: make test DEPLOYMENT_TYPE=ocp OPENSHIFT_RELEASE=$(make latest_version DEPLOYMENT_TYPE=ocp) 19 | - name: Login to Docker Registry 20 | run: docker login -u ${{ secrets.REGISTRY_USERNAME }} -p ${{ secrets.REGISTRY_PASSWORD }} ${{ secrets.REGISTRY_HOST }} 21 | - name: Push Docker image 22 | run: make push DEPLOYMENT_TYPE=ocp OPENSHIFT_RELEASE=$(make latest_version DEPLOYMENT_TYPE=ocp) 23 | -------------------------------------------------------------------------------- /.github/workflows/okd-feature.yml: -------------------------------------------------------------------------------- 1 | name: Docker Build 2 | 3 | on: 4 | push: 5 | branches-ignore: 6 | - master 7 | 8 | jobs: 9 | build-okd-feature: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v2 13 | - name: Fetch external dependencies 14 | run: make fetch OPENSHIFT_RELEASE=$(make latest_version) 15 | - name: Build the Docker image 16 | run: make build OPENSHIFT_RELEASE=$(make latest_version) 17 | - name: Test Docker image 18 | run: make test OPENSHIFT_RELEASE=$(make latest_version) 19 | -------------------------------------------------------------------------------- /.github/workflows/okd-master.yml: -------------------------------------------------------------------------------- 1 | name: Docker Build 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | 8 | jobs: 9 | build-okd: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v2 13 | - name: Fetch external dependencies 14 | run: make fetch OPENSHIFT_RELEASE=$(make latest_version) 15 | - name: Build the Docker image 16 | run: make build OPENSHIFT_RELEASE=$(make latest_version) 17 | - name: Test Docker image 18 | run: make test OPENSHIFT_RELEASE=$(make latest_version) 19 | - name: Login to Docker Registry 20 | run: docker login -u ${{ secrets.REGISTRY_USERNAME }} -p ${{ secrets.REGISTRY_PASSWORD }} ${{ secrets.REGISTRY_HOST }} 21 | - name: Push Docker image 22 | run: make push OPENSHIFT_RELEASE=$(make latest_version) 23 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.swp 2 | env.sh 3 | install-config.yaml 4 | inventory.ini 5 | config 6 | ignition 7 | terraform/.terraform 8 | terraform/.terraform* 9 | terraform/terraform.tfstate 10 | terraform/terraform.tfstate.* 11 | openshift-install-linux-*.tar.gz 12 | openshift-client-linux-*.tar.gz 13 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM docker.io/hashicorp/terraform:1.9.5@sha256:79336cfbc9113f41806e7b2061b913852f11d6bdbc0e188d184e6bdee40b84a7 AS terraform 2 | FROM docker.io/hashicorp/packer:1.11.2@sha256:12c441b8a3994e7df9f0e2692d9298f14c387e70bcc06139420977dbf80a137b AS packer 3 | FROM docker.io/alpine:3.20.2@sha256:0a4eaa0eecf5f8c050e5bba433f58c052be7587ee8af3e8b3910ef9ab5fbe9f5 4 | 5 | LABEL maintainer="simon@lauger.name" 6 | 7 | ARG OPENSHIFT_RELEASE 8 | ENV OPENSHIFT_RELEASE=${OPENSHIFT_RELEASE} 9 | 10 | ARG DEPLOYMENT_TYPE 11 | ENV DEPLOYMENT_TYPE=${DEPLOYMENT_TYPE} 12 | 13 | RUN apk update && \ 14 | apk add \ 15 | bash \ 16 | ca-certificates \ 17 | openssh-client \ 18 | openssl \ 19 | ansible \ 20 | make \ 21 | rsync \ 22 | curl \ 23 | git \ 24 | jq \ 25 | libc6-compat \ 26 | apache2-utils \ 27 | python3 \ 28 | py3-pip \ 29 | libvirt-client 30 | 31 | # OpenShift Installer 32 | COPY openshift-install-linux-${OPENSHIFT_RELEASE}.tar.gz . 33 | COPY openshift-client-linux-${OPENSHIFT_RELEASE}.tar.gz . 34 | 35 | RUN tar vxzf openshift-install-linux-${OPENSHIFT_RELEASE}.tar.gz openshift-install && \ 36 | tar vxzf openshift-client-linux-${OPENSHIFT_RELEASE}.tar.gz oc && \ 37 | tar vxzf openshift-client-linux-${OPENSHIFT_RELEASE}.tar.gz kubectl && \ 38 | mv openshift-install /usr/local/bin/openshift-install && \ 39 | mv oc /usr/local/bin/oc && \ 40 | mv kubectl /usr/local/bin/kubectl && \ 41 | rm openshift-install-linux-${OPENSHIFT_RELEASE}.tar.gz && \ 42 | rm openshift-client-linux-${OPENSHIFT_RELEASE}.tar.gz 43 | 44 | # External tools 45 | COPY --from=terraform /bin/terraform /usr/local/bin/terraform 46 | COPY --from=packer /bin/packer /usr/local/bin/packer 47 | 48 | # Install Packer Plugin 49 | RUN packer plugins install github.com/hashicorp/hcloud 50 | 51 | # Create workspace 52 | RUN mkdir /workspace 53 | WORKDIR /workspace 54 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright 2021 Simon Lauger 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 6 | 7 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 8 | 9 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 10 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .DEFAULT_GOAL := build 2 | 3 | # ocp 4 | OPENSHIFT_MIRROR?=https://mirror.openshift.com/pub/openshift-v4 5 | OCP_RELEASE_CHANNEL?=stable-4.13 6 | 7 | # okd 8 | OKD_MIRROR?=https://github.com/okd-project/okd/releases/download 9 | 10 | # either okd or ocp 11 | DEPLOYMENT_TYPE?=okd 12 | 13 | # fixed release version 14 | OPENSHIFT_RELEASE?=none 15 | 16 | # image name 17 | CONTAINER_NAME?=quay.io/slauger/hcloud-okd4 18 | CONTAINER_TAG?=$(OPENSHIFT_RELEASE) 19 | 20 | # coreos 21 | ifeq ($(DEPLOYMENT_TYPE),ocp) 22 | COREOS_IMAGE=rhcos 23 | else ifeq ($(DEPLOYMENT_TYPE),okd) 24 | COREOS_IMAGE=fcos 25 | else 26 | $(error installer only supports ocp or okd) 27 | endif 28 | 29 | # terraform switches 30 | BOOTSTRAP?=false 31 | MODE?=apply 32 | 33 | # openshift version 34 | .PHONY: latest_version 35 | latest_version: latest_version_$(DEPLOYMENT_TYPE) 36 | 37 | .PHONY: latest_version_okd 38 | latest_version_okd: 39 | @curl -s -H "Accept: application/vnd.github.v3+json" https://api.github.com/repos/okd-project/okd/tags | jq -j -r .[0].name 40 | 41 | .PHONY: latest_version_ocp 42 | latest_version_ocp: 43 | @curl -s https://raw.githubusercontent.com/openshift/cincinnati-graph-data/master/channels/$(OCP_RELEASE_CHANNEL).yaml | egrep '(4\.[0-9]+\.[0-9]+)' | tail -n1 | cut -d" " -f2 44 | 45 | # fetch 46 | .PHONY: fetch 47 | fetch: fetch_$(DEPLOYMENT_TYPE) 48 | 49 | .PHONY: fetch_okd 50 | fetch_okd: 51 | wget -O openshift-install-linux-$(OPENSHIFT_RELEASE).tar.gz $(OKD_MIRROR)/$(OPENSHIFT_RELEASE)/openshift-install-linux-$(OPENSHIFT_RELEASE).tar.gz 52 | wget -O openshift-client-linux-$(OPENSHIFT_RELEASE).tar.gz $(OKD_MIRROR)/$(OPENSHIFT_RELEASE)/openshift-client-linux-$(OPENSHIFT_RELEASE).tar.gz 53 | 54 | .PHONY: fetch_ocp 55 | fetch_ocp: 56 | wget -O openshift-install-linux-$(OPENSHIFT_RELEASE).tar.gz $(OPENSHIFT_MIRROR)/clients/ocp/$(OPENSHIFT_RELEASE)/openshift-install-linux-$(OPENSHIFT_RELEASE).tar.gz 57 | wget -O openshift-client-linux-$(OPENSHIFT_RELEASE).tar.gz $(OPENSHIFT_MIRROR)/clients/ocp/$(OPENSHIFT_RELEASE)/openshift-client-linux-$(OPENSHIFT_RELEASE).tar.gz 58 | 59 | .PHONY: build 60 | build: 61 | docker build --build-arg DEPLOYMENT_TYPE=$(DEPLOYMENT_TYPE) --build-arg OPENSHIFT_RELEASE=$(OPENSHIFT_RELEASE) -t $(CONTAINER_NAME):$(CONTAINER_TAG) . 62 | 63 | .PHONY: test 64 | test: 65 | docker run -v /var/run/docker.sock:/var/run/docker.sock -v $(shell pwd):/src:ro gcr.io/gcp-runtimes/container-structure-test:latest test --image $(CONTAINER_NAME):$(CONTAINER_TAG) --config /src/tests/image.tests.yaml 66 | 67 | .PHONY: push 68 | push: 69 | docker push $(CONTAINER_NAME):$(CONTAINER_TAG) 70 | 71 | .PHONY: run 72 | run: 73 | docker run -it --hostname openshift-toolbox --mount type=bind,source="$(shell pwd)",target=/workspace --mount type=bind,source="$(HOME)/.ssh,target=/root/.ssh" $(CONTAINER_NAME):$(CONTAINER_TAG) /bin/bash 74 | 75 | .PHONY: generate_manifests 76 | generate_manifests: 77 | mkdir config 78 | cp install-config.yaml config/install-config.yaml 79 | openshift-install create manifests --dir=config 80 | 81 | .PHONY: generate_ignition 82 | generate_ignition: 83 | rsync -av config/ ignition 84 | openshift-install create ignition-configs --dir=ignition 85 | 86 | .PHONY: hcloud_image 87 | hcloud_image: 88 | @if [ -z "$(HCLOUD_TOKEN)" ]; then echo "ERROR: HCLOUD_TOKEN is not set"; exit 1; fi 89 | if [ "$(DEPLOYMENT_TYPE)" == "okd" ]; then (cd packer && packer build -var fcos_url=$(shell openshift-install coreos print-stream-json | jq -r '.architectures.x86_64.artifacts.qemu.formats."qcow2.xz".disk.location') hcloud-fcos.json); fi 90 | if [ "$(DEPLOYMENT_TYPE)" == "ocp" ]; then (cd packer && packer build -var rhcos_url=$(shell openshift-install coreos print-stream-json | jq -r '.architectures.x86_64.artifacts.qemu.formats."qcow2.gz".disk.location') hcloud-rhcos.json); fi 91 | 92 | .PHONY: sign_csr 93 | sign_csr: 94 | @if [ ! -f "ignition/auth/kubeconfig" ]; then echo "ERROR: ignition/auth/kubeconfig not found"; exit 1; fi 95 | bash -c "export KUBECONFIG=$(shell pwd)/ignition/auth/kubeconfig; oc get csr -o name | xargs oc adm certificate approve || true" 96 | 97 | .PHONY: wait_bootstrap 98 | wait_bootstrap: 99 | openshift-install --dir=ignition/ wait-for bootstrap-complete --log-level=debug 100 | 101 | .PHONY: wait_completion 102 | wait_completion: 103 | openshift-install --dir=ignition/ wait-for install-complete --log-level=debug 104 | 105 | .PHONY: infrastructure 106 | infrastructure: 107 | @if [ -z "$(TF_VAR_dns_domain)" ]; then echo "ERROR: TF_VAR_dns_domain is not set"; exit 1; fi 108 | @if [ -z "$(TF_VAR_dns_zone_id)" ]; then echo "ERROR: TF_VAR_dns_zone_id is not set"; exit 1; fi 109 | @if [ -z "$(HCLOUD_TOKEN)" ]; then echo "ERROR: HCLOUD_TOKEN is not set"; exit 1; fi 110 | @if [ -z "$(CLOUDFLARE_EMAIL)" ]; then echo "ERROR: CLOUDFLARE_EMAIL is not set"; exit 1; fi 111 | (cd terraform && terraform init && terraform $(MODE) -var image=$(COREOS_IMAGE) -var bootstrap=$(BOOTSTRAP)) 112 | if [ "$(MODE)" == "apply" ]; then (cd ansible && ansible-playbook site.yml); fi 113 | 114 | .PHONY: destroy 115 | destroy: 116 | (cd terraform && terraform init && terraform destroy) 117 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![Docker Build](https://github.com/slauger/hcloud-okd4/workflows/Docker%20Build/badge.svg) 2 | 3 | 4 | # hcloud-okd4 5 | 6 | Deploy OKD4 (OpenShift) on Hetzner Cloud using Hashicorp Packer, Terraform and Ansible. 7 | 8 | ## Current status 9 | 10 | The Hetzner Cloud does not fulfill the I/O performance/latency requirements for etcd - even when using local SSDs (instead of ceph storage). This could result in different problems during the cluster bootstrap. You could check the I/O performance via `etcdctl check perf`. 11 | 12 | Because of that OpenShift on hcloud is only suitable for small test environments. Please do not use it for production clusters. 13 | 14 | ## Architecture 15 | 16 | The deployment defaults to a single node cluster. 17 | 18 | - 1x Master Node (CX41) 19 | - 1x Loadbalancer (LB11) 20 | - 1x Bootstrap Node (CX41) - deleted after cluster bootstrap 21 | - 1x Ignition Node (CX11) - deleted after cluster bootstrap 22 | 23 | ## Usage 24 | 25 | ### Build toolbox 26 | 27 | To ensure that the we have a proper build environment, we create a toolbox container first. 28 | 29 | ``` 30 | make fetch 31 | make build 32 | ``` 33 | 34 | If you do not want to build the container by your own, it is also available on [quay.io](https://quay.io/repository/slauger/hcloud-okd4). 35 | 36 | ### Run toolbox 37 | 38 | Use the following command to start the container. 39 | 40 | ``` 41 | make run 42 | ``` 43 | 44 | All the following commands will be executed inside the container. 45 | 46 | ### Set Version 47 | 48 | Set a target version of use the targets `latest_version` to fetch the latest available version. 49 | 50 | ``` 51 | export OPENSHIFT_RELEASE=$(make latest_version) 52 | ``` 53 | 54 | ### Create your install-config.yaml 55 | 56 | ``` 57 | --- 58 | apiVersion: v1 59 | baseDomain: 'example.com' 60 | metadata: 61 | name: 'okd4' 62 | compute: 63 | - hyperthreading: Enabled 64 | name: worker 65 | replicas: 0 66 | controlPlane: 67 | hyperthreading: Enabled 68 | name: master 69 | replicas: 1 70 | networking: 71 | clusterNetworks: 72 | - cidr: 10.128.0.0/14 73 | hostPrefix: 23 74 | networkType: OpenShiftSDN 75 | serviceNetwork: 76 | - 172.30.0.0/16 77 | machineCIDR: 78 | platform: 79 | none: {} 80 | pullSecret: '{"auths":{"none":{"auth": "none"}}}' 81 | sshKey: ssh-rsa AABBCC... Some_Service_User 82 | ``` 83 | 84 | ### Create cluster manifests 85 | 86 | ``` 87 | make generate_manifests 88 | ``` 89 | 90 | ### Create ignition config 91 | 92 | ``` 93 | make generate_ignition 94 | ``` 95 | 96 | ### Set required environment variables 97 | 98 | ``` 99 | # terraform variables 100 | export TF_VAR_dns_domain=okd4.example.com 101 | export TF_VAR_dns_zone_id=14758f1afd44c09b7992073ccf00b43d 102 | 103 | # credentials for hcloud 104 | export HCLOUD_TOKEN=14758f1afd44c09b7992073ccf00b43d14758f1afd44c09b7992073ccf00b43d 105 | 106 | # credentials for cloudflare 107 | export CLOUDFLARE_EMAIL=user@example.com 108 | export CLOUDFLARE_API_KEY=14758f1afd44c09b7992073ccf00b43d 109 | ``` 110 | 111 | ### Create Fedora CoreOS image 112 | 113 | Build a Fedora CoreOS hcloud image with Packer and embed the hcloud user data source (`http://169.254.169.254/hetzner/v1/userdata`). 114 | 115 | ``` 116 | make hcloud_image 117 | ``` 118 | 119 | ### Build infrastructure with Terraform 120 | 121 | ``` 122 | make infrastructure BOOTSTRAP=true 123 | ``` 124 | 125 | ### Wait for the bootstrap to complete 126 | 127 | ``` 128 | make wait_bootstrap 129 | ``` 130 | 131 | ### Cleanup bootstrap and ignition node 132 | 133 | ``` 134 | make infrastructure 135 | ``` 136 | 137 | ### Finish the installation process 138 | 139 | ``` 140 | make wait_completion 141 | ``` 142 | 143 | ### Sign Worker CSRs 144 | 145 | CSRs of the master nodes get signed by the bootstrap node automaticaly during the cluster bootstrap. CSRs from worker nodes must be signed manually. 146 | 147 | ``` 148 | make sign_csr 149 | sleep 60 150 | make sign_csr 151 | ``` 152 | 153 | This step is not necessary if you set `replicas_worker` to zero. 154 | 155 | ## Deployment of OCP 156 | 157 | It's also possible OCP (with RedHat CoreOS) instead of OKD. Just export `DEPLOYMENT_TYPE=ocp`. For example: 158 | 159 | ``` 160 | export DEPLOYMENT_TYPE=ocp 161 | export OPENSHIFT_RELEASE=4.6.35 162 | make fetch build run 163 | ``` 164 | 165 | You can also select the latest version from a specific channel via: 166 | 167 | ``` 168 | export OCP_RELEASE_CHANNEL=stable-4.11 169 | export OPENSHIFT_RELEASE=$(make latest_version) 170 | make fetch build run 171 | ``` 172 | 173 | To setup OCP a pull secret in your install-config.yaml is necessary, which could be obtained from [cloud.redhat.com](https://cloud.redhat.com/). 174 | 175 | ## Enforce Firewall rules 176 | 177 | As the Terraform module from Hetzer is currently unable to produce applied rules that contain hosts you deploy at the same time, you have to deploy them afterwards. 178 | 179 | In order to do that, you should visit your Hetzner Web Console and apply the `okd-master` firewall rule to all hosts with the label `okd.io/master: true`, the `okd-base` to the label `okd.io/node: true` and `okd-ingress` to all nodes with the `okd.io/ingress: true` label. Since terraform will ignore firewall changes, this should not interfere with your existing state. 180 | 181 | Note: This will keep hosts pingable, but isolate them complete from the internet, making the cluster only reachable through the load balancer. If you require direct SSH access, you can add another rule, that you apply nodes that allows access to port 22. 182 | 183 | ## Cloudflare API Token 184 | 185 | Checkout [this issue](https://github.com/slauger/hcloud-okd4/issues/176) to get details about how to obtain an API token for the Cloudflare API. 186 | 187 | ## Author 188 | 189 | - [slauger](https://github.com/slauger) 190 | -------------------------------------------------------------------------------- /ansible/ansible.cfg: -------------------------------------------------------------------------------- 1 | [defaults] 2 | nocows = 1 3 | inventory = ../inventory.ini 4 | roles_path = roles 5 | remote_user = root 6 | host_key_checking = False 7 | gather_facts = False 8 | hash_behaviour = merge 9 | interpreter_python = /usr/bin/python3 10 | -------------------------------------------------------------------------------- /ansible/playbooks/setup_haproxy.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: install and configure haproxy 3 | hosts: 4 | - haproxy 5 | gather_facts: no 6 | tasks: 7 | - name: install and configure haproxy 8 | include_role: 9 | name: haproxy 10 | - name: configure floating ip via netplan 11 | copy: 12 | dest: /etc/netplan/99-floating-ip.yaml 13 | content: | 14 | network: 15 | version: 2 16 | ethernets: 17 | eth0: 18 | addresses: 19 | - "{{ floating_ip }}/32" 20 | set-name: eth0 21 | - name: activate floating ip for instance 22 | shell: netplan apply 23 | -------------------------------------------------------------------------------- /ansible/playbooks/setup_ignition.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: prepare bootstrap ignition webserver 3 | hosts: 4 | - ignition 5 | gather_facts: no 6 | tasks: 7 | - name: ensure that directories are present 8 | file: 9 | state: directory 10 | path: "{{ item }}" 11 | owner: root 12 | group: root 13 | mode: 0755 14 | with_items: 15 | - "/var/www" 16 | - "/var/www/html" 17 | - name: upload bootstrap ignition file 18 | copy: 19 | src: "../../ignition/bootstrap.ign" 20 | dest: "/var/www/html/bootstrap.ign" 21 | owner: root 22 | group: root 23 | mode: 0644 24 | - name: install apache webserver 25 | package: 26 | name: apache2 27 | -------------------------------------------------------------------------------- /ansible/playbooks/wait_for_ssh.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: wait until ssh is available on all nodes 3 | hosts: 4 | - ignition 5 | - haproxy 6 | gather_facts: no 7 | tasks: 8 | - name: wait until ssh is available 9 | wait_for_connection: 10 | delay: 0 11 | timeout: 300 12 | retries: 10 13 | -------------------------------------------------------------------------------- /ansible/site.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - import_playbook: playbooks/wait_for_ssh.yml 3 | - import_playbook: playbooks/setup_ignition.yml 4 | -------------------------------------------------------------------------------- /packer/config-2.2.0.ign: -------------------------------------------------------------------------------- 1 | { 2 | "ignition": { 3 | "config": { 4 | "append": [ 5 | { 6 | "source": "http://169.254.169.254/hetzner/v1/userdata", 7 | "verification": {} 8 | } 9 | ] 10 | }, 11 | "timeouts": {}, 12 | "version": "2.2.0" 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /packer/config-3.0.0.ign: -------------------------------------------------------------------------------- 1 | { 2 | "ignition": { 3 | "config": { 4 | "merge": [ 5 | { 6 | "source": "http://169.254.169.254/hetzner/v1/userdata", 7 | "verification": {} 8 | } 9 | ] 10 | }, 11 | "timeouts": {}, 12 | "version": "3.0.0" 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /packer/hcloud-fcos.json: -------------------------------------------------------------------------------- 1 | { 2 | "variables": { 3 | "location": "nbg1", 4 | "server_type": "cx21", 5 | "snapshot_prefix": "fcos", 6 | "image_type": "generic", 7 | "ignition_config": "config-3.0.0.ign" 8 | }, 9 | "builders": [ 10 | { 11 | "type": "hcloud", 12 | "image": "ubuntu-22.04", 13 | "location": "{{user `location`}}", 14 | "server_type": "{{user `server_type`}}", 15 | "ssh_username": "root", 16 | "rescue": "linux64", 17 | "snapshot_name": "{{user `snapshot_prefix`}}-{{timestamp}}", 18 | "snapshot_labels": { 19 | "os": "fcos", 20 | "image_type": "{{user `image_type`}}", 21 | "fcos_stream": "{{user `fcos_stream`}}", 22 | "fcos_release": "{{user `fcos_release`}}" 23 | } 24 | } 25 | ], 26 | "provisioners": [ 27 | { 28 | "type": "shell", 29 | "inline": [ 30 | "set -x", 31 | "mkdir /source", 32 | "mount -t tmpfs -o size=2G none /source", 33 | "cd /source", 34 | "curl -sfL {{user `fcos_url`}} | unxz > fedora-coreos-qemu.x86_64.qcow2", 35 | "qemu-img convert fedora-coreos-qemu.x86_64.qcow2 -O raw /dev/sda", 36 | "partprobe /dev/sda", 37 | "mkdir /target", 38 | "mount /dev/sda3 /target", 39 | "mkdir /target/ignition" 40 | ] 41 | }, 42 | { 43 | "type": "file", 44 | "source": "{{user `ignition_config`}}", 45 | "destination": "/target/ignition/config.ign" 46 | }, 47 | { 48 | "type": "shell", 49 | "inline": [ 50 | "set -x", 51 | "cd /", 52 | "umount /source", 53 | "umount /target" 54 | ] 55 | } 56 | ] 57 | } 58 | -------------------------------------------------------------------------------- /packer/hcloud-rhcos.json: -------------------------------------------------------------------------------- 1 | { 2 | "variables": { 3 | "location": "nbg1", 4 | "server_type": "cx31", 5 | "snapshot_prefix": "rhcos", 6 | "image_type": "generic", 7 | "ignition_config": "config-2.2.0.ign" 8 | }, 9 | "builders": [ 10 | { 11 | "type": "hcloud", 12 | "image": "ubuntu-22.04", 13 | "location": "{{user `location`}}", 14 | "server_type": "{{user `server_type`}}", 15 | "ssh_username": "root", 16 | "rescue": "linux64", 17 | "snapshot_name": "{{user `snapshot_prefix`}}-{{timestamp}}", 18 | "snapshot_labels": { 19 | "os": "rhcos", 20 | "image_type": "{{user `image_type`}}", 21 | "fcos_stream": "{{user `fcos_stream`}}", 22 | "rhcos_release": "{{user `rhcos_release`}}", 23 | "rhcos_release_minor": "{{user `rhcos_release_minor`}}" 24 | } 25 | } 26 | ], 27 | "provisioners": [ 28 | { 29 | "type": "shell", 30 | "inline": [ 31 | "set -x", 32 | "mkdir /source", 33 | "mount -t tmpfs -o size=4G none /source", 34 | "cd /source", 35 | "curl -sfL -o rhcos-x86_64-qemu.x86_64.qcow2.gz {{user `rhcos_url`}}", 36 | "gzip -d rhcos-x86_64-qemu.x86_64.qcow2.gz", 37 | "qemu-img convert rhcos-x86_64-qemu.x86_64.qcow2 -O raw /dev/sda", 38 | "partprobe /dev/sda", 39 | "mkdir /target", 40 | "mount /dev/sda2 /target", 41 | "mkdir /target/ignition" 42 | ] 43 | }, 44 | { 45 | "type": "file", 46 | "source": "{{user `ignition_config`}}", 47 | "destination": "/target/ignition/config.ign" 48 | }, 49 | { 50 | "type": "shell", 51 | "inline": [ 52 | "set -x", 53 | "cd /", 54 | "umount /source", 55 | "umount /target" 56 | ] 57 | } 58 | ] 59 | } 60 | -------------------------------------------------------------------------------- /renovate.json: -------------------------------------------------------------------------------- 1 | { 2 | "packageRules": [ 3 | { 4 | "updateTypes": ["minor", "patch", "pin", "digest"], 5 | "automerge": true 6 | } 7 | ] 8 | } 9 | -------------------------------------------------------------------------------- /terraform/cacert.tf: -------------------------------------------------------------------------------- 1 | data "local_file" "ignition_master_file" { 2 | filename = "${path.root}/../ignition/master.ign" 3 | } 4 | 5 | data "local_file" "ignition_worker_file" { 6 | filename = "${path.root}/../ignition/worker.ign" 7 | } 8 | 9 | locals { 10 | ignition_master_cacert = jsondecode(data.local_file.ignition_master_file.content).ignition.security.tls.certificateAuthorities[0].source 11 | ignition_worker_cacert = jsondecode(data.local_file.ignition_master_file.content).ignition.security.tls.certificateAuthorities[0].source 12 | } 13 | 14 | #resource "local_file" "ignition_cacert" { 15 | # content = local.ignition_cacert 16 | # filename = "${path.root}/ignition_ca.crt" 17 | #} 18 | -------------------------------------------------------------------------------- /terraform/dns.tf: -------------------------------------------------------------------------------- 1 | resource "cloudflare_record" "dns_a_ignition" { 2 | zone_id = var.dns_zone_id 3 | name = "ignition.${var.dns_domain}" 4 | value = module.ignition.ipv4_addresses[0] 5 | type = "A" 6 | ttl = 120 7 | count = var.bootstrap == true ? 1 : 0 8 | } 9 | 10 | resource "cloudflare_record" "dns_a_api" { 11 | zone_id = var.dns_zone_id 12 | name = "api.${var.dns_domain}" 13 | value = hcloud_load_balancer.lb.ipv4 14 | type = "A" 15 | ttl = 120 16 | } 17 | 18 | resource "cloudflare_record" "dns_a_api_int" { 19 | zone_id = var.dns_zone_id 20 | name = "api-int.${var.dns_domain}" 21 | value = hcloud_load_balancer.lb.ipv4 22 | type = "A" 23 | ttl = 120 24 | } 25 | 26 | resource "cloudflare_record" "dns_a_apps" { 27 | zone_id = var.dns_zone_id 28 | name = "apps.${var.dns_domain}" 29 | value = hcloud_load_balancer.lb.ipv4 30 | type = "A" 31 | ttl = 120 32 | } 33 | 34 | resource "cloudflare_record" "dns_a_apps_wc" { 35 | zone_id = var.dns_zone_id 36 | name = "*.apps.${var.dns_domain}" 37 | value = hcloud_load_balancer.lb.ipv4 38 | type = "A" 39 | ttl = 120 40 | } 41 | 42 | resource "cloudflare_record" "dns_a_etcd" { 43 | zone_id = var.dns_zone_id 44 | name = "etcd-${count.index}.${var.dns_domain}" 45 | value = module.master.ipv4_addresses[count.index] 46 | type = "A" 47 | ttl = 120 48 | 49 | count = length(module.master.ipv4_addresses) 50 | } 51 | 52 | resource "cloudflare_record" "dns_srv_etcd" { 53 | zone_id = var.dns_zone_id 54 | name = "_etcd-server-ssl._tcp.${var.dns_domain}" 55 | type = "SRV" 56 | 57 | data = { 58 | service = "_etcd-server-ssl" 59 | proto = "_tcp" 60 | name = "_etcd-server-ssl._tcp.${var.dns_domain}" 61 | priority = 0 62 | weight = 0 63 | port = 2380 64 | target = "etcd-${count.index}.${var.dns_domain}" 65 | } 66 | 67 | count = length(module.master.ipv4_addresses) 68 | } 69 | -------------------------------------------------------------------------------- /terraform/firewall.tf: -------------------------------------------------------------------------------- 1 | # https://docs.okd.io/latest/installing/installing_platform_agnostic/installing-platform-agnostic.html#installation-network-connectivity-user-infra_installing-platform-agnostic 2 | resource "hcloud_firewall" "okd-base" { 3 | name = "okd-base" 4 | # ICMP is always a good idea 5 | # 6 | # Network reachability tests 7 | rule { 8 | direction = "in" 9 | protocol = "icmp" 10 | source_ips = [ 11 | "0.0.0.0/0", 12 | "::/0" 13 | ] 14 | } 15 | # Metrics 16 | rule { 17 | direction = "in" 18 | protocol = "tcp" 19 | port = 1936 20 | source_ips = [for s in concat(module.master.ipv4_addresses, module.worker.ipv4_addresses, module.bootstrap.ipv4_addresses) : "${s}/32"] 21 | } 22 | # Host level services, including the node exporter on ports 9100-9101 and the Cluster Version Operator on port 9099. 23 | rule { 24 | direction = "in" 25 | protocol = "tcp" 26 | port = "9000-9999" 27 | source_ips = [for s in concat(module.master.ipv4_addresses, module.worker.ipv4_addresses, module.bootstrap.ipv4_addresses) : "${s}/32"] 28 | } 29 | # The default ports that Kubernetes reserves 30 | rule { 31 | direction = "in" 32 | protocol = "tcp" 33 | port = "10250-10259" 34 | source_ips = [for s in concat(module.master.ipv4_addresses, module.worker.ipv4_addresses, module.bootstrap.ipv4_addresses) : "${s}/32"] 35 | } 36 | # openshift-sdn 37 | rule { 38 | direction = "in" 39 | protocol = "tcp" 40 | port = "10256" 41 | source_ips = [for s in concat(module.master.ipv4_addresses, module.worker.ipv4_addresses, module.bootstrap.ipv4_addresses) : "${s}/32"] 42 | } 43 | # VXLAN and Geneve 44 | rule { 45 | direction = "in" 46 | protocol = "udp" 47 | port = "4789" 48 | source_ips = [for s in concat(module.master.ipv4_addresses, module.worker.ipv4_addresses, module.bootstrap.ipv4_addresses) : "${s}/32"] 49 | } 50 | # VXLAN and Geneve 51 | rule { 52 | direction = "in" 53 | protocol = "udp" 54 | port = "6081" 55 | source_ips = [for s in concat(module.master.ipv4_addresses, module.worker.ipv4_addresses, module.bootstrap.ipv4_addresses) : "${s}/32"] 56 | } 57 | # Host level services, including the node exporter on ports 9100-9101. 58 | rule { 59 | direction = "in" 60 | protocol = "udp" 61 | port = "9000-9999" 62 | source_ips = [for s in concat(module.master.ipv4_addresses, module.worker.ipv4_addresses, module.bootstrap.ipv4_addresses) : "${s}/32"] 63 | } 64 | # Kubernetes node port 65 | rule { 66 | direction = "in" 67 | protocol = "tcp" 68 | port = "30000-32767" 69 | source_ips = [for s in concat(module.master.ipv4_addresses, module.worker.ipv4_addresses, module.bootstrap.ipv4_addresses) : "${s}/32"] 70 | } 71 | # Kubernetes node port 72 | rule { 73 | direction = "in" 74 | protocol = "udp" 75 | port = "30000-32767" 76 | source_ips = [for s in concat(module.master.ipv4_addresses, module.worker.ipv4_addresses, module.bootstrap.ipv4_addresses) : "${s}/32"] 77 | } 78 | } 79 | 80 | 81 | resource "hcloud_firewall" "okd-master" { 82 | name = "okd-master" 83 | 84 | # ICMP is always a good idea 85 | # 86 | # Network reachability tests 87 | rule { 88 | direction = "in" 89 | protocol = "icmp" 90 | source_ips = [ 91 | "0.0.0.0/0", 92 | "::/0" 93 | ] 94 | } 95 | # Kubernetes API 96 | rule { 97 | direction = "in" 98 | protocol = "tcp" 99 | port = "6443" 100 | source_ips = [for s in concat([hcloud_load_balancer.lb.ipv4],module.master.ipv4_addresses, module.worker.ipv4_addresses, module.bootstrap.ipv4_addresses) : "${s}/32"] 101 | } 102 | # Machine config server 103 | rule { 104 | direction = "in" 105 | protocol = "tcp" 106 | port = "22623" 107 | source_ips = [for s in concat([hcloud_load_balancer.lb.ipv4]) : "${s}/32"] 108 | } 109 | # etcd server and peer ports 110 | rule { 111 | direction = "in" 112 | protocol = "tcp" 113 | port = "2379-2380" 114 | source_ips = [for s in module.master.ipv4_addresses : "${s}/32"] 115 | } 116 | } 117 | 118 | resource "hcloud_firewall" "okd-ingress" { 119 | name = "okd-ingress" 120 | 121 | # ICMP is always a good idea 122 | # 123 | # Network reachability tests 124 | rule { 125 | direction = "in" 126 | protocol = "icmp" 127 | source_ips = [ 128 | "0.0.0.0/0", 129 | "::/0" 130 | ] 131 | } 132 | rule { 133 | direction = "in" 134 | protocol = "tcp" 135 | port = "80" 136 | source_ips = [for s in [hcloud_load_balancer.lb.ipv4] : "${s}/32"] 137 | } 138 | rule { 139 | direction = "in" 140 | protocol = "tcp" 141 | port = "443" 142 | source_ips = [for s in [hcloud_load_balancer.lb.ipv4] : "${s}/32"] 143 | } 144 | } 145 | -------------------------------------------------------------------------------- /terraform/image.tf: -------------------------------------------------------------------------------- 1 | data "hcloud_image" "image" { 2 | with_selector = "os=${var.image},image_type=generic" 3 | with_status = ["available"] 4 | most_recent = true 5 | } 6 | -------------------------------------------------------------------------------- /terraform/inventory.tf: -------------------------------------------------------------------------------- 1 | data "template_file" "ansible_inventory" { 2 | template = file("./templates/inventory.tpl") 3 | vars = { 4 | ignition_node = join("\n", module.ignition.server_names) 5 | bootstrap_node = join("\n", module.bootstrap.server_names) 6 | master_nodes = join("\n", module.master.server_names) 7 | worker_nodes = join("\n", module.worker.server_names) 8 | } 9 | } 10 | 11 | resource "local_file" "ansible_inventory" { 12 | content = data.template_file.ansible_inventory.rendered 13 | filename = "../inventory.ini" 14 | } 15 | -------------------------------------------------------------------------------- /terraform/loadbalancer.tf: -------------------------------------------------------------------------------- 1 | resource "hcloud_load_balancer" "lb" { 2 | name = "lb.${var.dns_domain}" 3 | load_balancer_type = "lb11" 4 | location = var.location 5 | dynamic "target" { 6 | for_each = concat(module.master.server_ids, module.worker.server_ids, module.bootstrap.server_ids) 7 | content { 8 | type = "server" 9 | server_id = target.value 10 | } 11 | } 12 | } 13 | 14 | resource "hcloud_load_balancer_network" "lb_network" { 15 | load_balancer_id = hcloud_load_balancer.lb.id 16 | subnet_id = hcloud_network_subnet.lb_subnet.id 17 | ip = "192.168.254.254" 18 | } 19 | 20 | resource "hcloud_load_balancer_service" "lb_api" { 21 | load_balancer_id = hcloud_load_balancer.lb.id 22 | protocol = "tcp" 23 | listen_port = 6443 24 | destination_port = 6443 25 | 26 | health_check { 27 | protocol = "tcp" 28 | port = 6443 29 | interval = 10 30 | timeout = 1 31 | retries = 3 32 | } 33 | } 34 | 35 | resource "hcloud_load_balancer_service" "lb_mcs" { 36 | load_balancer_id = hcloud_load_balancer.lb.id 37 | protocol = "tcp" 38 | listen_port = 22623 39 | destination_port = 22623 40 | 41 | health_check { 42 | protocol = "tcp" 43 | port = 22623 44 | interval = 10 45 | timeout = 1 46 | retries = 3 47 | } 48 | } 49 | 50 | resource "hcloud_load_balancer_service" "lb_ingress_http" { 51 | load_balancer_id = hcloud_load_balancer.lb.id 52 | protocol = "tcp" 53 | listen_port = 80 54 | destination_port = 80 55 | 56 | health_check { 57 | protocol = "tcp" 58 | port = 80 59 | interval = 10 60 | timeout = 1 61 | retries = 3 62 | } 63 | } 64 | 65 | resource "hcloud_load_balancer_service" "lb_ingress_https" { 66 | load_balancer_id = hcloud_load_balancer.lb.id 67 | protocol = "tcp" 68 | listen_port = 443 69 | destination_port = 443 70 | 71 | health_check { 72 | protocol = "tcp" 73 | port = 443 74 | interval = 10 75 | timeout = 1 76 | retries = 3 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /terraform/main.tf: -------------------------------------------------------------------------------- 1 | module "ignition" { 2 | source = "./modules/hcloud_instance" 3 | instance_count = var.bootstrap == true ? 1 : 0 4 | location = var.location 5 | name = "ignition" 6 | dns_domain = var.dns_domain 7 | dns_zone_id = var.dns_zone_id 8 | image = "ubuntu-22.04" 9 | user_data = file("templates/cloud-init.tpl") 10 | ssh_keys = data.hcloud_ssh_keys.all_keys.ssh_keys.*.name 11 | server_type = "cx11" 12 | subnet = hcloud_network_subnet.subnet.id 13 | } 14 | 15 | module "bootstrap" { 16 | source = "./modules/hcloud_coreos" 17 | instance_count = var.bootstrap == true ? 1 : 0 18 | location = var.location 19 | name = "bootstrap" 20 | dns_domain = var.dns_domain 21 | dns_zone_id = var.dns_zone_id 22 | dns_internal_ip = false 23 | image = data.hcloud_image.image.id 24 | image_name = var.image 25 | server_type = "cx41" 26 | subnet = hcloud_network_subnet.subnet.id 27 | ignition_url = var.bootstrap == true ? "http://${cloudflare_record.dns_a_ignition[0].name}/bootstrap.ign" : "" 28 | } 29 | 30 | module "master" { 31 | source = "./modules/hcloud_coreos" 32 | instance_count = var.replicas_master 33 | location = var.location 34 | name = "master" 35 | dns_domain = var.dns_domain 36 | dns_zone_id = var.dns_zone_id 37 | dns_internal_ip = false 38 | image = data.hcloud_image.image.id 39 | image_name = var.image 40 | server_type = "cx41" 41 | labels = { 42 | "okd.io/node" = "true", 43 | "okd.io/master" = "true", 44 | "okd.io/ingress" = "true" 45 | } 46 | # Manually add apply_to for the labels, until tf_hcloud allows apply_to in the firewall 47 | # firewall_ids = [hcloud_firewall.okd-base.id, hcloud_firewall.okd-master.id, hcloud_firewall.okd-ingress.id] 48 | subnet = hcloud_network_subnet.subnet.id 49 | ignition_url = "https://api-int.${var.dns_domain}:22623/config/master" 50 | ignition_cacert = local.ignition_master_cacert 51 | } 52 | 53 | module "worker" { 54 | source = "./modules/hcloud_coreos" 55 | instance_count = var.replicas_worker 56 | location = var.location 57 | name = "worker" 58 | dns_domain = var.dns_domain 59 | dns_zone_id = var.dns_zone_id 60 | dns_internal_ip = false 61 | image = data.hcloud_image.image.id 62 | image_name = var.image 63 | server_type = "cx41" 64 | labels = { 65 | "okd.io/node" = "true", 66 | "okd.io/ingress" = "true" 67 | "okd.io/worker" = "true" 68 | } 69 | # Manually add apply_to for the labels, until tf_hcloud allows apply_to in the firewall 70 | # firewall_ids = [hcloud_firewall.okd-base.id, hcloud_firewall.okd-ingress.id] 71 | subnet = hcloud_network_subnet.subnet.id 72 | ignition_url = "https://api-int.${var.dns_domain}:22623/config/worker" 73 | ignition_cacert = local.ignition_worker_cacert 74 | } 75 | -------------------------------------------------------------------------------- /terraform/modules/hcloud_coreos/dns.tf: -------------------------------------------------------------------------------- 1 | resource "cloudflare_record" "dns-a" { 2 | count = var.instance_count 3 | zone_id = var.dns_zone_id 4 | name = element(hcloud_server.server.*.name, count.index) 5 | # value = var.dns_internal_ip == true ? element(hcloud_server_network.server_network.*.ip, count.index) : element(hcloud_server.server.*.ipv4_address, count.index) 6 | value = element(hcloud_server.server.*.ipv4_address, count.index) 7 | type = "A" 8 | ttl = 120 9 | } 10 | -------------------------------------------------------------------------------- /terraform/modules/hcloud_coreos/ignition.tf: -------------------------------------------------------------------------------- 1 | data "template_file" "ignition_config" { 2 | template = file("${path.module}/templates/ignition.ign") 3 | vars = { 4 | hostname = "${format("${var.name}%02d", count.index + 1)}.${var.dns_domain}" 5 | hostname_b64 = base64encode("${format("${var.name}%02d", count.index + 1)}.${var.dns_domain}") 6 | resolvconf_b64 = base64encode(file("${path.module}/templates/resolv.conf")) 7 | ignition_url = var.ignition_url 8 | ignition_version = var.ignition_version 9 | ignition_cacert = var.ignition_cacert 10 | } 11 | count = var.instance_count 12 | } 13 | 14 | resource "local_file" "ignition_config" { 15 | content = data.template_file.ignition_config[count.index].rendered 16 | filename = "${path.root}/../ignition/${format("${var.name}%02d", count.index + 1)}.${var.dns_domain}.ign" 17 | count = var.instance_count 18 | } 19 | -------------------------------------------------------------------------------- /terraform/modules/hcloud_coreos/main.tf: -------------------------------------------------------------------------------- 1 | resource "hcloud_server" "server" { 2 | count = var.instance_count 3 | name = "${format("${var.name}%02d", count.index + 1)}.${var.dns_domain}" 4 | image = var.image 5 | server_type = var.server_type 6 | keep_disk = var.keep_disk 7 | ssh_keys = var.ssh_keys 8 | user_data = data.template_file.ignition_config[count.index].rendered 9 | location = var.location 10 | labels = var.labels 11 | backups = var.backups 12 | firewall_ids = var.firewall_ids 13 | lifecycle { 14 | ignore_changes = [user_data, image, firewall_ids] 15 | } 16 | } 17 | 18 | resource "hcloud_rdns" "dns-ptr-ipv4" { 19 | count = var.instance_count 20 | server_id = element(hcloud_server.server.*.id, count.index) 21 | ip_address = element(hcloud_server.server.*.ipv4_address, count.index) 22 | dns_ptr = element(hcloud_server.server.*.name, count.index) 23 | } 24 | 25 | resource "hcloud_rdns" "dns-ptr-ipv6" { 26 | count = var.instance_count 27 | server_id = element(hcloud_server.server.*.id, count.index) 28 | ip_address = "${element(hcloud_server.server.*.ipv6_address, count.index)}1" 29 | dns_ptr = element(hcloud_server.server.*.name, count.index) 30 | } 31 | -------------------------------------------------------------------------------- /terraform/modules/hcloud_coreos/network.tf: -------------------------------------------------------------------------------- 1 | #resource "hcloud_server_network" "server_network" { 2 | # server_id = element(hcloud_server.server.*.id, count.index) 3 | # subnet_id = var.subnet 4 | # count = length(hcloud_server.server.*.id) 5 | #} 6 | -------------------------------------------------------------------------------- /terraform/modules/hcloud_coreos/output.tf: -------------------------------------------------------------------------------- 1 | output "server_ids" { 2 | value = hcloud_server.server.*.id 3 | } 4 | 5 | output "server_names" { 6 | value = hcloud_server.server.*.name 7 | } 8 | 9 | #output "internal_ipv4_addresses" { 10 | # value = hcloud_server_network.server_network.*.ip 11 | #} 12 | 13 | output "ipv4_addresses" { 14 | value = hcloud_server.server.*.ipv4_address 15 | } 16 | 17 | output "ipv6_addresses" { 18 | value = hcloud_server.server.*.ipv6_address 19 | } 20 | -------------------------------------------------------------------------------- /terraform/modules/hcloud_coreos/templates/ignition.ign: -------------------------------------------------------------------------------- 1 | { 2 | "ignition": { 3 | "config": { 4 | %{ if ignition_version == "3.0.0" }"merge": [%{ else }"append": [%{ endif } 5 | { 6 | "source": "${ignition_url}"%{ if ignition_cacert == "" }, 7 | "verification": {}%{ endif } 8 | } 9 | ] 10 | },%{ if ignition_cacert != "" } 11 | "security": { 12 | "tls": { 13 | "certificateAuthorities": [ 14 | { 15 | "source": "${ignition_cacert}" 16 | } 17 | ] 18 | } 19 | },%{ endif } 20 | "timeouts": {}, 21 | "version": "${ignition_version}" 22 | }, 23 | "networkd": {}, 24 | "passwd": {}, 25 | "storage": { 26 | "files": [ 27 | { 28 | "filesystem": "root", 29 | "group": {}, 30 | "path": "/etc/hostname", 31 | "user": {}, 32 | "contents": { 33 | "source": "data:text/plain;charset=utf-8;base64,${hostname_b64}", 34 | "verification": {} 35 | }, 36 | "mode": 420 37 | } 38 | ] 39 | }, 40 | "storage": { 41 | "files": [ 42 | { 43 | "filesystem": "root", 44 | "group": {}, 45 | "overwrite": true, 46 | "path": "/etc/resolv.conf", 47 | "user": {}, 48 | "contents": { 49 | "source": "data:text/plain;charset=utf-8;base64,${resolvconf_b64}", 50 | "verification": {} 51 | }, 52 | "mode": 420 53 | } 54 | ] 55 | }, 56 | "systemd": { 57 | "units": [ 58 | { 59 | "contents": "[Unit]\nConditionFirstBoot=yes\n[Service]\nType=idle\nExecStart=/usr/bin/hostnamectl set-hostname ${hostname}\n[Install]\nWantedBy=multi-user.target\n", 60 | "enabled": true, 61 | "name": "set-hostname-firstboot.service" 62 | } 63 | ] 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /terraform/modules/hcloud_coreos/templates/resolv.conf: -------------------------------------------------------------------------------- 1 | # Managed via Terraform 2 | nameserver 1.1.1.1 3 | nameserver 1.0.0.1 4 | -------------------------------------------------------------------------------- /terraform/modules/hcloud_coreos/variables.tf: -------------------------------------------------------------------------------- 1 | variable "name" { 2 | type = string 3 | description = "Instance nam" 4 | } 5 | 6 | variable "dns_domain" { 7 | type = string 8 | description = "DNS domain" 9 | } 10 | 11 | variable "dns_zone_id" { 12 | description = "Zone ID" 13 | default = null 14 | } 15 | 16 | variable "dns_internal_ip" { 17 | description = "Point DNS record to internal ip" 18 | default = false 19 | } 20 | 21 | variable "instance_count" { 22 | type = number 23 | description = "Number of instances to deploy" 24 | default = 1 25 | } 26 | 27 | variable "server_type" { 28 | type = string 29 | description = "Hetzner Cloud instance type" 30 | default = "cx11" 31 | } 32 | 33 | variable "image" { 34 | type = string 35 | description = "Hetzner Cloud system image" 36 | default = "ubuntu-22.04" 37 | } 38 | 39 | variable "user_data" { 40 | description = "Cloud-Init user data to use during server creation" 41 | default = null 42 | } 43 | 44 | variable "ssh_keys" { 45 | type = list(any) 46 | description = "SSH key IDs or names which should be injected into the server at creation time" 47 | default = [] 48 | } 49 | 50 | variable "keep_disk" { 51 | type = bool 52 | description = "If true, do not upgrade the disk. This allows downgrading the server type later." 53 | default = true 54 | } 55 | 56 | variable "location" { 57 | type = string 58 | description = "The location name to create the server in. nbg1, fsn1 or hel1" 59 | default = "nbg1" 60 | } 61 | 62 | variable "labels" { 63 | type = map(string) 64 | description = "Labels that the instance is tagged with." 65 | default = {} 66 | } 67 | 68 | variable "backups" { 69 | type = bool 70 | description = "Enable or disable backups" 71 | default = false 72 | } 73 | 74 | variable "firewall_ids" { 75 | type = list(number) 76 | description = "Assigned firewalls" 77 | default = [] 78 | } 79 | 80 | variable "volume" { 81 | type = bool 82 | description = "Enable or disable an additional volume" 83 | default = false 84 | } 85 | 86 | variable "volume_size" { 87 | type = number 88 | description = "Size of the additional data volume" 89 | default = 20 90 | } 91 | 92 | variable "ignition_url" { 93 | type = string 94 | description = "URL to the external ignition webserver" 95 | } 96 | 97 | variable "ignition_cacert" { 98 | type = string 99 | description = "CA certificate for the machine config server" 100 | default = "" 101 | } 102 | 103 | variable "subnet" { 104 | type = string 105 | description = "Id of the additional internal network" 106 | } 107 | 108 | variable "image_name" { 109 | type = string 110 | description = "Either fcos or rhcos (necessary for ignition rendering)" 111 | default = "fcos" 112 | } 113 | 114 | variable "ignition_version" { 115 | type = string 116 | description = "Ignition Version" 117 | default = "3.0.0" 118 | } 119 | -------------------------------------------------------------------------------- /terraform/modules/hcloud_coreos/versions.tf: -------------------------------------------------------------------------------- 1 | ../../versions.tf -------------------------------------------------------------------------------- /terraform/modules/hcloud_coreos/volumes.tf: -------------------------------------------------------------------------------- 1 | resource "hcloud_volume" "volumes" { 2 | name = "${element(hcloud_server.server.*.name, count.index)}-data" 3 | size = var.volume_size 4 | format = "xfs" 5 | automount = false 6 | server_id = element(hcloud_server.server.*.id, count.index) 7 | count = var.volume == true ? var.instance_count : 0 8 | } 9 | -------------------------------------------------------------------------------- /terraform/modules/hcloud_instance/main.tf: -------------------------------------------------------------------------------- 1 | resource "hcloud_server" "server" { 2 | count = var.instance_count 3 | name = "${format("${var.name}%02d", count.index + 1)}.${var.dns_domain}" 4 | image = var.image 5 | server_type = var.server_type 6 | keep_disk = var.keep_disk 7 | ssh_keys = var.ssh_keys 8 | user_data = var.user_data 9 | location = var.location 10 | backups = var.backups 11 | lifecycle { 12 | ignore_changes = [user_data, image] 13 | } 14 | } 15 | 16 | resource "cloudflare_record" "dns-a" { 17 | count = var.instance_count 18 | zone_id = var.dns_zone_id 19 | name = element(hcloud_server.server.*.name, count.index) 20 | value = element(hcloud_server.server.*.ipv4_address, count.index) 21 | type = "A" 22 | ttl = 120 23 | } 24 | 25 | resource "cloudflare_record" "dns-aaaa" { 26 | count = var.instance_count 27 | zone_id = var.dns_zone_id 28 | name = element(hcloud_server.server.*.name, count.index) 29 | value = "${element(hcloud_server.server.*.ipv6_address, count.index)}1" 30 | type = "AAAA" 31 | ttl = 120 32 | } 33 | 34 | resource "hcloud_rdns" "dns-ptr-ipv4" { 35 | count = var.instance_count 36 | server_id = element(hcloud_server.server.*.id, count.index) 37 | ip_address = element(hcloud_server.server.*.ipv4_address, count.index) 38 | dns_ptr = element(hcloud_server.server.*.name, count.index) 39 | } 40 | 41 | resource "hcloud_rdns" "dns-ptr-ipv6" { 42 | count = var.instance_count 43 | server_id = element(hcloud_server.server.*.id, count.index) 44 | ip_address = "${element(hcloud_server.server.*.ipv6_address, count.index)}1" 45 | dns_ptr = element(hcloud_server.server.*.name, count.index) 46 | } 47 | -------------------------------------------------------------------------------- /terraform/modules/hcloud_instance/network.tf: -------------------------------------------------------------------------------- 1 | #resource "hcloud_server_network" "server_network" { 2 | # server_id = element(hcloud_server.server.*.id, count.index) 3 | # subnet_id = var.subnet 4 | # count = length(hcloud_server.server.*.id) 5 | #} 6 | -------------------------------------------------------------------------------- /terraform/modules/hcloud_instance/output.tf: -------------------------------------------------------------------------------- 1 | output "server_ids" { 2 | value = hcloud_server.server.*.id 3 | } 4 | 5 | output "server_names" { 6 | value = hcloud_server.server.*.name 7 | } 8 | 9 | #output "internal_ipv4_addresses" { 10 | # value = hcloud_server_network.server_network.*.ip 11 | #} 12 | 13 | output "ipv4_addresses" { 14 | value = hcloud_server.server.*.ipv4_address 15 | } 16 | 17 | output "ipv6_addresses" { 18 | value = hcloud_server.server.*.ipv6_address 19 | } 20 | -------------------------------------------------------------------------------- /terraform/modules/hcloud_instance/variables.tf: -------------------------------------------------------------------------------- 1 | variable "name" { 2 | type = string 3 | description = "Instance nam" 4 | } 5 | 6 | variable "dns_domain" { 7 | type = string 8 | description = "DNS domain" 9 | } 10 | 11 | variable "dns_zone_id" { 12 | description = "Zone ID" 13 | default = null 14 | } 15 | 16 | variable "dns_internal_ip" { 17 | description = "Point DNS record to internal ip" 18 | default = false 19 | } 20 | 21 | variable "instance_count" { 22 | type = number 23 | description = "Number of instances to deploy" 24 | default = 1 25 | } 26 | 27 | variable "server_type" { 28 | type = string 29 | description = "Hetzner Cloud instance type" 30 | default = "cx11" 31 | } 32 | 33 | variable "image" { 34 | type = string 35 | description = "Hetzner Cloud system image" 36 | default = "ubuntu-22.04" 37 | } 38 | 39 | variable "user_data" { 40 | description = "Cloud-Init user data to use during server creation" 41 | default = null 42 | } 43 | 44 | variable "ssh_keys" { 45 | type = list(any) 46 | description = "SSH key IDs or names which should be injected into the server at creation time" 47 | default = [] 48 | } 49 | 50 | variable "keep_disk" { 51 | type = bool 52 | description = "If true, do not upgrade the disk. This allows downgrading the server type later." 53 | default = true 54 | } 55 | 56 | variable "location" { 57 | type = string 58 | description = "The location name to create the server in. nbg1, fsn1 or hel1" 59 | default = "nbg1" 60 | } 61 | 62 | variable "backups" { 63 | type = bool 64 | description = "Enable or disable backups" 65 | default = false 66 | } 67 | 68 | variable "volume" { 69 | type = bool 70 | description = "Enable or disable an additional volume" 71 | default = false 72 | } 73 | 74 | variable "volume_size" { 75 | type = number 76 | description = "Size of the additional data volume" 77 | default = 20 78 | } 79 | 80 | variable "subnet" { 81 | type = string 82 | description = "Id of the additional internal network" 83 | } 84 | -------------------------------------------------------------------------------- /terraform/modules/hcloud_instance/versions.tf: -------------------------------------------------------------------------------- 1 | ../../versions.tf -------------------------------------------------------------------------------- /terraform/modules/hcloud_instance/volumes.tf: -------------------------------------------------------------------------------- 1 | resource "hcloud_volume" "volumes" { 2 | name = "${element(hcloud_server.server.*.name, count.index)}-data" 3 | size = var.volume_size 4 | format = "xfs" 5 | automount = false 6 | server_id = element(hcloud_server.server.*.id, count.index) 7 | count = var.volume == true ? var.instance_count : 0 8 | } 9 | -------------------------------------------------------------------------------- /terraform/network.tf: -------------------------------------------------------------------------------- 1 | resource "hcloud_network" "network" { 2 | name = var.dns_domain 3 | ip_range = var.network_cidr 4 | } 5 | 6 | resource "hcloud_network_subnet" "subnet" { 7 | network_id = hcloud_network.network.id 8 | type = "server" 9 | network_zone = "eu-central" 10 | ip_range = var.subnet_cidr 11 | } 12 | 13 | resource "hcloud_network_subnet" "lb_subnet" { 14 | network_id = hcloud_network.network.id 15 | type = "cloud" 16 | network_zone = "eu-central" 17 | ip_range = var.lb_subnet_cidr 18 | } 19 | -------------------------------------------------------------------------------- /terraform/output.tf: -------------------------------------------------------------------------------- 1 | output "ignition" { 2 | value = module.ignition 3 | } 4 | output "bootstrap" { 5 | value = module.bootstrap 6 | } 7 | output "master" { 8 | value = module.master 9 | } 10 | output "worker" { 11 | value = module.worker 12 | } 13 | -------------------------------------------------------------------------------- /terraform/provider.tf: -------------------------------------------------------------------------------- 1 | provider "cloudflare" { 2 | } 3 | 4 | provider "hcloud" { 5 | } 6 | 7 | provider "template" { 8 | } 9 | 10 | provider "local" { 11 | } 12 | 13 | provider "random" { 14 | } 15 | -------------------------------------------------------------------------------- /terraform/ssh_keys.tf: -------------------------------------------------------------------------------- 1 | data "hcloud_ssh_keys" "all_keys" { 2 | } 3 | -------------------------------------------------------------------------------- /terraform/templates/cloud-init.tpl: -------------------------------------------------------------------------------- 1 | #cloud-config 2 | 3 | # upgrade packages on boot 4 | package_update: true 5 | package_upgrade: false 6 | package_reboot_if_required: false 7 | 8 | write_files: 9 | - path: /etc/ssh/sshd_config 10 | content: | 11 | # manged by cloud-init 12 | AddressFamily any 13 | Port 22 14 | 15 | #HostKey /etc/ssh/ssh_host_dsa_key 16 | #HostKey /etc/ssh/ssh_host_ecdsa_key 17 | AcceptEnv LC_* 18 | Banner none 19 | ChallengeResponseAuthentication no 20 | Ciphers chacha20-poly1305@openssh.com,aes256-gcm@openssh.com,aes128-gcm@openssh.com,aes256-ctr,aes192-ctr,aes128-ctr 21 | #ClientAliveCountMax 0 22 | #ClientAliveInterval 900 23 | Compression no 24 | HostKey /etc/ssh/ssh_host_rsa_key 25 | HostKey /etc/ssh/ssh_host_ed25519_key 26 | HostbasedAuthentication no 27 | IgnoreRhosts yes 28 | KexAlgorithms curve25519-sha256@libssh.org,diffie-hellman-group18-sha512,diffie-hellman-group14-sha256,diffie-hellman-group16-sha512 29 | LogLevel VERBOSE 30 | MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com,umac-128-etm@openssh.com 31 | MaxAuthTries 2 32 | MaxSessions 10 33 | MaxStartups 10:30:100 34 | PasswordAuthentication no 35 | PermitEmptyPasswords no 36 | PermitRootLogin without-password 37 | PrintMotd no 38 | Protocol 2 39 | Subsystem sftp /usr/lib/openssh/sftp-server 40 | SyslogFacility AUTHPRIV 41 | UseDNS no 42 | UsePAM yes 43 | X11Forwarding no 44 | 45 | # only run 46 | runcmd: 47 | - "systemctl restart sshd" 48 | - "touch /etc/cloud-init.done" 49 | 50 | final_message: "The system is finally up, after $UPTIME seconds" 51 | 52 | #phone_home: 53 | # url: http://my.example.com/$INSTANCE_ID/ 54 | # post: [ pub_key_dsa, pub_key_rsa, pub_key_ecdsa, instance_id ] 55 | -------------------------------------------------------------------------------- /terraform/templates/inventory.tpl: -------------------------------------------------------------------------------- 1 | # generated by terraform 2 | 3 | [ignition] 4 | ${ignition_node} 5 | 6 | [bootstrap] 7 | ${bootstrap_node} 8 | 9 | [master] 10 | ${master_nodes} 11 | 12 | [worker] 13 | ${worker_nodes} 14 | -------------------------------------------------------------------------------- /terraform/variables.tf: -------------------------------------------------------------------------------- 1 | variable "replicas_master" { 2 | type = number 3 | default = 1 4 | description = "Count of master replicas" 5 | } 6 | 7 | variable "replicas_worker" { 8 | type = number 9 | default = 0 10 | description = "Count of worker replicas" 11 | } 12 | 13 | variable "bootstrap" { 14 | type = bool 15 | default = false 16 | description = "Whether to deploy a bootstrap instance" 17 | } 18 | 19 | variable "dns_domain" { 20 | type = string 21 | description = "Name of the Cloudflare domain" 22 | } 23 | 24 | variable "dns_zone_id" { 25 | type = string 26 | description = "Zone ID of the Cloudflare domain" 27 | } 28 | 29 | variable "ip_loadbalancer_api" { 30 | description = "IP of an external loadbalancer for api (optional)" 31 | default = null 32 | } 33 | 34 | variable "ip_loadbalancer_api_int" { 35 | description = "IP of an external loadbalancer for api-int (optional)" 36 | default = null 37 | } 38 | 39 | variable "ip_loadbalancer_apps" { 40 | description = "IP of an external loadbalancer for apps (optional)" 41 | default = null 42 | } 43 | 44 | variable "network_cidr" { 45 | type = string 46 | description = "CIDR for the network" 47 | default = "192.168.0.0/16" 48 | } 49 | 50 | variable "subnet_cidr" { 51 | type = string 52 | description = "CIDR for the subnet" 53 | default = "192.168.254.0/24" 54 | } 55 | 56 | variable "lb_subnet_cidr" { 57 | type = string 58 | description = "CIDR for the loadbalancer subnet" 59 | default = "192.168.253.0/24" 60 | } 61 | 62 | variable "location" { 63 | type = string 64 | description = "Region" 65 | default = "nbg1" 66 | } 67 | 68 | variable "image" { 69 | type = string 70 | description = "Image selector (either fcos or rhcos)" 71 | default = "fcos" 72 | } 73 | -------------------------------------------------------------------------------- /terraform/versions.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | cloudflare = { 4 | source = "cloudflare/cloudflare" 5 | version = "2.27.0" 6 | } 7 | hcloud = { 8 | source = "hetznercloud/hcloud" 9 | version = "1.48.0" 10 | } 11 | template = { 12 | source = "hashicorp/template" 13 | version = "2.2.0" 14 | } 15 | local = { 16 | source = "hashicorp/local" 17 | version = "2.5.1" 18 | } 19 | random = { 20 | source = "hashicorp/random" 21 | version = "3.6.2" 22 | } 23 | } 24 | required_version = ">= 0.14" 25 | } 26 | -------------------------------------------------------------------------------- /tests/image.tests.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | schemaVersion: "2.0.0" 3 | 4 | commandTests: 5 | - name: "openshift clients works" 6 | command: "oc" 7 | args: ["version"] 8 | - name: "openshift-install client works" 9 | command: "openshift-install" 10 | args: ["version"] 11 | - name: "kubectl works" 12 | command: "kubectl" 13 | args: ["version", "--client"] 14 | - name: "terraform works" 15 | command: "terraform" 16 | args: ["version"] 17 | - name: "packer works" 18 | command: "packer" 19 | args: ["version"] 20 | - name: "make exists" 21 | command: "which" 22 | args: ["make"] 23 | --------------------------------------------------------------------------------