├── .gitignore ├── tf-project ├── versions.tf └── cluster.tf ├── hosts.example ├── keystone_rc.sh.example ├── playbooks ├── workers.yml ├── master.yml └── kube-dependencies.yml ├── LICENSE.md └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | hosts 2 | keystone_rc.sh 3 | *.log 4 | /log[s] 5 | *.retry 6 | tf-project/.terraform/ 7 | tf-project/*terraform.* 8 | tf-project/ansible_inventory 9 | -------------------------------------------------------------------------------- /tf-project/versions.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | local = { 4 | source = "hashicorp/local" 5 | } 6 | openstack = { 7 | source = "terraform-provider-openstack/openstack" 8 | } 9 | } 10 | required_version = ">= 0.13" 11 | } 12 | -------------------------------------------------------------------------------- /hosts.example: -------------------------------------------------------------------------------- 1 | [master] 2 | master ansible_host= 3 | 4 | [workers] 5 | worker1 ansible_host= 6 | worker2 ansible_host= 7 | 8 | [all:vars] 9 | ansible_python_interpreter=/usr/bin/python3 10 | ansible_ssh_extra_args='-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null' 11 | ansible_ssh_private_key_file=/path/to/ssh-key 12 | ansible_user=ubuntu 13 | -------------------------------------------------------------------------------- /keystone_rc.sh.example: -------------------------------------------------------------------------------- 1 | export OS_USERNAME= 2 | export OS_PROJECT_NAME= 3 | export OS_PASSWORD= 4 | export OS_AUTH_URL=https://api.nrec.no:5000/v3 5 | export OS_IDENTITY_API_VERSION=3 6 | export OS_USER_DOMAIN_NAME=dataporten 7 | export OS_PROJECT_DOMAIN_NAME=dataporten 8 | export OS_REGION_NAME= 9 | export OS_INTERFACE=public 10 | export OS_NO_CACHE=1 11 | export TF_VAR_K8S_WORKER_COUNT=19 12 | export TF_VAR_K8S_WORKER_FLAVOR="m1.medium" 13 | export TF_VAR_K8S_IMAGE_NAME="GOLD Ubuntu 22.04 LTS" 14 | export TF_VAR_K8S_NETWORK_NAME="dualStack" 15 | export TF_VAR_K8S_KEY_PAIR="k8s-nodes" 16 | export TF_VAR_K8S_KEY_PAIR_LOCATION="~/.ssh" 17 | export TF_VAR_K8S_SECURITY_GROUP="SSH and ICMP" 18 | export ANSIBLE_NOCOWS=1 19 | -------------------------------------------------------------------------------- /playbooks/workers.yml: -------------------------------------------------------------------------------- 1 | - hosts: master 2 | become: yes 3 | #gather_facts: false 4 | tasks: 5 | - name: get join command 6 | shell: kubeadm token create --print-join-command 7 | register: join_command_raw 8 | 9 | - name: set join command 10 | set_fact: 11 | join_command: "{{ join_command_raw.stdout_lines[0] }}" 12 | 13 | 14 | - hosts: workers 15 | become: yes 16 | tasks: 17 | - name: TCP port 6443 on master is reachable from worker 18 | wait_for: "host={{ hostvars['k8s-master-1']['ansible_default_ipv4']['address'] }} port=6443 timeout=1" 19 | 20 | - name: join cluster 21 | shell: "{{ hostvars['k8s-master-1'].join_command }} >> node_joined.log" 22 | args: 23 | chdir: /home/ubuntu 24 | creates: node_joined.log 25 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Torgeir L 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /playbooks/master.yml: -------------------------------------------------------------------------------- 1 | - hosts: master 2 | become: yes 3 | tasks: 4 | - name: create an empty file for Kubeadm configuring 5 | copy: 6 | content: "" 7 | dest: /etc/kubernetes/kubeadm-config.yaml 8 | force: no 9 | 10 | - name: configuring the container runtime including its cgroup driver 11 | blockinfile: 12 | path: /etc/kubernetes/kubeadm-config.yaml 13 | block: | 14 | kind: ClusterConfiguration 15 | apiVersion: kubeadm.k8s.io/v1beta3 16 | networking: 17 | podSubnet: "10.244.0.0/16" 18 | --- 19 | kind: KubeletConfiguration 20 | apiVersion: kubelet.config.k8s.io/v1beta1 21 | runtimeRequestTimeout: "15m" 22 | cgroupDriver: "systemd" 23 | systemReserved: 24 | cpu: 100m 25 | memory: 350M 26 | kubeReserved: 27 | cpu: 100m 28 | memory: 50M 29 | enforceNodeAllocatable: 30 | - pods 31 | 32 | - name: initialize the cluster (this could take some time) 33 | shell: kubeadm init --config /etc/kubernetes/kubeadm-config.yaml >> cluster_initialized.log 34 | args: 35 | chdir: /home/ubuntu 36 | creates: cluster_initialized.log 37 | 38 | - name: create .kube directory 39 | become: yes 40 | become_user: ubuntu 41 | file: 42 | path: $HOME/.kube 43 | state: directory 44 | mode: 0755 45 | 46 | - name: copy admin.conf to user's kube config 47 | copy: 48 | src: /etc/kubernetes/admin.conf 49 | dest: /home/ubuntu/.kube/config 50 | remote_src: yes 51 | owner: ubuntu 52 | 53 | - name: install Pod network 54 | become: yes 55 | become_user: ubuntu 56 | shell: kubectl apply -f https://raw.githubusercontent.com/flannel-io/flannel/master/Documentation/kube-flannel.yml >> pod_network_setup.log 57 | args: 58 | chdir: $HOME 59 | creates: pod_network_setup.log 60 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | kubernetes-playbooks 2 | ============= 3 | 4 | Ansible playbooks that creates a Kubernetes 1.29 cluster of Openstack instances running Ubuntu 22.04 LTS. 5 | 6 | ## Prerequisites 7 | * Ansible and Python3 installed on the local machine (`# yum install ansible`). 8 | * An OpenStack security group for SSH and ICMP access named `SSH and ICMP`. 9 | * [Terraform](https://www.terraform.io/downloads.html) and [OpenStack CLI tools](https://docs.nrec.no/api.html) installed on the local machine. 10 | 11 | ## Create a `keystone_rc` file 12 | `$ cp keystone_rc.sh.example keystone_rc.sh` 13 | 14 | `$ chmod 0600 keystone_rc.sh` 15 | 16 | The `keystone_rc.sh` file will contain your API password so be careful with where you store it, and make sure it's private. Once it is, add your API password for OpenStack. You can also modify the worker count, network version, etc. 17 | 18 | Then load the file to the shell environment on the local computer: 19 | 20 | `$ source keystone_rc.sh` 21 | 22 | ## Add a public key for the cluster to the API user 23 | SSH key pairs are tied to users, and the dashboard and API user are technically different. The SSH public key therefore has to be [added to the API user](https://docs.openstack.org/python-openstackclient/latest/cli/command-objects/keypair.html#keypair-create) explicitly: 24 | 25 | `$ openstack keypair create --public-key /path/to/keyfile.pub k8s-nodes` 26 | 27 | ## Build the cluster using Terraform 28 | Change directory to `tf-project` and initialize Terraform: 29 | 30 | `$ cd tf-project` 31 | 32 | `$ terraform init` 33 | 34 | Then verify, plan and apply with Terraform: 35 | 36 | `$ terraform validate` 37 | 38 | `$ terraform plan` 39 | 40 | `$ terraform apply` 41 | 42 | Change directory back to the main directory: 43 | 44 | `$ cd ..` 45 | 46 | ## Create an inventory file for Ansible 47 | After creating the cluster on OpenStack, Terraform created a `ansible_inventory` file in the `tf-project` directory. It contains the machine names and IP addresses for the cluster. 48 | 49 | Alternatively, a `hosts` file can be created. Add the IP address to the master and workers in the `hosts` file using a text editor, and make sure each machine can be reached using SSH: 50 | 51 | `$ cp hosts.example hosts` 52 | 53 | `$ vim hosts` 54 | 55 | ## Install Kubernetes dependencies on all servers 56 | `$ ansible-playbook -i tf-project/ansible_inventory playbooks/kube-dependencies.yml` 57 | 58 | ## Initialize the master node 59 | `$ ansible-playbook -i tf-project/ansible_inventory playbooks/master.yml` 60 | 61 | `ssh` onto the master and verify that the master node get status `Ready`: 62 | ``` 63 | $ ssh -i /path/to/ssh-key ubuntu@ 64 | ubuntu@k8s-master-1:~$ kubectl get nodes 65 | NAME STATUS ROLES AGE VERSION 66 | k8s-master-1 Ready control-plane 30s v1.29.0 67 | ``` 68 | 69 | ## Add the worker nodes 70 | `$ ansible-playbook -i tf-project/ansible_inventory playbooks/workers.yml` 71 | 72 | Run `kubectl get nodes` once more on the master node to verify the worker nodes got added. 73 | 74 | ## Change or destroy the cluster 75 | Edit cluster settings in the `keystone_rc.sh` and source it again before re-running `terraform apply` to change the cluster, before re-running the playbooks to add new workers. 76 | 77 | Destroy the cluster when done: 78 | 79 | `$ cd tf-project` 80 | 81 | `$ terraform destroy` 82 | 83 | ## Credits 84 | Based on bsder's Digital Ocean tutorial «[How To Create a Kubernetes 1.11 Cluster Using Kubeadm on Ubuntu 18.04](https://www.digitalocean.com/community/tutorials/how-to-create-a-kubernetes-1-11-cluster-using-kubeadm-on-ubuntu-18-04)». 85 | 86 | ## License 87 | See the [LICENSE](LICENSE.md) file for license rights and limitations (MIT). 88 | -------------------------------------------------------------------------------- /playbooks/kube-dependencies.yml: -------------------------------------------------------------------------------- 1 | - hosts: all 2 | become: yes 3 | tasks: 4 | - fail: 5 | msg: "OS should be Ubuntu 22.04, not {{ ansible_distribution }} {{ ansible_distribution_version }}" 6 | when: ansible_distribution != 'Ubuntu' or ansible_distribution_version != '22.04' 7 | 8 | - name: update APT packages 9 | apt: 10 | update_cache: yes 11 | 12 | - name: reboot and wait for reboot to complete 13 | reboot: 14 | 15 | - name: disable SWAP (Kubeadm requirement) 16 | shell: | 17 | swapoff -a 18 | 19 | - name: disable SWAP in fstab (Kubeadm requirement) 20 | replace: 21 | path: /etc/fstab 22 | regexp: '^([^#].*?\sswap\s+sw\s+.*)$' 23 | replace: '# \1' 24 | 25 | - name: create an empty file for the Containerd module 26 | copy: 27 | content: "" 28 | dest: /etc/modules-load.d/containerd.conf 29 | force: no 30 | 31 | - name: configure modules for Containerd 32 | blockinfile: 33 | path: /etc/modules-load.d/containerd.conf 34 | block: | 35 | overlay 36 | br_netfilter 37 | 38 | - name: create an empty file for Kubernetes sysctl params 39 | copy: 40 | content: "" 41 | dest: /etc/sysctl.d/99-kubernetes-cri.conf 42 | force: no 43 | 44 | - name: configure sysctl params for Kubernetes 45 | lineinfile: 46 | path: /etc/sysctl.d/99-kubernetes-cri.conf 47 | line: "{{ item }}" 48 | with_items: 49 | - 'net.bridge.bridge-nf-call-iptables = 1' 50 | - 'net.ipv4.ip_forward = 1' 51 | - 'net.bridge.bridge-nf-call-ip6tables = 1' 52 | 53 | - name: apply sysctl params without reboot 54 | command: sysctl --system 55 | 56 | - name: install APT Transport HTTPS 57 | apt: 58 | name: apt-transport-https 59 | state: present 60 | 61 | - name: add Docker apt-key 62 | get_url: 63 | url: https://download.docker.com/linux/ubuntu/gpg 64 | dest: /etc/apt/keyrings/docker-apt-keyring.asc 65 | mode: '0644' 66 | force: true 67 | 68 | - name: add Docker's APT repository 69 | apt_repository: 70 | repo: "deb [arch={{ 'amd64' if ansible_architecture == 'x86_64' else 'arm64' }} signed-by=/etc/apt/keyrings/docker-apt-keyring.asc] https://download.docker.com/linux/ubuntu {{ ansible_distribution_release }} stable" 71 | state: present 72 | update_cache: yes 73 | 74 | - name: add Kubernetes apt-key 75 | get_url: 76 | url: https://pkgs.k8s.io/core:/stable:/v1.29/deb/Release.key 77 | dest: /etc/apt/keyrings/kubernetes-apt-keyring.asc 78 | mode: '0644' 79 | force: true 80 | 81 | - name: add Kubernetes' APT repository 82 | apt_repository: 83 | repo: "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.asc] https://pkgs.k8s.io/core:/stable:/v1.29/deb/ /" 84 | state: present 85 | update_cache: yes 86 | 87 | - name: install Containerd 88 | apt: 89 | name: containerd.io 90 | state: present 91 | 92 | - name: create Containerd directory 93 | file: 94 | path: /etc/containerd 95 | state: directory 96 | 97 | - name: add Containerd configuration 98 | shell: /usr/bin/containerd config default > /etc/containerd/config.toml 99 | 100 | - name: configuring the systemd cgroup driver for Containerd 101 | lineinfile: 102 | path: /etc/containerd/config.toml 103 | regexp: ' SystemdCgroup = false' 104 | line: ' SystemdCgroup = true' 105 | 106 | - name: enable the Containerd service and start it 107 | systemd: 108 | name: containerd 109 | state: restarted 110 | enabled: yes 111 | daemon-reload: yes 112 | 113 | - name: install Kubelet 114 | apt: 115 | name: kubelet=1.29.* 116 | state: present 117 | update_cache: true 118 | 119 | - name: install Kubeadm 120 | apt: 121 | name: kubeadm=1.29.* 122 | state: present 123 | 124 | - name: enable the Kubelet service, and enable it persistently 125 | service: 126 | name: kubelet 127 | enabled: yes 128 | 129 | - name: load br_netfilter kernel module 130 | modprobe: 131 | name: br_netfilter 132 | state: present 133 | 134 | - name: set bridge-nf-call-iptables 135 | sysctl: 136 | name: net.bridge.bridge-nf-call-iptables 137 | value: 1 138 | 139 | - name: set ip_forward 140 | sysctl: 141 | name: net.ipv4.ip_forward 142 | value: 1 143 | 144 | - name: reboot and wait for reboot to complete 145 | reboot: 146 | 147 | - hosts: master 148 | become: yes 149 | tasks: 150 | - name: install Kubectl 151 | apt: 152 | name: kubectl=1.29.* 153 | state: present 154 | force: yes # allow downgrades 155 | -------------------------------------------------------------------------------- /tf-project/cluster.tf: -------------------------------------------------------------------------------- 1 | # build instances on Openstack for a Kubernetes cluster 2 | variable "K8S_WORKER_COUNT" {} 3 | variable "K8S_WORKER_FLAVOR" {} 4 | variable "K8S_IMAGE_NAME" {} 5 | variable "K8S_NETWORK_NAME" {} 6 | variable "K8S_KEY_PAIR" {} 7 | variable "K8S_KEY_PAIR_LOCATION" {} 8 | variable "K8S_SECURITY_GROUP" {} 9 | 10 | terraform { 11 | required_version = ">= 0.13" 12 | } 13 | 14 | provider "openstack" {} 15 | 16 | resource "openstack_networking_secgroup_v2" "instance_comms" { 17 | name = "k8s-comms" 18 | description = "Security group for allowing TCP communication for Kubernetes" 19 | delete_default_rules = true 20 | } 21 | 22 | # Allow tcp on port 2379-2380 for IPv4 within security group (etcd) 23 | resource "openstack_networking_secgroup_rule_v2" "rule_k8s_tcp_etcd_ipv4" { 24 | direction = "ingress" 25 | ethertype = "IPv4" 26 | protocol = "tcp" 27 | port_range_min = 2379 28 | port_range_max = 2380 29 | remote_group_id = openstack_networking_secgroup_v2.instance_comms.id 30 | security_group_id = openstack_networking_secgroup_v2.instance_comms.id 31 | } 32 | 33 | # Allow tcp on port 6443 for IPv4 within security group (API server) 34 | resource "openstack_networking_secgroup_rule_v2" "rule_k8s_tcp_6443_ipv4" { 35 | direction = "ingress" 36 | ethertype = "IPv4" 37 | protocol = "tcp" 38 | port_range_min = 6443 39 | port_range_max = 6443 40 | remote_group_id = openstack_networking_secgroup_v2.instance_comms.id 41 | security_group_id = openstack_networking_secgroup_v2.instance_comms.id 42 | } 43 | 44 | # Allow tcp on port 10250 for IPv4 within security group (Kubelet API) 45 | resource "openstack_networking_secgroup_rule_v2" "rule_k8s_tcp_10250_ipv4" { 46 | direction = "ingress" 47 | ethertype = "IPv4" 48 | protocol = "tcp" 49 | port_range_min = 10250 50 | port_range_max = 10250 51 | remote_group_id = openstack_networking_secgroup_v2.instance_comms.id 52 | security_group_id = openstack_networking_secgroup_v2.instance_comms.id 53 | } 54 | 55 | # Allow tcp on port 10257 for IPv4 within security group (kube-controller-manager) 56 | resource "openstack_networking_secgroup_rule_v2" "rule_k8s_tcp_10257_ipv4" { 57 | direction = "ingress" 58 | ethertype = "IPv4" 59 | protocol = "tcp" 60 | port_range_min = 10257 61 | port_range_max = 10257 62 | remote_group_id = openstack_networking_secgroup_v2.instance_comms.id 63 | security_group_id = openstack_networking_secgroup_v2.instance_comms.id 64 | } 65 | 66 | # Allow tcp on port 10259 for IPv4 within security group (kube-scheduler) 67 | resource "openstack_networking_secgroup_rule_v2" "rule_k8s_tcp_10259_ipv4" { 68 | direction = "ingress" 69 | ethertype = "IPv4" 70 | protocol = "tcp" 71 | port_range_min = 10259 72 | port_range_max = 10259 73 | remote_group_id = openstack_networking_secgroup_v2.instance_comms.id 74 | security_group_id = openstack_networking_secgroup_v2.instance_comms.id 75 | } 76 | 77 | # Allow tcp on port 30000-32767 for IPv4 within security group (NodePort Services†) 78 | resource "openstack_networking_secgroup_rule_v2" "rule_k8s_tcp_nodeport_ipv4" { 79 | direction = "ingress" 80 | ethertype = "IPv4" 81 | protocol = "tcp" 82 | port_range_min = 30000 83 | port_range_max = 32767 84 | remote_group_id = openstack_networking_secgroup_v2.instance_comms.id 85 | security_group_id = openstack_networking_secgroup_v2.instance_comms.id 86 | } 87 | 88 | resource "openstack_compute_instance_v2" "master_instance" { 89 | count = 1 90 | name = "k8s-master-${count.index+1}" 91 | image_name = var.K8S_IMAGE_NAME 92 | flavor_name = var.K8S_WORKER_COUNT < 11 ? "m1.large" : "m1.xlarge" 93 | 94 | key_pair = var.K8S_KEY_PAIR 95 | security_groups = [ var.K8S_SECURITY_GROUP, openstack_networking_secgroup_v2.instance_comms.name ] 96 | 97 | network { 98 | name = var.K8S_NETWORK_NAME 99 | } 100 | } 101 | 102 | resource "openstack_compute_instance_v2" "worker_instance" { 103 | count = var.K8S_WORKER_COUNT 104 | name = "k8s-worker-${count.index+1}" 105 | image_name = var.K8S_IMAGE_NAME 106 | 107 | flavor_name = var.K8S_WORKER_FLAVOR 108 | 109 | key_pair = var.K8S_KEY_PAIR 110 | security_groups = [ var.K8S_SECURITY_GROUP, openstack_networking_secgroup_v2.instance_comms.name ] 111 | 112 | network { 113 | name = var.K8S_NETWORK_NAME 114 | } 115 | } 116 | 117 | resource "local_file" "ansible_inventory" { 118 | content = "[master]\n${openstack_compute_instance_v2.master_instance[0].name} ansible_host=${openstack_compute_instance_v2.master_instance[0].access_ip_v4}\n\n[workers]\n${join("\n", 119 | formatlist( 120 | "%s ansible_host=%s", 121 | openstack_compute_instance_v2.worker_instance.*.name, openstack_compute_instance_v2.worker_instance.*.access_ip_v4 122 | ))}\n\n[all:vars]\nansible_python_interpreter=/usr/bin/python3\nansible_ssh_extra_args='-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'\nansible_ssh_private_key_file=${var.K8S_KEY_PAIR_LOCATION}/${var.K8S_KEY_PAIR}\nansible_user=ubuntu" 123 | 124 | file_permission = "0600" 125 | filename = "ansible_inventory" 126 | } 127 | --------------------------------------------------------------------------------