├── .gitignore ├── CHANGELOG.md ├── LICENSE ├── README.md ├── group_vars └── all │ ├── infrastructure.yml │ └── kubernetes.yml ├── inventory.example ├── playbook-install-infrastructure.yml ├── playbook-install-kubernetes.yml ├── playbook-prepare-controlhost.yml └── roles ├── infrastructure ├── hostsfile │ ├── README.md │ ├── tasks │ │ └── main.yml │ └── templates │ │ └── hosts.j2 └── terraform │ ├── README.md │ ├── defaults │ └── main.yml │ ├── tasks │ └── main.yml │ ├── templates │ ├── cloud-init.cfg.j2 │ ├── inventory.j2 │ └── main.tf.j2 │ └── tf-data │ ├── outputs.tf │ └── variables.tf └── kubernetes ├── common ├── README.md ├── defaults │ └── main.yml ├── handlers │ └── main.yml ├── tasks │ ├── disable_swap.yml │ ├── kubernetes.yml │ ├── main.yml │ ├── runtime_crio.yml │ └── upgrade_kernel.yml └── vars │ └── main.yml ├── control ├── README.md ├── defaults │ └── main.yml └── tasks │ └── main.yml ├── master ├── README.md ├── defaults │ └── main.yml ├── files │ └── apps │ │ └── metallb-v0.9.4.yaml ├── tasks │ ├── apps │ │ ├── metallb.yml │ │ └── traefik.yml │ ├── bootstrap_token.yml │ ├── init.yml │ ├── main.yml │ └── network_plugin │ │ ├── cilium.yml │ │ └── main.yml └── vars │ └── main.yml └── worker ├── README.md ├── defaults └── main.yml ├── tasks └── main.yml └── vars └── main.yml /.gitignore: -------------------------------------------------------------------------------- 1 | **/.terraform/ 2 | **/terraform.tfstate 3 | **/terraform.tfstate 4 | **/terraform.tfstate.backup 5 | **/infrastructure/terraform/tf-data/ 6 | **/inventory 7 | **/cloud-init.cfg -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog mosibi-kubernetes 2 | 3 | ## November 12, 2023 4 | 5 | * Replace Centos 8 with Rocky Linux 9 6 | * Update Helm to version 3.13.2 7 | * Update Cilium to version 1.14.3 8 | * MetalLB and Traefik are not installed by default anymore 9 | * In the future mosibi-kubernetes will move to the Cilium interal load balancer and Gateway API 10 | * Remove compatibility with older distributions 11 | * Use FQCN for all Ansible modules calls 12 | * Drop Calico support 13 | * Drop Docker runtime support 14 | * YUM repos for Kubernetes and CRI-O are changed to https://pkgs.k8s.io/ 15 | * CRI-O packages are now from the pre-release repo! This will changed when the stable CRI-O repo is available on https://pkgs.k8s.io/ -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # mosibi-kubernetes 2 | This project contains several Ansible roles to create an virtual infrastructure and install Kubernetes on top of it. The focus is on using [CRI-O](https://cri-o.io/) as the container runtime and [Cilium](https://cilium.io/) for network connectivity and security although other container runtime engines and container network interfaces are also possible. 3 | 4 | The Ansible role `infrastructure` is available to install a cluster of virtual machines and role `kubernetes` and its sub-roles will install and configure the Kubernetes cluster on (virtual) machines. After installation you will have a Kubernetes cluster with the following components: 5 | 6 | * [CRI-O](https://cri-o.io): Container runtime engine. 7 | * [Cilium](https://cilium.io): Cilium provides secure networking and uses [eBPF](https://ebpf.io) to enforce policies and replaces the need for kube-proxy. 8 | * [Traefik](https://traefik.io): Traefik is a HTTP reverse proxy and load balancer that makes deploying microservices easy 9 | * [MetalLB](https://metallb.universe.tf): MetalLB is a load-balancer implementation for bare metal (and virtual) Kubernetes clusters, using standard routing protocols. 10 | 11 | This project is tested with Rocky Linux 9, it will probably also work with other Red Hat compatible distributions. 12 | 13 | On my [personal blog](https://blog.mosibi.nl) you will find an [article](https://blog.mosibi.nl/all/2020/12/27/mosibi-kubernetes.html) I wrote about this project with extra information. 14 | 15 | ## Create the infrastructure 16 | This **optional** step can be used to create a virtual infrastructure where Kubernetes can be installed on. By default it will create five virtual systems, one control host, one master and three workers. The control host is a host where the second part of the installation, installing the Kubernetes cluster, can be executed from. By using a control host, your workstation will not be tainted with software needed for the installation. 17 | 18 | ansible-playbook -i localhost, playbook-install-infrastructure.yml 19 | 20 | ### Custom configuration 21 | The `infrastructure/terraform` role support several parameters to change the configuration of the hosts, see [roles/infrastructure/terraform/README.md](roles/infrastructure/terraform/README.md). Those values can be placed in the file *group_vars/all/infrastructure.yml*, please be aware that the parameter `user.ssh_pub_key` **must be set**, otherwise you can't login in to the hosts after the installation. 22 | 23 | ## Prepare the control host 24 | This is also an **optional** step, the control host is the host from where the Kubernetes installation is instantiated and on which several extra (ansible) packages are installed which are used during and after the installation of Kubernetes. For example the `kubectl` binary will be downloaded and configured for the user running the playbook 25 | 26 | ansible-playbook -i inventory playbook-prepare-controlhost.yml 27 | 28 | ### Inventory 29 | When the `infrastructure` role is used, the file *inventory* is present, else you need to create it yourself, see the file *inventory.example* for an example. 30 | 31 | ### Sync only 32 | To speed up runtime, the playbook `playbook-prepare-controlhost.yml` supports the tag `sync` to only execute the synchronization instead of the complete playbook. 33 | 34 | ansible-playbook -i inventory playbook-prepare-controlhost.yml --tags sync 35 | 36 | ## Install Kubernetes 37 | After installing the nodes, with or without the `infrastructure` role, the installation of the cluster can be executed. If a separate control host is used, execute it from the that host instead of your workstation 38 | 39 | ansible-playbook -i inventory playbook-install-kubernetes.yml 40 | 41 | ### Inventory 42 | When the `infrastructure` role is used, the file *inventory* is present, else you need to create it yourself, see *inventory.example*. 43 | 44 | ### Custom configuration 45 | The subroles under Kubernetes, support several parameters, to set for example the [Kubernetes version](https://storage.googleapis.com/kubernetes-release/release/stable.txt) which must be used, see the README.md files in those subroles for more information and when needed, edit the file *group_vars/all/kubernetes.yml* to change the default values for a parameter. 46 | 47 | ## Cleanup everything 48 | Removing all the nodes and their disks is very easy if you created everything with the infrastructure role. Just change the directory to `roles/infrastructure/terraform/tf-data` and execute `terraform destroy` 49 | 50 | ```lang=shell 51 | cd roles/infrastructure/terraform/tf-data 52 | terraform destroy --var 'domain=example.com' --var 'installation_image=/dev/null' --var 'master_count=1' --var 'worker_count=3' --auto-approve 53 | ``` -------------------------------------------------------------------------------- /group_vars/all/infrastructure.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # domain: example.com 3 | # installation_image: "/tmp/CentOS-8-GenericCloud-8.3.2011-20201204.2.x86_64.qcow2" 4 | -------------------------------------------------------------------------------- /group_vars/all/kubernetes.yml: -------------------------------------------------------------------------------- 1 | --- 2 | kubernetes_version: "v1.20.1" 3 | -------------------------------------------------------------------------------- /inventory.example: -------------------------------------------------------------------------------- 1 | [masters] 2 | k8s-master1 ansible_host=192.168.122.100 ansible_ssh_common_args='-o StrictHostKeyChecking=no' ansible_python_interpreter="/usr/bin/python3" 3 | 4 | [workers] 5 | k8s-worker1 ansible_host=192.168.122.101 ansible_ssh_common_args='-o StrictHostKeyChecking=no' ansible_python_interpreter="/usr/bin/python3" 6 | k8s-worker2 ansible_host=192.168.122.102 ansible_ssh_common_args='-o StrictHostKeyChecking=no' ansible_python_interpreter="/usr/bin/python3" 7 | k8s-worker3 ansible_host=192.168.122.103 ansible_ssh_common_args='-o StrictHostKeyChecking=no' ansible_python_interpreter="/usr/bin/python3" 8 | 9 | [control] 10 | k8s-control ansible_host=192.168.122.99 ansible_ssh_common_args='-o StrictHostKeyChecking=no' ansible_python_interpreter="/usr/bin/python3" 11 | -------------------------------------------------------------------------------- /playbook-install-infrastructure.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Install the infrastructure 3 | hosts: localhost 4 | vars: 5 | ansible_connection: local 6 | ansible_python_interpreter: "/usr/bin/python3" 7 | gather_facts: true 8 | become: false 9 | roles: 10 | - { role: infrastructure/terraform, tags: infrastructure } 11 | 12 | - name: Configure /etc/hosts 13 | hosts: 14 | - control 15 | - masters 16 | - workers 17 | gather_facts: true 18 | become: true 19 | roles: 20 | - { role: infrastructure/hostsfile, tags: infrastructure } 21 | -------------------------------------------------------------------------------- /playbook-install-kubernetes.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Configure the control host 3 | hosts: 4 | - control 5 | become: true 6 | roles: 7 | - { role: kubernetes/control, tags: kubernetes-control } 8 | 9 | - name: Deploy Kubernetes common parts 10 | hosts: 11 | - masters 12 | - workers 13 | gather_facts: true 14 | become: true 15 | roles: 16 | - { role: kubernetes/common, tags: kubernetes-common } 17 | 18 | - name: Execute Kubernetes master parts 19 | environment: 20 | PATH: "{{ ansible_env.PATH }}:/usr/local/bin" 21 | hosts: 22 | - masters 23 | gather_facts: true 24 | become: true 25 | roles: 26 | - { role: kubernetes/master, tags: kubernetes-masters } 27 | 28 | - name: Execute Kubernetes worker parts 29 | hosts: 30 | - workers 31 | gather_facts: true 32 | become: true 33 | roles: 34 | - { role: kubernetes/worker, tags: kubernetes-workers } 35 | 36 | 37 | 38 | 39 | -------------------------------------------------------------------------------- /playbook-prepare-controlhost.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Configure the control host 3 | hosts: 4 | - control 5 | become: false 6 | tasks: 7 | - name: Install packages 8 | ansible.builtin.package: 9 | state: present 10 | name: [ 'rsync', 'epel-release', 'bash-completion' ] 11 | become: true 12 | 13 | - name: Install Ansible 14 | ansible.builtin.package: 15 | state: present 16 | name: ansible 17 | become: true 18 | 19 | - name: Ensure .ssh dir is present 20 | ansible.builtin.file: 21 | path: "{{ ansible_facts['user_dir'] }}/.ssh" 22 | state: directory 23 | mode: 0700 24 | 25 | - name: Install SSH key files 26 | ansible.builtin.copy: 27 | src: "{{ local_home }}/.ssh/{{ item.filename }}" 28 | dest: "{{ ansible_facts['user_dir'] }}/.ssh/{{ item.filename }}" 29 | mode: "{{ item.mode }}" 30 | vars: 31 | local_home: "{{ lookup('env','HOME') }}" 32 | loop: 33 | - { filename: id_rsa, mode: "0600" } 34 | - { filename: id_rsa.pub, mode: "0644" } 35 | 36 | - name: Install collections using ansible-galaxy 37 | ansible.builtin.command: "ansible-galaxy collection install {{ item.namespace }}.{{ item.name }}" 38 | args: 39 | creates: "{{ ansible_facts['user_dir'] }}/.ansible/collections/ansible_collections/{{ item.namespace }}/{{ item.name }}" 40 | loop: 41 | - { namespace: community, name: general } 42 | - { namespace: kubernetes, name: core } 43 | 44 | - name: Synchronize mosibi-kubernetes 45 | ansible.posix.synchronize: 46 | src: "{{ playbook_dir }}" 47 | dest: "{{ ansible_facts['user_dir'] }}" 48 | delegate_to: localhost 49 | tags: 50 | - sync 51 | -------------------------------------------------------------------------------- /roles/infrastructure/hostsfile/README.md: -------------------------------------------------------------------------------- 1 | # hostsfile 2 | Create the file /etc/hosts with information about every node in the inventory file -------------------------------------------------------------------------------- /roles/infrastructure/hostsfile/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: /etc/hosts 3 | ansible.builtin.template: 4 | src: hosts.j2 5 | dest: /etc/hosts 6 | owner: root 7 | group: root 8 | setype: etc_t 9 | mode: 0644 10 | -------------------------------------------------------------------------------- /roles/infrastructure/hostsfile/templates/hosts.j2: -------------------------------------------------------------------------------- 1 | 127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 2 | ::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 3 | {% for host in groups.all %} 4 | {% if host != 'localhost' %} 5 | {{ hostvars[host]['ansible_default_ipv4']['address'] }} {{ hostvars[host]['inventory_hostname_short'] }} {{ hostvars[host]['ansible_fqdn'] }} 6 | {% endif %} 7 | {% endfor %} 8 | -------------------------------------------------------------------------------- /roles/infrastructure/terraform/README.md: -------------------------------------------------------------------------------- 1 | # Terraform 2 | Install virtual machines using Terraform and libvirt 3 | 4 | ## Parameters 5 | * `domain`: Domain name for the virtual machine (Default: `example.com`) 6 | * `master_count`: Number of master nodes to create (Default: `1`) 7 | * `worker_count`: Number of worker nodes to create (Default: `3`) 8 | * `installation_image`: Image used to install the nodes (Default: `https://dl.rockylinux.org/pub/rocky/9/images/x86_64/Rocky-9-GenericCloud.latest.x86_64.qcow2`) 9 | * `user`: Dictionary with user information for the default user which is created on all nodes 10 | * `name`: User name (Default: `Username from user who executes the playbook`) 11 | * `gecos`: Gecos information (Default: `Gecos from user who executes the playbook`) 12 | * `group`: Group name (Default: `same as name`) 13 | * `ssh_pub_key`: Public key for the user (Default: `$HOME/.ssh/id_rsa.pub from user who executes the playbook`) 14 | * `tf_binary_path`: Path to the terraform binary (Default: `False`) 15 | * `debug_terraform`: Enable or disable debugging the role (Default: `false`) 16 | 17 | ## Dependencies 18 | * Terraform: https://www.terraform.io/downloads.html 19 | * Libvirt provider for Terraform: https://github.com/dmacvicar/terraform-provider-libvirt#installing 20 | 21 | 22 | ## Terraform internals 23 | The Ansible module [terraform](https://docs.ansible.com/ansible/latest/collections/community/general/terraform_module.html) is used to install the virtual infrastructure together with the [libvirt provider](https://github.com/dmacvicar/terraform-provider-libvirt) for Terraform. The Terraform state is saved in the directory mosibi-kubernetes/roles/infrastructure/terraform/tf-data. 24 | 25 | ### Remove the installed virtual systems 26 | If you need to remove the virtual infrastructure created with this Ansible role, change to the Terraform state directory (mosibi-kubernetes/roles/infrastructure/terraform/tf-data) and execute the following command, with the right values for the variables. The `installation_image` variable must be set, but is not used during removal, so the /dev/null value is okay. 27 | 28 | ```lang=shell 29 | terraform destroy \ 30 | -auto-approve \ 31 | -var 'domain=example.com' \ 32 | -var 'installation_image=/dev/null' \ 33 | -var 'worker_count=4' \ 34 | -var 'master_count=1' 35 | ``` 36 | -------------------------------------------------------------------------------- /roles/infrastructure/terraform/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | domain: example.com 3 | master_count: 1 4 | worker_count: 3 5 | installation_image: "https://dl.rockylinux.org/pub/rocky/9/images/x86_64/Rocky-9-GenericCloud.latest.x86_64.qcow2" 6 | user: 7 | name: "{{ ansible_facts['user_id'] }}" 8 | gecos: "{{ ansible_facts['user_gecos'] }}" 9 | group: "{{ ansible_facts['user_id'] }}" 10 | ssh_pub_key: "{{ lookup('file', ansible_facts['user_dir'] + '/.ssh/id_rsa.pub') }}" 11 | debug_terraform: false 12 | network_interface_type: false 13 | openvswitch_bridge: false -------------------------------------------------------------------------------- /roles/infrastructure/terraform/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Create cloud-init config for Terraform 3 | ansible.builtin.template: 4 | src: cloud-init.cfg.j2 5 | dest: "{{ playbook_dir }}/roles/infrastructure/terraform/tf-data/cloud-init.cfg" 6 | 7 | - name: Copy Terraform main.tf 8 | ansible.builtin.template: 9 | src: main.tf.j2 10 | dest: "{{ playbook_dir }}/roles/infrastructure/terraform/tf-data/main.tf" 11 | mode: 0644 12 | 13 | - name: Create nodes using Terraform 14 | community.general.terraform: 15 | project_path: "{{ playbook_dir }}/roles/infrastructure/terraform/tf-data" 16 | binary_path: "{{ tf_binary_path | default(omit) }}" 17 | force_init: true 18 | state: present 19 | variables: 20 | domain: "{{ domain }}" 21 | installation_image: "{{ installation_image }}" 22 | master_count: "{{ master_count }}" 23 | worker_count: "{{ worker_count }}" 24 | register: _terraform 25 | until: 26 | - (_terraform.outputs.masters_data.value | community.general.json_query('[*].network_interface[*].addresses') | length) == master_count 27 | - (_terraform.outputs.workers_data.value | community.general.json_query('[*].network_interface[*].addresses') | length) == worker_count 28 | - (_terraform.outputs.k8s_control_data.value.network_interface | community.general.json_query('[*].addresses') | length) == 1 29 | retries: 10 30 | delay: 10 31 | 32 | - name: Debug Terraform output 33 | ansible.builtin.debug: 34 | var: _terraform 35 | when: ( debug_terraform | bool ) 36 | 37 | - name: Create inventory file 38 | ansible.builtin.template: 39 | src: inventory.j2 40 | dest: "{{ playbook_dir }}/inventory" 41 | 42 | - name: Add control node to the running inventory 43 | ansible.builtin.add_host: 44 | name: "{{ _terraform.outputs.k8s_control_data.value.name }}" 45 | ansible_host: "{{ _terraform.outputs.k8s_control_data.value.network_interface[0].addresses[0] }}" 46 | ansible_ssh_common_args: '-o StrictHostKeyChecking=no' 47 | groups: control 48 | 49 | - name: Add master node(s) to the running inventory 50 | ansible.builtin.add_host: 51 | name: "{{ item.name }}" 52 | ansible_host: "{{ item.network_interface[0].addresses[0] }}" 53 | ansible_ssh_common_args: '-o StrictHostKeyChecking=no' 54 | groups: masters 55 | loop_control: 56 | label: "{{ item.name }}" 57 | loop: "{{ _terraform.outputs.masters_data.value }}" 58 | 59 | - name: Add worker node(s) to the running inventory 60 | ansible.builtin.add_host: 61 | name: "{{ item.name }}" 62 | ansible_host: "{{ item.network_interface[0].addresses[0] }}" 63 | ansible_ssh_common_args: '-o StrictHostKeyChecking=no' 64 | groups: workers 65 | loop_control: 66 | label: "{{ item.name }}" 67 | loop: "{{ _terraform.outputs.workers_data.value }}" 68 | 69 | - name: Wait max 300 seconds for port 22 to become open and contain "OpenSSH" 70 | ansible.builtin.wait_for: 71 | port: 22 72 | timeout: 300 73 | host: "{{ item[0][0] }}" 74 | search_regex: OpenSSH 75 | vars: 76 | _masters: "{{ _terraform.outputs.masters_data.value | community.general.json_query('[*].network_interface[*].addresses') }}" 77 | _workers: "{{ _terraform.outputs.workers_data.value | community.general.json_query('[*].network_interface[*].addresses') }}" 78 | _control: "{{ _terraform.outputs.k8s_control_data.value.network_interface | community.general.json_query('[*].addresses') }}" 79 | _all_ips: "{{ [ _control ] + _workers + _masters }}" 80 | loop_control: 81 | label: "{{ item[0][0] }}" 82 | loop: "{{ _all_ips }}" 83 | 84 | -------------------------------------------------------------------------------- /roles/infrastructure/terraform/templates/cloud-init.cfg.j2: -------------------------------------------------------------------------------- 1 | #cloud-config 2 | # vim: syntax=yaml 3 | # 4 | # *********************** 5 | # ---- for more examples look at: ------ 6 | # ---> https://cloudinit.readthedocs.io/en/latest/topics/examples.html 7 | # ****************************** 8 | # 9 | # This is the configuration syntax that the write_files module 10 | # will know how to understand. encoding can be given b64 or gzip or (gz+b64). 11 | # The content will be decoded accordingly and then written to the path that is 12 | # provided. 13 | # 14 | # Note: Content strings here are truncated for example purposes. 15 | ssh_pwauth: True 16 | groups: 17 | - {{ user.group }} 18 | users: 19 | - name: {{ user.name }} 20 | gecos: "{{ user.gecos }}" 21 | primary_group: {{ user.group }} 22 | sudo: ALL=(ALL) NOPASSWD:ALL 23 | ssh_authorized_keys: {{ user.ssh_pub_key }} 24 | hostname: "${hostname}.${domain}" 25 | packages: 26 | - wget 27 | - python3-pip 28 | -------------------------------------------------------------------------------- /roles/infrastructure/terraform/templates/inventory.j2: -------------------------------------------------------------------------------- 1 | [masters] 2 | {% for master in _terraform.outputs.masters_data.value %} 3 | {{ master.name }} ansible_host={{ master.network_interface[0].addresses[0] }} ansible_ssh_common_args='-o StrictHostKeyChecking=no' ansible_python_interpreter="/usr/bin/python3" 4 | {% endfor %} 5 | 6 | [workers] 7 | {% for worker in _terraform.outputs.workers_data.value %} 8 | {{ worker.name }} ansible_host={{ worker.network_interface[0].addresses[0] }} ansible_ssh_common_args='-o StrictHostKeyChecking=no' ansible_python_interpreter="/usr/bin/python3" 9 | {% endfor %} 10 | 11 | [control] 12 | {{ _terraform.outputs.k8s_control_data.value.name }} ansible_host={{ _terraform.outputs.k8s_control_data.value.network_interface[0].addresses[0] }} ansible_ssh_common_args='-o StrictHostKeyChecking=no' ansible_python_interpreter="/usr/bin/python3" 13 | -------------------------------------------------------------------------------- /roles/infrastructure/terraform/templates/main.tf.j2: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_version = ">= 0.13" 3 | required_providers { 4 | libvirt = { 5 | source = "dmacvicar/libvirt" 6 | version = "0.6.2" 7 | } 8 | } 9 | } 10 | 11 | provider "libvirt" { 12 | uri = "qemu:///system" 13 | } 14 | 15 | ### 16 | # k8s-control 17 | ### 18 | data "template_file" "k8s_control" { 19 | template = file("${path.module}/cloud-init.cfg") 20 | vars = { 21 | hostname = "k8s-control" 22 | domain = var.domain 23 | } 24 | } 25 | 26 | resource "libvirt_cloudinit_disk" "k8s_control" { 27 | name = "k8s_control.iso" 28 | user_data = data.template_file.k8s_control.rendered 29 | pool = "default" 30 | } 31 | 32 | resource "libvirt_volume" "k8s_control" { 33 | name = "k8s-control.qcow2" 34 | pool = "default" 35 | source = var.installation_image 36 | format = "qcow2" 37 | } 38 | 39 | resource "libvirt_domain" "k8s_control" { 40 | name = "k8s-control" 41 | memory = "2048" 42 | vcpu = 2 43 | 44 | cpu = { 45 | mode = "host-passthrough" 46 | } 47 | 48 | cloudinit = libvirt_cloudinit_disk.k8s_control.id 49 | 50 | disk { 51 | volume_id = libvirt_volume.k8s_control.id 52 | } 53 | 54 | console { 55 | type = "pty" 56 | target_type = "serial" 57 | target_port = "0" 58 | } 59 | 60 | graphics { 61 | type = "spice" 62 | listen_type = "address" 63 | autoport = true 64 | } 65 | 66 | network_interface { 67 | {% if network_interface_type == 'bridge' %} 68 | bridge = "switch0" 69 | {% else %} 70 | network_name = "default" 71 | {% endif %} 72 | wait_for_lease = true 73 | } 74 | 75 | {% if openvswitch_bridge is sameas true %} 76 | xml { 77 | xslt = file("ovs-port.xsl") 78 | } 79 | {% endif %} 80 | 81 | } 82 | 83 | ### 84 | # k8s-masters 85 | ### 86 | data "template_file" "k8s_master" { 87 | count = var.master_count 88 | template = file("${path.module}/cloud-init.cfg") 89 | vars = { 90 | hostname = "k8s-master${count.index + 1}" 91 | domain = var.domain 92 | } 93 | } 94 | 95 | resource "libvirt_cloudinit_disk" "k8s_master" { 96 | count = var.master_count 97 | name = "k8s_master${count.index + 1}.iso" 98 | user_data = data.template_file.k8s_master[count.index].rendered 99 | pool = "default" 100 | } 101 | 102 | resource "libvirt_volume" "k8s_master" { 103 | count = var.master_count 104 | name = "k8s-master${count.index + 1}.qcow2" 105 | pool = "default" 106 | source = var.installation_image 107 | format = "qcow2" 108 | } 109 | 110 | resource "libvirt_domain" "k8s_master" { 111 | count = var.master_count 112 | name = "k8s-master${count.index + 1}" 113 | memory = "4096" 114 | vcpu = 2 115 | 116 | cpu = { 117 | mode = "host-passthrough" 118 | } 119 | 120 | cloudinit = libvirt_cloudinit_disk.k8s_master[count.index].id 121 | 122 | disk { 123 | volume_id = libvirt_volume.k8s_master[count.index].id 124 | } 125 | 126 | console { 127 | type = "pty" 128 | target_type = "serial" 129 | target_port = "0" 130 | } 131 | 132 | graphics { 133 | type = "spice" 134 | listen_type = "address" 135 | autoport = true 136 | } 137 | 138 | network_interface { 139 | {% if network_interface_type == 'bridge' %} 140 | bridge = "switch0" 141 | {% else %} 142 | network_name = "default" 143 | {% endif %} 144 | wait_for_lease = true 145 | } 146 | 147 | {% if openvswitch_bridge is sameas true %} 148 | xml { 149 | xslt = file("ovs-port.xsl") 150 | } 151 | {% endif %} 152 | } 153 | 154 | ### 155 | # k8s-workers 156 | ### 157 | data "template_file" "k8s_worker" { 158 | count = var.worker_count 159 | template = file("${path.module}/cloud-init.cfg") 160 | vars = { 161 | hostname = "k8s-worker${count.index + 1}" 162 | domain = var.domain 163 | } 164 | } 165 | 166 | resource "libvirt_cloudinit_disk" "k8s_worker" { 167 | count = var.worker_count 168 | name = "k8s_worker${count.index + 1}.iso" 169 | user_data = data.template_file.k8s_worker[count.index].rendered 170 | pool = "default" 171 | } 172 | 173 | resource "libvirt_volume" "k8s_worker" { 174 | count = var.worker_count 175 | name = "k8s-worker${count.index + 1}.qcow2" 176 | pool = "default" 177 | source = var.installation_image 178 | format = "qcow2" 179 | } 180 | 181 | resource "libvirt_domain" "k8s_worker" { 182 | count = var.worker_count 183 | name = "k8s-worker${count.index + 1}" 184 | memory = "4096" 185 | vcpu = 2 186 | 187 | cpu = { 188 | mode = "host-passthrough" 189 | } 190 | 191 | cloudinit = libvirt_cloudinit_disk.k8s_worker[count.index].id 192 | 193 | disk { 194 | volume_id = libvirt_volume.k8s_worker[count.index].id 195 | } 196 | 197 | console { 198 | type = "pty" 199 | target_type = "serial" 200 | target_port = "0" 201 | } 202 | 203 | graphics { 204 | type = "spice" 205 | listen_type = "address" 206 | autoport = true 207 | } 208 | 209 | network_interface { 210 | {% if network_interface_type == 'bridge' %} 211 | bridge = "switch0" 212 | {% else %} 213 | network_name = "default" 214 | {% endif %} 215 | wait_for_lease = true 216 | } 217 | 218 | {% if openvswitch_bridge is sameas true %} 219 | xml { 220 | xslt = file("ovs-port.xsl") 221 | } 222 | {% endif %} 223 | } 224 | -------------------------------------------------------------------------------- /roles/infrastructure/terraform/tf-data/outputs.tf: -------------------------------------------------------------------------------- 1 | output "masters_data" { 2 | description = "Masters information" 3 | value = libvirt_domain.k8s_master.* 4 | } 5 | 6 | output "workers_data" { 7 | description = "Worker information" 8 | value = libvirt_domain.k8s_worker.* 9 | } 10 | 11 | output "k8s_control_data" { 12 | description = "k8s-control node information" 13 | value = libvirt_domain.k8s_control 14 | } 15 | -------------------------------------------------------------------------------- /roles/infrastructure/terraform/tf-data/variables.tf: -------------------------------------------------------------------------------- 1 | variable "domain" { 2 | type = string 3 | description = "Top Level Domain" 4 | } 5 | 6 | variable "installation_image" { 7 | type = string 8 | description = "Path to the installation images" 9 | } 10 | 11 | variable "master_count" { 12 | type = number 13 | description = "Number of masters" 14 | } 15 | 16 | variable "worker_count" { 17 | type = number 18 | description = "Number of workers" 19 | } -------------------------------------------------------------------------------- /roles/kubernetes/common/README.md: -------------------------------------------------------------------------------- 1 | # Common 2 | Common settings used for master and workers nodes which are installed using [mosibi-kubernetes](https://github.com/Mosibi/mosibi-kubernetes) 3 | 4 | ## Parameters 5 | * `kubernetes_version`: Kubernetes version to install, for example `v1.28.3` (No default value, must be set) 6 | * `network_plugin`: Configures the network plugin to use. Only cilium is supported (Default: `cilium`) 7 | * `container_runtime_engine`: Configures the container runtime to use. Only crio is supported (Default: `crio`) 8 | * `debug_common`: Enable or disable debugging the role (Default: `false`) 9 | -------------------------------------------------------------------------------- /roles/kubernetes/common/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | debug_common: false 3 | network_plugin: cilium 4 | container_runtime_engine: 'crio' -------------------------------------------------------------------------------- /roles/kubernetes/common/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Restart crio 3 | ansible.builtin.service: 4 | name: crio 5 | state: restarted 6 | 7 | - name: Restart kubelet 8 | ansible.builtin.service: 9 | name: kubelet 10 | state: restarted 11 | -------------------------------------------------------------------------------- /roles/kubernetes/common/tasks/disable_swap.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Kubeadm/kubernetes requires that swap is disabled 3 | 4 | - name: Get swap information 5 | ansible.builtin.command: grep -v ^Filename /proc/swaps 6 | ignore_errors: true 7 | failed_when: false 8 | changed_when: false 9 | register: _swapinfo 10 | 11 | - name: Disable system swap 12 | ansible.builtin.shell: "swapoff -a" 13 | vars: 14 | _nr: "{{ _swapinfo.stdout_lines | count }}" 15 | when: _nr | int > 0 16 | 17 | - name: Remove current swaps from fstab 18 | ansible.builtin.lineinfile: 19 | dest: /etc/fstab 20 | regexp: '(?i)^([^#][\S]+\s+(none|swap)\s+swap.*)' 21 | line: '# \1' 22 | backrefs: yes 23 | state: present 24 | 25 | - name: Disable swappiness 26 | ansible.posix.sysctl: 27 | name: "vm.swappiness" 28 | value: "1" 29 | state: present -------------------------------------------------------------------------------- /roles/kubernetes/common/tasks/kubernetes.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Install/configure the requirements needed by kubeadm/kubernetes 3 | 4 | - name: Kubernetes YUM repository 5 | ansible.builtin.yum_repository: 6 | name: kubernetes 7 | description: "Kubernetes YUM repo" 8 | baseurl: "https://pkgs.k8s.io/core:/stable:/{{ kubernetes_installation_version }}/rpm/" 9 | gpgkey: "https://pkgs.k8s.io/core:/stable:/{{ kubernetes_installation_version }}/rpm/repodata/repomd.xml.key" 10 | gpgcheck: true 11 | repo_gpgcheck: true 12 | 13 | - name: Install required packages for Kubernetes 14 | ansible.builtin.package: 15 | name: "{{ item }}" 16 | state: present 17 | loop: 18 | - "kubelet-{{ rpm_version }}" 19 | - "kubeadm-{{ rpm_version }}" 20 | - iproute-tc 21 | 22 | - name: Kubernetes sysctl settings 23 | ansible.posix.sysctl: 24 | name: "{{ item.name }}" 25 | value: "{{ item.value }}" 26 | loop: 27 | - { name: 'net.ipv4.ip_forward', value: 1 } 28 | - { name: 'net.bridge.bridge-nf-call-iptables', value: 1 } 29 | 30 | - name: Run the kubelet service 31 | ansible.builtin.service: 32 | name: kubelet 33 | state: started 34 | enabled: true 35 | -------------------------------------------------------------------------------- /roles/kubernetes/common/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # TODO: cilium does not like selinux atm, create a policy for it 3 | - name: Put SELinux in permissive mode, logging actions that would be blocked. 4 | ansible.posix.selinux: 5 | policy: targeted 6 | state: permissive 7 | tags: 8 | - common 9 | 10 | - name: Upgrade to newer kernel 11 | ansible.builtin.include_tasks: upgrade_kernel.yml 12 | when: 13 | - network_plugin == 'cilium' 14 | - ansible_kernel is version_compare('4.8','<') 15 | tags: 16 | - common 17 | 18 | - name: Disable firewalld 19 | ansible.builtin.service: 20 | name: firewalld 21 | state: stopped 22 | enabled: false 23 | failed_when: false 24 | tags: 25 | - common 26 | 27 | - name: Disable swap 28 | ansible.builtin.include_tasks: disable_swap.yml 29 | tags: 30 | - common 31 | 32 | - name: Install the container runtime engine 33 | ansible.builtin.include_tasks: runtime_{{ container_runtime_engine }}.yml 34 | tags: 35 | - common 36 | 37 | - name: Install and configure Kubernetes requirements 38 | ansible.builtin.include_tasks: kubernetes.yml 39 | tags: 40 | - common 41 | -------------------------------------------------------------------------------- /roles/kubernetes/common/tasks/runtime_crio.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Configure CRI-O repository 3 | ansible.builtin.yum_repository: 4 | name: crio 5 | description: CRI-O YUM repo 6 | baseurl: "https://pkgs.k8s.io/addons:/cri-o:/{{ _project_path }}/rpm/" 7 | gpgkey: "https://pkgs.k8s.io/addons:/cri-o:/{{ _project_path }}/rpm/repodata/repomd.xml.key" 8 | gpgcheck: yes 9 | vars: 10 | _project_path: 'prerelease:/main' 11 | 12 | - name: Load kernel modules needed by CRI-O 13 | community.general.modprobe: 14 | name: "{{ item }}" 15 | state: present 16 | loop: 17 | - "overlay" 18 | - "br_netfilter" 19 | 20 | - name: Install CRI-O 21 | ansible.builtin.package: 22 | name: cri-o 23 | state: present 24 | 25 | - name: Configure /etc/crictl.yaml 26 | ansible.builtin.copy: 27 | content: "runtime-endpoint: unix:///var/run/crio/crio.sock" 28 | dest: /etc/crictl.yaml 29 | notify: Restart crio 30 | 31 | - name: Service CRI-O 32 | ansible.builtin.service: 33 | name: crio 34 | state: started 35 | enabled: true 36 | -------------------------------------------------------------------------------- /roles/kubernetes/common/tasks/upgrade_kernel.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: elrepo yum repo 3 | ansible.builtin.yum_repository: 4 | name: elrepo 5 | description: 'ELRepo.org Community Enterprise Linux Repository' 6 | baseurl: 7 | - http://elrepo.org/linux/elrepo/el{{ ansible_distribution_major_version }}/$basearch/ 8 | - http://mirrors.coreix.net/elrepo/elrepo/el{{ ansible_distribution_major_version }}/$basearch/ 9 | - http://mirror.rackspace.com/elrepo/elrepo/el{{ ansible_distribution_major_version }}/$basearch/ 10 | - http://repos.lax-noc.com/elrepo/elrepo/el{{ ansible_distribution_major_version }}/$basearch/ 11 | enabled: true 12 | gpgcheck: true 13 | gpgkey: https://www.elrepo.org/RPM-GPG-KEY-elrepo.org 14 | protect: false 15 | 16 | - name: elrepo-kernel yum repo 17 | ansible.builtin.yum_repository: 18 | name: elrepo-kernel 19 | description: 'ELRepo.org Community Enterprise Linux Kernel Repository' 20 | baseurl: 21 | - http://elrepo.org/linux/kernel/el{{ ansible_distribution_major_version }}/$basearch/ 22 | - http://mirrors.coreix.net/elrepo/kernel/el{{ ansible_distribution_major_version }}/$basearch/ 23 | - http://mirror.rackspace.com/elrepo/kernel/el{{ ansible_distribution_major_version }}/$basearch/ 24 | - http://repos.lax-noc.com/elrepo/kernel/el{{ ansible_distribution_major_version }}/$basearch/ 25 | enabled: true 26 | gpgcheck: true 27 | gpgkey: https://www.elrepo.org/RPM-GPG-KEY-elrepo.org 28 | protect: false 29 | 30 | - name: Install kernel-ml package 31 | ansible.builtin.yum: 32 | name: kernel-ml 33 | state: installed 34 | register: _yum_kernel_upgrade 35 | 36 | - name: Set GRUB_DEFAULT to saved 37 | ansible.builtin.lineinfile: 38 | path: /etc/default/grub 39 | regexp: '^GRUB_DEFAULT' 40 | line: 'GRUB_DEFAULT=saved' 41 | register: _set_grub_default 42 | 43 | - name: Run grub2-mkconfig 44 | ansible.builtin.command: grub2-mkconfig -o /boot/grub2/grub.cfg 45 | when: ( _yum_kernel_upgrade.changed == true ) or ( _set_grub_default.changed == true ) 46 | 47 | - name: Set default kernel to new kernel 48 | ansible.builtin.command: grub2-set-default 0 49 | when: ( _yum_kernel_upgrade.changed == true ) or ( _set_grub_default.changed == true ) 50 | 51 | - name: reboot system with new kernel 52 | ansible.builtin.reboot: 53 | when: ( _yum_kernel_upgrade.changed == true ) or ( _set_grub_default.changed == true ) 54 | 55 | - name: Gather facts 56 | ansible.builtin.setup: 57 | when: ( _yum_kernel_upgrade.changed == true ) or ( _set_grub_default.changed == true ) 58 | 59 | - name: Check kernel version 60 | ansible.builtin.fail: 61 | msg: "Kernel version should be higher then 4.8 when Cilium is used as network plugin, but is {{ ansible_facts['kernel'] }}" 62 | when: 63 | - network_plugin == 'cilium' 64 | - ansible_kernel is version_compare('4.8','<') 65 | -------------------------------------------------------------------------------- /roles/kubernetes/common/vars/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Make 1.17.3-0 from v1.17.3 3 | rpm_version: "{{ kubernetes_version | regex_replace('^v', '') }}" 4 | crio_version: "{{ kubernetes_version | regex_replace('^v([0-9]+)\\.([0-9]+)\\.[0-9]+', '\\1.\\2') }}" 5 | kubernetes_installation_version: "{{ kubernetes_version | regex_replace('^(v[0-9]+)\\.([0-9]+)\\.[0-9]+', '\\1.\\2') }}" -------------------------------------------------------------------------------- /roles/kubernetes/control/README.md: -------------------------------------------------------------------------------- 1 | # Control 2 | Configure a control system which can be used to install Kubernetes from, using [mosibi-kubernetes](https://github.com/Mosibi/mosibi-kubernetes) 3 | 4 | ## Parameters 5 | * `kubernetes_version`: Kubernetes version to install, for example `v1.28.3` (No default value, must be set) 6 | * `kubectl_on_control_host`: Wether or not to install kubectl on the control host (Default: `true`) 7 | * `helm_on_control_host`: Wether or not to install helm on the control host (Default: `false`) 8 | * `debug_control`: Enable or disable debugging the role (Default: `false`) 9 | -------------------------------------------------------------------------------- /roles/kubernetes/control/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | debug_control: false 3 | kubectl_on_control_host: true 4 | 5 | # Helm 6 | helm_on_control_host: false 7 | helm_version: v3.13.2 # see https://github.com/helm/helm/releases -------------------------------------------------------------------------------- /roles/kubernetes/control/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Download kubectl and configure local kube config direcory 3 | block: 4 | - name: Download kubectl 5 | get_url: 6 | url: https://storage.googleapis.com/kubernetes-release/release/{{ kubernetes_version }}/bin/linux/amd64/kubectl 7 | dest: /usr/local/bin/kubectl 8 | mode: 0755 9 | tags: 10 | - control 11 | 12 | - name: Get user information 13 | ansible.builtin.command: "getent passwd {{ ansible_facts['env']['SUDO_UID'] | default(0) }}" 14 | changed_when: false 15 | failed_when: false 16 | register: _user_info 17 | tags: 18 | - control 19 | 20 | - name: Create kube config directory 21 | ansible.builtin.file: 22 | path: "{{ _user_info.stdout.split(':')[5] | default('/root') }}/.kube" 23 | state: directory 24 | owner: "{{ ansible_facts['env']['SUDO_UID'] | default(0) }}" 25 | group: "{{ ansible_facts['env']['SUDO_GID'] | default(0) }}" 26 | mode: 0700 27 | tags: 28 | - control 29 | 30 | - name: Configure bash completion 31 | ansible.builtin.lineinfile: 32 | path: "{{ _user_info.stdout.split(':')[5] | default('/root') }}/.bashrc" 33 | owner: "{{ ansible_facts['env']['SUDO_UID'] | default(0) }}" 34 | group: "{{ ansible_facts['env']['SUDO_GID'] | default(0) }}" 35 | create: True 36 | line: "source <(kubectl completion bash)" 37 | when: ansible_facts['user_shell'] == '/bin/bash' 38 | tags: 39 | - control 40 | 41 | when: kubectl_on_control_host == true 42 | 43 | - name: Download and extract Helm 44 | ansible.builtin.unarchive: 45 | src: "https://get.helm.sh/helm-{{ helm_version }}-linux-amd64.tar.gz" 46 | dest: /usr/local/bin 47 | extra_opts: 48 | - --strip=1 49 | - --wildcards 50 | - '*/helm' 51 | remote_src: True 52 | tags: 53 | - helm 54 | - control 55 | when: helm_on_control_host == true 56 | -------------------------------------------------------------------------------- /roles/kubernetes/master/README.md: -------------------------------------------------------------------------------- 1 | # Master 2 | Settings used for Masters nodes which are installed using [mosibi-kubernetes](https://github.com/Mosibi/mosibi-kubernetes) 3 | 4 | ## Parameters 5 | * `debug_master`: Enable or disable debugging the role (Default: `false`) 6 | * `kubernetes_version`: Kubernetes version to install, for example `v1.28.3` (No default value, must be set) 7 | * `apiserver_advertise_address`: Kubeadm setting, the apiserver ip address to advertise (Default: ``) 8 | * `kubeadm_opts`: Extra Kubeadm settings (Default: `""`) 9 | * `helm_version`: Helm version to use (Default: `v3.13.2`) 10 | * `cilium_release`: Cilium version to use (Default: `1.10.5`) 11 | * `cilium_enable_bgp`: Enable BGP (Default: `false`) 12 | * `install_hubble`: Enable or disable the installation of Hubble (Default: `true`) 13 | * `install_traefik`: Enable or disable the installation of Traefik (Default: `false`) 14 | * `install_metallb`: Enable or disable the installation of Metal-LB (Default: `false`) 15 | * `metallb_pool_start`: Start address of the MetalLB pool (Default: `192.168.122.240`) 16 | * `metallb_pool_end`: End address of the MetalLB pool (Default: `192.168.122.250`) 17 | -------------------------------------------------------------------------------- /roles/kubernetes/master/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | debug_master: false 3 | apiserver_advertise_address: "{{ first_master_node_ip_address }}" 4 | service_cidr: "10.96.0.0/12" 5 | pod_network_cidr: "10.217.0.0/16" 6 | kubectl_on_control_host: true 7 | 8 | # Kubeadm 9 | kubeadm_opts: "" 10 | 11 | # Helm 12 | helm_version: v3.13.2 # see https://github.com/helm/helm/releases 13 | 14 | # Cilium 15 | network_plugin: cilium 16 | cilium_release: 1.14.3 # see https://helm.cilium.io/ 17 | install_hubble: true 18 | cilium_enable_bgp: false 19 | hubble_ui_ingress_hostname: hubble.example.com 20 | 21 | # Traefik 22 | install_traefik: false 23 | traefik_ingress_hostname: traefik.example.com 24 | 25 | # MetalLB 26 | install_metallb: false 27 | metallb_pool_start: 192.168.122.240 28 | metallb_pool_end: 192.168.122.250 29 | -------------------------------------------------------------------------------- /roles/kubernetes/master/files/apps/metallb-v0.9.4.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: policy/v1beta1 2 | kind: PodSecurityPolicy 3 | metadata: 4 | labels: 5 | app: metallb 6 | name: controller 7 | namespace: metallb-system 8 | spec: 9 | allowPrivilegeEscalation: false 10 | allowedCapabilities: [] 11 | allowedHostPaths: [] 12 | defaultAddCapabilities: [] 13 | defaultAllowPrivilegeEscalation: false 14 | fsGroup: 15 | ranges: 16 | - max: 65535 17 | min: 1 18 | rule: MustRunAs 19 | hostIPC: false 20 | hostNetwork: false 21 | hostPID: false 22 | privileged: false 23 | readOnlyRootFilesystem: true 24 | requiredDropCapabilities: 25 | - ALL 26 | runAsUser: 27 | ranges: 28 | - max: 65535 29 | min: 1 30 | rule: MustRunAs 31 | seLinux: 32 | rule: RunAsAny 33 | supplementalGroups: 34 | ranges: 35 | - max: 65535 36 | min: 1 37 | rule: MustRunAs 38 | volumes: 39 | - configMap 40 | - secret 41 | - emptyDir 42 | --- 43 | apiVersion: policy/v1beta1 44 | kind: PodSecurityPolicy 45 | metadata: 46 | labels: 47 | app: metallb 48 | name: speaker 49 | namespace: metallb-system 50 | spec: 51 | allowPrivilegeEscalation: false 52 | allowedCapabilities: 53 | - NET_ADMIN 54 | - NET_RAW 55 | - SYS_ADMIN 56 | allowedHostPaths: [] 57 | defaultAddCapabilities: [] 58 | defaultAllowPrivilegeEscalation: false 59 | fsGroup: 60 | rule: RunAsAny 61 | hostIPC: false 62 | hostNetwork: true 63 | hostPID: false 64 | hostPorts: 65 | - max: 7472 66 | min: 7472 67 | privileged: true 68 | readOnlyRootFilesystem: true 69 | requiredDropCapabilities: 70 | - ALL 71 | runAsUser: 72 | rule: RunAsAny 73 | seLinux: 74 | rule: RunAsAny 75 | supplementalGroups: 76 | rule: RunAsAny 77 | volumes: 78 | - configMap 79 | - secret 80 | - emptyDir 81 | --- 82 | apiVersion: v1 83 | kind: ServiceAccount 84 | metadata: 85 | labels: 86 | app: metallb 87 | name: controller 88 | namespace: metallb-system 89 | --- 90 | apiVersion: v1 91 | kind: ServiceAccount 92 | metadata: 93 | labels: 94 | app: metallb 95 | name: speaker 96 | namespace: metallb-system 97 | --- 98 | apiVersion: rbac.authorization.k8s.io/v1 99 | kind: ClusterRole 100 | metadata: 101 | labels: 102 | app: metallb 103 | name: metallb-system:controller 104 | rules: 105 | - apiGroups: 106 | - '' 107 | resources: 108 | - services 109 | verbs: 110 | - get 111 | - list 112 | - watch 113 | - update 114 | - apiGroups: 115 | - '' 116 | resources: 117 | - services/status 118 | verbs: 119 | - update 120 | - apiGroups: 121 | - '' 122 | resources: 123 | - events 124 | verbs: 125 | - create 126 | - patch 127 | - apiGroups: 128 | - policy 129 | resourceNames: 130 | - controller 131 | resources: 132 | - podsecuritypolicies 133 | verbs: 134 | - use 135 | --- 136 | apiVersion: rbac.authorization.k8s.io/v1 137 | kind: ClusterRole 138 | metadata: 139 | labels: 140 | app: metallb 141 | name: metallb-system:speaker 142 | rules: 143 | - apiGroups: 144 | - '' 145 | resources: 146 | - services 147 | - endpoints 148 | - nodes 149 | verbs: 150 | - get 151 | - list 152 | - watch 153 | - apiGroups: 154 | - '' 155 | resources: 156 | - events 157 | verbs: 158 | - create 159 | - patch 160 | - apiGroups: 161 | - policy 162 | resourceNames: 163 | - speaker 164 | resources: 165 | - podsecuritypolicies 166 | verbs: 167 | - use 168 | --- 169 | apiVersion: rbac.authorization.k8s.io/v1 170 | kind: Role 171 | metadata: 172 | labels: 173 | app: metallb 174 | name: config-watcher 175 | namespace: metallb-system 176 | rules: 177 | - apiGroups: 178 | - '' 179 | resources: 180 | - configmaps 181 | verbs: 182 | - get 183 | - list 184 | - watch 185 | --- 186 | apiVersion: rbac.authorization.k8s.io/v1 187 | kind: Role 188 | metadata: 189 | labels: 190 | app: metallb 191 | name: pod-lister 192 | namespace: metallb-system 193 | rules: 194 | - apiGroups: 195 | - '' 196 | resources: 197 | - pods 198 | verbs: 199 | - list 200 | --- 201 | apiVersion: rbac.authorization.k8s.io/v1 202 | kind: ClusterRoleBinding 203 | metadata: 204 | labels: 205 | app: metallb 206 | name: metallb-system:controller 207 | roleRef: 208 | apiGroup: rbac.authorization.k8s.io 209 | kind: ClusterRole 210 | name: metallb-system:controller 211 | subjects: 212 | - kind: ServiceAccount 213 | name: controller 214 | namespace: metallb-system 215 | --- 216 | apiVersion: rbac.authorization.k8s.io/v1 217 | kind: ClusterRoleBinding 218 | metadata: 219 | labels: 220 | app: metallb 221 | name: metallb-system:speaker 222 | roleRef: 223 | apiGroup: rbac.authorization.k8s.io 224 | kind: ClusterRole 225 | name: metallb-system:speaker 226 | subjects: 227 | - kind: ServiceAccount 228 | name: speaker 229 | namespace: metallb-system 230 | --- 231 | apiVersion: rbac.authorization.k8s.io/v1 232 | kind: RoleBinding 233 | metadata: 234 | labels: 235 | app: metallb 236 | name: config-watcher 237 | namespace: metallb-system 238 | roleRef: 239 | apiGroup: rbac.authorization.k8s.io 240 | kind: Role 241 | name: config-watcher 242 | subjects: 243 | - kind: ServiceAccount 244 | name: controller 245 | - kind: ServiceAccount 246 | name: speaker 247 | --- 248 | apiVersion: rbac.authorization.k8s.io/v1 249 | kind: RoleBinding 250 | metadata: 251 | labels: 252 | app: metallb 253 | name: pod-lister 254 | namespace: metallb-system 255 | roleRef: 256 | apiGroup: rbac.authorization.k8s.io 257 | kind: Role 258 | name: pod-lister 259 | subjects: 260 | - kind: ServiceAccount 261 | name: speaker 262 | --- 263 | apiVersion: apps/v1 264 | kind: DaemonSet 265 | metadata: 266 | labels: 267 | app: metallb 268 | component: speaker 269 | name: speaker 270 | namespace: metallb-system 271 | spec: 272 | selector: 273 | matchLabels: 274 | app: metallb 275 | component: speaker 276 | ansible.builtin.template: 277 | metadata: 278 | annotations: 279 | prometheus.io/port: '7472' 280 | prometheus.io/scrape: 'true' 281 | labels: 282 | app: metallb 283 | component: speaker 284 | spec: 285 | containers: 286 | - args: 287 | - --port=7472 288 | - --config=config 289 | env: 290 | - name: METALLB_NODE_NAME 291 | valueFrom: 292 | fieldRef: 293 | fieldPath: spec.nodeName 294 | - name: METALLB_HOST 295 | valueFrom: 296 | fieldRef: 297 | fieldPath: status.hostIP 298 | - name: METALLB_ML_BIND_ADDR 299 | valueFrom: 300 | fieldRef: 301 | fieldPath: status.podIP 302 | # needed when another software is also using memberlist / port 7946 303 | #- name: METALLB_ML_BIND_PORT 304 | # value: "7946" 305 | - name: METALLB_ML_LABELS 306 | value: "app=metallb,component=speaker" 307 | - name: METALLB_ML_NAMESPACE 308 | valueFrom: 309 | fieldRef: 310 | fieldPath: metadata.namespace 311 | - name: METALLB_ML_SECRET_KEY 312 | valueFrom: 313 | secretKeyRef: 314 | name: memberlist 315 | key: secretkey 316 | image: metallb/speaker:v0.9.4 317 | imagePullPolicy: Always 318 | name: speaker 319 | ports: 320 | - containerPort: 7472 321 | name: monitoring 322 | resources: 323 | limits: 324 | cpu: 100m 325 | memory: 100Mi 326 | securityContext: 327 | allowPrivilegeEscalation: false 328 | capabilities: 329 | add: 330 | - NET_ADMIN 331 | - NET_RAW 332 | - SYS_ADMIN 333 | drop: 334 | - ALL 335 | readOnlyRootFilesystem: true 336 | hostNetwork: true 337 | nodeSelector: 338 | beta.kubernetes.io/os: linux 339 | serviceAccountName: speaker 340 | terminationGracePeriodSeconds: 2 341 | tolerations: 342 | - effect: NoSchedule 343 | key: node-role.kubernetes.io/master 344 | --- 345 | apiVersion: apps/v1 346 | kind: Deployment 347 | metadata: 348 | labels: 349 | app: metallb 350 | component: controller 351 | name: controller 352 | namespace: metallb-system 353 | spec: 354 | revisionHistoryLimit: 3 355 | selector: 356 | matchLabels: 357 | app: metallb 358 | component: controller 359 | ansible.builtin.template: 360 | metadata: 361 | annotations: 362 | prometheus.io/port: '7472' 363 | prometheus.io/scrape: 'true' 364 | labels: 365 | app: metallb 366 | component: controller 367 | spec: 368 | containers: 369 | - args: 370 | - --port=7472 371 | - --config=config 372 | image: metallb/controller:v0.9.4 373 | imagePullPolicy: Always 374 | name: controller 375 | ports: 376 | - containerPort: 7472 377 | name: monitoring 378 | resources: 379 | limits: 380 | cpu: 100m 381 | memory: 100Mi 382 | securityContext: 383 | allowPrivilegeEscalation: false 384 | capabilities: 385 | drop: 386 | - all 387 | readOnlyRootFilesystem: true 388 | nodeSelector: 389 | beta.kubernetes.io/os: linux 390 | securityContext: 391 | runAsNonRoot: true 392 | runAsUser: 65534 393 | serviceAccountName: controller 394 | terminationGracePeriodSeconds: 0 395 | -------------------------------------------------------------------------------- /roles/kubernetes/master/tasks/apps/metallb.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # https://metallb.universe.tf/installation/ 3 | 4 | - name: Create namespace metallb 5 | kubernetes.core.k8s: 6 | state: present 7 | definition: 8 | apiVersion: v1 9 | kind: Namespace 10 | metadata: 11 | name: metallb-system 12 | labels: 13 | app: metallb 14 | tags: 15 | - metallb 16 | 17 | - name: Install MetalLB 18 | kubernetes.core.k8s: 19 | state: present 20 | definition: "{{ lookup('file', 'apps/metallb-v0.9.4.yaml') }}" 21 | tags: 22 | - metallb 23 | 24 | - name: Check if the memberlist Secret is present 25 | kubernetes.core.k8s_info: 26 | api_version: v1 27 | kind: Secret 28 | namespace: metallb-system 29 | register: secrets 30 | tags: 31 | - metallb 32 | 33 | - name: Generate secretkey for the memberlist Secret 34 | ansible.builtin.command: 35 | cmd: openssl rand -hex 128 36 | vars: 37 | _secrets: "{{ secrets | community.general.json_query('resources[*].metadata.name') }}" 38 | register: openssl_output 39 | tags: 40 | - metallb 41 | when: not 'memberlist' in _secrets 42 | 43 | - name: Create a Secret memberlist 44 | kubernetes.core.k8s: 45 | state: present 46 | definition: 47 | apiVersion: v1 48 | kind: Secret 49 | metadata: 50 | creationTimestamp: null 51 | name: memberlist 52 | namespace: metallb-system 53 | data: 54 | secretkey: "{{ openssl_output.stdout }}" 55 | vars: 56 | _secrets: "{{ secrets | community.general.json_query('resources[*].metadata.name') }}" 57 | tags: 58 | - metallb 59 | when: not 'memberlist' in _secrets 60 | 61 | - name: Configure MetalLB using a ConfigMap 62 | kubernetes.core.k8s: 63 | state: present 64 | definition: 65 | apiVersion: v1 66 | kind: ConfigMap 67 | metadata: 68 | namespace: metallb-system 69 | name: config 70 | data: 71 | config: | 72 | address-pools: 73 | - name: default 74 | protocol: layer2 75 | addresses: 76 | - {{ metallb_pool_start }}-{{ metallb_pool_end }} 77 | tags: 78 | - metallb 79 | 80 | 81 | -------------------------------------------------------------------------------- /roles/kubernetes/master/tasks/apps/traefik.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # https://doc.traefik.io/traefik/getting-started/install-traefik/#use-the-helm-chart 3 | 4 | - name: Add Traefik Helm repo 5 | kubernetes.core.helm_repository: 6 | name: traefik 7 | repo_url: https://helm.traefik.io/traefik 8 | tags: 9 | - traefik 10 | - master 11 | 12 | - name: Create namespace traefik 13 | kubernetes.core.k8s: 14 | name: traefik 15 | api_version: v1 16 | kind: Namespace 17 | state: present 18 | tags: 19 | - traefik 20 | - master 21 | 22 | - name: Install Traefik 23 | kubernetes.core.helm: 24 | name: traefik 25 | chart_ref: traefik/traefik 26 | release_namespace: traefik 27 | release_values: 28 | ingressRoute: 29 | dashboard: 30 | enabled: false 31 | tags: 32 | - traefik 33 | - master 34 | 35 | - name: Configure an IngressRoute for the Treafik dashboard 36 | kubernetes.core.k8s: 37 | state: present 38 | definition: 39 | apiVersion: traefik.containo.us/v1alpha1 40 | kind: IngressRoute 41 | metadata: 42 | name: dashboard 43 | namespace: traefik 44 | spec: 45 | entryPoints: 46 | - web 47 | routes: 48 | - match: Host(`{{ traefik_ingress_hostname }}`) && (PathPrefix(`/dashboard`) || PathPrefix(`/api`)) 49 | kind: Rule 50 | services: 51 | - name: api@internal 52 | kind: TraefikService 53 | tags: 54 | - traefik 55 | - master 56 | -------------------------------------------------------------------------------- /roles/kubernetes/master/tasks/bootstrap_token.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Get bootstrap token(s) 3 | kubernetes.core.k8s_info: 4 | api_version: v1 5 | kind: Secret 6 | field_selectors: 7 | - type=bootstrap.kubernetes.io/token 8 | namespace: kube-system 9 | register: bootstrap_tokens 10 | 11 | - name: Set bootstrap token fact from existing token 12 | set_fact: 13 | bootstrap_token: "{{ token['token-id'] | b64decode }}.{{ token['token-secret'] | b64decode }}" 14 | vars: 15 | token: "{{ bootstrap_tokens.resources[0].data }}" 16 | when: bootstrap_tokens.resources | length > 0 17 | delegate_facts: True 18 | 19 | - name: Create bootstrap token 20 | ansible.builtin.command: kubeadm token create --ttl 10m 21 | register: bootstrap_token_created 22 | delegate_to: "{{ first_master_node_ip_address }}" 23 | when: bootstrap_tokens.resources | length < 1 24 | 25 | - name: Set bootstrap token fact from new create token 26 | ansible.builtin.set_fact: 27 | bootstrap_token: "{{ bootstrap_token_created.stdout }}" 28 | vars: 29 | token: "{{ bootstrap_tokens.resources[0].data }}" 30 | when: 31 | - bootstrap_tokens.resources | length < 1 32 | - bootstrap_token_created.changed == true 33 | delegate_facts: True -------------------------------------------------------------------------------- /roles/kubernetes/master/tasks/init.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Reset Kubernetes component 3 | ansible.builtin.shell: "kubeadm reset --force" 4 | register: reset_cluster 5 | 6 | - name: Init Kubernetes cluster 7 | when: reset_cluster is succeeded 8 | ansible.builtin.shell: | 9 | kubeadm init --service-cidr {{ service_cidr }} \ 10 | --kubernetes-version {{ kubernetes_version }} \ 11 | --pod-network-cidr {{ pod_network_cidr }} \ 12 | --apiserver-advertise-address {{ apiserver_advertise_address }} \ 13 | --node-name {{ ansible_hostname }} \ 14 | --skip-phases "addon/kube-proxy" \ 15 | {{ extended_kubeadm_opts }} 16 | register: init_cluster 17 | 18 | -------------------------------------------------------------------------------- /roles/kubernetes/master/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Install Openshift python module 3 | pip: 4 | name: openshift 5 | executable: /usr/bin/pip3 6 | 7 | - name: Get user information 8 | ansible.builtin.command: "getent passwd {{ ansible_facts['env']['SUDO_UID'] | default(0) }}" 9 | changed_when: false 10 | failed_when: false 11 | register: _user_info 12 | 13 | - name: Check if installation is already done 14 | stat: 15 | path: "/etc/kubernetes/pki/ca.key" 16 | register: kubeadm_ca_key 17 | 18 | - name: Init cluster if needed 19 | ansible.builtin.include_tasks: init.yml 20 | when: not kubeadm_ca_key.stat.exists 21 | run_once: yes 22 | 23 | - name: Get discovery-token-ca-cert-hash 24 | ansible.builtin.shell: openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt | openssl rsa -pubin -outform der 2>/dev/null | openssl dgst -sha256 -hex | sed 's/^.* //' 25 | register: _discovery_token_ca_cert_hash_content 26 | 27 | - name: Set discovery token ca cert hash fact 28 | set_fact: 29 | discovery_token_ca_cert_hash: "{{ _discovery_token_ca_cert_hash_content.stdout }}" 30 | delegate_facts: True 31 | 32 | - name: Create Kubernetes config directory for the root user 33 | ansible.builtin.file: 34 | path: "/root/.kube/" 35 | state: directory 36 | 37 | - name: Copy admin.conf to /root directory 38 | ansible.builtin.copy: 39 | src: "{{ kubeadmin_config }}" 40 | dest: "/root/.kube/config" 41 | owner: root 42 | group: root 43 | mode: 0600 44 | remote_src: true 45 | 46 | - name: Copy kube config to control host 47 | ansible.builtin.fetch: 48 | src: /root/.kube/config 49 | dest: "{{ _user_info.stdout.split(':')[5] | default('/root') }}/.kube/config" 50 | owner: "{{ ansible_facts['env']['SUDO_UID'] | default(0) }}" 51 | group: "{{ ansible_facts['env']['SUDO_GID'] | default(0) }}" 52 | mode: 0600 53 | flat: yes 54 | when: kubectl_on_control_host == true 55 | 56 | - name: Get or generate kubernetes bootstrap token 57 | ansible.builtin.include_tasks: bootstrap_token.yml 58 | 59 | - name: Download and extract Helm 60 | ansible.builtin.unarchive: 61 | src: "https://get.helm.sh/helm-{{ helm_version }}-linux-amd64.tar.gz" 62 | dest: /usr/local/bin 63 | extra_opts: 64 | - --strip=1 65 | - --wildcards 66 | - '*/helm' 67 | remote_src: True 68 | tags: 69 | - helm 70 | - master 71 | 72 | - name: Install networking 73 | ansible.builtin.include_tasks: network_plugin/main.yml 74 | tags: 75 | - master 76 | - network 77 | - cilium 78 | 79 | ### 80 | # Begin: Apps 81 | ### 82 | - name: Install MetalLB as LoadBalancer 83 | ansible.builtin.include_tasks: apps/metallb.yml 84 | tags: 85 | - metallb 86 | when: install_metallb 87 | 88 | - name: Install Traefik as Ingress Controller 89 | ansible.builtin.include_tasks: apps/traefik.yml 90 | tags: 91 | - traefik 92 | when: install_traefik 93 | 94 | ### 95 | # End: Apps 96 | ### 97 | 98 | - name: Configure an IngressRoute for the Hubble ui 99 | kubernetes.core.k8s: 100 | state: present 101 | definition: 102 | apiVersion: traefik.containo.us/v1alpha1 103 | kind: IngressRoute 104 | metadata: 105 | name: hubble-ui-ingress 106 | namespace: kube-system 107 | spec: 108 | entryPoints: 109 | - web 110 | routes: 111 | - match: Host(`{{ hubble_ui_ingress_hostname }}`) 112 | kind: Rule 113 | services: 114 | - kind: Service 115 | name: hubble-ui 116 | port: 80 117 | tags: 118 | - cilium 119 | - master 120 | when: 121 | - (install_traefik | bool) 122 | - (install_hubble | bool) 123 | -------------------------------------------------------------------------------- /roles/kubernetes/master/tasks/network_plugin/cilium.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # https://github.com/cilium/cilium/issues/10645 3 | # https://docs.cilium.io/en/v1.8/operations/system_requirements/ 4 | # echo 'net.ipv4.conf.lxc*.rp_filter = 0' > /etc/sysctl.d/99-override_cilium_rp_filter.conf 5 | # systemctl restart systemd-sysctl 6 | 7 | - name: Add Cilium Helm repo 8 | kubernetes.core.helm_repository: 9 | name: cilium 10 | repo_url: https://helm.cilium.io 11 | tags: 12 | - cilium 13 | - network 14 | - master 15 | 16 | - name: Install Cilium 17 | kubernetes.core.helm: 18 | name: cilium 19 | chart_ref: cilium/cilium 20 | chart_version: "{{ cilium_release }}" 21 | release_namespace: kube-system 22 | release_values: "{{ cilium_helm_values }}" 23 | tags: 24 | - cilium 25 | - network 26 | - master 27 | 28 | - name: Configure BPG ConfigMap 29 | kubernetes.core.k8s: 30 | state: present 31 | definition: 32 | apiVersion: v1 33 | kind: ConfigMap 34 | metadata: 35 | name: bgp-config 36 | namespace: kube-system 37 | data: 38 | config.yaml: | 39 | peers: 40 | - peer-address: 192.168.1.1 41 | peer-asn: 64512 42 | my-asn: 64512 43 | address-pools: 44 | - name: default 45 | protocol: bgp 46 | addresses: 47 | - 172.16.10.0/24 48 | when: (cilium_enable_bgp | bool) 49 | tags: 50 | - cilium 51 | - network 52 | - master 53 | -------------------------------------------------------------------------------- /roles/kubernetes/master/tasks/network_plugin/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Install network plugin 3 | ansible.builtin.include_tasks: "{{ network_plugin }}.yml" 4 | tags: 5 | - master 6 | - network 7 | - cilium -------------------------------------------------------------------------------- /roles/kubernetes/master/vars/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | kubeadmin_config: /etc/kubernetes/admin.conf 3 | 4 | # The ip address of the first master node is 5 | # used to bootstrap a new cluster 6 | first_master_node_ip_address: "{{ hostvars[groups['masters'][0]]['ansible_default_ipv4'].address | default(groups['masters'][0]) }}" 7 | 8 | # Extent kubeadm options when Cilium is used 9 | extended_kubeadm_opts: "{{ (network_plugin == 'cilium') | ternary(kubeadm_opts + ' --skip-phases=addon/kube-proxy', kubeadm_opts) }}" 10 | 11 | cilium_helm_values: 12 | externalIPs: 13 | enabled: true 14 | hostPort: 15 | enabled: true 16 | kubeProxyReplacement: true 17 | k8sServiceHost: "{{ ansible_facts['default_ipv4']['address'] }}" 18 | k8sServicePort: 6443 19 | loadBalancer: 20 | algorithm: maglev 21 | nodeinit: 22 | enabled: true 23 | nodePort: 24 | enabled: true 25 | operator: 26 | replicas: 1 27 | bgp: 28 | enabled: "{{ cilium_enable_bgp | bool }}" 29 | announce: 30 | loadbalancerIP: true 31 | hubble: 32 | enabled: "{{ install_hubble | bool }}" 33 | metrics: 34 | enabled: 35 | - dns 36 | - drop 37 | - tcp 38 | - flow 39 | - port-distribution 40 | - icmp 41 | - http 42 | listenAddress: ":4244" 43 | relay: 44 | enabled: true 45 | ui: 46 | enabled: true 47 | -------------------------------------------------------------------------------- /roles/kubernetes/worker/README.md: -------------------------------------------------------------------------------- 1 | # Worker 2 | Settings used for Worker nodes which are installed using [mosibi-kubernetes](https://github.com/Mosibi/mosibi-kubernetes) 3 | 4 | ## Parameters 5 | * `debug_worker`: Enable or disable debugging the role (Default: `false`) 6 | -------------------------------------------------------------------------------- /roles/kubernetes/worker/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | debug_worker: false -------------------------------------------------------------------------------- /roles/kubernetes/worker/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Debug join command 3 | ansible.builtin.debug: 4 | msg: "kubeadm join --token {{ bootstrap_token }} {{ first_master_node }}:6443 --discovery-token-ca-cert-hash sha256:{{ discovery_token_ca_cert_hash }}" 5 | when: 6 | - (debug_worker | bool) 7 | 8 | - name: Join cluster 9 | ansible.builtin.command: "kubeadm join --token {{ bootstrap_token }} {{ first_master_node }}:6443 --discovery-token-ca-cert-hash sha256:{{ discovery_token_ca_cert_hash }} --node-name {{ ansible_hostname }}" 10 | args: 11 | creates: /etc/kubernetes/kubelet.conf -------------------------------------------------------------------------------- /roles/kubernetes/worker/vars/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | first_master_node: "{{ groups['masters'][0] }}" 3 | bootstrap_token: "{{ hostvars[first_master_node]['bootstrap_token'] }}" 4 | discovery_token_ca_cert_hash: "{{ hostvars[first_master_node]['discovery_token_ca_cert_hash'] }}" --------------------------------------------------------------------------------