├── .ansible-lint ├── .github ├── ISSUE_TEMPLATE.md └── PULL_REQUEST_TEMPLATE.md ├── .gitignore ├── .yamllint ├── LICENSE ├── README.md ├── ansible.cfg ├── collections └── requirements.yml ├── deploy.sh ├── inventory ├── .gitignore └── sample │ ├── group_vars │ └── all.yml │ └── hosts.ini ├── renovate.json ├── reset.sh ├── reset.yml ├── roles ├── download │ ├── master │ │ └── tasks │ │ │ └── main.yml │ └── node │ │ └── tasks │ │ └── main.yml ├── k3s │ ├── master │ │ ├── defaults │ │ │ └── main.yml │ │ ├── tasks │ │ │ └── main.yml │ │ └── templates │ │ │ ├── k3s.service.j2 │ │ │ ├── metallb.configmap.j2 │ │ │ ├── metallb.ipaddresspool.j2 │ │ │ ├── metallb.namespace.j2 │ │ │ ├── metallb.yaml.j2 │ │ │ ├── vip.rbac.yaml.j2 │ │ │ └── vip.yaml.j2 │ └── node │ │ ├── tasks │ │ └── main.yml │ │ └── templates │ │ └── k3s.service.j2 ├── prereq │ └── tasks │ │ └── main.yml ├── raspberrypi │ ├── handlers │ │ └── main.yml │ └── tasks │ │ ├── main.yml │ │ └── prereq │ │ ├── CentOS.yml │ │ ├── Raspbian.yml │ │ ├── Ubuntu.yml │ │ └── default.yml └── reset │ └── tasks │ ├── main.yml │ └── umount_with_children.yml └── site.yml /.ansible-lint: -------------------------------------------------------------------------------- 1 | --- 2 | skip_list: 3 | - 'fqcn-builtins' 4 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | ## Expected Behavior 7 | 8 | 9 | 10 | ## Current Behavior 11 | 12 | 13 | ## Steps to Reproduce 14 | 15 | 16 | 17 | 1. 18 | 2. 19 | 3. 20 | 4. 21 | 22 | ## Context (variables) 23 | 24 | 25 | Operating system: 26 | 27 | Hardware: 28 | 29 | ### Variables Used: 30 | 31 | `all.yml` 32 | 33 | ```yml 34 | k3s_version: "" 35 | ansible_user: NA 36 | systemd_dir: "" 37 | 38 | cilium_iface: "" 39 | cilium_values: "" 40 | 41 | apiserver_endpoint: "" 42 | 43 | k3s_token: "NA" 44 | 45 | extra_server_args: "" 46 | extra_agent_args: "" 47 | 48 | kube_vip_tag_version: "" 49 | 50 | metal_lb_speaker_tag_version: "" 51 | metal_lb_controller_tag_version: "" 52 | 53 | metal_lb_ip_range: "" 54 | ``` 55 | 56 | ### Hosts 57 | 58 | `host.ini` 59 | 60 | ```ini 61 | [master] 62 | IP.ADDRESS.ONE 63 | IP.ADDRESS.TWO 64 | IP.ADDRESS.THREE 65 | 66 | [node] 67 | IP.ADDRESS.FOUR 68 | IP.ADDRESS.FIVE 69 | 70 | [k3s_cluster:children] 71 | master 72 | node 73 | ``` 74 | 75 | ## Possible Solution 76 | 77 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | # Proposed Changes 2 | 3 | 4 | - 5 | - 6 | - 7 | 8 | ## Checklist 9 | 10 | - [ ] Tested locally 11 | - [ ] Ran `site.yml` playbook 12 | - [ ] Ran `reset.yml` playbook 13 | - [ ] Did not add any unnecessary changes 14 | - [ ] 🚀 15 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .vagrant 2 | deploy-* 3 | reset-* 4 | vagrant/ 5 | -------------------------------------------------------------------------------- /.yamllint: -------------------------------------------------------------------------------- 1 | --- 2 | extends: default 3 | 4 | rules: 5 | line-length: 6 | max: 120 7 | level: warning 8 | truthy: 9 | allowed-values: ['true', 'false', 'yes', 'no'] 10 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Automated build of HA k3s Cluster with `kube-vip`, Cilium, and MetalLB 2 | 3 | This playbook will build a HA Kubernetes cluster with `k3s`, `kube-vip`, MetalLB, and Cilium via `ansible`. 4 | >Cilium installation is done via helm to ensure easy upgrades and future customization of values 5 | 6 | ## 📖 k3s Ansible Playbook 7 | 8 | Build a Kubernetes cluster with a built-in service mesh using Ansible with k3s. The goal is easily install a HA Kubernetes cluster on machines running: 9 | 10 | - [X] Debian 11 | - [X] Ubuntu 12 | - [X] CentOS 13 | 14 | on processor architecture: 15 | 16 | - [X] x64 17 | - [X] arm64 18 | - [X] armhf 19 | 20 | ## ✅ System requirements 21 | 22 | - Deployment environment must have Ansible 2.4.0+. 23 | - `server` and `agent` nodes should have passwordless SSH access, if not you can supply arguments to provide credentials `--ask-pass --ask-become-pass` to each command. 24 | 25 | ## 🚀 Getting Started 26 | 27 | ### 🍴 Preparation 28 | 29 | First create a new directory based on the `sample` directory within the `inventory` directory: 30 | 31 | ```bash 32 | cp -R inventory/sample inventory/my-cluster 33 | ``` 34 | 35 | Second, edit `inventory/my-cluster/hosts.ini` to match the system information gathered above 36 | 37 | For example: 38 | 39 | ```ini 40 | [master] 41 | 192.168.30.38 42 | 192.168.30.39 43 | 192.168.30.40 44 | 45 | [node] 46 | 192.168.30.41 47 | 192.168.30.42 48 | 49 | [k3s_cluster:children] 50 | master 51 | node 52 | ``` 53 | 54 | If multiple hosts are in the master group, the playbook will automatically set up k3s in [HA mode with etcd](https://rancher.com/docs/k3s/latest/en/installation/ha-embedded/). 55 | 56 | This requires at least k3s version `1.19.1` however the version is configurable by using the `k3s_version` variable. 57 | 58 | If needed, you can also edit `inventory/my-cluster/group_vars/all.yml` to match your environment. 59 | 60 | ### ☸️ Create Cluster 61 | 62 | Start provisioning of the cluster using the following command: 63 | 64 | ```bash 65 | ansible-playbook site.yml -i inventory/my-cluster/hosts.ini 66 | ``` 67 | 68 | After deployment control plane will be accessible via virtual ip-address which is defined in inventory/group_vars/all.yml as `apiserver_endpoint` 69 | 70 | ### 🔥 Remove k3s cluster 71 | 72 | ```bash 73 | ansible-playbook reset.yml -i inventory/my-cluster/hosts.ini 74 | ``` 75 | 76 | >You should also reboot these nodes due to the VIP not being destroyed 77 | 78 | ## ⚙️ Kube Config 79 | 80 | To copy your `kube config` locally so that you can access your **Kubernetes** cluster run: 81 | 82 | ```bash 83 | scp debian@master_ip:~/.kube/config ~/.kube/config 84 | ``` 85 | --- 86 | 87 | Thanks to these repos for code and ideas: 88 | 89 | - [k3s-io/k3s-ansible](https://github.com/k3s-io/k3s-ansible) 90 | - [techno-tim/k3s-ansible](https://github.com/techno-tim/k3s-ansible) 91 | -------------------------------------------------------------------------------- /ansible.cfg: -------------------------------------------------------------------------------- 1 | [defaults] 2 | nocows = True 3 | roles_path = ./roles 4 | inventory = ./hosts.ini 5 | 6 | remote_tmp = $HOME/.ansible/tmp 7 | local_tmp = $HOME/.ansible/tmp 8 | pipelining = True 9 | become = True 10 | host_key_checking = False 11 | deprecation_warnings = False 12 | callback_whitelist = profile_tasks 13 | -------------------------------------------------------------------------------- /collections/requirements.yml: -------------------------------------------------------------------------------- 1 | --- 2 | collections: 3 | - name: community.general 4 | - name: kubernetes.core 5 | -------------------------------------------------------------------------------- /deploy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ansible-playbook site.yml -i inventory/my-cluster/hosts.ini -------------------------------------------------------------------------------- /inventory/.gitignore: -------------------------------------------------------------------------------- 1 | /* 2 | !.gitignore 3 | !sample/ 4 | -------------------------------------------------------------------------------- /inventory/sample/group_vars/all.yml: -------------------------------------------------------------------------------- 1 | --- 2 | k3s_version: v1.24.3+k3s1 3 | # this is the user that has ssh access to these machines 4 | ansible_user: $USERNAME 5 | systemd_dir: /etc/systemd/system 6 | 7 | # Set your timezone 8 | system_timezone: "America/Los_Angeles" 9 | 10 | # interface which will be used for cilium 11 | cilium_iface: "eth0" 12 | 13 | # desired values for cilium helm install 14 | cilium_values: " --version 1.12.1 --set operator.replicas=1 " 15 | 16 | # apiserver_endpoint is virtual ip-address which will be configured on each master 17 | # if you have less than two master nodes you must set this to your master's ip 18 | apiserver_endpoint: "10.0.0.40" 19 | 20 | # k3s_token is required masters can talk together securely 21 | # this token should be alpha numeric only 22 | k3s_token: "$SUPER_SECRET_TOKEN" 23 | 24 | # change these to your liking, the only mandatory value is --disable servicelb 25 | extra_server_args: " --disable servicelb --disable traefik --disable local-storage" 26 | extra_agent_args: "" 27 | 28 | # image tag for kube-vip 29 | kube_vip_tag_version: "v0.5.0" 30 | 31 | # image tag for metal lb 32 | metal_lb_speaker_tag_version: "v0.13.4" 33 | metal_lb_controller_tag_version: "v0.13.4" 34 | 35 | # metallb ip range for load balancer 36 | metal_lb_ip_range: "10.0.0.20-10.0.0.30" 37 | -------------------------------------------------------------------------------- /inventory/sample/hosts.ini: -------------------------------------------------------------------------------- 1 | [master] 2 | 192.168.30.38 3 | 192.168.30.40 4 | 5 | [node] 6 | 192.168.30.41 7 | 192.168.30.42 8 | 9 | [k3s_cluster:children] 10 | master 11 | node -------------------------------------------------------------------------------- /renovate.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://docs.renovatebot.com/renovate-schema.json", 3 | "extends": [ 4 | "config:base" 5 | ] 6 | } 7 | -------------------------------------------------------------------------------- /reset.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ansible-playbook reset.yml -i inventory/my-cluster/hosts.ini 4 | -------------------------------------------------------------------------------- /reset.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - hosts: k3s_cluster 4 | gather_facts: yes 5 | become: yes 6 | roles: 7 | - role: reset 8 | -------------------------------------------------------------------------------- /roles/download/master/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: Download k3s binary x64 4 | get_url: 5 | url: https://github.com/k3s-io/k3s/releases/download/{{ k3s_version }}/k3s 6 | checksum: sha256:https://github.com/k3s-io/k3s/releases/download/{{ k3s_version }}/sha256sum-amd64.txt 7 | dest: /usr/local/bin/k3s 8 | owner: root 9 | group: root 10 | mode: 0755 11 | when: ansible_facts.architecture == "x86_64" 12 | 13 | - name: Download k3s binary arm64 14 | get_url: 15 | url: https://github.com/k3s-io/k3s/releases/download/{{ k3s_version }}/k3s-arm64 16 | checksum: sha256:https://github.com/k3s-io/k3s/releases/download/{{ k3s_version }}/sha256sum-arm64.txt 17 | dest: /usr/local/bin/k3s 18 | owner: root 19 | group: root 20 | mode: 0755 21 | when: 22 | - ( ansible_facts.architecture is search("arm") and 23 | ansible_facts.userspace_bits == "64" ) or 24 | ansible_facts.architecture is search("aarch64") 25 | 26 | - name: Download k3s binary armhf 27 | get_url: 28 | url: https://github.com/k3s-io/k3s/releases/download/{{ k3s_version }}/k3s-armhf 29 | checksum: sha256:https://github.com/k3s-io/k3s/releases/download/{{ k3s_version }}/sha256sum-arm.txt 30 | dest: /usr/local/bin/k3s 31 | owner: root 32 | group: root 33 | mode: 0755 34 | when: 35 | - ansible_facts.architecture is search("arm") 36 | - ansible_facts.userspace_bits == "32" 37 | 38 | - name: Download helm install script 39 | get_url: 40 | url: https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 41 | dest: ~{{ ansible_user }} 42 | owner: root 43 | group: root 44 | mode: 0700 45 | 46 | - name: Install helm 47 | command: >- 48 | ./get-helm-3 49 | 50 | - name: Confirm helm is installed 51 | wait_for: 52 | path: /usr/local/bin/helm 53 | state: present 54 | 55 | - name: Remove helm install script 56 | file: 57 | path: ~{{ ansible_user }}/get-helm-3 58 | state: absent 59 | 60 | -------------------------------------------------------------------------------- /roles/download/node/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: Download k3s binary x64 4 | get_url: 5 | url: https://github.com/k3s-io/k3s/releases/download/{{ k3s_version }}/k3s 6 | checksum: sha256:https://github.com/k3s-io/k3s/releases/download/{{ k3s_version }}/sha256sum-amd64.txt 7 | dest: /usr/local/bin/k3s 8 | owner: root 9 | group: root 10 | mode: 0755 11 | when: ansible_facts.architecture == "x86_64" 12 | 13 | - name: Download k3s binary arm64 14 | get_url: 15 | url: https://github.com/k3s-io/k3s/releases/download/{{ k3s_version }}/k3s-arm64 16 | checksum: sha256:https://github.com/k3s-io/k3s/releases/download/{{ k3s_version }}/sha256sum-arm64.txt 17 | dest: /usr/local/bin/k3s 18 | owner: root 19 | group: root 20 | mode: 0755 21 | when: 22 | - ( ansible_facts.architecture is search("arm") and 23 | ansible_facts.userspace_bits == "64" ) or 24 | ansible_facts.architecture is search("aarch64") 25 | 26 | - name: Download k3s binary armhf 27 | get_url: 28 | url: https://github.com/k3s-io/k3s/releases/download/{{ k3s_version }}/k3s-armhf 29 | checksum: sha256:https://github.com/k3s-io/k3s/releases/download/{{ k3s_version }}/sha256sum-arm.txt 30 | dest: /usr/local/bin/k3s 31 | owner: root 32 | group: root 33 | mode: 0755 34 | when: 35 | - ansible_facts.architecture is search("arm") 36 | - ansible_facts.userspace_bits == "32" -------------------------------------------------------------------------------- /roles/k3s/master/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | ansible_user: root 3 | server_init_args: >- 4 | {% if groups['master'] | length > 1 %} 5 | {% if ansible_host == hostvars[groups['master'][0]]['ansible_host'] | default(groups['master'][0]) %} 6 | --cluster-init 7 | {% else %} 8 | --server https://{{ hostvars[groups['master'][0]]['ansible_host'] | default(groups['master'][0]) }}:6443 9 | {% endif %} 10 | --token {{ k3s_token }} 11 | {% endif %} 12 | {{ extra_server_args | default('') }} 13 | -------------------------------------------------------------------------------- /roles/k3s/master/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: Clean previous runs of k3s-init 4 | systemd: 5 | name: k3s-init 6 | state: stopped 7 | failed_when: false 8 | 9 | - name: Clean previous runs of k3s-init 10 | command: systemctl reset-failed k3s-init 11 | failed_when: false 12 | changed_when: false 13 | args: 14 | warn: false # The ansible systemd module does not support reset-failed 15 | 16 | - name: Create manifests directory on first master 17 | file: 18 | path: /var/lib/rancher/k3s/server/manifests 19 | state: directory 20 | owner: root 21 | group: root 22 | mode: 0644 23 | when: ansible_host == hostvars[groups['master'][0]]['ansible_host'] | default(groups['master'][0]) 24 | 25 | - name: Copy vip rbac manifest to first master 26 | template: 27 | src: "vip.rbac.yaml.j2" 28 | dest: "/var/lib/rancher/k3s/server/manifests/vip-rbac.yaml" 29 | owner: root 30 | group: root 31 | mode: 0644 32 | when: ansible_host == ( hostvars[groups['master'][0]]['ansible_host'] | default(groups['master'][0]) ) and ( groups['master'] | length > 1 ) 33 | 34 | - name: Copy vip manifest to first master 35 | template: 36 | src: "vip.yaml.j2" 37 | dest: "/var/lib/rancher/k3s/server/manifests/vip.yaml" 38 | owner: root 39 | group: root 40 | mode: 0644 41 | when: ansible_host == ( hostvars[groups['master'][0]]['ansible_host'] | default(groups['master'][0]) ) and ( groups['master'] | length > 1 ) 42 | 43 | - name: Copy metallb namespace manifest to first master 44 | template: 45 | src: "metallb.namespace.j2" 46 | dest: "/var/lib/rancher/k3s/server/manifests/metallb-namespace.yaml" 47 | owner: root 48 | group: root 49 | mode: 0644 50 | when: ansible_host == hostvars[groups['master'][0]]['ansible_host'] | default(groups['master'][0]) 51 | 52 | - name: Copy metallb ConfigMap manifest to first master 53 | template: 54 | src: "metallb.ipaddresspool.j2" 55 | dest: "/var/lib/rancher/k3s/server/manifests/metallb-configmap.yaml" 56 | owner: root 57 | group: root 58 | mode: 0644 59 | when: ansible_host == hostvars[groups['master'][0]]['ansible_host'] | default(groups['master'][0]) 60 | 61 | - name: Copy metallb main manifest to first master 62 | template: 63 | src: "metallb.yaml.j2" 64 | dest: "/var/lib/rancher/k3s/server/manifests/metallb.yaml" 65 | owner: root 66 | group: root 67 | mode: 0644 68 | when: ansible_host == hostvars[groups['master'][0]]['ansible_host'] | default(groups['master'][0]) 69 | 70 | - name: Init cluster inside the transient k3s-init service 71 | command: 72 | cmd: "systemd-run -p RestartSec=2 \ 73 | -p Restart=on-failure \ 74 | --unit=k3s-init \ 75 | k3s server --flannel-backend=none --disable-network-policy {{ server_init_args }}" 76 | creates: "{{ systemd_dir }}/k3s.service" 77 | args: 78 | warn: false # The ansible systemd module does not support transient units 79 | 80 | - name: Verification 81 | block: 82 | - name: Verify that all nodes actually joined (check k3s-init.service if this fails) 83 | command: 84 | cmd: k3s kubectl get nodes -l "node-role.kubernetes.io/master=true" -o=jsonpath="{.items[*].metadata.name}" 85 | register: nodes 86 | until: nodes.rc == 0 and (nodes.stdout.split() | length) == (groups['master'] | length) 87 | retries: "{{ retry_count | default(20) }}" 88 | delay: 10 89 | changed_when: false 90 | always: 91 | - name: Kill the temporary service used for initialization 92 | systemd: 93 | name: k3s-init 94 | state: stopped 95 | failed_when: false 96 | 97 | - name: Copy K3s service file 98 | register: k3s_service 99 | template: 100 | src: "k3s.service.j2" 101 | dest: "{{ systemd_dir }}/k3s.service" 102 | owner: root 103 | group: root 104 | mode: 0644 105 | 106 | - name: Enable and check K3s service 107 | systemd: 108 | name: k3s 109 | daemon_reload: yes 110 | state: restarted 111 | enabled: yes 112 | 113 | - name: Wait for node-token 114 | wait_for: 115 | path: /var/lib/rancher/k3s/server/node-token 116 | 117 | - name: Register node-token file access mode 118 | stat: 119 | path: /var/lib/rancher/k3s/server 120 | register: p 121 | 122 | - name: Change file access node-token 123 | file: 124 | path: /var/lib/rancher/k3s/server 125 | mode: "g+rx,o+rx" 126 | 127 | - name: Read node-token from master 128 | slurp: 129 | src: /var/lib/rancher/k3s/server/node-token 130 | register: node_token 131 | 132 | - name: Store Master node-token 133 | set_fact: 134 | token: "{{ node_token.content | b64decode | regex_replace('\n', '') }}" 135 | 136 | - name: Restore node-token file access 137 | file: 138 | path: /var/lib/rancher/k3s/server 139 | mode: "{{ p.stat.mode }}" 140 | 141 | - name: Create directory .kube 142 | file: 143 | path: ~{{ ansible_user }}/.kube 144 | state: directory 145 | owner: "{{ ansible_user }}" 146 | mode: "u=rwx,g=rx,o=" 147 | 148 | - name: Copy config file to user home directory 149 | copy: 150 | src: /etc/rancher/k3s/k3s.yaml 151 | dest: ~{{ ansible_user }}/.kube/config 152 | remote_src: yes 153 | owner: "{{ ansible_user }}" 154 | mode: "u=rw,g=,o=" 155 | 156 | - name: Configure kubectl cluster to https://{{ apiserver_endpoint }}:6443 157 | command: >- 158 | k3s kubectl config set-cluster default 159 | --server=https://{{ apiserver_endpoint }}:6443 160 | --kubeconfig ~{{ ansible_user }}/.kube/config 161 | changed_when: true 162 | 163 | - name: Create kubectl symlink 164 | file: 165 | src: /usr/local/bin/k3s 166 | dest: /usr/local/bin/kubectl 167 | state: link 168 | 169 | - name: Create crictl symlink 170 | file: 171 | src: /usr/local/bin/k3s 172 | dest: /usr/local/bin/crictl 173 | state: link 174 | 175 | - name: Add cilium helm repo 176 | kubernetes.core.helm_repository: 177 | name: cilium 178 | repo_url: "https://helm.cilium.io/" 179 | 180 | - name: Install cilium chart 181 | command: 182 | cmd: "helm install cilium cilium/cilium -n kube-system {{ cilium_values }} --kubeconfig ~{{ ansible_user }}/.kube/config" -------------------------------------------------------------------------------- /roles/k3s/master/templates/k3s.service.j2: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Lightweight Kubernetes 3 | Documentation=https://k3s.io 4 | After=network-online.target 5 | 6 | [Service] 7 | Type=notify 8 | ExecStartPre=-/sbin/modprobe br_netfilter 9 | ExecStartPre=-/sbin/modprobe overlay 10 | ExecStart=/usr/local/bin/k3s server {{ extra_server_args | default("") }} 11 | KillMode=process 12 | Delegate=yes 13 | # Having non-zero Limit*s causes performance problems due to accounting overhead 14 | # in the kernel. We recommend using cgroups to do container-local accounting. 15 | LimitNOFILE=1048576 16 | LimitNPROC=infinity 17 | LimitCORE=infinity 18 | TasksMax=infinity 19 | TimeoutStartSec=0 20 | Restart=always 21 | RestartSec=5s 22 | 23 | [Install] 24 | WantedBy=multi-user.target 25 | -------------------------------------------------------------------------------- /roles/k3s/master/templates/metallb.configmap.j2: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | namespace: metallb-system 5 | name: config 6 | data: 7 | config: | 8 | address-pools: 9 | - name: default 10 | protocol: layer2 11 | addresses: 12 | - {{ metal_lb_ip_range }} 13 | -------------------------------------------------------------------------------- /roles/k3s/master/templates/metallb.ipaddresspool.j2: -------------------------------------------------------------------------------- 1 | apiVersion: metallb.io/v1beta1 2 | kind: IPAddressPool 3 | metadata: 4 | name: first-pool 5 | namespace: metallb-system 6 | spec: 7 | addresses: 8 | - {{ metal_lb_ip_range }} 9 | --- 10 | apiVersion: metallb.io/v1beta1 11 | kind: L2Advertisement 12 | metadata: 13 | name: default 14 | namespace: metallb-system -------------------------------------------------------------------------------- /roles/k3s/master/templates/metallb.namespace.j2: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: metallb-system 5 | labels: 6 | app: metallb 7 | -------------------------------------------------------------------------------- /roles/k3s/master/templates/metallb.yaml.j2: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | labels: 5 | pod-security.kubernetes.io/audit: privileged 6 | pod-security.kubernetes.io/enforce: privileged 7 | pod-security.kubernetes.io/warn: privileged 8 | name: metallb-system 9 | --- 10 | apiVersion: apiextensions.k8s.io/v1 11 | kind: CustomResourceDefinition 12 | metadata: 13 | annotations: 14 | controller-gen.kubebuilder.io/version: v0.7.0 15 | name: addresspools.metallb.io 16 | spec: 17 | conversion: 18 | strategy: Webhook 19 | webhook: 20 | clientConfig: 21 | caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tDQpNSUlGWlRDQ0EwMmdBd0lCQWdJVU5GRW1XcTM3MVpKdGkrMmlSQzk1WmpBV1MxZ3dEUVlKS29aSWh2Y05BUUVMDQpCUUF3UWpFTE1Ba0dBMVVFQmhNQ1dGZ3hGVEFUQmdOVkJBY01ERVJsWm1GMWJIUWdRMmwwZVRFY01Cb0dBMVVFDQpDZ3dUUkdWbVlYVnNkQ0JEYjIxd1lXNTVJRXgwWkRBZUZ3MHlNakEzTVRrd09UTXlNek5hRncweU1qQTRNVGd3DQpPVE15TXpOYU1FSXhDekFKQmdOVkJBWVRBbGhZTVJVd0V3WURWUVFIREF4RVpXWmhkV3gwSUVOcGRIa3hIREFhDQpCZ05WQkFvTUUwUmxabUYxYkhRZ1EyOXRjR0Z1ZVNCTWRHUXdnZ0lpTUEwR0NTcUdTSWIzRFFFQkFRVUFBNElDDQpEd0F3Z2dJS0FvSUNBUUNxVFpxMWZRcC9vYkdlenhES0o3OVB3Ny94azJwellualNzMlkzb1ZYSm5sRmM4YjVlDQpma2ZZQnY2bndscW1keW5PL2phWFBaQmRQSS82aFdOUDBkdVhadEtWU0NCUUpyZzEyOGNXb3F0MGNTN3pLb1VpDQpvcU1tQ0QvRXVBeFFNZjhRZDF2c1gvVllkZ0poVTZBRXJLZEpIaXpFOUJtUkNkTDBGMW1OVW55Rk82UnRtWFZUDQpidkxsTDVYeTc2R0FaQVBLOFB4aVlDa0NtbDdxN0VnTWNiOXlLWldCYmlxQ3VkTXE5TGJLNmdKNzF6YkZnSXV4DQo1L1pXK2JraTB2RlplWk9ZODUxb1psckFUNzJvMDI4NHNTWW9uN0pHZVZkY3NoUnh5R1VpSFpSTzdkaXZVTDVTDQpmM2JmSDFYbWY1ZDQzT0NWTWRuUUV2NWVaOG8zeWVLa3ZrbkZQUGVJMU9BbjdGbDlFRVNNR2dhOGFaSG1URSttDQpsLzlMSmdDYjBnQmtPT0M0WnV4bWh2aERKV1EzWnJCS3pMQlNUZXN0NWlLNVlwcXRWVVk2THRyRW9FelVTK1lsDQpwWndXY2VQWHlHeHM5ZURsR3lNVmQraW15Y3NTU1UvVno2Mmx6MnZCS21NTXBkYldDQWhud0RsRTVqU2dyMjRRDQp0eGNXLys2N3d5KzhuQlI3UXdqVTFITndVRjBzeERWdEwrZ1NHVERnSEVZSlhZelYvT05zMy94TkpoVFNPSkxNDQpoeXNVdyttaGdackdhbUdXcHVIVU1DUitvTWJzMTc1UkcrQjJnUFFHVytPTjJnUTRyOXN2b0ZBNHBBQm8xd1dLDQpRYjRhY3pmeVVscElBOVFoSmFsZEY3S3dPSHVlV3gwRUNrNXg0T2tvVDBvWVp0dzFiR0JjRGtaSmF3SURBUUFCDQpvMU13VVRBZEJnTlZIUTRFRmdRVW90UlNIUm9IWTEyRFZ4R0NCdEhpb1g2ZmVFQXdId1lEVlIwakJCZ3dGb0FVDQpvdFJTSFJvSFkxMkRWeEdDQnRIaW9YNmZlRUF3RHdZRFZSMFRBUUgvQkFVd0F3RUIvekFOQmdrcWhraUc5dzBCDQpBUXNGQUFPQ0FnRUFSbkpsWWRjMTFHd0VxWnh6RDF2R3BDR2pDN2VWTlQ3aVY1d3IybXlybHdPYi9aUWFEa0xYDQpvVStaOVVXT1VlSXJTdzUydDdmQUpvVVAwSm5iYkMveVIrU1lqUGhvUXNiVHduOTc2ZldBWTduM3FMOXhCd1Y0DQphek41OXNjeUp0dlhMeUtOL2N5ak1ReDRLajBIMFg0bWJ6bzVZNUtzWWtYVU0vOEFPdWZMcEd0S1NGVGgrSEFDDQpab1Q5YnZHS25adnNHd0tYZFF0Wnh0akhaUjVqK3U3ZGtQOTJBT051RFNabS8rWVV4b2tBK09JbzdSR3BwSHNXDQo1ZTdNY0FTVXRtb1FORXd6dVFoVkJaRWQ1OGtKYjUrV0VWbGNzanlXNnRTbzErZ25tTWNqR1BsMWgxR2hVbjV4DQpFY0lWRnBIWXM5YWo1NmpBSjk1MVQvZjhMaWxmTlVnanBLQ0c1bnl0SUt3emxhOHNtdGlPdm1UNEpYbXBwSkI2DQo4bmdHRVluVjUrUTYwWFJ2OEhSSGp1VG9CRHVhaERrVDA2R1JGODU1d09FR2V4bkZpMXZYWUxLVllWb1V2MXRKDQo4dVdUR1pwNllDSVJldlBqbzg5ZytWTlJSaVFYUThJd0dybXE5c0RoVTlqTjA0SjdVL1RvRDFpNHE3VnlsRUc5DQorV1VGNkNLaEdBeTJIaEhwVncyTGFoOS9lUzdZMUZ1YURrWmhPZG1laG1BOCtqdHNZamJadnR5Mm1SWlF0UUZzDQpUU1VUUjREbUR2bVVPRVRmeStpRHdzK2RkWXVNTnJGeVVYV2dkMnpBQU4ydVl1UHFGY2pRcFNPODFzVTJTU3R3DQoxVzAyeUtYOGJEYmZFdjBzbUh3UzliQnFlSGo5NEM1Mjg0YXpsdTBmaUdpTm1OUEM4ckJLRmhBPQ0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ== 22 | service: 23 | name: webhook-service 24 | namespace: metallb-system 25 | path: /convert 26 | conversionReviewVersions: 27 | - v1alpha1 28 | - v1beta1 29 | group: metallb.io 30 | names: 31 | kind: AddressPool 32 | listKind: AddressPoolList 33 | plural: addresspools 34 | singular: addresspool 35 | scope: Namespaced 36 | versions: 37 | - deprecated: true 38 | deprecationWarning: metallb.io v1alpha1 AddressPool is deprecated 39 | name: v1alpha1 40 | schema: 41 | openAPIV3Schema: 42 | description: AddressPool is the Schema for the addresspools API. 43 | properties: 44 | apiVersion: 45 | description: 'APIVersion defines the versioned schema of this representation 46 | of an object. Servers should convert recognized schemas to the latest 47 | internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' 48 | type: string 49 | kind: 50 | description: 'Kind is a string value representing the REST resource this 51 | object represents. Servers may infer this from the endpoint the client 52 | submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' 53 | type: string 54 | metadata: 55 | type: object 56 | spec: 57 | description: AddressPoolSpec defines the desired state of AddressPool. 58 | properties: 59 | addresses: 60 | description: A list of IP address ranges over which MetalLB has authority. 61 | You can list multiple ranges in a single pool, they will all share 62 | the same settings. Each range can be either a CIDR prefix, or an 63 | explicit start-end range of IPs. 64 | items: 65 | type: string 66 | type: array 67 | autoAssign: 68 | default: true 69 | description: AutoAssign flag used to prevent MetallB from automatic 70 | allocation for a pool. 71 | type: boolean 72 | bgpAdvertisements: 73 | description: When an IP is allocated from this pool, how should it 74 | be translated into BGP announcements? 75 | items: 76 | properties: 77 | aggregationLength: 78 | default: 32 79 | description: The aggregation-length advertisement option lets 80 | you “roll up” the /32s into a larger prefix. 81 | format: int32 82 | minimum: 1 83 | type: integer 84 | aggregationLengthV6: 85 | default: 128 86 | description: Optional, defaults to 128 (i.e. no aggregation) 87 | if not specified. 88 | format: int32 89 | type: integer 90 | communities: 91 | description: BGP communities 92 | items: 93 | type: string 94 | type: array 95 | localPref: 96 | description: BGP LOCAL_PREF attribute which is used by BGP best 97 | path algorithm, Path with higher localpref is preferred over 98 | one with lower localpref. 99 | format: int32 100 | type: integer 101 | type: object 102 | type: array 103 | protocol: 104 | description: Protocol can be used to select how the announcement is 105 | done. 106 | enum: 107 | - layer2 108 | - bgp 109 | type: string 110 | required: 111 | - addresses 112 | - protocol 113 | type: object 114 | status: 115 | description: AddressPoolStatus defines the observed state of AddressPool. 116 | type: object 117 | required: 118 | - spec 119 | type: object 120 | served: true 121 | storage: false 122 | subresources: 123 | status: {} 124 | - deprecated: true 125 | deprecationWarning: metallb.io v1beta1 AddressPool is deprecated, consider using 126 | IPAddressPool 127 | name: v1beta1 128 | schema: 129 | openAPIV3Schema: 130 | description: AddressPool represents a pool of IP addresses that can be allocated 131 | to LoadBalancer services. AddressPool is deprecated and being replaced by 132 | IPAddressPool. 133 | properties: 134 | apiVersion: 135 | description: 'APIVersion defines the versioned schema of this representation 136 | of an object. Servers should convert recognized schemas to the latest 137 | internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' 138 | type: string 139 | kind: 140 | description: 'Kind is a string value representing the REST resource this 141 | object represents. Servers may infer this from the endpoint the client 142 | submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' 143 | type: string 144 | metadata: 145 | type: object 146 | spec: 147 | description: AddressPoolSpec defines the desired state of AddressPool. 148 | properties: 149 | addresses: 150 | description: A list of IP address ranges over which MetalLB has authority. 151 | You can list multiple ranges in a single pool, they will all share 152 | the same settings. Each range can be either a CIDR prefix, or an 153 | explicit start-end range of IPs. 154 | items: 155 | type: string 156 | type: array 157 | autoAssign: 158 | default: true 159 | description: AutoAssign flag used to prevent MetallB from automatic 160 | allocation for a pool. 161 | type: boolean 162 | bgpAdvertisements: 163 | description: Drives how an IP allocated from this pool should translated 164 | into BGP announcements. 165 | items: 166 | properties: 167 | aggregationLength: 168 | default: 32 169 | description: The aggregation-length advertisement option lets 170 | you “roll up” the /32s into a larger prefix. 171 | format: int32 172 | minimum: 1 173 | type: integer 174 | aggregationLengthV6: 175 | default: 128 176 | description: Optional, defaults to 128 (i.e. no aggregation) 177 | if not specified. 178 | format: int32 179 | type: integer 180 | communities: 181 | description: BGP communities to be associated with the given 182 | advertisement. 183 | items: 184 | type: string 185 | type: array 186 | localPref: 187 | description: BGP LOCAL_PREF attribute which is used by BGP best 188 | path algorithm, Path with higher localpref is preferred over 189 | one with lower localpref. 190 | format: int32 191 | type: integer 192 | type: object 193 | type: array 194 | protocol: 195 | description: Protocol can be used to select how the announcement is 196 | done. 197 | enum: 198 | - layer2 199 | - bgp 200 | type: string 201 | required: 202 | - addresses 203 | - protocol 204 | type: object 205 | status: 206 | description: AddressPoolStatus defines the observed state of AddressPool. 207 | type: object 208 | required: 209 | - spec 210 | type: object 211 | served: true 212 | storage: true 213 | subresources: 214 | status: {} 215 | status: 216 | acceptedNames: 217 | kind: "" 218 | plural: "" 219 | conditions: [] 220 | storedVersions: [] 221 | --- 222 | apiVersion: apiextensions.k8s.io/v1 223 | kind: CustomResourceDefinition 224 | metadata: 225 | annotations: 226 | controller-gen.kubebuilder.io/version: v0.7.0 227 | creationTimestamp: null 228 | name: bfdprofiles.metallb.io 229 | spec: 230 | group: metallb.io 231 | names: 232 | kind: BFDProfile 233 | listKind: BFDProfileList 234 | plural: bfdprofiles 235 | singular: bfdprofile 236 | scope: Namespaced 237 | versions: 238 | - name: v1beta1 239 | schema: 240 | openAPIV3Schema: 241 | description: BFDProfile represents the settings of the bfd session that can 242 | be optionally associated with a BGP session. 243 | properties: 244 | apiVersion: 245 | description: 'APIVersion defines the versioned schema of this representation 246 | of an object. Servers should convert recognized schemas to the latest 247 | internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' 248 | type: string 249 | kind: 250 | description: 'Kind is a string value representing the REST resource this 251 | object represents. Servers may infer this from the endpoint the client 252 | submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' 253 | type: string 254 | metadata: 255 | type: object 256 | spec: 257 | description: BFDProfileSpec defines the desired state of BFDProfile. 258 | properties: 259 | detectMultiplier: 260 | description: Configures the detection multiplier to determine packet 261 | loss. The remote transmission interval will be multiplied by this 262 | value to determine the connection loss detection timer. 263 | format: int32 264 | maximum: 255 265 | minimum: 2 266 | type: integer 267 | echoInterval: 268 | description: Configures the minimal echo receive transmission interval 269 | that this system is capable of handling in milliseconds. Defaults 270 | to 50ms 271 | format: int32 272 | maximum: 60000 273 | minimum: 10 274 | type: integer 275 | echoMode: 276 | description: Enables or disables the echo transmission mode. This 277 | mode is disabled by default, and not supported on multi hops setups. 278 | type: boolean 279 | minimumTtl: 280 | description: 'For multi hop sessions only: configure the minimum expected 281 | TTL for an incoming BFD control packet.' 282 | format: int32 283 | maximum: 254 284 | minimum: 1 285 | type: integer 286 | passiveMode: 287 | description: 'Mark session as passive: a passive session will not 288 | attempt to start the connection and will wait for control packets 289 | from peer before it begins replying.' 290 | type: boolean 291 | receiveInterval: 292 | description: The minimum interval that this system is capable of receiving 293 | control packets in milliseconds. Defaults to 300ms. 294 | format: int32 295 | maximum: 60000 296 | minimum: 10 297 | type: integer 298 | transmitInterval: 299 | description: The minimum transmission interval (less jitter) that 300 | this system wants to use to send BFD control packets in milliseconds. 301 | Defaults to 300ms 302 | format: int32 303 | maximum: 60000 304 | minimum: 10 305 | type: integer 306 | type: object 307 | status: 308 | description: BFDProfileStatus defines the observed state of BFDProfile. 309 | type: object 310 | type: object 311 | served: true 312 | storage: true 313 | subresources: 314 | status: {} 315 | status: 316 | acceptedNames: 317 | kind: "" 318 | plural: "" 319 | conditions: [] 320 | storedVersions: [] 321 | --- 322 | apiVersion: apiextensions.k8s.io/v1 323 | kind: CustomResourceDefinition 324 | metadata: 325 | annotations: 326 | controller-gen.kubebuilder.io/version: v0.7.0 327 | creationTimestamp: null 328 | name: bgpadvertisements.metallb.io 329 | spec: 330 | group: metallb.io 331 | names: 332 | kind: BGPAdvertisement 333 | listKind: BGPAdvertisementList 334 | plural: bgpadvertisements 335 | singular: bgpadvertisement 336 | scope: Namespaced 337 | versions: 338 | - name: v1beta1 339 | schema: 340 | openAPIV3Schema: 341 | description: BGPAdvertisement allows to advertise the IPs coming from the 342 | selected IPAddressPools via BGP, setting the parameters of the BGP Advertisement. 343 | properties: 344 | apiVersion: 345 | description: 'APIVersion defines the versioned schema of this representation 346 | of an object. Servers should convert recognized schemas to the latest 347 | internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' 348 | type: string 349 | kind: 350 | description: 'Kind is a string value representing the REST resource this 351 | object represents. Servers may infer this from the endpoint the client 352 | submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' 353 | type: string 354 | metadata: 355 | type: object 356 | spec: 357 | description: BGPAdvertisementSpec defines the desired state of BGPAdvertisement. 358 | properties: 359 | aggregationLength: 360 | default: 32 361 | description: The aggregation-length advertisement option lets you 362 | “roll up” the /32s into a larger prefix. Defaults to 32. Works for 363 | IPv4 addresses. 364 | format: int32 365 | minimum: 1 366 | type: integer 367 | aggregationLengthV6: 368 | default: 128 369 | description: The aggregation-length advertisement option lets you 370 | “roll up” the /128s into a larger prefix. Defaults to 128. Works 371 | for IPv6 addresses. 372 | format: int32 373 | type: integer 374 | communities: 375 | description: The BGP communities to be associated with the announcement. 376 | Each item can be a community of the form 1234:1234 or the name of 377 | an alias defined in the Community CRD. 378 | items: 379 | type: string 380 | type: array 381 | ipAddressPoolSelectors: 382 | description: A selector for the IPAddressPools which would get advertised 383 | via this advertisement. If no IPAddressPool is selected by this 384 | or by the list, the advertisement is applied to all the IPAddressPools. 385 | items: 386 | description: A label selector is a label query over a set of resources. 387 | The result of matchLabels and matchExpressions are ANDed. An empty 388 | label selector matches all objects. A null label selector matches 389 | no objects. 390 | properties: 391 | matchExpressions: 392 | description: matchExpressions is a list of label selector requirements. 393 | The requirements are ANDed. 394 | items: 395 | description: A label selector requirement is a selector that 396 | contains values, a key, and an operator that relates the 397 | key and values. 398 | properties: 399 | key: 400 | description: key is the label key that the selector applies 401 | to. 402 | type: string 403 | operator: 404 | description: operator represents a key's relationship 405 | to a set of values. Valid operators are In, NotIn, Exists 406 | and DoesNotExist. 407 | type: string 408 | values: 409 | description: values is an array of string values. If the 410 | operator is In or NotIn, the values array must be non-empty. 411 | If the operator is Exists or DoesNotExist, the values 412 | array must be empty. This array is replaced during a 413 | strategic merge patch. 414 | items: 415 | type: string 416 | type: array 417 | required: 418 | - key 419 | - operator 420 | type: object 421 | type: array 422 | matchLabels: 423 | additionalProperties: 424 | type: string 425 | description: matchLabels is a map of {key,value} pairs. A single 426 | {key,value} in the matchLabels map is equivalent to an element 427 | of matchExpressions, whose key field is "key", the operator 428 | is "In", and the values array contains only "value". The requirements 429 | are ANDed. 430 | type: object 431 | type: object 432 | type: array 433 | ipAddressPools: 434 | description: The list of IPAddressPools to advertise via this advertisement, 435 | selected by name. 436 | items: 437 | type: string 438 | type: array 439 | localPref: 440 | description: The BGP LOCAL_PREF attribute which is used by BGP best 441 | path algorithm, Path with higher localpref is preferred over one 442 | with lower localpref. 443 | format: int32 444 | type: integer 445 | nodeSelectors: 446 | description: NodeSelectors allows to limit the nodes to announce as 447 | next hops for the LoadBalancer IP. When empty, all the nodes having are 448 | announced as next hops. 449 | items: 450 | description: A label selector is a label query over a set of resources. 451 | The result of matchLabels and matchExpressions are ANDed. An empty 452 | label selector matches all objects. A null label selector matches 453 | no objects. 454 | properties: 455 | matchExpressions: 456 | description: matchExpressions is a list of label selector requirements. 457 | The requirements are ANDed. 458 | items: 459 | description: A label selector requirement is a selector that 460 | contains values, a key, and an operator that relates the 461 | key and values. 462 | properties: 463 | key: 464 | description: key is the label key that the selector applies 465 | to. 466 | type: string 467 | operator: 468 | description: operator represents a key's relationship 469 | to a set of values. Valid operators are In, NotIn, Exists 470 | and DoesNotExist. 471 | type: string 472 | values: 473 | description: values is an array of string values. If the 474 | operator is In or NotIn, the values array must be non-empty. 475 | If the operator is Exists or DoesNotExist, the values 476 | array must be empty. This array is replaced during a 477 | strategic merge patch. 478 | items: 479 | type: string 480 | type: array 481 | required: 482 | - key 483 | - operator 484 | type: object 485 | type: array 486 | matchLabels: 487 | additionalProperties: 488 | type: string 489 | description: matchLabels is a map of {key,value} pairs. A single 490 | {key,value} in the matchLabels map is equivalent to an element 491 | of matchExpressions, whose key field is "key", the operator 492 | is "In", and the values array contains only "value". The requirements 493 | are ANDed. 494 | type: object 495 | type: object 496 | type: array 497 | peers: 498 | description: Peers limits the bgppeer to advertise the ips of the 499 | selected pools to. When empty, the loadbalancer IP is announced 500 | to all the BGPPeers configured. 501 | items: 502 | type: string 503 | type: array 504 | type: object 505 | status: 506 | description: BGPAdvertisementStatus defines the observed state of BGPAdvertisement. 507 | type: object 508 | type: object 509 | served: true 510 | storage: true 511 | subresources: 512 | status: {} 513 | status: 514 | acceptedNames: 515 | kind: "" 516 | plural: "" 517 | conditions: [] 518 | storedVersions: [] 519 | --- 520 | apiVersion: apiextensions.k8s.io/v1 521 | kind: CustomResourceDefinition 522 | metadata: 523 | annotations: 524 | controller-gen.kubebuilder.io/version: v0.7.0 525 | name: bgppeers.metallb.io 526 | spec: 527 | conversion: 528 | strategy: Webhook 529 | webhook: 530 | clientConfig: 531 | caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tDQpNSUlGWlRDQ0EwMmdBd0lCQWdJVU5GRW1XcTM3MVpKdGkrMmlSQzk1WmpBV1MxZ3dEUVlKS29aSWh2Y05BUUVMDQpCUUF3UWpFTE1Ba0dBMVVFQmhNQ1dGZ3hGVEFUQmdOVkJBY01ERVJsWm1GMWJIUWdRMmwwZVRFY01Cb0dBMVVFDQpDZ3dUUkdWbVlYVnNkQ0JEYjIxd1lXNTVJRXgwWkRBZUZ3MHlNakEzTVRrd09UTXlNek5hRncweU1qQTRNVGd3DQpPVE15TXpOYU1FSXhDekFKQmdOVkJBWVRBbGhZTVJVd0V3WURWUVFIREF4RVpXWmhkV3gwSUVOcGRIa3hIREFhDQpCZ05WQkFvTUUwUmxabUYxYkhRZ1EyOXRjR0Z1ZVNCTWRHUXdnZ0lpTUEwR0NTcUdTSWIzRFFFQkFRVUFBNElDDQpEd0F3Z2dJS0FvSUNBUUNxVFpxMWZRcC9vYkdlenhES0o3OVB3Ny94azJwellualNzMlkzb1ZYSm5sRmM4YjVlDQpma2ZZQnY2bndscW1keW5PL2phWFBaQmRQSS82aFdOUDBkdVhadEtWU0NCUUpyZzEyOGNXb3F0MGNTN3pLb1VpDQpvcU1tQ0QvRXVBeFFNZjhRZDF2c1gvVllkZ0poVTZBRXJLZEpIaXpFOUJtUkNkTDBGMW1OVW55Rk82UnRtWFZUDQpidkxsTDVYeTc2R0FaQVBLOFB4aVlDa0NtbDdxN0VnTWNiOXlLWldCYmlxQ3VkTXE5TGJLNmdKNzF6YkZnSXV4DQo1L1pXK2JraTB2RlplWk9ZODUxb1psckFUNzJvMDI4NHNTWW9uN0pHZVZkY3NoUnh5R1VpSFpSTzdkaXZVTDVTDQpmM2JmSDFYbWY1ZDQzT0NWTWRuUUV2NWVaOG8zeWVLa3ZrbkZQUGVJMU9BbjdGbDlFRVNNR2dhOGFaSG1URSttDQpsLzlMSmdDYjBnQmtPT0M0WnV4bWh2aERKV1EzWnJCS3pMQlNUZXN0NWlLNVlwcXRWVVk2THRyRW9FelVTK1lsDQpwWndXY2VQWHlHeHM5ZURsR3lNVmQraW15Y3NTU1UvVno2Mmx6MnZCS21NTXBkYldDQWhud0RsRTVqU2dyMjRRDQp0eGNXLys2N3d5KzhuQlI3UXdqVTFITndVRjBzeERWdEwrZ1NHVERnSEVZSlhZelYvT05zMy94TkpoVFNPSkxNDQpoeXNVdyttaGdackdhbUdXcHVIVU1DUitvTWJzMTc1UkcrQjJnUFFHVytPTjJnUTRyOXN2b0ZBNHBBQm8xd1dLDQpRYjRhY3pmeVVscElBOVFoSmFsZEY3S3dPSHVlV3gwRUNrNXg0T2tvVDBvWVp0dzFiR0JjRGtaSmF3SURBUUFCDQpvMU13VVRBZEJnTlZIUTRFRmdRVW90UlNIUm9IWTEyRFZ4R0NCdEhpb1g2ZmVFQXdId1lEVlIwakJCZ3dGb0FVDQpvdFJTSFJvSFkxMkRWeEdDQnRIaW9YNmZlRUF3RHdZRFZSMFRBUUgvQkFVd0F3RUIvekFOQmdrcWhraUc5dzBCDQpBUXNGQUFPQ0FnRUFSbkpsWWRjMTFHd0VxWnh6RDF2R3BDR2pDN2VWTlQ3aVY1d3IybXlybHdPYi9aUWFEa0xYDQpvVStaOVVXT1VlSXJTdzUydDdmQUpvVVAwSm5iYkMveVIrU1lqUGhvUXNiVHduOTc2ZldBWTduM3FMOXhCd1Y0DQphek41OXNjeUp0dlhMeUtOL2N5ak1ReDRLajBIMFg0bWJ6bzVZNUtzWWtYVU0vOEFPdWZMcEd0S1NGVGgrSEFDDQpab1Q5YnZHS25adnNHd0tYZFF0Wnh0akhaUjVqK3U3ZGtQOTJBT051RFNabS8rWVV4b2tBK09JbzdSR3BwSHNXDQo1ZTdNY0FTVXRtb1FORXd6dVFoVkJaRWQ1OGtKYjUrV0VWbGNzanlXNnRTbzErZ25tTWNqR1BsMWgxR2hVbjV4DQpFY0lWRnBIWXM5YWo1NmpBSjk1MVQvZjhMaWxmTlVnanBLQ0c1bnl0SUt3emxhOHNtdGlPdm1UNEpYbXBwSkI2DQo4bmdHRVluVjUrUTYwWFJ2OEhSSGp1VG9CRHVhaERrVDA2R1JGODU1d09FR2V4bkZpMXZYWUxLVllWb1V2MXRKDQo4dVdUR1pwNllDSVJldlBqbzg5ZytWTlJSaVFYUThJd0dybXE5c0RoVTlqTjA0SjdVL1RvRDFpNHE3VnlsRUc5DQorV1VGNkNLaEdBeTJIaEhwVncyTGFoOS9lUzdZMUZ1YURrWmhPZG1laG1BOCtqdHNZamJadnR5Mm1SWlF0UUZzDQpUU1VUUjREbUR2bVVPRVRmeStpRHdzK2RkWXVNTnJGeVVYV2dkMnpBQU4ydVl1UHFGY2pRcFNPODFzVTJTU3R3DQoxVzAyeUtYOGJEYmZFdjBzbUh3UzliQnFlSGo5NEM1Mjg0YXpsdTBmaUdpTm1OUEM4ckJLRmhBPQ0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ== 532 | service: 533 | name: webhook-service 534 | namespace: metallb-system 535 | path: /convert 536 | conversionReviewVersions: 537 | - v1beta1 538 | - v1beta2 539 | group: metallb.io 540 | names: 541 | kind: BGPPeer 542 | listKind: BGPPeerList 543 | plural: bgppeers 544 | singular: bgppeer 545 | scope: Namespaced 546 | versions: 547 | - name: v1beta1 548 | schema: 549 | openAPIV3Schema: 550 | description: BGPPeer is the Schema for the peers API. 551 | properties: 552 | apiVersion: 553 | description: 'APIVersion defines the versioned schema of this representation 554 | of an object. Servers should convert recognized schemas to the latest 555 | internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' 556 | type: string 557 | kind: 558 | description: 'Kind is a string value representing the REST resource this 559 | object represents. Servers may infer this from the endpoint the client 560 | submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' 561 | type: string 562 | metadata: 563 | type: object 564 | spec: 565 | description: BGPPeerSpec defines the desired state of Peer. 566 | properties: 567 | bfdProfile: 568 | type: string 569 | ebgpMultiHop: 570 | description: EBGP peer is multi-hops away 571 | type: boolean 572 | holdTime: 573 | description: Requested BGP hold time, per RFC4271. 574 | type: string 575 | keepaliveTime: 576 | description: Requested BGP keepalive time, per RFC4271. 577 | type: string 578 | myASN: 579 | description: AS number to use for the local end of the session. 580 | format: int32 581 | maximum: 4294967295 582 | minimum: 0 583 | type: integer 584 | nodeSelectors: 585 | description: Only connect to this peer on nodes that match one of 586 | these selectors. 587 | items: 588 | properties: 589 | matchExpressions: 590 | items: 591 | properties: 592 | key: 593 | type: string 594 | operator: 595 | type: string 596 | values: 597 | items: 598 | type: string 599 | minItems: 1 600 | type: array 601 | required: 602 | - key 603 | - operator 604 | - values 605 | type: object 606 | type: array 607 | matchLabels: 608 | additionalProperties: 609 | type: string 610 | type: object 611 | type: object 612 | type: array 613 | password: 614 | description: Authentication password for routers enforcing TCP MD5 615 | authenticated sessions 616 | type: string 617 | peerASN: 618 | description: AS number to expect from the remote end of the session. 619 | format: int32 620 | maximum: 4294967295 621 | minimum: 0 622 | type: integer 623 | peerAddress: 624 | description: Address to dial when establishing the session. 625 | type: string 626 | peerPort: 627 | description: Port to dial when establishing the session. 628 | maximum: 16384 629 | minimum: 0 630 | type: integer 631 | routerID: 632 | description: BGP router ID to advertise to the peer 633 | type: string 634 | sourceAddress: 635 | description: Source address to use when establishing the session. 636 | type: string 637 | required: 638 | - myASN 639 | - peerASN 640 | - peerAddress 641 | type: object 642 | status: 643 | description: BGPPeerStatus defines the observed state of Peer. 644 | type: object 645 | type: object 646 | served: true 647 | storage: false 648 | subresources: 649 | status: {} 650 | - name: v1beta2 651 | schema: 652 | openAPIV3Schema: 653 | description: BGPPeer is the Schema for the peers API. 654 | properties: 655 | apiVersion: 656 | description: 'APIVersion defines the versioned schema of this representation 657 | of an object. Servers should convert recognized schemas to the latest 658 | internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' 659 | type: string 660 | kind: 661 | description: 'Kind is a string value representing the REST resource this 662 | object represents. Servers may infer this from the endpoint the client 663 | submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' 664 | type: string 665 | metadata: 666 | type: object 667 | spec: 668 | description: BGPPeerSpec defines the desired state of Peer. 669 | properties: 670 | bfdProfile: 671 | description: The name of the BFD Profile to be used for the BFD session 672 | associated to the BGP session. If not set, the BFD session won't 673 | be set up. 674 | type: string 675 | ebgpMultiHop: 676 | description: To set if the BGPPeer is multi-hops away. Needed for 677 | FRR mode only. 678 | type: boolean 679 | holdTime: 680 | description: Requested BGP hold time, per RFC4271. 681 | type: string 682 | keepaliveTime: 683 | description: Requested BGP keepalive time, per RFC4271. 684 | type: string 685 | myASN: 686 | description: AS number to use for the local end of the session. 687 | format: int32 688 | maximum: 4294967295 689 | minimum: 0 690 | type: integer 691 | nodeSelectors: 692 | description: Only connect to this peer on nodes that match one of 693 | these selectors. 694 | items: 695 | description: A label selector is a label query over a set of resources. 696 | The result of matchLabels and matchExpressions are ANDed. An empty 697 | label selector matches all objects. A null label selector matches 698 | no objects. 699 | properties: 700 | matchExpressions: 701 | description: matchExpressions is a list of label selector requirements. 702 | The requirements are ANDed. 703 | items: 704 | description: A label selector requirement is a selector that 705 | contains values, a key, and an operator that relates the 706 | key and values. 707 | properties: 708 | key: 709 | description: key is the label key that the selector applies 710 | to. 711 | type: string 712 | operator: 713 | description: operator represents a key's relationship 714 | to a set of values. Valid operators are In, NotIn, Exists 715 | and DoesNotExist. 716 | type: string 717 | values: 718 | description: values is an array of string values. If the 719 | operator is In or NotIn, the values array must be non-empty. 720 | If the operator is Exists or DoesNotExist, the values 721 | array must be empty. This array is replaced during a 722 | strategic merge patch. 723 | items: 724 | type: string 725 | type: array 726 | required: 727 | - key 728 | - operator 729 | type: object 730 | type: array 731 | matchLabels: 732 | additionalProperties: 733 | type: string 734 | description: matchLabels is a map of {key,value} pairs. A single 735 | {key,value} in the matchLabels map is equivalent to an element 736 | of matchExpressions, whose key field is "key", the operator 737 | is "In", and the values array contains only "value". The requirements 738 | are ANDed. 739 | type: object 740 | type: object 741 | type: array 742 | password: 743 | description: Authentication password for routers enforcing TCP MD5 744 | authenticated sessions 745 | type: string 746 | passwordSecret: 747 | description: passwordSecret is name of the authentication secret for 748 | BGP Peer. the secret must be of type "kubernetes.io/basic-auth", 749 | and created in the same namespace as the MetalLB deployment. The 750 | password is stored in the secret as the key "password". 751 | properties: 752 | name: 753 | description: name is unique within a namespace to reference a 754 | secret resource. 755 | type: string 756 | namespace: 757 | description: namespace defines the space within which the secret 758 | name must be unique. 759 | type: string 760 | type: object 761 | peerASN: 762 | description: AS number to expect from the remote end of the session. 763 | format: int32 764 | maximum: 4294967295 765 | minimum: 0 766 | type: integer 767 | peerAddress: 768 | description: Address to dial when establishing the session. 769 | type: string 770 | peerPort: 771 | default: 179 772 | description: Port to dial when establishing the session. 773 | maximum: 16384 774 | minimum: 0 775 | type: integer 776 | routerID: 777 | description: BGP router ID to advertise to the peer 778 | type: string 779 | sourceAddress: 780 | description: Source address to use when establishing the session. 781 | type: string 782 | required: 783 | - myASN 784 | - peerASN 785 | - peerAddress 786 | type: object 787 | status: 788 | description: BGPPeerStatus defines the observed state of Peer. 789 | type: object 790 | type: object 791 | served: true 792 | storage: true 793 | subresources: 794 | status: {} 795 | status: 796 | acceptedNames: 797 | kind: "" 798 | plural: "" 799 | conditions: [] 800 | storedVersions: [] 801 | --- 802 | apiVersion: apiextensions.k8s.io/v1 803 | kind: CustomResourceDefinition 804 | metadata: 805 | annotations: 806 | controller-gen.kubebuilder.io/version: v0.7.0 807 | creationTimestamp: null 808 | name: communities.metallb.io 809 | spec: 810 | group: metallb.io 811 | names: 812 | kind: Community 813 | listKind: CommunityList 814 | plural: communities 815 | singular: community 816 | scope: Namespaced 817 | versions: 818 | - name: v1beta1 819 | schema: 820 | openAPIV3Schema: 821 | description: Community is a collection of aliases for communities. Users can 822 | define named aliases to be used in the BGPPeer CRD. 823 | properties: 824 | apiVersion: 825 | description: 'APIVersion defines the versioned schema of this representation 826 | of an object. Servers should convert recognized schemas to the latest 827 | internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' 828 | type: string 829 | kind: 830 | description: 'Kind is a string value representing the REST resource this 831 | object represents. Servers may infer this from the endpoint the client 832 | submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' 833 | type: string 834 | metadata: 835 | type: object 836 | spec: 837 | description: CommunitySpec defines the desired state of Community. 838 | properties: 839 | communities: 840 | items: 841 | properties: 842 | name: 843 | description: The name of the alias for the community. 844 | type: string 845 | value: 846 | description: The BGP community value corresponding to the given 847 | name. 848 | type: string 849 | type: object 850 | type: array 851 | type: object 852 | status: 853 | description: CommunityStatus defines the observed state of Community. 854 | type: object 855 | type: object 856 | served: true 857 | storage: true 858 | subresources: 859 | status: {} 860 | status: 861 | acceptedNames: 862 | kind: "" 863 | plural: "" 864 | conditions: [] 865 | storedVersions: [] 866 | --- 867 | apiVersion: apiextensions.k8s.io/v1 868 | kind: CustomResourceDefinition 869 | metadata: 870 | annotations: 871 | controller-gen.kubebuilder.io/version: v0.7.0 872 | creationTimestamp: null 873 | name: ipaddresspools.metallb.io 874 | spec: 875 | group: metallb.io 876 | names: 877 | kind: IPAddressPool 878 | listKind: IPAddressPoolList 879 | plural: ipaddresspools 880 | singular: ipaddresspool 881 | scope: Namespaced 882 | versions: 883 | - name: v1beta1 884 | schema: 885 | openAPIV3Schema: 886 | description: IPAddressPool represents a pool of IP addresses that can be allocated 887 | to LoadBalancer services. 888 | properties: 889 | apiVersion: 890 | description: 'APIVersion defines the versioned schema of this representation 891 | of an object. Servers should convert recognized schemas to the latest 892 | internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' 893 | type: string 894 | kind: 895 | description: 'Kind is a string value representing the REST resource this 896 | object represents. Servers may infer this from the endpoint the client 897 | submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' 898 | type: string 899 | metadata: 900 | type: object 901 | spec: 902 | description: IPAddressPoolSpec defines the desired state of IPAddressPool. 903 | properties: 904 | addresses: 905 | description: A list of IP address ranges over which MetalLB has authority. 906 | You can list multiple ranges in a single pool, they will all share 907 | the same settings. Each range can be either a CIDR prefix, or an 908 | explicit start-end range of IPs. 909 | items: 910 | type: string 911 | type: array 912 | autoAssign: 913 | default: true 914 | description: AutoAssign flag used to prevent MetallB from automatic 915 | allocation for a pool. 916 | type: boolean 917 | avoidBuggyIPs: 918 | default: false 919 | description: AvoidBuggyIPs prevents addresses ending with .0 and .255 920 | to be used by a pool. 921 | type: boolean 922 | required: 923 | - addresses 924 | type: object 925 | status: 926 | description: IPAddressPoolStatus defines the observed state of IPAddressPool. 927 | type: object 928 | required: 929 | - spec 930 | type: object 931 | served: true 932 | storage: true 933 | subresources: 934 | status: {} 935 | status: 936 | acceptedNames: 937 | kind: "" 938 | plural: "" 939 | conditions: [] 940 | storedVersions: [] 941 | --- 942 | apiVersion: apiextensions.k8s.io/v1 943 | kind: CustomResourceDefinition 944 | metadata: 945 | annotations: 946 | controller-gen.kubebuilder.io/version: v0.7.0 947 | creationTimestamp: null 948 | name: l2advertisements.metallb.io 949 | spec: 950 | group: metallb.io 951 | names: 952 | kind: L2Advertisement 953 | listKind: L2AdvertisementList 954 | plural: l2advertisements 955 | singular: l2advertisement 956 | scope: Namespaced 957 | versions: 958 | - name: v1beta1 959 | schema: 960 | openAPIV3Schema: 961 | description: L2Advertisement allows to advertise the LoadBalancer IPs provided 962 | by the selected pools via L2. 963 | properties: 964 | apiVersion: 965 | description: 'APIVersion defines the versioned schema of this representation 966 | of an object. Servers should convert recognized schemas to the latest 967 | internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' 968 | type: string 969 | kind: 970 | description: 'Kind is a string value representing the REST resource this 971 | object represents. Servers may infer this from the endpoint the client 972 | submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' 973 | type: string 974 | metadata: 975 | type: object 976 | spec: 977 | description: L2AdvertisementSpec defines the desired state of L2Advertisement. 978 | properties: 979 | ipAddressPoolSelectors: 980 | description: A selector for the IPAddressPools which would get advertised 981 | via this advertisement. If no IPAddressPool is selected by this 982 | or by the list, the advertisement is applied to all the IPAddressPools. 983 | items: 984 | description: A label selector is a label query over a set of resources. 985 | The result of matchLabels and matchExpressions are ANDed. An empty 986 | label selector matches all objects. A null label selector matches 987 | no objects. 988 | properties: 989 | matchExpressions: 990 | description: matchExpressions is a list of label selector requirements. 991 | The requirements are ANDed. 992 | items: 993 | description: A label selector requirement is a selector that 994 | contains values, a key, and an operator that relates the 995 | key and values. 996 | properties: 997 | key: 998 | description: key is the label key that the selector applies 999 | to. 1000 | type: string 1001 | operator: 1002 | description: operator represents a key's relationship 1003 | to a set of values. Valid operators are In, NotIn, Exists 1004 | and DoesNotExist. 1005 | type: string 1006 | values: 1007 | description: values is an array of string values. If the 1008 | operator is In or NotIn, the values array must be non-empty. 1009 | If the operator is Exists or DoesNotExist, the values 1010 | array must be empty. This array is replaced during a 1011 | strategic merge patch. 1012 | items: 1013 | type: string 1014 | type: array 1015 | required: 1016 | - key 1017 | - operator 1018 | type: object 1019 | type: array 1020 | matchLabels: 1021 | additionalProperties: 1022 | type: string 1023 | description: matchLabels is a map of {key,value} pairs. A single 1024 | {key,value} in the matchLabels map is equivalent to an element 1025 | of matchExpressions, whose key field is "key", the operator 1026 | is "In", and the values array contains only "value". The requirements 1027 | are ANDed. 1028 | type: object 1029 | type: object 1030 | type: array 1031 | ipAddressPools: 1032 | description: The list of IPAddressPools to advertise via this advertisement, 1033 | selected by name. 1034 | items: 1035 | type: string 1036 | type: array 1037 | nodeSelectors: 1038 | description: NodeSelectors allows to limit the nodes to announce as 1039 | next hops for the LoadBalancer IP. When empty, all the nodes having are 1040 | announced as next hops. 1041 | items: 1042 | description: A label selector is a label query over a set of resources. 1043 | The result of matchLabels and matchExpressions are ANDed. An empty 1044 | label selector matches all objects. A null label selector matches 1045 | no objects. 1046 | properties: 1047 | matchExpressions: 1048 | description: matchExpressions is a list of label selector requirements. 1049 | The requirements are ANDed. 1050 | items: 1051 | description: A label selector requirement is a selector that 1052 | contains values, a key, and an operator that relates the 1053 | key and values. 1054 | properties: 1055 | key: 1056 | description: key is the label key that the selector applies 1057 | to. 1058 | type: string 1059 | operator: 1060 | description: operator represents a key's relationship 1061 | to a set of values. Valid operators are In, NotIn, Exists 1062 | and DoesNotExist. 1063 | type: string 1064 | values: 1065 | description: values is an array of string values. If the 1066 | operator is In or NotIn, the values array must be non-empty. 1067 | If the operator is Exists or DoesNotExist, the values 1068 | array must be empty. This array is replaced during a 1069 | strategic merge patch. 1070 | items: 1071 | type: string 1072 | type: array 1073 | required: 1074 | - key 1075 | - operator 1076 | type: object 1077 | type: array 1078 | matchLabels: 1079 | additionalProperties: 1080 | type: string 1081 | description: matchLabels is a map of {key,value} pairs. A single 1082 | {key,value} in the matchLabels map is equivalent to an element 1083 | of matchExpressions, whose key field is "key", the operator 1084 | is "In", and the values array contains only "value". The requirements 1085 | are ANDed. 1086 | type: object 1087 | type: object 1088 | type: array 1089 | type: object 1090 | status: 1091 | description: L2AdvertisementStatus defines the observed state of L2Advertisement. 1092 | type: object 1093 | type: object 1094 | served: true 1095 | storage: true 1096 | subresources: 1097 | status: {} 1098 | status: 1099 | acceptedNames: 1100 | kind: "" 1101 | plural: "" 1102 | conditions: [] 1103 | storedVersions: [] 1104 | --- 1105 | apiVersion: v1 1106 | kind: ServiceAccount 1107 | metadata: 1108 | labels: 1109 | app: metallb 1110 | name: controller 1111 | namespace: metallb-system 1112 | --- 1113 | apiVersion: v1 1114 | kind: ServiceAccount 1115 | metadata: 1116 | labels: 1117 | app: metallb 1118 | name: speaker 1119 | namespace: metallb-system 1120 | --- 1121 | apiVersion: policy/v1beta1 1122 | kind: PodSecurityPolicy 1123 | metadata: 1124 | labels: 1125 | app: metallb 1126 | name: controller 1127 | spec: 1128 | allowPrivilegeEscalation: false 1129 | allowedCapabilities: [] 1130 | allowedHostPaths: [] 1131 | defaultAddCapabilities: [] 1132 | defaultAllowPrivilegeEscalation: false 1133 | fsGroup: 1134 | ranges: 1135 | - max: 65535 1136 | min: 1 1137 | rule: MustRunAs 1138 | hostIPC: false 1139 | hostNetwork: false 1140 | hostPID: false 1141 | privileged: false 1142 | readOnlyRootFilesystem: true 1143 | requiredDropCapabilities: 1144 | - ALL 1145 | runAsUser: 1146 | ranges: 1147 | - max: 65535 1148 | min: 1 1149 | rule: MustRunAs 1150 | seLinux: 1151 | rule: RunAsAny 1152 | supplementalGroups: 1153 | ranges: 1154 | - max: 65535 1155 | min: 1 1156 | rule: MustRunAs 1157 | volumes: 1158 | - configMap 1159 | - secret 1160 | - emptyDir 1161 | --- 1162 | apiVersion: policy/v1beta1 1163 | kind: PodSecurityPolicy 1164 | metadata: 1165 | labels: 1166 | app: metallb 1167 | name: speaker 1168 | spec: 1169 | allowPrivilegeEscalation: false 1170 | allowedCapabilities: 1171 | - NET_RAW 1172 | allowedHostPaths: [] 1173 | defaultAddCapabilities: [] 1174 | defaultAllowPrivilegeEscalation: false 1175 | fsGroup: 1176 | rule: RunAsAny 1177 | hostIPC: false 1178 | hostNetwork: true 1179 | hostPID: false 1180 | hostPorts: 1181 | - max: 7472 1182 | min: 7472 1183 | - max: 7946 1184 | min: 7946 1185 | privileged: true 1186 | readOnlyRootFilesystem: true 1187 | requiredDropCapabilities: 1188 | - ALL 1189 | runAsUser: 1190 | rule: RunAsAny 1191 | seLinux: 1192 | rule: RunAsAny 1193 | supplementalGroups: 1194 | rule: RunAsAny 1195 | volumes: 1196 | - configMap 1197 | - secret 1198 | - emptyDir 1199 | --- 1200 | apiVersion: rbac.authorization.k8s.io/v1 1201 | kind: Role 1202 | metadata: 1203 | labels: 1204 | app: metallb 1205 | name: controller 1206 | namespace: metallb-system 1207 | rules: 1208 | - apiGroups: 1209 | - "" 1210 | resources: 1211 | - secrets 1212 | verbs: 1213 | - create 1214 | - delete 1215 | - get 1216 | - list 1217 | - patch 1218 | - update 1219 | - watch 1220 | - apiGroups: 1221 | - "" 1222 | resourceNames: 1223 | - memberlist 1224 | resources: 1225 | - secrets 1226 | verbs: 1227 | - list 1228 | - apiGroups: 1229 | - apps 1230 | resourceNames: 1231 | - controller 1232 | resources: 1233 | - deployments 1234 | verbs: 1235 | - get 1236 | - apiGroups: 1237 | - metallb.io 1238 | resources: 1239 | - bgppeers 1240 | verbs: 1241 | - get 1242 | - list 1243 | - apiGroups: 1244 | - metallb.io 1245 | resources: 1246 | - addresspools 1247 | verbs: 1248 | - get 1249 | - list 1250 | - watch 1251 | - apiGroups: 1252 | - metallb.io 1253 | resources: 1254 | - bfdprofiles 1255 | verbs: 1256 | - get 1257 | - list 1258 | - watch 1259 | - apiGroups: 1260 | - metallb.io 1261 | resources: 1262 | - ipaddresspools 1263 | verbs: 1264 | - get 1265 | - list 1266 | - watch 1267 | - apiGroups: 1268 | - metallb.io 1269 | resources: 1270 | - bgpadvertisements 1271 | verbs: 1272 | - get 1273 | - list 1274 | - watch 1275 | - apiGroups: 1276 | - metallb.io 1277 | resources: 1278 | - l2advertisements 1279 | verbs: 1280 | - get 1281 | - list 1282 | - watch 1283 | - apiGroups: 1284 | - metallb.io 1285 | resources: 1286 | - communities 1287 | verbs: 1288 | - get 1289 | - list 1290 | - watch 1291 | --- 1292 | apiVersion: rbac.authorization.k8s.io/v1 1293 | kind: Role 1294 | metadata: 1295 | labels: 1296 | app: metallb 1297 | name: pod-lister 1298 | namespace: metallb-system 1299 | rules: 1300 | - apiGroups: 1301 | - "" 1302 | resources: 1303 | - pods 1304 | verbs: 1305 | - list 1306 | - apiGroups: 1307 | - "" 1308 | resources: 1309 | - secrets 1310 | verbs: 1311 | - get 1312 | - list 1313 | - watch 1314 | - apiGroups: 1315 | - metallb.io 1316 | resources: 1317 | - addresspools 1318 | verbs: 1319 | - get 1320 | - list 1321 | - watch 1322 | - apiGroups: 1323 | - metallb.io 1324 | resources: 1325 | - bfdprofiles 1326 | verbs: 1327 | - get 1328 | - list 1329 | - watch 1330 | - apiGroups: 1331 | - metallb.io 1332 | resources: 1333 | - bgppeers 1334 | verbs: 1335 | - get 1336 | - list 1337 | - watch 1338 | - apiGroups: 1339 | - metallb.io 1340 | resources: 1341 | - l2advertisements 1342 | verbs: 1343 | - get 1344 | - list 1345 | - watch 1346 | - apiGroups: 1347 | - metallb.io 1348 | resources: 1349 | - bgpadvertisements 1350 | verbs: 1351 | - get 1352 | - list 1353 | - watch 1354 | - apiGroups: 1355 | - metallb.io 1356 | resources: 1357 | - ipaddresspools 1358 | verbs: 1359 | - get 1360 | - list 1361 | - watch 1362 | - apiGroups: 1363 | - metallb.io 1364 | resources: 1365 | - communities 1366 | verbs: 1367 | - get 1368 | - list 1369 | - watch 1370 | --- 1371 | apiVersion: rbac.authorization.k8s.io/v1 1372 | kind: ClusterRole 1373 | metadata: 1374 | labels: 1375 | app: metallb 1376 | name: metallb-system:controller 1377 | rules: 1378 | - apiGroups: 1379 | - "" 1380 | resources: 1381 | - services 1382 | verbs: 1383 | - get 1384 | - list 1385 | - watch 1386 | - apiGroups: 1387 | - "" 1388 | resources: 1389 | - services/status 1390 | verbs: 1391 | - update 1392 | - apiGroups: 1393 | - "" 1394 | resources: 1395 | - events 1396 | verbs: 1397 | - create 1398 | - patch 1399 | - apiGroups: 1400 | - policy 1401 | resourceNames: 1402 | - controller 1403 | resources: 1404 | - podsecuritypolicies 1405 | verbs: 1406 | - use 1407 | - apiGroups: 1408 | - admissionregistration.k8s.io 1409 | resources: 1410 | - validatingwebhookconfigurations 1411 | - mutatingwebhookconfigurations 1412 | verbs: 1413 | - create 1414 | - delete 1415 | - get 1416 | - list 1417 | - patch 1418 | - update 1419 | - watch 1420 | - apiGroups: 1421 | - apiextensions.k8s.io 1422 | resources: 1423 | - customresourcedefinitions 1424 | verbs: 1425 | - create 1426 | - delete 1427 | - get 1428 | - list 1429 | - patch 1430 | - update 1431 | - watch 1432 | --- 1433 | apiVersion: rbac.authorization.k8s.io/v1 1434 | kind: ClusterRole 1435 | metadata: 1436 | labels: 1437 | app: metallb 1438 | name: metallb-system:speaker 1439 | rules: 1440 | - apiGroups: 1441 | - "" 1442 | resources: 1443 | - services 1444 | - endpoints 1445 | - nodes 1446 | verbs: 1447 | - get 1448 | - list 1449 | - watch 1450 | - apiGroups: 1451 | - discovery.k8s.io 1452 | resources: 1453 | - endpointslices 1454 | verbs: 1455 | - get 1456 | - list 1457 | - watch 1458 | - apiGroups: 1459 | - "" 1460 | resources: 1461 | - events 1462 | verbs: 1463 | - create 1464 | - patch 1465 | - apiGroups: 1466 | - policy 1467 | resourceNames: 1468 | - speaker 1469 | resources: 1470 | - podsecuritypolicies 1471 | verbs: 1472 | - use 1473 | --- 1474 | apiVersion: rbac.authorization.k8s.io/v1 1475 | kind: RoleBinding 1476 | metadata: 1477 | labels: 1478 | app: metallb 1479 | name: controller 1480 | namespace: metallb-system 1481 | roleRef: 1482 | apiGroup: rbac.authorization.k8s.io 1483 | kind: Role 1484 | name: controller 1485 | subjects: 1486 | - kind: ServiceAccount 1487 | name: controller 1488 | namespace: metallb-system 1489 | --- 1490 | apiVersion: rbac.authorization.k8s.io/v1 1491 | kind: RoleBinding 1492 | metadata: 1493 | labels: 1494 | app: metallb 1495 | name: pod-lister 1496 | namespace: metallb-system 1497 | roleRef: 1498 | apiGroup: rbac.authorization.k8s.io 1499 | kind: Role 1500 | name: pod-lister 1501 | subjects: 1502 | - kind: ServiceAccount 1503 | name: speaker 1504 | namespace: metallb-system 1505 | --- 1506 | apiVersion: rbac.authorization.k8s.io/v1 1507 | kind: ClusterRoleBinding 1508 | metadata: 1509 | labels: 1510 | app: metallb 1511 | name: metallb-system:controller 1512 | roleRef: 1513 | apiGroup: rbac.authorization.k8s.io 1514 | kind: ClusterRole 1515 | name: metallb-system:controller 1516 | subjects: 1517 | - kind: ServiceAccount 1518 | name: controller 1519 | namespace: metallb-system 1520 | --- 1521 | apiVersion: rbac.authorization.k8s.io/v1 1522 | kind: ClusterRoleBinding 1523 | metadata: 1524 | labels: 1525 | app: metallb 1526 | name: metallb-system:speaker 1527 | roleRef: 1528 | apiGroup: rbac.authorization.k8s.io 1529 | kind: ClusterRole 1530 | name: metallb-system:speaker 1531 | subjects: 1532 | - kind: ServiceAccount 1533 | name: speaker 1534 | namespace: metallb-system 1535 | --- 1536 | apiVersion: v1 1537 | kind: Secret 1538 | metadata: 1539 | name: webhook-server-cert 1540 | namespace: metallb-system 1541 | --- 1542 | apiVersion: v1 1543 | kind: Service 1544 | metadata: 1545 | name: webhook-service 1546 | namespace: metallb-system 1547 | spec: 1548 | ports: 1549 | - port: 443 1550 | targetPort: 9443 1551 | selector: 1552 | component: controller 1553 | --- 1554 | apiVersion: apps/v1 1555 | kind: Deployment 1556 | metadata: 1557 | labels: 1558 | app: metallb 1559 | component: controller 1560 | name: controller 1561 | namespace: metallb-system 1562 | spec: 1563 | revisionHistoryLimit: 3 1564 | selector: 1565 | matchLabels: 1566 | app: metallb 1567 | component: controller 1568 | template: 1569 | metadata: 1570 | annotations: 1571 | prometheus.io/port: "7472" 1572 | prometheus.io/scrape: "true" 1573 | labels: 1574 | app: metallb 1575 | component: controller 1576 | spec: 1577 | containers: 1578 | - args: 1579 | - --port=7472 1580 | - --log-level=info 1581 | env: 1582 | - name: METALLB_ML_SECRET_NAME 1583 | value: memberlist 1584 | - name: METALLB_DEPLOYMENT 1585 | value: controller 1586 | image: quay.io/metallb/controller:{{ metal_lb_controller_tag_version }} 1587 | livenessProbe: 1588 | failureThreshold: 3 1589 | httpGet: 1590 | path: /metrics 1591 | port: monitoring 1592 | initialDelaySeconds: 10 1593 | periodSeconds: 10 1594 | successThreshold: 1 1595 | timeoutSeconds: 1 1596 | name: controller 1597 | ports: 1598 | - containerPort: 7472 1599 | name: monitoring 1600 | - containerPort: 9443 1601 | name: webhook-server 1602 | protocol: TCP 1603 | readinessProbe: 1604 | failureThreshold: 3 1605 | httpGet: 1606 | path: /metrics 1607 | port: monitoring 1608 | initialDelaySeconds: 10 1609 | periodSeconds: 10 1610 | successThreshold: 1 1611 | timeoutSeconds: 1 1612 | securityContext: 1613 | allowPrivilegeEscalation: false 1614 | capabilities: 1615 | drop: 1616 | - all 1617 | readOnlyRootFilesystem: true 1618 | volumeMounts: 1619 | - mountPath: /tmp/k8s-webhook-server/serving-certs 1620 | name: cert 1621 | readOnly: true 1622 | nodeSelector: 1623 | kubernetes.io/os: linux 1624 | securityContext: 1625 | fsGroup: 65534 1626 | runAsNonRoot: true 1627 | runAsUser: 65534 1628 | serviceAccountName: controller 1629 | terminationGracePeriodSeconds: 0 1630 | volumes: 1631 | - name: cert 1632 | secret: 1633 | defaultMode: 420 1634 | secretName: webhook-server-cert 1635 | --- 1636 | apiVersion: apps/v1 1637 | kind: DaemonSet 1638 | metadata: 1639 | labels: 1640 | app: metallb 1641 | component: speaker 1642 | name: speaker 1643 | namespace: metallb-system 1644 | spec: 1645 | selector: 1646 | matchLabels: 1647 | app: metallb 1648 | component: speaker 1649 | template: 1650 | metadata: 1651 | annotations: 1652 | prometheus.io/port: "7472" 1653 | prometheus.io/scrape: "true" 1654 | labels: 1655 | app: metallb 1656 | component: speaker 1657 | spec: 1658 | containers: 1659 | - args: 1660 | - --port=7472 1661 | - --log-level=info 1662 | env: 1663 | - name: METALLB_NODE_NAME 1664 | valueFrom: 1665 | fieldRef: 1666 | fieldPath: spec.nodeName 1667 | - name: METALLB_HOST 1668 | valueFrom: 1669 | fieldRef: 1670 | fieldPath: status.hostIP 1671 | - name: METALLB_ML_BIND_ADDR 1672 | valueFrom: 1673 | fieldRef: 1674 | fieldPath: status.podIP 1675 | - name: METALLB_ML_LABELS 1676 | value: app=metallb,component=speaker 1677 | - name: METALLB_ML_SECRET_KEY 1678 | valueFrom: 1679 | secretKeyRef: 1680 | key: secretkey 1681 | name: memberlist 1682 | image: quay.io/metallb/speaker:{{ metal_lb_speaker_tag_version }} 1683 | livenessProbe: 1684 | failureThreshold: 3 1685 | httpGet: 1686 | path: /metrics 1687 | port: monitoring 1688 | initialDelaySeconds: 10 1689 | periodSeconds: 10 1690 | successThreshold: 1 1691 | timeoutSeconds: 1 1692 | name: speaker 1693 | ports: 1694 | - containerPort: 7472 1695 | name: monitoring 1696 | - containerPort: 7946 1697 | name: memberlist-tcp 1698 | - containerPort: 7946 1699 | name: memberlist-udp 1700 | protocol: UDP 1701 | readinessProbe: 1702 | failureThreshold: 3 1703 | httpGet: 1704 | path: /metrics 1705 | port: monitoring 1706 | initialDelaySeconds: 10 1707 | periodSeconds: 10 1708 | successThreshold: 1 1709 | timeoutSeconds: 1 1710 | securityContext: 1711 | allowPrivilegeEscalation: false 1712 | capabilities: 1713 | add: 1714 | - NET_RAW 1715 | drop: 1716 | - ALL 1717 | readOnlyRootFilesystem: true 1718 | hostNetwork: true 1719 | nodeSelector: 1720 | kubernetes.io/os: linux 1721 | serviceAccountName: speaker 1722 | terminationGracePeriodSeconds: 2 1723 | tolerations: 1724 | - effect: NoSchedule 1725 | key: node-role.kubernetes.io/master 1726 | operator: Exists 1727 | - effect: NoSchedule 1728 | key: node-role.kubernetes.io/control-plane 1729 | operator: Exists 1730 | --- 1731 | apiVersion: admissionregistration.k8s.io/v1 1732 | kind: ValidatingWebhookConfiguration 1733 | metadata: 1734 | creationTimestamp: null 1735 | name: metallb-webhook-configuration 1736 | webhooks: 1737 | - admissionReviewVersions: 1738 | - v1 1739 | clientConfig: 1740 | service: 1741 | name: webhook-service 1742 | namespace: metallb-system 1743 | path: /validate-metallb-io-v1beta2-bgppeer 1744 | failurePolicy: Fail 1745 | name: bgppeersvalidationwebhook.metallb.io 1746 | rules: 1747 | - apiGroups: 1748 | - metallb.io 1749 | apiVersions: 1750 | - v1beta2 1751 | operations: 1752 | - CREATE 1753 | - UPDATE 1754 | resources: 1755 | - bgppeers 1756 | sideEffects: None 1757 | - admissionReviewVersions: 1758 | - v1 1759 | clientConfig: 1760 | service: 1761 | name: webhook-service 1762 | namespace: metallb-system 1763 | path: /validate-metallb-io-v1beta1-addresspool 1764 | failurePolicy: Fail 1765 | name: addresspoolvalidationwebhook.metallb.io 1766 | rules: 1767 | - apiGroups: 1768 | - metallb.io 1769 | apiVersions: 1770 | - v1beta1 1771 | operations: 1772 | - CREATE 1773 | - UPDATE 1774 | resources: 1775 | - addresspools 1776 | sideEffects: None 1777 | - admissionReviewVersions: 1778 | - v1 1779 | clientConfig: 1780 | service: 1781 | name: webhook-service 1782 | namespace: metallb-system 1783 | path: /validate-metallb-io-v1beta1-bfdprofile 1784 | failurePolicy: Fail 1785 | name: bfdprofilevalidationwebhook.metallb.io 1786 | rules: 1787 | - apiGroups: 1788 | - metallb.io 1789 | apiVersions: 1790 | - v1beta1 1791 | operations: 1792 | - DELETE 1793 | resources: 1794 | - bfdprofiles 1795 | sideEffects: None 1796 | - admissionReviewVersions: 1797 | - v1 1798 | clientConfig: 1799 | service: 1800 | name: webhook-service 1801 | namespace: metallb-system 1802 | path: /validate-metallb-io-v1beta1-bgpadvertisement 1803 | failurePolicy: Fail 1804 | name: bgpadvertisementvalidationwebhook.metallb.io 1805 | rules: 1806 | - apiGroups: 1807 | - metallb.io 1808 | apiVersions: 1809 | - v1beta1 1810 | operations: 1811 | - CREATE 1812 | - UPDATE 1813 | resources: 1814 | - bgpadvertisements 1815 | sideEffects: None 1816 | - admissionReviewVersions: 1817 | - v1 1818 | clientConfig: 1819 | service: 1820 | name: webhook-service 1821 | namespace: metallb-system 1822 | path: /validate-metallb-io-v1beta1-community 1823 | failurePolicy: Fail 1824 | name: communityvalidationwebhook.metallb.io 1825 | rules: 1826 | - apiGroups: 1827 | - metallb.io 1828 | apiVersions: 1829 | - v1beta1 1830 | operations: 1831 | - CREATE 1832 | - UPDATE 1833 | resources: 1834 | - communities 1835 | sideEffects: None 1836 | - admissionReviewVersions: 1837 | - v1 1838 | clientConfig: 1839 | service: 1840 | name: webhook-service 1841 | namespace: metallb-system 1842 | path: /validate-metallb-io-v1beta1-ipaddresspool 1843 | failurePolicy: Fail 1844 | name: ipaddresspoolvalidationwebhook.metallb.io 1845 | rules: 1846 | - apiGroups: 1847 | - metallb.io 1848 | apiVersions: 1849 | - v1beta1 1850 | operations: 1851 | - CREATE 1852 | - UPDATE 1853 | resources: 1854 | - ipaddresspools 1855 | sideEffects: None 1856 | - admissionReviewVersions: 1857 | - v1 1858 | clientConfig: 1859 | service: 1860 | name: webhook-service 1861 | namespace: metallb-system 1862 | path: /validate-metallb-io-v1beta1-l2advertisement 1863 | failurePolicy: Fail 1864 | name: l2advertisementvalidationwebhook.metallb.io 1865 | rules: 1866 | - apiGroups: 1867 | - metallb.io 1868 | apiVersions: 1869 | - v1beta1 1870 | operations: 1871 | - CREATE 1872 | - UPDATE 1873 | resources: 1874 | - l2advertisements 1875 | sideEffects: None -------------------------------------------------------------------------------- /roles/k3s/master/templates/vip.rbac.yaml.j2: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: kube-vip 5 | namespace: kube-system 6 | --- 7 | apiVersion: rbac.authorization.k8s.io/v1 8 | kind: ClusterRole 9 | metadata: 10 | annotations: 11 | rbac.authorization.kubernetes.io/autoupdate: "true" 12 | name: system:kube-vip-role 13 | rules: 14 | - apiGroups: [""] 15 | resources: ["services", "services/status", "nodes"] 16 | verbs: ["list","get","watch", "update"] 17 | - apiGroups: ["coordination.k8s.io"] 18 | resources: ["leases"] 19 | verbs: ["list", "get", "watch", "update", "create"] 20 | --- 21 | kind: ClusterRoleBinding 22 | apiVersion: rbac.authorization.k8s.io/v1 23 | metadata: 24 | name: system:kube-vip-binding 25 | roleRef: 26 | apiGroup: rbac.authorization.k8s.io 27 | kind: ClusterRole 28 | name: system:kube-vip-role 29 | subjects: 30 | - kind: ServiceAccount 31 | name: kube-vip 32 | namespace: kube-system 33 | -------------------------------------------------------------------------------- /roles/k3s/master/templates/vip.yaml.j2: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: DaemonSet 3 | metadata: 4 | creationTimestamp: null 5 | name: kube-vip-ds 6 | namespace: kube-system 7 | spec: 8 | selector: 9 | matchLabels: 10 | name: kube-vip-ds 11 | template: 12 | metadata: 13 | creationTimestamp: null 14 | labels: 15 | name: kube-vip-ds 16 | spec: 17 | affinity: 18 | nodeAffinity: 19 | requiredDuringSchedulingIgnoredDuringExecution: 20 | nodeSelectorTerms: 21 | - matchExpressions: 22 | - key: node-role.kubernetes.io/master 23 | operator: Exists 24 | - matchExpressions: 25 | - key: node-role.kubernetes.io/control-plane 26 | operator: Exists 27 | containers: 28 | - args: 29 | - manager 30 | env: 31 | - name: vip_arp 32 | value: "true" 33 | - name: port 34 | value: "6443" 35 | - name: vip_interface 36 | value: {{ cilium_iface }} 37 | - name: vip_cidr 38 | value: "32" 39 | - name: cp_enable 40 | value: "true" 41 | - name: cp_namespace 42 | value: kube-system 43 | - name: vip_ddns 44 | value: "false" 45 | - name: svc_enable 46 | value: "false" 47 | - name: vip_leaderelection 48 | value: "true" 49 | - name: vip_leaseduration 50 | value: "15" 51 | - name: vip_renewdeadline 52 | value: "10" 53 | - name: vip_retryperiod 54 | value: "2" 55 | - name: address 56 | value: {{ apiserver_endpoint }} 57 | image: ghcr.io/kube-vip/kube-vip:{{ kube_vip_tag_version }} 58 | imagePullPolicy: Always 59 | name: kube-vip 60 | resources: {} 61 | securityContext: 62 | capabilities: 63 | add: 64 | - NET_ADMIN 65 | - NET_RAW 66 | - SYS_TIME 67 | hostNetwork: true 68 | serviceAccountName: kube-vip 69 | tolerations: 70 | - effect: NoSchedule 71 | operator: Exists 72 | - effect: NoExecute 73 | operator: Exists 74 | updateStrategy: {} 75 | status: 76 | currentNumberScheduled: 0 77 | desiredNumberScheduled: 0 78 | numberMisscheduled: 0 79 | numberReady: 0 80 | -------------------------------------------------------------------------------- /roles/k3s/node/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: Copy K3s service file 4 | template: 5 | src: "k3s.service.j2" 6 | dest: "{{ systemd_dir }}/k3s-node.service" 7 | owner: root 8 | group: root 9 | mode: 0755 10 | 11 | - name: Enable and check K3s service 12 | systemd: 13 | name: k3s-node 14 | daemon_reload: yes 15 | state: restarted 16 | enabled: yes 17 | -------------------------------------------------------------------------------- /roles/k3s/node/templates/k3s.service.j2: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Lightweight Kubernetes 3 | Documentation=https://k3s.io 4 | After=network-online.target 5 | 6 | [Service] 7 | Type=notify 8 | ExecStartPre=-/sbin/modprobe br_netfilter 9 | ExecStartPre=-/sbin/modprobe overlay 10 | ExecStart=/usr/local/bin/k3s agent --server https://{{ apiserver_endpoint }}:6443 --token {{ hostvars[groups['master'][0]]['token'] | default(k3s_token) }} {{ extra_agent_args | default("") }} 11 | KillMode=process 12 | Delegate=yes 13 | # Having non-zero Limit*s causes performance problems due to accounting overhead 14 | # in the kernel. We recommend using cgroups to do container-local accounting. 15 | LimitNOFILE=1048576 16 | LimitNPROC=infinity 17 | LimitCORE=infinity 18 | TasksMax=infinity 19 | TimeoutStartSec=0 20 | Restart=always 21 | RestartSec=5s 22 | 23 | [Install] 24 | WantedBy=multi-user.target 25 | -------------------------------------------------------------------------------- /roles/prereq/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Set same timezone on every Server 3 | timezone: 4 | name: "{{ system_timezone }}" 5 | when: (system_timezone is defined) and (system_timezone != "Your/Timezone") 6 | 7 | - name: Set SELinux to disabled state 8 | selinux: 9 | state: disabled 10 | when: ansible_os_family == "RedHat" 11 | 12 | - name: Enable IPv4 forwarding 13 | sysctl: 14 | name: net.ipv4.ip_forward 15 | value: "1" 16 | state: present 17 | reload: yes 18 | 19 | - name: Enable IPv6 forwarding 20 | sysctl: 21 | name: net.ipv6.conf.all.forwarding 22 | value: "1" 23 | state: present 24 | reload: yes 25 | 26 | - name: Add br_netfilter to /etc/modules-load.d/ 27 | copy: 28 | content: "br_netfilter" 29 | dest: /etc/modules-load.d/br_netfilter.conf 30 | mode: "u=rw,g=,o=" 31 | when: ansible_os_family == "RedHat" 32 | 33 | - name: Load br_netfilter 34 | modprobe: 35 | name: br_netfilter 36 | state: present 37 | when: ansible_os_family == "RedHat" 38 | 39 | - name: Set bridge-nf-call-iptables (just to be sure) 40 | sysctl: 41 | name: "{{ item }}" 42 | value: "1" 43 | state: present 44 | reload: yes 45 | when: ansible_os_family == "RedHat" 46 | loop: 47 | - net.bridge.bridge-nf-call-iptables 48 | - net.bridge.bridge-nf-call-ip6tables 49 | 50 | - name: Add /usr/local/bin to sudo secure_path 51 | lineinfile: 52 | line: 'Defaults secure_path = /sbin:/bin:/usr/sbin:/usr/bin:/usr/local/bin' 53 | regexp: "Defaults(\\s)*secure_path(\\s)*=" 54 | state: present 55 | insertafter: EOF 56 | path: /etc/sudoers 57 | validate: 'visudo -cf %s' 58 | when: ansible_os_family == "RedHat" 59 | -------------------------------------------------------------------------------- /roles/raspberrypi/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: reboot 3 | reboot: 4 | -------------------------------------------------------------------------------- /roles/raspberrypi/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Test for raspberry pi /proc/cpuinfo 3 | command: grep -E "Raspberry Pi|BCM2708|BCM2709|BCM2835|BCM2836" /proc/cpuinfo 4 | register: grep_cpuinfo_raspberrypi 5 | failed_when: false 6 | changed_when: false 7 | 8 | - name: Test for raspberry pi /proc/device-tree/model 9 | command: grep -E "Raspberry Pi" /proc/device-tree/model 10 | register: grep_device_tree_model_raspberrypi 11 | failed_when: false 12 | changed_when: false 13 | 14 | - name: Set raspberry_pi fact to true 15 | set_fact: 16 | raspberry_pi: true 17 | when: 18 | grep_cpuinfo_raspberrypi.rc == 0 or grep_device_tree_model_raspberrypi.rc == 0 19 | 20 | - name: Set detected_distribution to Raspbian 21 | set_fact: 22 | detected_distribution: Raspbian 23 | when: > 24 | raspberry_pi|default(false) and 25 | ( ansible_facts.lsb.id|default("") == "Raspbian" or 26 | ansible_facts.lsb.description|default("") is match("[Rr]aspbian.*") ) 27 | 28 | - name: Set detected_distribution to Raspbian (ARM64 on Debian Buster) 29 | set_fact: 30 | detected_distribution: Raspbian 31 | when: 32 | - ansible_facts.architecture is search("aarch64") 33 | - raspberry_pi|default(false) 34 | - ansible_facts.lsb.description|default("") is match("Debian.*buster") 35 | 36 | - name: Set detected_distribution_major_version 37 | set_fact: 38 | detected_distribution_major_version: "{{ ansible_facts.lsb.major_release }}" 39 | when: 40 | - detected_distribution | default("") == "Raspbian" 41 | 42 | - name: Set detected_distribution to Raspbian (ARM64 on Debian Bullseye) 43 | set_fact: 44 | detected_distribution: Raspbian 45 | when: 46 | - ansible_facts.architecture is search("aarch64") 47 | - raspberry_pi|default(false) 48 | - ansible_facts.lsb.description|default("") is match("Debian.*bullseye") 49 | 50 | - name: execute OS related tasks on the Raspberry Pi 51 | include_tasks: "{{ item }}" 52 | with_first_found: 53 | - "prereq/{{ detected_distribution }}-{{ detected_distribution_major_version }}.yml" 54 | - "prereq/{{ detected_distribution }}.yml" 55 | - "prereq/{{ ansible_distribution }}-{{ ansible_distribution_major_version }}.yml" 56 | - "prereq/{{ ansible_distribution }}.yml" 57 | - "prereq/default.yml" 58 | when: 59 | - raspberry_pi|default(false) 60 | -------------------------------------------------------------------------------- /roles/raspberrypi/tasks/prereq/CentOS.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Enable cgroup via boot commandline if not already enabled for Centos 3 | lineinfile: 4 | path: /boot/cmdline.txt 5 | backrefs: yes 6 | regexp: '^((?!.*\bcgroup_enable=cpuset cgroup_memory=1 cgroup_enable=memory\b).*)$' 7 | line: '\1 cgroup_enable=cpuset cgroup_memory=1 cgroup_enable=memory' 8 | notify: reboot 9 | -------------------------------------------------------------------------------- /roles/raspberrypi/tasks/prereq/Raspbian.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Activating cgroup support 3 | lineinfile: 4 | path: /boot/cmdline.txt 5 | regexp: '^((?!.*\bcgroup_enable=cpuset cgroup_memory=1 cgroup_enable=memory\b).*)$' 6 | line: '\1 cgroup_enable=cpuset cgroup_memory=1 cgroup_enable=memory' 7 | backrefs: true 8 | notify: reboot 9 | 10 | - name: Install iptables 11 | apt: name=iptables state=present 12 | 13 | - name: Flush iptables before changing to iptables-legacy 14 | iptables: 15 | flush: true 16 | changed_when: false # iptables flush always returns changed 17 | 18 | - name: Changing to iptables-legacy 19 | alternatives: 20 | path: /usr/sbin/iptables-legacy 21 | name: iptables 22 | register: ip4_legacy 23 | 24 | - name: Changing to ip6tables-legacy 25 | alternatives: 26 | path: /usr/sbin/ip6tables-legacy 27 | name: ip6tables 28 | register: ip6_legacy 29 | -------------------------------------------------------------------------------- /roles/raspberrypi/tasks/prereq/Ubuntu.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Enable cgroup via boot commandline if not already enabled for Ubuntu on a Raspberry Pi 3 | lineinfile: 4 | path: /boot/firmware/cmdline.txt 5 | backrefs: yes 6 | regexp: '^((?!.*\bcgroup_enable=cpuset cgroup_memory=1 cgroup_enable=memory\b).*)$' 7 | line: '\1 cgroup_enable=cpuset cgroup_memory=1 cgroup_enable=memory' 8 | notify: reboot 9 | -------------------------------------------------------------------------------- /roles/raspberrypi/tasks/prereq/default.yml: -------------------------------------------------------------------------------- 1 | --- 2 | -------------------------------------------------------------------------------- /roles/reset/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Disable services 3 | systemd: 4 | name: "{{ item }}" 5 | state: stopped 6 | enabled: no 7 | failed_when: false 8 | with_items: 9 | - k3s 10 | - k3s-node 11 | - k3s-init 12 | 13 | - name: pkill -9 -f "k3s/data/[^/]+/bin/containerd-shim-runc" 14 | register: pkill_containerd_shim_runc 15 | command: pkill -9 -f "k3s/data/[^/]+/bin/containerd-shim-runc" 16 | changed_when: "pkill_containerd_shim_runc.rc == 0" 17 | failed_when: false 18 | 19 | - name: Umount k3s filesystems 20 | include_tasks: umount_with_children.yml 21 | with_items: 22 | - /run/k3s 23 | - /run/cilium 24 | - /sys/fs/bpf 25 | - /var/lib/kubelet 26 | - /run/netns 27 | - /var/lib/rancher/k3s 28 | - /var/lib/kubelet/pods 29 | - /var/lib/kubelet/plugins 30 | - /run/netns/cni- 31 | loop_control: 32 | loop_var: mounted_fs 33 | 34 | - name: Remove service files, binaries and data 35 | file: 36 | name: "{{ item }}" 37 | state: absent 38 | with_items: 39 | - /usr/local/bin/k3s 40 | - "{{ systemd_dir }}/k3s.service" 41 | - "{{ systemd_dir }}/k3s-node.service" 42 | - /etc/rancher/k3s 43 | - /run/k3s 44 | - /run/flannel 45 | - /run/cilium 46 | - /etc/rancher/ 47 | - /var/lib/kubelet 48 | - /var/lib/rancher/k3s 49 | - /var/lib/rancher/ 50 | - /usr/local/bin/k3s 51 | - /var/lib/cni/ 52 | - /usr/local/bin/helm 53 | 54 | - name: daemon_reload 55 | systemd: 56 | daemon_reload: yes 57 | 58 | - name: Reboot and wait for node to come back up 59 | reboot: 60 | reboot_timeout: 3600 61 | -------------------------------------------------------------------------------- /roles/reset/tasks/umount_with_children.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Get the list of mounted filesystems 3 | shell: set -o pipefail && cat /proc/mounts | awk '{ print $2}' | grep -E "^{{ mounted_fs }}" 4 | register: get_mounted_filesystems 5 | args: 6 | executable: /bin/bash 7 | failed_when: false 8 | changed_when: get_mounted_filesystems.stdout | length > 0 9 | check_mode: false 10 | 11 | - name: Umount filesystem 12 | mount: 13 | path: "{{ item }}" 14 | state: unmounted 15 | with_items: 16 | "{{ get_mounted_filesystems.stdout_lines | reverse | list }}" 17 | -------------------------------------------------------------------------------- /site.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - hosts: k3s_cluster 4 | gather_facts: yes 5 | become: yes 6 | roles: 7 | - role: prereq 8 | - role: raspberrypi 9 | 10 | - hosts: master 11 | become: yes 12 | roles: 13 | - role: download/master 14 | - role: k3s/master 15 | 16 | - hosts: node 17 | become: yes 18 | roles: 19 | - role: download/node 20 | - role: k3s/node 21 | --------------------------------------------------------------------------------