├── K8s-v1.14 ├── configmap-073.yaml ├── deploy.sh └── onprem.yaml ├── K8s-v1.17 ├── deploy.sh ├── metallb-configmap-083.yaml └── onprem-v17.yaml ├── K8s-v1.18 ├── deploy.sh ├── metallb-configmap-083.yaml ├── onprem186.yaml └── reset.sh ├── K8s-v1.19 ├── deploy.sh ├── metallb-configmap-083.yaml ├── onprem192.yaml └── reset.sh ├── K8s-v1.23 ├── deploy.sh ├── destroy.sh └── onprem1230.yaml ├── LICENSE ├── README.md └── ansible ├── ansible.cfg ├── deploy.yml ├── destroy.yml ├── do.sh ├── hosts ├── inv.yml ├── playbook.yml ├── reboot.yml ├── setup-containerd.yml └── site.yml /K8s-v1.14/configmap-073.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | namespace: metallb-system 5 | name: config 6 | data: 7 | config: | 8 | address-pools: 9 | - name: default 10 | protocol: layer2 11 | addresses: 12 | - 192.168.7.130-192.168.7.134 13 | -------------------------------------------------------------------------------- /K8s-v1.14/deploy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -xe 4 | 5 | echo Deploying Kubernetes 6 | sudo kubeadm init --config=$HOME/onprem.yaml 7 | 8 | echo Setting Credentials for kubectl access 9 | mkdir -p $HOME/.kube 10 | sudo -H cp /etc/kubernetes/admin.conf $HOME/.kube/config 11 | sudo -H chown $(id -u):$(id -g) $HOME/.kube/config 12 | 13 | kubectl taint nodes --all node-role.kubernetes.io/master- 14 | 15 | sleep 5 16 | 17 | # https://docs.projectcalico.org/v3.7/getting-started/kubernetes/installation/calico 18 | # As we are running AIO (or at most 3 nodes in my lab) - I am using the 50 19 | # nodes or less installation method 20 | 21 | echo Deploying Calico 3.7 CNI 22 | kubectl apply -f https://docs.projectcalico.org/v3.7/manifests/calico.yaml 23 | 24 | # https://metallb.universe.tf/installation/ 25 | # Ensure the IP address range in configmap-073.yaml is reserved 26 | # by your firewall/router, and are part of the POD subnet. 27 | 28 | echo Deploying metallb 0.7.3 bare metal load balancer provider 29 | kubectl apply -f https://raw.githubusercontent.com/google/metallb/v0.7.3/manifests/metallb.yaml 30 | kubectl apply -f $HOME/configmap-073.yaml 31 | 32 | echo Please use kubeadm on any extra nodes you wish in your cluster. 33 | echo wait for all Kubernetes nodes to enter the "Ready" state. 34 | echo Finally deploy Istio. 35 | -------------------------------------------------------------------------------- /K8s-v1.14/onprem.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kubelet.config.k8s.io/v1beta1 2 | kind: KubeletConfiguration 3 | maxPods: 400 4 | --- 5 | apiVersion: kubeadm.k8s.io/v1beta1 6 | kind: ClusterConfiguration 7 | apiServer: 8 | extraArgs: 9 | authorization-mode: Node,RBAC 10 | timeoutForControlPlane: 4m0s 11 | certSANs: 12 | # Put your public IP address for your Internet connection in this 13 | # field. 14 | # Ensure port 8443 and 15443 are port-forwarded by your firewall/router. 15 | # WARNING: This port forward will enable kubeapi and the Istio gateway 16 | # port to be accessible anywhere on the Internet, so use caution when using 17 | # the sample certificates as they are widely known. It is best to generate 18 | # your own root CA and intermediate certificates for Istio to use. 19 | - "105.1.1.1" 20 | certificatesDir: /etc/kubernetes/pki 21 | clusterName: kubernetes 22 | controlPlaneEndpoint: "" 23 | controllerManager: {} 24 | dns: 25 | type: CoreDNS 26 | etcd: 27 | local: 28 | dataDir: /var/lib/etcd 29 | imageRepository: k8s.gcr.io 30 | kubernetesVersion: v1.14.1 31 | networking: 32 | dnsDomain: cluster.local 33 | podSubnet: "" 34 | serviceSubnet: 10.96.0.0/12 35 | scheduler: {} 36 | --- 37 | -------------------------------------------------------------------------------- /K8s-v1.17/deploy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -xe 4 | 5 | echo Deploying Kubernetes 6 | sudo kubeadm init --config=$HOME/onprem-v17.yaml 7 | 8 | echo Setting Credentials for kubectl access 9 | mkdir -p $HOME/.kube 10 | sudo -H cp /etc/kubernetes/admin.conf $HOME/.kube/config 11 | sudo -H chown $(id -u):$(id -g) $HOME/.kube/config 12 | 13 | kubectl taint nodes --all node-role.kubernetes.io/master- 14 | 15 | sleep 5 16 | 17 | # https://docs.projectcalico.org/v3.7/getting-started/kubernetes/installation/calico 18 | # As we are running AIO (or at most 3 nodes in my lab) - I am using the 50 19 | # nodes or less installation method 20 | 21 | echo Deploying Calico 3.8 CNI 22 | kubectl apply -f https://docs.projectcalico.org/v3.8/manifests/calico.yaml 23 | 24 | # https://metallb.universe.tf/installation/ 25 | # Ensure the IP address range in configmap-073.yaml is reserved 26 | # by your firewall/router, and are part of the POD subnet. 27 | 28 | echo Deploying metallb 0.7.3 bare metal load balancer provider 29 | kubectl apply -f https://raw.githubusercontent.com/google/metallb/v0.8.3/manifests/metallb.yaml 30 | kubectl apply -f $HOME/metallb-configmap-083.yaml 31 | 32 | echo Please use kubeadm on any extra nodes you wish in your cluster. 33 | echo wait for all Kubernetes nodes to enter the "Ready" state. 34 | echo Finally deploy Istio. 35 | -------------------------------------------------------------------------------- /K8s-v1.17/metallb-configmap-083.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | namespace: metallb-system 5 | name: config 6 | data: 7 | config: | 8 | address-pools: 9 | - name: default 10 | protocol: layer2 11 | addresses: 12 | - 192.168.7.150-192.168.7.154 13 | -------------------------------------------------------------------------------- /K8s-v1.17/onprem-v17.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kubeadm.k8s.io/v1beta2 3 | kind: ClusterConfiguration 4 | controllerManager: 5 | extraArgs: 6 | cluster-signing-cert-file: /etc/kubernetes/pki/ca.crt 7 | cluster-signing-key-file: /etc/kubernetes/pki/ca.key 8 | kubernetesVersion: v1.17.4 9 | networking: 10 | dnsDomain: cluster.local 11 | serviceSubnet: 10.96.0.0/12 12 | apiServer: 13 | certSANs: 14 | - 105.1.1.1 15 | extraArgs: 16 | authorization-mode: Node,RBAC 17 | service-account-issuer: kubernetes.default.svc 18 | service-account-signing-key-file: /etc/kubernetes/pki/sa.key 19 | --- 20 | apiVersion: kubeadm.k8s.io/v1beta2 21 | kind: InitConfiguration 22 | localAPIEndpoint: 23 | advertiseAddress: 192.168.7.105 24 | bindPort: 6443 25 | --- 26 | -------------------------------------------------------------------------------- /K8s-v1.18/deploy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -xe 4 | 5 | echo Deploying Kubernetes 6 | sudo kubeadm init --config=$HOME/onprem186.yaml 7 | 8 | echo Setting Credentials for kubectl access 9 | mkdir -p $HOME/.kube 10 | sudo -H cp /etc/kubernetes/admin.conf $HOME/.kube/config 11 | sudo -H chown $(id -u):$(id -g) $HOME/.kube/config 12 | 13 | kubectl taint nodes --all node-role.kubernetes.io/master- 14 | 15 | sleep 5 16 | 17 | # https://docs.projectcalico.org/v3.7/getting-started/kubernetes/installation/calico 18 | # As we are running AIO (or at most 3 nodes in my lab) - I am using the 50 19 | # nodes or less installation method 20 | 21 | echo Deploying Calico 3.14 CNI 22 | kubectl apply -f https://docs.projectcalico.org/v3.15/manifests/calico.yaml 23 | 24 | # https://metallb.universe.tf/installation/ 25 | # Ensure the IP address range in configmap-073.yaml is reserved 26 | # by your firewall/router, and are part of the POD subnet. 27 | 28 | echo Deploying metallb 0.8.3 bare metal load balancer provider 29 | kubectl apply -f https://raw.githubusercontent.com/google/metallb/v0.8.3/manifests/metallb.yaml 30 | kubectl apply -f $HOME/metallb-configmap-083.yaml 31 | 32 | echo Please use kubeadm on any extra nodes you wish in your cluster. 33 | echo wait for all Kubernetes nodes to enter the "Ready" state. 34 | echo Finally deploy Istio. 35 | -------------------------------------------------------------------------------- /K8s-v1.18/metallb-configmap-083.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | namespace: metallb-system 5 | name: config 6 | data: 7 | config: | 8 | address-pools: 9 | - name: default 10 | protocol: layer2 11 | addresses: 12 | - 192.168.7.156-192.168.7.160 13 | -------------------------------------------------------------------------------- /K8s-v1.18/onprem186.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kubeadm.k8s.io/v1beta2 3 | kind: InitConfiguration 4 | bootstrapTokens: 5 | - groups: 6 | - system:bootstrappers:kubeadm:default-node-token 7 | token: abcdef.0123456789abcdef 8 | ttl: 24h0m0s 9 | usages: 10 | - signing 11 | - authentication 12 | localAPIEndpoint: 13 | advertiseAddress: 192.168.7.140 14 | bindPort: 6443 15 | nodeRegistration: 16 | criSocket: /var/run/dockershim.sock 17 | name: scale-cp 18 | kubeletExtraArgs: 19 | "feature-gates": "EphemeralContainers=true" 20 | taints: 21 | - effect: NoSchedule 22 | key: node-role.kubernetes.io/master 23 | --- 24 | apiVersion: kubeadm.k8s.io/v1beta2 25 | kind: ClusterConfiguration 26 | apiServer: 27 | timeoutForControlPlane: 4m0s 28 | extraArgs: 29 | "service-account-issuer": "kubernetes.default.svc" 30 | "service-account-signing-key-file": "/etc/kubernetes/pki/sa.key" 31 | "feature-gates": "EphemeralContainers=true" 32 | scheduler: 33 | extraArgs: 34 | "feature-gates": "EphemeralContainers=true" 35 | controllerManager: 36 | extraArgs: 37 | "feature-gates": "EphemeralContainers=true" 38 | certificatesDir: /etc/kubernetes/pki 39 | clusterName: kubernetes 40 | controllerManager: {} 41 | dns: 42 | type: CoreDNS 43 | etcd: 44 | local: 45 | dataDir: /var/lib/etcd 46 | imageRepository: k8s.gcr.io 47 | kubernetesVersion: v1.18.6 48 | networking: 49 | dnsDomain: cluster.local 50 | serviceSubnet: 10.96.0.0/12 51 | scheduler: {} 52 | --- 53 | apiVersion: kubelet.config.k8s.io/v1beta1 54 | kind: KubeletConfiguration 55 | maxPods: 100 56 | --- 57 | -------------------------------------------------------------------------------- /K8s-v1.18/reset.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | sudo kubeadm reset --skip-phases preflight,update-cluster-status,remove-etcd-member 4 | sudo rm -rf /var/lib/etcd 5 | sudo iptables -F 6 | -------------------------------------------------------------------------------- /K8s-v1.19/deploy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -xe 4 | 5 | echo Deploying Kubernetes 6 | sudo kubeadm init --config=$HOME/onprem192.yaml 7 | 8 | echo Setting Credentials for kubectl access 9 | mkdir -p $HOME/.kube 10 | sudo -H cp /etc/kubernetes/admin.conf $HOME/.kube/config 11 | sudo -H chown $(id -u):$(id -g) $HOME/.kube/config 12 | 13 | kubectl taint nodes --all node-role.kubernetes.io/master- 14 | 15 | sleep 5 16 | 17 | # https://docs.projectcalico.org/v3.7/getting-started/kubernetes/installation/calico 18 | # As we are running AIO (or at most 3 nodes in my lab) - I am using the 50 19 | # nodes or less installation method 20 | 21 | echo Deploying Calico 3.14 CNI 22 | kubectl apply -f https://docs.projectcalico.org/v3.15/manifests/calico.yaml 23 | 24 | # https://metallb.universe.tf/installation/ 25 | # Ensure the IP address range in configmap-073.yaml is reserved 26 | # by your firewall/router, and are part of the POD subnet. 27 | 28 | echo Deploying metallb 0.8.3 bare metal load balancer provider 29 | kubectl apply -f https://raw.githubusercontent.com/google/metallb/v0.8.3/manifests/metallb.yaml 30 | kubectl apply -f $HOME/metallb-configmap-083.yaml 31 | 32 | echo Please use kubeadm on any extra nodes you wish in your cluster. 33 | echo wait for all Kubernetes nodes to enter the "Ready" state. 34 | echo Finally deploy Istio. 35 | -------------------------------------------------------------------------------- /K8s-v1.19/metallb-configmap-083.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | namespace: metallb-system 5 | name: config 6 | data: 7 | config: | 8 | address-pools: 9 | - name: default 10 | protocol: layer2 11 | addresses: 12 | - 192.168.7.156-192.168.7.160 13 | -------------------------------------------------------------------------------- /K8s-v1.19/onprem192.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kubeadm.k8s.io/v1beta2 3 | kind: InitConfiguration 4 | bootstrapTokens: 5 | - groups: 6 | - system:bootstrappers:kubeadm:default-node-token 7 | token: abcdef.0123456789abcdef 8 | ttl: 24h0m0s 9 | usages: 10 | - signing 11 | - authentication 12 | localAPIEndpoint: 13 | advertiseAddress: 192.168.7.140 14 | bindPort: 6443 15 | nodeRegistration: 16 | criSocket: /var/run/containerd/containerd.sock 17 | name: scale-cp 18 | kubeletExtraArgs: 19 | "feature-gates": "EphemeralContainers=true" 20 | taints: 21 | - effect: NoSchedule 22 | key: node-role.kubernetes.io/master 23 | --- 24 | apiVersion: kubeadm.k8s.io/v1beta2 25 | kind: ClusterConfiguration 26 | apiServer: 27 | timeoutForControlPlane: 4m0s 28 | extraArgs: 29 | "service-account-issuer": "kubernetes.default.svc" 30 | "service-account-signing-key-file": "/etc/kubernetes/pki/sa.key" 31 | "feature-gates": "EphemeralContainers=true" 32 | scheduler: 33 | extraArgs: 34 | "feature-gates": "EphemeralContainers=true" 35 | controllerManager: 36 | extraArgs: 37 | "feature-gates": "EphemeralContainers=true" 38 | certificatesDir: /etc/kubernetes/pki 39 | clusterName: kubernetes 40 | dns: 41 | type: CoreDNS 42 | etcd: 43 | local: 44 | dataDir: /var/lib/etcd 45 | imageRepository: k8s.gcr.io 46 | kubernetesVersion: v1.19.2 47 | networking: 48 | dnsDomain: cluster.local 49 | serviceSubnet: 10.96.0.0/12 50 | --- 51 | apiVersion: kubelet.config.k8s.io/v1beta1 52 | kind: KubeletConfiguration 53 | maxPods: 100 54 | cgroupDriver: systemd 55 | --- 56 | -------------------------------------------------------------------------------- /K8s-v1.19/reset.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | sudo kubeadm reset --skip-phases preflight,update-cluster-status,remove-etcd-member 4 | sudo rm -rf /var/lib/etcd 5 | sudo iptables -F 6 | -------------------------------------------------------------------------------- /K8s-v1.23/deploy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -xe 4 | 5 | echo Deploying Kubernetes 6 | sudo kubeadm init --config=./onprem1230.yaml 7 | 8 | echo Setting Credentials for kubectl access 9 | mkdir -p $HOME/.kube 10 | sudo -H cp /etc/kubernetes/admin.conf $HOME/.kube/config 11 | sudo -H chown $(id -u):$(id -g) $HOME/.kube/config 12 | 13 | #kubectl taint nodes --all node-role.kubernetes.io/master- 14 | 15 | sleep 5 16 | 17 | echo Deploying Calico 3.14 CNI 18 | kubectl apply -f https://docs.projectcalico.org/manifests/calico.yaml 19 | 20 | echo Deploying metallb 0.11.0 bare metal load balancer provider 21 | kubectl apply -f https://raw.githubusercontent.com/metallb/metallb/v0.11.0/manifests/namespace.yaml 22 | kubectl apply -f https://raw.githubusercontent.com/metallb/metallb/v0.11.0/manifests/metallb.yaml 23 | -------------------------------------------------------------------------------- /K8s-v1.23/destroy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | sudo kubeadm reset 4 | sudo rm -rf /var/lib/etcd 5 | sudo iptables -F 6 | sudo iptables -X 7 | sudo iptables -Z 8 | sudo iptables -t nat -F 9 | sudo iptables -t nat -X 10 | sudo iptables -t mangle -F 11 | sudo iptables -t mangle -X 12 | sudo iptables -t raw -F 13 | -------------------------------------------------------------------------------- /K8s-v1.23/onprem1230.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kubeadm.k8s.io/v1beta2 3 | kind: InitConfiguration 4 | localAPIEndpoint: 5 | advertiseAddress: 192.168.33.179 6 | bindPort: 6443 7 | nodeRegistration: 8 | criSocket: /var/run/containerd/containerd.sock 9 | name: cilium-01 10 | --- 11 | apiVersion: kubeadm.k8s.io/v1beta2 12 | kind: ClusterConfiguration 13 | apiServer: 14 | timeoutForControlPlane: 4m0s 15 | extraArgs: 16 | "service-account-issuer": "kubernetes.default.svc" 17 | "service-account-signing-key-file": "/etc/kubernetes/pki/sa.key" 18 | certificatesDir: /etc/kubernetes/pki 19 | clusterName: kubernetes 20 | dns: 21 | type: CoreDNS 22 | etcd: 23 | local: 24 | dataDir: /var/lib/etcd 25 | imageRepository: k8s.gcr.io 26 | kubernetesVersion: v1.23.0 27 | networking: 28 | dnsDomain: cluster.local 29 | podSubnet: 10.1.0.0/16 30 | serviceSubnet: 10.2.0.0/16 31 | --- 32 | apiVersion: kubelet.config.k8s.io/v1beta1 33 | kind: KubeletConfiguration 34 | maxPods: 100 35 | cgroupDriver: systemd 36 | --- 37 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Deploy Containerd, Kubernetes, Calico, MetalLB for development ONLY 2 | 3 | ## Overview 4 | 5 | This is not a deployment project, but simply my tooling for doing development work. 6 | I don't feel the world needs yet another K8s deployment system, however, I need 7 | something simple and understandable for my own development which involves 8 | contribution to [Istio project](https://istio.io) and the cloud native ecosystem. I 9 | wrote the Ansible code in about an hour so its not great, and is probably buggy. 10 | If you submit PRs to fix bugs, I will accept them. PRs for features may not be 11 | accepted. Please slack me if you are thinking of adding anything complex, so you 12 | don't waste your time :) 13 | 14 | This code deploys the following: 15 | - containerd 16 | - Kubernetes 17 | - Calico 18 | - MetalLB 19 | 20 | Please read the license. There is ABSOLUTELY NO WARRANTY. If you run these playbooks 21 | on an existing cluster, they may eat your system. 22 | 23 | ## Directory Layout 24 | 25 | | Directory | Purpose 26 | |-----------|-------- 27 | | K8s-v1.14 | deploy K8s v1.14 with kubeadm AIO 28 | | K8s-v1.17 | deploy K8s v1.17 with kubeadm AIO 29 | | K8s-v1.18 | deploy K8s v1.18 with kubeadm AIO 30 | | K8s-v1.19 | deploy K8s v1.19 with kubeadm AIO 31 | | ansible | Deploy a control plane and a bunch of workers using an ansible playbook with kubeadm 32 | 33 | ## Using Ansible 34 | 35 | ### Prereqs 36 | 37 | - Ensure you have passwordless sudo setup on every node in your inventory. 38 | - Do not run as user root. 39 | - Instead run as your local user. 40 | - Ensure containerd 1.3.3 or later is installed. This tooling does not depend on Docker, only containerd from ubuntu. You may be able to install containerd with ansible-playbook -f setup-containerd.yml. This may or may not work. I have not done a final validation of my system from initial deployment to determine if the code is correct. 41 | 42 | ### Usage 43 | 44 | 1. Copy the yaml files form the `K8s-v1.y` directory to your home directory. 45 | 1. Modify the inventory with the ip addresses of your machines. 46 | 1. Modify the advertise address to your control plane IP address in `onprem-1y.yaml`. 47 | 1. Run the `do.sh` script with `-c create` or `-c destroy`. 48 | 49 | **DO OR DO NOT** 50 | -------------------------------------------------------------------------------- /ansible/ansible.cfg: -------------------------------------------------------------------------------- 1 | [defaults] 2 | inventory = hosts 3 | remote_user = sdake 4 | ask_pass = False 5 | host_key_checking = False 6 | ansible_port = 22 7 | -------------------------------------------------------------------------------- /ansible/deploy.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: cp_nodes 3 | tasks: 4 | - name: Initialize the Kubernetes cluster using kubeadm 5 | command: sudo kubeadm init --config=$HOME/onprem.yaml --skip-phases=addon/kube-proxy 6 | 7 | - name: Initialize the Kubernetes cluster using kubeadm 8 | command: mkdir -p $HOME/.kube 9 | 10 | - name: Copy admin.conf to .kube/config 11 | command: sudo -H cp /etc/kubernetes/admin.conf $HOME/.kube/config 12 | 13 | - name: Chown the file to the uid and gid of this user 14 | command: sudo -H chown 1000:1000 $HOME/.kube/config 15 | 16 | # - name: Install calico pod network 17 | # become: false 18 | # command: kubectl apply -f https://docs.projectcalico.org/v3.15/manifests/calico.yaml 19 | 20 | # - name: Install MetalLB Load Balancer 21 | #become: false 22 | #command: kubectl apply -f https://raw.githubusercontent.com/google/metallb/v0.8.3/manifests/metallb.yaml 23 | 24 | # - name: Install MetalLB Load Balancer Configuration 25 | # become: false 26 | # command: kubectl apply -f $HOME/metallb-configmap-083.yaml 27 | 28 | - name: Generate join command 29 | command: kubeadm token create --print-join-command 30 | register: join_command 31 | 32 | - name: Register join command as a dummy host 33 | add_host: 34 | name: "DUMMY_HOST" 35 | JOINVAR: "{{ join_command.stdout }}" 36 | 37 | - hosts: worker_nodes 38 | become: true 39 | tasks: 40 | - name: Record the dummy host join command 41 | shell: echo -n "{{ hostvars['DUMMY_HOST']['JOINVAR'] }} --cri-socket=/run/containerd/containerd.sock" 42 | register: join 43 | 44 | - name: Join the node to cluster 45 | command: "{{join.stdout}}" 46 | -------------------------------------------------------------------------------- /ansible/destroy.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: all 3 | become: true 4 | 5 | tasks: 6 | 7 | - name: Resetting Kubernetes 8 | command: kubeadm reset -f 9 | 10 | - name: Reset etcd state 11 | command: rm -rf /var/lib/etcd 12 | 13 | - name: Resetting cni state 14 | command: rm -rf /etc/cni/net.d 15 | 16 | - name: Flushing iptables rules 17 | command: iptables -F 18 | 19 | - name: Flushing iptables chains 20 | command: iptables -X 21 | 22 | - name: Flushing iptables counters 23 | command: iptables -Z 24 | 25 | - name: Flushing all nat rules 26 | command: iptables -t nat -F 27 | 28 | - name: Flushing all nat chains 29 | command: iptables -t nat -X 30 | 31 | - name: Flushing all mangle rules 32 | command: iptables -t mangle -F 33 | 34 | - name: Flushing all mangle chains 35 | command: iptables -t mangle -X 36 | 37 | - name: Flushing all raw rules 38 | command: iptables -t raw -F 39 | 40 | - name: Flushing all raw chains 41 | command: iptables -t raw -X 42 | -------------------------------------------------------------------------------- /ansible/do.sh: -------------------------------------------------------------------------------- 1 | usage() { echo "Usage: $0 -c [create|destroy]" 1>&2; exit 1; } 2 | 3 | while getopts ":c:" o; do 4 | case "${o}" in 5 | c) 6 | cmd="${OPTARG}" 7 | ;; 8 | *) 9 | usage 10 | ;; 11 | esac 12 | done 13 | 14 | if [ "${cmd}" != "create" ] && [ "${cmd}" != "destroy" ]; then 15 | usage 16 | fi 17 | 18 | if [ "${cmd}" == "create" ]; then 19 | ansible-playbook site.yml -i inv.yml 20 | fi 21 | 22 | if [ "${cmd}" == "destroy" ]; then 23 | ansible-playbook destroy.yml -i inv.yml 24 | fi 25 | -------------------------------------------------------------------------------- /ansible/hosts: -------------------------------------------------------------------------------- 1 | [cp_nodes] 2 | cilium-01 ansible_host=192.168.33.179 3 | 4 | [worker_nodes] 5 | cilium-02 ansible_host=192.168.33.181 6 | cilium-03 ansible_host=192.168.33.243 7 | cilium-04 ansible_host=192.168.33.240 8 | cilium-05 ansible_host=192.168.33.223 9 | cilium-06 ansible_host=192.168.33.201 10 | cilium-07 ansible_host=192.168.33.202 11 | cilium-08 ansible_host=192.168.33.241 12 | cilium-09 ansible_host=192.168.33.244 13 | cilium-10 ansible_host=192.168.33.185 14 | -------------------------------------------------------------------------------- /ansible/inv.yml: -------------------------------------------------------------------------------- 1 | [cp_nodes] 2 | scale-cp ansible_host=192.168.7.140 3 | 4 | [worker_nodes] 5 | scale-01 ansible_host=192.168.7.141 6 | scale-02 ansible_host=192.168.7.142 7 | scale-03 ansible_host=192.168.7.143 8 | scale-04 ansible_host=192.168.7.144 9 | scale-05 ansible_host=192.168.7.145 10 | scale-06 ansible_host=192.168.7.146 11 | scale-07 ansible_host=192.168.7.147 12 | scale-08 ansible_host=192.168.7.148 13 | scale-09 ansible_host=192.168.7.149 14 | scale-10 ansible_host=192.168.7.150 15 | scale-11 ansible_host=192.168.7.151 16 | scale-12 ansible_host=192.168.7.152 17 | scale-13 ansible_host=192.168.7.153 18 | scale-14 ansible_host=192.168.7.154 19 | scale-15 ansible_host=192.168.7.155 20 | -------------------------------------------------------------------------------- /ansible/playbook.yml: -------------------------------------------------------------------------------- 1 | - name: Add users 2 | hosts: all 3 | become: True 4 | tasks: 5 | - group: 6 | name: ansible 7 | gid: 2000 8 | state: present 9 | 10 | - user: 11 | name: ansible 12 | comment: "Ansible" 13 | uid: 2000 14 | group: ansible 15 | groups: sudo 16 | 17 | - name: Set authorized key took from file 18 | authorized_key: 19 | user: ansible 20 | state: present 21 | key: "{{ lookup('file', '/home/sdake/.ssh/id_ed25519.pub') }}" 22 | 23 | - name: Set authorized key took from file 24 | authorized_key: 25 | user: sdake 26 | state: present 27 | key: "{{ lookup('file', '/home/sdake/.ssh/id_ed25519.pub') }}" 28 | 29 | - name: Allow 'sudo' group to have passwordless sudo 30 | lineinfile: 31 | dest: /etc/sudoers 32 | state: present 33 | regexp: '^# %sudo' 34 | line: '%sudo ALL=(ALL) NOPASSWD: ALL' 35 | 36 | - name: Remove resolv.conf 37 | file: 38 | state: absent 39 | path: /etc/resolv.conf 40 | 41 | - name: Replace resolv.conf with my resolver cache 42 | copy: 43 | dest: /etc/resolv.conf 44 | content: | 45 | nameserver 192.168.33.1 46 | 47 | # - name: Update the cache and upgrade systems 48 | # register: updatesys 49 | # apt: 50 | # name: "*" 51 | # state: latest 52 | # update_cache: yes 53 | 54 | - name: Install HWE kernel 55 | apt: 56 | name: linux-generic-hwe-20.04 57 | state: present 58 | 59 | - name: Install containerd 60 | apt: 61 | name: containerd 62 | state: present 63 | 64 | - name: Install apt-transport-https 65 | apt: 66 | name: apt-transport-https 67 | state: present 68 | 69 | - name: Install ca-certificates 70 | apt: 71 | name: ca-certificates 72 | state: present 73 | 74 | - name: Install curl 75 | apt: 76 | name: curl 77 | state: present 78 | 79 | - name: Install Kubernetes signing key 80 | ansible.builtin.apt_key: 81 | url: https://packages.cloud.google.com/apt/doc/apt-key.gpg 82 | state: present 83 | 84 | - name: add Kubernetes apt repository 85 | apt_repository: 86 | repo: deb https://apt.kubernetes.io/ kubernetes-xenial main 87 | state: present 88 | filename: kubernetes 89 | update_cache: yes 90 | 91 | - name: Install kubeadm 92 | apt: 93 | name: kubeadm 94 | state: present 95 | 96 | - name: Install kubelet 97 | apt: 98 | name: kubelet 99 | state: present 100 | 101 | - name: Install kubectl 102 | apt: 103 | name: kubectl 104 | state: present 105 | 106 | - name: Disable swap permanently 107 | replace: 108 | path: /etc/fstab 109 | regexp: '^([^#].*?\sswap\s+sw\s+.*)$' 110 | replace: '# \1' 111 | 112 | - name: Turn swap off 113 | shell: | 114 | swapoff -a 115 | 116 | - name: Install br netfilter module 117 | modprobe: 118 | name: br_netfilter 119 | state: present 120 | 121 | - sysctl: 122 | name: net.bridge.bridge-nf-call-ip6tables 123 | value: 1 124 | sysctl_set: yes 125 | state: present 126 | reload: yes 127 | 128 | - sysctl: 129 | name: net.bridge.bridge-nf-call-iptables 130 | value: 1 131 | sysctl_set: yes 132 | state: present 133 | reload: yes 134 | 135 | - sysctl: 136 | name: net.ipv4.ip_forward 137 | value: 1 138 | sysctl_set: yes 139 | state: present 140 | reload: yes 141 | -------------------------------------------------------------------------------- /ansible/reboot.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: worker_nodes 3 | become: true 4 | 5 | tasks: 6 | - name: Reboot the machine with all defaults using Ansible 7 | reboot: 8 | -------------------------------------------------------------------------------- /ansible/setup-containerd.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: all 3 | tasks: 4 | # If docker is installed, uncomment this and run it to REMOVE docker on 5 | # your inventory hosts 6 | # - name: prune 7 | # command: docker system prune -f -a 8 | # - name: remove 9 | # command: sudo systemd stop docker 10 | # - name: remove 11 | # command: sudo apt -y remove docker-ce-cli 12 | # - name: remove 13 | # command: sudo apt -y remove containerd.io 14 | - name: Update the debian package cache 15 | command: sudo apt -y update 16 | - name: Upgrade all nodes to the latest packaging 17 | command: sudo apt -y upgrade 18 | - name: Install containerd 19 | command: sudo apt -y install containerd 20 | - name: Create containerd dir 21 | command: sudo mkdir -p /etc/containerd 22 | - name: Create the default configuration for containerd 23 | command: sudo bash -c "containerd config default > /etc/containerd/config.toml" 24 | - name: restart containerd 25 | command: sudo systemctl restart containerd 26 | - name: Add runtime args in kubelet configuraiton 27 | become: true 28 | lineinfile: 29 | dest: "/etc/systemd/system/kubelet.service.d/10-kubeadm.conf" 30 | line: "Environment=\"KUBELET_EXTRA_ARGS= --runtime-cgroups=/system.slice/containerd.service --container-runtime=remote --runtime-request-timeout=15m --container-runtime-endpoint=unix:///run/containerd/containerd.sock\"" 31 | insertafter: '\[Service\]' 32 | - name: Start containerd 33 | command: sudo systemctl start containerd 34 | - name: Pull the pause container (no idea why this is needed, it may not be) 35 | command: sudo ctr images pull k8s.gcr.io/pause:3.2 36 | -------------------------------------------------------------------------------- /ansible/site.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: cp_nodes 3 | tasks: 4 | - name: Initialize the Kubernetes cluster using kubeadm 5 | command: sudo kubeadm init --config=$HOME/onprem192.yaml 6 | 7 | - name: Initialize the Kubernetes cluster using kubeadm 8 | command: mkdir -p $HOME/.kube 9 | 10 | - name: Copy admin.conf to .kube/config 11 | command: sudo -H cp /etc/kubernetes/admin.conf $HOME/.kube/config 12 | 13 | - name: Chown the file to the uid and gid of this user 14 | command: sudo -H chown 1000:1000 $HOME/.kube/config 15 | 16 | - name: Install calico pod network 17 | become: false 18 | command: kubectl apply -f https://docs.projectcalico.org/v3.15/manifests/calico.yaml 19 | 20 | - name: Install MetalLB Load Balancer 21 | become: false 22 | command: kubectl apply -f https://raw.githubusercontent.com/google/metallb/v0.8.3/manifests/metallb.yaml 23 | 24 | - name: Install MetalLB Load Balancer Configuration 25 | become: false 26 | command: kubectl apply -f $HOME/metallb-configmap-083.yaml 27 | 28 | - name: Generate join command 29 | command: kubeadm token create --print-join-command 30 | register: join_command 31 | 32 | - name: Register join command as a dummy host 33 | add_host: 34 | name: "DUMMY_HOST" 35 | JOINVAR: "{{ join_command.stdout }}" 36 | 37 | - hosts: worker_nodes 38 | become: true 39 | tasks: 40 | - name: Record the dummy host join command 41 | shell: echo -n "{{ hostvars['DUMMY_HOST']['JOINVAR'] }} --cri-socket=/run/containerd/containerd.sock" 42 | register: join 43 | 44 | - name: Join the node to cluster 45 | command: "{{join.stdout}}" 46 | --------------------------------------------------------------------------------