├── .gitignore
├── README.md
├── config
    ├── cilium-values.yaml.tpl
    ├── k0s_metallb_push.yaml.tpl
    └── metallb-config.yaml.tpl
├── playground.sh
├── playground_functions.sh
└── playground_vars.sh


/.gitignore:
--------------------------------------------------------------------------------
1 | ./bkps
2 | ./bkps/*
3 | *.yaml
4 | *.yml
5 | config/*.yaml
6 | config/*.yml
7 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # WORK IN PROGRESS 
 2 | # contributions highly appreciated  ;)
 3 | 
 4 | ## K0S Cilium Playground
 5 | 
 6 | ### Intro
 7 | This repository holds set of scripts that helps building and configuring a full cilium playground with :
 8 | - cluster-mesh enabled
 9 | - metallb as an IPAM solution 
10 | 
11 | This toolkit has been tested on MACOS but should behave the exact same way on Linux hosts.
12 | ### Requirements
13 | For this project to run successfully you will need the following binaries deployment on your host:
14 | - multipass
15 | - k0sctl
16 | - brew 
17 | - kubectl 
18 | - helm
19 | 
20 | Note: on MACOS/Linux you just need `brew` installed, the scripts will install the rest of the needed requirements.
21 | 
22 | ### Variables
23 | Several variables as been declared in the `playground_vars.sh`  script, some of them can be overridden:
24 | 
25 | | name | description | type | default value | comment |
26 | |------|-------------|------|---------------|---------|
27 | |DEBUGENABLED| Enables Debug Mode i.e: k0s controller visible and debug log enabled|Boolean|true| If you don't need to see the controller(s) nor the full logs, pass the variable to 'false'|
28 | |K0SCTLVERS|Version of the k0sctl binary|string|v0.14.0|if the binary doesn't exist AND brew installed, it will automatically deploy the latest version [UNUSED FOR NOW]|
29 | |METALLBVERS|Version of metallb used|string|v0.13.7|         |
30 | |CILIUMENABLED|Enable or not the Cilium CNI AND clustermesh configuration|Boolean|true|if cilium is not deployed i.e:CILIUMENABLED=false <br> the default CNI will be used <br>(as of now kube-router)       |
31 | |CILIUMVERS|Version of cilium used|number|1.12.4|Cilium version <1.13 **NEEDS** metallb , the 1.13 is yet to be tested|
32 | |CUSTOMCILIUM|Enables usage of a custom Cilium helm charts configuration|Boolean|false|if set to `true` <br>the CUSTOMCILIUMVALUESFILES should be filled with the values.yaml content to be used with cilium|
33 | |CUSTOMCILIUMVALUESFILES|Values to be passed to the cilium helm charts|string|[empty]|values to be passed in yaml format|
34 | |NUMBER_OF_VMS|number of VMs for each cluster|number|3| the number of VMs counts both controllers and workers nodes|
35 | |SSHPKEYPATH|path to access the ssh key file |string|~/.ssh/|      |
36 | |SSHKEYNAME|name of the ssh key file|string|id_rsa|if the file doesn't exist the scripts will create it and initialize it with a blank passphrase <br><br> if the keypair already exist make sure it has a **BLANK** passphrase or use ssh-agent and load the keypair before running the scripts|
37 | |K0SADMINUSERNAME|system acountname to be created in each VMs|string|k0s|      |
38 | |K0SCONTROLLERCOUNT|number of controller nodes to be configued|number|1|      |
39 | |K0SCLUSTERNUMBER|number of cluster to be deployed and configured|number|2|      |
40 | |K0SCLUSTERNAMEVAR|name to be for each cluster|string|k0scilium|      |
41 | 
42 | 
43 | ### deployment
44 | if you don't feel the need to tweak the default variables, you just need to run the `init_playground` :
45 | ```
46 | git clone https://github.com/xinity/k0s_cilium_playground
47 | chmod u+x playground.sh
48 | ./playground.sh -i
49 | 
50 | or
51 | 
52 | ./playground.sh --install
53 | ```
54 | 
55 | ### TODO
56 | - code refactoring
57 | - log management 
58 | - FULL debug mode
59 | - add full https local FQDN management
60 | - add cilium full ingress support
61 | - add support for Cilium 1.13 IPAM (without metallb)


--------------------------------------------------------------------------------
/config/cilium-values.yaml.tpl:
--------------------------------------------------------------------------------
 1 |         extensions: 
 2 |           helm: 
 3 |             repositories: 
 4 |             - name: metallb  
 5 |               url: https://metallb.github.io/metallb 
 6 |             - name: cilium 
 7 |               url: https://helm.cilium.io/ 
 8 |             charts: 
 9 |             - name: metallb 
10 |               chartname: metallb/metallb 
11 |               namespace: metallb
12 |               version: METALLBVERS
13 |             - name: cilium 
14 |               chartname: cilium/cilium 
15 |               version: CILIUMVERS
16 |               values: |2 
17 |                 cluster: 
18 |                   name: CLUSTERNAME 
19 |                   id: CLUSTERID 
20 |                 rollOutCiliumPods: true 
21 |                 hubble: 
22 |                   enabled: true 
23 |                   metrics: 
24 |                     enabled:  
25 |                     - dns:query;ignoreAAAA 
26 |                     - drop 
27 |                     - tcp 
28 |                     - flow 
29 |                     - icmp 
30 |                     - http    
31 |                     port: 9965 
32 |                     serviceAnnotations: {} 
33 |                     serviceMonitor: 
34 |                       enabled: false 
35 |                       labels: {} 
36 |                       annotations: {} 
37 |                       metricRelabelings: ~ 
38 |                   relay: 
39 |                     enabled: true 
40 |                     rollOutPods: true 
41 |                     prometheus: 
42 |                       enabled: true 
43 |                       port: 9966 
44 |                       serviceMonitor: 
45 |                         enabled: false 
46 |                         labels: {} 
47 |                         annotations: {} 
48 |                         interval: "10s" 
49 |                         metricRelabelings: ~ 
50 |                   ui: 
51 |                     enabled: true 
52 |                     standalone: 
53 |                       enabled: false 
54 |                       tls: 
55 |                         certsVolume: {} 
56 |                     rollOutPods: true 
57 |                 ipam: 
58 |                   mode: "cluster-pool" 
59 |                   operator: 
60 |                     clusterPoolIPv4PodCIDR: "10.244.0.0/16" 
61 |                     clusterPoolIPv4PodCIDRList: ["10.244.0.0/16"] 
62 |                     clusterPoolIPv4MaskSize: 24 
63 |                     clusterPoolIPv6PodCIDR: "fd00::/104" 
64 |                     clusterPoolIPv6PodCIDRList: [] 
65 |                     clusterPoolIPv6MaskSize: 120 
66 |                 prometheus: 
67 |                   enabled: true 
68 |                   port: 9962 
69 |                   serviceMonitor: 
70 |                     enabled: false 
71 |                     labels: {} 
72 |                     annotations: {} 
73 |                     metricRelabelings: ~ 
74 |                 operator: 
75 |                   enabled: true 
76 |                   rollOutPods: true 
77 |                   prometheus: 
78 |                     enabled: true 
79 |                     port: 9963 
80 |                     serviceMonitor: 
81 |                       enabled: false 
82 |                       labels: {} 
83 |                       annotations: {} 
84 |                       metricRelabelings: ~ 
85 |                   skipCRDCreation: false 
86 |                   removeNodeTaints: true 
87 |                   setNodeNetworkStatus: true 
88 |                   unmanagedPodWatcher: 
89 |                     restart: true 
90 |                     intervalSeconds: 15 
91 |                 k8sServiceHost: CNODE
92 |                 k8sServicePort: 6443 
93 |                 kubeProxyReplacement: "strict" 
94 |                 kubeProxyReplacementHealthzBindAddr: "0.0.0.0:10256" 
95 |               namespace: cilium
96 | 


--------------------------------------------------------------------------------
/config/k0s_metallb_push.yaml.tpl:
--------------------------------------------------------------------------------
1 |     files:
2 |     - name: metallb-cm-manifest
3 |       src: METALLBRANGEFILE
4 |       dstDir: /var/lib/k0s/manifests/metallb/
5 |       perm: 0700
6 | 


--------------------------------------------------------------------------------
/config/metallb-config.yaml.tpl:
--------------------------------------------------------------------------------
 1 | apiVersion: metallb.io/v1beta1
 2 | kind: IPAddressPool
 3 | metadata:
 4 |   name: first-pool
 5 |   namespace: metallb
 6 | spec:
 7 |   addresses:
 8 |   - METALLBIPRANGE
 9 | ---
10 | apiVersion: metallb.io/v1beta1
11 | kind: L2Advertisement
12 | metadata:
13 |   name: cilium-metallb
14 |   namespace: metallb


--------------------------------------------------------------------------------
/playground.sh:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env bash
 2 | # set -x 
 3 | # source playground_vars.sh
 4 | source playground_functions.sh
 5 | 
 6 |   # while getopts 'idh' OPTION; do
 7 |   #   case "$OPTION" in 
 8 |   #     i) 
 9 |   #       create_platform
10 |   #       ;;
11 |   #     d)
12 |   #       purge_all
13 |   #       ;;
14 |   #     h)
15 |   #       playground_help
16 |   #       ;;
17 |   #     ?) 
18 |   #       playground_help
19 |   #       ;;
20 |   #   esac
21 |   # done
22 | 
23 | while true; do
24 |   case "$1" in
25 |     -i | --install ) create_platform ; break ;;
26 |     -d | --delete ) purge_all ;;
27 |     -h | --help ) playground_help ;;
28 |     -v | --verbose ) printf "${CL}${RED}${BOLD}I SAID NOT IMPLEMENTED YET ;-) \n${NORMAL}"  ; exec_spinner "count_down 3" "" "reboot!" ; clear ; playground_help ;;
29 |     * ) playground_help ;;
30 |   esac
31 | done


--------------------------------------------------------------------------------
/playground_functions.sh:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env bash
  2 | # set -x 
  3 | 
  4 | source playground_vars.sh  
  5 | 
  6 | NORMAL=$(tput sgr0)
  7 | BOLD=$(tput bold)
  8 | BLACK=$(tput setaf 0)
  9 | RED=$(tput setaf 1)
 10 | GREEN=$(tput setaf 2)
 11 | YELLOW=$(tput setaf 3)
 12 | BLUE=$(tput setaf 4)
 13 | MAGENTA=$(tput setaf 5)
 14 | CYAN=$(tput setaf 6)
 15 | WHITE=$(tput setaf 7)
 16 | 
 17 | # Clear Line
 18 | CL="\e[2K"
 19 | # Spinner Character
 20 | SPINNER="⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏"
 21 | 
 22 | function spinner() {
 23 |   tput civis
 24 |   task=$1
 25 |   msg=$2
 26 |   while :; do
 27 |     jobs %1 > /dev/null 2>&1
 28 |     [ $? = 0 ] || {
 29 |       printf "${CL}${GREEN}${BOLD}✓${WHITE}${BOLD} ${task} ${GREEN}${BOLD}Done\n${NORMAL}"
 30 |       break
 31 |     }
 32 |     for (( i=0; i<${#SPINNER}; i++ )); do
 33 |       sleep 0.05
 34 |       printf "${CL}${CYAN}${BLD}${SPINNER:$i:1}${NORMAL} ${task} ${BLUE}${BOLD}${msg}\r${NORMAL}"
 35 |       # tput cuu1
 36 |       # printf ""
 37 |     done
 38 |   done
 39 | }
 40 | 
 41 | function exec_spinner() {
 42 | msg="${2-InProgress}"
 43 | task="${3-$1}"
 44 | $1 > /dev/null 2>/dev/null & spinner "$task" "$msg"
 45 | # $1 & spinner "$task" "$msg"
 46 | 
 47 | tput cnorm
 48 | }
 49 | 
 50 | function check_bins() {
 51 | # checking brew binary
 52 | if [ "${CILIUMENABLED}" == 'true' ];
 53 |   then
 54 |     BINLIST+=" cilium"
 55 | fi
 56 | 
 57 | for b in $BINLIST ;
 58 | do 
 59 | if ! command -v "${b}" &> /dev/null
 60 | then
 61 |     echo "${b} could not be found"
 62 |     if [ "${b}" == brew ]
 63 |     then
 64 |       echo "Check <https://github.com/Homebrew/brew> on how to install it"
 65 |       exit
 66 |     else
 67 |         echo "Installing ${b} ..."
 68 |           case $i in 
 69 |             "cilium")
 70 |               brew install "${b}"-cli
 71 |               ;;
 72 |             "k0sctl")
 73 |               brew install k0sproject/tap/"${b}"@@
 74 |               ;;
 75 |             *)
 76 |               brew install "${b}"
 77 |               ;;
 78 |           esac
 79 |       echo ""${b}" successfully installed, next !" 
 80 | fi
 81 | fi
 82 | done
 83 | }
 84 | 
 85 | function check_gen_ssh_key(){
 86 |   if [ -s "${SSHPKEYPATH}""${SSHKEYNAME}" ];
 87 |     then
 88 | #    echo "checking existing key without passphrase"
 89 |     if ! ssh-keygen -y -P "" -f "${SSHPKEYPATH}""${SSHKEYNAME}" > /dev/null 2>&1;
 90 |       then
 91 |         echo "ssh key exist BUT WITH PASSPHRASE"
 92 |         echo "either remove the passphrase using: ssh-keygen -p ${SSHPKEYPATH}${SSHKEYNAME}"
 93 |         echo "or update the playground_vars script and change SSHKEYNAME value"
 94 |         echo "or load the ssh keypair using ssh-agent"
 95 |         exit
 96 |     fi
 97 |     else
 98 |     echo "generating ssh key ${SSHPKEYPATH}${SSHKEYNAME}"
 99 |       ssh-keygen -b 2048 -t rsa -f "${SSHPKEYPATH}""${SSHKEYNAME}" -q -N ""
100 |   fi
101 | }
102 | 
103 | function gen_multipass_cloud-init(){
104 | #  echo "Create cloud-init to import ssh key..."
105 | 
106 | # https://github.com/canonical/multipass/issues/965#issuecomment-591284180
107 | cat <<EOF > config/multipass-cloud-init.yml
108 | ---
109 | users:
110 |   - name: "${K0SADMINUSERNAME}"
111 |     sudo: ALL=(ALL) NOPASSWD:ALL
112 |     shell: /usr/bin/bash
113 |     ssh_authorized_keys:
114 |       - $( cat "${SSHPKEYPATH}""${SSHKEYNAME}.pub")
115 | EOF
116 | }
117 | 
118 | function check_instances() {
119 |   for knode in ${K0SHOSTLIST}; 
120 |   do 
121 |     timeout 5 bash -c "</dev/tcp/"${knode}"/22"
122 | #  ssh -q -o BatchMode=yes  -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${knode}" 'exit 0'
123 |     if [ $? == 0 ];then
124 |       echo "SSH Connection to "${knode}" is possible"
125 |     else
126 |       echo "SSH Connection to "${knode}" is broken"
127 |       exit 1
128 |     fi
129 |   done
130 | }
131 | 
132 | function create_instances(){
133 | K0SHOSTLIST=()
134 | 
135 | for ((n = 1 ; n <= "${NUMBER_OF_VMS}" ; n++)); do
136 | #echo "["${n}"/${NUMBER_OF_VMS}] Creating instance "${K0SNODENAME}"-${n} with multipass..."
137 | 
138 |   exec_spinner "multipass launch --name "${K0SNODENAME}"-"${n}" --cloud-init config/multipass-cloud-init.yml --cpus 2 --mem 6000M --disk 20G focal" "Work in Progress" "["${n}"/${NUMBER_OF_VMS}] Creating instance "${K0SNODENAME}"-${n} ..."
139 | 
140 |   # echo "adding node "${K0SNODENAME}"-${n} ip to k0sctl host list "
141 |   nodeip="$(multipass info "${K0SNODENAME}"-"${n}" --format csv | awk -F',' '{print$3}' | tail -1)"
142 |   K0SHOSTLIST+="${nodeip} "
143 | done
144 | check_instances
145 | }
146 | 
147 | function gen_k0s_config(){
148 |   k0sctl init --k0s -n "${K0SCLUSTERNAME}" -u "${K0SADMINUSERNAME}" -i "${SSHPKEYPATH}""${SSHKEYNAME}" -C "${K0SCONTROLLERCOUNT}" ${K0SHOSTLIST} > k0s-"${K0SCLUSTERNAME}"-config.yaml
149 | 
150 | }
151 | 
152 | function tweak_k0s_config(){
153 |     echo "tweaking ${K0SCLUSTERNAME} config file for cilium playground" 
154 |     
155 |     if [  "${DEBUGENABLED}" == 'true' ];
156 |     then
157 |       echo "enabling k0s controller(s) debug mode "
158 |       sed -i '' -e '/controller/a\'$'\n''    installFlags: \
159 |       - --enable-worker \
160 |       - --debug=true' k0s-"${K0SCLUSTERNAME}"-config.yaml
161 |     fi
162 |     
163 |     if [ "${CILIUMENABLED}" == 'true' ];
164 |     then
165 |       echo "disabling kube-proxy"
166 |       sed -i '' -e '/          kubeProxy:/ {' -e 'n; s/.*/            disabled: true/' -e '}' k0s-"${K0SCLUSTERNAME}"-config.yaml
167 |       echo "changing kuberouter to custom"
168 |       sed -i '' -e 's/provider: kuberouter/provider: custom/' k0s-"${K0SCLUSTERNAME}"-config.yaml
169 |       sed -i '' -e 's/10\.244\.0\.0\/16/10\.'"${p}"'44\.0\.0\/16/' k0s-"${K0SCLUSTERNAME}"-config.yaml
170 |       sed -i '' -e 's/10\.96\.0\.0\/12/10\.'"${p}"'6\.0\.0\/12/' k0s-"${K0SCLUSTERNAME}"-config.yaml
171 |       inject_cilium_config
172 |     fi
173 | 
174 | }
175 | 
176 | function inject_cilium_config() {
177 |   CNODEARRAY=(${K0SHOSTLIST})
178 |   CNODE=${CNODEARRAY[0]}
179 | 
180 |     if [ "${CUSTOMCILIUM}" == 'true' ];
181 |     then
182 |       sed -i '' -e '/       api:/ {
183 |       r config/'"${CUSTOMCILIUMVALUESFILES}"'
184 |       N
185 |       }' k0s-"${K0SCLUSTERNAME}"-config.yaml  
186 |     else 
187 |       sed -e 's/METALLBVERS/'"${METALLBVERS}"'/g; s/CLUSTERNAME/'"${K0SCLUSTERNAME}"'/g; s/CLUSTERID/'"${p}"'/g; s/CILIUMVERS/'"${CILIUMVERS}"'/g; s/CNODE/'"${CNODE}"'/g' config/cilium-values.yaml.tpl > config/cilium-values-"${K0SCLUSTERNAME}".yaml
188 |       sed -i '' -e 's/10\.244\.0\.0\/16/10\.'"${p}"'44\.0\.0\/16/' config/cilium-values-"${K0SCLUSTERNAME}".yaml
189 | 
190 |       sed -i '' -e '/       api:/ {
191 |       r config/cilium-values-'"${K0SCLUSTERNAME}"'.yaml
192 |       N
193 |       }' k0s-"${K0SCLUSTERNAME}"-config.yaml
194 |       
195 |     fi
196 | }
197 | 
198 | ###################
199 | # metallb functions
200 | ###################
201 | 
202 | function gen_metallb_iprange(){
203 |   CNODEARRAY=(${K0SHOSTLIST})
204 |   CNODE=${CNODEARRAY[0]}
205 |   NODERANGE=$(echo "${CNODE}" | awk -F'.' '{print$1"."$2"."$3}')
206 |   STARTRANGE="${NODERANGE}.${p}00"
207 |   ENDRANGE="${NODERANGE}.${p}50"
208 | }
209 | 
210 | function gen_metallb_config(){
211 | gen_metallb_iprange
212 | sed -e 's/METALLBIPRANGE/'"${STARTRANGE}"'-'"${ENDRANGE}"'/' config/metallb-config.yaml.tpl > config/metallb-config-"${K0SCLUSTERNAME}".yaml
213 | }
214 | 
215 | function apply_metallb_config(){
216 | export KUBECONFIG=~/.kube/"${K0SCLUSTERNAME}".config
217 | readynbr=''
218 | check_deployment_status metallb
219 | gen_metallb_config
220 | kubectl apply -f config/metallb-config-"${K0SCLUSTERNAME}".yaml > /dev/null 2>&1
221 | # echo $(echo ${FUNCNAME} | awk -F "_" '{print$2}')" configuration:"  $(printf "${BOLDGREEN}DEPLOYED${ENDCOLOR}")
222 | }
223 | 
224 | 
225 | function build_k0s_cluster(){
226 | if [ "${CILIUMENABLED}" == 'true' ];
227 |   then
228 |     OPT='--no-wait'
229 | fi
230 |     k0sctl apply --config k0s-"${K0SCLUSTERNAME}"-config.yaml "${OPT}" > /dev/null 2>&1
231 | #    echo $(echo ${FUNCNAME} | awk -F "_" '{print$2}')":"  $(printf "${BOLDGREEN}DEPLOYED${ENDCOLOR}")
232 | 
233 | }
234 | 
235 | function gen_kube_config() {
236 | if [ ! -d ~/.kube ]
237 | then
238 |   mkdir ~/.kube
239 | fi
240 | k0sctl kubeconfig --config k0s-"${K0SCLUSTERNAME}"-config.yaml > ~/.kube/"${K0SCLUSTERNAME}".config
241 | if [ "${CILIUMENABLED}" == 'true' ];
242 |   then
243 |     sed -i '' -e 's/admin/admin-'"${K0SCLUSTERNAME}"'/g' ~/.kube/"${K0SCLUSTERNAME}".config 
244 | fi
245 | }
246 | 
247 | function merge_kube_config() {
248 |   export KUBECONFIG=$(find  ~/.kube/k0scilium*.config | awk 'BEGIN { ORS = ":" } { print }') \
249 |   && kubectl config view --merge --flatten > ~/.kube/config-cilium \
250 |   && export KUBECONFIG=~/.kube/config-cilium
251 | }
252 | 
253 | function valid_ip()
254 | {
255 |     local  ip=$1
256 |     local  stat=1
257 | 
258 |     if [[ $ip =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then
259 |         OIFS=$IFS
260 |         IFS='.'
261 |         ip=($ip)
262 |         IFS=$OIFS
263 |         [[ ${ip[0]} -le 255 && ${ip[1]} -le 255 \
264 |             && ${ip[2]} -le 255 && ${ip[3]} -le 255 ]]
265 |         stat=$?
266 |     fi
267 |     return $stat
268 | }
269 | 
270 | # function check_pods_ready_status(){
271 | 
272 | # until [ $(kubectl -n ${1} get pods -o jsonpath='{range .items[*]}{.status.containerStatuses[*].ready.true}{.metadata.name}{ "\n"}{end}' | wc -l ) -eq $(kubectl get po -n ${1} | tail -n +2 | wc -l  )  ];
273 | # do 
274 | #   sleep 10
275 | #   echo "waiting for ${1} pods to be fully ready"
276 | # done
277 | #   echo "${1} pods status: " $(printf "${BOLDGREEN}READY${ENDCOLOR}")
278 | 
279 | # }
280 | 
281 | function check_deployment_status(){
282 | 
283 | 
284 | # echo "waiting for ${1} services to be deployed"
285 | until kubectl get svc -n ${1} -o custom-columns=NAME:.metadata.name | tail -n +2 | grep -E -o '[A-Za-z]' > /dev/null 2>&1 ;
286 | do 
287 |         sleep 5
288 | done
289 | # echo "${1} service status: " $(printf "${BOLDGREEN}DEPLOYED${ENDCOLOR}")
290 | SVCLIST=$(kubectl get svc -n ${1} -o custom-columns=NAME:.metadata.name | tail -n +2)
291 | SVCNUMBER="$(echo "${SVCLIST}" | wc -l)"
292 | 
293 | while [[ ${readynbr} -lt "${SVCNUMBER}" ]]
294 | do
295 | 
296 |   
297 |   sleep 5
298 |   # echo "waiting for ${1} services to be up and running:"$(echo ${SVCLIST} | tr '\n' ' ')
299 |   for svc in ${SVCLIST};
300 |     do 
301 |         until kubectl get endpoints ${svc} -n ${1} -o=jsonpath='{.subsets[0].addresses[0].ip}' | grep -E -o '([0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3})' > /dev/null 2>&1 ; 
302 |         do 
303 |         sleep 5
304 |         done
305 |         readynbr=$((readynbr+1))
306 | #        echo "${svc} status: " $(printf "${BOLDGREEN}READY${ENDCOLOR}")
307 |   done
308 | done  
309 | }
310 | 
311 | 
312 | function enable_cilium_clustermesh() {
313 | merge_kube_config
314 | if [ "${CILIUMENABLED}" == 'true' ];
315 |   then
316 |   CLUSTERLIST=()
317 | #  while IFS=$'\n' read -r line; do CLUSTERLIST+=("$line"); done < <(kubectl config get-contexts --output=name)
318 |   while IFS= read -r line; do CLUSTERLIST+=( "$line" ); done < <(kubectl config get-contexts --output=name)
319 |   # echo knames: $(kubectl config get-contexts --output=name)
320 |   # echo CLUSTER_LIST: ${CLUSTERLIST}
321 |   # echo C1: ${CLUSTERLIST[0]}
322 |   # echo C2: ${CLUSTERLIST[1]}
323 |   for c in ${CLUSTERLIST[@]};
324 |   do 
325 |   kubectl config use-context "${c}"
326 |   kubectl get endpoints clustermesh-apiserver -n cilium -o=jsonpath='{.subsets[0].addresses[0].ip}' | grep -E -o '([0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3})' > /dev/null 2>&1
327 |   if [ $? -ne 0 ] && [ "${c}" == "${CLUSTERLIST[0]}" ];
328 |     then
329 |     readynbr=''
330 |     check_deployment_status cilium
331 | #    echo "enable cilium clustermesh on cluster:" "${c}"
332 |     cilium clustermesh enable --context "${c}" --service-type=LoadBalancer -n cilium
333 |   fi
334 |     if [ "${c}" == "${CLUSTERLIST[1]}" ];
335 |     then
336 |       readynbr=''
337 |       check_deployment_status cilium
338 |       cilium clustermesh enable --context "${c}" --service-type=LoadBalancer -n cilium
339 | #      echo "connecting cluster "${CLUSTERLIST[0]}" and "${CLUSTERLIST[1]}"" 
340 |       cilium clustermesh connect --context "${CLUSTERLIST[0]}" --destination-context "${CLUSTERLIST[1]}"  -n cilium
341 |     fi        
342 |   done
343 |   # check_pods_ready_status cilium
344 | fi
345 | }
346 | 
347 | 
348 | function create_platform() {
349 |   exec_spinner "check_bins" "Work in progress" "checking binary:"
350 |   exec_spinner "check_gen_ssh_key" "checking ssh keys" "check_gen_ssh_key:"
351 |   exec_spinner "gen_multipass_cloud-init" "Work in Progress" "Generate Cloud init Config:"
352 |   
353 |   for ((p=1; p <= K0SCLUSTERNUMBER; p++))
354 |   do
355 |     K0SCLUSTERNAME=${K0SCLUSTERNAMEVAR}-"${p}"
356 |     K0SNODENAME=${K0SCLUSTERNAME}-node
357 |     printf "${CL}${BLUE}${BOLD}Cluster: ${K0SCLUSTERNAME} \n${NORMAL}"
358 |     create_instances
359 |     exec_spinner "gen_k0s_config" "Work in Progress" "Generating k0s yaml"
360 |       if [ ${CILIUMENABLED} == 'true' ];
361 |         then
362 |             printf "${CL}${BLUE}${BOLD}Cluster: ${K0SCLUSTERNAME} will use Cilium Clustermesh configuration \n${NORMAL}"
363 |             exec_spinner "tweak_k0s_config" "Work in Progress" "tweaking k0s config"
364 |             exec_spinner "build_k0s_cluster" "Work in Progress" "Building k0s cluster"
365 |             exec_spinner "gen_kube_config" "Work in Progress" "Generating kubeconfig"
366 |             exec_spinner "apply_metallb_config" "Work in Progress" "Deploy MetalLB"
367 |             exec_spinner "enable_cilium_clustermesh" "Work in Progress" "Deploying Cilium"           
368 |         else
369 |             printf "${CL}${BLUE}${BOLD}Cluster: ${K0SCLUSTERNAME} will use default CNI \n${NORMAL}"
370 |             exec_spinner "build_k0s_cluster" "Work in Progress" "Building k0s cluster"
371 |             exec_spinner "gen_kube_config" "Work in Progress" "Generating kubeconfig"
372 |             exec_spinner "merge_kube_config" "Work in Progress" "merging kubeconfig"
373 |       fi
374 |   done
375 | }
376 | function playground_help() {
377 |   # Display Help
378 | 
379 |   printf "${CL}${GREEN}${BOLD}Cilium playground scripts \n${NORMAL}"
380 | 
381 |   # printf "${CL}${GREEN}${BOLD}✓${WHITE}${BOLD} ${task} ${GREEN}${BOLD}Done\n${NORMAL}"
382 | 
383 |    echo ""
384 |    echo
385 |    printf "${CL}Syntax: ./playground [${GREEN}${BOLD} -i / --install || -d / --delete || ${RED}${BOLD} -v / --verbose ${GREEN}${BOLD} || -h / --help]\n${NORMAL}"
386 |    echo ""
387 |    echo "options:"
388 |    printf "${GREEN} -i / --install     deploys the playground\n${NORMAL}"
389 |    printf "${GREEN} -d / --delete      delete the playground , but leave '${BINLIST}' installed\n${NORMAL}"
390 |    printf "${GREEN} -h / --help        Print this Help.\n${NORMAL}"
391 |    printf "${RED} -v / --verbose     Verbose mode [ NOT IMPLEMENTED YET ]\n${NORMAL}"
392 |    echo
393 |    exit 0
394 | }
395 | 
396 | function purge_all() {
397 |   echo "${K0SCLUSTERNAMEVAR}"
398 |   nodelist="$(multipass list --format csv | grep "${K0SCLUSTERNAMEVAR}" |  awk -F',' '{print$1}' | awk 'BEGIN { ORS = " " } { print }'  )"
399 |   printf "${RED} HERE THE NODE LIST TO BE DELETED: \n${NORMAL}" "${nodelist}"
400 |   printf "${BOLD}${RED}Do you wish to delete them? \n${NORMAL}"
401 |   read -p "are you REALLY SURE ? (Delete_ALL/No) " yn
402 |   case $yn in 
403 |     Delete_ALL ) exec_spinner "multipass delete ${nodelist}" "Work in Progress" "deleting nodes" && exec_spinner "multipass purge" "Work in Progress" "Purging multipass" && rm -f "${HOME}"/.kube/"${K0SCLUSTERNAMEVAR}"-*.config *-"${K0SCLUSTERNAMEVAR}"-* config/*-"${K0SCLUSTERNAMEVAR}"-* config/multipass-cloud-init.yml  && \
404 |     printf "\n${BOLD}${RED} /!\/!\/!\/!\/!\/!\/!\/!\/!\/!\/!\/!\/!\/!\/!\/!\/!\/!\/!\/!\/!\/!\ ${NORMAL}" \
405 |     && printf "\n${BOLD}${RED} /!\ you might need to manually Cleanup your ~/.ssh/known_hosts /!\ ${NORMAL}" \
406 |     && printf "\n${BOLD}${RED} /!\/!\/!\/!\/!\/!\/!\/!\/!\/!\/!\/!\/!\/!\/!\/!\/!\/!\/!\/!\/!\/!\ \n${NORMAL}" ; break ;;
407 |     No ) echo "OK i won't change anything ( yet? :p )"; exit ;;
408 |     * ) echo "you need to either enter: Delete_ALL or No ";;
409 |   esac
410 | }
411 | 
412 | function count_down() {
413 | COUNT=$1
414 | # bash while loop
415 | while [ $COUNT -gt 0 ]; do
416 |         echo $COUNT
417 |         let COUNT=COUNT-1
418 |         sleep 1
419 | done
420 | }


--------------------------------------------------------------------------------
/playground_vars.sh:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env bash
 2 | ### DO NOT CHANGES THOSE VARIABLES ####
 3 | 
 4 | #### COLORS:
 5 | GREEN="32"
 6 | BOLDGREEN="\e[1;${GREEN}m"
 7 | ENDCOLOR="\e[0m"
 8 | 
 9 | #### list of binaries required
10 | BINLIST="brew multipass k0sctl kubectl helm"
11 | 
12 | ### DO NOT CHANGES THOSE VARIABLES ####
13 | 
14 | ### VARIABLES BELOW CAN BE CHANGED ####
15 | DEBUGENABLED='true'
16 | 
17 | K0SCTLVERS='v0.14.0'
18 | 
19 | METALLBVERS='v0.13.7'
20 | 
21 | CILIUMENABLED='true'
22 | 
23 | CILIUMVERS='1.12.4'
24 | 
25 | CUSTOMCILIUM='false'
26 | 
27 | CUSTOMCILIUMVALUESFILES=''
28 | # Default to 5 VMs
29 | NUMBER_OF_VMS='3'
30 | 
31 | # ssh pub key(s) path 
32 | SSHPKEYPATH=~/.ssh/
33 | 
34 | # ssh pub key name   
35 | SSHKEYNAME=id_rsa
36 | 
37 | K0SADMINUSERNAME=k0s
38 | 
39 | K0SCONTROLLERCOUNT='1'
40 | 
41 | K0SCLUSTERNUMBER='2'
42 | 
43 | K0SCLUSTERNAMEVAR='k0scilium'
44 | 
45 | 
46 | 


--------------------------------------------------------------------------------