├── .gitignore ├── manifests └── nginx.yaml ├── helpers.sh ├── README.md └── .github └── workflows └── cluster.yml /.gitignore: -------------------------------------------------------------------------------- 1 | config_encoded 2 | config.yaml 3 | -------------------------------------------------------------------------------- /manifests/nginx.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: nginx-deployment 5 | labels: 6 | app: nginx 7 | spec: 8 | replicas: 3 9 | selector: 10 | matchLabels: 11 | app: nginx 12 | template: 13 | metadata: 14 | labels: 15 | app: nginx 16 | spec: 17 | containers: 18 | - name: nginx 19 | image: nginx:1.14.2 20 | ports: 21 | - containerPort: 80 -------------------------------------------------------------------------------- /helpers.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | start_vpn() { 4 | curl https://get.mocaccino.org/luet/get_luet_root.sh | sudo sh 5 | sudo luet install -y repository/mocaccino-extra 6 | sudo luet install -y utils/edgevpn container/k3s 7 | echo "$EDGEVPN" | base64 -d > config.yaml 8 | sudo -E EDGEVPNCONFIG=config.yaml IFACE=edgevpn0 edgevpn > /dev/null 2>&1 & 9 | } 10 | 11 | wait_master() { 12 | while ! nc -z $MASTER 6443; do 13 | echo "K3s server not ready yet.." 14 | sleep 1 15 | done 16 | } 17 | 18 | start_server() { 19 | while ! ip a | grep $ADDRESS ; do 20 | echo "VPN not ready yet.." 21 | sleep 1 22 | done 23 | sudo ip a 24 | 25 | ( 26 | set +e 27 | while true; do 28 | 29 | sudo -E k3s server --flannel-iface=edgevpn0 --node-ip $IP --node-external-ip $IP 30 | done 31 | ) & 32 | 33 | while ! nc -z localhost 6443; do 34 | echo "K3s server not ready yet.." 35 | sleep 1 36 | done 37 | while [ ! -f /etc/rancher/k3s/k3s.yaml ]; do 38 | echo "KUBECONFIG not available yet.." 39 | sleep 1 40 | done 41 | 42 | sudo cat /etc/rancher/k3s/k3s.yaml 43 | sudo luet serve-repo --address $IP --dir /var/lib/rancher/k3s/server/ & 44 | sudo luet serve-repo --address $IP --port 9091 --dir /etc/rancher/k3s 45 | } 46 | 47 | start_agent() { 48 | wait_master 49 | 50 | while ! nc -z $MASTER 9090; do 51 | echo "certs not ready yet.." 52 | sleep 1 53 | done 54 | 55 | ( 56 | set +e 57 | while true; do 58 | export K3S_TOKEN=$( curl --silent -L http://$MASTER:9090/node-token ) 59 | 60 | echo "Node token $K3S_TOKEN" 61 | sudo -E k3s agent --server https://$MASTER:6443 --flannel-iface=edgevpn0 --node-ip $IP 62 | done 63 | ) 64 | } 65 | 66 | start_jumpbox() { 67 | sudo luet install -y utils/k9s container/kubectl 68 | } 69 | 70 | prepare_jumpbox() { 71 | wait_master 72 | curl http://10.1.0.20:9091/k3s.yaml | sed 's/127\.0\.0\.1/10.1.0.20/g' > k3s.yaml 73 | } 74 | 75 | install_helm() { 76 | curl https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash 77 | } 78 | 79 | install_fleet() { 80 | export KUBECONFIG=k3s.yaml 81 | helm -n fleet-system install --create-namespace --wait \ 82 | fleet-crd https://github.com/rancher/fleet/releases/download/v0.3.3/fleet-crd-0.3.3.tgz 83 | helm -n fleet-system install --create-namespace --wait \ 84 | fleet https://github.com/rancher/fleet/releases/download/v0.3.3/fleet-0.3.3.tgz 85 | 86 | while : ; do 87 | kubectl get pods -n fleet-local && break 88 | sleep 5 89 | done 90 | 91 | cat < config_encoded 33 | ``` 34 | 35 | now set the content of `config_encoded` to the repository as `EDGEVPN` secret. 36 | 37 | ### 3) Run the workflow 38 | 39 | Commit something, it will run automatically. Or just check out the latest run and click on the "Re-run job" button. 40 | 41 | 42 | That's it, now go into the "jumpbox" job and wait for it to display an ssh connection string. 43 | 44 | Once inside, you can use kubectl with: 45 | 46 | ```bash 47 | KUBECONFIG=k3s.yaml kubectl get pods -A 48 | ``` 49 | 50 | ## More 51 | 52 | ### Persistency 53 | 54 | The cluster is not persistent. Github Action has a limit timeout of 6h. Nothing forbids to reschedule another job each 6h our, and you can decomment that in the workflow file. 55 | 56 | ### Access the cluster 57 | 58 | To be able to access the cluster from your system, you need to connect via VPN. 59 | 60 | In the terminal, run: 61 | 62 | ```bash 63 | sudo IFACE=edgevpn0 ADDRESS=10.1.0.2/24 EDGEVPNCONFIG=$PWD/config.yaml edgevpn 64 | ``` 65 | 66 | _Note_ that the `ADDRESS` we are setting here is the one we will have in the VPN. We are not setting any public IP. Addresses are internal. 67 | 68 | Open another terminal, and wait for connection to be available, monitor ```ping 10.1.0.20```. 69 | 70 | The setup is fixed, you will find the master node on the `10.1.0.20` ip over VPN. 71 | 72 | ### Grab cluster kube config 73 | 74 | From inside the VPN you can always get the latest kubeconfig with: 75 | 76 | ```bash 77 | curl http://10.1.0.20:9091/k3s.yaml | sed 's/127\.0\.0\.1/10.1.0.20/g' > k3s.yaml 78 | ``` 79 | 80 | ### `manifests` 81 | 82 | Each kubernetes resource in the manifests folder gets automatically deployed with fleet. You can disable this behavior by commenting the `install_helm` and `install_fleet` line in the cluster workflow file. -------------------------------------------------------------------------------- /.github/workflows/cluster.yml: -------------------------------------------------------------------------------- 1 | name: Cluster 2 | 3 | on: 4 | push: 5 | # Decomment to run it scheduled (it will recreate each 6h) 6 | # schedule: 7 | # - cron: "0 */6 * * *" 8 | 9 | # This block ensures only one instance is running 10 | concurrency: 11 | group: ci-${{ github.head_ref || github.ref }}-${{ github.repository }} 12 | 13 | jobs: 14 | jumpbox: 15 | runs-on: ubuntu-latest 16 | strategy: 17 | matrix: 18 | include: 19 | - ip: "10.1.0.3/24" 20 | ip_addr: "10.1.0.3" 21 | steps: 22 | - name: Release space from worker 23 | run: | 24 | sudo rm -rf /usr/local/lib/android # will release about 10 GB if you don't need Android 25 | sudo rm -rf /usr/share/dotnet # will release about 20GB if you don't need .NET 26 | - name: Checkout 27 | uses: actions/checkout@v2 28 | with: 29 | fetch-depth: 0 30 | - name: server up 31 | env: 32 | EDGEVPN: ${{ secrets.EDGEVPN }} 33 | ADDRESS: ${{ matrix.ip }} 34 | IP: ${{ matrix.ip_addr }} 35 | MASTER: 10.1.0.20 36 | run: | 37 | source helpers.sh 38 | start_vpn 39 | start_jumpbox 40 | prepare_jumpbox 41 | # Decomment to avoid installing fleet by default 42 | install_helm 43 | install_fleet 44 | - name: Setup tmate session 45 | uses: mxschmitt/action-tmate@v3 46 | with: 47 | limit-access-to-actor: true 48 | 49 | master: 50 | runs-on: ubuntu-latest 51 | strategy: 52 | matrix: 53 | include: 54 | - ip: "10.1.0.20/24" 55 | ip_addr: "10.1.0.20" 56 | steps: 57 | - name: Release space from worker 58 | run: | 59 | sudo rm -rf /usr/local/lib/android # will release about 10 GB if you don't need Android 60 | sudo rm -rf /usr/share/dotnet # will release about 20GB if you don't need .NET 61 | - name: Checkout 62 | uses: actions/checkout@v2 63 | with: 64 | fetch-depth: 0 65 | - name: server up 66 | env: 67 | EDGEVPN: ${{ secrets.EDGEVPN }} 68 | ADDRESS: ${{ matrix.ip }} 69 | IP: ${{ matrix.ip_addr }} 70 | run: | 71 | source helpers.sh 72 | start_vpn 73 | start_server 74 | workers: 75 | runs-on: ubuntu-latest 76 | strategy: 77 | matrix: 78 | include: 79 | # Add more lines here to have more worker nodes. 80 | # Just take care of bumping the IP address. 81 | # The IP address is the VPN address of each node. 82 | - ip: "10.1.0.21/24" 83 | ip_addr: "10.1.0.21" 84 | - ip: "10.1.0.22/24" 85 | ip_addr: "10.1.0.22" 86 | - ip: "10.1.0.23/24" 87 | ip_addr: "10.1.0.23" 88 | - ip: "10.1.0.24/24" 89 | ip_addr: "10.1.0.24" 90 | steps: 91 | - name: Release space from worker 92 | run: | 93 | sudo rm -rf /usr/local/lib/android # will release about 10 GB if you don't need Android 94 | sudo rm -rf /usr/share/dotnet # will release about 20GB if you don't need .NET 95 | - name: Checkout 96 | uses: actions/checkout@v2 97 | with: 98 | fetch-depth: 0 99 | - name: server up 100 | env: 101 | EDGEVPN: ${{ secrets.EDGEVPN }} 102 | ADDRESS: ${{ matrix.ip }} 103 | IP: ${{ matrix.ip_addr }} 104 | MASTER: 10.1.0.20 105 | run: | 106 | source helpers.sh 107 | start_vpn 108 | 109 | start_agent 110 | --------------------------------------------------------------------------------