├── .dockerignore ├── .github └── workflows │ ├── main.yaml │ ├── reusable-multi-node.yaml │ └── reusable-single-node.yaml ├── .gitignore ├── Dockerfile ├── Dockerfile.d ├── SHA256SUMS.d │ ├── cni-plugins-v1.7.1 │ ├── flannel-v0.26.7 │ └── helm-v3.17.3 ├── etc_udev_rules.d_90-flannel.rules └── u7s-entrypoint.sh ├── LICENSE ├── Makefile ├── Makefile.d ├── check-preflight.sh ├── detect-container-engine.sh ├── install-flannel.sh ├── node-subnet.sh └── sync-external-ip.sh ├── README.md ├── docker-compose.yaml ├── docs └── images │ └── multi-tenancy.png ├── hack ├── README.md ├── create-cluster-lima.sh └── test-smoke.sh ├── init-host ├── README.md ├── init-host.root.d │ ├── install-nerdctl.sh │ └── install-podman.sh ├── init-host.root.sh └── init-host.rootless.sh └── kubeadm-config.yaml /.dockerignore: -------------------------------------------------------------------------------- 1 | /kubeconfig 2 | /kubectl 3 | /join-command 4 | -------------------------------------------------------------------------------- /.github/workflows/main.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: Main 3 | on: [push, pull_request] 4 | jobs: 5 | single-node: 6 | name: "Single node" 7 | strategy: 8 | fail-fast: false 9 | matrix: 10 | container_engine: [docker, nerdctl, podman] 11 | uses: ./.github/workflows/reusable-single-node.yaml 12 | with: 13 | container_engine: ${{ matrix.container_engine }} 14 | 15 | multi-node: 16 | name: "Multi node" 17 | strategy: 18 | fail-fast: false 19 | matrix: 20 | include: 21 | - lima_template: template://ubuntu-24.04 22 | container_engine: docker 23 | - lima_template: template://ubuntu-24.04 24 | container_engine: nerdctl 25 | - lima_template: template://centos-stream-9 26 | container_engine: podman 27 | - lima_template: template://fedora 28 | container_engine: podman 29 | uses: ./.github/workflows/reusable-multi-node.yaml 30 | with: 31 | lima_template: ${{ matrix.lima_template }} 32 | container_engine: ${{ matrix.container_engine }} 33 | 34 | # TODO: this test should create multiple instances of Usernetes on each of the hosts 35 | multi-node-custom-ports: 36 | name: "Multi node with custom service ports" 37 | uses: ./.github/workflows/reusable-multi-node.yaml 38 | with: 39 | lima_template: "template://ubuntu-24.04" 40 | container_engine: "docker" 41 | # Defaults to 6443 42 | kube_apiserver_port: "8080" 43 | # Defaults to 8472 44 | flannel_port: "9072" 45 | # Defaults to 10250 46 | kubelet_port: "20250" 47 | # Defaults to 2379 48 | etcd_port: "9090" 49 | -------------------------------------------------------------------------------- /.github/workflows/reusable-multi-node.yaml: -------------------------------------------------------------------------------- 1 | name: Multi Node 2 | on: 3 | workflow_call: 4 | # allow reuse of this workflow in other files here 5 | inputs: 6 | lima_template: 7 | description: lima template 8 | type: string 9 | default: "template://ubuntu-24.04" 10 | container_engine: 11 | description: container engine 12 | type: string 13 | default: "docker" 14 | kubelet_port: 15 | description: kubelet serving port 16 | type: string 17 | default: "10250" 18 | flannel_port: 19 | description: flannel vxlan port 20 | type: string 21 | default: "8472" 22 | etcd_port: 23 | description: etcd service port 24 | type: string 25 | default: "2379" 26 | kube_apiserver_port: 27 | description: Kubernetes API server port 28 | # Using string, might be bug with number 29 | # https://github.com/orgs/community/discussions/67182 30 | type: string 31 | default: "6443" 32 | 33 | # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idpermissions 34 | permissions: read-all 35 | 36 | jobs: 37 | multi-node: 38 | name: "Multi node (emulated using Lima)" 39 | runs-on: ubuntu-24.04 40 | timeout-minutes: 30 41 | env: 42 | LIMA_TEMPLATE: "${{ inputs.lima_template }}" 43 | CONTAINER_ENGINE: "${{ inputs.container_engine }}" 44 | PORT_KUBE_APISERVER: "${{ inputs.kube_apiserver_port }}" 45 | PORT_FLANNEL: "${{ inputs.flannel_port }}" 46 | PORT_KUBELET: "${{ inputs.kubelet_port }}" 47 | PORT_ETCD: "${{ inputs.etcd_port }}" 48 | KUBECONFIG: ./kubeconfig 49 | steps: 50 | - uses: actions/checkout@v4 51 | 52 | - uses: lima-vm/lima-actions/setup@v1 53 | id: lima-actions-setup 54 | 55 | - name: "Cache ~/.cache/lima" 56 | uses: actions/cache@v4 57 | with: 58 | path: ~/.cache/lima 59 | key: lima-${{ steps.lima-actions-setup.outputs.version }} 60 | 61 | - run: ./hack/create-cluster-lima.sh 62 | - run: kubectl taint nodes --all node-role.kubernetes.io/control-plane- || true 63 | - run: ./hack/test-smoke.sh 64 | - if: failure() 65 | name: "kubectl get nodes" 66 | run: | 67 | set -x 68 | kubectl get nodes -o wide 69 | kubectl get nodes -o yaml 70 | limactl shell host0 df -h 71 | limactl shell host1 df -h 72 | - if: failure() 73 | name: "kubectl get pods" 74 | run: | 75 | set -x 76 | kubectl get pods -A -o yaml 77 | limactl shell host0 journalctl --user --no-pager --since "10 min ago" 78 | - name: "Test data persistency after restarting the node" 79 | run: | 80 | limactl stop host0 81 | limactl stop host1 82 | limactl start host0 83 | limactl start host1 84 | # The plain mode of Lima disables automatic port forwarding 85 | ssh -q -f -N -L ${{ inputs.kube_apiserver_port }}:127.0.0.1:${{ inputs.kube_apiserver_port }} -F ~/.lima/host0/ssh.config lima-host0 86 | sleep 30 87 | ./hack/test-smoke.sh 88 | -------------------------------------------------------------------------------- /.github/workflows/reusable-single-node.yaml: -------------------------------------------------------------------------------- 1 | name: Single Node 2 | on: 3 | workflow_call: 4 | inputs: 5 | container_engine: 6 | description: container engine 7 | type: string 8 | default: "docker" 9 | kubelet_port: 10 | description: kubelet serving port 11 | type: string 12 | default: "10250" 13 | flannel_port: 14 | description: flannel vxlan port 15 | type: string 16 | default: "8472" 17 | etcd_port: 18 | description: etcd service port 19 | type: string 20 | default: "2379" 21 | kube_apiserver_port: 22 | description: Kubernetes API server port 23 | # Using string, might be bug with number 24 | # https://github.com/orgs/community/discussions/67182 25 | type: string 26 | default: "6443" 27 | 28 | # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idpermissions 29 | permissions: read-all 30 | 31 | jobs: 32 | single-node: 33 | name: "Single node" 34 | runs-on: ubuntu-24.04 35 | timeout-minutes: 40 36 | env: 37 | CONTAINER_ENGINE: "${{ inputs.container_engine }}" 38 | PORT_KUBE_APISERVER: "${{ inputs.kube_apiserver_port }}" 39 | PORT_FLANNEL: "${{ inputs.flannel_port }}" 40 | PORT_KUBELET: "${{ inputs.kubelet_port }}" 41 | PORT_ETCD: "${{ inputs.etcd_port }}" 42 | KUBECONFIG: ./kubeconfig 43 | steps: 44 | - uses: actions/checkout@v4 45 | - name: Set up cgroup v2 delegation 46 | run: | 47 | sudo mkdir -p /etc/systemd/system/user@.service.d 48 | cat <>/etc/hosts 9 | 10 | exec "$@" 11 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Run `make help` to show usage 2 | .DEFAULT_GOAL := help 3 | 4 | # Change ports for different kubernetes services 5 | export PORT_ETCD ?= 2379 6 | export PORT_KUBELET ?= 10250 7 | export PORT_FLANNEL ?= 8472 8 | export PORT_KUBE_APISERVER ?= 6443 9 | 10 | # HOSTNAME is the name of the physical host 11 | export HOSTNAME ?= $(shell hostname) 12 | # HOST_IP is the IP address of the physical host. Accessible from other hosts. 13 | export HOST_IP ?= $(shell ip --json route get 1 | jq -r .[0].prefsrc) 14 | # NODE_NAME is the host name of the Kubernetes node running in Rootless Docker. 15 | # Not accessible from other hosts. 16 | export NODE_NAME ?= u7s-$(HOSTNAME) 17 | # NODE_SUBNET is the subnet of the Kubernetes node running in Rootless Docker. 18 | # Not accessible from other hosts. 19 | export NODE_SUBNET ?= $(shell $(CURDIR)/Makefile.d/node-subnet.sh) 20 | # NODE_IP is the IP address of the Kubernetes node running in Rootless Docker. 21 | # Not accessible from other hosts. 22 | export NODE_IP := $(subst .0/24,.100,$(NODE_SUBNET)) 23 | 24 | export CONTAINER_ENGINE ?= $(shell $(CURDIR)/Makefile.d/detect-container-engine.sh CONTAINER_ENGINE) 25 | 26 | export CONTAINER_ENGINE_TYPE ?= $(shell $(CURDIR)/Makefile.d/detect-container-engine.sh CONTAINER_ENGINE_TYPE) 27 | 28 | COMPOSE ?= $(shell $(CURDIR)/Makefile.d/detect-container-engine.sh COMPOSE) 29 | 30 | NODE_SERVICE_NAME := node 31 | NODE_SHELL := $(COMPOSE) exec \ 32 | -e HOST_IP=$(HOST_IP) \ 33 | -e NODE_NAME=$(NODE_NAME) \ 34 | -e NODE_SUBNET=$(NODE_SUBNET) \ 35 | -e NODE_IP=$(NODE_IP) \ 36 | -e PORT_KUBE_APISERVER=$(PORT_KUBE_APISERVER) \ 37 | -e PORT_FLANNEL=$(PORT_FLANNEL) \ 38 | -e PORT_KUBELET=$(PORT_KUBELET) \ 39 | -e PORT_ETCD=$(PORT_ETCD) \ 40 | $(NODE_SERVICE_NAME) 41 | 42 | ifeq ($(CONTAINER_ENGINE),nerdctl) 43 | ifneq (,$(wildcard $(XDG_RUNTIME_DIR)/bypass4netnsd.sock)) 44 | export BYPASS4NETNS := true 45 | export BYPASS4NETNS_IGNORE_SUBNETS := ["10.96.0.0/16", "10.244.0.0/16", "$(NODE_SUBNET)"] 46 | endif 47 | endif 48 | 49 | .PHONY: help 50 | help: 51 | @echo '# Bootstrap a cluster' 52 | @echo 'make up' 53 | @echo 'make kubeadm-init' 54 | @echo 'make install-flannel' 55 | @echo 56 | @echo '# Enable kubectl' 57 | @echo 'make kubeconfig' 58 | @echo 'export KUBECONFIG=$$(pwd)/kubeconfig' 59 | @echo 'kubectl get pods -A' 60 | @echo 61 | @echo '# Multi-host' 62 | @echo 'make join-command' 63 | @echo 'scp join-command another-host:~/usernetes' 64 | @echo 'ssh another-host make -C ~/usernetes up kubeadm-join' 65 | @echo 'make sync-external-ip' 66 | @echo 67 | @echo '# Debug' 68 | @echo 'make logs' 69 | @echo 'make shell' 70 | @echo 'make kubeadm-reset' 71 | @echo 'make down-v' 72 | @echo 'kubectl taint nodes --all node-role.kubernetes.io/control-plane-' 73 | 74 | .PHONY: check-preflight 75 | check-preflight: 76 | ./Makefile.d/check-preflight.sh 77 | 78 | .PHONY: render 79 | render: check-preflight 80 | $(COMPOSE) config 81 | 82 | .PHONY: up 83 | up: check-preflight 84 | $(COMPOSE) up --build -d 85 | 86 | .PHONY: down 87 | down: 88 | $(COMPOSE) down 89 | 90 | .PHONY: down-v 91 | down-v: 92 | $(COMPOSE) down -v 93 | 94 | .PHONY: shell 95 | shell: 96 | $(NODE_SHELL) bash 97 | 98 | .PHONY: logs 99 | logs: 100 | $(NODE_SHELL) journalctl --follow --since="1 day ago" 101 | 102 | .PHONY: kubeconfig 103 | kubeconfig: 104 | $(COMPOSE) exec -T $(NODE_SERVICE_NAME) sed -e "s/$(NODE_NAME)/127.0.0.1/g" /etc/kubernetes/admin.conf >kubeconfig 105 | @echo "# Run the following command by yourself:" 106 | @echo "export KUBECONFIG=$(shell pwd)/kubeconfig" 107 | ifeq ($(shell command -v kubectl 2> /dev/null),) 108 | @echo "# To install kubectl, run the following command too:" 109 | @echo "make kubectl" 110 | endif 111 | 112 | .PHONY: kubectl 113 | kubectl: 114 | $(COMPOSE) exec -T --workdir=/usr/bin $(NODE_SERVICE_NAME) tar c kubectl | tar xv 115 | @echo "# Run the following command by yourself:" 116 | @echo "export PATH=$(shell pwd):\$$PATH" 117 | @echo "source <(kubectl completion bash)" 118 | 119 | .PHONY: join-command 120 | join-command: 121 | echo "#!/bin/bash" >join-command 122 | echo "set -eux -o pipefail" >>join-command 123 | echo "echo \"$(HOST_IP) $(NODE_NAME)\" >/etc/hosts.u7s" >>join-command 124 | echo "cat /etc/hosts.u7s >>/etc/hosts" >>join-command 125 | $(NODE_SHELL) kubeadm token create --print-join-command | tr -d '\r' >>join-command 126 | @echo "# Copy the 'join-command' file to another host, and run the following commands:" 127 | @echo "# On the other host (the new worker):" 128 | @echo "# make kubeadm-join" 129 | @echo "# On this host (the control plane):" 130 | @echo "# make sync-external-ip" 131 | 132 | .PHONY: kubeadm-init 133 | kubeadm-init: 134 | $(NODE_SHELL) sh -euc "envsubst /tmp/kubeadm-config.yaml" 135 | $(NODE_SHELL) kubeadm init --config /tmp/kubeadm-config.yaml --skip-token-print 136 | $(MAKE) sync-external-ip 137 | @echo "# Run 'make join-command' to print the join command" 138 | 139 | .PHONY: sync-external-ip 140 | sync-external-ip: 141 | $(NODE_SHELL) /usernetes/Makefile.d/sync-external-ip.sh 142 | 143 | .PHONY: kubeadm-join 144 | kubeadm-join: 145 | $(NODE_SHELL) /bin/bash /usernetes/join-command 146 | @echo "# Run 'make sync-external-ip' on the control plane" 147 | 148 | .PHONY: kubeadm-reset 149 | kubeadm-reset: 150 | $(NODE_SHELL) kubeadm reset --force 151 | 152 | .PHONY: install-flannel 153 | install-flannel: 154 | $(NODE_SHELL) /usernetes/Makefile.d/install-flannel.sh 155 | -------------------------------------------------------------------------------- /Makefile.d/check-preflight.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -eu 3 | 4 | function INFO() { 5 | echo >&2 -e "\e[104m\e[97m[INFO]\e[49m\e[39m $@" 6 | } 7 | function WARNING() { 8 | echo >&2 -e "\e[101m\e[97m[WARNING]\e[49m\e[39m $@" 9 | } 10 | 11 | function ERROR() { 12 | echo >&2 -e "\e[101m\e[97m[ERROR]\e[49m\e[39m $@" 13 | } 14 | 15 | script_dir="$(dirname "$0")" 16 | detect_engine="${script_dir}"/detect-container-engine.sh 17 | : "${CONTAINER_ENGINE:=$("${detect_engine}" CONTAINER_ENGINE)}" 18 | : "${CONTAINER_ENGINE_TYPE:=$("${detect_engine}" CONTAINER_ENGINE_TYPE)}" 19 | : "${QUICK:=0}" 20 | : "${BUSYBOX_IMAGE:=docker.io/library/busybox:latest}" 21 | 22 | if [ -z "${CONTAINER_ENGINE}" ] || [ -z "${CONTAINER_ENGINE_TYPE}" ]; then 23 | ERROR "No container engine was detected" 24 | exit 1 25 | fi 26 | INFO "Detected container engine type: ${CONTAINER_ENGINE_TYPE}" 27 | 28 | # Check hard dependency commands 29 | for f in make jq "${CONTAINER_ENGINE}"; do 30 | if ! command -v "${f}" >/dev/null 2>&1; then 31 | ERROR "Command \"${f}\" is not installed" 32 | exit 1 33 | fi 34 | done 35 | 36 | # Check soft dependency commands 37 | for f in kubectl; do 38 | if ! command -v "${f}" >/dev/null 2>&1; then 39 | WARNING "Command \"${f}\" is not installed" 40 | fi 41 | done 42 | 43 | rootless= 44 | case "${CONTAINER_ENGINE_TYPE}" in 45 | "podman") 46 | if [ "$(${CONTAINER_ENGINE} info --format '{{.Host.Security.Rootless}}')" = "true" ]; then 47 | rootless=1 48 | fi 49 | ;; 50 | *) 51 | if ${CONTAINER_ENGINE} info --format '{{json .SecurityOptions}}' | grep -q "name=rootless"; then 52 | rootless=1 53 | fi 54 | ;; 55 | esac 56 | 57 | # Check if the container engine is running in Rootless mode 58 | if [ "${rootless}" = "1" ]; then 59 | # Check systemd lingering: https://rootlesscontaine.rs/getting-started/common/login/ 60 | if command -v loginctl >/dev/null 2>&1; then 61 | if [ "$(loginctl show-user --property Linger "${UID}")" != "Linger=yes" ]; then 62 | WARNING 'systemd lingering is not enabled. Run `sudo loginctl enable-linger $(whoami)` to enable it, otherwise Kubernetes will exit on logging out.' 63 | fi 64 | else 65 | WARNING "systemd lingering is not enabled?" 66 | fi 67 | 68 | # Check cgroup config 69 | if [[ ! -f /sys/fs/cgroup/cgroup.controllers ]]; then 70 | ERROR "Needs cgroup v2, see https://rootlesscontaine.rs/getting-started/common/cgroup2/" 71 | exit 1 72 | else 73 | f="/sys/fs/cgroup/user.slice/user-$(id -u).slice/user@$(id -u).service/cgroup.controllers" 74 | if [[ ! -f $f ]]; then 75 | ERROR "systemd not running? file not found: $f" 76 | exit 1 77 | fi 78 | if ! grep -q cpu "${f}"; then 79 | WARNING "cpu controller might not be enabled, you need to configure /etc/systemd/system/user@.service.d , see https://rootlesscontaine.rs/getting-started/common/cgroup2/" 80 | elif ! grep -q memory "${f}"; then 81 | WARNING "memory controller might not be enabled, you need to configure /etc/systemd/system/user@.service.d , see https://rootlesscontaine.rs/getting-started/common/cgroup2/" 82 | fi 83 | fi 84 | else 85 | WARNING "Container engine (${CONTAINER_ENGINE}) does not seem running in Rootless mode" 86 | fi 87 | 88 | # Check kernel modules 89 | for f in br_netfilter ip6_tables ip6table_nat ip_tables iptable_nat vxlan; do 90 | if ! grep -qw "^$f" /proc/modules; then 91 | WARNING "Kernel module \"${f}\" does not seem loaded? (negligible if built-in to the kernel)" 92 | fi 93 | done 94 | 95 | if [ "$QUICK" != "1" ]; then 96 | # Check net.ipv4.conf.default.rp_filter in the container engine's network namespace. (e.g., netns of dockerd) 97 | # The value can be 0 (disabled) or 2 (loose), must not be 1 (strict). 98 | if [ "$(${CONTAINER_ENGINE} run --rm --net=host "${BUSYBOX_IMAGE}" sysctl -n net.ipv4.conf.default.rp_filter)" == "1" ]; then 99 | ERROR "sysctl value \"net.ipv4.conf.default.rp_filter\" must be 0 (disabled) or 2 (loose) in the container engine's network namespace" 100 | exit 1 101 | fi 102 | fi 103 | -------------------------------------------------------------------------------- /Makefile.d/detect-container-engine.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -eu -o pipefail 3 | : "${CONTAINER_ENGINE:=}" 4 | : "${COMPOSE:=}" 5 | 6 | if [ -z "${CONTAINER_ENGINE}" ]; then 7 | if command -v dockerd-rootless.sh >/dev/null 2>&1; then 8 | CONTAINER_ENGINE=docker 9 | elif command -v containerd-rootless.sh >/dev/null 2>&1; then 10 | CONTAINER_ENGINE=nerdctl 11 | elif command -v podman >/dev/null 2>&1; then 12 | CONTAINER_ENGINE=podman 13 | else 14 | echo >&2 "$0: no container engine was detected" 15 | exit 1 16 | fi 17 | fi 18 | 19 | CONTAINER_ENGINE_TYPE=docker 20 | if [[ "${CONTAINER_ENGINE}" = *"podman"* ]]; then 21 | CONTAINER_ENGINE_TYPE=podman 22 | elif [[ "${CONTAINER_ENGINE}" = *"nerdctl"* ]]; then 23 | CONTAINER_ENGINE_TYPE=nerdctl 24 | fi 25 | 26 | if [ -z "${COMPOSE}" ]; then 27 | COMPOSE="${CONTAINER_ENGINE} compose" 28 | if [ "${CONTAINER_ENGINE_TYPE}" = "podman" ]; then 29 | COMPOSE=podman-compose 30 | fi 31 | fi 32 | 33 | case "$#" in 34 | 0) 35 | echo "CONTAINER_ENGINE=${CONTAINER_ENGINE}" 36 | echo "CONTAINER_ENGINE_TYPE=${CONTAINER_ENGINE_TYPE}" 37 | echo "COMPOSE=${COMPOSE}" 38 | ;; 39 | 1) 40 | case "$1" in 41 | "CONTAINER_ENGINE" | "CONTAINER_ENGINE_TYPE" | "COMPOSE") 42 | echo "${!1}" 43 | ;; 44 | *) 45 | echo >&2 "$0: unknown argument: $1" 46 | exit 1 47 | ;; 48 | esac 49 | ;; 50 | *) 51 | echo >&2 "$0: too many arguments" 52 | exit 1 53 | ;; 54 | esac 55 | -------------------------------------------------------------------------------- /Makefile.d/install-flannel.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -eu -o pipefail 3 | 4 | # See chart values, 0 indicates default for platform 5 | # https://github.com/flannel-io/flannel/blob/v0.26.1/chart/kube-flannel/values.yaml 6 | : "${PORT_FLANNEL:='0'}" 7 | 8 | if ! helm -n kube-flannel list -q | grep flannel; then 9 | kubectl create namespace kube-flannel 10 | kubectl label --overwrite namespace kube-flannel pod-security.kubernetes.io/enforce=privileged 11 | helm install flannel --namespace kube-flannel --set-json flannel.backendPort=${PORT_FLANNEL} /flannel 12 | fi 13 | -------------------------------------------------------------------------------- /Makefile.d/node-subnet.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -eu -o pipefail 3 | : "${HOSTNAME:=$(hostname)}" 4 | NODE_SUBNET_ID=$((16#$(echo "${HOSTNAME}" | sha256sum | head -c2))) 5 | NODE_SUBNET=10.100.${NODE_SUBNET_ID}.0/24 6 | echo "${NODE_SUBNET}" 7 | -------------------------------------------------------------------------------- /Makefile.d/sync-external-ip.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -eu -o pipefail 3 | 4 | for node in $(kubectl get nodes -o name); do 5 | # Set ExternalIP 6 | host_ip="$(kubectl get "${node}" -o jsonpath='{.metadata.labels.usernetes/host-ip}')" 7 | kubectl patch "${node}" --type=merge --subresource status --patch \ 8 | "\"status\": {\"addresses\": [{\"type\":\"ExternalIP\", \"address\": \"${host_ip}\"}]}" 9 | 10 | # Propagate ExternalIP to flannel 11 | # https://github.com/flannel-io/flannel/blob/v0.24.4/Documentation/kubernetes.md#annotations 12 | kubectl annotate "${node}" flannel.alpha.coreos.com/public-ip-overwrite=${host_ip} 13 | 14 | # Remove taints 15 | taints="$(kubectl get "${node}" -o jsonpath='{.spec.taints}')" 16 | if echo "${taints}" | grep -q node.cloudprovider.kubernetes.io/uninitialized; then 17 | kubectl taint nodes "${node}" node.cloudprovider.kubernetes.io/uninitialized- 18 | fi 19 | done 20 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Usernetes: Kubernetes without the root privileges (Generation 2) 2 | 3 | Usernetes (Gen2) deploys a Kubernetes cluster inside [Rootless Docker](https://rootlesscontaine.rs/getting-started/docker/), 4 | so as to mitigate potential container-breakout vulnerabilities. 5 | 6 | > [!NOTE] 7 | > 8 | > Usernetes (Gen2) has *significantly* diverged from the original Usernetes (Gen1), 9 | > which did not require Rootless Docker to be installed on hosts. 10 | > 11 | > See the [`gen1`](https://github.com/rootless-containers/usernetes/tree/gen1) branch for 12 | > the original Usernetes (Gen1). 13 | 14 | Usernetes (Gen2) is similar to [Rootless `kind`](https://kind.sigs.k8s.io/docs/user/rootless/) and [Rootless minikube](https://minikube.sigs.k8s.io/docs/drivers/docker/), 15 | but Usernetes (Gen 2) supports creating a cluster with multiple hosts. 16 | 17 | ## Components 18 | - Cluster configuration: kubeadm 19 | - CRI: containerd 20 | - OCI: runc 21 | - CNI: Flannel 22 | 23 | ## Requirements 24 | 25 | - One of the following host operating system: 26 | 27 | |Host operating system|Minimum version| 28 | |---------------------|---------------| 29 | |Ubuntu (recommended) |22.04 | 30 | |Rocky Linux |9 | 31 | |AlmaLinux |9 | 32 | |Fedora |(?) | 33 | 34 | - One of the following container engines: 35 | 36 | |Container Engine |Minimum version| 37 | |------------------------------------------------------------------------------------|---------------| 38 | |[Rootless Docker](https://rootlesscontaine.rs/getting-started/docker/) (recommended)|v20.10 | 39 | |[Rootless Podman](https://rootlesscontaine.rs/getting-started/podman/) |v4.x | 40 | |[Rootless nerdctl](https://rootlesscontaine.rs/getting-started/containerd/) |v1.6 | 41 | 42 | ```bash 43 | curl -o install.sh -fsSL https://get.docker.com 44 | sudo sh install.sh 45 | dockerd-rootless-setuptool.sh install 46 | ``` 47 | 48 | - systemd lingering: 49 | ```bash 50 | sudo loginctl enable-linger $(whoami) 51 | ``` 52 | 53 | - cgroup v2 delegation: 54 | ```bash 55 | sudo mkdir -p /etc/systemd/system/user@.service.d 56 | 57 | sudo tee /etc/systemd/system/user@.service.d/delegate.conf </dev/null 58 | [Service] 59 | Delegate=cpu cpuset io memory pids 60 | EOF 61 | 62 | sudo systemctl daemon-reload 63 | ``` 64 | 65 | - Kernel modules: 66 | ``` 67 | sudo tee /etc/modules-load.d/usernetes.conf </dev/null 68 | br_netfilter 69 | vxlan 70 | EOF 71 | 72 | sudo systemctl restart systemd-modules-load.service 73 | ``` 74 | 75 | - sysctl: 76 | ``` 77 | sudo tee /etc/sysctl.d/99-usernetes.conf </dev/null 78 | net.ipv4.conf.default.rp_filter = 2 79 | EOF 80 | 81 | sudo sysctl --system 82 | ``` 83 | 84 | - slirp4netns, not Pasta: 85 | ``` 86 | # Podman v5 (or later) users have to change the network mode from pasta to slirp4netns. 87 | # This step is not needed for Docker, nerdctl, and Podman v4. 88 | 89 | mkdir -p "$HOME/.config/containers/containers.conf.d" 90 | cat <"$HOME/.config/containers/containers.conf.d/slirp4netns.conf" 91 | [network] 92 | default_rootless_network_cmd="slirp4netns" 93 | EOF 94 | ``` 95 | 102 | 103 | Use scripts in [`./init-host`](./init-host) for automating these steps. 104 | 105 | ## Usage 106 | See `make help`. 107 | 108 | ```bash 109 | # Bootstrap a cluster 110 | make up 111 | make kubeadm-init 112 | make install-flannel 113 | 114 | # Enable kubectl 115 | make kubeconfig 116 | export KUBECONFIG=$(pwd)/kubeconfig 117 | kubectl get pods -A 118 | 119 | # Multi-host 120 | make join-command 121 | scp join-command another-host:~/usernetes 122 | ssh another-host make -C ~/usernetes up kubeadm-join 123 | make sync-external-ip 124 | 125 | # Debug 126 | make logs 127 | make shell 128 | make kubeadm-reset 129 | make down-v 130 | kubectl taint nodes --all node-role.kubernetes.io/control-plane- 131 | ``` 132 | 133 | The container engine defaults to Docker. 134 | To change the container engine, set `export CONTAINER_ENGINE=podman` or `export CONTAINER_ENGINE=nerdctl`. 135 | 136 | ### Customization 137 | 138 | The following environment variables are recognized: 139 | 140 | Name | Type | Default value 141 | ----------------------|---------|---------------------------------------------------------------- 142 | `CONTAINER_ENGINE` | String | automatically resolved to "docker", "podman", or "nerdctl" 143 | `HOST_IP` | String | automatically resolved to the host's IP address 144 | `NODE_NAME` | String | "u7s-" + the host's hostname 145 | `NODE_SUBNET` | String | "10.100.%d.0/24" (%d is computed from the hash of the hostname) 146 | `PORT_ETCD` | Integer | 2379 147 | `PORT_KUBELET` | Integer | 10250 148 | `PORT_FLANNEL` | Integer | 8472 149 | `PORT_KUBE_APISERVER` | Integer | 6443 150 | 151 | ## Limitations 152 | - Node ports cannot be exposed automatically. Edit [`docker-compose.yaml`](./docker-compose.yaml) for exposing additional node ports. 153 | - Most of host files are not visible with `hostPath` mounts. Edit [`docker-compose.yaml`](./docker-compose.yaml) for mounting additional files. 154 | - Some [volume drivers](https://kubernetes.io/docs/concepts/storage/volumes/) such as `nfs` do not work. 155 | 156 | ## Advanced topics 157 | ### Network 158 | When `CONTAINER_ENGINE` is set to `nerdctl`, [bypass4netns](https://github.com/rootless-containers/bypass4netns) can be enabled for accelerating `connect(2)` syscalls. 159 | The acceleration currently does not apply to VXLAN packets. 160 | 161 | ```bash 162 | containerd-rootless-setuptool.sh install-bypass4netnsd 163 | export CONTAINER_ENGINE=nerdctl 164 | make up 165 | ``` 166 | 167 | > [!NOTE] 168 | > 169 | > The support for bypass4netns is still experimental 170 | 171 | ### Multi-tenancy 172 | 173 | Multiple users on the hosts may create their own instances of Usernetes, but the port numbers have to be changed to avoid conflicts. 174 | 175 | ```bash 176 | # Default: 2379 177 | export PORT_ETCD=12379 178 | # Default: 10250 179 | export PORT_KUBELET=20250 180 | # Default: 8472 181 | export PORT_FLANNEL=18472 182 | # Default: 6443 183 | export PORT_KUBE_APISERVER=16443 184 | 185 | make up 186 | ``` 187 | 188 | ![docs/images/multi-tenancy.png](./docs/images/multi-tenancy.png) 189 | 190 | ### Rootful mode 191 | Although Usernetes (Gen2) is designed to be used with Rootless Docker, it should work with the regular "rootful" Docker too. 192 | This might be useful for some people who are looking for "multi-host" version of [`kind`](https://kind.sigs.k8s.io/) and [minikube](https://minikube.sigs.k8s.io/). 193 | -------------------------------------------------------------------------------- /docker-compose.yaml: -------------------------------------------------------------------------------- 1 | # Use `make up`, not `docker compose up`, 2 | # as this YAML requires ${...} variables to be set. 3 | --- 4 | services: 5 | node: 6 | build: . 7 | hostname: ${NODE_NAME} 8 | privileged: true 9 | restart: always 10 | networks: 11 | default: 12 | ipv4_address: ${NODE_IP} 13 | ports: 14 | # : 15 | # etcd (default: 2379) 16 | - ${PORT_ETCD}:${PORT_ETCD} 17 | # kube-apiserver (default: 6443) 18 | - ${PORT_KUBE_APISERVER}:${PORT_KUBE_APISERVER} 19 | # kubelet (default: 10250) 20 | - ${PORT_KUBELET}:${PORT_KUBELET} 21 | # flannel (default: 8472) 22 | - ${PORT_FLANNEL}:${PORT_FLANNEL}/udp 23 | volumes: 24 | - .:/usernetes:ro 25 | - /boot:/boot:ro 26 | - /lib/modules:/lib/modules:ro 27 | - node-var:/var 28 | - node-opt:/opt 29 | - node-etc:/etc 30 | - type: tmpfs 31 | target: /run 32 | - type: tmpfs 33 | target: /tmp 34 | working_dir: /usernetes 35 | environment: 36 | KUBECONFIG: /etc/kubernetes/admin.conf 37 | HOST_IP: ${HOST_IP} 38 | sysctls: 39 | - net.ipv4.ip_forward=1 40 | # In addition, `net.ipv4.conf.default.rp_filter` 41 | # has to be set to 0 (disabled) or 2 (loose) 42 | # in the daemon's network namespace. 43 | annotations: 44 | # Accelerate network for nerdctl >= 2.0.0-beta.4 with bypass4netns >= 0.4.1 45 | "nerdctl/bypass4netns": "${BYPASS4NETNS:-false}" 46 | "nerdctl/bypass4netns-ignore-bind": "true" 47 | "nerdctl/bypass4netns-ignore-subnets": "${BYPASS4NETNS_IGNORE_SUBNETS:-}" 48 | networks: 49 | default: 50 | ipam: 51 | config: 52 | # Each of the nodes has to have a different IP. 53 | # The node IP here is not accessible from other nodes. 54 | - subnet: ${NODE_SUBNET} 55 | volumes: 56 | node-var: {} 57 | node-opt: {} 58 | node-etc: {} 59 | -------------------------------------------------------------------------------- /docs/images/multi-tenancy.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rootless-containers/usernetes/e337e8deedb7e00bf76501e3f3f7d9a6ec0d4bdd/docs/images/multi-tenancy.png -------------------------------------------------------------------------------- /hack/README.md: -------------------------------------------------------------------------------- 1 | This directory contains testing utilities. 2 | -------------------------------------------------------------------------------- /hack/create-cluster-lima.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -eux -o pipefail 3 | 4 | : "${LIMACTL:=limactl --tty=false}" 5 | : "${LIMACTL_CREATE_ARGS:=}" 6 | : "${LIMA_TEMPLATE:=template://default}" 7 | : "${CONTAINER_ENGINE:=docker}" 8 | : "${LOCKDOWN_SUDO:=1}" 9 | : "${PORT_KUBE_APISERVER:=6443}" 10 | : "${PORT_ETCD:=2379}" 11 | : "${PORT_FLANNEL:=8472}" 12 | : "${PORT_KUBELET:=10250}" 13 | 14 | guest_home="/home/${USER}.linux" 15 | 16 | if [ "$(id -u)" -le 1000 ]; then 17 | # In --plain mode, UID has to be >= 1000 to populate subuids 18 | # https://github.com/lima-vm/lima/issues/3001 19 | LIMACTL_CREATE_ARGS="${LIMACTL_CREATE_ARGS} --set=.user.uid=1001" 20 | fi 21 | 22 | # Create Rootless Docker hosts 23 | for host in host0 host1; do 24 | # Set --plain to minimize Limaism 25 | ${LIMACTL} start --plain --network lima:user-v2 --name="${host}" ${LIMACTL_CREATE_ARGS} "${LIMA_TEMPLATE}" 26 | ${LIMACTL} copy -r "$(pwd)" "${host}:${guest_home}/usernetes" 27 | ${LIMACTL} shell "${host}" sudo CONTAINER_ENGINE="${CONTAINER_ENGINE}" "${guest_home}/usernetes/init-host/init-host.root.sh" 28 | # Terminate the current session so that the cgroup delegation takes an effect. This command exits with status 255 as SSH terminates. 29 | ${LIMACTL} shell "${host}" sudo loginctl terminate-user "${USER}" || true 30 | ${LIMACTL} shell "${host}" sudo loginctl enable-linger "${USER}" 31 | if [ "${LOCKDOWN_SUDO}" = "1" ]; then 32 | # Lockdown sudo to ensure rootless-ness 33 | ${LIMACTL} shell "${host}" sudo sh -euxc 'rm -rf /etc/sudoers.d/*-cloud-init-users' 34 | fi 35 | ${LIMACTL} shell "${host}" CONTAINER_ENGINE="${CONTAINER_ENGINE}" "${guest_home}/usernetes/init-host/init-host.rootless.sh" 36 | done 37 | 38 | SERVICE_PORTS="PORT_KUBE_APISERVER=${PORT_KUBE_APISERVER} PORT_ETCD=${PORT_ETCD} PORT_FLANNEL=${PORT_FLANNEL} PORT_KUBELET=${PORT_KUBELET}" 39 | 40 | # Launch a Kubernetes node inside a Rootless Docker host 41 | for host in host0 host1; do 42 | ${LIMACTL} shell "${host}" ${SERVICE_PORTS} CONTAINER_ENGINE="${CONTAINER_ENGINE}" make -C "${guest_home}/usernetes" up 43 | done 44 | 45 | # Bootstrap a cluster with host0 46 | ${LIMACTL} shell host0 ${SERVICE_PORTS} CONTAINER_ENGINE="${CONTAINER_ENGINE}" make -C "${guest_home}/usernetes" kubeadm-init install-flannel kubeconfig join-command 47 | 48 | # Let host1 join the cluster 49 | ${LIMACTL} copy host0:~/usernetes/join-command host1:~/usernetes/join-command 50 | ${LIMACTL} shell host1 ${SERVICE_PORTS} CONTAINER_ENGINE="${CONTAINER_ENGINE}" make -C "${guest_home}/usernetes" kubeadm-join 51 | ${LIMACTL} shell host0 ${SERVICE_PORTS} CONTAINER_ENGINE="${CONTAINER_ENGINE}" make -C "${guest_home}/usernetes" sync-external-ip 52 | 53 | # Enable kubectl 54 | ssh -q -f -N -L ${PORT_KUBE_APISERVER}:127.0.0.1:${PORT_KUBE_APISERVER} -F ~/.lima/host0/ssh.config lima-host0 55 | ${LIMACTL} copy host0:${guest_home}/usernetes/kubeconfig ./kubeconfig 56 | KUBECONFIG="$(pwd)/kubeconfig" 57 | export KUBECONFIG 58 | kubectl get nodes -o wide 59 | kubectl get pods -A 60 | -------------------------------------------------------------------------------- /hack/test-smoke.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -eu -o pipefail 3 | 4 | INFO() { 5 | echo -e "\e[104m\e[97m[INFO]\e[49m\e[39m $@" 6 | } 7 | 8 | INFO "Waiting for nodes to be ready" 9 | for node in $(kubectl get node -o name); do 10 | kubectl wait --timeout=5m --for=condition=ready "${node}" 11 | done 12 | 13 | function smoketest_dns() { 14 | INFO "Creating StatefulSet \"dnstest\" and headless Service \"dnstest\"" 15 | kubectl apply -f - <&2 "Unsupported architecture" 21 | exit 1 22 | ;; 23 | esac 24 | 25 | mkdir -p /root/nerdctl.tmp 26 | ( 27 | cd /root/nerdctl.tmp 28 | curl -fSLO https://github.com/containerd/nerdctl/releases/download/v${VERSION}/nerdctl-full-${VERSION}-linux-${arch}.tar.gz 29 | curl -fSLO https://github.com/containerd/nerdctl/releases/download/v${VERSION}/SHA256SUMS 30 | [ "$(sha256sum SHA256SUMS | awk '{print $1}')" = "${SHASHA}" ] 31 | sha256sum --check --ignore-missing SHA256SUMS 32 | tar Cxzvvf /usr/local nerdctl-full-${VERSION}-linux-${arch}.tar.gz 33 | ) 34 | rm -rf /root/nerdctl.tmp 35 | 36 | if [ -e /etc/apparmor.d/rootlesskit ]; then 37 | # https://rootlesscontaine.rs/getting-started/common/apparmor/ 38 | sed -e s@/usr/bin/rootlesskit@/usr/local/bin/rootlesskit@g /etc/apparmor.d/rootlesskit >/etc/apparmor.d/usr.local.bin.rootlesskit 39 | systemctl restart apparmor 40 | fi 41 | -------------------------------------------------------------------------------- /init-host/init-host.root.d/install-podman.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # This script installs Podman. 3 | set -eux -o pipefail 4 | if [ "$(id -u)" != "0" ]; then 5 | echo "Must run as the root" 6 | exit 1 7 | fi 8 | 9 | # Needs slirp4netns, not pasta: 10 | # 11 | # > 2024-12-02T17:15:40.070018488Z stderr F E1202 17:15:40.068621 1 main.go:228] Failed to create SubnetManager: 12 | # > error retrieving pod spec for 'kube-flannel/kube-flannel-ds-ms2d9': Get "https://10.96.0.1:443/api/v1/namespaces/kube-flannel/pods/kube-flannel-ds-ms2d9": 13 | # > dial tcp 10.96.0.1:443: i/o timeout 14 | 15 | if command -v dnf >/dev/null 2>&1; then 16 | dnf install -y --best podman podman-compose slirp4netns 17 | else 18 | apt-get update -qq 19 | apt-get -qq -y install podman podman-compose slirp4netns 20 | fi 21 | -------------------------------------------------------------------------------- /init-host/init-host.root.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -eux -o pipefail 3 | 4 | if [ "$(id -u)" != "0" ]; then 5 | echo "Must run as the root" 6 | exit 1 7 | fi 8 | 9 | : "${CONTAINER_ENGINE:=docker}" 10 | script_dir="$(dirname "$0")" 11 | 12 | if [ ! -e /etc/systemd/system/user@.service.d/delegate.conf ]; then 13 | mkdir -p /etc/systemd/system/user@.service.d 14 | cat </etc/systemd/system/user@.service.d/delegate.conf 15 | [Service] 16 | Delegate=cpu cpuset io memory pids 17 | EOF 18 | systemctl daemon-reload 19 | fi 20 | 21 | cat >/etc/modules-load.d/usernetes.conf </etc/sysctl.d/99-usernetes.conf </dev/null 2>&1; then 56 | dnf install -y --best git shadow-utils make jq 57 | # podman-compose requires EPEL 58 | if grep -q centos /etc/os-release; then 59 | # Works with Rocky and Alma too 60 | dnf install -y --best epel-release 61 | fi 62 | else 63 | apt-get update 64 | apt-get install -y git uidmap make jq 65 | fi 66 | 67 | case "${CONTAINER_ENGINE}" in 68 | "docker") 69 | if ! command -v dockerd-rootless-setuptool.sh >/dev/null 2>&1; then 70 | if grep -q centos /etc/os-release; then 71 | # Works with Rocky and Alma too 72 | dnf config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo 73 | dnf -y install docker-ce 74 | else 75 | curl https://get.docker.com | sh 76 | fi 77 | fi 78 | systemctl disable --now docker 79 | ;; 80 | "podman") 81 | if ! command -v podman-compose >/dev/null 2>&1; then 82 | "${script_dir}"/init-host.root.d/install-podman.sh 83 | fi 84 | ;; 85 | "nerdctl") 86 | if ! command -v nerdctl >/dev/null 2>&1; then 87 | "${script_dir}"/init-host.root.d/install-nerdctl.sh 88 | fi 89 | ;; 90 | *) 91 | echo >&2 "Unsupported container engine: ${CONTAINER_ENGINE}" 92 | exit 1 93 | ;; 94 | esac 95 | -------------------------------------------------------------------------------- /init-host/init-host.rootless.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -eux -o pipefail 3 | 4 | if [ "$(id -u)" == "0" ]; then 5 | echo "Must not run as the root" 6 | exit 1 7 | fi 8 | 9 | : "${CONTAINER_ENGINE:=docker}" 10 | : "${XDG_CONFIG_HOME:=${HOME}/.config}" 11 | case "${CONTAINER_ENGINE}" in 12 | "docker") 13 | dockerd-rootless-setuptool.sh install || (journalctl --user --since "10 min ago"; exit 1) 14 | ;; 15 | "nerdctl") 16 | containerd-rootless-setuptool.sh install 17 | containerd-rootless-setuptool.sh install-buildkit-containerd 18 | containerd-rootless-setuptool.sh install-bypass4netnsd 19 | ;; 20 | "podman") 21 | # pasta does not seem to work well 22 | # > 2024-12-02T17:15:40.070018488Z stderr F E1202 17:15:40.068621 1 main.go:228] Failed to create SubnetManager: 23 | # > error retrieving pod spec for 'kube-flannel/kube-flannel-ds-ms2d9': Get "https://10.96.0.1:443/api/v1/namespaces/kube-flannel/pods/kube-flannel-ds-ms2d9": 24 | # > dial tcp 10.96.0.1:443: i/o timeout 25 | mkdir -p "${XDG_CONFIG_HOME}/containers/containers.conf.d" 26 | cat <"${XDG_CONFIG_HOME}/containers/containers.conf.d/slirp4netns.conf" 27 | [network] 28 | default_rootless_network_cmd="slirp4netns" 29 | EOF 30 | systemctl --user enable --now podman-restart 31 | ;; 32 | *) 33 | # NOP 34 | ;; 35 | esac 36 | 37 | ${CONTAINER_ENGINE} info 38 | -------------------------------------------------------------------------------- /kubeadm-config.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kubeadm.k8s.io/v1beta4 2 | kind: InitConfiguration 3 | localAPIEndpoint: 4 | bindPort: ${PORT_KUBE_APISERVER} 5 | --- 6 | apiVersion: kubeadm.k8s.io/v1beta4 7 | kind: ClusterConfiguration 8 | controlPlaneEndpoint: "${NODE_NAME}:${PORT_KUBE_APISERVER}" 9 | apiServer: 10 | certSANs: 11 | - localhost 12 | - 127.0.0.1 13 | - "${NODE_NAME}" 14 | - "${HOST_IP}" 15 | extraArgs: 16 | - name: etcd-servers 17 | value: https://127.0.0.1:${PORT_ETCD} 18 | - name: advertise-address 19 | value: ${HOST_IP} 20 | - name: secure-port 21 | value: "${PORT_KUBE_APISERVER}" 22 | # Default: "Hostname,InternalDNS,InternalIP,ExternalDNS,ExternalIP" 23 | - name: kubelet-preferred-address-types 24 | value: ExternalIP 25 | controllerManager: 26 | extraArgs: 27 | - name: cloud-provider 28 | value: external 29 | networking: 30 | serviceSubnet: "10.96.0.0/16" 31 | podSubnet: "10.244.0.0/16" 32 | etcd: 33 | local: 34 | extraArgs: 35 | - name: listen-client-urls 36 | value: "https://127.0.0.1:${PORT_ETCD},https://${NODE_IP}:${PORT_ETCD}" 37 | - name: advertise-client-urls 38 | value: https://${NODE_IP}:${PORT_ETCD} 39 | --- 40 | kind: KubeletConfiguration 41 | apiVersion: kubelet.config.k8s.io/v1beta1 42 | failSwapOn: false 43 | port: ${PORT_KUBELET} 44 | featureGates: 45 | KubeletInUserNamespace: true 46 | --- 47 | apiVersion: kubeproxy.config.k8s.io/v1alpha1 48 | kind: KubeProxyConfiguration 49 | mode: "iptables" 50 | conntrack: 51 | # Skip setting sysctl value "net.netfilter.nf_conntrack_max" 52 | maxPerCore: 0 53 | # Skip setting "net.netfilter.nf_conntrack_tcp_timeout_established" 54 | tcpEstablishedTimeout: 0s 55 | # Skip setting "net.netfilter.nf_conntrack_tcp_timeout_close" 56 | tcpCloseWaitTimeout: 0s 57 | --------------------------------------------------------------------------------