├── .github └── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature_request.md ├── .gitignore ├── BUILD.md ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── Makefile ├── README.md ├── mok-image ├── Dockerfile ├── files │ ├── LICENSES │ │ └── README │ ├── etc │ │ ├── crictl.yaml │ │ ├── crio │ │ │ └── crio.conf.d │ │ │ │ └── 11-crio.conf │ │ ├── default │ │ │ └── kubelet │ │ ├── sysctl.d │ │ │ ├── 10-network-magic.conf │ │ │ └── 10-network-security.conf │ │ └── systemd │ │ │ └── system │ │ │ ├── kubelet.service │ │ │ ├── kubelet.service.d │ │ │ ├── 10-kubeadm.conf │ │ │ └── 11-kind.conf │ │ │ ├── kubelet.slice │ │ │ ├── node-address-update.service │ │ │ └── undo-mount-hacks.service │ ├── kind │ │ ├── README.txt │ │ └── bin │ │ │ ├── create-kubelet-cgroup-v2.sh │ │ │ ├── mount-product-files.sh │ │ │ └── undo-mount-hacks.sh │ ├── kube-flannel.yml │ ├── mok │ │ └── README │ └── usr │ │ └── local │ │ └── bin │ │ ├── clean-install │ │ ├── entrypoint │ │ └── update_kube-proxy.sh └── scripts │ ├── target-cc │ └── third_party │ └── gimme │ ├── LICENSE │ ├── README.md │ └── gimme ├── package └── mok ├── src ├── arch.sh ├── buildimage.sh ├── containerutils.sh ├── createcluster.sh ├── deletecluster.sh ├── embed-dockerfile.sh ├── error.sh ├── exec.sh ├── getcluster.sh ├── globals.sh ├── lib │ ├── JSONPath.sh │ └── parser.sh ├── machine.sh ├── machinecreate.sh ├── machinedestroy.sh ├── machinelist.sh ├── machinesetup.sh ├── machinestart.sh ├── machinestop.sh ├── macos.sh ├── main.sh ├── util.sh └── versions.sh └── tests ├── README.md ├── build-tests.sh ├── e2e-logs ├── 2020-06-05-1591359733_test-1_BAD_1.0000.log ├── 2020-06-05-1591359996_test-1_BAD_1.0000.log ├── 2020-06-05-1591360319_test-1_GOOD_1.0000.log ├── 2020-06-05-1591365638_test-1_GOOD_1.0000.log ├── 2020-06-05-1591366321_test-1_GOOD_1.0000.log ├── 2020-06-05-1591366441_test-2_GOOD_1.0000.log ├── 2020-06-05-1591366586_test-1_GOOD_1.0000.log ├── 2020-06-05-1591367000_test-2_GOOD_1.0000.log ├── 2020-06-05-1591372712_test-1_GOOD_1.0000.log ├── 2020-06-05-1591372979_test-2_GOOD_1.0000.log ├── 2020-06-05-1591377275_test-1_GOOD_1.0000.log ├── 2020-06-05-1591377693_test-2_GOOD_1.0000.log ├── 2020-06-05-1591396993_test-1_BAD_0.9927.log ├── 2020-06-05-1591397543_test-1_GOOD_1.0000.log ├── 2020-06-05-1591397979_test-2_GOOD_1.0000.log ├── 2020-06-06-1591400029_test-1_BAD_0.9913.log ├── 2020-06-06-1591401501_test-1_BAD_1.0000.log ├── 2020-06-06-1591401794_test-2_BAD_1.0000.log ├── 2020-06-06-1591403459_test-2_BAD_1.0000.log ├── 2020-06-06-1591403901_test-1_BAD_1.0000.log ├── 2020-06-06-1591404157_test-2_BAD_1.0000.log ├── 2020-06-06-1591404451_test-1_GOOD_1.0000.log ├── 2020-06-06-1591404854_test-2_GOOD_1.0000.log ├── 2020-06-07-1591548681_test-1_GOOD_1.0000.log ├── 2020-06-07-1591549005_test-2_GOOD_1.0000.log ├── 2020-06-07-1591552672_test-1_GOOD_1.0000.log ├── 2020-06-07-1591553096_test-2_GOOD_1.0000.log ├── 2020-06-08-1591651739_test-1_GOOD_1.0000.log ├── 2020-06-08-1591652144_test-2_GOOD_.log ├── 2020-06-25-1593124118_test-1_GOOD_1.0000.log ├── 2020-06-26-1593173882_test-1_BAD_1.0000.log ├── 2020-06-26-1593173916_test-1_BAD_1.0000.log ├── 2020-06-26-1593173991_test-1_BAD_1.0000.log ├── 2020-06-26-1593175450_test-1_GOOD_1.0000.log ├── 2020-06-26-1593175717_test-2_GOOD_1.0000.log ├── 2020-06-27-1593276521_test-1_BAD_1.0000.log ├── 2020-06-27-1593276576_test-1_BAD_1.0000.log ├── 2020-06-27-1593277981_test-1_GOOD_.log ├── 2020-06-27-1593278281_test-2_GOOD_1.0000.log ├── 2020-06-27-1593281825_test-1_GOOD_1.0000.log ├── 2020-06-27-1593282103_test-2_GOOD_1.0000.log ├── 2020-09-16-1600251508_test-1_GOOD_1.0000.log ├── 2020-09-16-1600251786_test-2_BAD_1.0000.log ├── 2020-09-16-1600270058_test-1_BAD_1.0000.log ├── 2020-09-16-1600270127_test-1_BAD_1.0000.log ├── 2020-09-16-1600280454_test-1_GOOD_1.0000.log ├── 2020-09-16-1600280636_test-2_GOOD_1.0000.log ├── 2020-09-16-1600282808_test-1_BAD_1.0000.log ├── 2020-09-16-1600283125_test-1_GOOD_1.0000.log ├── 2020-09-16-1600283316_test-2_GOOD_1.0000.log ├── 2020-09-16-1600284989_test-1_GOOD_1.0000.log ├── 2020-09-16-1600285349_test-2_GOOD_1.0000.log └── training │ ├── 2020-06-05-1591359418_test-1_BAD_1.0000.log │ ├── 2020-06-05-1591359857_test-1_BAD_1.0000.log │ ├── 2020-06-05-1591360319_test-1_GOOD_1.0000.log │ ├── 2020-06-06-1591400374_test-2_BAD_1.0000.log │ ├── 2020-06-06-1591403178_test-1_BAD_1.0000.log │ └── README.md ├── e2e-tests.sh ├── e2e ├── crm114 │ ├── BAD │ ├── BUILD_USAGE │ ├── CREATE_USAGE │ ├── DELETE_USAGE │ ├── EXEC_USAGE │ ├── GET_USAGE │ ├── GOOD │ ├── UNKNOWN_USAGE │ ├── classify.crm │ ├── learn.crm │ ├── unlearn.crm │ └── usage_match.crm ├── e2e-test-1.scr └── e2e-test-2.scr ├── shunit2 ├── testfiles └── docker_inspect_container.txt ├── unit-tests.sh └── usage-checks.sh /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | 1. Go to '...' 16 | 2. Click on '....' 17 | 3. Scroll down to '....' 18 | 4. See error 19 | 20 | **Expected behavior** 21 | A clear and concise description of what you expected to happen. 22 | 23 | **Screenshots** 24 | If applicable, add screenshots to help explain your problem. 25 | 26 | **Desktop (please complete the following information):** 27 | - OS: [e.g. iOS] 28 | - Browser [e.g. chrome, safari] 29 | - Version [e.g. 22] 30 | 31 | **Smartphone (please complete the following information):** 32 | - Device: [e.g. iPhone6] 33 | - OS: [e.g. iOS8.1] 34 | - Browser [e.g. stock browser, safari] 35 | - Version [e.g. 22] 36 | 37 | **Additional context** 38 | Add any other context about the problem here. 39 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | mok.deploy 2 | mok.deploy.noconst 3 | buildimage.deploy 4 | latest.hardcopy 5 | tags 6 | *.swp 7 | -------------------------------------------------------------------------------- /BUILD.md: -------------------------------------------------------------------------------- 1 | Update versions in: 2 | - `Makefile` 3 | - `VERSION = 0.8.21` 4 | - `README.md` 5 | - `Current kubernetes version: 1.x.x` 6 | - `src/globals.sh` 7 | - `declare -rg MOKVERSION="0.8.x"` 8 | + - `declare -rg K8SVERSION="1.x.x"` 9 | 10 | Check versions in: 11 | - https://github.com/cri-o/cri-o/tags 12 | - https://github.com/opencontainers/runc/tags 13 | - https://github.com/kubernetes-sigs/cri-tools/tags 14 | 15 | and update `mok-image/Dockerfile` 16 | 17 | For kubernetes MINOR version upgrades, update: 18 | - `src/createcluster.sh` 19 | - `_CC_set_up_master_node()` 20 | - `"1.31."* | "1.32."*)` 21 | - `_CC_set_up_worker_node()` 22 | - `"1.31."* | "1.32."*)` 23 | 24 | Remove all images: 25 | ``` 26 | sudo podman images | tail -n +2 | awk '{ print $3 }' | xargs sudo podman rmi --force 27 | ``` 28 | Update mok 29 | ``` 30 | touch mok-image/Dockerfile 31 | make package; sudo cp package/mok /usr/local/bin 32 | ``` 33 | Build prebuilt image for upload and check version as it's building: 34 | ``` 35 | mok build image --tailf 36 | ``` 37 | Tag and upload for Arm (built on MacOS M4): 38 | ``` 39 | VERSION=1.32.3 40 | podman tag localhost/local/mok-image_arm64:${VERSION} docker.io/myownkind/mok-image_arm64:${VERSION} 41 | podman login docker.io 42 | podman push docker.io/myownkind/mok-image_arm64:${VERSION} 43 | ``` 44 | Tag and upload for Intel (built on Fedora): 45 | ``` 46 | VERSION=1.32.3 47 | sudo podman tag localhost/local/mok-image_x86_64:${VERSION} docker.io/myownkind/mok-image_x86_64:${VERSION} 48 | sudo podman login docker.io 49 | sudo podman push docker.io/myownkind/mok-image_x86_64:${VERSION} 50 | ``` 51 | 52 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as 6 | contributors and maintainers pledge to making participation in our project and 7 | our community a harassment-free experience for everyone, regardless of age, body 8 | size, disability, ethnicity, sex characteristics, gender identity and expression, 9 | level of experience, education, socio-economic status, nationality, personal 10 | appearance, race, religion, or sexual identity and orientation. 11 | 12 | ## Our Standards 13 | 14 | Examples of behavior that contributes to creating a positive environment 15 | include: 16 | 17 | * Using welcoming and inclusive language 18 | * Being respectful of differing viewpoints and experiences 19 | * Gracefully accepting constructive criticism 20 | * Focusing on what is best for the community 21 | * Showing empathy towards other community members 22 | 23 | Examples of unacceptable behavior by participants include: 24 | 25 | * The use of sexualized language or imagery and unwelcome sexual attention or 26 | advances 27 | * Trolling, insulting/derogatory comments, and personal or political attacks 28 | * Public or private harassment 29 | * Publishing others' private information, such as a physical or electronic 30 | address, without explicit permission 31 | * Other conduct which could reasonably be considered inappropriate in a 32 | professional setting 33 | 34 | ## Our Responsibilities 35 | 36 | Project maintainers are responsible for clarifying the standards of acceptable 37 | behavior and are expected to take appropriate and fair corrective action in 38 | response to any instances of unacceptable behavior. 39 | 40 | Project maintainers have the right and responsibility to remove, edit, or 41 | reject comments, commits, code, wiki edits, issues, and other contributions 42 | that are not aligned to this Code of Conduct, or to ban temporarily or 43 | permanently any contributor for other behaviors that they deem inappropriate, 44 | threatening, offensive, or harmful. 45 | 46 | ## Scope 47 | 48 | This Code of Conduct applies both within project spaces and in public spaces 49 | when an individual is representing the project or its community. Examples of 50 | representing a project or community include using an official project e-mail 51 | address, posting via an official social media account, or acting as an appointed 52 | representative at an online or offline event. Representation of a project may be 53 | further defined and clarified by project maintainers. 54 | 55 | ## Enforcement 56 | 57 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 58 | reported by contacting the project team at mark.clarkson@smorg.co.uk. All 59 | complaints will be reviewed and investigated and will result in a response that 60 | is deemed necessary and appropriate to the circumstances. The project team is 61 | obligated to maintain confidentiality with regard to the reporter of an incident. 62 | Further details of specific enforcement policies may be posted separately. 63 | 64 | Project maintainers who do not follow or enforce the Code of Conduct in good 65 | faith may face temporary or permanent repercussions as determined by other 66 | members of the project's leadership. 67 | 68 | ## Attribution 69 | 70 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 71 | available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html 72 | 73 | [homepage]: https://www.contributor-covenant.org 74 | 75 | For answers to common questions about this code of conduct, see 76 | https://www.contributor-covenant.org/faq 77 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | ## `make test` must pass 4 | 5 | When `make test` is run it will: 6 | 7 | 1. Run end-to-end and usage tests. 8 | 9 | 2. Run `shellcheck` on mok/mok. 10 | 11 | This must pass when run with no options to be approved. 12 | 13 | 3. Run `shfmt`. 14 | 15 | This must pass when run with `-s -i 2` to be approved 16 | `-s` = simplify code 17 | `-i 2` = indention using two spaces 18 | 19 | Get `shfmt` from https://github.com/mvdan/sh. 20 | 21 | I use the vim-shfmt plugin from https://github.com/z0mbix/vim-shfmt, 22 | with the following lines added to `~/.vimrc`: 23 | 24 | ``` 25 | " shfmt 26 | let g:shfmt_extra_args = '-s -i 2' 27 | let g:shfmt_fmt_on_save = 1 28 | ``` 29 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | VERSION = 0.8.30 2 | 3 | .PHONY: all 4 | all: mok.deploy tags 5 | 6 | mok.deploy: src/*.sh src/lib/*.sh mok-image mok-image/* mok-image/files/* 7 | bash src/embed-dockerfile.sh 8 | cd src && ( echo '#!/usr/bin/env bash'; cat macos.sh arch.sh \ 9 | lib/parser.sh globals.sh error.sh util.sh getcluster.sh machine.sh machinestop.sh \ 10 | machinestart.sh machinecreate.sh machinelist.sh machinedestroy.sh machinesetup.sh \ 11 | exec.sh deletecluster.sh createcluster.sh versions.sh containerutils.sh \ 12 | buildimage.deploy lib/JSONPath.sh main.sh; \ 13 | printf 'if [ "$$0" = "$${BASH_SOURCE[0]}" ] || [ -z "$${BASH_SOURCE[0]}" ]; then\n MA_main "$$@"\nfi\n' \ 14 | ) >../mok.deploy 15 | chmod +x mok.deploy 16 | # cp mok.deploy package/ 17 | 18 | .PHONY: package 19 | package: mok.deploy 20 | cp -v mok.deploy package/mok 21 | 22 | .PHONY: install 23 | install: all 24 | sudo install mok.deploy /usr/local/bin/mok 25 | 26 | .PHONY: uninstall 27 | uninstall: 28 | rm -f /usr/local/bin/mok 29 | 30 | .PHONY: clean 31 | clean: 32 | rm -f mok.deploy src/buildimage.deploy package/mok.deploy \ 33 | tests/hardcopy tests/screenlog.0 tags 34 | 35 | .PHONY: test 36 | test: clean mok.deploy 37 | shellcheck mok.deploy 38 | shfmt -s -i 2 -d src/*.sh 39 | cd tests && ./usage-checks.sh && ./e2e-tests.sh 40 | 41 | .PHONY: buildtest 42 | buildtest: clean mok.deploy 43 | ./tests/build-tests.sh 44 | 45 | tags: src/*.sh 46 | ctags --language-force=sh src/*.sh src/lib/*.sh tests/unit-tests.sh 47 | 48 | # vim:noet:ts=2:sw=2 49 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # MOK - Run Kubernetes on your laptop 2 | 3 | ![image](https://github.com/user-attachments/assets/0750910e-d6da-4c65-92ea-f7bc64b116cc) 4 | 5 | 6 | Default kubernetes version: 1.33.1 7 | 8 | Available kubernetes versions: 9 | 10 | | Minor version | versions | 11 | | ------------- | -------- | 12 | | 1.31.x | not supported | 13 | | 1.32.x | 1.32.0, 1.32.1, 1.32.2, 1.32.3, 1.32.4 | 14 | | 1.33.x | 1.33.0, 1.33.1 | 15 | 16 | ## TL;DR Quick Start 17 | 18 | Install: 19 | 20 | ```bash 21 | curl -O https://raw.githubusercontent.com/bashtools/mok/refs/heads/master/package/mok 22 | chmod +x mok 23 | sudo mv mok /usr/local/bin/ 24 | ``` 25 | 26 | Create cluster: 27 | 28 | ``` 29 | mok build image --get-prebuilt-image 30 | mok create cluster myk8s --masters 1 --publish 31 | ``` 32 | 33 | Use cluser: 34 | 35 | ``` 36 | export KUBECONFIG=/var/tmp/admin-myk8s.conf 37 | kubectl get nodes 38 | kubectl get pods --all-namespaces 39 | kubectl run --privileged --rm -ti alpine --image alpine /bin/sh 40 | ``` 41 | 42 | Delete cluster: 43 | 44 | ``` 45 | mok delete cluster myk8s 46 | ``` 47 | 48 | ## Requirements 49 | 50 | **MacOS** 51 | * Mok will will install any required packages using Homebrew, and will prompt you before doing so. 52 | * To see exactly how and what will be installed, see `src/macos.sh`. 53 | 54 | **Fedora Desktop or Server** 55 | * Install Podman. 56 | 57 | ## Install 58 | 59 | ### Installation for Linux and Mac 60 | 61 | Use `curl` to download `mok` and move it to `/usr/local/bin`: 62 | 63 | ```bash 64 | curl -O https://raw.githubusercontent.com/bashtools/mok/refs/heads/master/package/mok 65 | chmod +x mok 66 | sudo mv mok /usr/local/bin/ 67 | ``` 68 | 69 | ## Using Mok 70 | 71 | ### Build a container image 72 | 73 | ```bash 74 | mok build image --get-prebuilt-image 75 | ``` 76 | 77 | ### Create a single node kuberenetes cluster 78 | 79 | ```bash 80 | mok create cluster myk8s --masters 1 --publish 81 | ``` 82 | 83 | For Mac users `--publish` must be used - but it's optional for Linux users: 84 | 85 | ### Run some kubectl commands 86 | 87 | Naturally, the [kubectl command](https://kubernetes.io/docs/tasks/tools/) is needed for this. 88 | 89 | ```bash 90 | export KUBECONFIG=/var/tmp/admin-myk8s.conf 91 | kubectl get nodes 92 | kubectl get pods --all-namespaces 93 | ``` 94 | 95 | ```bash 96 | # --privileged is required if you want to `ping` a host 97 | kubectl run --privileged --rm -ti alpine --image alpine /bin/sh 98 | ``` 99 | 100 | ### Get help 101 | 102 | ```bash 103 | mok -h 104 | mok create -h 105 | mok build -h 106 | mok machine -h 107 | # ... etc ... 108 | ``` 109 | 110 | ### Delete the cluster 111 | 112 | ```bash 113 | mok delete cluster myk8s 114 | ``` 115 | 116 | On Mac OS do, `mok machine stop`, to stop the podman machine and free up resources. 117 | 118 | ## To Uninstall mok completely 119 | 120 | ### Mac 121 | 122 | ```bash 123 | mok machine destroy 124 | ``` 125 | 126 | If `mok` installed Homebrew, then remove homebrew and all its installed packages with: 127 | 128 | ``` 129 | /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/uninstall.sh)" 130 | ``` 131 | 132 | Then to completely remove any left over directories run: 133 | 134 | ``` 135 | sudo rm -rf /opt/homebrew 136 | ``` 137 | 138 | Finally, delete `mok`, with: 139 | 140 | ```bash 141 | sudo rm /usr/local/bin/mok 142 | ``` 143 | 144 | ### Linux 145 | 146 | ```bash 147 | sudo rm /usr/local/bin/mok 148 | 149 | # If you used git and make, then delete the git repo 150 | rm -rf mok/ 151 | ``` 152 | 153 | Then delete the podman images that were built by `mok build`. 154 | 155 | ## Known Issues 156 | 157 | * With multiple master nodes only the first master is set up 158 | * Currently only single node clusters can be stopped and restarted 159 | * For Mac, if you installed Homebrew with `mok` then you should run 160 | `/opt/homebrew/bin/brew doctor` and follow the instructions shown there if 161 | you want to use Homebrew outside of Mok, or if you want to run the utilities 162 | that mok installed (podman for example). 163 | 164 | ## Some Features 165 | 166 | * Podman Desktop is not required 167 | * On Mac OS all the required packages are installed for you 168 | * On Mac OS it uses a non-default podman machine, so won't mess up your existing podman installation 169 | * Builds kubernetes master and worker nodes in containers 170 | * Very simple to use without need for YAML files 171 | * After creating the image a single node cluster builds in under 60 seconds 172 | * For multi-node clusters the 'create cluster' command returns only when kubernetes is completely ready, with all nodes and pods up and ready. 173 | * Can skip setting up kubernetes on the master and/or worker node (good for learning!) 174 | * In this case the set-up scripts are placed in `/root` in the containers and can be run by hand 175 | * Can do kubernetes the hard way (see [kthwic](https://github.com/my-own-kind/kubernetes-the-hard-way-in-containers)) 176 | * `mok build` and `mok create` can show extensive kubernetes logs with `--tailf` 177 | 178 | * [Full Documentation](https://github.com/bashtools/mokctl-docs/tree/master/docs) 179 | 180 | ## Support Mok 181 | 182 | Follow [Mok on BlueSky](https://bsky.app/profile/github-mok.bsky.social) or give Mok a star. 183 | -------------------------------------------------------------------------------- /mok-image/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2018 The Kubernetes Authors. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # kind node base image 16 | # 17 | # For systemd + docker configuration used below, see the following references: 18 | # https://systemd.io/CONTAINER_INTERFACE/ 19 | 20 | # start from debian slim, this image is reasonably small as a starting point 21 | # for a kubernetes node image, it doesn't contain much (anything?) we don't need 22 | # this stage will install basic files and packages 23 | ARG BASE_IMAGE=debian:bookworm-slim 24 | FROM $BASE_IMAGE AS base 25 | 26 | ARG KUBERNETES_VERSION=v1.32 27 | ARG CRIO_VERSION=v1.32 28 | 29 | # copy in static files 30 | # all scripts and directories are 0755 (rwx r-x r-x) 31 | # all non-scripts are 0644 (rw- r-- r--) 32 | COPY --chmod=0755 files/usr/local/bin/* /usr/local/bin/ 33 | COPY --chmod=0644 files/kind/ /kind/ 34 | COPY --chmod=0644 files/mok/ /mok/ 35 | # COPY only applies to files, not the directory itself, so the permissions are 36 | # fixed in RUN below with a chmod. 37 | COPY --chmod=0755 files/mok/ /mok/ 38 | COPY --chmod=0755 files/kind/bin/ /kind/bin/ 39 | COPY --chmod=0644 files/LICENSES/* /LICENSES/* 40 | COPY --chmod=0644 files/etc/crictl.yaml /etc/ 41 | COPY --chmod=0644 files/etc/crio/crio.conf.d/* /etc/crio/crio.conf.d/ 42 | COPY --chmod=0644 files/etc/sysctl.d/* /etc/sysctl.d/ 43 | COPY --chmod=0644 files/etc/systemd/system/* /etc/systemd/system/ 44 | COPY --chmod=0644 files/etc/systemd/system/kubelet.service.d/* /etc/systemd/system/kubelet.service.d/ 45 | COPY --chmod=0644 files/kube-flannel.yml /root/kube-flannel.yml 46 | 47 | # Install dependencies, first from apt, then from release tarballs. 48 | # NOTE: we use one RUN to minimize layers. 49 | # 50 | # The base image already has a basic userspace + apt but we need to install more packages. 51 | # Packages installed are broken down into (each on a line): 52 | # - packages needed to run services (systemd) 53 | # - packages needed for kubernetes components 54 | # - packages needed for networked backed storage with kubernetes 55 | # - packages needed by the container runtime 56 | # - misc packages kind uses itself 57 | # - packages that provide semi-core kubernetes functionality 58 | # After installing packages we cleanup by: 59 | # - removing unwanted systemd services 60 | # - disabling kmsg in journald (these log entries would be confusing) 61 | # 62 | # Next we download and extract crictl and CNI plugin binaries from upstream. 63 | # 64 | # Next we ensure the /etc/kubernetes/manifests directory exists. Normally 65 | # a kubeadm debian / rpm package would ensure that this exists but we install 66 | # freshly built binaries directly when we build the node image. 67 | # 68 | # Finally we adjust tempfiles cleanup to be 1 minute after "boot" instead of 15m 69 | # This is plenty after we've done initial setup for a node, but before we are 70 | # likely to try to export logs etc. 71 | RUN chmod 755 /kind/bin && \ 72 | echo "Installing Packages ..." \ 73 | && DEBIAN_FRONTEND=noninteractive clean-install \ 74 | systemd apt-transport-https ca-certificates gpg socat \ 75 | conntrack iptables nftables iproute2 ethtool util-linux mount kmod \ 76 | libseccomp2 pigz fuse-overlayfs \ 77 | nfs-common open-iscsi \ 78 | bash ca-certificates curl jq procps \ 79 | && find /lib/systemd/system/sysinit.target.wants/ -name "systemd-tmpfiles-setup.service" -delete \ 80 | && rm -f /lib/systemd/system/multi-user.target.wants/* \ 81 | && rm -f /etc/systemd/system/*.wants/* \ 82 | && rm -f /lib/systemd/system/local-fs.target.wants/* \ 83 | && rm -f /lib/systemd/system/sockets.target.wants/*udev* \ 84 | && rm -f /lib/systemd/system/sockets.target.wants/*initctl* \ 85 | && rm -f /lib/systemd/system/basic.target.wants/* \ 86 | && echo "ReadKMsg=no" >> /etc/systemd/journald.conf \ 87 | && ln -s "$(which systemd)" /sbin/init 88 | 89 | # Install CRI-O (and delete the config it creates in /etc/cni/net.d) 90 | RUN curl -fsSL https://pkgs.k8s.io/core:/stable:/$KUBERNETES_VERSION/deb/Release.key | \ 91 | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg \ 92 | && echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/$KUBERNETES_VERSION/deb/ /" | \ 93 | tee /etc/apt/sources.list.d/kubernetes.list \ 94 | && curl -fsSL https://pkgs.k8s.io/addons:/cri-o:/stable:/$CRIO_VERSION/deb/Release.key | \ 95 | gpg --dearmor -o /etc/apt/keyrings/cri-o-apt-keyring.gpg \ 96 | && echo "deb [signed-by=/etc/apt/keyrings/cri-o-apt-keyring.gpg] https://pkgs.k8s.io/addons:/cri-o:/stable:/$CRIO_VERSION/deb/ /" | \ 97 | tee /etc/apt/sources.list.d/cri-o.list \ 98 | && apt-get update \ 99 | && apt-get install -y cri-o kubelet kubeadm kubectl \ 100 | && rm -f /etc/cni/net.d/* 101 | 102 | # NOTE: systemd-binfmt.service will register things into binfmt_misc which is kernel-global 103 | RUN echo "Enabling / Disabling services ... " \ 104 | && systemctl enable kubelet.service \ 105 | && systemctl enable undo-mount-hacks.service \ 106 | && systemctl mask systemd-binfmt.service \ 107 | && systemctl enable crio.service \ 108 | && systemctl enable node-address-update.service 109 | 110 | RUN echo "Ensuring /etc/kubernetes/manifests" \ 111 | && mkdir -p /etc/kubernetes/manifests 112 | 113 | # shared stage to setup go version for building binaries 114 | # NOTE we will be cross-compiling for performance reasons 115 | # This is also why we start again FROM the same base image but a different 116 | # platform and only the files needed for building 117 | # We will copy the built binaries from later stages to the final stage(s) 118 | FROM --platform=$BUILDPLATFORM $BASE_IMAGE AS go-build 119 | COPY --chmod=0755 files/usr/local/bin/* /usr/local/bin/ 120 | COPY --chmod=0755 scripts/third_party/gimme/gimme /usr/local/bin/ 121 | COPY --chmod=0755 scripts/target-cc /usr/local/bin/ 122 | # tools needed at build-time only 123 | # first ensure we can install packages for both architectures 124 | RUN dpkg --add-architecture arm64 && dpkg --add-architecture amd64 \ 125 | && clean-install bash ca-certificates curl git make pkg-config \ 126 | crossbuild-essential-amd64 crossbuild-essential-arm64 \ 127 | libseccomp-dev:amd64 libseccomp-dev:arm64 128 | # set by makefile to .go-version 129 | ARG GO_VERSION 130 | RUN eval "$(gimme "${GO_VERSION}")" \ 131 | && export GOTOOLCHAIN="go${GO_VERSION}" \ 132 | && GOBIN=/usr/local/bin go install github.com/google/go-licenses@latest 133 | 134 | # stage for building runc 135 | FROM go-build AS build-runc 136 | ARG TARGETARCH GO_VERSION 137 | ARG RUNC_VERSION="v1.2.6" 138 | ARG RUNC_CLONE_URL="https://github.com/opencontainers/runc" 139 | RUN git clone --filter=tree:0 "${RUNC_CLONE_URL}" /runc \ 140 | && cd /runc \ 141 | && git checkout "${RUNC_VERSION}" \ 142 | && eval "$(gimme "${GO_VERSION}")" \ 143 | && export GOTOOLCHAIN="go${GO_VERSION}" \ 144 | && export GOARCH=$TARGETARCH && export CC=$(target-cc) && export CGO_ENABLED=1 \ 145 | && make runc \ 146 | && GOARCH=$TARGETARCH go-licenses save --save_path=/_LICENSES . 147 | 148 | # stage for building crictl 149 | FROM go-build AS build-crictl 150 | ARG TARGETARCH GO_VERSION 151 | ARG CRI_TOOLS_CLONE_URL="https://github.com/kubernetes-sigs/cri-tools" 152 | ARG CRICTL_VERSION="v1.32.0" 153 | RUN git clone --filter=tree:0 "${CRI_TOOLS_CLONE_URL}" /cri-tools \ 154 | && cd /cri-tools \ 155 | && git checkout "${CRICTL_VERSION}" \ 156 | && eval "$(gimme "${GO_VERSION}")" \ 157 | && export GOTOOLCHAIN="go${GO_VERSION}" \ 158 | && export GOARCH=$TARGETARCH && export CC=$(target-cc) && export CGO_ENABLED=1 \ 159 | && make BUILD_BIN_PATH=./build crictl \ 160 | && GOARCH=$TARGETARCH go-licenses save --save_path=/_LICENSES ./cmd/crictl 161 | 162 | # stage for building cni-plugins 163 | FROM go-build AS build-cni 164 | ARG TARGETARCH GO_VERSION 165 | ARG CNI_PLUGINS_VERSION="v1.5.1" 166 | ARG CNI_PLUGINS_CLONE_URL="https://github.com/containernetworking/plugins" 167 | RUN git clone --filter=tree:0 "${CNI_PLUGINS_CLONE_URL}" /cni-plugins \ 168 | && cd /cni-plugins \ 169 | && git checkout "${CNI_PLUGINS_VERSION}" \ 170 | && eval "$(gimme "${GO_VERSION}")" \ 171 | && export GOTOOLCHAIN="go${GO_VERSION}" \ 172 | && mkdir ./bin \ 173 | && export GOARCH=$TARGETARCH && export CC=$(target-cc) && export CGO_ENABLED=1 \ 174 | && go build -o ./bin/host-local -mod=vendor ./plugins/ipam/host-local \ 175 | && go build -o ./bin/loopback -mod=vendor ./plugins/main/loopback \ 176 | && go build -o ./bin/ptp -mod=vendor ./plugins/main/ptp \ 177 | && go build -o ./bin/portmap -mod=vendor ./plugins/meta/portmap \ 178 | && GOARCH=$TARGETARCH go-licenses save --save_path=/_LICENSES \ 179 | ./plugins/ipam/host-local \ 180 | ./plugins/main/loopback ./plugins/main/ptp \ 181 | ./plugins/meta/portmap 182 | 183 | # build final image layout from other stages 184 | FROM base AS build 185 | COPY --from=build-runc /runc/runc /usr/local/sbin/runc 186 | RUN runc --version 187 | COPY --from=build-runc /_LICENSES/* /LICENSES/ 188 | # copy over crictl build and install 189 | COPY --from=build-crictl /cri-tools/build/crictl /usr/local/bin/ 190 | COPY --from=build-crictl /_LICENSES/* /LICENSES/ 191 | # copy over CNI plugins build and install 192 | RUN mkdir -p /opt/cni/bin 193 | COPY --from=build-cni /cni-plugins/bin/host-local /opt/cni/bin/ 194 | COPY --from=build-cni /cni-plugins/bin/loopback /opt/cni/bin/ 195 | COPY --from=build-cni /cni-plugins/bin/ptp /opt/cni/bin/ 196 | COPY --from=build-cni /cni-plugins/bin/portmap /opt/cni/bin/ 197 | COPY --from=build-cni /_LICENSES/* /LICENSES/ 198 | 199 | # squash down to one compressed layer, without any lingering whiteout files etc 200 | FROM scratch 201 | COPY --from=build / / 202 | # add metadata, must be done after the squashing 203 | # first tell systemd that it is in docker (it will check for the container env) 204 | # https://systemd.io/CONTAINER_INTERFACE/ 205 | ENV container=docker 206 | # systemd exits on SIGRTMIN+3, not SIGTERM (which re-executes it) 207 | # https://bugzilla.redhat.com/show_bug.cgi?id=1201657 208 | STOPSIGNAL SIGRTMIN+3 209 | # Using the full explicit command seemed to be required. Amazingly 210 | # things work on Fedora 37 (with cgroupsv1 enabled and using Podman) with no entrypoint script!! 211 | #ENTRYPOINT [ "/usr/local/bin/entrypoint", "/sbin/init" ] 212 | CMD [ "/lib/systemd/systemd", "log-level=info", "unit=sysinit.target" ] 213 | -------------------------------------------------------------------------------- /mok-image/files/LICENSES/README: -------------------------------------------------------------------------------- 1 | Go license files 2 | -------------------------------------------------------------------------------- /mok-image/files/etc/crictl.yaml: -------------------------------------------------------------------------------- 1 | runtime-endpoint: "unix:///var/run/crio/crio.sock" 2 | timeout: 0 3 | debug: false 4 | -------------------------------------------------------------------------------- /mok-image/files/etc/crio/crio.conf.d/11-crio.conf: -------------------------------------------------------------------------------- 1 | [crio.runtime] 2 | default_runtime = "crun" 3 | conmon_cgroup = "pod" 4 | cgroup_manager = "cgroupfs" 5 | -------------------------------------------------------------------------------- /mok-image/files/etc/default/kubelet: -------------------------------------------------------------------------------- 1 | KUBELET_EXTRA_ARGS= 2 | -------------------------------------------------------------------------------- /mok-image/files/etc/sysctl.d/10-network-magic.conf: -------------------------------------------------------------------------------- 1 | # Do not consider loopback addresses as martian source or destination while routing 2 | # - Docker with custom networks uses an embedded DNS server with address 172.0.0.11 3 | # - Kubernetes pods mount the node resolv.conf, so they can't use a loopack address 4 | # that is only reachable from the node. 5 | # 6 | # KIND rewrites the well-known docker DNS address 127.0.0.11 by a non-loopback address. 7 | # The DNS traffic coming from a pod will have to route to a localhost address to be NATed, 8 | # hence we have to enable route_localnet or pods DNS will not work. 9 | # Kubernetes mitigates the possible security issue caused by enabling this option. 10 | # ref: https://nvd.nist.gov/vuln/detail/CVE-2020-8558 11 | net.ipv4.conf.all.route_localnet=1 12 | -------------------------------------------------------------------------------- /mok-image/files/etc/sysctl.d/10-network-security.conf: -------------------------------------------------------------------------------- 1 | # Turn on Source Address Verification in all interfaces to 2 | # prevent some spoofing attacks. 3 | net.ipv4.conf.default.rp_filter=1 4 | net.ipv4.conf.all.rp_filter=1 5 | -------------------------------------------------------------------------------- /mok-image/files/etc/systemd/system/kubelet.service: -------------------------------------------------------------------------------- 1 | # slightly modified from: 2 | # https://github.com/kubernetes/kubernetes/blob/ba8fcafaf8c502a454acd86b728c857932555315/build/debs/kubelet.service 3 | [Unit] 4 | Description=kubelet: The Kubernetes Node Agent 5 | Documentation=http://kubernetes.io/docs/ 6 | # NOTE: kind deviates from upstream here to avoid crashlooping 7 | # This does *not* support altering the kubelet config path though. 8 | # We intend to upstream this change but first need to solve the upstream 9 | # Packaging problem (all kubernetes versions use the same files out of tree). 10 | ConditionPathExists=/var/lib/kubelet/config.yaml 11 | 12 | [Service] 13 | ExecStart=/usr/bin/kubelet 14 | Restart=always 15 | StartLimitInterval=0 16 | # NOTE: kind deviates from upstream here with a lower RestartSec 17 | RestartSec=1s 18 | # And by adding the [Service] lines below 19 | CPUAccounting=true 20 | MemoryAccounting=true 21 | Slice=kubelet.slice 22 | KillMode=process 23 | 24 | [Install] 25 | WantedBy=multi-user.target 26 | -------------------------------------------------------------------------------- /mok-image/files/etc/systemd/system/kubelet.service.d/10-kubeadm.conf: -------------------------------------------------------------------------------- 1 | # https://github.com/kubernetes/kubernetes/blob/ba8fcafaf8c502a454acd86b728c857932555315/build/debs/10-kubeadm.conf 2 | # Note: This dropin only works with kubeadm and kubelet v1.11+ 3 | [Service] 4 | Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf" 5 | Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml" 6 | # This is a file that "kubeadm init" and "kubeadm join" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically 7 | EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env 8 | # This is a file that the user can use for overrides of the kubelet args as a last resort. Preferably, the user should use 9 | # the .NodeRegistration.KubeletExtraArgs object in the configuration files instead. KUBELET_EXTRA_ARGS should be sourced from this file. 10 | EnvironmentFile=-/etc/default/kubelet 11 | ExecStart= 12 | ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS 13 | -------------------------------------------------------------------------------- /mok-image/files/etc/systemd/system/kubelet.service.d/11-kind.conf: -------------------------------------------------------------------------------- 1 | # kind specific additions go in this file 2 | [Service] 3 | # On cgroup v1, the /kubelet cgroup is created in the entrypoint script before running systemd. 4 | # On cgroup v2, the /kubelet cgroup is created here. (See the comments in the entrypoint script for the reason.) 5 | ExecStartPre=/bin/sh -euc "if [ -f /sys/fs/cgroup/cgroup.controllers ]; then /kind/bin/create-kubelet-cgroup-v2.sh; fi" 6 | # on WSL2 (and potentially other distros without systemd) /sys/fs/cgroup/systemd is created after the entrypoint, during /sbin/init. 7 | # This eventually leads to kubelet failing to start, see: https://github.com/kubernetes-sigs/kind/issues/2323 8 | ExecStartPre=/bin/sh -euc "if [ ! -f /sys/fs/cgroup/cgroup.controllers ] && [ ! -d /sys/fs/cgroup/systemd/kubelet ]; then mkdir -p /sys/fs/cgroup/systemd/kubelet; fi" 9 | -------------------------------------------------------------------------------- /mok-image/files/etc/systemd/system/kubelet.slice: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=slice used to run Kubernetes / Kubelet 3 | Before=slices.target 4 | 5 | [Slice] 6 | MemoryAccounting=true 7 | CPUAccounting=true 8 | -------------------------------------------------------------------------------- /mok-image/files/etc/systemd/system/node-address-update.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Check for NODEADDRESSCHANGED and run update script 3 | After=network.target 4 | 5 | [Service] 6 | Type=oneshot 7 | ExecStart=/bin/bash -c 'if [ -n "$NODEADDRESSCHANGED" ]; then /usr/local/bin/update_kube-proxy.sh; fi' 8 | EnvironmentFile=/etc/environment 9 | 10 | [Install] 11 | WantedBy=multi-user.target 12 | -------------------------------------------------------------------------------- /mok-image/files/etc/systemd/system/undo-mount-hacks.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Undo KIND mount hacks 3 | After=slices.target 4 | Before=kubelet.service 5 | 6 | [Service] 7 | Type=oneshot 8 | ExecStart=/kind/bin/undo-mount-hacks.sh 9 | 10 | [Install] 11 | WantedBy=multi-user.target 12 | -------------------------------------------------------------------------------- /mok-image/files/kind/README.txt: -------------------------------------------------------------------------------- 1 | This directory is reserved for KIND [^1] internal purposes. 2 | 3 | Modifying or depending on the contents of this directory is NOT supported. 4 | 5 | Here be dragons. 6 | 7 | [^1]: https://kind.sigs.k8s.io/ 8 | -------------------------------------------------------------------------------- /mok-image/files/kind/bin/create-kubelet-cgroup-v2.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2021 The Kubernetes Authors. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | set -o errexit 18 | set -o nounset 19 | set -o pipefail 20 | if [[ ! -f "/sys/fs/cgroup/cgroup.controllers" ]]; then 21 | echo 'ERROR: this script should not be called on cgroup v1 hosts' >&2 22 | exit 1 23 | fi 24 | 25 | # NOTE: we can't use `test -s` because cgroup.procs is not a regular file. 26 | if grep -qv '^0$' /sys/fs/cgroup/cgroup.procs ; then 27 | echo 'ERROR: this script needs /sys/fs/cgroup/cgroup.procs to be empty (for writing the top-level cgroup.subtree_control)' >&2 28 | # So, this script needs to be called after launching systemd. 29 | # This script cannot be called from /usr/local/bin/entrypoint. 30 | exit 1 31 | fi 32 | 33 | ensure_subtree_control() { 34 | local group=$1 35 | # When cgroup.controllers is like "cpu cpuset memory io pids", 36 | # cgroup.subtree_control is written with "+cpu +cpuset +memory +io +pids" . 37 | sed -e 's/ / +/g' -e 's/^/+/' <"/sys/fs/cgroup/$group/cgroup.controllers" >"/sys/fs/cgroup/$group/cgroup.subtree_control" 38 | } 39 | 40 | # kubelet requires all the controllers (including hugetlb) in /sys/fs/cgroup/cgroup.controllers to be available in 41 | # /sys/fs/cgroup/kubelet/cgroup.subtree_control. 42 | # 43 | # We need to update the top-level cgroup.subtree_controllers as well, because hugetlb is not present in the file by default. 44 | ensure_subtree_control / 45 | mkdir -p /sys/fs/cgroup/kubelet 46 | ensure_subtree_control /kubelet 47 | # again for kubelet.slice for systemd cgroup driver 48 | mkdir -p /sys/fs/cgroup/kubelet.slice 49 | ensure_subtree_control /kubelet.slice 50 | -------------------------------------------------------------------------------- /mok-image/files/kind/bin/mount-product-files.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2021 The Kubernetes Authors. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # This script is a createContainer hook [1] that replicates the functionality from entrypoint script to mount product_name and product_uuid but from a product_name and product_uuid copied into the contianer rootfs to prevent all the containers from bind mounting the same file. Sharing the same bind mount between all the containers increases the latency accessing the container, preventing it from accessing in some cases. 18 | # 19 | # [1] https://github.com/opencontainers/runtime-spec/blob/master/config.md#createcontainer-hooks 20 | 21 | set -o errexit 22 | set -o nounset 23 | set -o pipefail 24 | 25 | # Explicitly set PATH so as not to be inherited from the container 26 | # We have no reason to be using any binary from the container, and the 27 | # container PATH may lack normal "host" (kind node container in this case) 28 | # system paths. 29 | # 30 | # See: https://github.com/kubernetes-sigs/kind/issues/2551 31 | # 32 | # All of the binaries this script needs are in /usr/bin currently, but this is 33 | # the full normal PATH for this image with pretty standard linux paths. 34 | export PATH='/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' 35 | 36 | # The bundle represents the dir path to container filesystem, container runtime state [1] is 37 | # passed to the hook's stdin 38 | # 39 | # [1] https://github.com/opencontainers/runtime-spec/blob/master/runtime.md#state 40 | # 41 | bundle=$(jq -r .bundle) 42 | 43 | cp /kind/product_* "${bundle:?}/rootfs/" 44 | if [[ -f /sys/class/dmi/id/product_name ]]; then 45 | mount -o ro,bind "${bundle:?}"/rootfs/product_name "${bundle:?}"/rootfs/sys/class/dmi/id/product_name 46 | fi 47 | 48 | if [[ -f /sys/class/dmi/id/product_uuid ]]; then 49 | mount -o ro,bind "${bundle:?}"/rootfs/product_uuid "${bundle:?}"/rootfs/sys/class/dmi/id/product_uuid 50 | fi 51 | 52 | if [[ -f /sys/devices/virtual/dmi/id/product_uuid ]]; then 53 | mount -o ro,bind "${bundle:?}"/rootfs/product_uuid "${bundle:?}"/rootfs/sys/devices/virtual/dmi/id/product_uuid 54 | fi 55 | -------------------------------------------------------------------------------- /mok-image/files/kind/bin/undo-mount-hacks.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2023 The Kubernetes Authors. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | set -o errexit 18 | set -o nounset 19 | set -o pipefail 20 | 21 | # don't run on cgroups v2 22 | if [[ -f "/sys/fs/cgroup/cgroup.controllers" ]]; then 23 | exit 0 24 | fi 25 | 26 | # on cgroups v1 we want to undo bind mounting over this to hide misc 27 | # see entrypoint 28 | umount /proc/cgroups || true 29 | -------------------------------------------------------------------------------- /mok-image/files/kube-flannel.yml: -------------------------------------------------------------------------------- 1 | # This manifest installs the required infra for flannel 2 | # https://github.com/flannel-io/flannel/releases/latest/download/kube-flannel.yml 3 | # The following sed commands are applied to the above manifest 4 | # sed 's/privileged: false/privileged: true/' 5 | # sed 's/vxlan/host-gw/' 6 | apiVersion: v1 7 | kind: Namespace 8 | metadata: 9 | labels: 10 | k8s-app: flannel 11 | pod-security.kubernetes.io/enforce: privileged 12 | name: kube-flannel 13 | --- 14 | apiVersion: v1 15 | kind: ServiceAccount 16 | metadata: 17 | labels: 18 | k8s-app: flannel 19 | name: flannel 20 | namespace: kube-flannel 21 | --- 22 | apiVersion: rbac.authorization.k8s.io/v1 23 | kind: ClusterRole 24 | metadata: 25 | labels: 26 | k8s-app: flannel 27 | name: flannel 28 | rules: 29 | - apiGroups: 30 | - "" 31 | resources: 32 | - pods 33 | verbs: 34 | - get 35 | - apiGroups: 36 | - "" 37 | resources: 38 | - nodes 39 | verbs: 40 | - get 41 | - list 42 | - watch 43 | - apiGroups: 44 | - "" 45 | resources: 46 | - nodes/status 47 | verbs: 48 | - patch 49 | --- 50 | apiVersion: rbac.authorization.k8s.io/v1 51 | kind: ClusterRoleBinding 52 | metadata: 53 | labels: 54 | k8s-app: flannel 55 | name: flannel 56 | roleRef: 57 | apiGroup: rbac.authorization.k8s.io 58 | kind: ClusterRole 59 | name: flannel 60 | subjects: 61 | - kind: ServiceAccount 62 | name: flannel 63 | namespace: kube-flannel 64 | --- 65 | apiVersion: v1 66 | data: 67 | cni-conf.json: | 68 | { 69 | "name": "cni0", 70 | "cniVersion": "0.3.1", 71 | "plugins": [ 72 | { 73 | "type": "flannel", 74 | "delegate": { 75 | "hairpinMode": true, 76 | "isDefaultGateway": true 77 | } 78 | }, 79 | { 80 | "type": "portmap", 81 | "capabilities": { 82 | "portMappings": true 83 | } 84 | } 85 | ] 86 | } 87 | net-conf.json: | 88 | { 89 | "Network": "10.244.0.0/16", 90 | "EnableNFTables": false, 91 | "Backend": { 92 | "Type": "host-gw" 93 | } 94 | } 95 | kind: ConfigMap 96 | metadata: 97 | labels: 98 | app: flannel 99 | k8s-app: flannel 100 | tier: node 101 | name: kube-flannel-cfg 102 | namespace: kube-flannel 103 | --- 104 | apiVersion: apps/v1 105 | kind: DaemonSet 106 | metadata: 107 | labels: 108 | app: flannel 109 | k8s-app: flannel 110 | tier: node 111 | name: kube-flannel-ds 112 | namespace: kube-flannel 113 | spec: 114 | selector: 115 | matchLabels: 116 | app: flannel 117 | k8s-app: flannel 118 | template: 119 | metadata: 120 | labels: 121 | app: flannel 122 | k8s-app: flannel 123 | tier: node 124 | spec: 125 | affinity: 126 | nodeAffinity: 127 | requiredDuringSchedulingIgnoredDuringExecution: 128 | nodeSelectorTerms: 129 | - matchExpressions: 130 | - key: kubernetes.io/os 131 | operator: In 132 | values: 133 | - linux 134 | containers: 135 | - args: 136 | - --ip-masq 137 | - --kube-subnet-mgr 138 | command: 139 | - /opt/bin/flanneld 140 | env: 141 | - name: POD_NAME 142 | valueFrom: 143 | fieldRef: 144 | fieldPath: metadata.name 145 | - name: POD_NAMESPACE 146 | valueFrom: 147 | fieldRef: 148 | fieldPath: metadata.namespace 149 | - name: EVENT_QUEUE_DEPTH 150 | value: "5000" 151 | image: docker.io/flannel/flannel:v0.25.7 152 | name: kube-flannel 153 | resources: 154 | requests: 155 | cpu: 100m 156 | memory: 50Mi 157 | securityContext: 158 | capabilities: 159 | add: 160 | - NET_ADMIN 161 | - NET_RAW 162 | privileged: true 163 | volumeMounts: 164 | - mountPath: /run/flannel 165 | name: run 166 | - mountPath: /etc/kube-flannel/ 167 | name: flannel-cfg 168 | - mountPath: /run/xtables.lock 169 | name: xtables-lock 170 | hostNetwork: true 171 | initContainers: 172 | - args: 173 | - -f 174 | - /flannel 175 | - /opt/cni/bin/flannel 176 | command: 177 | - cp 178 | image: docker.io/flannel/flannel-cni-plugin:v1.5.1-flannel2 179 | name: install-cni-plugin 180 | volumeMounts: 181 | - mountPath: /opt/cni/bin 182 | name: cni-plugin 183 | - args: 184 | - -f 185 | - /etc/kube-flannel/cni-conf.json 186 | - /etc/cni/net.d/10-flannel.conflist 187 | command: 188 | - cp 189 | image: docker.io/flannel/flannel:v0.25.7 190 | name: install-cni 191 | volumeMounts: 192 | - mountPath: /etc/cni/net.d 193 | name: cni 194 | - mountPath: /etc/kube-flannel/ 195 | name: flannel-cfg 196 | priorityClassName: system-node-critical 197 | serviceAccountName: flannel 198 | tolerations: 199 | - effect: NoSchedule 200 | operator: Exists 201 | volumes: 202 | - hostPath: 203 | path: /run/flannel 204 | name: run 205 | - hostPath: 206 | path: /opt/cni/bin 207 | name: cni-plugin 208 | - hostPath: 209 | path: /etc/cni/net.d 210 | name: cni 211 | - configMap: 212 | name: kube-flannel-cfg 213 | name: flannel-cfg 214 | - hostPath: 215 | path: /run/xtables.lock 216 | type: FileOrCreate 217 | name: xtables-lock 218 | -------------------------------------------------------------------------------- /mok-image/files/mok/README: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bashtools/mok/5ccedb9aea7b05d47e310003c157efdd7df782af/mok-image/files/mok/README -------------------------------------------------------------------------------- /mok-image/files/usr/local/bin/clean-install: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Copyright 2017 The Kubernetes Authors. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # A script encapsulating a common Dockerimage pattern for installing packages 18 | # and then cleaning up the unnecessary install artifacts. 19 | # e.g. clean-install iptables ebtables conntrack 20 | 21 | set -o errexit 22 | 23 | if [ $# = 0 ]; then 24 | echo >&2 "No packages specified" 25 | exit 1 26 | fi 27 | 28 | apt-get update 29 | apt-get upgrade -y 30 | apt-get install -y --no-install-recommends "$@" 31 | apt-get clean -y 32 | rm -rf \ 33 | /var/cache/debconf/* \ 34 | /var/lib/apt/lists/* \ 35 | /var/log/* \ 36 | /tmp/* \ 37 | /var/tmp/* \ 38 | /usr/share/doc/* \ 39 | /usr/share/doc-base/* \ 40 | /usr/share/man/* \ 41 | /usr/share/local/* 42 | -------------------------------------------------------------------------------- /mok-image/files/usr/local/bin/update_kube-proxy.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | export KUBECONFIG=/etc/kubernetes/super-admin.conf 4 | # Wait for api server to become ready 5 | # shellcheck disable=SC2034 6 | for i in {1..60}; do 7 | if kubectl get pods -n kube-system -l k8s-app=kube-proxy --no-headers=true | grep -qs Running; then 8 | break 9 | fi 10 | sleep 1 11 | done 12 | sleep 2 13 | ipv4=$(cat /mok/old-ipv4) 14 | kubectl get configmap kube-proxy -n kube-system -o yaml > /tmp/kube-proxy.yaml 15 | sed -i '/^ *creationTimestamp:/d;/^ *resourceVersion:/d;/^ *uid:/d' /tmp/kube-proxy.yaml 16 | sed -i "s#\(^ *server: \).*#\1https://$ipv4:6443#" /tmp/kube-proxy.yaml 17 | kubectl apply -f /tmp/kube-proxy.yaml -n kube-system 18 | kubectl delete pod -l k8s-app=kube-proxy -n kube-system 19 | sleep 2 20 | systemctl restart kubelet 21 | -------------------------------------------------------------------------------- /mok-image/scripts/target-cc: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright 2023 The Kubernetes Authors. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # this script maps buildx TARGETARCH to $CC 18 | 19 | set -o errexit -o nounset -o pipefail 20 | 21 | case $TARGETARCH in 22 | arm64) 23 | echo -n 'aarch64-linux-gnu-gcc' ;; 24 | amd64) 25 | echo -n 'x86_64-linux-gnu-gcc' ;; 26 | *) 27 | exit 1 ;; 28 | esac 29 | -------------------------------------------------------------------------------- /mok-image/scripts/third_party/gimme/LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015-2018 gimme contributors 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /mok-image/scripts/third_party/gimme/README.md: -------------------------------------------------------------------------------- 1 | # gimme 2 | 3 | This is an unmodified copy of [gimme], so we don't have to download it 4 | from the internet. 5 | 6 | [gimme]: https://github.com/travis-ci/gimme -------------------------------------------------------------------------------- /src/arch.sh: -------------------------------------------------------------------------------- 1 | # shellcheck shell=bash disable=SC2148 2 | 3 | declare __mokostype 4 | 5 | set_arch() { 6 | if [[ ${__mokostype} == "linux" ]]; then 7 | # Is this machine x86 or arm? 8 | if [[ $(uname -m) == "x86_64" ]]; then 9 | export __mokarch=x86_64 10 | elif [[ $(uname -m) == "aarch64" ]]; then 11 | export __mokarch=arm64 12 | fi 13 | elif [[ ${__mokostype} == "macos" ]]; then 14 | # Is this machine x86 or arm? 15 | if [[ $(uname -m) == "x86_64" ]]; then 16 | export __mokarch=x86_64 17 | elif [[ $(uname -m) == "arm64" ]]; then 18 | export __mokarch=arm64 19 | fi 20 | else 21 | printf 'ERROR: Unknown OS type: %s\n' "${__mokostype}" >"${STDERR}" 22 | exit "${ERROR}" 23 | fi 24 | } 25 | 26 | set_arch 27 | -------------------------------------------------------------------------------- /src/buildimage.sh: -------------------------------------------------------------------------------- 1 | # shellcheck shell=bash disable=SC2148 2 | # BI - Build Image 3 | 4 | # _BI is an associative array that holds data specific to building an image. 5 | declare -A _BI 6 | 7 | # Declare externally defined variables ---------------------------------------- 8 | 9 | # Defined in GL (globals.sh) 10 | declare OK ERROR STDERR STOP TRUE FALSE 11 | 12 | # Getters/Setters ------------------------------------------------------------- 13 | 14 | # BI_setflag_useprebuiltimage setter sets the useprebuiltimage array item. 15 | # This is called by the parser, via a callback in _BI_new. 16 | BI_setflag_useprebuiltimage() { 17 | _BI[useprebuiltimage]="$1" 18 | } 19 | 20 | # BI_baseimagename getter outputs the value of the _BI[baseimagename]. 21 | # Args: None expected. 22 | BI_baseimagename() { 23 | printf '%s' "${_BI[baseimagename]}" 24 | } 25 | 26 | # Public Functions ------------------------------------------------------------ 27 | 28 | # BI_usage outputs help text for the build image component. 29 | # It is called by PA_usage(), via a callback in _BI_new. 30 | # Args: None expected. 31 | BI_usage() { 32 | 33 | cat <<'EnD' 34 | BUILD subcommands are: 35 | 36 | image - Creates the docker 'mok-image' container image. 37 | 38 | build image options: 39 | 40 | Format: 41 | build image [flags] 42 | 43 | Flags: 44 | --get-prebuilt-image - Instead of building a 'node' image 45 | locally, download it from a container registry instead. 46 | --k8sver VERSION - The Kubernetes version, e.g. "1.32.0" 47 | A previous version cannot be built from scratch hence this option 48 | must be used with '--get-prebuilt-image'. To build a previous 49 | version of kubernetes it must be built with a previous version 50 | of mok. 51 | --tailf - Show the build output whilst building. 52 | 53 | EnD 54 | } 55 | 56 | # BI_cleanup removes temporary files created during the build. 57 | # This function is called by the 'MA_cleanup' trap only. 58 | # Args: None expected. 59 | BI_cleanup() { 60 | [[ -e ${_BI[dockerbuildtmpdir]} ]] && 61 | [[ ${_BI[dockerbuildtmpdir]} == "/var/tmp/"* ]] && { 62 | rm -rf "${_BI[dockerbuildtmpdir]}" || { 63 | printf 'ERROR: "rm -rf %s" failed.\n' "${_BI[dockerbuildtmpdir]}" \ 64 | >"${STDERR}" 65 | err || return 66 | } 67 | } 68 | } 69 | 70 | # BI_check_valid_options checks if arg1 is in a list of valid build image 71 | # options. This function is called by the parser, via a callback in _BI_new. 72 | # Args: arg1 - the option to check. 73 | # arg2 - value of the item to be set, optional 74 | BI_process_options() { 75 | 76 | case "$1" in 77 | -h | --help) 78 | BI_usage 79 | return "${STOP}" 80 | ;; 81 | --tailf) 82 | _BI[tailf]="${TRUE}" 83 | return "${OK}" 84 | ;; 85 | --k8sver) 86 | _BI[k8sver]="$2" 87 | return "$(PA_shift)" 88 | ;; 89 | --get-prebuilt-image) 90 | _BI[useprebuiltimage]="${TRUE}" 91 | return "${OK}" 92 | ;; 93 | *) 94 | BI_usage 95 | printf 'ERROR: "%s" is not a valid "build" option.\n' "${1}" \ 96 | >"${STDERR}" 97 | return "${ERROR}" 98 | ;; 99 | esac 100 | } 101 | 102 | # BI_run builds the base image used for masters and workers. 103 | # This function is called in main.sh. 104 | # Args: None expected. 105 | BI_run() { 106 | 107 | _BI_sanity_checks || return 108 | 109 | [[ ${_BI[tailf]} == "${TRUE}" ]] && { 110 | UT_set_tailf "${TRUE}" || err || return 111 | } 112 | 113 | local retval=0 114 | _BI_build_container_image 115 | retval=$? 116 | 117 | if [[ ${retval} -eq ${OK} ]]; then 118 | : # We only need the tick - no text 119 | else 120 | printf 'Image build failed\n' >"${STDERR}" 121 | err 122 | fi 123 | 124 | return "${retval}" 125 | } 126 | 127 | # Private Functions ----------------------------------------------------------- 128 | 129 | # _BI_new resets the initial values for the _BI associative array. 130 | # Args: None expected. 131 | _BI_new() { 132 | _BI[tailf]="${FALSE}" 133 | _BI[baseimagename]="mok-image" 134 | _BI[useprebuiltimage]="${FALSE}" 135 | _BI[dockerbuildtmpdir]= 136 | _BI[k8sver]="${K8SVERSION}" 137 | 138 | # Program the parser's state machine 139 | PA_add_state "COMMAND" "build" "SUBCOMMAND" "" 140 | PA_add_state "SUBCOMMAND" "buildimage" "END" "" 141 | 142 | # Set up the parser's option callbacks 143 | PA_add_option_callback "build" "BI_process_options" || return 144 | PA_add_option_callback "buildimage" "BI_process_options" || return 145 | 146 | # Set up the parser's usage callbacks 147 | PA_add_usage_callback "build" "BI_usage" || return 148 | PA_add_usage_callback "buildimage" "BI_usage" || return 149 | 150 | # Set up the parser's run callbacks 151 | PA_add_run_callback "buildimage" "BI_run" 152 | } 153 | 154 | # BI_sanity_checks is expected to run some quick and simple checks to 155 | # see if it has all it's key components. For build image this does nothing. 156 | # This function should not be deleted as it is called in main.sh. 157 | # Args: None expected. 158 | _BI_sanity_checks() { 159 | 160 | if [[ -z $(PA_subcommand) ]]; then 161 | BI_usage 162 | exit "${OK}" 163 | fi 164 | } 165 | 166 | # _BI_build_container_image creates the docker build directory in 167 | # dockerbuildtmpdir then calls docker build to build the image. 168 | # Args: No args expected. 169 | _BI_build_container_image() { 170 | 171 | local cmd retval tagname buildargs text buildtype basename 172 | 173 | _BI_create_docker_build_dir || return 174 | 175 | buildargs=$(_BI_get_build_args_for_latest) || return 176 | basename="${_BI[baseimagename]}_$(MA_arch)" 177 | tagname="${_BI[k8sver]}" 178 | 179 | local imgprefix 180 | imgprefix=$(CU_imgprefix) || err || return 181 | if [[ ${_BI[useprebuiltimage]} == "${FALSE}" ]]; then 182 | # Each mok release can build the hardcoded version only 183 | # so reset the tagname to that version 184 | tagname="${K8SVERSION}" 185 | buildtype="create" 186 | cmd="docker build \ 187 | -t "${imgprefix}local/${basename}:${tagname}" \ 188 | --force-rm \ 189 | ${buildargs} \ 190 | ${_BI[dockerbuildtmpdir]}/${_BI[baseimagename]}" 191 | text="Creating" 192 | else 193 | # We can download and run any available version 194 | buildtype="download" 195 | cmd="docker pull docker.io/myownkind/${basename}:${tagname}" 196 | text="Downloading" 197 | fi 198 | 199 | UT_run_with_progress \ 200 | " ${text} base image, '${basename}:${tagname}'" "${cmd}" 201 | 202 | retval=$? 203 | [[ ${retval} -ne ${OK} ]] && { 204 | local runlogfile 205 | runlogfile=$(UT_runlogfile) || err || return 206 | printf 'ERROR: Docker returned an error, shown below\n\n' >"${STDERR}" 207 | cat "${runlogfile}" >"${STDERR}" 208 | printf '\n' >"${STDERR}" 209 | return "${ERROR}" 210 | } 211 | 212 | [[ ${buildtype} == "create" ]] && { 213 | cmd="_BI_modify_container_image" 214 | 215 | UT_run_with_progress \ 216 | " Modifying base image (pulling kubernetes images)" "${cmd}" 217 | 218 | retval=$? 219 | [[ ${retval} -ne ${OK} ]] && { 220 | local runlogfile 221 | runlogfile=$(UT_runlogfile) || err || return 222 | printf 'ERROR: Docker returned an error, shown below\n\n' >"${STDERR}" 223 | cat "${runlogfile}" >"${STDERR}" 224 | printf '\n' >"${STDERR}" 225 | return "${ERROR}" 226 | } 227 | } 228 | 229 | return "${OK}" 230 | } 231 | 232 | # _BI_modify_container_image starts a container suitable for running kubernetes 233 | # components (since the docker/podman build environment isn't suitable), makes 234 | # some modifications and then 'commits' the image. The modifications allow 235 | # `mok create ...` to complete more quickly. 236 | # Args: No args expected. 237 | _BI_modify_container_image() { 238 | 239 | # Delete container, just in case it was left behind 240 | docker stop -t 5 mok-build-modify &>/dev/null 241 | docker rm mok-build-modify &>/dev/null 242 | 243 | CC_set_clustername "build" || err || return 244 | 245 | # Start container 246 | CU_create_container "mok-build-modify" "mok-build-modify" "${K8SVERSION}" || 247 | return 248 | 249 | # Wait for crio to become ready 250 | local counter 251 | printf '\n\n ** WAITING FOR CRIO TO BECOME READY **\n\n' 252 | while ! docker exec mok-build-modify systemctl status crio; do 253 | sleep 1 254 | [[ ${counter} -gt 10 ]] && { 255 | printf '\nERROR: CRI-O did not start after %d tries.\n' "${counter}" 256 | err || return 257 | } 258 | ((counter++)) 259 | done 260 | 261 | # Modify container 262 | docker exec mok-build-modify kubeadm config images pull \ 263 | --kubernetes-version "${K8SVERSION}" || err || return 264 | 265 | # Stop container 266 | docker stop -t 5 "mok-build-modify" 267 | 268 | # Write image 269 | local imgprefix tagname basename 270 | imgprefix=$(CU_imgprefix) || err || return 271 | basename="${_BI[baseimagename]}_$(MA_arch)" 272 | tagname="${K8SVERSION}" 273 | docker commit mok-build-modify "${imgprefix}local/${basename}:${tagname}" || err || return 274 | 275 | # Delete container, just in case it was left behind 276 | docker stop -t 5 mok-build-modify 277 | docker rm mok-build-modify || err || return 278 | 279 | return "${OK}" 280 | } 281 | 282 | # _BI_get_build_args_for_latest sets the buildargs variable that is added 283 | # to the 'podman build ...' command line. Only the latest version is built 284 | # from scratch. Earlier versions are downloaded using '--get-prebuilt-image'. 285 | # Args: None expected 286 | _BI_get_build_args_for_latest() { 287 | 288 | local buildargs 289 | 290 | buildargs="--build-arg GO_VERSION=${GO_VERSION}" 291 | 292 | printf '%s' "${buildargs}" 293 | } 294 | 295 | # _BI_create_docker_build_dir creates a docker build directory in 296 | # /var/tmp/tmp.XXXXXXXX 297 | # Args: None expected 298 | _BI_create_docker_build_dir() { 299 | 300 | _BI[dockerbuildtmpdir]="$(mktemp -d -p /var/tmp)" || { 301 | printf 'ERROR: mktmp failed.\n' >"${STDERR}" 302 | err || return 303 | } 304 | 305 | # The following comments should not be removed or changed. 306 | # embed-dockerfile.sh adds a base64 encoded tarball and 307 | # unpacking code between them. 308 | 309 | #mok-image-tarball-start 310 | #mok-image-tarball-end 311 | } 312 | 313 | # Initialise _BI 314 | _BI_new || exit 1 315 | 316 | # vim helpers ----------------------------------------------------------------- 317 | #include globals.sh 318 | # vim:ft=sh:sw=2:et:ts=2: 319 | -------------------------------------------------------------------------------- /src/deletecluster.sh: -------------------------------------------------------------------------------- 1 | # shellcheck shell=bash disable=SC2148 2 | # DC - Delete Cluster 3 | 4 | # _DC is an associative array that holds data specific to deleting a cluster. 5 | declare -A _DC 6 | 7 | # Declare externally defined variables ---------------------------------------- 8 | 9 | declare OK ERROR STDERR STOP TRUE 10 | 11 | # Getters/Setters ------------------------------------------------------------- 12 | 13 | # DC_set_clustername setter sets the cluster name to be deleted. This is 14 | # called by the parser, via a callback in _DC_new. 15 | DC_set_clustername() { 16 | _DC[clustername]="$1" 17 | } 18 | 19 | # Public Functions ------------------------------------------------------------ 20 | 21 | # DC_process_options checks if arg1 is in a list of valid delete cluster 22 | # options. This function is called by the parser, via a callback in _DC_new. 23 | # Args: arg1 - the option to check. 24 | # arg2 - value of the item to be set, optional 25 | DC_process_options() { 26 | 27 | case "$1" in 28 | -h | --help) 29 | DC_usage 30 | return "${STOP}" 31 | ;; 32 | *) 33 | DC_usage 34 | printf 'ERROR: "%s" is not a valid "build" option.\n' "${1}" \ 35 | >"${STDERR}" 36 | return "${ERROR}" 37 | ;; 38 | esac 39 | } 40 | 41 | # DC_usage outputs help text for the create cluster component. 42 | # It is called by PA_usage(), via a callback in _DC_new. 43 | # Args: None expected. 44 | DC_usage() { 45 | 46 | cat <<'EnD' 47 | DELETE subcommands are: 48 | 49 | cluster - Delete a local kubernetes cluster. 50 | 51 | delete cluster options: 52 | 53 | Format: 54 | delete cluster NAME 55 | NAME - The name of the cluster to delete 56 | 57 | EnD 58 | } 59 | 60 | # DC_run starts the delete cluster process. 61 | # Args: None expected. 62 | DC_run() { 63 | 64 | _DC_sanity_checks || return 65 | 66 | # Mutate functions make system changes. 67 | 68 | declare -i numnodes=0 69 | local id ids r 70 | 71 | numnodes=$(CU_get_cluster_size "${_DC[clustername]}") || return 72 | 73 | [[ ${numnodes} -eq 0 ]] && { 74 | printf '\nERROR: No cluster exists with name, "%s". Aborting.\n\n' \ 75 | "${_DC[clustername]}" >"${STDERR}" 76 | return "${ERROR}" 77 | } 78 | 79 | ids=$(CU_get_cluster_container_ids "${_DC[clustername]}") || return 80 | 81 | printf 'The following containers will be deleted:\n\n' 82 | 83 | GC_set_showheader "${TRUE}" 84 | GC_set_clustername "${_DC[clustername]}" 85 | GC_run "${_DC[clustername]}" || return 86 | 87 | printf "\nAre you sure you want to delete the cluster? (y/N) >" 88 | 89 | read -r ans 90 | 91 | [[ ${ans} != "y" ]] && { 92 | printf '\nCancelling by user request.\n' 93 | return "${OK}" 94 | } 95 | 96 | printf '\n' 97 | 98 | for id in ${ids}; do 99 | UT_run_with_progress \ 100 | " Deleting id, '${id}' from cluster '${_DC[clustername]}'." \ 101 | _DC_delete "${id}" 102 | r=$? 103 | [[ ${r} -ne 0 ]] && { 104 | runlogfile=$(UT_runlogfile) || err || return 105 | cat "${runlogfile}" >"${STDERR}" 106 | printf '\nERROR: Docker failed.\n\n' >"${STDERR}" 107 | err 108 | return "${r}" 109 | } 110 | done 111 | 112 | printf '\n' 113 | } 114 | 115 | # Private Functions ----------------------------------------------------------- 116 | 117 | # _DC_new sets the initial values for the _DC associative array and sets up the 118 | # parser ready for processing the command line arguments, options and usage. 119 | # Args: None expected. 120 | _DC_new() { 121 | _DC[clustername]= 122 | 123 | # Program the parser's state machine 124 | PA_add_state "COMMAND" "delete" "SUBCOMMAND" "" 125 | PA_add_state "SUBCOMMAND" "deletecluster" "ARG1" "" 126 | PA_add_state "ARG1" "deletecluster" "END" "DC_set_clustername" 127 | 128 | # Set up the parser's option callbacks 129 | PA_add_option_callback "delete" "DC_process_options" || return 130 | PA_add_option_callback "deletecluster" "DC_process_options" || return 131 | 132 | # Set up the parser's usage callbacks 133 | PA_add_usage_callback "delete" "DC_usage" || return 134 | PA_add_usage_callback "deletecluster" "DC_usage" || return 135 | 136 | # Set up the parser's run callbacks 137 | PA_add_run_callback "deletecluster" "DC_run" 138 | } 139 | 140 | # DC_sanity_checks is expected to run some quick and simple checks to see if it 141 | # has it's main requirements before DC_run is called. 142 | # Args: None expected. 143 | _DC_sanity_checks() { 144 | 145 | if [[ -z $(PA_subcommand) ]]; then 146 | DC_usage 147 | exit "${OK}" 148 | fi 149 | 150 | if [[ -z ${_DC[clustername]} ]]; then 151 | DC_usage 152 | printf 'Please provide the Cluster NAME to delete.\n' >"${STDERR}" 153 | return "${ERROR}" 154 | fi 155 | } 156 | 157 | # _DC_delete stops and removes a docker container. 158 | # Args: arg1 - docker id to delete. 159 | _DC_delete() { 160 | 161 | docker stop -t 5 "${id}" 162 | docker rm "${id}" || err 163 | } 164 | 165 | # Initialise _DC 166 | _DC_new || exit 1 167 | 168 | # vim helpers ----------------------------------------------------------------- 169 | #include globals.sh 170 | # vim:ft=sh:sw=2:et:ts=2: 171 | -------------------------------------------------------------------------------- /src/embed-dockerfile.sh: -------------------------------------------------------------------------------- 1 | # shellcheck shell=bash disable=SC2148 disable=SC1078 disable=SC1079 disable=SC2026 2 | a=$(tar cz "mok-image$1" | base64 | sed 's/$/\\/') 3 | sed -r '/mok-image-tarball-start/, /mok-image-tarball-end/ c \ 4 | #mok-image-tarball-start \ 5 | cat <<'EnD' | base64 -d | tar xz -C "${_BI[dockerbuildtmpdir]}" \ 6 | '"$a"' 7 | EnD\ 8 | #mok-image-tarball-end' src/buildimage.sh >src/buildimage.deploy 9 | -------------------------------------------------------------------------------- /src/error.sh: -------------------------------------------------------------------------------- 1 | # shellcheck shell=bash disable=SC2148 2 | # ER - Error handling 3 | 4 | # ER is an associative array that holds data specific to error handling. 5 | declare -A ER 6 | 7 | # Declare externally defined associative arrays ------------------------------- 8 | 9 | # Defined in GL (globals.sh) 10 | declare ERROR TRUE STDERR 11 | 12 | # Public Functions ------------------------------------------------------------ 13 | 14 | # ER_err outputs a stacktrace and returns ERROR status. 15 | # Args: None expected. 16 | ER_err() { 17 | 18 | # In case of error print the function call stack 19 | # No args expected 20 | 21 | [[ ${ER[errcalled]} == "${TRUE}" ]] && return "${ERROR}" 22 | ER[errcalled]="${TRUE}" 23 | local frame=0 24 | printf '\n' >"${STDERR}" 25 | while caller "${frame}"; do 26 | ((frame++)) 27 | done | tac >"${STDERR}" 28 | printf '\n' >"${STDERR}" 29 | return "${ERROR}" 30 | } 31 | 32 | # err is a synonym for ER_err to aid with code readability. 33 | err() { 34 | ER_err 35 | return 36 | } 37 | 38 | # Private Functions ----------------------------------------------------------- 39 | 40 | # ER_new sets the initial values for the ER associative array. 41 | # Args: None expected. 42 | _ER_new() { 43 | ER[errcalled]= 44 | } 45 | 46 | # Initialise ER 47 | _ER_new || exit 1 48 | 49 | # vim helpers ----------------------------------------------------------------- 50 | #include globals.sh 51 | # vim:ft=sh:sw=2:et:ts=2: 52 | -------------------------------------------------------------------------------- /src/exec.sh: -------------------------------------------------------------------------------- 1 | # shellcheck shell=bash disable=SC2148 2 | # EX - EXec 3 | 4 | # EX is an associative array that holds data specific to the exec cluster command. 5 | declare -A _EX 6 | 7 | # Declare externally defined variables ---------------------------------------- 8 | 9 | declare OK ERROR STDERR TRUE STOP 10 | 11 | # Getters/Setters ------------------------------------------------------------- 12 | 13 | # EX_set_containername setter sets the containername to be used to 'log in' to. 14 | EX_set_containername() { 15 | _EX[containername]="$1" 16 | } 17 | 18 | # Public Functions ------------------------------------------------------------ 19 | 20 | # EX_process_options checks if arg1 is in a list of valid exec options. This 21 | # function is called by the parser, via a callback in _EX_new. 22 | # Args: arg1 - the option to check. 23 | # arg2 - value of the item to be set, optional 24 | EX_process_options() { 25 | 26 | case "$1" in 27 | -h | --help) 28 | EX_usage 29 | return "${STOP}" 30 | ;; 31 | *) 32 | EX_usage 33 | printf 'ERROR: "%s" is not a valid "exec" option.\n' "${1}" \ 34 | >"${STDERR}" 35 | return "${ERROR}" 36 | ;; 37 | esac 38 | } 39 | 40 | # EX_usage outputs help text for the create exec component. It is called in 41 | # this file and by PA_usage(), via a callback in _EX_new. 42 | # Args: None expected. 43 | EX_usage() { 44 | 45 | cat <<'EnD' 46 | 'Log in' to a container. EXEC has no subcommands. 47 | 48 | exec options: 49 | 50 | Format: 51 | exec [NAME] 52 | NAME - (optional) The name of the container to 'log in' to. 53 | If NAME is not given then the user will be offered 54 | a choice of containers to log in to. If there is only 55 | one cluster and one node then it will 'log in' to the 56 | only available container. 57 | 58 | EnD 59 | } 60 | 61 | # Execs 'bash' in the container referenced by _EX[containername]. If just 62 | # 'mok exec' is called without options then the user is offered a selection 63 | # of existing clusters to exec into. 64 | # Args: None expected. 65 | # --------------------------------------------------------------------------- 66 | EX_run() { 67 | 68 | _EX_sanity_checks || return 69 | 70 | local names int ans lines containernames gcoutput 71 | 72 | GC_set_showheader "${TRUE}" || err || return 73 | gcoutput=$(GC_run) || return 74 | names=$(printf '%s' "${gcoutput}" | awk '{ print $3; }') 75 | readarray -t containernames <<<"${names}" 76 | 77 | if [[ -n ${_EX[containername]} ]]; then 78 | 79 | # The caller gave a specific name for exec. 80 | # Check if the container name exists 81 | 82 | if grep -qs "^${_EX[containername]}$" <<<"${names}"; then 83 | _EX_exec "${_EX[containername]}" || return 84 | return "${OK}" 85 | else 86 | printf 'ERROR: Cannot exec into non-existent container: "%s".\n' \ 87 | "${_EX[containername]}" 88 | return "${ERROR}" 89 | fi 90 | 91 | elif [[ ${#containernames[*]} == 2 ]]; then 92 | 93 | # If there's only one container just log into it without asking 94 | _EX_exec "${containernames[1]}" 95 | 96 | elif [[ ${#containernames[*]} == 1 ]]; then 97 | 98 | printf 'No containers found.\n' 99 | return "${OK}" 100 | 101 | else 102 | 103 | # The caller supplied no container name. 104 | # Present some choices 105 | 106 | printf 'Choose the container to log in to:\n\n' 107 | 108 | readarray -t lines <<<"${gcoutput}" || return 109 | 110 | # Print the header then print the items in the loop 111 | printf ' %s\n' "${lines[0]}" 112 | for int in $(seq 1 $((${#lines[*]} - 1))); do 113 | printf '%00d) %s\n' "${int}" "${lines[int]}" 114 | done | sort -k 4 115 | 116 | printf "\nChoose a number (Enter to cancel)> " 117 | read -r ans 118 | 119 | [[ -z ${ans} || ${ans} -lt 0 || ${ans} -gt $((${#lines[*]} - 1)) ]] && { 120 | printf '\nInvalid choice. Aborting.\n' 121 | return "${OK}" 122 | } 123 | 124 | _EX_exec "${containernames[ans]}" 125 | fi 126 | } 127 | 128 | # Private Functions ----------------------------------------------------------- 129 | 130 | # _EX_new sets the initial values for the _EX associative array and sets up the 131 | # parser ready for processing the command line arguments, options and usage. 132 | _EX_new() { 133 | _EX[containername]= 134 | 135 | # Program the parser's state machine 136 | PA_add_state "COMMAND" "exec" "ARG1" "" 137 | PA_add_state "ARG1" "exec" "END" "EX_set_containername" 138 | 139 | # Set up the parser's option callbacks 140 | PA_add_option_callback "exec" "EX_process_options" || return 141 | PA_add_usage_callback "exec" "EX_usage" || return 142 | 143 | # Set up the parser's run callbacks 144 | PA_add_run_callback "exec" "EX_run" 145 | } 146 | 147 | # EX_sanity_checks is expected to run some quick and simple checks to 148 | # see if it has all it's key components. For exec this does nothing. 149 | # Args: None expected. 150 | _EX_sanity_checks() { :; } 151 | 152 | # _EX_exec runs a command in a docker container using bash as the command by 153 | # default. 154 | # Args: arg1 - docker container name 155 | # arg2 - command to run 156 | _EX_exec() { 157 | 158 | local cmd=${2:-bash} url port 159 | 160 | containerrt=$(CU_containerrt) || err || return 161 | podmantype=$(CU_podmantype) || err || return 162 | 163 | if [[ $containerrt == "podman" && $podmantype == "machine" ]]; then 164 | # TODO: A named podman machine does not work the same way 165 | # as the default podman machine. Rootful doesn't work 166 | # for a named machine so need to use '--url'. Ask 167 | # Podman developers about this. 168 | port=$(podman machine inspect mok-machine | grep Port | grep -o '[0-9]\+') 169 | url="ssh://root@127.0.0.1:$port/run/podman/podman.sock" 170 | # TODO: This would be preferred to using '--url' 171 | # podman -c mok-machine exec -ti "$1" "${cmd}" 172 | read -rt 0.1 173 | podman --url "${url}" exec -ti "$1" "${cmd}" 174 | elif [[ $containerrt == "podman" ]]; then 175 | read -rt 0.1 176 | exec podman exec -ti "$1" "${cmd}" 177 | elif [[ $containerrt == "docker" ]]; then 178 | read -rt 0.1 179 | exec docker exec -ti "$1" "${cmd}" 180 | fi 181 | } 182 | 183 | # Initialise EX 184 | _EX_new || exit 1 185 | 186 | # vim helpers ----------------------------------------------------------------- 187 | #include globals.sh 188 | # vim:ft=sh:sw=2:et:ts=2: 189 | -------------------------------------------------------------------------------- /src/getcluster.sh: -------------------------------------------------------------------------------- 1 | # shellcheck shell=bash disable=SC2148 2 | # GC - Get Cluster 3 | 4 | # _GC is an associative array that holds data specific to the get cluster command. 5 | declare -A _GC 6 | 7 | # Declare externally defined variables ---------------------------------------- 8 | 9 | declare OK ERROR STDERR TRUE STOP 10 | 11 | # Getters/Setters ------------------------------------------------------------- 12 | 13 | # GC_showheader getter outputs the value of the showheader array member. 14 | GC_showheader() { 15 | printf '%s' "${_GC[showheader]}" 16 | } 17 | 18 | # GC_set_showheader setter sets the value of the showheader array member. 19 | # If $TRUE the header will be shown, or $FALSE, then it won't be shown. 20 | GC_set_showheader() { 21 | _GC[showheader]="$1" 22 | } 23 | 24 | # GC_set_clustername setter sets the cluster name to show information about. 25 | GC_set_clustername() { 26 | _GC[clustername]="$1" 27 | } 28 | 29 | # Public Functions ------------------------------------------------------------ 30 | 31 | # GC_cleanup removes artifacts that were created during execution. Currently 32 | # this does nothing and this function could be deleted. It is called by 33 | # MA_cleanup. 34 | GC_cleanup() { 35 | : 36 | } 37 | 38 | # GC_process_options checks if arg1 is in a list of valid get cluster 39 | # options. This function is called by the parser. 40 | # Args: arg1 - the option to check. 41 | # arg2 - value of the item to be set, optional 42 | GC_process_options() { 43 | 44 | case "$1" in 45 | -h | --help) 46 | GC_usage 47 | return "${STOP}" 48 | ;; 49 | *) 50 | GC_usage 51 | printf 'ERROR: "%s" is not a valid "build" option.\n' "${1}" \ 52 | >"${STDERR}" 53 | return "${ERROR}" 54 | ;; 55 | esac 56 | } 57 | 58 | # GC_usage outputs help text for the create cluster component. 59 | # It is called by PA_usage(). 60 | # Args: None expected. 61 | GC_usage() { 62 | 63 | cat <<'EnD' 64 | GET subcommands are: 65 | 66 | cluster[s] - list all mok managed clusters. 67 | 68 | get cluster[s] options: 69 | 70 | Format: 71 | get cluster(s) [NAME] 72 | NAME - (optional) The name of the cluster to get 73 | details about. 74 | 75 | EnD 76 | } 77 | 78 | # GC_run gets information about cluster(s). 79 | # This function is called in main.sh. 80 | # Args: None expected. 81 | GC_run() { 82 | 83 | _GC_sanity_checks || return 84 | 85 | local ids id info output labelkey 86 | local containerhostname containerip 87 | 88 | declare -a nodes 89 | 90 | ids=$(CU_get_cluster_container_ids "${_GC[clustername]}") || return 91 | 92 | if [[ -z ${ids} ]]; then 93 | printf 'No clusters found\n' 94 | return "${OK}" 95 | fi 96 | 97 | readarray -t nodes <<<"${ids}" 98 | 99 | # Use 'docker inspect' to get the value of the label $LABELKEY 100 | 101 | output=$(mktemp -p /var/tmp) || { 102 | printf 'ERROR: mktmp failed.\n' >"${STDERR}" 103 | return "${ERROR}" 104 | } 105 | 106 | # Output the header 107 | [[ ${_GC[showheader]} -eq ${TRUE} ]] && { 108 | printf 'MOK_Cluster Docker_ID Container_Name IP_Address\n' >"${output}" 109 | } 110 | 111 | labelkey=$(CU_labelkey) || err || return 112 | 113 | for id in "${nodes[@]}"; do 114 | 115 | info=$(CU_get_container_info "${id}") || return 116 | 117 | clustname=$(JSONPath ".[0].Config.Labels.${labelkey}" -b <<<"${info}") || 118 | err || return 119 | 120 | containerhostname=$(JSONPath '.[0].Config.Hostname' -b <<<"${info}") || 121 | err || return 122 | 123 | containerip=$(JSONPath '.[0].NetworkSettings.Networks.*.IPAddress' -b <<<"${info}") || 124 | err || return 125 | 126 | printf '%s %s %s %s\n' "${clustname}" "${id}" "${containerhostname}" "${containerip}" 127 | 128 | done | sort -k 3 >>"${output}" 129 | 130 | column -t "${output}" || err 131 | } 132 | 133 | # Private Functions ----------------------------------------------------------- 134 | 135 | # _GC_new sets the initial values for the _GC associative array. 136 | # Args: None expected. 137 | _GC_new() { 138 | _GC[showheader]="${TRUE}" 139 | 140 | # Program the parser's state machine 141 | PA_add_state "COMMAND" "get" "SUBCOMMAND" "" 142 | PA_add_state "SUBCOMMAND" "getcluster" "ARG1" "" 143 | PA_add_state "SUBCOMMAND" "getclusters" "ARG1" "" 144 | PA_add_state "ARG1" "getcluster" "END" "GC_set_clustername" 145 | PA_add_state "ARG1" "getclusters" "END" "GC_set_clustername" 146 | 147 | # Set up the parser's option callbacks 148 | PA_add_option_callback "get" "GC_process_options" || return 149 | PA_add_option_callback "getcluster" "GC_process_options" || return 150 | PA_add_option_callback "getclusters" "GC_process_options" || return 151 | 152 | # Set up the parser's usage callbacks 153 | PA_add_usage_callback "get" "GC_usage" || return 154 | PA_add_usage_callback "getcluster" "GC_usage" || return 155 | PA_add_usage_callback "getclusters" "GC_usage" || return 156 | 157 | # Set up the parser's run callbacks 158 | PA_add_run_callback "getcluster" "GC_run" 159 | PA_add_run_callback "getclusters" "GC_run" 160 | } 161 | 162 | # GC_sanity_checks is expected to run some quick and simple checks to 163 | # see if it has all it's key components. For build image this does nothing. 164 | # This function should not be deleted as it is called in main.sh. 165 | # Args: None expected. 166 | _GC_sanity_checks() { 167 | 168 | # Only show the usage if we are in 'get' mode and have not specified a 169 | # subcommand. This allows internal code to call GC_run() 170 | if [[ $(PA_command) == "get" && -z $(PA_subcommand) ]]; then 171 | GC_usage 172 | exit "${OK}" 173 | fi 174 | } 175 | 176 | # Initialise _GC 177 | _GC_new || exit 1 178 | 179 | # vim helpers ----------------------------------------------------------------- 180 | #include globals.sh 181 | # vim:ft=sh:sw=2:et:ts=2: 182 | -------------------------------------------------------------------------------- /src/globals.sh: -------------------------------------------------------------------------------- 1 | # shellcheck shell=bash disable=SC2148 2 | # GL - Globals 3 | 4 | # Private Functions ----------------------------------------------------------- 5 | 6 | # GL_new sets the only global variables that should be in use anywhere. The 7 | # only other globals are the associative arrays. All the globals are constant 8 | # (readonly) variables. Declare these globals, where needed, in other files 9 | # so that shellcheck passes. 10 | # Args: None expected. 11 | _GL_new() { 12 | 13 | declare -rg MOKVERSION="0.8.30" 14 | declare -rg K8SVERSION="1.33.1" 15 | declare -rg GO_VERSION="1.23.4" 16 | 17 | # Returns, exit codes 18 | declare -rg OK=0 19 | declare -rg SUCCESS=0 20 | declare -rg ERROR=1 21 | declare -rg FAILURE=1 22 | 23 | # For setting flags 24 | declare -rg TRUE=1 25 | declare -rg FALSE=0 26 | 27 | # To signal main() to exit with SUCCESS in the PArser 28 | declare -rg STOP=2 29 | 30 | declare -rg STDOUT="/dev/stdout" 31 | declare -rg STDERR="/dev/stderr" 32 | 33 | # The following just keep shellcheck happy 34 | local dummy dumb 35 | dummy="${OK}" 36 | dummy="${ERROR}" 37 | dummy="${STOP}" 38 | dummy="${STDOUT}" 39 | dummy="${STDERR}" 40 | dummy="${TRUE}" 41 | dummy="${FALSE}" 42 | dummy="${SUCCESS}" 43 | dummy="${FAILURE}" 44 | dummy="${MOKVERSION}" 45 | dummy="${GO_VERSION}" 46 | dummy="${K8SVERSION}" 47 | dumb="${dummy}${dumb}" 48 | } 49 | 50 | # Initialise GL 51 | _GL_new || exit 1 52 | 53 | # vim helpers ----------------------------------------------------------------- 54 | 55 | # The following lines allow the use of '[C-i' and '[I' (do ':help [I') in vim. 56 | #include buildimage.sh 57 | #include containerutils.sh 58 | #include createcluster.sh 59 | #include deletecluster.sh 60 | #include embed-dockerfile.sh 61 | #include error.sh 62 | #include exec.sh 63 | #include getcluster.sh 64 | #include main.sh 65 | #include lib/parser.sh 66 | #include util.sh 67 | #include versions.sh 68 | 69 | # vim:ft=sh:sw=2:et:ts=2: 70 | -------------------------------------------------------------------------------- /src/lib/parser.sh: -------------------------------------------------------------------------------- 1 | # shellcheck shell=bash disable=SC2148 2 | # PA - PArser 3 | 4 | # _PA holds data specific to parsing the command line arguments. 5 | declare -A _PA 6 | 7 | # Declare externally defined variables ---------------------------------------- 8 | 9 | # Defined in ER (globals.sh) 10 | declare OK ERROR STDERR 11 | 12 | # Getters/Setters ------------------------------------------------------------- 13 | 14 | # PA_command getter outputs the _PA[command] array member. This contains the 15 | # command the user requested. 16 | PA_command() { 17 | printf '%s' "${_PA[command]}" 18 | } 19 | 20 | # PA_subcommand getter outputs the _PA[subcommand] array member. This contains 21 | # the subcommand the user requested. 22 | PA_subcommand() { 23 | printf '%s' "${_PA[subcommand]}" 24 | } 25 | 26 | # PA_shift outputs the _PA[shift] array member, returned by the caller 27 | # when an extra shift is required whilst consuming option values. 28 | # 29 | # Example code: 30 | # ``` 31 | #XX_process_options_callback() { 32 | # 33 | # case "$1" in 34 | # -h | --help) 35 | # CC_usage 36 | # return "${STOP}" 37 | # ;; 38 | # ... omitted ... 39 | # --k8sver) 40 | # _CC[k8sver]="$2" 41 | # return "$(PA_shift)" 42 | # ;; 43 | # --with-lb) 44 | # ... omitted .. 45 | # ``` 46 | PA_shift() { 47 | printf '%s' "${_PA[shift]}" 48 | } 49 | 50 | # PA_set_state setter sets the initial state of the parser, which should be one 51 | # of COMMAND, SUBCOMMAND, or ARG1. 52 | # Args: arg1 - the initial state to set. 53 | PA_set_state() { 54 | _PA[state]="$1" 55 | } 56 | 57 | # Public Functions ------------------------------------------------------------ 58 | 59 | # PA_add_option_callback adds a callback to the list of callbacks used for 60 | # processing options. 61 | # Args: arg1 - Null string (for global options), COMMAND or COMMANDSUBCOMMAND. 62 | # arg2 - The function to call. 63 | PA_add_option_callback() { 64 | _PA[optscallbacks]+="$1,$2 " 65 | } 66 | 67 | # PA_add_usage_callback adds a callback to the list of callbacks used for 68 | # output of help text. 69 | # Args: arg1 - Null string (for global help), COMMAND or COMMANDSUBCOMMAND. 70 | # arg2 - The function to call. 71 | PA_add_usage_callback() { 72 | _PA[usagecallbacks]+="$1,$2 " 73 | } 74 | 75 | # PA_add_state adds a callback to the list of callbacks used for 76 | # programming the state machine. 77 | # Args: arg1 - Current state to match. 78 | # arg2 - The value of the state to match. 79 | # arg3 - The new state if arg1 and arg2 match. 80 | # arg4 - The function to call, optional. 81 | PA_add_state() { 82 | _PA[statecallbacks]+="$1,$2,$3,$4 " 83 | } 84 | 85 | # PA_add_run_callback adds a callback to the list of callbacks used for 86 | # running the user defined entrypoint function. 87 | # Args: arg1 - The value of the state to match. Probably like: 88 | # "commandsubcommand" "function" 89 | # arg2 - The function to call. 90 | PA_add_run_callback() { 91 | _PA[runcallbacks]+="$1,$2 " 92 | } 93 | 94 | # PA_run implements an interleaved state machine to process the 95 | # user request. It allows for strict checking of arguments and args. All 96 | # command line arguments are processed in order from left to right. 97 | # 98 | # Each COMMAND can have a different set of requirements which are controlled 99 | # by setting the next state at each transition. 100 | # 101 | # --global-options COMMAND --command-options SUBCOMMAND --subcommand-options \ 102 | # ARG1 --subcommand-options ARG2 --subcommand-options ... 103 | # 104 | # --global-options are those before COMMAND. 105 | # --command-options can be after the COMMAND but before SUBCOMMAND. 106 | # --subcommand-options can be anywhere after the SUBCOMMAND. 107 | # 108 | # Args: arg1-N - The arguments given to mok by the user on the command line 109 | PA_run() { 110 | 111 | local retval=${OK} 112 | 113 | set -- "$@" 114 | local ARGN=$# ARGNUM=0 retval=0 115 | while [ "${ARGN}" -ne 0 ]; do 116 | case "$1" in 117 | --* | -*) 118 | _PA_process_option "$1" "$2" || { 119 | retval=$? 120 | if [[ ${retval} == "${_PA[shift]}" ]]; then 121 | ARGN=$((ARGN - 1)) 122 | shift 123 | else 124 | return "${retval}" 125 | fi 126 | } 127 | ;; 128 | *) 129 | case "${_PA[state]}" in 130 | COMMAND) 131 | _PA_check_token "${1}" "COMMAND" "command" 132 | [[ $? -eq ${ERROR} ]] && { 133 | PA_usage 134 | printf 'Invalid COMMAND, "%s".\n\n' "$1" >"${STDERR}" 135 | return "${ERROR}" 136 | } 137 | ;; 138 | SUBCOMMAND) 139 | _PA_check_token "$1" "SUBCOMMAND" "subcommand" 140 | [[ $? -eq ${ERROR} ]] && { 141 | PA_usage 142 | printf 'Invalid SUBCOMMAND for %s, "%s".\n\n' "${_PA[command]}" "${1}" \ 143 | >"${STDERR}" 144 | return "${ERROR}" 145 | } 146 | ;; 147 | ARG*) 148 | ((ARGNUM++)) 149 | _PA_check_token "${1}" "ARG${ARGNUM}" 150 | [[ $? -eq ${ERROR} ]] && { 151 | PA_usage 152 | printf 'Invalid ARG1 for %s %s, "%s".\n\n' "${_PA[command]}" \ 153 | "${_PA[subcommand]}" "${1}" >"${STDERR}" 154 | return "${ERROR}" 155 | } 156 | ;; 157 | END) 158 | PA_usage 159 | printf 'ERROR No more args expected, "%s" is unexpected for "%s %s"\n' \ 160 | "${1}" "${_PA[command]}" "${_PA[subcommand]}" >"${STDERR}" 161 | return "${ERROR}" 162 | ;; 163 | *) 164 | printf 'Internal ERROR. Invalid state "%s"\n' "${_PA[state]}" >"${STDERR}" 165 | return "${ERROR}" 166 | ;; 167 | esac 168 | ;; 169 | esac 170 | shift 1 171 | ARGN=$((ARGN - 1)) 172 | done 173 | 174 | # Run the user defined entrypoint function: 175 | for item in ${_PA[runcallbacks]}; do 176 | IFS=, read -r stateval func <<<"${item}" 177 | [[ ${stateval} == "${_PA[command]}${_PA[subcommand]}" ]] && { 178 | eval "${func}" || retval=$? 179 | return "${retval}" 180 | } 181 | done 182 | 183 | PA_usage 184 | 185 | return "${OK}" 186 | } 187 | 188 | # PA_usage outputs help text for a single component if help was asked for when 189 | # a command was specified, or for all components otherwise. 190 | # Args: None expected. 191 | PA_usage() { 192 | 193 | curcmdsubcmd="${_PA[command]}${_PA[subcommand]}" 194 | 195 | for item in ${_PA[usagecallbacks]}; do 196 | IFS=, read -r cmdsubcmd func <<<"${item}" 197 | [[ ${curcmdsubcmd} == "${cmdsubcmd}" ]] && { 198 | eval "${func}" 199 | return 200 | } 201 | done 202 | 203 | eval "${_PA[usage]}" 204 | } 205 | 206 | # Private Functions ----------------------------------------------------------- 207 | 208 | # PA_new sets the initial values for the PArser's associative array. 209 | # Args: None expected. 210 | _PA_new() { 211 | _PA[command]= 212 | _PA[subcommand]= 213 | _PA[state]="COMMAND" 214 | _PA[optscallbacks]= 215 | _PA[usagecallbacks]= 216 | _PA[statecallbacks]= 217 | _PA[runcallbacks]= 218 | # The return value if the caller asked for an extra shift: 219 | _PA[shift]=126 220 | } 221 | 222 | # _PA_check_token checks for a valid token in arg2 state. The logic is 223 | # most easily understood by reading the full original version at: 224 | # https://github.com/mclarkson/my-own-kind/blob/master/docs/package.md#scripted-cluster-creation-and-deletion 225 | # This function is a reduction of all the check_xxx_token functions. 226 | # Args: arg1 - the token to check. 227 | # arg2 - the current state. 228 | # arg3 - the state value to set, optional. This should only be sent 229 | # for command and subcommand states. 230 | _PA_check_token() { 231 | 232 | local item 233 | 234 | if [[ -n ${_PA[subcommand]} ]]; then 235 | cmdsubcommand="${_PA[command]}${_PA[subcommand]}" 236 | elif [[ -n ${_PA[command]} && ${_PA[state]} != "ARG"* ]]; then 237 | cmdsubcommand="${_PA[command]}$1" 238 | elif [[ -n ${_PA[command]} && ${_PA[state]} == "ARG"* ]]; then 239 | cmdsubcommand="${_PA[command]}" 240 | elif [[ -z ${_PA[command]} && ${_PA[state]} == "ARG"* ]]; then 241 | cmdsubcommand= 242 | else 243 | cmdsubcommand="$1" 244 | fi 245 | 246 | for item in ${_PA[statecallbacks]}; do 247 | IFS=, read -r state component newstate func <<<"${item}" 248 | [[ ${state} == "$2" ]] && { 249 | [[ ${component} == "${cmdsubcommand}" ]] && { 250 | [[ -n $3 ]] && _PA["$3"]="$1" 251 | _PA[state]="${newstate}" 252 | [[ -n ${func} ]] && { 253 | eval "${func} $1" || return 254 | } 255 | return "${OK}" 256 | } 257 | } 258 | done 259 | } 260 | 261 | # _PA_process_option checks that the user-provided option is valid for the 262 | # command-subcommand or global states. 263 | # Args: arg1 - The option to check. 264 | # arg2 - TODO The value of the option if present, optional. 265 | _PA_process_option() { 266 | 267 | local item curcmdsubcmd 268 | 269 | curcmdsubcmd="${_PA[command]}${_PA[subcommand]}" 270 | 271 | for item in ${_PA[optscallbacks]}; do 272 | IFS=, read -r cmdsubcmd func <<<"${item}" 273 | [[ ${curcmdsubcmd} == "${cmdsubcmd}" ]] && { 274 | eval "${func} \"$1\" \"$2\"" 275 | return $? 276 | } 277 | done 278 | 279 | return "${ERROR}" 280 | } 281 | 282 | # Initialise _PA 283 | _PA_new 284 | 285 | # vim helpers ----------------------------------------------------------------- 286 | #include globals.sh 287 | # vim:ft=sh:sw=2:et:ts=2: 288 | -------------------------------------------------------------------------------- /src/machine.sh: -------------------------------------------------------------------------------- 1 | # shellcheck shell=bash disable=SC2148 2 | # MU - Machine Utilities for Podman 3 | 4 | # _MU is an associative array that holds data specific to machine utils. 5 | declare -A _MU 6 | 7 | # Declare externally defined variables ---------------------------------------- 8 | 9 | declare ERROR STDERR STOP 10 | 11 | # Getters/Setters ------------------------------------------------------------- 12 | 13 | # Public Functions ------------------------------------------------------------ 14 | 15 | # MU_cleanup removes artifacts that were created during execution. This function 16 | # is run automatically by the Parser library. Currently this does nothing and 17 | # this function could be deleted. 18 | MU_cleanup() { :; } 19 | 20 | # MU_process_options checks if arg1 is in a list of valid machine utility 21 | # options. This function is called by the parser. 22 | # Args: arg1 - the option to check. 23 | # arg2 - value of the option to be set, optional. This depends on the 24 | # type of option. 25 | MU_process_options() { 26 | case "$1" in 27 | -h | --help) 28 | MU_usage 29 | return "${STOP}" 30 | ;; 31 | *) 32 | MU_usage 33 | printf 'ERROR: "%s" is not a valid "create" option.\n' "${1}" \ 34 | >"${STDERR}" 35 | return "${ERROR}" 36 | ;; 37 | esac 38 | } 39 | 40 | # MU_usage outputs help text for the machine utilities component. 41 | # Args: None expected. 42 | MU_usage() { 43 | 44 | cat <<'EnD' 45 | MACHINE subcommands are: 46 | 47 | list - List the currently created Podman machines. Only one machine 48 | is currently supported and it will be named 'mok-machine'. 49 | create - Create a Podman machine named 'mok-machine'. This will also 50 | setup the Podman machine to run Kubernetes so there is no 51 | need to run 'machine setup'. 52 | destroy - Completely removes the Podman machine named 'mok-machine'. 53 | start - Start the Podman machine named 'mok-machine'. 54 | stop - Stop the Podman machine named 'mok-machine'. 55 | setup - Apply the correct settings to the Podman machine to run 56 | Kubernetes. 57 | 58 | For more information: 59 | 60 | machine list -h 61 | machine create -h 62 | machine destroy -h 63 | machine start -h 64 | machine stop -h 65 | machine setup -h 66 | 67 | EnD 68 | } 69 | 70 | # Private Functions ----------------------------------------------------------- 71 | 72 | # _MU_new sets the initial values for the Machine Utils associative array and 73 | # sets up the Parser to call functions in machine*.sh files. 74 | # Args: None expected. 75 | _MU_new() { 76 | # Program the parser's state machine 77 | PA_add_state "COMMAND" "machine" "SUBCOMMAND" "" 78 | PA_add_state "SUBCOMMAND" "machinelist" "END" "" 79 | PA_add_state "SUBCOMMAND" "machinecreate" "END" "" 80 | PA_add_state "SUBCOMMAND" "machinedestroy" "END" "" 81 | PA_add_state "SUBCOMMAND" "machinesetup" "END" "" 82 | PA_add_state "SUBCOMMAND" "machinestart" "END" "" 83 | PA_add_state "SUBCOMMAND" "machinestop" "END" "" 84 | 85 | # Set up the parser's option callbacks 86 | PA_add_option_callback "machine" "MU_process_options" || return 87 | PA_add_option_callback "machinelist" "MU_list_process_options" || return 88 | PA_add_option_callback "machinecreate" "MU_create_process_options" || return 89 | PA_add_option_callback "machinedestroy" "MU_destroy_process_options" || return 90 | PA_add_option_callback "machinesetup" "MU_setup_process_options" || return 91 | PA_add_option_callback "machinestart" "MU_start_process_options" || return 92 | PA_add_option_callback "machinestop" "MU_stop_process_options" || return 93 | 94 | # Set up the parser's usage callbacks 95 | PA_add_usage_callback "machine" "MU_usage" || return 96 | PA_add_usage_callback "machinelist" "MU_list_usage" || return 97 | PA_add_usage_callback "machinecreate" "MU_create_usage" || return 98 | PA_add_usage_callback "machinedestroy" "MU_destroy_usage" || return 99 | PA_add_usage_callback "machinesetup" "MU_setup_usage" || return 100 | PA_add_usage_callback "machinestart" "MU_start_usage" || return 101 | PA_add_usage_callback "machinestop" "MU_stop_usage" || return 102 | 103 | # Set up the parser's run callbacks 104 | PA_add_run_callback "machinelist" "MU_list_run" 105 | PA_add_run_callback "machinecreate" "MU_create_run" 106 | PA_add_run_callback "machinedestroy" "MU_destroy_run" 107 | PA_add_run_callback "machinesetup" "MU_setup_run" 108 | PA_add_run_callback "machinestart" "MU_start_run" 109 | PA_add_run_callback "machinestop" "MU_stop_run" 110 | } 111 | 112 | # Initialise _MU 113 | _MU_new || exit 1 114 | 115 | # vim helpers ----------------------------------------------------------------- 116 | #include globals.sh 117 | # vim:ft=sh:sw=2:et:ts=2: 118 | -------------------------------------------------------------------------------- /src/machinecreate.sh: -------------------------------------------------------------------------------- 1 | # shellcheck shell=bash disable=SC2148 2 | 3 | # MACHINE CREATE ============================================================== 4 | 5 | # MU_create_process_options checks if arg1 is in a list of valid machine utility 6 | # options. This function is called by the parser. 7 | # Args: arg1 - the option to check. 8 | # arg2 - value of the option to be set, optional. This depends on the 9 | # type of option. 10 | MU_create_process_options() { 11 | case "$1" in 12 | -h | --help) 13 | MU_create_usage 14 | return "${STOP}" 15 | ;; 16 | -v | --verbose) 17 | UT_set_tailf "${TRUE}" || err || return 18 | ;; 19 | *) 20 | MU_create_usage 21 | printf 'ERROR: "%s" is not a valid "create" option.\n' "${1}" \ 22 | >"${STDERR}" 23 | return "${ERROR}" 24 | ;; 25 | esac 26 | } 27 | 28 | # MU_create_usage outputs help text for the machine utilities component. 29 | # Args: None expected. 30 | MU_create_usage() { 31 | 32 | cat <"${STDERR}" 58 | return "${ERROR}" 59 | } 60 | 61 | UT_run_with_progress \ 62 | " Loading module in mok-machine" \ 63 | podman machine ssh --username root mok-machine modprobe nf_conntrack 64 | r=$? 65 | [[ ${r} -ne 0 ]] && { 66 | runlogfile=$(UT_runlogfile) || err || return 67 | cat "${runlogfile}" >"${STDERR}" 68 | return "${ERROR}" 69 | } 70 | 71 | UT_run_with_progress \ 72 | " Persisting nf_conntrack module in mok-machine" \ 73 | podman machine ssh --username root mok-machine \ 74 | bash -c \ 75 | "\"\\\"echo nf_conntrack \ 76 | >/etc/modules-load.d/nf_conntrack.conf\\\"\"" 77 | r=$? 78 | [[ ${r} -ne 0 ]] && { 79 | runlogfile=$(UT_runlogfile) || err || return 80 | cat "${runlogfile}" >"${STDERR}" 81 | return "${ERROR}" 82 | } 83 | 84 | UT_run_with_progress \ 85 | " Setting nf_conntrack_max in mok-machine" \ 86 | podman machine ssh --username root mok-machine sysctl -q \ 87 | -w net.netfilter.nf_conntrack_max=163840 88 | r=$? 89 | [[ ${r} -ne 0 ]] && { 90 | runlogfile=$(UT_runlogfile) || err || return 91 | cat "${runlogfile}" >"${STDERR}" 92 | return "${ERROR}" 93 | } 94 | 95 | UT_run_with_progress \ 96 | " Persisting nf_conntrack_max in mok-machine" \ 97 | podman machine ssh --username root mok-machine \ 98 | bash -c \ 99 | "\"\\\"echo net.netfilter.nf_conntrack_max=163840 \ 100 | >/etc/sysctl.d/99-nf_conntrack_max.conf\\\"\"" 101 | r=$? 102 | [[ ${r} -ne 0 ]] && { 103 | runlogfile=$(UT_runlogfile) || err || return 104 | cat "${runlogfile}" >"${STDERR}" 105 | return "${ERROR}" 106 | } 107 | } 108 | 109 | -------------------------------------------------------------------------------- /src/machinedestroy.sh: -------------------------------------------------------------------------------- 1 | # shellcheck shell=bash disable=SC2148 2 | 3 | # MACHINE DESTROY ============================================================== 4 | 5 | # MU_destroy_process_options checks if arg1 is in a list of valid machine utility 6 | # options. This function is called by the parser. 7 | # Args: arg1 - the option to check. 8 | # arg2 - value of the option to be set, optional. This depends on the 9 | # type of option. 10 | MU_destroy_process_options() { 11 | case "$1" in 12 | -h | --help) 13 | MU_destroy_usage 14 | return "${STOP}" 15 | ;; 16 | -v | --verbose) 17 | UT_set_tailf "${TRUE}" || err || return 18 | ;; 19 | *) 20 | MU_destroy_usage 21 | printf 'ERROR: "%s" is not a valid "create" option.\n' "${1}" \ 22 | >"${STDERR}" 23 | return "${ERROR}" 24 | ;; 25 | esac 26 | } 27 | 28 | # MU_destroy_usage outputs help text for the machine utilities component. 29 | # Args: None expected. 30 | MU_destroy_usage() { 31 | 32 | cat <<'EnD' 33 | machine destroy options: 34 | 35 | Format: 36 | machine destroy [flags] 37 | 38 | Flags: 39 | -h, --help - This help text. 40 | -v, --verbose - Verbose output. Useful if there was a problem. 41 | 42 | Description: 43 | This will completely remove a Podman machine named 'mok-machine'. 44 | A machine must be stopped before it can be destroyed. 45 | 46 | EnD 47 | } 48 | 49 | # MU_destroy_run deletes a Podman machine. 50 | # Args: None expected. 51 | MU_destroy_run() { 52 | printf 'This will completely remove the podman machine named "mok-machine"\n' 53 | printf "Are you sure you want to destroy the machine? (y/N) >" 54 | read -r ans 55 | [[ ${ans} != "y" ]] && { 56 | printf 'Cancelling by user request.\n' 57 | return "${OK}" 58 | } 59 | UT_run_with_progress \ 60 | " Deleting the Podman machine: mok-machine" \ 61 | podman machine rm -f mok-machine 62 | r=$? 63 | [[ ${r} -ne 0 ]] && { 64 | runlogfile=$(UT_runlogfile) || err || return 65 | cat "${runlogfile}" >"${STDERR}" 66 | return "${ERROR}" 67 | } 68 | } 69 | 70 | -------------------------------------------------------------------------------- /src/machinelist.sh: -------------------------------------------------------------------------------- 1 | # shellcheck shell=bash disable=SC2148 2 | 3 | # MACHINE LIST ================================================================ 4 | 5 | # MU_list_process_options checks if arg1 is in a list of valid machine utility 6 | # options. This function is called by the parser. 7 | # Args: arg1 - the option to check. 8 | # arg2 - value of the option to be set, optional. This depends on the 9 | # type of option. 10 | MU_list_process_options() { 11 | case "$1" in 12 | -h | --help) 13 | MU_list_usage 14 | return "${STOP}" 15 | ;; 16 | *) 17 | MU_list_usage 18 | printf 'ERROR: "%s" is not a valid "create" option.\n' "${1}" \ 19 | >"${STDERR}" 20 | return "${ERROR}" 21 | ;; 22 | esac 23 | } 24 | 25 | # MU_list_usage outputs help text for the machine utilities component. 26 | # Args: None expected. 27 | MU_list_usage() { 28 | 29 | cat <<'EnD' 30 | machine list options: 31 | 32 | Format: 33 | machine list [flags] 34 | 35 | Flags: 36 | -h - This help text. 37 | 38 | EnD 39 | } 40 | 41 | # MU_list_run lists Podman machines. 42 | # Args: None expected. 43 | MU_list_run() { 44 | info=$(podman machine list --format json) || err || return 45 | 46 | running=$(UT_sed_json_block "${info}" \ 47 | 'Name' 'mok-machine' '}' \ 48 | 'Running' \ 49 | ) || err || return 50 | 51 | exists=$(UT_sed_json_block "${info}" \ 52 | 'Name' 'mok-machine' '}' \ 53 | 'Name' \ 54 | ) || err || return 55 | 56 | if [[ ${exists} != "mok-machine" ]]; then 57 | printf 'mok-machine: does not exist.\n' 58 | else 59 | if [[ ${running} == "true" ]]; then 60 | printf 'mok-machine: running.\n' 61 | else 62 | printf 'mok-machine: stopped.\n' 63 | fi 64 | fi 65 | } 66 | 67 | -------------------------------------------------------------------------------- /src/machinesetup.sh: -------------------------------------------------------------------------------- 1 | # shellcheck shell=bash disable=SC2148 2 | 3 | # MACHINE SETUP =============================================================== 4 | 5 | # MU_setup_process_options checks if arg1 is in a list of valid machine utility 6 | # options. This function is called by the parser. 7 | # Args: arg1 - the option to check. 8 | # arg2 - value of the option to be set, optional. This depends on the 9 | # type of option. 10 | MU_setup_process_options() { 11 | case "$1" in 12 | -h | --help) 13 | MU_setup_usage 14 | return "${STOP}" 15 | ;; 16 | -v | --verbose) 17 | UT_set_tailf "${TRUE}" || err || return 18 | ;; 19 | *) 20 | MU_setup_usage 21 | printf 'ERROR: "%s" is not a valid "create" option.\n' "${1}" \ 22 | >"${STDERR}" 23 | return "${ERROR}" 24 | ;; 25 | esac 26 | } 27 | 28 | 29 | # MU_setup_usage outputs help text for the machine utilities component. 30 | # Args: None expected. 31 | MU_setup_usage() { 32 | 33 | cat <<'EnD' 34 | machine setup options: 35 | 36 | Format: 37 | machine setup [flags] 38 | 39 | Flags: 40 | -h, --help - This help text. 41 | -v, --verbose - Verbose output. Useful if there was a problem. 42 | 43 | EnD 44 | } 45 | 46 | # MU_setup_run sets up a Podman machine. 47 | # Args: None expected. 48 | MU_setup_run() { 49 | UT_run_with_progress \ 50 | " Loading module in mok-machine" \ 51 | podman machine ssh --username root mok-machine modprobe nf_conntrack 52 | r=$? 53 | [[ ${r} -ne 0 ]] && { 54 | runlogfile=$(UT_runlogfile) || err || return 55 | cat "${runlogfile}" >"${STDERR}" 56 | return "${ERROR}" 57 | } 58 | 59 | UT_run_with_progress \ 60 | " Persisting nf_conntrack module in mok-machine" \ 61 | podman machine ssh --username root mok-machine \ 62 | bash -c \ 63 | "\"\\\"echo nf_conntrack \ 64 | >/etc/modules-load.d/nf_conntrack.conf\\\"\"" 65 | r=$? 66 | [[ ${r} -ne 0 ]] && { 67 | runlogfile=$(UT_runlogfile) || err || return 68 | cat "${runlogfile}" >"${STDERR}" 69 | return "${ERROR}" 70 | } 71 | 72 | UT_run_with_progress \ 73 | " Setting nf_conntrack_max in mok-machine" \ 74 | podman machine ssh --username root mok-machine sysctl -q \ 75 | -w net.netfilter.nf_conntrack_max=163840 76 | r=$? 77 | [[ ${r} -ne 0 ]] && { 78 | runlogfile=$(UT_runlogfile) || err || return 79 | cat "${runlogfile}" >"${STDERR}" 80 | return "${ERROR}" 81 | } 82 | 83 | UT_run_with_progress \ 84 | " Persisting nf_conntrack_max in mok-machine" \ 85 | podman machine ssh --username root mok-machine \ 86 | bash -c \ 87 | "\"\\\"echo net.netfilter.nf_conntrack_max=163840 \ 88 | >/etc/sysctl.d/99-nf_conntrack_max.conf\\\"\"" 89 | r=$? 90 | [[ ${r} -ne 0 ]] && { 91 | runlogfile=$(UT_runlogfile) || err || return 92 | cat "${runlogfile}" >"${STDERR}" 93 | return "${ERROR}" 94 | } 95 | } 96 | 97 | -------------------------------------------------------------------------------- /src/machinestart.sh: -------------------------------------------------------------------------------- 1 | # shellcheck shell=bash disable=SC2148 2 | 3 | # MACHINE START =============================================================== 4 | 5 | # MU_start_process_options checks if arg1 is in a list of valid machine utility 6 | # options. This function is called by the parser. 7 | # Args: arg1 - the option to check. 8 | # arg2 - value of the option to be set, optional. This depends on the 9 | # type of option. 10 | MU_start_process_options() { 11 | case "$1" in 12 | -h | --help) 13 | MU_start_usage 14 | return "${STOP}" 15 | ;; 16 | -v | --verbose) 17 | UT_set_tailf "${TRUE}" || err || return 18 | ;; 19 | *) 20 | MU_start_usage 21 | printf 'ERROR: "%s" is not a valid "start" option.\n' "${1}" \ 22 | >"${STDERR}" 23 | return "${ERROR}" 24 | ;; 25 | esac 26 | } 27 | 28 | # MU_start_usage outputs help text for the machine utilities component. 29 | # Args: None expected. 30 | MU_start_usage() { 31 | 32 | cat <"${STDERR}" 58 | return "${ERROR}" 59 | } 60 | } 61 | 62 | -------------------------------------------------------------------------------- /src/machinestop.sh: -------------------------------------------------------------------------------- 1 | # shellcheck shell=bash disable=SC2148 2 | 3 | # MACHINE STOP ================================================================ 4 | 5 | # MU_stop_process_options checks if arg1 is in a list of valid machine utility 6 | # options. This function is called by the parser. 7 | # Args: arg1 - the option to check. 8 | # arg2 - value of the option to be set, optional. This depends on the 9 | # type of option. 10 | MU_stop_process_options() { 11 | case "$1" in 12 | -h | --help) 13 | MU_stop_usage 14 | return "${STOP}" 15 | ;; 16 | *) 17 | MU_stop_usage 18 | printf 'ERROR: "%s" is not a valid "stop" option.\n' "${1}" \ 19 | >"${STDERR}" 20 | return "${ERROR}" 21 | ;; 22 | esac 23 | } 24 | 25 | # MU_stop_usage outputs help text for the machine utilities component. 26 | # Args: None expected. 27 | MU_stop_usage() { 28 | 29 | cat <"${STDERR}" 54 | return "${ERROR}" 55 | } 56 | } 57 | 58 | 59 | -------------------------------------------------------------------------------- /src/macos.sh: -------------------------------------------------------------------------------- 1 | # shellcheck shell=bash disable=SC2148 2 | 3 | __check_os() { 4 | local majorversion system_ok=1 install_bash=0 install_coreutils=0 install_gawk=0 5 | local install_podman=0 install_sed=0 __mokostype=linux 6 | 7 | if [[ -d /System/Library/CoreServices ]]; then 8 | [[ -e /opt/homebrew/bin/bash ]] && eval "$(/opt/homebrew/bin/brew shellenv)" 9 | # We're on macOS, check for a recent Bash 10 | if command -v bash &>/dev/null; then 11 | majorversion=$(bash --version | head -n 1 | sed 's/^.*version \([0-9]*\).*/\1/') 12 | else 13 | echo "No version of Bash was found which is strange. Aborting." 14 | exit 1 15 | fi 16 | if [[ ${majorversion} -lt 5 ]]; then 17 | install_bash=1 18 | system_ok=0 19 | fi 20 | if ! command -v tac &>/dev/null; then 21 | install_coreutils=1 22 | system_ok=0 23 | fi 24 | if ! command -v gawk &>/dev/null; then 25 | install_gawk=1 26 | system_ok=0 27 | fi 28 | if ! command -v gsed &>/dev/null; then 29 | install_sed=1 30 | system_ok=0 31 | fi 32 | if ! command -v podman &>/dev/null; then 33 | install_podman=1 34 | system_ok=0 35 | fi 36 | 37 | if [[ ${system_ok} -eq 0 ]]; then 38 | echo "Your system requires some extra packages to be installed." 39 | echo "This script can install them for you." 40 | echo 41 | read -r -p 'Install extra packages? (y/N) ' ans 42 | if [[ ${ans} == "y" ]]; then 43 | if ! command -v brew &>/dev/null; then 44 | echo 'Homebrew is required to install extra packages.' 45 | echo 'You can install this yourself by following the instructions at https://brew.sh' 46 | echo 47 | read -r -p 'Should I install Homebrew now? (y/N) ' ans 48 | if [[ ${ans} == "y" ]]; then 49 | # Install Homebrew 50 | echo -n "Installing Homebrew, with:" 51 | echo " /bin/bash -c \"\$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)\"" 52 | echo 53 | /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" 54 | retval=$? 55 | if [[ retval -ne 0 ]]; then 56 | echo "Homebrew install failed." 57 | echo "Please try to install Homebrew manually using the above commmand." 58 | exit 1 59 | fi 60 | if [[ ! -e /opt/homebrew/bin/brew ]]; then 61 | echo "Homebrew did not seem install to /opt/homebrew." 62 | echo "Mok expects 'brew' to be in /opt/homebrew/bin/brew but it's not there." 63 | echo "Cannot continue. Please report this issue on GitHub." 64 | exit 1 65 | fi 66 | eval "$(/opt/homebrew/bin/brew shellenv)" 67 | else 68 | echo "Aborting due to missing 'Homebrew'." 69 | exit 1 70 | fi 71 | fi 72 | if [[ ! -e /opt/homebrew/bin/brew ]]; then 73 | echo "Mok expects 'brew' to be in /opt/homebrew/bin/brew but it's not there." 74 | echo "Cannot continue. Please report this issue on GitHub." 75 | exit 1 76 | fi 77 | eval "$(/opt/homebrew/bin/brew shellenv)" 78 | if [[ ${install_bash} -eq 1 ]]; then 79 | read -r -p 'Bash 5 is not installed. Install? (y/N) ' ans 80 | if [[ ${ans} == "y" ]]; then 81 | brew install bash 82 | else 83 | echo "Aborting due to missing 'Bash 5'." 84 | exit 1 85 | fi 86 | fi 87 | if [[ ${install_coreutils} -eq 1 ]]; then 88 | read -r -p 'Coreutils is not installed. Install? (y/N) ' ans 89 | if [[ ${ans} == "y" ]]; then 90 | brew install coreutils 91 | else 92 | echo "Aborting due to missing 'coreutils'." 93 | exit 1 94 | fi 95 | fi 96 | if [[ ${install_gawk} -eq 1 ]]; then 97 | read -r -p 'Gawk is not installed. Install? (y/N) ' ans 98 | if [[ ${ans} == "y" ]]; then 99 | brew install gawk 100 | else 101 | echo "Aborting due to missing 'gawk'." 102 | exit 1 103 | fi 104 | fi 105 | if [[ ${install_sed} -eq 1 ]]; then 106 | read -r -p 'GNU-Sed is not installed. Install? (y/N) ' ans 107 | if [[ ${ans} == "y" ]]; then 108 | brew install gnu-sed 109 | else 110 | echo "Aborting due to missing 'sed'." 111 | exit 1 112 | fi 113 | fi 114 | if [[ ${install_podman} -eq 1 ]]; then 115 | read -r -p 'Podman is not installed. Install? (y/N) ' ans 116 | if [[ ${ans} == "y" ]]; then 117 | brew install podman 118 | else 119 | echo "Aborting due to missing 'podman'." 120 | exit 1 121 | fi 122 | fi 123 | else 124 | echo "Aborting due to missing packages." 125 | exit 1 126 | fi 127 | fi 128 | fi 129 | } 130 | 131 | [[ __homebrew -ne 1 ]] && __check_os 132 | 133 | [[ -d /System/Library/CoreServices && -e /opt/homebrew/bin/bash && __homebrew -ne 1 ]] && { 134 | eval "$(/opt/homebrew/bin/brew shellenv)" 135 | export __mokostype=macos 136 | export __homebrew=1 137 | sed() { 138 | gsed "$@" 139 | } 140 | export -f sed 141 | exec bash "$0" "$@" 142 | } 143 | 144 | __mokostype="${__mokostype:-linux}" 145 | -------------------------------------------------------------------------------- /src/main.sh: -------------------------------------------------------------------------------- 1 | # shellcheck shell=bash disable=SC2148 2 | # MA - execution starts here 3 | 4 | # MA is an associative array that holds data specific to this file 5 | declare -A _MA 6 | 7 | # Defined in GL (globals.sh) 8 | declare OK ERROR STOP STDERR TRUE __mokostype __mokarch 9 | 10 | # Getters/Setters ------------------------------------------------------------- 11 | 12 | # MA_program_args getter outputs the program arguments sent by the user. 13 | MA_program_args() { 14 | printf '%s' "${_MA[program_args]}" 15 | } 16 | 17 | # MA_arg_1 getter outputs the first argument sent by the user. 18 | # Used by _CU_podman_checks so it doesn't check for a podman machine 19 | # if we're running 'machine' commands 20 | # FIXME: The command might not be the first argument. It could be '-p' 21 | MA_arg_1() { 22 | printf '%s' "${_MA[arg_1]}" 23 | } 24 | 25 | # MA_program_name getter outputs the name of the program. 26 | MA_program_name() { 27 | printf '%s' "${_MA[arg_0]##*/}" 28 | } 29 | 30 | # MA_ostype getter outputs the OS type. 31 | MA_ostype() { 32 | printf '%s' "${_MA[ostype]}" 33 | } 34 | 35 | # MA_ostype getter outputs the machine architecture. 36 | MA_arch() { 37 | printf '%s' "${_MA[arch]}" 38 | } 39 | 40 | # Public Functions ------------------------------------------------------------ 41 | 42 | # main is the start point for this application. 43 | # Args: arg1-N - the command line arguments entered by the user. 44 | MA_main() { 45 | local retval="${OK}" 46 | 47 | CU_podman_or_docker 48 | 49 | trap MA_cleanup EXIT 50 | MA_sanity_checks || return 51 | PA_run "$@" || retval=$? 52 | 53 | exit "${retval}" 54 | } 55 | 56 | # MA_version outputs the version information the exits. 57 | MA_version() { 58 | printf 'Version: %s\n' "${MOKVERSION}" 59 | exit "${OK}" 60 | } 61 | 62 | # MA_process_global_options is called by the parser when a global option is 63 | # encountered for processing. 64 | # Args: arg1 - the global option that was found. 65 | MA_process_global_options() { 66 | case "$1" in 67 | -h | --help) 68 | PA_usage 69 | return "${STOP}" 70 | ;; 71 | -p | --plain) 72 | UT_set_plain "${TRUE}" 73 | ;; 74 | *) 75 | printf 'INTERNAL ERROR: Invalid global option, "%s".' "$1" 76 | return "${ERROR}" 77 | ;; 78 | esac 79 | } 80 | 81 | # MA_cleanup is called from an EXIT trap only, when the program exits, and 82 | # calls every other function matching the pattern '^.._cleanup'. 83 | # Args: No args expected. 84 | MA_cleanup() { 85 | local retval="${OK}" funcs func 86 | funcs=$(declare -F | awk '{print $NF;}' | grep '^.._cleanup') 87 | for func in ${funcs}; do 88 | [[ ${func} == "${FUNCNAME[0]}" ]] && continue 89 | eval "${func}" || retval=$? 90 | done 91 | return "${retval}" 92 | } 93 | 94 | # MA_sanity_checks is expected to run some quick and simple checks to 95 | # see if key components are available before MA_main is called. 96 | # Args: No args expected. 97 | MA_sanity_checks() { 98 | 99 | local binary binaries 100 | 101 | if [[ $(CU_podmantype) == "native" ]]; then 102 | binaries="gawk tac column tput grep sed ip cut" 103 | else 104 | binaries="gawk tac column tput grep sed cut" 105 | fi 106 | 107 | for binary in ${binaries}; do 108 | if ! command -v "${binary}" >&/dev/null; then 109 | printf 'ERROR: "%s" binary not found in path. Aborting.' "${binary}" \ 110 | >"${STDERR}" 111 | return "${ERROR}" 112 | fi 113 | done 114 | 115 | # Disable terminal escapes (colours) if stdout is not a terminal 116 | [ -t 1 ] || UT_disable_colours 117 | } 118 | 119 | # MA_usage outputs help text for all components then quits with no error. This 120 | # is registered as a callback in the Parser, see _MA_new 121 | # Args: None expected. 122 | MA_usage() { 123 | 124 | cat <<'EnD' 125 | Usage: mok [-h] [subcommand] [ARGS...] 126 | 127 | Global options: 128 | --help 129 | -h - This help text 130 | --plain 131 | -p - Plain output. No colours or animations. 132 | 133 | Where command can be one of: 134 | create - Add item(s) to the system. 135 | delete - Delete item(s) from the system. 136 | build - Build item(s) used by the system. 137 | get - Get details about items in the system. 138 | exec - 'Log in' to the container. 139 | machine - Manage a podman machine (MacOS only). 140 | version - Display version information. 141 | 142 | For help on a specific command, run: 143 | mok --help 144 | EnD 145 | } 146 | 147 | # MA_new sets the initial values for the _MA associative array 148 | _MA_new() { 149 | _MA[program_args]="$*" 150 | _MA[arg_0]="$0" 151 | _MA[arg_1]="$1" 152 | _MA[ostype]="${__mokostype}" # linux or macos 153 | _MA[arch]="${__mokarch}" # x86_64 or arm64 154 | 155 | # Program the parser's state machine 156 | PA_add_state "COMMAND" "version" "END" "MA_version" 157 | 158 | # Set up the parser's option callbacks 159 | PA_add_option_callback "" "MA_process_global_options" 160 | 161 | # Set up the parser's usage callbacks 162 | PA_add_usage_callback "" "MA_usage" 163 | } 164 | 165 | # Initialise _MA 166 | _MA_new "$@" || exit 1 167 | 168 | # vim helpers ----------------------------------------------------------------- 169 | #include globals.sh 170 | # vim:ft=sh:sw=2:et:ts=2: 171 | -------------------------------------------------------------------------------- /src/util.sh: -------------------------------------------------------------------------------- 1 | # shellcheck shell=bash disable=SC2148 2 | # UT - Utilities 3 | 4 | # _UT is an associative array that holds data specific to general utilities. 5 | declare -A _UT 6 | 7 | # Declare externally defined global variables --------------------------------- 8 | 9 | declare OK STDERR FALSE TRUE 10 | 11 | # Getters/Setters ------------------------------------------------------------- 12 | 13 | # UT_runlogfile getter outputs the value of _UT[runlogfile], which contains the 14 | # name of the file written to when running UT_run_with_progress. 15 | UT_runlogfile() { 16 | printf '%s' "${_UT[runlogfile]}" 17 | } 18 | 19 | # UT_tailf getter indicates whether a log will be shown in real-time during the 20 | # 'run', TRUE, or not, FALSE. 21 | # Args 22 | UT_tailf() { 23 | printf '%s' "${_UT[tailf]}" 24 | } 25 | 26 | # UT_tailf setter indicates whether a log should be shown during the 'run', 27 | # TRUE, or not, FALSE. 28 | # Args: arg1 - Whether to tail, TRUE or FALSE. 29 | UT_set_tailf() { 30 | _UT[tailf]="$1" 31 | } 32 | 33 | # UT_set_plain setter sets plain output, with no colour, animations, or UTF 34 | # high byte characters. 35 | # Args: arg1 - Whether to show plain output, TRUE or FALSE. 36 | UT_set_plain() { 37 | _UT[plain]="$1" 38 | UT_disable_colours 39 | } 40 | 41 | # Public Functions ------------------------------------------------------------ 42 | 43 | # UT_disable_colours resets variables used for colourised output so that they 44 | # contain no colour terminal escapes. Useful if stdin is not a terminal. 45 | UT_disable_colours() { 46 | _UT[yellow]= 47 | _UT[green]= 48 | _UT[red]= 49 | _UT[normal]= 50 | _UT[probablysuccess]="!" 51 | _UT[success]="✓" 52 | _UT[failure]="✕" 53 | } 54 | 55 | # UT_run_with_progress displays a progress spinner, item text, and a tick or 56 | # cross based on the exit code. 57 | # Args: arg1 - the text to display. 58 | # arg2-N - remaining args are the program to run and its arguments. 59 | UT_run_with_progress() { 60 | 61 | local displaytext=$1 retval int spinner=() 62 | 63 | _UT[runlogfile]=$(mktemp -p /var/tmp) || { 64 | printf 'ERROR: mktmp failed.\n' >"${STDERR}" 65 | err || return 66 | } 67 | shift 68 | 69 | if [[ ${_UT[tailf]} == "${FALSE}" && ${_UT[plain]} == "${TRUE}" ]]; then 70 | 71 | # Output with no colours or spinny 72 | 73 | printf '%s' "${displaytext}" 74 | 75 | eval "$*" &>"${_UT[runlogfile]}" 76 | retval=$? 77 | 78 | # Mark success/fail 79 | if [[ ${retval} -eq 0 ]]; then 80 | printf ' .. %s\n' "${_UT[success]}" 81 | else 82 | printf ' .. %s\n' "${_UT[failure]}" 83 | fi 84 | 85 | elif [[ ${_UT[tailf]} == "${FALSE}" ]]; then 86 | 87 | # Output with colours and spinny 88 | 89 | while read -r char; do 90 | spinner+=("${char}") 91 | done <<<"$(grep -o . <<<"${_UT[spinnerchars]}")" 92 | 93 | ( 94 | eval "$*" &>"${_UT[runlogfile]}" 95 | ) & 96 | 97 | # Turn the cursor off 98 | tput civis 99 | 100 | # Start the spin animation 101 | printf ' %s' "${displaytext}" 102 | (while true; do 103 | for int in {0..3}; do 104 | printf '\r %s ' "${spinner[int]}" 105 | sleep .1 106 | done 107 | done) & 108 | _UT[spinnerpid]=$! 109 | disown 110 | 111 | # Wait for the command to finish 112 | wait %1 2>/dev/null 113 | retval=$? 114 | 115 | # Kill the spinner 116 | kill "${_UT[spinnerpid]}" 2>/dev/null 117 | _UT[spinnerpid]= 118 | 119 | sleep .5 120 | 121 | # Mark success/fail 122 | if [[ ${retval} -eq 127 ]]; then 123 | # The job finished before we started waiting for it 124 | printf '\r %s\n' "${_UT[probablysuccess]}" 125 | elif [[ ${retval} -eq 0 ]]; then 126 | printf '\r %s\n' "${_UT[success]}" 127 | else 128 | printf '\r %s\n' "${_UT[failure]}" 129 | fi 130 | 131 | # Restore the cursor 132 | tput cnorm 133 | sleep .5 134 | 135 | else 136 | 137 | # Tailf output 138 | 139 | printf 'COMMAND: %s\n\n' "$*" 140 | 141 | ( 142 | eval "$*" &>/dev/stdout 143 | ) & 144 | sleep 1 145 | 146 | # Wait for the command to finish 147 | wait %1 2>/dev/null 148 | retval=$? 149 | 150 | # Mark success/fail 151 | if [[ ${retval} -eq 127 ]]; then 152 | # The job finished before we started waiting for it 153 | printf '\n\nSTATUS: OK - (Probably)\n\n' 154 | elif [[ ${retval} -eq 0 ]]; then 155 | printf '\n\nSTATUS: OK\n\n' 156 | else 157 | printf '\n\nSTATUS: FAIL\n\n' 158 | fi 159 | fi 160 | 161 | return "${retval}" 162 | } 163 | 164 | # UT_cleanup removes any artifacts that were created during execution. 165 | # This is called by 'MA_cleanup' trap only. 166 | UT_cleanup() { 167 | 168 | # Called when the script exits. 169 | 170 | local int 171 | 172 | # Kill the spinny, and anything else, if they're running 173 | if [[ ${_UT[tailf]} == "${FALSE}" ]]; then 174 | [[ -n ${_UT[spinnerpid]} ]] && { 175 | printf '%s\r ✕%s\n' "${_UT[red]}" "${_UT[normal]}" 176 | kill "${_UT[spinnerpid]}" 177 | } 178 | # If progress spinner crashed make sure the cursor is shown 179 | [ -t 1 ] && tput cnorm 180 | else 181 | kill "${_UT[spinnerpid]}" &>/dev/null 182 | fi 183 | 184 | return "${OK}" 185 | } 186 | 187 | # UT_sed_json_block returns the value of the query_key between 188 | # block_start_key:block_start_val and block_end_key. 189 | # Args: 190 | # Arg 1: data 191 | # Arg 2: block_start_key 192 | # Arg 3: block_start_val 193 | # Arg 4: block_end_key 194 | # Arg 5: query_key 195 | UT_sed_json_block() { 196 | local data block_start_key block_start_val block_end_key query_key 197 | data="$1" 198 | block_start_key="$2" 199 | block_start_val="$3" 200 | block_end_key="$4" 201 | query_key="$5" 202 | echo "$data" | sed -rn \ 203 | '/'"${block_start_key}"'.*'"${block_start_val}"'/,/'"${block_end_key}"'/ { s/^ *"'"${query_key}"'": *//p }' \ 204 | | tr -d ',"' 205 | } 206 | 207 | # Private Functions ----------------------------------------------------------- 208 | 209 | # _UT_new sets the initial values for the _UT associative array. 210 | # Args: None expected. 211 | _UT_new() { 212 | _UT[tailf]="${FALSE}" 213 | _UT[yellow]=$(tput setaf 3) 214 | _UT[green]=$(tput setaf 2) 215 | _UT[red]=$(tput setaf 1) 216 | _UT[normal]=$(tput sgr0) 217 | _UT[probablysuccess]="${_UT[yellow]}✓${_UT[normal]}" 218 | _UT[success]="${_UT[green]}✓${_UT[normal]}" 219 | _UT[failure]="${_UT[red]}✕${_UT[normal]}" 220 | _UT[runlogfile]= 221 | _UT[spinnerchars]="|/-\\" 222 | _UT[spinnerpid]= 223 | } 224 | 225 | # Initialise _UT 226 | _UT_new || exit 1 227 | 228 | # vim helpers ----------------------------------------------------------------- 229 | #include globals.sh 230 | # vim:ft=sh:sw=2:et:ts=2: 231 | -------------------------------------------------------------------------------- /src/versions.sh: -------------------------------------------------------------------------------- 1 | # shellcheck shell=bash disable=SC2148 2 | # CC - Create Cluster - versions file 3 | 4 | # CC is an associative array that holds data specific to creating a cluster. 5 | declare -A _CC 6 | 7 | # Declare externally defined variables ---------------------------------------- 8 | 9 | # Defined in GL (globals.sh) 10 | declare OK ERROR STDERR TRUE 11 | 12 | # _CC_set_up_master_node_v1_30_0 uses kubeadm to set up the master node. 13 | # Args: arg1 - the container to set up. 14 | _CC_set_up_master_node_v1_30_0() { 15 | 16 | local setupfile lbaddr certSANs="{}" certkey masternum t lbip 17 | # Set by _CC_get_master_join_details: 18 | local cahash token masterip 19 | 20 | setupfile=$(mktemp -p /var/tmp) || { 21 | printf 'ERROR: mktmp failed.\n' >"${STDERR}" 22 | return "${ERROR}" 23 | } 24 | 25 | masternum="${1##*-}" # <- eg. for xxx-master-1, masternum=1 26 | 27 | if [[ ${_CC[skipmastersetup]} != "${TRUE}" ]]; then 28 | 29 | if [[ ${_CC[withlb]} == "${TRUE}" && ${masternum} -eq 1 ]]; then 30 | 31 | # This is the first master node 32 | 33 | # Sets cahash, token, and masterip: 34 | lbip=$(CU_get_container_ip "${_CC[clustername]}-lb") 35 | lbaddr=", '${lbip}'" 36 | uploadcerts="--upload-certs" 37 | certkey="CertificateKey: f8802e114ef118304e561c3acd4d0b543adc226b7a27f675f56564185ffe0c07" 38 | 39 | elif [[ ${_CC[withlb]} == "${TRUE}" && ${masternum} -ne 1 ]]; then 40 | 41 | # This is not the first master node, so join with the master 42 | # https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/high-availability/ 43 | 44 | # Keep trying to get join details until apiserver is ready or we run out of tries 45 | for try in $(seq 1 9); do 46 | # Runs a script on master node to get join details 47 | t=$(_CC_get_master_join_details "${_CC[clustername]}-master-1") 48 | retval=$? 49 | [[ ${retval} -eq 0 ]] && break 50 | [[ ${try} -eq 9 ]] && { 51 | printf '\nERROR: Problem with "_CC_get_master_join_details". Tried %d times\n\n' "${try}" \ 52 | >"${STDERR}" 53 | return "${ERROR}" 54 | } 55 | sleep 5 56 | done 57 | eval "${t}" 58 | fi 59 | fi 60 | 61 | # Always set certSANs for the host whether we need it or not 62 | hostaddr=$(ip ro get 8.8.8.8 | cut -d" " -f 7) || err || return 63 | certSANs=" 64 | certSANs: [ '${hostaddr}'${lbaddr}, '127.0.0.1', 'localhost' ]" 65 | 66 | # Write the file regardless, so the user can use it if required 67 | 68 | cat <"${setupfile}" 69 | # Disable ipv6 70 | sysctl -w net.ipv6.conf.all.disable_ipv6=1 71 | sysctl -w net.ipv6.conf.default.disable_ipv6=1 72 | 73 | # Use a custom configuration. Default config created from kubeadm with: 74 | # kubeadm config print init-defaults 75 | # then edited. 76 | 77 | ipaddr=\$(ip ro get default 8.8.8.8 | head -n 1 | cut -f 7 -d " ") 78 | podsubnet="10.244.0.0/16" 79 | servicesubnet="10.96.0.0/16" 80 | 81 | cat <kubeadm-init-defaults.yaml 82 | apiVersion: kubeadm.k8s.io/v1beta4 83 | bootstrapTokens: 84 | - groups: 85 | - system:bootstrappers:kubeadm:default-node-token 86 | token: abcdef.0123456789abcdef 87 | ttl: 24h0m0s 88 | usages: 89 | - signing 90 | - authentication 91 | kind: InitConfiguration 92 | ${certkey} 93 | localAPIEndpoint: 94 | advertiseAddress: \$ipaddr 95 | bindPort: 6443 96 | nodeRegistration: 97 | criSocket: unix:///var/run/crio/crio.sock 98 | imagePullPolicy: IfNotPresent 99 | imagePullSerial: true 100 | name: $1 101 | taints: 102 | - effect: NoSchedule 103 | key: node-role.kubernetes.io/control-plane 104 | timeouts: 105 | controlPlaneComponentHealthCheck: 4m0s 106 | discovery: 5m0s 107 | etcdAPICall: 2m0s 108 | kubeletHealthCheck: 4m0s 109 | kubernetesAPICall: 1m0s 110 | tlsBootstrap: 5m0s 111 | upgradeManifests: 5m0s 112 | --- 113 | apiServer: ${certSANs} 114 | apiVersion: kubeadm.k8s.io/v1beta4 115 | caCertificateValidityPeriod: 87600h0m0s 116 | certificateValidityPeriod: 8760h0m0s 117 | certificatesDir: /etc/kubernetes/pki 118 | clusterName: ${_CC[clustername]}-cluster 119 | controlPlaneEndpoint: ${lbaddr:-\$ipaddr}:6443 120 | controllerManager: {} 121 | dns: {} 122 | encryptionAlgorithm: RSA-2048 123 | etcd: 124 | local: 125 | dataDir: /var/lib/etcd 126 | imageRepository: registry.k8s.io 127 | kind: ClusterConfiguration 128 | kubernetesVersion: v${_CC[k8sver]} 129 | networking: 130 | dnsDomain: cluster.local 131 | podSubnet: \${podsubnet} 132 | serviceSubnet: \${servicesubnet} 133 | proxy: {} 134 | scheduler: {} 135 | EOF 136 | 137 | mkdir -p /mok 138 | cp kubeadm-init-defaults.yaml /mok/kubeadm.yaml 139 | 140 | if [[ -z "${masterip}" ]]; then 141 | # Run the preflight phase 142 | kubeadm init \\ 143 | --ignore-preflight-errors Swap \\ 144 | --config=kubeadm-init-defaults.yaml \\ 145 | phase preflight 146 | 147 | # Set up the kubelet 148 | kubeadm init phase kubelet-start 149 | 150 | # Edit the kubelet configuration file 151 | echo "failSwapOn: false" >>/var/lib/kubelet/config.yaml 152 | sed -i 's/cgroupDriver: systemd/cgroupDriver: cgroupfs/' /var/lib/kubelet/config.yaml 153 | 154 | # Tell kubeadm to carry on from here 155 | kubeadm init \\ 156 | --config=kubeadm-init-defaults.yaml \\ 157 | --ignore-preflight-errors Swap \\ 158 | ${uploadcerts} \\ 159 | --skip-phases=preflight,kubelet-start 160 | 161 | export KUBECONFIG=/etc/kubernetes/super-admin.conf 162 | 163 | # Flannel - 10.244.0.0./16 164 | kubectl apply -f /root/kube-flannel.yml 165 | else 166 | kubeadm join ${masterip}:6443 \\ 167 | --ignore-preflight-errors Swap \\ 168 | --control-plane \\ 169 | --token ${token} \\ 170 | --discovery-token-ca-cert-hash sha256:${cahash} \\ 171 | --certificate-key f8802e114ef118304e561c3acd4d0b543adc226b7a27f675f56564185ffe0c07 172 | fi 173 | 174 | systemctl enable kubelet 175 | EnD 176 | 177 | docker cp "${setupfile}" "$1":/root/setup.sh || err || { 178 | rm -f "${setupfile}" 179 | return "${ERROR}" 180 | } 181 | 182 | # Run the file 183 | [[ -z ${_CC[skipmastersetup]} ]] && { 184 | docker exec "$1" bash /root/setup.sh || err || return 185 | 186 | # Remove the taint if we're setting up a single node cluster 187 | 188 | [[ ${_CC[numworkers]} -eq 0 ]] && { 189 | 190 | removetaint=$(mktemp -p /var/tmp) || { 191 | printf 'ERROR: mktmp failed.\n' >"${STDERR}" 192 | return "${ERROR}" 193 | } 194 | 195 | # Write the file 196 | cat <<'EnD' >"${removetaint}" 197 | export KUBECONFIG=/etc/kubernetes/super-admin.conf 198 | kubectl taint nodes --all node-role.kubernetes.io/control-plane:NoSchedule- 199 | EnD 200 | 201 | docker cp "${removetaint}" "$1":/root/removetaint.sh || err || { 202 | rm -f "${removetaint}" 203 | return "${ERROR}" 204 | } 205 | 206 | # Run the file 207 | docker exec "$1" bash /root/removetaint.sh || err 208 | } 209 | } 210 | 211 | return "${OK}" 212 | } 213 | 214 | # _CC_set_up_worker_node_v1_30_0 uses kubeadm to set up the worker node. 215 | # Args: arg1 - the container to set up. 216 | _CC_set_up_worker_node_v1_30_0() { 217 | 218 | local setupfile cahash="$2" token="$3" masterip="$4" 219 | 220 | setupfile=$(mktemp -p /var/tmp) || { 221 | printf 'ERROR: mktmp failed.\n' >"${STDERR}" 222 | return "${ERROR}" 223 | } 224 | 225 | if [[ ${_CC[withlb]} == "${TRUE}" ]]; then 226 | masterip=$(CU_get_container_ip "${_CC[clustername]}-lb") || return 227 | fi 228 | 229 | cat <"${setupfile}" 230 | # Wait for the master API to become ready 231 | while true; do 232 | curl -k https://${masterip}:6443/ 233 | [[ \$? -eq 0 ]] && break 234 | sleep 1 235 | done 236 | 237 | # Do the preflight tests (ignoring swap error) 238 | kubeadm join \\ 239 | phase preflight \\ 240 | --token ${token} \\ 241 | --discovery-token-ca-cert-hash sha256:${cahash} \\ 242 | --ignore-preflight-errors Swap \\ 243 | ${masterip}:6443 244 | 245 | # Set up the kubelet 246 | kubeadm join \\ 247 | phase kubelet-start \\ 248 | --token ${token} \\ 249 | --discovery-token-ca-cert-hash sha256:${cahash} \\ 250 | ${masterip}:6443 & 251 | 252 | while true; do 253 | [[ -e /var/lib/kubelet/config.yaml ]] && break 254 | sleep 1 255 | done 256 | 257 | # Edit the kubelet configuration file 258 | echo "failSwapOn: false" >>/var/lib/kubelet/config.yaml 259 | sed -i 's/cgroupDriver: systemd/cgroupDriver: cgroupfs/' /var/lib/kubelet/config.yaml 260 | 261 | systemctl enable --now kubelet 262 | EnD 263 | 264 | docker cp "${setupfile}" "$1":/root/setup.sh 2>"${STDERR}" || err || { 265 | rm -f "${setupfile}" 266 | return "${ERROR}" 267 | } 268 | 269 | docker exec "$1" bash /root/setup.sh || err 270 | } 271 | 272 | # vim helpers ----------------------------------------------------------------- 273 | #include globals.sh 274 | # vim:ft=sh:sw=2:et:ts=2: 275 | -------------------------------------------------------------------------------- /tests/README.md: -------------------------------------------------------------------------------- 1 | # Tests 2 | 3 | ## unit-tests.sh 4 | 5 | Currently broken. Unit tests make programming so tedious that I'm hoping end-to-end tests will be enough. 6 | 7 | ## build-tests.sh 8 | 9 | I'm not sure that this needs to be kept either. 10 | 11 | ## e2e-tests.sh 12 | 13 | This script starts the end-to-end tests. Tests are run using `cmdline-player`, installed during `sudo make install`. 14 | 15 | Test output is saved in `e2e-logs` in files with format as: 16 | 17 | ```none 18 | 2020-06-05-1591367000_test-2_GOOD_1.0000.log 19 | ^ ^ ^ ^ 20 | unix timestamp | | | 21 | Test number | | 22 | CRM114 status | 23 | CRM114 confidence (pR) 24 | ``` 25 | 26 | In CRM114 we use OSB Classification to train good and bad output from tests. 27 | 28 | About CRM 114 confidence (pR): 29 | 30 | > CRM114 classifiers are generally set up to give pR values in this scale; experimental 31 | > classifiers may need to be adjusted to get their typical results into the following ranges: 32 | > 33 | > * pR values above +100 signify “high chance this text is a member of this class” 34 | > * pR values between +100 and +10 signify “moderate chance this text is a member of 35 | > this class” 36 | > * pR values between +10 and ­-10 signify “unsure” (and typically should be retrained if 37 | > using either SSTTT or DSTTT training. 38 | > * pR values between ­-10 and ­-100 signify “moderate chance this text is not a member of 39 | > this class” 40 | > * pR values of below ­-100 signify “high chance this text is not a member of this class” 41 | 42 | ### Usage texts 43 | 44 | Sanity checks for usage texts. CSS files were created with: 45 | 46 | ```bash 47 | # Must use bash, not zsh, for these: 48 | 49 | # Create css files 50 | for i in build create delete exec get unknown; do crm learn.crm ${i^^}_USAGE <<<$(:); done 51 | 52 | # Train css files 53 | for i in build create delete exec get; do crm learn.crm ${i^^}_USAGE <<<$(mok ${i} -h); done 54 | 55 | ``` 56 | 57 | The CSS files are matched to actual `mok command -h` usage in `usage-checks.sh`. 58 | 59 | Output from `usage-checks.sh`: 60 | 61 | ```none 62 | GOOD - CRM114 thinks output from 'mok build -h' is BUILD_USAGE. 63 | GOOD - CRM114 thinks output from 'mok create -h' is CREATE_USAGE. 64 | GOOD - CRM114 thinks output from 'mok delete -h' is DELETE_USAGE. 65 | GOOD - CRM114 thinks output from 'mok exec -h' is EXEC_USAGE. 66 | GOOD - CRM114 thinks output from 'mok get -h' is GET_USAGE. 67 | ``` 68 | -------------------------------------------------------------------------------- /tests/build-tests.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # =========================================================================== 4 | # Build tests 5 | # =========================================================================== 6 | 7 | # --------------------------------------------------------------------------- 8 | testDockerBuildNoCache(){ 9 | # --------------------------------------------------------------------------- 10 | # Create directory with create_docker_build_dir then manually 11 | # build the image from scratch (--no-cache) 12 | 13 | create_docker_build_dir >/dev/null 14 | docker build --no-cache -t local/mok-centos-7 \ 15 | "$DOCKERBUILDTMPDIR/mok-centos-7" >/dev/null 16 | assertEquals \ 17 | "Check docker build status" \ 18 | "0" "$?" 19 | cleanup 20 | } 21 | 22 | # --------------------------------------------------------------------------- 23 | testDockerBuildExitStatus(){ 24 | # --------------------------------------------------------------------------- 25 | # Check that build_container_image works 26 | 27 | build_container_image >/dev/null 28 | assertEquals \ 29 | "Check docker build status" \ 30 | "0" "$?" 31 | cleanup 32 | } 33 | 34 | # =========================================================================== 35 | # shUnit2 funcs 36 | # =========================================================================== 37 | 38 | # --------------------------------------------------------------------------- 39 | setUp() { 40 | # --------------------------------------------------------------------------- 41 | # source mok.deploy and disable output of usage(). 42 | 43 | . ./mok.deploy 44 | usage() { :; } 45 | } 46 | 47 | # --------------------------------------------------------------------------- 48 | grabMainOutput() { 49 | # --------------------------------------------------------------------------- 50 | # Helper function. Sets LINES array to script output. 51 | 52 | local tmpname=`mktemp --tmpdir=/var/tmp` 53 | main "$@" >$tmpname 54 | readarray -t LINES <$tmpname 55 | } 56 | 57 | # Load and run shUnit2. 58 | . tests/shunit2 59 | 60 | # vim:ft=bash:sw=2:et:ts=2: 61 | -------------------------------------------------------------------------------- /tests/e2e-logs/2020-06-05-1591359733_test-1_BAD_1.0000.log: -------------------------------------------------------------------------------- 1 | 2 | tests(master*)$ errstop() { [[ $1 -gt 0 ]] && echo "SCREENCAST STOP &"; read; } 3 | tests(master*)$ date 4 | Fri 5 Jun 13:22:08 BST 2020 5 | tests(master*)$ alias mokbox='docker run --rm -ti --hostname mokbox --name mokbox -v /var/run/docker.sock:/var/run/dock 6 | er.sock -v /var/tmp:/var/tmp myownkind/mokbox' 7 | tests(master*)$ mokbox 8 | docker: Error response from daemon: write /run/media/mclarkson/ICYBOX/var-lib-docker/containerd/daemon/io.containerd.me 9 | tadata.v1.bolt/meta.db: read-only file system: unknown. 10 | tests(master*)$ retval=$? 11 | tests(master*)$ errstop() { [[ $1 -gt 0 ]] && { printf 'SCREENCAST STOP &'; read; }; } 12 | tests(master*)$ errstop $retval 13 | SCREENCAST STOP & 14 | 15 | -------------------------------------------------------------------------------- /tests/e2e-logs/2020-06-05-1591359996_test-1_BAD_1.0000.log: -------------------------------------------------------------------------------- 1 | 2 | tests(master*)$ errstop() { [[ $1 -gt 0 ]] && echo "SCREENCAST STOP &"; read; } 3 | tests(master*)$ date 4 | Fri 5 Jun 13:26:20 BST 2020 5 | tests(master*)$ alias mokbox='docker run --rm -ti --hostname mokbox --name mokbox -v /var/run/docker.sock:/var/run/dock 6 | er.sock -v /var/tmp:/var/tmp myownkind/mokbox' 7 | tests(master*)$ mokbox 8 | [root@mokbox /]# retval=$? 9 | [root@mokbox /]# errstop() { [[ $1 -gt 0 ]] && { printf 'SCREENCAST STOP &'; read; }; } 10 | [root@mokbox /]# errstop $retval 11 | [root@mokbox /]# 12 | [root@mokbox /]# # Test 1 - download the prebuilt image 13 | [root@mokbox /]# 14 | [root@mokbox /]# mok -p build image --get-prebuilt-image 15 | Downloading base image, 'mok-centos-7-v1.18.3' .. SUCCESS 16 | [root@mokbox /]# errstop $? 17 | [root@mokbox /]# time mok -p create cluster myk8s --masters 1 --tailf 18 | 19 | ERROR: Cluster, "myk8s", exists! Aborting. 20 | 21 | real 0m0.109s 22 | user 0m0.064s 23 | sys 0m0.055s 24 | [root@mokbox /]# errstop $? 25 | SCREENCAST STOP & 26 | -------------------------------------------------------------------------------- /tests/e2e-logs/2020-06-05-1591367000_test-2_GOOD_1.0000.log: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bashtools/mok/5ccedb9aea7b05d47e310003c157efdd7df782af/tests/e2e-logs/2020-06-05-1591367000_test-2_GOOD_1.0000.log -------------------------------------------------------------------------------- /tests/e2e-logs/2020-06-05-1591372979_test-2_GOOD_1.0000.log: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bashtools/mok/5ccedb9aea7b05d47e310003c157efdd7df782af/tests/e2e-logs/2020-06-05-1591372979_test-2_GOOD_1.0000.log -------------------------------------------------------------------------------- /tests/e2e-logs/2020-06-05-1591377693_test-2_GOOD_1.0000.log: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bashtools/mok/5ccedb9aea7b05d47e310003c157efdd7df782af/tests/e2e-logs/2020-06-05-1591377693_test-2_GOOD_1.0000.log -------------------------------------------------------------------------------- /tests/e2e-logs/2020-06-05-1591396993_test-1_BAD_0.9927.log: -------------------------------------------------------------------------------- 1 | 2 | tests(master*)$ errstop() { [[ $1 -gt 0 ]] && { printf 'SCREENCAST STOP &'; read 3 | ; }; } 4 | tests(master*)$ date 5 | Fri 5 Jun 23:42:58 BST 2020 6 | tests(master*)$ alias mokbox='docker run --rm -ti --hostname mokbox --name mokbo 7 | x -v /var/run/docker.sock:/var/run/docker.sock -v /var/tmp:/var/tmp myownkind/mo 8 | kbox' 9 | tests(master*)$ mokbox 10 | [root@mokbox /]# retval=$? 11 | [root@mokbox /]# errstop() { [[ $1 -gt 0 ]] && { printf 'SCREENCAST STOP &'; rea 12 | d; }; } 13 | [root@mokbox /]# errstop $retval 14 | [root@mokbox /]# 15 | [root@mokbox /]# # Test 1 - download the prebuilt image and test 16 | [root@mokbox /]# 17 | [root@mokbox /]# mok -p build image --get-prebuilt-image 18 | Downloading base image, 'mok-centos-7-v1.18.3' .. SUCCESS 19 | [root@mokbox /]# errstop $? 20 | [root@mokbox /]# time mok -p create cluster myk8s --masters 1 --tailf 21 | 22 | ERROR: Cluster, "myk8s", exists! Aborting. 23 | 24 | real 0m0.111s 25 | user 0m0.076s 26 | sys 0m0.045s 27 | [root@mokbox /]# errstop $? 28 | SCREENCAST STOP & 29 | -------------------------------------------------------------------------------- /tests/e2e-logs/2020-06-05-1591397979_test-2_GOOD_1.0000.log: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bashtools/mok/5ccedb9aea7b05d47e310003c157efdd7df782af/tests/e2e-logs/2020-06-05-1591397979_test-2_GOOD_1.0000.log -------------------------------------------------------------------------------- /tests/e2e-logs/2020-06-06-1591400029_test-1_BAD_0.9913.log: -------------------------------------------------------------------------------- 1 | 2 | tests(master*)$ errstop() { [[ $1 -gt 0 ]] && { printf 'SCREENCAST STOP &'; read 3 | ; }; } 4 | tests(master*)$ date 5 | Sat 6 Jun 00:33:35 BST 2020 6 | tests(master*)$ alias mokbox='docker run --rm -ti --hostname mokbox --name mokbo 7 | x -v /var/run/docker.sock:/var/run/docker.sock -v /var/tmp:/var/tmp myownkind/mo 8 | kbox' 9 | tests(master*)$ mokbox 10 | [root@mokbox /]# retval=$? 11 | [root@mokbox /]# errstop() { [[ $1 -gt 0 ]] && { printf 'SCREENCAST STOP &'; rea 12 | d; }; } 13 | [root@mokbox /]# errstop $retval 14 | [root@mokbox /]# 15 | [root@mokbox /]# # Test 1 - download the prebuilt image and test 16 | [root@mokbox /]# 17 | [root@mokbox /]# mok -p build image --get-prebuilt-image 18 | Downloading base image, 'mok-centos-7-v1.18.3' .. SUCCESS 19 | [root@mokbox /]# errstop $? 20 | [root@mokbox /]# time mok -p create cluster myk8s --masters 1 --tailf 21 | 22 | ERROR: Cluster, "myk8s", exists! Aborting. 23 | 24 | real 0m0.116s 25 | user 0m0.073s 26 | sys 0m0.048s 27 | [root@mokbox /]# errstop $? 28 | SCREENCAST STOP & 29 | -------------------------------------------------------------------------------- /tests/e2e-logs/2020-06-06-1591401501_test-1_BAD_1.0000.log: -------------------------------------------------------------------------------- 1 | 2 | tests(master*)$ errstop() { [[ $1 -gt 0 ]] && { printf 'SCREENCAST STOP &'; read 3 | ; }; } 4 | tests(master*)$ date 5 | Sat 6 Jun 00:58:07 BST 2020 6 | tests(master*)$ alias mokbox='docker run --rm -ti --hostname mokbox --name mokbo 7 | x -v /var/run/docker.sock:/var/run/docker.sock -v /var/tmp:/var/tmp myownkind/mo 8 | kbox' 9 | tests(master*)$ mokbox 10 | [root@mokbox /]# retval=$? 11 | [root@mokbox /]# errstop() { [[ $1 -gt 0 ]] && { printf 'SCREENCAST STOP &'; rea 12 | d; }; } 13 | [root@mokbox /]# errstop $retval 14 | [root@mokbox /]# 15 | [root@mokbox /]# # Test 1 - download the prebuilt image and test 16 | [root@mokbox /]# 17 | [root@mokbox /]# mok -p build image --get-prebuilt-image 18 | Downloading base image, 'mok-centos-7-v1.18.3' .. SUCCESS 19 | [root@mokbox /]# errstop $? 20 | [root@mokbox /]# time mok -p create cluster myk8s --masters 1 --tailf 21 | 22 | ERROR: Cluster, "myk8s", exists! Aborting. 23 | 24 | real 0m0.111s 25 | user 0m0.071s 26 | sys 0m0.049s 27 | [root@mokbox /]# errstop $? 28 | SCREENCAST STOP & 29 | -------------------------------------------------------------------------------- /tests/e2e-logs/2020-06-06-1591401794_test-2_BAD_1.0000.log: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bashtools/mok/5ccedb9aea7b05d47e310003c157efdd7df782af/tests/e2e-logs/2020-06-06-1591401794_test-2_BAD_1.0000.log -------------------------------------------------------------------------------- /tests/e2e-logs/2020-06-06-1591403459_test-2_BAD_1.0000.log: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bashtools/mok/5ccedb9aea7b05d47e310003c157efdd7df782af/tests/e2e-logs/2020-06-06-1591403459_test-2_BAD_1.0000.log -------------------------------------------------------------------------------- /tests/e2e-logs/2020-06-06-1591403901_test-1_BAD_1.0000.log: -------------------------------------------------------------------------------- 1 | 2 | tests(master*)$ errstop() { [[ $1 -gt 0 ]] && { printf 'SCREENCAST STOP &'; read 3 | ; }; } 4 | tests(master*)$ date 5 | Sat 6 Jun 01:38:06 BST 2020 6 | tests(master*)$ alias mokbox='docker run --rm -ti --hostname mokbox --name mokbo 7 | x -v /var/run/docker.sock:/var/run/docker.sock -v /var/tmp:/var/tmp myownkind/mo 8 | kbox' 9 | tests(master*)$ mokbox 10 | [root@mokbox /]# retval=$? 11 | [root@mokbox /]# errstop() { [[ $1 -gt 0 ]] && { printf 'SCREENCAST STOP &'; rea 12 | d; }; } 13 | [root@mokbox /]# errstop $retval 14 | [root@mokbox /]# 15 | [root@mokbox /]# # Test 1 - download the prebuilt image and test 16 | [root@mokbox /]# 17 | [root@mokbox /]# mok -p build image --get-prebuilt-image 18 | Downloading base image, 'mok-centos-7-v1.18.3' .. SUCCESS 19 | [root@mokbox /]# errstop $? 20 | [root@mokbox /]# time mok -p create cluster myk8s --masters 1 --tailf 21 | 22 | ERROR: Cluster, "myk8s", exists! Aborting. 23 | 24 | real 0m0.122s 25 | user 0m0.086s 26 | sys 0m0.039s 27 | [root@mokbox /]# errstop $? 28 | SCREENCAST STOP & 29 | -------------------------------------------------------------------------------- /tests/e2e-logs/2020-06-06-1591404157_test-2_BAD_1.0000.log: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bashtools/mok/5ccedb9aea7b05d47e310003c157efdd7df782af/tests/e2e-logs/2020-06-06-1591404157_test-2_BAD_1.0000.log -------------------------------------------------------------------------------- /tests/e2e-logs/2020-06-06-1591404854_test-2_GOOD_1.0000.log: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bashtools/mok/5ccedb9aea7b05d47e310003c157efdd7df782af/tests/e2e-logs/2020-06-06-1591404854_test-2_GOOD_1.0000.log -------------------------------------------------------------------------------- /tests/e2e-logs/2020-06-07-1591549005_test-2_GOOD_1.0000.log: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bashtools/mok/5ccedb9aea7b05d47e310003c157efdd7df782af/tests/e2e-logs/2020-06-07-1591549005_test-2_GOOD_1.0000.log -------------------------------------------------------------------------------- /tests/e2e-logs/2020-06-07-1591553096_test-2_GOOD_1.0000.log: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bashtools/mok/5ccedb9aea7b05d47e310003c157efdd7df782af/tests/e2e-logs/2020-06-07-1591553096_test-2_GOOD_1.0000.log -------------------------------------------------------------------------------- /tests/e2e-logs/2020-06-08-1591652144_test-2_GOOD_.log: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bashtools/mok/5ccedb9aea7b05d47e310003c157efdd7df782af/tests/e2e-logs/2020-06-08-1591652144_test-2_GOOD_.log -------------------------------------------------------------------------------- /tests/e2e-logs/2020-06-26-1593173882_test-1_BAD_1.0000.log: -------------------------------------------------------------------------------- 1 | 2 | $ errstop() { [[ $1 -gt 0 ]] && { printf 'SCREENCAST STOP &'; read; }; } 3 | $ date 4 | Fri 26 Jun 13:17:49 BST 2020 5 | $ alias mokbox='docker run --rm -ti --hostname mokbox --name mokbox -v /var/run/docker. 6 | sock:/var/run/docker.sock -v /var/tmp:/var/tmp myownkind/mokbox' 7 | $ mokbox 8 | [root@mokbox /]# retval=$? 9 | [root@mokbox /]# errstop() { [[ $1 -gt 0 ]] && { printf 'SCREENCAST STOP &'; read; }; } 10 | [root@mokbox /]# errstop $retval 11 | [root@mokbox /]# 12 | [root@mokbox /]# # Test 1 - download the prebuilt image and test 13 | [root@mokbox /]# 14 | [root@mokbox /]# mok -p build image --get-prebuilt-image 15 | Downloading base image, 'mok-centos-7-v1.18.4' .. FAIL 16 | ERROR: Docker returned an error, shown below 17 | 18 | Using default tag: latest 19 | Error response from daemon: pull access denied for myownkind/mok-centos-7-v1.18.4, repo 20 | sitory does not exist or may require 'docker login': denied: requested access to the re 21 | source is denied 22 | 23 | Image build failed 24 | 25 | 3653 main /usr/bin/mok 26 | 29 MA_main /usr/bin/mok 27 | 2515 BI_run /usr/bin/mok 28 | 517 err /usr/bin/mok 29 | 30 | [root@mokbox /]# errstop $? 31 | SCREENCAST STOP & 32 | -------------------------------------------------------------------------------- /tests/e2e-logs/2020-06-26-1593173916_test-1_BAD_1.0000.log: -------------------------------------------------------------------------------- 1 | 2 | $ errstop() { [[ $1 -gt 0 ]] && { printf 'SCREENCAST STOP &'; read; }; } 3 | $ date 4 | Fri 26 Jun 13:18:24 BST 2020 5 | $ alias mokbox='docker run --rm -ti --hostname mokbox --name mokbox -v /var/run/docker. 6 | sock:/var/run/docker.sock -v /var/tmp:/var/tmp myownkind/mokbox' 7 | $ mokbox 8 | [root@mokbox /]# retval=$? 9 | [root@mokbox /]# errstop() { [[ $1 -gt 0 ]] && { printf 'SCREENCAST STOP &'; read; }; } 10 | [root@mokbox /]# errstop $retval 11 | [root@mokbox /]# 12 | [root@mokbox /]# # Test 1 - download the prebuilt image and test 13 | [root@mokbox /]# 14 | [root@mokbox /]# mok -p build image --get-prebuilt-image 15 | Downloading base image, 'mok-centos-7-v1.18.4' .. FAIL 16 | ERROR: Docker returned an error, shown below 17 | 18 | Using default tag: latest 19 | Error response from daemon: pull access denied for myownkind/mok-centos-7-v1.18.4, repo 20 | sitory does not exist or may require 'docker login': denied: requested access to the re 21 | source is denied 22 | 23 | Image build failed 24 | 25 | 3653 main /usr/bin/mok 26 | 29 MA_main /usr/bin/mok 27 | 2515 BI_run /usr/bin/mok 28 | 517 err /usr/bin/mok 29 | 30 | [root@mokbox /]# errstop $? 31 | SCREENCAST STOP & 32 | -------------------------------------------------------------------------------- /tests/e2e-logs/2020-06-26-1593173991_test-1_BAD_1.0000.log: -------------------------------------------------------------------------------- 1 | 2 | $ errstop() { [[ $1 -gt 0 ]] && { printf 'SCREENCAST STOP &'; read; }; } 3 | $ date 4 | Fri 26 Jun 13:19:39 BST 2020 5 | $ alias mokbox='docker run --rm -ti --hostname mokbox --name mokbox -v /var/run/docker. 6 | sock:/var/run/docker.sock -v /var/tmp:/var/tmp myownkind/mokbox' 7 | $ mokbox 8 | [root@mokbox /]# retval=$? 9 | [root@mokbox /]# errstop() { [[ $1 -gt 0 ]] && { printf 'SCREENCAST STOP &'; read; }; } 10 | [root@mokbox /]# errstop $retval 11 | [root@mokbox /]# 12 | [root@mokbox /]# # Test 1 - download the prebuilt image and test 13 | [root@mokbox /]# 14 | [root@mokbox /]# mok -p build image --get-prebuilt-image 15 | Downloading base image, 'mok-centos-7-v1.18.4' .. FAIL 16 | ERROR: Docker returned an error, shown below 17 | 18 | Using default tag: latest 19 | Error response from daemon: pull access denied for myownkind/mok-centos-7-v1.18.4, repo 20 | sitory does not exist or may require 'docker login': denied: requested access to the re 21 | source is denied 22 | 23 | Image build failed 24 | 25 | 3653 main /usr/bin/mok 26 | 29 MA_main /usr/bin/mok 27 | 2515 BI_run /usr/bin/mok 28 | 517 err /usr/bin/mok 29 | 30 | [root@mokbox /]# errstop $? 31 | SCREENCAST STOP & 32 | -------------------------------------------------------------------------------- /tests/e2e-logs/2020-06-26-1593175717_test-2_GOOD_1.0000.log: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bashtools/mok/5ccedb9aea7b05d47e310003c157efdd7df782af/tests/e2e-logs/2020-06-26-1593175717_test-2_GOOD_1.0000.log -------------------------------------------------------------------------------- /tests/e2e-logs/2020-06-27-1593276521_test-1_BAD_1.0000.log: -------------------------------------------------------------------------------- 1 | $ errstop() { [[ $1 -gt 0 ]] && { printf 'SCREENCAST STOP &'; read; }; } 2 | $ date 3 | Sat 27 Jun 17:48:29 BST 2020 4 | $ alias mokbox='docker run --rm -ti --hostname mokbox --name mokbox -v /var/run/ 5 | docker.sock:/var/run/docker.sock -v /var/tmp:/var/tmp myownkind/mokbox' 6 | $ mokbox 7 | [root@mokbox /]# retval=$? 8 | [root@mokbox /]# errstop() { [[ $1 -gt 0 ]] && { printf 'SCREENCAST STOP &'; rea 9 | d; }; } 10 | [root@mokbox /]# errstop $retval 11 | [root@mokbox /]# 12 | [root@mokbox /]# # Test 1 - download the prebuilt image and test 13 | [root@mokbox /]# 14 | [root@mokbox /]# mok -p build image --get-prebuilt-image 15 | Downloading base image, 'mok-centos-7-v1.18.5' .. FAIL 16 | ERROR: Docker returned an error, shown below 17 | 18 | Using default tag: latest 19 | Error response from daemon: pull access denied for myownkind/mok-centos-7-v1.18. 20 | 5, repository does not exist or may require 'docker login': denied: requested ac 21 | cess to the resource is denied 22 | 23 | Image build failed 24 | 25 | 3652 main /usr/bin/mok 26 | 29 MA_main /usr/bin/mok 27 | 2521 BI_run /usr/bin/mok 28 | 530 err /usr/bin/mok 29 | 30 | [root@mokbox /]# errstop $? 31 | SCREENCAST STOP & 32 | -------------------------------------------------------------------------------- /tests/e2e-logs/2020-06-27-1593276576_test-1_BAD_1.0000.log: -------------------------------------------------------------------------------- 1 | 2 | $ errstop() { [[ $1 -gt 0 ]] && { printf 'SCREENCAST STOP &'; read; }; } 3 | $ date 4 | Sat 27 Jun 17:49:23 BST 2020 5 | $ alias mokbox='docker run --rm -ti --hostname mokbox --name mokbox -v /var/run/ 6 | docker.sock:/var/run/docker.sock -v /var/tmp:/var/tmp myownkind/mokbox' 7 | $ mokbox 8 | [root@mokbox /]# retval=$? 9 | [root@mokbox /]# errstop() { [[ $1 -gt 0 ]] && { printf 'SCREENCAST STOP &'; rea 10 | d; }; } 11 | [root@mokbox /]# errstop $retval 12 | [root@mokbox /]# 13 | [root@mokbox /]# # Test 1 - download the prebuilt image and test 14 | [root@mokbox /]# 15 | [root@mokbox /]# mok -p build image --get-prebuilt-image 16 | Downloading base image, 'mok-centos-7-v1.18.5' .. FAIL 17 | ERROR: Docker returned an error, shown below 18 | 19 | Using default tag: latest 20 | Error response from daemon: pull access denied for myownkind/mok-centos-7-v1.18. 21 | 5, repository does not exist or may require 'docker login': denied: requested ac 22 | cess to the resource is denied 23 | 24 | Image build failed 25 | 26 | 3652 main /usr/bin/mok 27 | 29 MA_main /usr/bin/mok 28 | 2521 BI_run /usr/bin/mok 29 | 530 err /usr/bin/mok 30 | 31 | [root@mokbox /]# errstop $? 32 | SCREENCAST STOP & 33 | -------------------------------------------------------------------------------- /tests/e2e-logs/2020-06-27-1593278281_test-2_GOOD_1.0000.log: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bashtools/mok/5ccedb9aea7b05d47e310003c157efdd7df782af/tests/e2e-logs/2020-06-27-1593278281_test-2_GOOD_1.0000.log -------------------------------------------------------------------------------- /tests/e2e-logs/2020-06-27-1593282103_test-2_GOOD_1.0000.log: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bashtools/mok/5ccedb9aea7b05d47e310003c157efdd7df782af/tests/e2e-logs/2020-06-27-1593282103_test-2_GOOD_1.0000.log -------------------------------------------------------------------------------- /tests/e2e-logs/2020-09-16-1600251508_test-1_GOOD_1.0000.log: -------------------------------------------------------------------------------- 1 | 2 | % errstop() { [[ $1 -gt 0 ]] && { printf 'SCREENCAST STOP &'; read; }; } 3 | % date 4 | Wed 16 Sep 11:17:31 BST 2020 5 | % alias mokbox='docker run --rm -ti --hostname mokbox --name mokbox -v /var/run/ 6 | docker.sock:/var/run/docker.sock -v /var/tmp:/var/tmp myownkind/mokbox' 7 | % mokbox 8 | [root@mokbox /]# retval=$? 9 | [root@mokbox /]# errstop() { [[ $1 -gt 0 ]] && { printf 'SCREENCAST STOP &'; rea 10 | d; }; } 11 | [root@mokbox /]# errstop $retval 12 | [root@mokbox /]# 13 | [root@mokbox /]# # Test 1 - download the prebuilt image and test 14 | [root@mokbox /]# 15 | [root@mokbox /]# mok -p build image --get-prebuilt-image 16 | Downloading base image, 'mok-centos-7-v1.19.1' .. SUCCESS 17 | [root@mokbox /]# errstop $? 18 | [root@mokbox /]# time mok -p create cluster myk8s --masters 1 --tailf 19 | COMMAND: CU_create_container myk8s-master-1 MokCluster=myk8s 1.19.1 20 | 21 | 314ec414359e67fad38379dd75e7c986a85910f49bf6f0e5e5d9f1ef1a7d323a 22 | 23 | STATUS: OK 24 | 25 | COMMAND: _CC_set_up_master_node myk8s-master-1 26 | 27 | net.ipv6.conf.all.disable_ipv6 = 1 28 | net.ipv6.conf.default.disable_ipv6 = 1 29 | W0916 10:17:47.714671 129 configset.go:348] WARNING: kubeadm cannot validate 30 | component configs for API groups [kubelet.config.k8s.io kubeproxy.config.k8s.io 31 | ] 32 | [init] Using Kubernetes version: v1.19.1 33 | [preflight] Running pre-flight checks 34 | [WARNING Swap]: running with swap on is not supported. Please disable sw 35 | ap 36 | [WARNING Service-Kubelet]: kubelet service is not enabled, please run 's 37 | ystemctl enable kubelet.service' 38 | [preflight] Pulling images required for setting up a Kubernetes cluster 39 | [preflight] This might take a minute or two, depending on the speed of your inte 40 | rnet connection 41 | [preflight] You can also perform this action in beforehand using 'kubeadm config 42 | images pull' 43 | [certs] Using certificateDir folder "/etc/kubernetes/pki" 44 | [certs] Generating "ca" certificate and key 45 | [certs] Generating "apiserver" certificate and key 46 | [certs] apiserver serving cert is signed for DNS names [kubernetes kubernetes.de 47 | fault kubernetes.default.svc kubernetes.default.svc.cluster.local myk8s-master-1 48 | ] and IPs [10.96.0.1 172.17.0.3] 49 | [certs] Generating "apiserver-kubelet-client" certificate and key 50 | [certs] Generating "front-proxy-ca" certificate and key 51 | [certs] Generating "front-proxy-client" certificate and key 52 | [certs] Generating "etcd/ca" certificate and key 53 | [certs] Generating "etcd/server" certificate and key 54 | [certs] etcd/server serving cert is signed for DNS names [localhost myk8s-master 55 | -1] and IPs [172.17.0.3 127.0.0.1 ::1] 56 | [certs] Generating "etcd/peer" certificate and key 57 | [certs] etcd/peer serving cert is signed for DNS names [localhost myk8s-master-1 58 | ] and IPs [172.17.0.3 127.0.0.1 ::1] 59 | [certs] Generating "etcd/healthcheck-client" certificate and key 60 | [certs] Generating "apiserver-etcd-client" certificate and key 61 | [certs] Generating "sa" key and public key 62 | [kubeconfig] Using kubeconfig folder "/etc/kubernetes" 63 | [kubeconfig] Writing "admin.conf" kubeconfig file 64 | [kubeconfig] Writing "kubelet.conf" kubeconfig file 65 | [kubeconfig] Writing "controller-manager.conf" kubeconfig file 66 | [kubeconfig] Writing "scheduler.conf" kubeconfig file 67 | [kubelet-start] Writing kubelet environment file with flags to file "/var/lib/ku 68 | belet/kubeadm-flags.env" 69 | [kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.y 70 | aml" 71 | [kubelet-start] Starting the kubelet 72 | [control-plane] Using manifest folder "/etc/kubernetes/manifests" 73 | [control-plane] Creating static Pod manifest for "kube-apiserver" 74 | [control-plane] Creating static Pod manifest for "kube-controller-manager" 75 | [control-plane] Creating static Pod manifest for "kube-scheduler" 76 | [etcd] Creating static Pod manifest for local etcd in "/etc/kubernetes/manifests 77 | " 78 | [wait-control-plane] Waiting for the kubelet to boot up the control plane as sta 79 | tic Pods from directory "/etc/kubernetes/manifests". This can take up to 4m0s 80 | [apiclient] All control plane components are healthy after 22.004893 seconds 81 | [upload-config] Storing the configuration used in ConfigMap "kubeadm-config" in 82 | the "kube-system" Namespace 83 | [kubelet] Creating a ConfigMap "kubelet-config-1.19" in namespace kube-system wi 84 | th the configuration for the kubelets in the cluster 85 | [upload-certs] Skipping phase. Please see --upload-certs 86 | [mark-control-plane] Marking the node myk8s-master-1 as control-plane by adding 87 | the label "node-role.kubernetes.io/master=''" 88 | [mark-control-plane] Marking the node myk8s-master-1 as control-plane by adding 89 | the taints [node-role.kubernetes.io/master:NoSchedule] 90 | [bootstrap-token] Using token: abcdef.0123456789abcdef 91 | [bootstrap-token] Configuring bootstrap tokens, cluster-info ConfigMap, RBAC Rol 92 | es 93 | [bootstrap-token] configured RBAC rules to allow Node Bootstrap tokens to get no 94 | des 95 | [bootstrap-token] configured RBAC rules to allow Node Bootstrap tokens to post C 96 | SRs in order for nodes to get long term certificate credentials 97 | [bootstrap-token] configured RBAC rules to allow the csrapprover controller auto 98 | matically approve CSRs from a Node Bootstrap Token 99 | [bootstrap-token] configured RBAC rules to allow certificate rotation for all no 100 | de client certificates in the cluster 101 | [bootstrap-token] Creating the "cluster-info" ConfigMap in the "kube-public" nam 102 | espace 103 | [kubelet-finalize] Updating "/etc/kubernetes/kubelet.conf" to point to a rotatab 104 | le kubelet client certificate and key 105 | [addons] Applied essential addon: CoreDNS 106 | [addons] Applied essential addon: kube-proxy 107 | 108 | Your Kubernetes control-plane has initialized successfully! 109 | 110 | To start using your cluster, you need to run the following as a regular user: 111 | 112 | mkdir -p $HOME/.kube 113 | sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config 114 | sudo chown $(id -u):$(id -g) $HOME/.kube/config 115 | 116 | You should now deploy a pod network to the cluster. 117 | Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at: 118 | https://kubernetes.io/docs/concepts/cluster-administration/addons/ 119 | 120 | You can now join any number of control-plane nodes by copying certificate author 121 | ities 122 | and service account keys on each node and then running the following as root: 123 | 124 | kubeadm join 172.17.0.3:6443 --token abcdef.0123456789abcdef \ 125 | --discovery-token-ca-cert-hash sha256:16d12b4d52c3d6831d83c67f30aa000c0df10d 126 | 40eebb6f7e3bb1528c126be4f3 \ 127 | --control-plane 128 | 129 | Then you can join any number of worker nodes by running the following on each as 130 | root: 131 | 132 | kubeadm join 172.17.0.3:6443 --token abcdef.0123456789abcdef \ 133 | --discovery-token-ca-cert-hash sha256:16d12b4d52c3d6831d83c67f30aa000c0df10d 134 | 40eebb6f7e3bb1528c126be4f3 135 | % Total % Received % Xferd Average Speed Time Time Time Current 136 | Dload Upload Total Spent Left Speed 137 | 100 4831 100 4831 0 0 9801 0 --:--:-- --:--:-- --:--:-- 9799 138 | podsecuritypolicy.policy/psp.flannel.unprivileged created 139 | Warning: rbac.authorization.k8s.io/v1beta1 ClusterRole is deprecated in v1.17+, 140 | unavailable in v1.22+; use rbac.authorization.k8s.io/v1 ClusterRole 141 | clusterrole.rbac.authorization.k8s.io/flannel created 142 | Warning: rbac.authorization.k8s.io/v1beta1 ClusterRoleBinding is deprecated in v 143 | 1.17+, unavailable in v1.22+; use rbac.authorization.k8s.io/v1 ClusterRoleBindin 144 | g 145 | clusterrolebinding.rbac.authorization.k8s.io/flannel created 146 | serviceaccount/flannel created 147 | configmap/kube-flannel-cfg created 148 | daemonset.apps/kube-flannel-ds created 149 | Created symlink from /etc/systemd/system/multi-user.target.wants/kubelet.service 150 | to /usr/lib/systemd/system/kubelet.service. 151 | node/myk8s-master-1 untainted 152 | 153 | STATUS: OK 154 | 155 | Cluster, "myk8s", can be accessed using: 156 | 157 | export KUBECONFIG=/var/tmp/admin-myk8s.conf 158 | 159 | real 0m31.855s 160 | user 0m0.416s 161 | sys 0m0.299s 162 | [root@mokbox /]# errstop $? 163 | [root@mokbox /]# export KUBECONFIG=/var/tmp/admin-myk8s.conf 164 | [root@mokbox /]# for i in `seq 1 20`; do if kubectl get pods &>/dev/null; then b 165 | reak; fi; echo -n "."; sleep 1; done 166 | [root@mokbox /]# while kubectl get pods -A | tail -n +2 | awk '{ print $3; }' | 167 | grep -qs 0; do echo -n "."; sleep 2; done; echo 168 | No resources found 169 | 170 | [root@mokbox /]# kubectl get pods -A 171 | No resources found 172 | [root@mokbox /]# errstop $? 173 | [root@mokbox /]# kubectl run --rm -ti --restart=Never --image=alpine shell2 sh | 174 | | errstop $? 175 | Error from server (Forbidden): pods "shell2" is forbidden: error looking up serv 176 | ice account default/default: serviceaccount "default" not found 177 | SCREENCAST STOP & 178 | -------------------------------------------------------------------------------- /tests/e2e-logs/2020-09-16-1600251786_test-2_BAD_1.0000.log: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bashtools/mok/5ccedb9aea7b05d47e310003c157efdd7df782af/tests/e2e-logs/2020-09-16-1600251786_test-2_BAD_1.0000.log -------------------------------------------------------------------------------- /tests/e2e-logs/2020-09-16-1600270058_test-1_BAD_1.0000.log: -------------------------------------------------------------------------------- 1 | 2 | % errstop() { [[ $1 -gt 0 ]] && { printf 'SCREENCAST STOP &'; read; }; } 3 | % date 4 | Wed 16 Sep 16:27:22 BST 2020 5 | % alias mokbox='docker run --rm -ti --hostname mokbox --name mokbox -v /var/run 6 | /docker.sock:/var/run/docker.sock -v /var/tmp:/var/tmp myownkind/mokbox' 7 | % mokbox 8 | [root@mokbox /]# retval=$? 9 | [root@mokbox /]# errstop() { [[ $1 -gt 0 ]] && { printf 'SCREENCAST STOP &'; re 10 | ad; }; } 11 | [root@mokbox /]# errstop $retval 12 | [root@mokbox /]# 13 | [root@mokbox /]# # Test 1 - download the prebuilt image and test 14 | [root@mokbox /]# 15 | [root@mokbox /]# mok -p build image --get-prebuilt-image 16 | Downloading base image, 'mok-centos-7-v1.19.1' .. FAIL 17 | ERROR: Docker returned an error, shown below 18 | 19 | Using default tag: latest 20 | Error response from daemon: Get "https://registry-1.docker.io/v2/": dial tcp: l 21 | ookup registry-1.docker.io on 192.168.0.254:53: read udp 192.168.0.100:48538->1 22 | 92.168.0.254:53: i/o timeout 23 | 24 | Image build failed 25 | 26 | 3653 main /usr/bin/mok 27 | 29 MA_main /usr/bin/mok 28 | 2522 BI_run /usr/bin/mok 29 | 530 err /usr/bin/mok 30 | 31 | [root@mokbox /]# errstop $? 32 | SCREENCAST STOP & 33 | -------------------------------------------------------------------------------- /tests/e2e-logs/2020-09-16-1600270127_test-1_BAD_1.0000.log: -------------------------------------------------------------------------------- 1 | 2 | % errstop() { [[ $1 -gt 0 ]] && { printf 'SCREENCAST STOP &'; read; }; } 3 | % date 4 | Wed 16 Sep 16:28:27 BST 2020 5 | % alias mokbox='docker run --rm -ti --hostname mokbox --name mokbox -v /var/run 6 | /docker.sock:/var/run/docker.sock -v /var/tmp:/var/tmp myownkind/mokbox' 7 | % mokbox 8 | [root@mokbox /]# retval=$? 9 | [root@mokbox /]# errstop() { [[ $1 -gt 0 ]] && { printf 'SCREENCAST STOP &'; re 10 | ad; }; } 11 | [root@mokbox /]# errstop $retval 12 | [root@mokbox /]# 13 | [root@mokbox /]# # Test 1 - download the prebuilt image and test 14 | [root@mokbox /]# 15 | [root@mokbox /]# mok -p build image --get-prebuilt-image 16 | Downloading base image, 'mok-centos-7-v1.19.1' .. FAIL 17 | ERROR: Docker returned an error, shown below 18 | 19 | Using default tag: latest 20 | Error response from daemon: Get "https://registry-1.docker.io/v2/": dial tcp: l 21 | ookup registry-1.docker.io on 192.168.0.254:53: read udp 192.168.0.100:39134->1 22 | 92.168.0.254:53: i/o timeout 23 | 24 | Image build failed 25 | 26 | 3653 main /usr/bin/mok 27 | 29 MA_main /usr/bin/mok 28 | 2522 BI_run /usr/bin/mok 29 | 530 err /usr/bin/mok 30 | 31 | [root@mokbox /]# errstop $? 32 | SCREENCAST STOP & 33 | -------------------------------------------------------------------------------- /tests/e2e-logs/2020-09-16-1600280636_test-2_GOOD_1.0000.log: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bashtools/mok/5ccedb9aea7b05d47e310003c157efdd7df782af/tests/e2e-logs/2020-09-16-1600280636_test-2_GOOD_1.0000.log -------------------------------------------------------------------------------- /tests/e2e-logs/2020-09-16-1600282808_test-1_BAD_1.0000.log: -------------------------------------------------------------------------------- 1 | 2 | % errstop() { [[ $1 -gt 0 ]] && { printf 'SCREENCAST STOP &'; read; }; } 3 | % date 4 | Wed 16 Sep 19:59:54 BST 2020 5 | % alias mokbox='docker run --rm -ti --hostname mokbox --name mokbox -v /var/run/docker.sock:/var/run/docker.sock -v /var/tmp:/var/tmp myownkind/mokbox' 6 | % mokbox 7 | [root@mokbox /]# retval=$? 8 | [root@mokbox /]# errstop() { [[ $1 -gt 0 ]] && { printf 'SCREENCAST STOP &'; read; }; } 9 | [root@mokbox /]# errstop $retval 10 | [root@mokbox /]# 11 | [root@mokbox /]# # Test 1 - download the prebuilt image and test 12 | [root@mokbox /]# 13 | [root@mokbox /]# mok -p build image --get-prebuilt-image 14 | Downloading base image, 'mok-centos-7-v1.19.1' .. SUCCESS 15 | [root@mokbox /]# errstop $? 16 | [root@mokbox /]# time mok -p create cluster myk8s --masters 1 --tailf 17 | 18 | ERROR: Cluster, "myk8s", exists! Aborting. 19 | 20 | real 0m0.177s 21 | user 0m0.114s 22 | sys 0m0.076s 23 | [root@mokbox /]# errstop $? 24 | SCREENCAST STOP & 25 | -------------------------------------------------------------------------------- /tests/e2e-logs/2020-09-16-1600283125_test-1_GOOD_1.0000.log: -------------------------------------------------------------------------------- 1 | 2 | % errstop() { [[ $1 -gt 0 ]] && { printf 'SCREENCAST STOP &'; read; }; } 3 | % date 4 | Wed 16 Sep 20:02:05 BST 2020 5 | % alias mokbox='docker run --rm -ti --hostname mokbox --name mokbox -v /var/run/docker.sock:/var/run/docker.sock -v /var/tmp:/var/tmp myownkind/mokbox' 6 | % mokbox 7 | [root@mokbox /]# retval=$? 8 | [root@mokbox /]# errstop() { [[ $1 -gt 0 ]] && { printf 'SCREENCAST STOP &'; read; }; } 9 | [root@mokbox /]# errstop $retval 10 | [root@mokbox /]# 11 | [root@mokbox /]# # Test 1 - download the prebuilt image and test 12 | [root@mokbox /]# 13 | [root@mokbox /]# mok -p build image --get-prebuilt-image 14 | Downloading base image, 'mok-centos-7-v1.19.1' .. SUCCESS 15 | [root@mokbox /]# errstop $? 16 | [root@mokbox /]# time mok -p create cluster myk8s --masters 1 --tailf 17 | COMMAND: CU_create_container myk8s-master-1 MokCluster=myk8s 1.19.1 18 | 19 | 22c503a7cccc7de956b6886e660abd653a7ded68a138badcb0154e2eb0f238d0 20 | 21 | STATUS: OK 22 | 23 | COMMAND: _CC_set_up_master_node myk8s-master-1 24 | 25 | net.ipv6.conf.all.disable_ipv6 = 1 26 | net.ipv6.conf.default.disable_ipv6 = 1 27 | W0916 19:02:33.546301 128 configset.go:348] WARNING: kubeadm cannot validate component configs for API groups [kubelet.config.k8s.io kubeproxy.config.k8s.io] 28 | [init] Using Kubernetes version: v1.19.1 29 | [preflight] Running pre-flight checks 30 | [WARNING Swap]: running with swap on is not supported. Please disable swap 31 | [WARNING Service-Kubelet]: kubelet service is not enabled, please run 'systemctl enable kubelet.service' 32 | [preflight] Pulling images required for setting up a Kubernetes cluster 33 | [preflight] This might take a minute or two, depending on the speed of your internet connection 34 | [preflight] You can also perform this action in beforehand using 'kubeadm config images pull' 35 | [certs] Using certificateDir folder "/etc/kubernetes/pki" 36 | [certs] Generating "ca" certificate and key 37 | [certs] Generating "apiserver" certificate and key 38 | [certs] apiserver serving cert is signed for DNS names [kubernetes kubernetes.default kubernetes.default.svc kubernetes.default.svc.cluster.local myk8s-master-1] and IPs [10 39 | .96.0.1 172.17.0.3] 40 | [certs] Generating "apiserver-kubelet-client" certificate and key 41 | [certs] Generating "front-proxy-ca" certificate and key 42 | [certs] Generating "front-proxy-client" certificate and key 43 | [certs] Generating "etcd/ca" certificate and key 44 | [certs] Generating "etcd/server" certificate and key 45 | [certs] etcd/server serving cert is signed for DNS names [localhost myk8s-master-1] and IPs [172.17.0.3 127.0.0.1 ::1] 46 | [certs] Generating "etcd/peer" certificate and key 47 | [certs] etcd/peer serving cert is signed for DNS names [localhost myk8s-master-1] and IPs [172.17.0.3 127.0.0.1 ::1] 48 | [certs] Generating "etcd/healthcheck-client" certificate and key 49 | [certs] Generating "apiserver-etcd-client" certificate and key 50 | [certs] Generating "sa" key and public key 51 | [kubeconfig] Using kubeconfig folder "/etc/kubernetes" 52 | [kubeconfig] Writing "admin.conf" kubeconfig file 53 | [kubeconfig] Writing "kubelet.conf" kubeconfig file 54 | [kubeconfig] Writing "controller-manager.conf" kubeconfig file 55 | [kubeconfig] Writing "scheduler.conf" kubeconfig file 56 | [kubelet-start] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env" 57 | [kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml" 58 | [kubelet-start] Starting the kubelet 59 | [control-plane] Using manifest folder "/etc/kubernetes/manifests" 60 | [control-plane] Creating static Pod manifest for "kube-apiserver" 61 | [control-plane] Creating static Pod manifest for "kube-controller-manager" 62 | [control-plane] Creating static Pod manifest for "kube-scheduler" 63 | [etcd] Creating static Pod manifest for local etcd in "/etc/kubernetes/manifests" 64 | [wait-control-plane] Waiting for the kubelet to boot up the control plane as static Pods from directory "/etc/kubernetes/manifests". This can take up to 4m0s 65 | [apiclient] All control plane components are healthy after 23.007884 seconds 66 | [upload-config] Storing the configuration used in ConfigMap "kubeadm-config" in the "kube-system" Namespace 67 | [kubelet] Creating a ConfigMap "kubelet-config-1.19" in namespace kube-system with the configuration for the kubelets in the cluster 68 | [upload-certs] Skipping phase. Please see --upload-certs 69 | [mark-control-plane] Marking the node myk8s-master-1 as control-plane by adding the label "node-role.kubernetes.io/master=''" 70 | [mark-control-plane] Marking the node myk8s-master-1 as control-plane by adding the taints [node-role.kubernetes.io/master:NoSchedule] 71 | [bootstrap-token] Using token: abcdef.0123456789abcdef 72 | [bootstrap-token] Configuring bootstrap tokens, cluster-info ConfigMap, RBAC Roles 73 | [bootstrap-token] configured RBAC rules to allow Node Bootstrap tokens to get nodes 74 | [bootstrap-token] configured RBAC rules to allow Node Bootstrap tokens to post CSRs in order for nodes to get long term certificate credentials 75 | [bootstrap-token] configured RBAC rules to allow the csrapprover controller automatically approve CSRs from a Node Bootstrap Token 76 | [bootstrap-token] configured RBAC rules to allow certificate rotation for all node client certificates in the cluster 77 | [bootstrap-token] Creating the "cluster-info" ConfigMap in the "kube-public" namespace 78 | [kubelet-finalize] Updating "/etc/kubernetes/kubelet.conf" to point to a rotatable kubelet client certificate and key 79 | [addons] Applied essential addon: CoreDNS 80 | [addons] Applied essential addon: kube-proxy 81 | 82 | Your Kubernetes control-plane has initialized successfully! 83 | 84 | To start using your cluster, you need to run the following as a regular user: 85 | 86 | mkdir -p $HOME/.kube 87 | sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config 88 | sudo chown $(id -u):$(id -g) $HOME/.kube/config 89 | 90 | You should now deploy a pod network to the cluster. 91 | Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at: 92 | https://kubernetes.io/docs/concepts/cluster-administration/addons/ 93 | 94 | You can now join any number of control-plane nodes by copying certificate authorities 95 | and service account keys on each node and then running the following as root: 96 | 97 | kubeadm join 172.17.0.3:6443 --token abcdef.0123456789abcdef \ 98 | --discovery-token-ca-cert-hash sha256:93c6d54d9260f2238ba4a7adde6b416767264d32c91aeddf1485dc2d63261ebc \ 99 | --control-plane 100 | 101 | Then you can join any number of worker nodes by running the following on each as root: 102 | 103 | kubeadm join 172.17.0.3:6443 --token abcdef.0123456789abcdef \ 104 | --discovery-token-ca-cert-hash sha256:93c6d54d9260f2238ba4a7adde6b416767264d32c91aeddf1485dc2d63261ebc 105 | % Total % Received % Xferd Average Speed Time Time Time Current 106 | Dload Upload Total Spent Left Speed 107 | 100 4831 100 4831 0 0 8579 0 --:--:-- --:--:-- --:--:-- 8580 108 | podsecuritypolicy.policy/psp.flannel.unprivileged created 109 | Warning: rbac.authorization.k8s.io/v1beta1 ClusterRole is deprecated in v1.17+, unavailable in v1.22+; use rbac.authorization.k8s.io/v1 ClusterRole 110 | clusterrole.rbac.authorization.k8s.io/flannel created 111 | Warning: rbac.authorization.k8s.io/v1beta1 ClusterRoleBinding is deprecated in v1.17+, unavailable in v1.22+; use rbac.authorization.k8s.io/v1 ClusterRoleBinding 112 | clusterrolebinding.rbac.authorization.k8s.io/flannel created 113 | serviceaccount/flannel created 114 | configmap/kube-flannel-cfg created 115 | daemonset.apps/kube-flannel-ds created 116 | Created symlink from /etc/systemd/system/multi-user.target.wants/kubelet.service to /usr/lib/systemd/system/kubelet.service. 117 | node/myk8s-master-1 untainted 118 | 119 | STATUS: OK 120 | 121 | Cluster, "myk8s", can be accessed using: 122 | 123 | export KUBECONFIG=/var/tmp/admin-myk8s.conf 124 | 125 | real 0m36.815s 126 | user 0m0.509s 127 | sys 0m0.349s 128 | [root@mokbox /]# errstop $? 129 | [root@mokbox /]# export KUBECONFIG=/var/tmp/admin-myk8s.conf 130 | [root@mokbox /]# sleep 5 131 | [root@mokbox /]# for i in `seq 1 20`; do if kubectl get pods &>/dev/null; then break; fi; echo -n "."; sleep 1; done 132 | [root@mokbox /]# sleep 5 133 | [root@mokbox /]# while kubectl get pods -A | tail -n +2 | awk '{ print $3; }' | grep -qs 0; do echo -n "."; sleep 2; done; echo 134 | ...................................... 135 | [root@mokbox /]# kubectl get pods -A 136 | NAMESPACE NAME READY STATUS RESTARTS AGE 137 | kube-system coredns-f9fd979d6-4dggb 1/1 Running 0 86s 138 | kube-system coredns-f9fd979d6-kdxmr 1/1 Running 0 86s 139 | kube-system etcd-myk8s-master-1 1/1 Running 0 91s 140 | kube-system kube-apiserver-myk8s-master-1 1/1 Running 0 91s 141 | kube-system kube-controller-manager-myk8s-master-1 1/1 Running 0 91s 142 | kube-system kube-flannel-ds-qdlnr 1/1 Running 0 86s 143 | kube-system kube-proxy-s56nr 1/1 Running 0 86s 144 | kube-system kube-scheduler-myk8s-master-1 1/1 Running 0 91s 145 | [root@mokbox /]# errstop $? 146 | [root@mokbox /]# kubectl run --rm -ti --restart=Never --image=alpine shell2 sh || errstop $? 147 | If you don't see a command prompt, try pressing enter. 148 | / # ip address 149 | 1: lo: mtu 65536 qdisc noqueue state UNKNOWN qlen 1000 150 | link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 151 | inet 127.0.0.1/8 scope host lo 152 | valid_lft forever preferred_lft forever 153 | inet6 ::1/128 scope host 154 | valid_lft forever preferred_lft forever 155 | 3: eth0@if6: mtu 1450 qdisc noqueue state UP 156 | link/ether 66:f8:f4:ce:34:78 brd ff:ff:ff:ff:ff:ff 157 | inet 10.244.0.4/24 brd 10.244.0.255 scope global eth0 158 | valid_lft forever preferred_lft forever 159 | inet6 fe80::64f8:f4ff:fece:3478/64 scope link 160 | valid_lft forever preferred_lft forever 161 | / # apk add fortune 162 | fetch http://dl-cdn.alpinelinux.org/alpine/v3.12/main/x86_64/APKINDEX.tar.gz 163 | fetch http://dl-cdn.alpinelinux.org/alpine/v3.12/community/x86_64/APKINDEX.tar.gz 164 | (1/2) Installing libbsd (0.10.0-r0) 165 | (2/2) Installing fortune (0.1-r1) 166 | 3% ###### 167 | Executing busybox-1.31.1-r16.trigger 168 | ERROR: busybox-1.31.1-r16.trigger: script exited with error 127 169 | OK: 9 MiB in 16 packages 170 | / # exit 0 171 | pod "shell2" deleted 172 | [root@mokbox /]# errstop $? 173 | [root@mokbox /]# mok -p get cluster 174 | MOK_Cluster Docker_ID Container_Name IP_Address 175 | myk8s 22c503a7cccc myk8s-master-1 172.17.0.3 176 | [root@mokbox /]# errstop $? 177 | [root@mokbox /]# mok -p delete cluster myk8s || errstop $? 178 | The following containers will be deleted: 179 | 180 | MOK_Cluster Docker_ID Container_Name IP_Address 181 | myk8s 22c503a7cccc myk8s-master-1 172.17.0.3 182 | 183 | Are you sure you want to delete the cluster? (y/N) >y 184 | 185 | Deleting id, '22c503a7cccc' from cluster 'myk8s'. .. SUCCESS 186 | 187 | [root@mokbox /]# mok -p get cluster 188 | [root@mokbox /]# errstop $? 189 | [root@mokbox /]# exit 190 | exit 191 | % 192 | % # Test complete 193 | % date 194 | Wed 16 Sep 20:05:20 BST 2020 195 | % 196 | -------------------------------------------------------------------------------- /tests/e2e-logs/2020-09-16-1600283316_test-2_GOOD_1.0000.log: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bashtools/mok/5ccedb9aea7b05d47e310003c157efdd7df782af/tests/e2e-logs/2020-09-16-1600283316_test-2_GOOD_1.0000.log -------------------------------------------------------------------------------- /tests/e2e-logs/2020-09-16-1600285349_test-2_GOOD_1.0000.log: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bashtools/mok/5ccedb9aea7b05d47e310003c157efdd7df782af/tests/e2e-logs/2020-09-16-1600285349_test-2_GOOD_1.0000.log -------------------------------------------------------------------------------- /tests/e2e-logs/training/2020-06-05-1591359418_test-1_BAD_1.0000.log: -------------------------------------------------------------------------------- 1 | 2 | tests(master*)$ errstop() { [[ $1 -gt 0 ]] && echo "SCREENCAST STOP &"; read; } 3 | tests(master*)$ date 4 | Fri 5 Jun 13:16:53 BST 2020 5 | tests(master*)$ alias mokbox='docker run --rm -ti --hostname mokbox --name mokbox -v /var/run/docker.sock:/var/run/dock 6 | er.sock -v /var/tmp:/var/tmp myownkind/mokbox' 7 | tests(master*)$ mokbox 8 | docker: Error response from daemon: write /run/media/mclarkson/ICYBOX/var-lib-docker/containerd/daemon/io.containerd.me 9 | tadata.v1.bolt/meta.db: read-only file system: unknown. 10 | tests(master*)$ retval=$? 11 | tests(master*)$ errstop() { [[ $1 -gt 0 ]] && { printf 'SCREENCAST STOP &'; read; }; } 12 | tests(master*)$ errstop $retval 13 | SCREENCAST STOP & 14 | 15 | -------------------------------------------------------------------------------- /tests/e2e-logs/training/2020-06-05-1591359857_test-1_BAD_1.0000.log: -------------------------------------------------------------------------------- 1 | 2 | tests(master*)$ errstop() { [[ $1 -gt 0 ]] && echo "SCREENCAST STOP &"; read; } 3 | tests(master*)$ date 4 | Fri 5 Jun 13:23:28 BST 2020 5 | tests(master*)$ alias mokbox='docker run --rm -ti --hostname mokbox --name mokbox -v /var/run/docker.sock:/var/run/dock 6 | er.sock -v /var/tmp:/var/tmp myownkind/mokbox' 7 | tests(master*)$ mokbox 8 | [root@mokbox /]# retval=$? 9 | [root@mokbox /]# errstop() { [[ $1 -gt 0 ]] && { printf 'SCREENCAST STOP &'; read; }; } 10 | [root@mokbox /]# errstop $retval 11 | [root@mokbox /]# 12 | [root@mokbox /]# # Test 1 - download the prebuilt image 13 | [root@mokbox /]# 14 | [root@mokbox /]# mok -p build image --get-prebuilt-image 15 | Downloading base image, 'mok-centos-7-v1.18.3' .. SUCCESS 16 | [root@mokbox /]# errstop $? 17 | [root@mokbox /]# time mok -p create cluster myk8s --masters 1 --tailf 18 | 2db21b0060eb6674b2aa1cb3d7f7a82c4962a9d24d8119028fa3a8e5bc2dc681 19 | 20 | STATUS: OK 21 | 22 | net.ipv6.conf.all.disable_ipv6 = 1 23 | net.ipv6.conf.default.disable_ipv6 = 1 24 | W0605 12:23:57.541472 52 configset.go:202] WARNING: kubeadm cannot validate component configs for API groups [kube 25 | let.config.k8s.io kubeproxy.config.k8s.io] 26 | [init] Using Kubernetes version: v1.18.3 27 | [preflight] Running pre-flight checks 28 | [WARNING Swap]: running with swap on is not supported. Please disable swap 29 | [WARNING Service-Kubelet]: kubelet service is not enabled, please run 'systemctl enable kubelet.service' 30 | error execution phase preflight: [preflight] Some fatal errors occurred: 31 | [ERROR CRI]: container runtime is not running: output: time="2020-06-05T12:24:03Z" level=fatal msg="failed to c 32 | onnect: failed to connect, make sure you are running as root and the runtime has been started: context deadline exceede 33 | d" 34 | , error: exit status 1 35 | [preflight] If you know what you are doing, you can make a check non-fatal with `--ignore-preflight-errors=...` 36 | To see the stack trace of this error execute with --v=5 or higher 37 | % Total % Received % Xferd Average Speed Time Time Time Current 38 | Dload Upload Total Spent Left Speed 39 | 100 14366 100 14366 0 0 6515 0 0:00:02 0:00:02 --:--:-- 6515 40 | W0605 12:24:14.028712 180 loader.go:223] Config not found: /etc/kubernetes/admin.conf 41 | The connection to the server localhost:8080 was refused - did you specify the right host or port? 42 | Created symlink from /etc/systemd/system/multi-user.target.wants/kubelet.service to /usr/lib/systemd/system/kubelet.ser 43 | vice. 44 | W0605 12:24:15.621325 206 loader.go:223] Config not found: /etc/kubernetes/admin.conf 45 | The connection to the server localhost:8080 was refused - did you specify the right host or port? 46 | 47 | 3549 main /usr/bin/mok 48 | 27 MA_main /usr/bin/mok 49 | 1381 CC_run /usr/bin/mok 50 | 1500 _CC_setup_master_nodes /usr/bin/mok 51 | 661 UT_run_with_progress /usr/bin/mok 52 | 1544 _CC_set_up_master_node /usr/bin/mok 53 | 1984 _CC_set_up_master_node_v1_18_3 /usr/bin/mok 54 | 517 err /usr/bin/mok 55 | 56 | STATUS: OK 57 | 58 | Error: No such container:path: myk8s-master-1:/etc/kubernetes/admin.conf 59 | 60 | 3549 main /usr/bin/mok 61 | 27 MA_main /usr/bin/mok 62 | 1381 CC_run /usr/bin/mok 63 | 1526 _CC_setup_master_nodes /usr/bin/mok 64 | 517 err /usr/bin/mok 65 | 66 | real 0m30.338s 67 | user 0m0.433s 68 | sys 0m0.323s 69 | [root@mokbox /]# errstop $? 70 | SCREENCAST STOP & 71 | -------------------------------------------------------------------------------- /tests/e2e-logs/training/2020-06-06-1591400374_test-2_BAD_1.0000.log: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bashtools/mok/5ccedb9aea7b05d47e310003c157efdd7df782af/tests/e2e-logs/training/2020-06-06-1591400374_test-2_BAD_1.0000.log -------------------------------------------------------------------------------- /tests/e2e-logs/training/2020-06-06-1591403178_test-1_BAD_1.0000.log: -------------------------------------------------------------------------------- 1 | 2 | tests(master*)$ errstop() { [[ $1 -gt 0 ]] && { printf 'SCREENCAST STOP &'; read 3 | ; }; } 4 | tests(master*)$ date 5 | Sat 6 Jun 01:24:03 BST 2020 6 | tests(master*)$ alias mokbox='docker run --rm -ti --hostname mokbox --name mokbo 7 | x -v /var/run/docker.sock:/var/run/docker.sock -v /var/tmp:/var/tmp myownkind/mo 8 | kbox' 9 | tests(master*)$ mokbox 10 | [root@mokbox /]# retval=$? 11 | [root@mokbox /]# errstop() { [[ $1 -gt 0 ]] && { printf 'SCREENCAST STOP &'; rea 12 | d; }; } 13 | [root@mokbox /]# errstop $retval 14 | [root@mokbox /]# 15 | [root@mokbox /]# # Test 1 - download the prebuilt image and test 16 | [root@mokbox /]# 17 | [root@mokbox /]# mok -p build image --get-prebuilt-image 18 | Downloading base image, 'mok-centos-7-v1.18.3' .. SUCCESS 19 | [root@mokbox /]# errstop $? 20 | [root@mokbox /]# time mok -p create cluster myk8s --masters 1 --tailf 21 | 19455b1e691ee6dd6e41aeac0977a8ba3754db426c9f221bc777ef1f90655262 22 | 23 | STATUS: OK 24 | 25 | net.ipv6.conf.all.disable_ipv6 = 1 26 | net.ipv6.conf.default.disable_ipv6 = 1 27 | W0606 00:24:18.892519 146 configset.go:202] WARNING: kubeadm cannot validate 28 | component configs for API groups [kubelet.config.k8s.io kubeproxy.config.k8s.io 29 | ] 30 | [init] Using Kubernetes version: v1.18.3 31 | [preflight] Running pre-flight checks 32 | [WARNING Swap]: running with swap on is not supported. Please disable sw 33 | ap 34 | [WARNING Service-Kubelet]: kubelet service is not enabled, please run 's 35 | ystemctl enable kubelet.service' 36 | [preflight] Pulling images required for setting up a Kubernetes cluster 37 | [preflight] This might take a minute or two, depending on the speed of your inte 38 | rnet connection 39 | [preflight] You can also perform this action in beforehand using 'kubeadm config 40 | images pull' 41 | [kubelet-start] Writing kubelet environment file with flags to file "/var/lib/ku 42 | belet/kubeadm-flags.env" 43 | [kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.y 44 | aml" 45 | [kubelet-start] Starting the kubelet 46 | [certs] Using certificateDir folder "/etc/kubernetes/pki" 47 | [certs] Generating "ca" certificate and key 48 | [certs] Generating "apiserver" certificate and key 49 | [certs] apiserver serving cert is signed for DNS names [myk8s-master-1 kubernete 50 | s kubernetes.default kubernetes.default.svc kubernetes.default.svc.cluster.local 51 | ] and IPs [10.96.0.1 172.17.0.3 172.17.0.3] 52 | [certs] Generating "apiserver-kubelet-client" certificate and key 53 | [certs] Generating "front-proxy-ca" certificate and key 54 | [certs] Generating "front-proxy-client" certificate and key 55 | [certs] Generating "etcd/ca" certificate and key 56 | [certs] Generating "etcd/server" certificate and key 57 | [certs] etcd/server serving cert is signed for DNS names [myk8s-master-1 localho 58 | st] and IPs [172.17.0.3 127.0.0.1 ::1] 59 | [certs] Generating "etcd/peer" certificate and key 60 | [certs] etcd/peer serving cert is signed for DNS names [myk8s-master-1 localhost 61 | ] and IPs [172.17.0.3 127.0.0.1 ::1] 62 | [certs] Generating "etcd/healthcheck-client" certificate and key 63 | [certs] Generating "apiserver-etcd-client" certificate and key 64 | [certs] Generating "sa" key and public key 65 | [kubeconfig] Using kubeconfig folder "/etc/kubernetes" 66 | [kubeconfig] Writing "admin.conf" kubeconfig file 67 | [kubeconfig] Writing "kubelet.conf" kubeconfig file 68 | [kubeconfig] Writing "controller-manager.conf" kubeconfig file 69 | [kubeconfig] Writing "scheduler.conf" kubeconfig file 70 | [control-plane] Using manifest folder "/etc/kubernetes/manifests" 71 | [control-plane] Creating static Pod manifest for "kube-apiserver" 72 | [control-plane] Creating static Pod manifest for "kube-controller-manager" 73 | W0606 00:24:24.620135 146 manifests.go:225] the default kube-apiserver autho 74 | rization-mode is "Node,RBAC"; using "Node,RBAC" 75 | [control-plane] Creating static Pod manifest for "kube-scheduler" 76 | W0606 00:24:24.621558 146 manifests.go:225] the default kube-apiserver autho 77 | rization-mode is "Node,RBAC"; using "Node,RBAC" 78 | [etcd] Creating static Pod manifest for local etcd in "/etc/kubernetes/manifests 79 | " 80 | [wait-control-plane] Waiting for the kubelet to boot up the control plane as sta 81 | tic Pods from directory "/etc/kubernetes/manifests". This can take up to 4m0s 82 | [apiclient] All control plane components are healthy after 26.505639 seconds 83 | [upload-config] Storing the configuration used in ConfigMap "kubeadm-config" in 84 | the "kube-system" Namespace 85 | [kubelet] Creating a ConfigMap "kubelet-config-1.18" in namespace kube-system wi 86 | th the configuration for the kubelets in the cluster 87 | [upload-certs] Skipping phase. Please see --upload-certs 88 | [mark-control-plane] Marking the node myk8s-master-1 as control-plane by adding 89 | the label "node-role.kubernetes.io/master=''" 90 | [mark-control-plane] Marking the node myk8s-master-1 as control-plane by adding 91 | the taints [node-role.kubernetes.io/master:NoSchedule] 92 | [bootstrap-token] Using token: abcdef.0123456789abcdef 93 | [bootstrap-token] Configuring bootstrap tokens, cluster-info ConfigMap, RBAC Rol 94 | es 95 | [bootstrap-token] configured RBAC rules to allow Node Bootstrap tokens to get no 96 | des 97 | [bootstrap-token] configured RBAC rules to allow Node Bootstrap tokens to post C 98 | SRs in order for nodes to get long term certificate credentials 99 | [bootstrap-token] configured RBAC rules to allow the csrapprover controller auto 100 | matically approve CSRs from a Node Bootstrap Token 101 | [bootstrap-token] configured RBAC rules to allow certificate rotation for all no 102 | de client certificates in the cluster 103 | [bootstrap-token] Creating the "cluster-info" ConfigMap in the "kube-public" nam 104 | espace 105 | [kubelet-finalize] Updating "/etc/kubernetes/kubelet.conf" to point to a rotatab 106 | le kubelet client certificate and key 107 | [addons] Applied essential addon: CoreDNS 108 | [addons] Applied essential addon: kube-proxy 109 | 110 | Your Kubernetes control-plane has initialized successfully! 111 | 112 | To start using your cluster, you need to run the following as a regular user: 113 | 114 | mkdir -p $HOME/.kube 115 | sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config 116 | sudo chown $(id -u):$(id -g) $HOME/.kube/config 117 | 118 | You should now deploy a pod network to the cluster. 119 | Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at: 120 | https://kubernetes.io/docs/concepts/cluster-administration/addons/ 121 | 122 | You can now join any number of control-plane nodes by copying certificate author 123 | ities 124 | and service account keys on each node and then running the following as root: 125 | 126 | kubeadm join 172.17.0.3:6443 --token abcdef.0123456789abcdef \ 127 | --discovery-token-ca-cert-hash sha256:8d148a593a78ee901e994f8a009585c13159da 128 | 80f7b92115493860a47659ed50 \ 129 | --control-plane 130 | 131 | Then you can join any number of worker nodes by running the following on each as 132 | root: 133 | 134 | kubeadm join 172.17.0.3:6443 --token abcdef.0123456789abcdef \ 135 | --discovery-token-ca-cert-hash sha256:8d148a593a78ee901e994f8a009585c13159da 136 | 80f7b92115493860a47659ed50 137 | % Total % Received % Xferd Average Speed Time Time Time Current 138 | Dload Upload Total Spent Left Speed 139 | 100 14366 100 14366 0 0 30209 0 --:--:-- --:--:-- --:--:-- 30180 140 | podsecuritypolicy.policy/psp.flannel.unprivileged created 141 | clusterrole.rbac.authorization.k8s.io/flannel created 142 | clusterrolebinding.rbac.authorization.k8s.io/flannel created 143 | serviceaccount/flannel created 144 | configmap/kube-flannel-cfg created 145 | daemonset.apps/kube-flannel-ds-amd64 created 146 | daemonset.apps/kube-flannel-ds-arm64 created 147 | daemonset.apps/kube-flannel-ds-arm created 148 | daemonset.apps/kube-flannel-ds-ppc64le created 149 | daemonset.apps/kube-flannel-ds-s390x created 150 | Created symlink from /etc/systemd/system/multi-user.target.wants/kubelet.service 151 | to /usr/lib/systemd/system/kubelet.service. 152 | node/myk8s-master-1 untainted 153 | 154 | STATUS: OK 155 | 156 | Cluster, "myk8s", can be accessed using: 157 | 158 | export KUBECONFIG=/var/tmp/admin-myk8s.conf 159 | 160 | real 0m37.667s 161 | user 0m0.383s 162 | sys 0m0.282s 163 | [root@mokbox /]# errstop $? 164 | [root@mokbox /]# export KUBECONFIG=/var/tmp/admin-myk8s.conf 165 | [root@mokbox /]# for i in `seq 1 20`; do if kubectl get pods &>/dev/null; then b 166 | reak; fi; echo -n "."; sleep 1; done 167 | [root@mokbox /]# while kubectl get pods -A | tail -n +2 | awk '{ print $3; }' | 168 | grep -qs 0; do echo -n "."; sleep 2; done; echo 169 | ..................... 170 | [root@mokbox /]# kubectl get pods -A 171 | NAMESPACE NAME READY STATUS RESTART 172 | S AGE 173 | kube-system coredns-66bff467f8-bssr8 1/1 Running 0 174 | 36s 175 | kube-system coredns-66bff467f8-cv6ns 1/1 Running 0 176 | 36s 177 | kube-system etcd-myk8s-master-1 1/1 Running 0 178 | 46s 179 | kube-system kube-apiserver-myk8s-master-1 1/1 Running 0 180 | 46s 181 | kube-system kube-controller-manager-myk8s-master-1 1/1 Running 0 182 | 46s 183 | kube-system kube-flannel-ds-amd64-k5v5q 1/1 Running 0 184 | 36s 185 | kube-system kube-proxy-7d7hc 1/1 Running 0 186 | 36s 187 | kube-system kube-scheduler-myk8s-master-1 1/1 Running 0 188 | 46s 189 | [root@mokbox /]# errstop $? 190 | [root@mokbox /]# kubectl run --rm -ti --restart=Never --image=alpine shell2 sh | 191 | | errstop $? 192 | If you don't see a command prompt, try pressing enter. 193 | / # ip address 194 | 1: lo: mtu 65536 qdisc noqueue state UNKNOWN qlen 1000 195 | link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 196 | inet 127.0.0.1/8 scope host lo 197 | valid_lft forever preferred_lft forever 198 | inet6 ::1/128 scope host 199 | valid_lft forever preferred_lft forever 200 | 3: eth0@if6: mtu 1450 qdisc noqueue sta 201 | te UP 202 | link/ether 72:59:26:45:23:49 brd ff:ff:ff:ff:ff:ff 203 | inet 10.244.0.4/24 brd 10.244.0.255 scope global eth0 204 | valid_lft forever preferred_lft forever 205 | inet6 fe80::7059:26ff:fe45:2349/64 scope link 206 | valid_lft forever preferred_lft forever 207 | / # apk add fortune 208 | fetch http://dl-cdn.alpinelinux.org/alpine/v3.12/main/x86_64/APKINDEX.tar.gz 209 | fetch http://dl-cdn.alpinelinux.org/alpine/v3.12/community/x86_64/APKINDEX.tar.g 210 | z 211 | (1/2) Installing libbsd (0.10.0-r0) 212 | 0% 213 | (2/2) Installing fortune (0.1-r1) 214 | 3% ## 215 | 23% ################# 216 | 73% ###################################################### 217 | Executing busybox-1.31.1-r16.trigger 218 | ERROR: busybox-1.31.1-r16.trigger: script exited with error 127 219 | OK: 9 MiB in 16 packages 220 | / # exit 0 221 | 222 | E0606 00:26:07.796913 475 v2.go:105] EOF 223 | pod "shell2" deleted 224 | [root@mokbox /]# y 225 | bash: y: command not found 226 | [root@mokbox /]# mok -p get cluster 227 | MOK_Cluster Docker_ID Container_Name IP_Address 228 | myk8s 19455b1e691e myk8s-master-1 172.17.0.3 229 | [root@mokbox /]# exit 230 | exit 231 | tests(master*)$ 232 | tests(master*)$ # Test complete 233 | tests(master*)$ date 234 | Sat 6 Jun 01:26:14 BST 2020 235 | tests(master*)$ 236 | -------------------------------------------------------------------------------- /tests/e2e-logs/training/README.md: -------------------------------------------------------------------------------- 1 | These logs were ioriginally misclassified. 2 | Train crm114 with these files according to the GOOD/BAD status in the file's name. 3 | -------------------------------------------------------------------------------- /tests/e2e-tests.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | declare -r TRUE=1 ERROR=1 STDERR="/dev/stderr" 4 | 5 | err() { 6 | 7 | # In case of error print the function call stack 8 | # No args expected 9 | 10 | [[ ${ER[errcalled]} == "${TRUE}" ]] && return "${ERROR}" 11 | ER[errcalled]="${TRUE}" 12 | local frame=0 13 | printf '\n' >"${STDERR}" 14 | while caller "${frame}"; do 15 | ((frame++)) 16 | done | tac >"${STDERR}" 17 | printf '\n' >"${STDERR}" 18 | return "${ERROR}" 19 | } 20 | 21 | classify_status() { 22 | cd e2e/crm114/ || err || exit 1 23 | crm classify.crm <../../hardcopy | sed -n '2p' | grep -Eo '(GOOD|BAD)' 24 | } 25 | 26 | classify_probability() { 27 | cd e2e/crm114/ || err || exit 1 28 | crm classify.crm <../../hardcopy | sed -n '2p' | sed -r 's/^.*prob: ([^ ]+).*/\1/' 29 | } 30 | 31 | save_hardcopy() { 32 | local date testnum status probability 33 | 34 | date=$(date +%Y-%m-%d-%s) 35 | testnum="$1" 36 | status="$2" 37 | probability="$3" 38 | 39 | mkdir -p e2e-logs 40 | cat -s hardcopy >"e2e-logs/${date}_test-${testnum}_${status}_${probability}.log" 41 | } 42 | 43 | main() { 44 | export PATH="../cmdline-player/:${PATH}" 45 | 46 | local status probability 47 | 48 | for x in 1 2; do 49 | cmdline-player -n "e2e/e2e-test-${x}.scr" 50 | while true; do 51 | hardcopy=no 52 | for i in {1..5}; do 53 | [[ -e hardcopy ]] && { 54 | i="${i}" 55 | hardcopy=yes 56 | printf 'hardcopy found\n' 57 | break 58 | } 59 | printf 'hardcopy not found\n' 60 | sleep .5 61 | done 62 | [[ ${hardcopy} == "yes" ]] && break 63 | printf 'creating hardcopy - why!' 64 | screen -S screencast -X hardcopy -h hardcopy 65 | done 66 | status=$(classify_status) 67 | probability=$(classify_probability) 68 | save_hardcopy "${x}" "${status}" "${probability}" 69 | printf 'Test %d status: %s (%s)\n' "${x}" "${status}" "${probability}" 70 | done 71 | } 72 | 73 | main "$@" 74 | 75 | # vim:ft=sh:sw=2:et:ts=2: 76 | -------------------------------------------------------------------------------- /tests/e2e/crm114/BAD: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bashtools/mok/5ccedb9aea7b05d47e310003c157efdd7df782af/tests/e2e/crm114/BAD -------------------------------------------------------------------------------- /tests/e2e/crm114/BUILD_USAGE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bashtools/mok/5ccedb9aea7b05d47e310003c157efdd7df782af/tests/e2e/crm114/BUILD_USAGE -------------------------------------------------------------------------------- /tests/e2e/crm114/CREATE_USAGE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bashtools/mok/5ccedb9aea7b05d47e310003c157efdd7df782af/tests/e2e/crm114/CREATE_USAGE -------------------------------------------------------------------------------- /tests/e2e/crm114/DELETE_USAGE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bashtools/mok/5ccedb9aea7b05d47e310003c157efdd7df782af/tests/e2e/crm114/DELETE_USAGE -------------------------------------------------------------------------------- /tests/e2e/crm114/EXEC_USAGE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bashtools/mok/5ccedb9aea7b05d47e310003c157efdd7df782af/tests/e2e/crm114/EXEC_USAGE -------------------------------------------------------------------------------- /tests/e2e/crm114/GET_USAGE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bashtools/mok/5ccedb9aea7b05d47e310003c157efdd7df782af/tests/e2e/crm114/GET_USAGE -------------------------------------------------------------------------------- /tests/e2e/crm114/GOOD: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bashtools/mok/5ccedb9aea7b05d47e310003c157efdd7df782af/tests/e2e/crm114/GOOD -------------------------------------------------------------------------------- /tests/e2e/crm114/UNKNOWN_USAGE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bashtools/mok/5ccedb9aea7b05d47e310003c157efdd7df782af/tests/e2e/crm114/UNKNOWN_USAGE -------------------------------------------------------------------------------- /tests/e2e/crm114/classify.crm: -------------------------------------------------------------------------------- 1 | { 2 | isolate (:stats:) 3 | classify (GOOD BAD) (:stats:) 4 | output /:*:stats:/ 5 | } 6 | -------------------------------------------------------------------------------- /tests/e2e/crm114/learn.crm: -------------------------------------------------------------------------------- 1 | { 2 | learn (:*:_arg2:) 3 | } 4 | -------------------------------------------------------------------------------- /tests/e2e/crm114/unlearn.crm: -------------------------------------------------------------------------------- 1 | { 2 | learn (:*:_arg2:) 3 | } 4 | -------------------------------------------------------------------------------- /tests/e2e/crm114/usage_match.crm: -------------------------------------------------------------------------------- 1 | { 2 | isolate (:stats:) 3 | classify (UNKNOWN_USAGE CREATE_USAGE DELETE_USAGE BUILD_USAGE GET_USAGE EXEC_USAGE) (:stats:) 4 | output /:*:stats:/ 5 | } 6 | -------------------------------------------------------------------------------- /tests/e2e/e2e-test-1.scr: -------------------------------------------------------------------------------- 1 | screencast delay 1 2 | screencast prompt #$%> 3 | errstop() { [[ $1 -gt 0 ]] && { printf 'SCREENCAST STOP &'; read; }; } 4 | date 5 | screencast paste 6 | alias mokbox='docker run --rm -ti --hostname mokbox --name mokbox -v /var/run/docker.sock:/var/run/docker.sock -v /var/tmp:/var/tmp myownkind/mokbox' 7 | mokbox 8 | retval=$? 9 | errstop() { [[ $1 -gt 0 ]] && { printf 'SCREENCAST STOP &'; read; }; } 10 | errstop $retval 11 | 12 | # Test 1 - download the prebuilt image and test 13 | 14 | mok -p build image --get-prebuilt-image 15 | errstop $? 16 | time mok -p create cluster myk8s --masters 1 --tailf 17 | errstop $? 18 | export KUBECONFIG=/var/tmp/admin-myk8s.conf 19 | sleep 5 20 | for i in `seq 1 20`; do if kubectl get pods &>/dev/null; then break; fi; echo -n "."; sleep 1; done 21 | sleep 5 22 | while kubectl get pods -A | tail -n +2 | awk '{ print $3; }' | grep -qs 0; do echo -n "."; sleep 2; done; echo 23 | kubectl get pods -A 24 | errstop $? 25 | kubectl run --rm -ti --restart=Never --image=alpine shell2 sh || errstop $? 26 | ip address 27 | apk add fortune 28 | exit 0 29 | errstop $? 30 | mok -p get cluster 31 | errstop $? 32 | mok -p delete cluster myk8s || errstop $? 33 | y 34 | mok -p get cluster 35 | errstop $? 36 | exit 37 | 38 | # Test complete 39 | date 40 | -------------------------------------------------------------------------------- /tests/e2e/e2e-test-2.scr: -------------------------------------------------------------------------------- 1 | screencast delay 1 2 | screencast prompt #$%> 3 | errstop() { [[ $1 -gt 0 ]] && { printf 'SCREENCAST STOP &'; read; }; } 4 | date 5 | screencast paste 6 | alias mokbox='docker run --rm -ti --hostname mokbox --name mokbox -v /var/run/docker.sock:/var/run/docker.sock -v /var/tmp:/var/tmp myownkind/mokbox' 7 | mokbox 8 | retval=$? 9 | errstop() { [[ $1 -gt 0 ]] && { printf 'SCREENCAST STOP &'; read; }; } 10 | errstop $retval 11 | 12 | # Test 2 - create the prebuilt image and test 13 | 14 | mok -p build image --tailf 15 | errstop $? 16 | time mok -p create cluster myk8s --masters 1 --workers 1 --tailf 17 | errstop $? 18 | export KUBECONFIG=/var/tmp/admin-myk8s.conf 19 | for i in `seq 1 20`; do if kubectl get pods &>/dev/null; then break; fi; echo -n "."; sleep 1; done 20 | while kubectl get pods -A | tail -n +2 | awk '{ print $3; }' | grep -qs 0; do echo -n "."; sleep 2; done; echo 21 | kubectl get pods -A 22 | errstop $? 23 | kubectl run --rm -ti --restart=Never --image=alpine shell2 sh || errstop $? 24 | ip address 25 | apk add fortune 26 | exit 0 27 | errstop $? 28 | mok -p get cluster 29 | errstop $? 30 | mok -p delete cluster myk8s || errstop $? 31 | y 32 | mok -p get cluster 33 | errstop $? 34 | exit 35 | 36 | # Test complete 37 | date 38 | -------------------------------------------------------------------------------- /tests/testfiles/docker_inspect_container.txt: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "Id": "5c844b362d2aa45fb82223abe911bdb120c044567ce64de776898c892eacd192", 4 | "Created": "2020-04-27T03:08:28.211972301Z", 5 | "Path": "/usr/local/bin/entrypoint", 6 | "Args": [ 7 | "/usr/sbin/init" 8 | ], 9 | "State": { 10 | "Status": "running", 11 | "Running": true, 12 | "Paused": false, 13 | "Restarting": false, 14 | "OOMKilled": false, 15 | "Dead": false, 16 | "Pid": 176154, 17 | "ExitCode": 0, 18 | "Error": "", 19 | "StartedAt": "2020-04-27T03:08:29.093065984Z", 20 | "FinishedAt": "0001-01-01T00:00:00Z" 21 | }, 22 | "Image": "sha256:2af34d07c8d4a7475b23d254d822100a85f4f04b4b6b32bf774d071123231770", 23 | "ResolvConfPath": "/home/var-lib-docker/containers/5c844b362d2aa45fb82223abe911bdb120c044567ce64de776898c892eacd192/resolv.conf", 24 | "HostnamePath": "/home/var-lib-docker/containers/5c844b362d2aa45fb82223abe911bdb120c044567ce64de776898c892eacd192/hostname", 25 | "HostsPath": "/home/var-lib-docker/containers/5c844b362d2aa45fb82223abe911bdb120c044567ce64de776898c892eacd192/hosts", 26 | "LogPath": "/home/var-lib-docker/containers/5c844b362d2aa45fb82223abe911bdb120c044567ce64de776898c892eacd192/5c844b362d2aa45fb82223abe911bdb120c044567ce64de776898c892eacd192-json.log", 27 | "Name": "/mycluster2-master-1", 28 | "RestartCount": 0, 29 | "Driver": "devicemapper", 30 | "Platform": "linux", 31 | "MountLabel": "", 32 | "ProcessLabel": "", 33 | "AppArmorProfile": "", 34 | "ExecIDs": null, 35 | "HostConfig": { 36 | "Binds": [ 37 | "/sys/fs/cgroup:/sys/fs/cgroup:ro", 38 | "/lib/modules:/lib/modules:ro" 39 | ], 40 | "ContainerIDFile": "", 41 | "LogConfig": { 42 | "Type": "json-file", 43 | "Config": {} 44 | }, 45 | "NetworkMode": "default", 46 | "PortBindings": {}, 47 | "RestartPolicy": { 48 | "Name": "no", 49 | "MaximumRetryCount": 0 50 | }, 51 | "AutoRemove": false, 52 | "VolumeDriver": "", 53 | "VolumesFrom": null, 54 | "CapAdd": null, 55 | "CapDrop": null, 56 | "Capabilities": null, 57 | "Dns": [], 58 | "DnsOptions": [], 59 | "DnsSearch": [], 60 | "ExtraHosts": null, 61 | "GroupAdd": null, 62 | "IpcMode": "private", 63 | "Cgroup": "", 64 | "Links": null, 65 | "OomScoreAdj": 0, 66 | "PidMode": "", 67 | "Privileged": true, 68 | "PublishAllPorts": false, 69 | "ReadonlyRootfs": false, 70 | "SecurityOpt": [ 71 | "label=disable" 72 | ], 73 | "Tmpfs": { 74 | "/run": "", 75 | "/tmp": "" 76 | }, 77 | "UTSMode": "", 78 | "UsernsMode": "", 79 | "ShmSize": 67108864, 80 | "Runtime": "runc", 81 | "ConsoleSize": [ 82 | 0, 83 | 0 84 | ], 85 | "Isolation": "", 86 | "CpuShares": 0, 87 | "Memory": 0, 88 | "NanoCpus": 0, 89 | "CgroupParent": "", 90 | "BlkioWeight": 0, 91 | "BlkioWeightDevice": [], 92 | "BlkioDeviceReadBps": null, 93 | "BlkioDeviceWriteBps": null, 94 | "BlkioDeviceReadIOps": null, 95 | "BlkioDeviceWriteIOps": null, 96 | "CpuPeriod": 0, 97 | "CpuQuota": 0, 98 | "CpuRealtimePeriod": 0, 99 | "CpuRealtimeRuntime": 0, 100 | "CpusetCpus": "", 101 | "CpusetMems": "", 102 | "Devices": [], 103 | "DeviceCgroupRules": null, 104 | "DeviceRequests": null, 105 | "KernelMemory": 0, 106 | "KernelMemoryTCP": 0, 107 | "MemoryReservation": 0, 108 | "MemorySwap": 0, 109 | "MemorySwappiness": null, 110 | "OomKillDisable": false, 111 | "PidsLimit": null, 112 | "Ulimits": null, 113 | "CpuCount": 0, 114 | "CpuPercent": 0, 115 | "IOMaximumIOps": 0, 116 | "IOMaximumBandwidth": 0, 117 | "MaskedPaths": null, 118 | "ReadonlyPaths": null 119 | }, 120 | "GraphDriver": { 121 | "Data": { 122 | "DeviceId": "1312", 123 | "DeviceName": "docker-8:3-393989-1750080ecbc1cb845b70c30ceb3c63438212c36e7ffc3f8b3667c8e2943f4531", 124 | "DeviceSize": "10737418240" 125 | }, 126 | "Name": "devicemapper" 127 | }, 128 | "Mounts": [ 129 | { 130 | "Type": "bind", 131 | "Source": "/lib/modules", 132 | "Destination": "/lib/modules", 133 | "Mode": "ro", 134 | "RW": false, 135 | "Propagation": "rprivate" 136 | }, 137 | { 138 | "Type": "bind", 139 | "Source": "/sys/fs/cgroup", 140 | "Destination": "/sys/fs/cgroup", 141 | "Mode": "ro", 142 | "RW": false, 143 | "Propagation": "rprivate" 144 | } 145 | ], 146 | "Config": { 147 | "Hostname": "mycluster2-master-1", 148 | "Domainname": "", 149 | "User": "", 150 | "AttachStdin": false, 151 | "AttachStdout": false, 152 | "AttachStderr": false, 153 | "Tty": false, 154 | "OpenStdin": false, 155 | "StdinOnce": false, 156 | "Env": [ 157 | "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", 158 | "container=docker" 159 | ], 160 | "Cmd": null, 161 | "Image": "local/mok-centos-7", 162 | "Volumes": { 163 | "/sys/fs/cgroup": {} 164 | }, 165 | "WorkingDir": "", 166 | "Entrypoint": [ 167 | "/usr/local/bin/entrypoint", 168 | "/usr/sbin/init" 169 | ], 170 | "OnBuild": null, 171 | "Labels": { 172 | "MokCluster": "mycluster2", 173 | "org.label-schema.build-date": "20191001", 174 | "org.label-schema.license": "GPLv2", 175 | "org.label-schema.name": "CentOS Base Image", 176 | "org.label-schema.schema-version": "1.0", 177 | "org.label-schema.vendor": "CentOS" 178 | }, 179 | "StopSignal": "SIGRTMIN+3" 180 | }, 181 | "NetworkSettings": { 182 | "Bridge": "", 183 | "SandboxID": "63829a5cdf5242be39fb6c2cc8ae1c1a37c834d4a880933a93d33548e06671fe", 184 | "HairpinMode": false, 185 | "LinkLocalIPv6Address": "", 186 | "LinkLocalIPv6PrefixLen": 0, 187 | "Ports": {}, 188 | "SandboxKey": "/var/run/docker/netns/63829a5cdf52", 189 | "SecondaryIPAddresses": null, 190 | "SecondaryIPv6Addresses": null, 191 | "EndpointID": "fd20ba006090ffa8f7e52803f12b4503d239ad0b937a84f5b43edb172024b76e", 192 | "Gateway": "172.17.0.1", 193 | "GlobalIPv6Address": "", 194 | "GlobalIPv6PrefixLen": 0, 195 | "IPAddress": "172.17.0.3", 196 | "IPPrefixLen": 16, 197 | "IPv6Gateway": "", 198 | "MacAddress": "02:42:ac:11:00:03", 199 | "Networks": { 200 | "bridge": { 201 | "IPAMConfig": null, 202 | "Links": null, 203 | "Aliases": null, 204 | "NetworkID": "5fb08cc86dcb6e7ce8c65e63f7de3e237acc8a25db4e1cc8c813091195443528", 205 | "EndpointID": "fd20ba006090ffa8f7e52803f12b4503d239ad0b937a84f5b43edb172024b76e", 206 | "Gateway": "172.17.0.1", 207 | "IPAddress": "172.17.0.3", 208 | "IPPrefixLen": 16, 209 | "IPv6Gateway": "", 210 | "GlobalIPv6Address": "", 211 | "GlobalIPv6PrefixLen": 0, 212 | "MacAddress": "02:42:ac:11:00:03", 213 | "DriverOpts": null 214 | } 215 | } 216 | } 217 | } 218 | ] 219 | -------------------------------------------------------------------------------- /tests/usage-checks.sh: -------------------------------------------------------------------------------- 1 | cd e2e/crm114/ || exit 1 2 | 3 | retval=0 4 | 5 | for i in build create delete exec get; do 6 | crmthinks=$(crm usage_match.crm <<<"$(mok "${i}" -h)" | sed -n '2p' | 7 | grep -Eo '(BUILD_USAGE|CREATE_USAGE|DELETE_USAGE|EXEC_USAGE|GET_USAGE)') 8 | status="GOOD" 9 | [[ ${crmthinks} != "${i^^}_USAGE" ]] && { 10 | status="!! BAD !!" 11 | retval=1 12 | } 13 | printf '%s - CRM114 thinks output from '\''mok %s -h'\'' is %s.\n' \ 14 | "${status}" "${i}" "${crmthinks}" 15 | done 16 | 17 | exit "${retval}" 18 | --------------------------------------------------------------------------------