├── .gitignore ├── .travis.yml ├── Dockerfile.build ├── LICENSE ├── Makefile ├── README.md ├── Vagrantfile ├── cmd └── kube-spawn │ ├── cni-spawn.go │ ├── create.go │ ├── destroy.go │ ├── kube-spawn.go │ ├── list.go │ ├── start.go │ ├── stop.go │ └── up.go ├── doc ├── dev-workflow.md ├── devel │ └── release.md ├── distro.md ├── kube-spawn-cluster-lifecycle.svg ├── kube-spawn-dev-workflow.svg ├── rktlet.md ├── troubleshooting.md └── vagrant.md ├── go.mod ├── go.sum ├── logos ├── PNG │ ├── kube_spawn-horz_prpblkonTRSP.png │ └── kube_spawn-horz_prpblkonwht.png ├── SVG │ ├── kube_spawn-horz_blkonwht.svg │ ├── kube_spawn-horz_prpblkonwht.svg │ ├── kube_spawn-horz_prponwht.svg │ ├── kube_spawn-horz_redblkonwht.svg │ ├── kube_spawn-horz_redonwht.svg │ ├── kube_spawn-horz_whtonblk.svg │ ├── kube_spawn-horz_whtonprp.svg │ ├── kube_spawn-horz_whtonred.svg │ ├── kube_spawn-vert_blkonwht.svg │ ├── kube_spawn-vert_prpblkonwht.svg │ ├── kube_spawn-vert_prponwht.svg │ ├── kube_spawn-vert_redblkonwht.svg │ ├── kube_spawn-vert_redonwht.svg │ ├── kube_spawn-vert_whtonblk.svg │ ├── kube_spawn-vert_whtonprp.svg │ └── kube_spawn-vert_whtonred.svg ├── kube_spawn-horz_blkonwht.svg ├── kube_spawn-horz_prpblkonwht.svg ├── kube_spawn-horz_prponwht.svg ├── kube_spawn-horz_redblkonwht.svg ├── kube_spawn-horz_redonwht.svg ├── kube_spawn-horz_whtonblk.svg ├── kube_spawn-horz_whtonprp.svg ├── kube_spawn-horz_whtonred.svg ├── kube_spawn-vert_blkonwht.svg ├── kube_spawn-vert_prpblkonwht.svg ├── kube_spawn-vert_prponwht.svg ├── kube_spawn-vert_redblkonwht.svg ├── kube_spawn-vert_redonwht.svg ├── kube_spawn-vert_whtonblk.svg ├── kube_spawn-vert_whtonprp.svg └── kube_spawn-vert_whtonred.svg ├── pkg ├── bootstrap │ ├── cninet.go │ ├── download.go │ ├── node.go │ └── util.go ├── cache │ └── cache.go ├── cluster │ ├── cluster.go │ └── clusterfiles.go ├── cnispawn │ ├── netns.go │ └── spawn.go ├── machinectl │ └── machinectl.go ├── multiprint │ └── multiprint.go ├── nspawntool │ └── run.go └── utils │ ├── fs │ └── fs.go │ ├── hash.go │ └── terminal.go ├── scripts ├── vagrant-mod-env.sh └── vagrant-setup-env.sh ├── vagrant-all.sh └── vagrant-fetch-kubeconfig.sh /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | .vagrant 10 | 11 | # Architecture specific extensions/prefixes 12 | *.[568vq] 13 | [568vq].out 14 | 15 | *.cgo1.go 16 | *.cgo2.c 17 | _cgo_defun.c 18 | _cgo_gotypes.go 19 | _cgo_export.* 20 | 21 | _testmain.go 22 | 23 | *.exe 24 | *.test 25 | *.prof 26 | 27 | *.lck 28 | *.log 29 | 30 | # binaries 31 | /cni-noop 32 | /cnispawn 33 | /kube-spawn 34 | 35 | # Vagrant artifacts 36 | .ssh_config 37 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | services: 4 | - docker 5 | 6 | install: true 7 | 8 | go: 9 | - 1.11.x 10 | 11 | before_script: 12 | - curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | bash -s -- -b $GOPATH/bin latest 13 | 14 | script: 15 | - echo run lint 16 | - golangci-lint run --disable-all -E errcheck 17 | - "[ -n $(gofmt -s -l pkg cmd) ] || (echo Code format error; gofmt -s -d -e pkg cmd; false)" 18 | - DOCKERIZED=y make 19 | 20 | notifications: 21 | email: false 22 | 23 | sudo: false 24 | -------------------------------------------------------------------------------- /Dockerfile.build: -------------------------------------------------------------------------------- 1 | FROM golang:latest 2 | 3 | ENV GOCACHE /tmp/.cache 4 | ENV PATH "/go/bin:${PATH}" 5 | 6 | WORKDIR /usr/src/kube-spawn 7 | 8 | ENTRYPOINT ["make", "DOCKERIZED=n"] 9 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | BIN = kube-spawn 2 | DOCKERIZED ?= y 3 | DOCKER_TAG ?= latest 4 | PREFIX ?= /usr 5 | BINDIR ?= ${PREFIX}/bin 6 | UID=$(shell id -u) 7 | GOCACHEDIR=$(shell which go >/dev/null 2>&1 && go env GOCACHE || echo "$(HOME)/.cache/go-build") 8 | 9 | .PHONY: all clean install 10 | 11 | VERSION=$(shell git describe --tags --always --dirty) 12 | 13 | ifeq ($(DOCKERIZED),y) 14 | all: 15 | docker build -t kube-spawn-build:$(DOCKER_TAG) -f Dockerfile.build . 16 | mkdir -p $(GOCACHEDIR) 17 | docker run --rm -ti \ 18 | -v `pwd`:/usr/src/kube-spawn:Z \ 19 | -v $(GOCACHEDIR):/tmp/.cache:Z \ 20 | --user $(UID):$(UID) \ 21 | kube-spawn-build 22 | else 23 | all: 24 | GO111MODULE=on go mod download 25 | GO111MODULE=on go build -ldflags "-X main.version=$(VERSION)" \ 26 | ./cmd/kube-spawn 27 | endif 28 | 29 | update-vendor: 30 | GO111MODULE=on go get -u 31 | GO111MODULE=on go mod tidy 32 | 33 | clean: 34 | rm -f \ 35 | kube-spawn \ 36 | 37 | install: 38 | install kube-spawn "$(BINDIR)" 39 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![kube-spawn Logo](logos/PNG/kube_spawn-horz_prpblkonwht.png) 2 | 3 | # kube-spawn 4 | 5 | `kube-spawn` is a tool for creating a multi-node Kubernetes (>= 1.8) cluster on a single Linux machine, created mostly for developers __of__ Kubernetes but is also a [Certified Kubernetes Distribution](https://kubernetes.io/partners/#dist) and, therefore, perfect for running and testing deployments locally. 6 | 7 | It attempts to mimic production setups by making use of OS containers to set up nodes. 8 | 9 | ## Demo 10 | 11 | [![asciicast](https://asciinema.org/a/132605.png)](https://asciinema.org/a/132605) 12 | 13 | ## Requirements 14 | 15 | * `systemd-nspawn` in at least version 233 16 | * Large enough `/var/lib/machines` partition. 17 | 18 | If /var/lib/machines is not its own filesystem, systemd-nspawn 19 | will create /var/lib/machines.raw and loopback mount it as a 20 | btrfs filesystem. You may wish to increase the default size: 21 | 22 | `machinectl set-limit 20G` 23 | 24 | We recommend you create a partition of sufficient size, format 25 | it as btrfs, and mount it on /var/lib/machines, rather than 26 | letting the loopback mechanism take hold. 27 | 28 | In the event there is a loopback file mounted on /var/lib/machines, 29 | kube-spawn will attempt to enlarge the underlying image `/var/lib/machines.raw` 30 | on cluster start, but this can only succeed when the image is not in use by 31 | another cluster or machine. Not enough disk space is a common source 32 | of error. See [doc/troubleshooting](doc/troubleshooting.md#varlibmachines-partition-too-small) for 33 | instructions on how to increase the size manually. 34 | * `qemu-img` 35 | 36 | ## Installation 37 | 38 | `kube-spawn` should run well on a modern Linux system (for example Fedora 27 or 39 | Debian testing). If you want to test it in a controlled environment, you can 40 | use [Vagrant](doc/vagrant.md). 41 | 42 | To install `kube-spawn` on your machine, download a single binary release 43 | or [build from source](#building). 44 | 45 | kube-spawn uses CNI to setup networking for its containers. For that, you need 46 | to download the CNI plugins (v.0.6.0 or later) from GitHub. 47 | 48 | Example: 49 | 50 | ``` 51 | cd /tmp 52 | curl -fsSL -O https://github.com/containernetworking/plugins/releases/download/v0.6.0/cni-plugins-amd64-v0.6.0.tgz 53 | sudo mkdir -p /opt/cni/bin 54 | sudo tar -C /opt/cni/bin -xvf cni-plugins-amd64-v0.6.0.tgz 55 | ``` 56 | 57 | By default, kube-spawn expects the plugins in `/opt/cni/bin`. The location 58 | can be configured with `--cni-plugin-dir=` from the command line or 59 | by setting `cni-plugin-dir: ...` in the configuration file. 60 | 61 | Alternatively, you can use `go get` to fetch the plugins into your `GOPATH`: 62 | 63 | ``` 64 | go get -u github.com/containernetworking/plugins/plugins/... 65 | ``` 66 | 67 | ## Quickstart 68 | 69 | Create and start a 3 node cluster with the name "default": 70 | 71 | ``` 72 | sudo ./kube-spawn create 73 | sudo ./kube-spawn start [--nodes 3] 74 | ``` 75 | 76 | Reminder: if the CNI plugins can't be found in `/opt/cni/bin`, you need 77 | to pass `--cni-plugin-dir path/to/plugins`. 78 | 79 | `create` prepares the cluster environment in `/var/lib/kube-spawn/clusters`. 80 | 81 | `start` brings up the nodes and configures the cluster using 82 | [kubeadm](https://github.com/kubernetes/kubeadm). 83 | 84 | Shortly after, the cluster should be initialized: 85 | 86 | ``` 87 | [...] 88 | 89 | Cluster "default" initialized 90 | Export $KUBECONFIG as follows for kubectl: 91 | 92 | export KUBECONFIG=/var/lib/kube-spawn/clusters/default/admin.kubeconfig 93 | ``` 94 | 95 | After another 1-2 minutes the nodes should be ready: 96 | 97 | ``` 98 | export KUBECONFIG=/var/lib/kube-spawn/clusters/default/admin.kubeconfig 99 | kubectl get nodes 100 | NAME STATUS ROLES AGE VERSION 101 | kube-spawn-c1-master-q9fd4y Ready master 5m v1.9.6 102 | kube-spawn-c1-worker-dj7xou Ready 4m v1.9.6 103 | kube-spawn-c1-worker-etbxnu Ready 4m v1.9.6 104 | ``` 105 | 106 | ## Configuration 107 | 108 | kube-spawn can be configured by command line flags, configuration file 109 | (default `/etc/kube-spawn/config.yaml` or `--config path/to/config.yaml`), 110 | environment variables or a mix thereof. 111 | 112 | Example: 113 | 114 | ``` 115 | # /etc/kube-spawn/config.yaml 116 | cni-plugin-dir: /home/user/code/go/bin 117 | cluster-name: cluster1 118 | container-runtime: rkt 119 | rktlet-binary-path: /home/user/code/go/src/github.com/kubernetes-incubator/rktlet/bin/rktlet 120 | ``` 121 | 122 | ## CNI plugins 123 | 124 | kube-spawn supports weave, flannel, calico. It defaults to weave. 125 | 126 | To configure with flannel: 127 | ``` 128 | kube-spawn create --pod-network-cidr 10.244.0.0/16 --cni-plugin flannel --kubernetes-version=v1.10.5 129 | kube-spawn start --cni-plugin flannel --nodes 5 130 | ``` 131 | 132 | To configure with calico: 133 | ``` 134 | kube-spawn create --pod-network-cidr 192.168.0.0/16 --cni-plugin calico --kubernetes-version=v1.10.5 135 | kube-spawn start --cni-plugin calico --nodes 5 136 | ``` 137 | 138 | To configure with canal: 139 | ``` 140 | kube-spawn create --pod-network-cidr 10.244.0.0/16 --cni-plugin canal --kubernetes-version=v1.10.5 141 | kube-spawn start --cni-plugin canal --nodes 5 142 | ``` 143 | 144 | ## Accessing kube-spawn nodes 145 | 146 | All nodes can be seen with `machinectl list`. `machinectl shell` can be 147 | used to access a node, for example: 148 | 149 | ``` 150 | sudo machinectl shell kube-spawn-c1-master-fubo3j 151 | ``` 152 | 153 | The password is `root`. 154 | 155 | ## Documentation 156 | 157 | See [doc/](doc/) 158 | 159 | ## Building 160 | 161 | To build kube-spawn in a Docker build container, simply run: 162 | 163 | ``` 164 | make 165 | ``` 166 | 167 | Optionally, install kube-spawn under a system directory: 168 | 169 | ``` 170 | sudo make install 171 | ``` 172 | 173 | `PREFIX` can be set to override the default target `/usr`. 174 | 175 | ## Troubleshooting 176 | 177 | See [doc/troubleshooting](doc/troubleshooting.md) 178 | 179 | ## Community 180 | 181 | Discuss the project on [Slack](https://kubernetes.slack.com/messages/C9ZMJH2NL/). 182 | -------------------------------------------------------------------------------- /Vagrantfile: -------------------------------------------------------------------------------- 1 | # -*- mode: ruby -*- 2 | # vi: set ft=ruby sw=2 ts=2 : 3 | 4 | ENV["TERM"] = "xterm-256color" 5 | ENV["LC_ALL"] = "en_US.UTF-8" 6 | 7 | Vagrant.configure("2") do |config| 8 | config.vm.box = "fedora/30-cloud-base" # defaults to fedora 9 | 10 | # common parts 11 | if Vagrant.has_plugin?("vagrant-vbguest") 12 | config.vbguest.auto_update = false 13 | end 14 | config.vm.provider :libvirt do |libvirt| 15 | libvirt.cpus = 2 16 | libvirt.memory = 4096 17 | end 18 | config.vm.provider :virtualbox do |vb| 19 | vb.check_guest_additions = false 20 | vb.functional_vboxsf = false 21 | vb.customize ["modifyvm", :id, "--memory", "4096"] 22 | vb.customize ["modifyvm", :id, "--cpus", "2"] 23 | end 24 | 25 | # Fedora 30 26 | config.vm.define "fedora", primary: true do |fedora| 27 | config.vm.provision "shell", inline: "dnf install -y btrfs-progs git go iptables libselinux-utils make polkit qemu-img rinetd systemd-container" 28 | 29 | config.vm.synced_folder ".", "/vagrant", disabled: true 30 | config.vm.synced_folder ".", "/home/vagrant/go/src/github.com/kinvolk/kube-spawn", 31 | create: true, 32 | owner: "vagrant", 33 | group: "vagrant", 34 | type: "rsync", 35 | rsync__exclude: ".kube-spawn/" 36 | 37 | # NOTE: chown is explicitly needed, even when synced_folder is configured 38 | # with correct owner/group. Maybe a vagrant issue? 39 | config.vm.provision "shell", inline: "mkdir -p /home/vagrant/go ; chown -R vagrant:vagrant /home/vagrant/go" 40 | 41 | config.vm.provision "shell", env: {"GOPATH" => "/home/vagrant/go", "KUBESPAWN_REDIRECT_TRAFFIC" => ENV["KUBESPAWN_REDIRECT_TRAFFIC"]}, privileged: false, path: "scripts/vagrant-setup-env.sh" 42 | config.vm.provision "shell", env: {"VUSER" => "vagrant"}, path: "scripts/vagrant-mod-env.sh" 43 | if ENV["KUBESPAWN_AUTOBUILD"] <=> "true" 44 | config.vm.provision "shell", env: {"GOPATH" => "/home/vagrant/go", "KUBESPAWN_REDIRECT_TRAFFIC" => ENV["KUBESPAWN_REDIRECT_TRAFFIC"]}, inline: "bash /home/vagrant/build.sh" 45 | end 46 | end 47 | 48 | # Ubuntu 19.04 (Disco) 49 | config.vm.define "ubuntu", autostart: false do |ubuntu| 50 | config.vm.box = "generic/ubuntu1904" 51 | config.vm.provision "shell", inline: "apt-get update; DEBIAN_FRONTEND=noninteractive apt-get install -y btrfs-progs git golang-1.12 iptables make policykit-1 qemu-utils rinetd selinux-utils systemd-container" 52 | 53 | config.vm.synced_folder ".", "/vagrant", disabled: true 54 | config.vm.synced_folder ".", "/home/vagrant/go/src/github.com/kinvolk/kube-spawn", 55 | create: true, 56 | owner: "vagrant", 57 | group: "vagrant", 58 | type: "rsync", 59 | rsync__exclude: ".kube-spawn/" 60 | 61 | config.vm.provision "shell", inline: "mkdir -p /home/vagrant/go ; chown -R vagrant:vagrant /home/vagrant/go" 62 | config.vm.provision "shell", env: {"GOPATH" => "/home/vagrant/go", "KUBESPAWN_REDIRECT_TRAFFIC" => ENV["KUBESPAWN_REDIRECT_TRAFFIC"]}, privileged: false, path: "scripts/vagrant-setup-env.sh" 63 | config.vm.provision "shell", env: {"VUSER" => "vagrant"}, path: "scripts/vagrant-mod-env.sh" 64 | if ENV["KUBESPAWN_AUTOBUILD"] <=> "true" 65 | config.vm.provision "shell", env: {"GOPATH" => "/home/vagrant/go", "KUBESPAWN_REDIRECT_TRAFFIC" => ENV["KUBESPAWN_REDIRECT_TRAFFIC"]}, inline: "bash /home/vagrant/build.sh" 66 | end 67 | end 68 | 69 | # Debian testing 70 | config.vm.define "debian", autostart: false do |debian| 71 | config.vm.box = "debian/testing64" 72 | config.vm.provision "shell", inline: "echo deb http://httpredir.debian.org/debian unstable main >> /etc/apt/sources.list; apt-get update; DEBIAN_FRONTEND=noninteractive apt-get install -y btrfs-progs git golang iptables make policykit-1 qemu-utils rinetd selinux-utils systemd-container" 73 | 74 | config.vm.synced_folder ".", "/vagrant", disabled: true 75 | config.vm.synced_folder ".", "/home/vagrant/go/src/github.com/kinvolk/kube-spawn", 76 | create: true, 77 | owner: "vagrant", 78 | group: "vagrant", 79 | type: "rsync", 80 | rsync__exclude: ".kube-spawn/" 81 | 82 | config.vm.provision "shell", inline: "mkdir -p /home/vagrant/go ; chown -R vagrant:vagrant /home/vagrant/go" 83 | config.vm.provision "shell", env: {"GOPATH" => "/home/vagrant/go", "KUBESPAWN_REDIRECT_TRAFFIC" => ENV["KUBESPAWN_REDIRECT_TRAFFIC"]}, privileged: false, path: "scripts/vagrant-setup-env.sh" 84 | config.vm.provision "shell", env: {"VUSER" => "vagrant"}, path: "scripts/vagrant-mod-env.sh" 85 | if ENV["KUBESPAWN_AUTOBUILD"] <=> "true" 86 | config.vm.provision "shell", env: {"GOPATH" => "/home/vagrant/go", "KUBESPAWN_REDIRECT_TRAFFIC" => ENV["KUBESPAWN_REDIRECT_TRAFFIC"]}, inline: "bash /home/vagrant/build.sh" 87 | end 88 | end 89 | 90 | config.vm.network "forwarded_port", guest: 6443, host: 6443 91 | end 92 | -------------------------------------------------------------------------------- /cmd/kube-spawn/cni-spawn.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 Kinvolk GmbH 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "fmt" 21 | "os" 22 | 23 | "github.com/spf13/cobra" 24 | 25 | "github.com/kinvolk/kube-spawn/pkg/cnispawn" 26 | ) 27 | 28 | var ( 29 | cniSpawnCmd = &cobra.Command{ 30 | Use: "cni-spawn", 31 | Short: "Spawn systemd-nspawn containers in a new network namespace", 32 | Hidden: true, 33 | Run: runCNISpawn, 34 | } 35 | cniPluginDir string 36 | ) 37 | 38 | func init() { 39 | kubespawnCmd.AddCommand(cniSpawnCmd) 40 | cniSpawnCmd.Flags().StringVar(&cniPluginDir, "cni-plugin-dir", "/opt/cni/bin", "path to CNI plugin directory") 41 | } 42 | 43 | func runCNISpawn(cmd *cobra.Command, args []string) { 44 | if err := cnispawn.Spawn(cniPluginDir, args); err != nil { 45 | fmt.Fprintf(os.Stderr, "%s\n", err) 46 | os.Exit(1) 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /cmd/kube-spawn/create.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 Kinvolk GmbH 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "log" 21 | "path" 22 | 23 | "github.com/spf13/cobra" 24 | "github.com/spf13/viper" 25 | 26 | "github.com/kinvolk/kube-spawn/pkg/bootstrap" 27 | "github.com/kinvolk/kube-spawn/pkg/cache" 28 | "github.com/kinvolk/kube-spawn/pkg/cluster" 29 | "github.com/kinvolk/kube-spawn/pkg/utils/fs" 30 | ) 31 | 32 | var ( 33 | createCmd = &cobra.Command{ 34 | Use: "create", 35 | Short: "Create a new cluster", 36 | Example: ` 37 | # Create a cluster using a custom hyperkube image 38 | $ sudo ./kube-spawn create --hyperkube-image 10.22.0.1:5000/me/my-hyperkube-amd64-image:my-test 39 | 40 | # Create a cluster using rkt as the container runtime 41 | $ sudo ./kube-spawn create --container-runtime rkt --rktlet-binary-path $GOPATH/src/github.com/kubernetes-incubator/rktlet/bin/rktlet`, 42 | Run: runCreate, 43 | } 44 | ) 45 | 46 | func init() { 47 | kubespawnCmd.AddCommand(createCmd) 48 | 49 | createCmd.Flags().String("container-runtime", "docker", "Runtime to use for the cluster (can be docker or rkt)") 50 | createCmd.Flags().String("kubernetes-version", "v1.12.3", "Kubernetes version to install") 51 | createCmd.Flags().String("kubernetes-source-dir", "", "Path to directory with Kubernetes sources") 52 | createCmd.Flags().String("hyperkube-image", "", "Kubernetes hyperkube image to use (if unset, upstream k8s is installed)") 53 | createCmd.Flags().String("cni-plugin-dir", "/opt/cni/bin", "Path to directory with CNI plugins") 54 | createCmd.Flags().String("cni-plugin", "weave", "CNI plugin to use (weave, flannel, calico, canal)") 55 | createCmd.Flags().String("cluster-cidr", "", "Cluster CIDR to use") 56 | createCmd.Flags().String("pod-network-cidr", "", "Pod Network CIDR to use") 57 | createCmd.Flags().String("rkt-binary-path", "/usr/local/bin/rkt", "Path to rkt binary") 58 | createCmd.Flags().String("rkt-stage1-image-path", "/usr/local/bin/stage1-coreos.aci", "Path to rkt stage1-coreos.aci image") 59 | createCmd.Flags().String("rktlet-binary-path", "/usr/local/bin/rktlet", "Path to rktlet binary") 60 | } 61 | 62 | func runCreate(cmd *cobra.Command, args []string) { 63 | if len(args) > 0 { 64 | log.Fatalf("Command create doesn't take arguments, got: %v", args) 65 | } 66 | 67 | doCreate() 68 | } 69 | 70 | func doCreate() { 71 | kubespawnDir := viper.GetString("dir") 72 | clusterName := viper.GetString("cluster-name") 73 | clusterDir := path.Join(kubespawnDir, "clusters", clusterName) 74 | if exists, err := fs.PathExists(clusterDir); err != nil { 75 | log.Fatalf("Failed to stat directory %q: %s\n", err) 76 | } else if exists { 77 | log.Fatalf("Cluster directory exists already at %q", clusterDir) 78 | } 79 | 80 | // TODO 81 | if err := bootstrap.PathSupportsOverlay(kubespawnDir); err != nil { 82 | log.Fatalf("Unable to use overlayfs on underlying filesystem of %q: %v", kubespawnDir, err) 83 | } 84 | 85 | kluster, err := cluster.New(clusterDir, clusterName) 86 | if err != nil { 87 | log.Fatalf("Failed to create cluster object: %v", err) 88 | } 89 | 90 | clusterSettings := &cluster.ClusterSettings{ 91 | KubernetesVersion: viper.GetString("kubernetes-version"), 92 | KubernetesSourceDir: viper.GetString("kubernetes-source-dir"), 93 | CNIPluginDir: viper.GetString("cni-plugin-dir"), 94 | CNIPlugin: viper.GetString("cni-plugin"), 95 | ContainerRuntime: viper.GetString("container-runtime"), 96 | ClusterCIDR: viper.GetString("cluster-cidr"), 97 | PodNetworkCIDR: viper.GetString("pod-network-cidr"), 98 | RktBinaryPath: viper.GetString("rkt-binary-path"), 99 | RktStage1ImagePath: viper.GetString("rkt-stage1-image-path"), 100 | RktletBinaryPath: viper.GetString("rktlet-binary-path"), 101 | HyperkubeImage: viper.GetString("hyperkube-image"), 102 | } 103 | 104 | clusterCache, err := cache.New(path.Join(kubespawnDir, "cache")) 105 | if err != nil { 106 | log.Fatalf("Failed to create cache object: %v", err) 107 | } 108 | 109 | if err := kluster.Create(clusterSettings, clusterCache); err != nil { 110 | log.Fatalf("Failed to create cluster: %v", err) 111 | } 112 | 113 | log.Printf("Cluster %s created", clusterName) 114 | } 115 | -------------------------------------------------------------------------------- /cmd/kube-spawn/destroy.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 Kinvolk GmbH 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "log" 21 | "path" 22 | 23 | "github.com/spf13/cobra" 24 | "github.com/spf13/viper" 25 | 26 | "github.com/kinvolk/kube-spawn/pkg/cluster" 27 | ) 28 | 29 | var ( 30 | destroyCmd = &cobra.Command{ 31 | Use: "destroy", 32 | Short: "Destroy a cluster", 33 | Long: "Destroy a cluster", 34 | Run: runDestroy, 35 | } 36 | ) 37 | 38 | func init() { 39 | kubespawnCmd.AddCommand(destroyCmd) 40 | } 41 | 42 | func runDestroy(cmd *cobra.Command, args []string) { 43 | if len(args) > 0 { 44 | log.Fatalf("Command destroy doesn't take arguments, got: %v", args) 45 | } 46 | 47 | kubespawnDir := viper.GetString("dir") 48 | clusterName := viper.GetString("cluster-name") 49 | clusterDir := path.Join(kubespawnDir, "clusters", clusterName) 50 | 51 | kluster, err := cluster.New(clusterDir, clusterName) 52 | if err != nil { 53 | log.Fatalf("Failed to create cluster object: %v", err) 54 | } 55 | 56 | log.Printf("Destroying cluster %s ...", clusterName) 57 | 58 | if err := kluster.Destroy(); err != nil { 59 | log.Fatalf("Failed to destroy cluster: %v", err) 60 | } 61 | 62 | log.Printf("Cluster %s destroyed", clusterName) 63 | } 64 | -------------------------------------------------------------------------------- /cmd/kube-spawn/kube-spawn.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 Kinvolk GmbH 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "fmt" 21 | "log" 22 | "os" 23 | 24 | "github.com/spf13/cobra" 25 | "github.com/spf13/viper" 26 | "golang.org/x/sys/unix" 27 | ) 28 | 29 | var ( 30 | kubespawnCmd = &cobra.Command{ 31 | Use: "kube-spawn", 32 | Short: "kube-spawn is a tool for creating a multi-node dev Kubernetes cluster", 33 | Long: `kube-spawn is a tool for creating a multi-node dev Kubernetes cluster 34 | You can run a release-version cluster or spawn one from a custom hyperkube image`, 35 | Run: func(cmd *cobra.Command, args []string) { 36 | if printVersion { 37 | fmt.Printf("kube-spawn %s\n", version) 38 | os.Exit(0) 39 | } 40 | if err := cmd.Usage(); err != nil { 41 | log.Fatal(err) 42 | } 43 | }, 44 | } 45 | // set from ldflags to current git version during build 46 | version string 47 | 48 | printVersion bool 49 | 50 | cfgFile string 51 | ) 52 | 53 | func init() { 54 | log.SetFlags(0) 55 | 56 | cobra.OnInitialize(initConfig) 57 | 58 | kubespawnCmd.Flags().BoolVarP(&printVersion, "version", "V", false, "Output version and exit") 59 | kubespawnCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default \"/etc/kube-spawn/config.yaml\")") 60 | kubespawnCmd.PersistentFlags().StringP("dir", "d", "/var/lib/kube-spawn", "Path to kube-spawn asset directory") 61 | kubespawnCmd.PersistentFlags().StringP("cluster-name", "c", "default", "Name for the cluster") 62 | 63 | kubespawnCmd.PersistentPreRunE = func(cmd *cobra.Command, args []string) error { 64 | cmdName := cmd.Use 65 | if cmdName == "create" || cmdName == "destroy" || cmdName == "start" || cmdName == "stop" || cmdName == "up" { 66 | if unix.Geteuid() != 0 { 67 | cmd.SilenceUsage = true 68 | return fmt.Errorf("root privileges required for command %q, aborting", cmdName) 69 | } 70 | } 71 | err := viper.BindPFlags(cmd.Flags()) 72 | return err 73 | } 74 | } 75 | 76 | func initConfig() { 77 | if cfgFile != "" { 78 | viper.SetConfigFile(cfgFile) 79 | } else { 80 | viper.SetConfigName("config") 81 | viper.SetConfigType("yaml") 82 | config := fmt.Sprintf("/etc/kube-spawn") 83 | viper.AddConfigPath(config) 84 | } 85 | viper.SetEnvPrefix("KUBE_SPAWN") 86 | viper.AutomaticEnv() 87 | if err := viper.ReadInConfig(); err == nil { 88 | log.Printf("Using config file %q", viper.ConfigFileUsed()) 89 | } 90 | } 91 | 92 | func main() { 93 | if err := kubespawnCmd.Execute(); err != nil { 94 | os.Exit(1) 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /cmd/kube-spawn/list.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 Kinvolk GmbH 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "fmt" 21 | "io/ioutil" 22 | "log" 23 | "os" 24 | "path" 25 | 26 | "github.com/spf13/cobra" 27 | "github.com/spf13/viper" 28 | ) 29 | 30 | var ( 31 | listCmd = &cobra.Command{ 32 | Use: "list", 33 | Short: "List all kube-spawn clusters", 34 | Run: runList, 35 | } 36 | ) 37 | 38 | func init() { 39 | kubespawnCmd.AddCommand(listCmd) 40 | } 41 | 42 | func runList(cmd *cobra.Command, args []string) { 43 | if len(args) > 0 { 44 | log.Fatalf("Command list doesn't take arguments, got: %v", args) 45 | } 46 | 47 | clusterDir := path.Join(viper.GetString("dir"), "clusters/") 48 | 49 | entries, err := ioutil.ReadDir(clusterDir) 50 | if err != nil && !os.IsNotExist(err) { 51 | log.Fatalf("Failed to read cluster directory: %v", err) 52 | } 53 | 54 | if len(entries) == 0 { 55 | log.Printf("No clusters yet") 56 | } else { 57 | fmt.Println("Available clusters:") 58 | for _, entry := range entries { 59 | fmt.Printf(" %s\n", entry.Name()) 60 | } 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /cmd/kube-spawn/start.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 Kinvolk GmbH 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "log" 21 | "path" 22 | 23 | "github.com/spf13/cobra" 24 | "github.com/spf13/viper" 25 | 26 | "github.com/kinvolk/kube-spawn/pkg/cluster" 27 | ) 28 | 29 | var ( 30 | startCmd = &cobra.Command{ 31 | Use: "start", 32 | Short: "Start a cluster that was created with 'kube-spawn create' before", 33 | Run: runStart, 34 | } 35 | ) 36 | 37 | func init() { 38 | kubespawnCmd.AddCommand(startCmd) 39 | 40 | startCmd.Flags().IntP("nodes", "n", 3, "Number of nodes to start") 41 | startCmd.Flags().String("cni-plugin-dir", "/opt/cni/bin", "Path to directory with CNI plugins") 42 | startCmd.Flags().String("cni-plugin", "weave", "CNI plugin (weave, flannel, calico, canal)") 43 | startCmd.Flags().String("flatcar-channel", "alpha", "Channel for Flatcar Linux (alpha, beta, stable)") 44 | } 45 | 46 | func runStart(cmd *cobra.Command, args []string) { 47 | if len(args) > 0 { 48 | log.Fatalf("Command start doesn't take arguments, got: %v", args) 49 | } 50 | 51 | doStart() 52 | 53 | } 54 | 55 | func doStart() { 56 | kubespawnDir := viper.GetString("dir") 57 | clusterName := viper.GetString("cluster-name") 58 | numberNodes := viper.GetInt("nodes") 59 | cniPluginDir := viper.GetString("cni-plugin-dir") 60 | cniPlugin := viper.GetString("cni-plugin") 61 | flatcarChannel := viper.GetString("flatcar-channel") 62 | 63 | kluster, err := cluster.New(path.Join(kubespawnDir, "clusters", clusterName), clusterName) 64 | if err != nil { 65 | log.Fatalf("Failed to create cluster object: %v", err) 66 | } 67 | 68 | if err := kluster.Start(numberNodes, cniPluginDir, cniPlugin, flatcarChannel); err != nil { 69 | log.Fatalf("Failed to start cluster: %v", err) 70 | } 71 | 72 | log.Printf("Cluster %q initialized", clusterName) 73 | log.Println("Export $KUBECONFIG as follows for kubectl:") 74 | log.Printf("\n\texport KUBECONFIG=%s\n\n", kluster.AdminKubeconfigPath()) 75 | } 76 | -------------------------------------------------------------------------------- /cmd/kube-spawn/stop.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 Kinvolk GmbH 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "log" 21 | "path" 22 | 23 | "github.com/spf13/cobra" 24 | "github.com/spf13/viper" 25 | 26 | "github.com/kinvolk/kube-spawn/pkg/cluster" 27 | ) 28 | 29 | var ( 30 | stopCmd = &cobra.Command{ 31 | Use: "stop", 32 | Short: "Stop a running cluster", 33 | Run: runStop, 34 | } 35 | flagForce bool 36 | ) 37 | 38 | func init() { 39 | kubespawnCmd.AddCommand(stopCmd) 40 | stopCmd.Flags().BoolVarP(&flagForce, "force", "f", false, "terminate machines instead of trying graceful shutdown") 41 | } 42 | 43 | func runStop(cmd *cobra.Command, args []string) { 44 | if len(args) > 0 { 45 | log.Fatalf("Command stop doesn't take arguments, got: %v", args) 46 | } 47 | 48 | kubespawnDir := viper.GetString("dir") 49 | clusterName := viper.GetString("cluster-name") 50 | clusterDir := path.Join(kubespawnDir, "clusters", clusterName) 51 | 52 | kluster, err := cluster.New(clusterDir, clusterName) 53 | if err != nil { 54 | log.Fatalf("Failed to create cluster object: %v", err) 55 | } 56 | 57 | log.Printf("Stopping cluster %s ...", clusterName) 58 | 59 | if err := kluster.Stop(); err != nil { 60 | log.Fatalf("Failed to stop cluster: %v", err) 61 | } 62 | 63 | log.Printf("Cluster %s stopped", clusterName) 64 | } 65 | -------------------------------------------------------------------------------- /cmd/kube-spawn/up.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Kinvolk GmbH 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "log" 21 | 22 | "github.com/spf13/cobra" 23 | ) 24 | 25 | var ( 26 | upCmd = &cobra.Command{ 27 | Use: "up", 28 | Short: "Create and start a new cluster", 29 | Example: ` 30 | # Create and start a Kubernetes v1.10.0 cluster with 4 nodes (master + 3 worker) 31 | sudo ./kube-spawn up --kubernetes-version v1.10.0 --nodes 4`, 32 | Run: runUp, 33 | } 34 | ) 35 | 36 | func init() { 37 | kubespawnCmd.AddCommand(upCmd) 38 | 39 | // Flags should be kept in sync with `start` and `create` 40 | 41 | upCmd.Flags().String("container-runtime", "docker", "Runtime to use for the cluster (can be docker or rkt)") 42 | upCmd.Flags().String("kubernetes-version", "v1.12.3", "Kubernetes version to install") 43 | upCmd.Flags().String("kubernetes-source-dir", "", "Path to directory with Kubernetes sources") 44 | upCmd.Flags().String("hyperkube-image", "", "Kubernetes hyperkube image to use (if unset, upstream k8s is installed)") 45 | upCmd.Flags().String("cni-plugin-dir", "/opt/cni/bin", "Path to directory with CNI plugins") 46 | upCmd.Flags().String("cni-plugin", "weave", "CNI plugin to use (weave, flannel, calico, canal)") 47 | upCmd.Flags().String("rkt-binary-path", "/usr/local/bin/rkt", "Path to rkt binary") 48 | upCmd.Flags().String("rkt-stage1-image-path", "/usr/local/bin/stage1-coreos.aci", "Path to rkt stage1-coreos.aci image") 49 | upCmd.Flags().String("rktlet-binary-path", "/usr/local/bin/rktlet", "Path to rktlet binary") 50 | upCmd.Flags().IntP("nodes", "n", 3, "Number of nodes to start") 51 | } 52 | 53 | func runUp(cmd *cobra.Command, args []string) { 54 | if len(args) > 0 { 55 | log.Fatalf("Command up doesn't take arguments, got: %v", args) 56 | } 57 | 58 | doCreate() 59 | doStart() 60 | } 61 | -------------------------------------------------------------------------------- /doc/dev-workflow.md: -------------------------------------------------------------------------------- 1 | # Kubernetes development workflow example 2 | 3 | This article describes a step-by-step workflow that a Kubernetes developer might follow when testing a Kubernetes patch with kube-spawn. 4 | 5 | For the purpose of the article, we will write a new [admission controller](https://kubernetes.io/docs/admin/admission-controllers/) named `DenyAttach` that inconditionally denies all attaching to a container. The end result will be: 6 | 7 | ```bash 8 | $ kubectl attach mypod-74c9fd65cb-n5hsg 9 | If you don't see a command prompt, try pressing enter. 10 | Error from server (Forbidden): pods "mypod-74c9fd65cb-n5hsg" is forbidden: cannot attach to a container, rejected by admission controller 11 | ``` 12 | 13 | The implementation of `DenyAttach` will be reusing code from the existing admission controller [DenyEscalatingExec](https://kubernetes.io/docs/admin/admission-controllers/#denyescalatingexec). 14 | 15 | ## Compiling locally 16 | 17 | We will first fetch the [patch](https://github.com/kinvolk/kubernetes/commit/c117bd71672b2da7c7777cddf0287b07d29b90e5). 18 | 19 | ```bash 20 | $ cd $GOPATH/src/k8s.io/kubernetes 21 | 22 | # Add git kinvolk remote if not already done 23 | $ git remote |grep -q kinvolk || git remote add kinvolk https://github.com/kinvolk/kubernetes 24 | 25 | # Fetch the branch 26 | $ git pull kinvolk alban/v1.8.5-beta.0-denyattach 27 | $ git checkout kinvolk/alban/v1.8.5-beta.0-denyattach 28 | 29 | # Build Kubernetes 30 | $ build/run.sh make 31 | 32 | # Build a Hyperkube Docker image with a tag 33 | $ make -C cluster/images/hyperkube VERSION=v1.8.5-beta.0-denyattach 34 | 35 | $ docker images | grep hyperkube-amd64 36 | ``` 37 | 38 | ## Pushing the new hyperkube image to a registry of your choice 39 | 40 | In this example, we will spawn a local registry: 41 | 42 | ``` 43 | docker run -d -p 5000:5000 --name registry registry:2 44 | ``` 45 | 46 | Tag the hyperkube image and push it, for example: 47 | 48 | ``` 49 | docker tag e0d598144aa3 127.0.0.1:5000/me/hyperkube-amd64:v1.8.5-beta.0-denyattach 50 | docker push 127.0.0.1:5000/me/hyperkube-amd64:v1.8.5-beta.0-denyattach 51 | ``` 52 | 53 | ## Deploying your build on kube-spawn 54 | 55 | ``` 56 | $ sudo ./kube-spawn create --kubernetes-source-dir $GOPATH/src/k8s.io/kubernetes --hyperkube-image 10.22.0.1:5000/me/hyperkube-amd64:v1.8.5-beta.0-denyattach -c denyattach 57 | $ sudo ./kube-spawn start -c denyattach 58 | ``` 59 | 60 | Note that the registry IP address must be `10.22.0.1` here, which is the 61 | address of the host `cni0` interface by kube-spawn. 62 | 63 | Since the hyperkube image contains the API server, controller manager and 64 | scheduler but not e.g. kubeadm, we also pass `--kubernetes-source-dir` 65 | to point kube-spawn to the location from where to copy the necessary 66 | binaries. If not given, kube-spawn would use the vanilla upstream version 67 | (`--kubernetes-version` default). 68 | 69 | Let's test if it works: 70 | 71 | ``` 72 | $ export KUBECONFIG=/var/lib/kube-spawn/clusters/denyattach/admin.kubeconfig 73 | $ kubectl run mypod --image=busybox --command -- /bin/sh -c 'while true ; do sleep 1 ; date ; done' 74 | ... 75 | $ kubectl get pods 76 | NAME READY STATUS RESTARTS AGE 77 | mypod-74c9fd65cb-n9rfs 1/1 Running 0 11s 78 | $ kubectl attach mypod-74c9fd65cb-n9rfs 79 | If you don't see a command prompt, try pressing enter. 80 | Error from server (Forbidden): pods "mypod-74c9fd65cb-n9rfs" is forbidden: cannot attach to a container, rejected by admission controller 81 | ``` 82 | 83 | ## Testing a different DenyAttach admission controller 84 | 85 | Someone might not like the error message, saying "rejected by admission controller": 86 | Kubernetes has plenty of admission controllers and it does not say which one rejected the request. 87 | 88 | Luckily, a colleague fixed that already. Let's test her patch: 89 | 90 | ``` 91 | $ sudo ./kube-spawn create --kubernetes-source-dir $GOPATH/src/k8s.io/kubernetes --hyperkube-image docker.io/kinvolk/hyperkube-amd64:v1.8.5-beta.0-denyattachfix -c denyattachfix 92 | $ sudo ./kube-spawn start -c denyattachfix 93 | ``` 94 | 95 | ``` 96 | $ export KUBECONFIG=/var/lib/kube-spawn/clusters/denyattachfix/admin.kubeconfig 97 | $ kubectl run mypod --image=busybox --command -- /bin/sh -c 'while true ; do sleep 1 ; date ; done' 98 | ... 99 | $ kubectl get pods 100 | NAME READY STATUS RESTARTS AGE 101 | mypod-74c9fd65cb-gbrd9 1/1 Running 0 11s 102 | $ kubectl attach mypod-74c9fd65cb-gbrd9 103 | If you don't see a command prompt, try pressing enter. 104 | Error from server (Forbidden): pods "mypod-74c9fd65cb-gbrd9" is forbidden: cannot attach to a container, rejected by DenyAttach 105 | ``` 106 | -------------------------------------------------------------------------------- /doc/devel/release.md: -------------------------------------------------------------------------------- 1 | # kube-spawn release guide 2 | 3 | ## Release cycle 4 | 5 | This section describes the typical release cycle of kube-spawn: 6 | 7 | 1. A GitHub [milestone][milestones] sets the target date for a future kube-spawn release. 8 | 2. Issues grouped into the next release milestone are worked on in order of priority. 9 | 3. Changes are submitted for review in the form of a GitHub Pull Request (PR). Each PR undergoes review and must pass continuous integration (CI) tests before being accepted and merged into the main line of kube-spawn source code. 10 | 4. The day before each release is a short code freeze during which no new code or dependencies may be merged. Instead, this period focuses on polishing the release, with tasks concerning: 11 | * Documentation 12 | * Usability tests 13 | * Issues triaging 14 | * Roadmap planning and scheduling the next release milestone 15 | * Organizational and backlog review 16 | * Build, distribution, and install testing by release manager 17 | 18 | ## Release process 19 | 20 | This section shows how to perform a release of kube-spawn. 21 | Only parts of the procedure are automated; this is somewhat intentional (manual steps for sanity checking) but it can probably be further scripted, help is appreciated. 22 | The following example assumes we're going from version 0.1.1 (`v0.1.1`) to 0.2.0 (`v0.2.0`). 23 | 24 | Let's get started: 25 | 26 | - Start at the relevant milestone on GitHub (e.g. https://github.com/kinvolk/kube-spawn/milestones/v0.2.0): ensure all referenced issues are closed (or moved elsewhere, if they're not done). 27 | - Make sure your git status is clean: `git status` 28 | - Create a tag locally: `git tag v0.2.0 -m "kube-spawn v0.2.0"` 29 | - Build the release: 30 | - `git clean -ffdx && make` should work 31 | - check that the version is correct: `./kube-spawn --version` 32 | - smoke test the release 33 | - Integration tests on CI should be green 34 | - Run the [CNCF conformance tests][conformance-tests] and keep the results 35 | - Prepare the release notes. See [the previous release notes][release-notes] for example. 36 | Try to capture most of the salient changes since the last release, but don't go into unnecessary detail (better to link/reference the documentation wherever possible). 37 | 38 | Push the tag to GitHub: 39 | 40 | - Push the tag to GitHub: `git push --tags` 41 | 42 | Now we switch to the GitHub web UI to conduct the release: 43 | 44 | - Start a [new release][gh-new-release] on Github 45 | - Tag "v0.2.0", release title "v0.2.0" 46 | - Copy-paste the release notes you prepared earlier 47 | - Attach the release. 48 | This is a simple tarball: 49 | 50 | ``` 51 | export KSVER="0.2.0" 52 | export NAME="kube-spawn-v$KSVER" 53 | mkdir $NAME 54 | cp kube-spawn $NAME/ 55 | sudo chown -R root:root $NAME/ 56 | tar czvf $NAME.tar.gz --numeric-owner $NAME/ 57 | ``` 58 | 59 | - Ensure the milestone on GitHub is closed (e.g. https://github.com/kinvolk/kube-spawn/milestones/v0.2.0) 60 | 61 | - Publish the release! 62 | 63 | - Submit the [CNCF conformance tests results][conformance-tests] 64 | 65 | - Clean your git tree: `sudo git clean -ffdx`. 66 | 67 | [conformance-tests]: https://github.com/cncf/k8s-conformance/blob/master/instructions.md 68 | [release-notes]: https://github.com/kinvolk/kube-spawn/releases 69 | [gh-new-release]: https://github.com/kinvolk/kube-spawn/releases/new 70 | [milestones]: https://github.com/kinvolk/kube-spawn/milestones 71 | -------------------------------------------------------------------------------- /doc/distro.md: -------------------------------------------------------------------------------- 1 | ## How to set up kube-spawn on various Linux distros 2 | 3 | ### Common 4 | 5 | First of all, to be able to run kube-spawn, you need to make sure that the 6 | following things are done on your system, no matter which distro you run on. 7 | 8 | * make sure that systemd v233 or newer is installed 9 | * set SELinux mode to either `Permissive` or `Disabled`, e.g.: `sudo setenforce 0` 10 | * set env variables correctly 11 | * set `GOPATH` correctly, e.g. `export GOPATH=$HOME/go` 12 | * set `KUBECONFIG`: `export KUBECONFIG=/var/lib/kube-spawn/clusters/default/admin.kubeconfig` 13 | 14 | ### Fedora 15 | 16 | Fedora 26 or newer is needed, mainly for systemd. 17 | kube-spawn works fine on Fedora as long as the following dependencies are installed. 18 | 19 | #### install required packages 20 | 21 | ``` 22 | sudo dnf install -y btrfs-progs git go iptables libselinux-utils polkit qemu-img systemd-container 23 | ``` 24 | 25 | ### Ubuntu 26 | 27 | Ubuntu 17.10 (Artful) or newer is needed, mainly for systemd. 28 | 29 | #### install required packages 30 | 31 | ``` 32 | sudo apt-get install -y btrfs-progs git golang iptables policykit-1 qemu-utils selinux-utils systemd-container 33 | ``` 34 | 35 | #### systemd-resolved 36 | 37 | On Ubuntu 17.10, systemd-resolved is enabled by default, as well as its stub 38 | listener, which listens on 127.0.0.53:53 for the local DNS resolution. 39 | Unfortunately, systemd v234, the default version on Ubuntu 17.10, has bugs 40 | regarding DNS resolution. Thus we need to disable systemd-resolved on the host, 41 | which will make systemd-resolved disabled inside nspawn containers as well. 42 | So it's recommended to run the following commands on the host, before starting 43 | kube-spawn clusters. 44 | 45 | ``` 46 | sudo sed -i -e 's/^#*.*DNSStubListener=.*$/DNSStubListener=no/' /etc/systemd/resolved.conf 47 | sudo sed -i -e 's/nameserver 127.0.0.53/nameserver 8.8.8.8/' /etc/resolv.conf 48 | systemctl is-active systemd-resolved >& /dev/null && sudo systemctl stop systemd-resolved 49 | systemctl is-enabled systemd-resolved >& /dev/null && sudo systemctl disable systemd-resolved 50 | ``` 51 | 52 | ### Debian 53 | 54 | Normally it should be similar to Ubuntu. 55 | 56 | ### openSUSE Kubic 57 | 58 | All versions of openSUSE Kubic should work. 59 | 60 | #### install required packages 61 | 62 | ``` 63 | transactional-update pkg install kubernetes-client systemd-container cni-plugins 64 | systemctl reboot 65 | ``` 66 | 67 | #### CNI plugins 68 | 69 | openSUSE has a cni-plugin RPM. If this should be used, CNI_PATH 70 | has to be set: 71 | 72 | ``` 73 | export CNI_PATH=/usr/lib/cni 74 | ``` 75 | 76 | -------------------------------------------------------------------------------- /doc/rktlet.md: -------------------------------------------------------------------------------- 1 | # Run Kubernetes with rkt as container runtime 2 | 3 | kube-spawn supports rktlet, the rkt implementation of the Kubernetes Container 4 | Runtime Interface: https://github.com/kubernetes-incubator/rktlet) 5 | 6 | To use rkt as the container runtime, set `--container-runtime=rkt` with the 7 | `create` command. rkt and rktlet must be available on the host. 8 | 9 | Example: 10 | 11 | ``` 12 | sudo ./kube-spawn create --container-runtime rkt --rktlet-binary-path ~/code/go/src/github.com/kubernetes-incubator/rktlet/bin/rktlet -c rktcluster 13 | sudo ./kube-spawn start -c rktcluster -n 5 14 | ... 15 | export KUBECONFIG=/var/lib/kube-spawn/clusters/rktcluster/admin.kubeconfig 16 | kubectl get nodes -o wide 17 | NAME STATUS ROLES AGE VERSION EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME 18 | kube-spawn-rktcluster-master-yomfri Ready master 1m v1.9.6 Flatcar Linux by Kinvolk 1729.0.0 (Rhyolite) 4.15.0-2-amd64 rkt://0.1.0 19 | kube-spawn-rktcluster-worker-4u9fsu Ready 41s v1.9.6 Flatcar Linux by Kinvolk 1729.0.0 (Rhyolite) 4.15.0-2-amd64 rkt://0.1.0 20 | kube-spawn-rktcluster-worker-mysslr Ready 41s v1.9.6 Flatcar Linux by Kinvolk 1729.0.0 (Rhyolite) 4.15.0-2-amd64 rkt://0.1.0 21 | kube-spawn-rktcluster-worker-ogrm8l Ready 40s v1.9.6 Flatcar Linux by Kinvolk 1729.0.0 (Rhyolite) 4.15.0-2-amd64 rkt://0.1.0 22 | kube-spawn-rktcluster-worker-yxspu2 Ready 41s v1.9.6 Flatcar Linux by Kinvolk 1729.0.0 (Rhyolite) 4.15.0-2-amd64 rkt://0.1.0 23 | ``` 24 | 25 | `--rkt-binary-path` and `--rkt-stage1-image-path` can be used to specify 26 | non-default location for the rkt / stage1 binaries. 27 | 28 | NB: rktlet doesn't support Kubernetes 1.10 at the time of writing, see 29 | https://github.com/kubernetes-incubator/rktlet/issues/183 30 | -------------------------------------------------------------------------------- /doc/troubleshooting.md: -------------------------------------------------------------------------------- 1 | # Troubleshooting 2 | 3 | Here are some common issues that we encountered and how we work around or 4 | fix them. If you discover more, please create an issue or submit a PR. 5 | 6 | - [`/var/lib/machines` partition too small](#varlibmachines-partition-too-small) 7 | - [SELinux](#selinux) 8 | - [Restarting machines fails without removing machine images](#restarting-machines-fails-without-removing-machine-images) 9 | - [Running on a version of systemd \< 233](#running-on-a-version-of-systemd--233) 10 | - [kubeadm init looks like it is hanging](#kubeadm-init-looks-like-it-is-hanging) 11 | - [Inotify problems with many nodes](#inotify-problems-with-many-nodes) 12 | - [Issues with ISPs hijacking DNS requests](#issues-with-isps-hijacking-dns-requests) 13 | 14 | ## `/var/lib/machines` partition too small 15 | 16 | Run the following commands to enlarge the storage pool where `POOL_SIZE` 17 | is the disk image size in bytes: 18 | 19 | ``` 20 | # umount /var/lib/machines 21 | # qemu-img resize -f raw /var/lib/machines.raw POOL_SIZE 22 | # mount -t btrfs -o loop /var/lib/machines.raw /var/lib/machines 23 | # btrfs filesystem resize max /var/lib/machines 24 | # btrfs quota disable /var/lib/machines 25 | ``` 26 | 27 | Note that the commands above can fail for some reasons. For example, `umount` can fail because `/var/lib/machines` does not exist. In that case, you might need to create the directory. Or `umount` can fail with `EBUSY`, then you might need to figure out which process blocks umount. 28 | 29 | If `/var/lib/machines.raw` does not exist at all, then it means probably that systemd-machined has never initialized the storage pool. So you might need to do the initialization, for example: 30 | 31 | ``` 32 | sudo truncate -s 20G /var/lib/machines.raw 33 | sudo mkfs -t btrfs /var/lib/machines.raw 34 | sudo mount -o loop -t btrfs /var/lib/machines.raw /var/lib/machines 35 | ``` 36 | 37 | You might also want to set an upper limit for the volume by running `sudo machinectl set-limit 20G`. 38 | 39 | ## SELinux 40 | 41 | To run `kube-spawn`, it is recommended to turn off SELinux enforcing mode: 42 | 43 | ``` 44 | $ sudo setenforce 0 45 | ``` 46 | 47 | However, it is also true that disabling security framework is not always desirable. So it is planned to handle security policy instead of disabling them. Until then, there's no easy way to get around. 48 | 49 | ## Restarting machines fails without removing machine images 50 | 51 | If the `start` command fails, make sure to remove all created images 52 | (`machinectl remove ...`) before trying again. 53 | 54 | ## Running on a version of systemd < 233 55 | 56 | You can build `systemd-nspawn` yourself and include these patches: 57 | 58 | * `SYSTEMD_NSPAWN_USE_CGNS` https://github.com/systemd/systemd/pull/3809 59 | * `SYSTEMD_NSPAWN_MOUNT_RW` and `SYSTEMD_NSPAWN_USE_NETNS` https://github.com/systemd/systemd/pull/4395 60 | 61 | We highly recommend using CoreOS' fork which backported that feature 62 | to the 231 version of systemd (which is the one that Fedora and 63 | the other popular distributions are using in its stable releases). 64 | 65 | In order to do that, please use the following commands: 66 | 67 | ``` 68 | $ git clone git@github.com:coreos/systemd.git 69 | $ cd systemd 70 | $ git checkout v231 71 | $ ./autogen.sh 72 | $ ./configure 73 | $ make 74 | ``` 75 | 76 | You **shouldn't** do `make install` after that! Using the custom 77 | `systemd-nspawn` binary with the other components of systemd being 78 | in another version is totally fine. 79 | 80 | You may try to use master branch from upstream systemd repository, but we 81 | don't encourage it. 82 | 83 | You can pass `kube-spawn` an alternative `systemd-nspawn` binary by setting the 84 | environment variable `SYSTEMD_NSPAWN_PATH` to where you have built your own. 85 | 86 | ## kubeadm init looks like it is hanging 87 | 88 | Usually it takes 1-3 minutes until `kubeadm init` initialized the 89 | cluster nodes and finished bootstrapping on the master node. While waiting 90 | for it to finish, it shows a message like: 91 | 92 | ``` 93 | [init] This often takes around a minute; or longer if the control plane images have to be pulled. 94 | ``` 95 | 96 | kubeadm does not give many hints to users. Possible reasons are: 97 | 98 | * container runtime (docker or rktlet) is not running or running incorrectly 99 | * kubelet is not running or running incorrectly 100 | * any other fundamental errors like filesystem being full 101 | 102 | So in that case, users should find out the underlying reasons by doing e.g.: 103 | 104 | ``` 105 | $ sudo machinectl shell kubespawn0 106 | ``` 107 | 108 | Then inside kubespawn0, do debugging like: 109 | 110 | ``` 111 | # systemctl status docker 112 | # systemctl status kubelet 113 | # journalctl -u docker -xe --no-pager | less 114 | # journalctl -u kubelet -xe --no-pager | less 115 | ``` 116 | 117 | ## Inotify problems with many nodes 118 | 119 | Running a big amount of nodes (many-node clusters or many clusters) can cause inotify limits to be reached, making new nodes fail to start. 120 | 121 | The symptom is a message like this in the kubelet logs on nodes with the `NotReady` state: 122 | 123 | ``` 124 | Failed to start cAdvisor inotify_add_watch /sys/fs/cgroup/blkio/machine.slice/machine-kubespawndefault0.scope/system.slice/var-lib-docker-overlay2-0646d006ef5cf6c4d61c1ad51f958d0891d184ba70a2816d30462175a80beeaa-merged.mount: no space left on device 125 | ``` 126 | 127 | To increase inotify limits you can use the sysctl tool on the host: 128 | 129 | ``` 130 | # sysctl fs.inotify.max_user_watches=524288 131 | # sysctl fs.inotify.max_user_instances=8192 132 | ``` 133 | 134 | ## Issues with ISPs hijacking DNS requests 135 | 136 | Some ISPs use [DNS 137 | hijacking](https://en.wikipedia.org/wiki/DNS_hijacking#Manipulation_by_ISPs), 138 | violating the DNS protocol. Please check if your DNS server correctly returns the `NXDOMAIN` error on non-existent domains: 139 | 140 | ``` 141 | $ host non-existent-domain-name-7932432687432.com 142 | Host non-existent-domain-name-7932432687432.com not found: 3(NXDOMAIN) 143 | ``` 144 | 145 | If it's not the case, the kube-dns pod might not start correctly or might be very slow: 146 | 147 | ``` 148 | $ kubectl get pods --all-namespaces 149 | NAMESPACE NAME READY STATUS RESTARTS AGE 150 | kube-system kube-dns-2425271678-t7mrw 0/3 ContainerCreating 0 5m 151 | ``` 152 | 153 | To fix this issue, please specify valid DNS servers on the host. Example: 154 | ``` 155 | $ cat /etc/resolv.conf 156 | nameserver 8.8.8.8 157 | ``` 158 | -------------------------------------------------------------------------------- /doc/vagrant.md: -------------------------------------------------------------------------------- 1 | ## Testing `kube-spawn` with [Vagrant](https://www.vagrantup.com/) 2 | 3 | ### With script 4 | 5 | There is a script called `vagrant-all.sh` which does the following things: 6 | 7 | - sets up the Vagrant VM (`vagrant up`) 8 | - automatically builds the project during provisioning 9 | - runs the Kubernetes cluster 10 | - redirects traffic from 6443 port on VM to the container with k8s API 11 | - copies the `kubeconfig` to host 12 | 13 | ``` 14 | $ ./vagrant-all.sh 15 | ``` 16 | 17 | Then you can use the Kubernetes cluster both from inside the VM: 18 | 19 | ``` 20 | $ vagrant ssh 21 | $ kubectl get nodes 22 | NAME STATUS AGE VERSION 23 | kubespawndefault0 Ready 12m v1.7.5 24 | kubespawndefault1 Ready 11m v1.7.5 25 | ``` 26 | 27 | or from host, if you have `kubectl` installed on it: 28 | 29 | ``` 30 | $ kubectl get nodes 31 | NAME STATUS AGE VERSION 32 | kubespawndefault0 Ready 12m v1.7.5 33 | kubespawndefault1 Ready 11m v1.7.5 34 | ``` 35 | 36 | ### Manually 37 | 38 | The provided Vagrantfile is used to test `kube-spawn` on various Linux distributions. 39 | 40 | ``` 41 | $ vagrant up 42 | $ vagrant ssh 43 | $ ./build.sh # sets up environment, runs build and setup/init command 44 | ``` 45 | 46 | You can set the following environment variables: 47 | 48 | - `KUBESPAWN_AUTOBUILD` - runs `build.sh` script (which builds kube-spawn) during machine 49 | provisioning 50 | - `KUBESPAWN_REDIRECT_TRAFFIC` - redirects traffic from 6443 port on VM to the container 51 | with k8s API 52 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/kinvolk/kube-spawn 2 | 3 | go 1.12 4 | 5 | require ( 6 | github.com/Masterminds/semver v1.4.2 7 | github.com/containernetworking/cni v0.7.0 8 | github.com/containernetworking/plugins v0.7.0 9 | github.com/golang/protobuf v1.3.1 // indirect 10 | github.com/kr/pretty v0.1.0 // indirect 11 | github.com/magiconair/properties v1.8.1 // indirect 12 | github.com/onsi/ginkgo v1.8.0 // indirect 13 | github.com/onsi/gomega v1.5.0 // indirect 14 | github.com/pelletier/go-toml v1.4.0 // indirect 15 | github.com/pkg/errors v0.8.1 16 | github.com/spf13/afero v1.2.2 // indirect 17 | github.com/spf13/cobra v0.0.4 18 | github.com/spf13/jwalterweatherman v1.1.0 // indirect 19 | github.com/spf13/viper v1.3.2 20 | github.com/stretchr/testify v1.3.0 // indirect 21 | golang.org/x/net v0.0.0-20190520210107-018c4d40a106 // indirect 22 | golang.org/x/sys v0.0.0-20190522044717-8097e1b27ff5 23 | golang.org/x/text v0.3.2 // indirect 24 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect 25 | ) 26 | -------------------------------------------------------------------------------- /logos/PNG/kube_spawn-horz_prpblkonTRSP.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kinvolk/kube-spawn/6173ce6eeebef80bbc2ce0d29243b4bf13c97ff2/logos/PNG/kube_spawn-horz_prpblkonTRSP.png -------------------------------------------------------------------------------- /logos/PNG/kube_spawn-horz_prpblkonwht.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kinvolk/kube-spawn/6173ce6eeebef80bbc2ce0d29243b4bf13c97ff2/logos/PNG/kube_spawn-horz_prpblkonwht.png -------------------------------------------------------------------------------- /logos/SVG/kube_spawn-horz_blkonwht.svg: -------------------------------------------------------------------------------- 1 | Artboard 15 -------------------------------------------------------------------------------- /logos/SVG/kube_spawn-horz_prpblkonwht.svg: -------------------------------------------------------------------------------- 1 | Artboard 27 -------------------------------------------------------------------------------- /logos/SVG/kube_spawn-horz_prponwht.svg: -------------------------------------------------------------------------------- 1 | Artboard 19 -------------------------------------------------------------------------------- /logos/SVG/kube_spawn-horz_redblkonwht.svg: -------------------------------------------------------------------------------- 1 | Artboard 29 -------------------------------------------------------------------------------- /logos/SVG/kube_spawn-horz_redonwht.svg: -------------------------------------------------------------------------------- 1 | Artboard 21 -------------------------------------------------------------------------------- /logos/SVG/kube_spawn-horz_whtonblk.svg: -------------------------------------------------------------------------------- 1 | Artboard 17 -------------------------------------------------------------------------------- /logos/SVG/kube_spawn-horz_whtonprp.svg: -------------------------------------------------------------------------------- 1 | Artboard 25 -------------------------------------------------------------------------------- /logos/SVG/kube_spawn-horz_whtonred.svg: -------------------------------------------------------------------------------- 1 | Artboard 23 -------------------------------------------------------------------------------- /logos/SVG/kube_spawn-vert_blkonwht.svg: -------------------------------------------------------------------------------- 1 | Artboard 16 -------------------------------------------------------------------------------- /logos/kube_spawn-horz_blkonwht.svg: -------------------------------------------------------------------------------- 1 | Artboard 15 -------------------------------------------------------------------------------- /logos/kube_spawn-horz_prpblkonwht.svg: -------------------------------------------------------------------------------- 1 | Artboard 27 -------------------------------------------------------------------------------- /logos/kube_spawn-horz_prponwht.svg: -------------------------------------------------------------------------------- 1 | Artboard 19 -------------------------------------------------------------------------------- /logos/kube_spawn-horz_redblkonwht.svg: -------------------------------------------------------------------------------- 1 | Artboard 29 -------------------------------------------------------------------------------- /logos/kube_spawn-horz_redonwht.svg: -------------------------------------------------------------------------------- 1 | Artboard 21 -------------------------------------------------------------------------------- /logos/kube_spawn-horz_whtonblk.svg: -------------------------------------------------------------------------------- 1 | Artboard 17 -------------------------------------------------------------------------------- /logos/kube_spawn-horz_whtonprp.svg: -------------------------------------------------------------------------------- 1 | Artboard 25 -------------------------------------------------------------------------------- /logos/kube_spawn-horz_whtonred.svg: -------------------------------------------------------------------------------- 1 | Artboard 23 -------------------------------------------------------------------------------- /logos/kube_spawn-vert_blkonwht.svg: -------------------------------------------------------------------------------- 1 | Artboard 16 -------------------------------------------------------------------------------- /logos/kube_spawn-vert_prpblkonwht.svg: -------------------------------------------------------------------------------- 1 | Artboard 28 -------------------------------------------------------------------------------- /logos/kube_spawn-vert_redblkonwht.svg: -------------------------------------------------------------------------------- 1 | Artboard 30 -------------------------------------------------------------------------------- /pkg/bootstrap/cninet.go: -------------------------------------------------------------------------------- 1 | package bootstrap 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "os" 7 | "path" 8 | ) 9 | 10 | const NspawnNetPath string = "/etc/cni/net.d/10-kube-spawn-net.conf" 11 | const NspawnNetConf string = ` 12 | { 13 | "cniVersion": "0.2.0", 14 | "name": "kube-spawn-net", 15 | "type": "bridge", 16 | "bridge": "cni0", 17 | "isGateway": true, 18 | "ipMasq": true, 19 | "ipam": { 20 | "type": "host-local", 21 | "subnet": "10.22.0.0/16", 22 | "routes": [ 23 | { "dst": "0.0.0.0/0" } 24 | ] 25 | } 26 | }` 27 | const LoopbackNetPath string = "/etc/cni/net.d/10-loopback.conf" 28 | const LoopbackNetConf string = ` 29 | { 30 | "cniVersion": "0.2.0", 31 | "type": "loopback" 32 | }` 33 | 34 | func writeNetConf(fpath, content string) error { 35 | if _, err := os.Stat(fpath); os.IsExist(err) { 36 | return nil 37 | } 38 | dir, _ := path.Split(fpath) 39 | if err := os.MkdirAll(dir, os.ModePerm); err != nil { 40 | return nil 41 | } 42 | if err := ioutil.WriteFile(fpath, []byte(content), os.ModePerm); err != nil { 43 | return fmt.Errorf("error writing %s: %s", fpath, err) 44 | } 45 | return nil 46 | } 47 | 48 | func WriteNetConf() error { 49 | if err := writeNetConf(NspawnNetPath, NspawnNetConf); err != nil { 50 | return err 51 | } 52 | if err := writeNetConf(LoopbackNetPath, LoopbackNetConf); err != nil { 53 | return err 54 | } 55 | return nil 56 | } 57 | -------------------------------------------------------------------------------- /pkg/bootstrap/download.go: -------------------------------------------------------------------------------- 1 | package bootstrap 2 | 3 | import ( 4 | "log" 5 | "net/http" 6 | "os" 7 | "path" 8 | "strings" 9 | "sync" 10 | 11 | "github.com/kinvolk/kube-spawn/pkg/utils" 12 | "github.com/kinvolk/kube-spawn/pkg/utils/fs" 13 | "github.com/pkg/errors" 14 | ) 15 | 16 | const ( 17 | k8sURL string = "https://dl.k8s.io/$VERSION/bin/linux/amd64/" 18 | k8sGithubURL string = "https://raw.githubusercontent.com/kubernetes/kubernetes/$VERSION/build/rpms/" 19 | staticSocatUrl string = "https://raw.githubusercontent.com/andrew-d/static-binaries/530df977dd38ba3b4197878b34466d49fce69d8e/binaries/linux/x86_64/socat" 20 | 21 | sha1Suffix string = ".sha1" 22 | ) 23 | 24 | var ( 25 | // note: we are downloading these in parallel (limit number or improve DownloadK8sBins func) 26 | // 27 | // key: full URL for a file to be downloaded. 28 | // value: whether it has to be verified with checksum or not 29 | k8sBinaryFiles = map[string]bool{ 30 | k8sURL + "kubelet": true, 31 | k8sURL + "kubeadm": true, 32 | k8sURL + "kubectl": true, 33 | k8sGithubURL + "kubelet.service": false, 34 | k8sGithubURL + "10-kubeadm.conf": false, 35 | } 36 | ) 37 | 38 | func Download(url, fpath string) error { 39 | resp, err := http.Get(url) 40 | if err != nil { 41 | return err 42 | } 43 | defer resp.Body.Close() 44 | 45 | if resp.StatusCode != 200 { 46 | return errors.Errorf("server returned [%d] %q", resp.StatusCode, resp.Status) 47 | } 48 | return fs.CreateFileFromReader(fpath, resp.Body) 49 | } 50 | 51 | func DownloadKubernetesBinaries(k8sVersion, targetDir string) error { 52 | var err error 53 | versionPath := path.Join(targetDir, k8sVersion) 54 | if exists, err := fs.PathExists(versionPath); err != nil { 55 | return err 56 | } else if !exists { 57 | if err := os.MkdirAll(versionPath, 0755); err != nil { 58 | return err 59 | } 60 | } 61 | 62 | var wg sync.WaitGroup 63 | wg.Add(len(k8sBinaryFiles)) 64 | for url, verifyHash := range k8sBinaryFiles { 65 | // replace placeholder $VERSION with actual version parameter 66 | go func(url string, verifyHash bool) { 67 | defer wg.Done() 68 | url = strings.Replace(url, "$VERSION", k8sVersion, 1) 69 | inCachePath := path.Join(versionPath, path.Base(url)) 70 | if exists, err := fs.PathExists(inCachePath); err != nil { 71 | log.Printf("Error checking if path %q exists: %v\n", inCachePath, err) 72 | return 73 | } else if !exists { 74 | log.Printf("Downloading %s", path.Base(inCachePath)) 75 | if err = Download(url, inCachePath); err != nil { 76 | err = errors.Wrapf(err, "error downloading %s", url) 77 | return 78 | } 79 | 80 | if verifyHash { 81 | sha1URL := url + sha1Suffix 82 | sha1CachePath := inCachePath + sha1Suffix 83 | if err = Download(sha1URL, sha1CachePath); err != nil { 84 | err = errors.Wrapf(err, "error downloading %s", sha1URL) 85 | return 86 | } 87 | 88 | if err = utils.VerifySha1(inCachePath, sha1CachePath); err != nil { 89 | err = errors.Wrapf(err, "error verifying checksum of %s", sha1URL) 90 | return 91 | } 92 | } 93 | } 94 | }(url, verifyHash) 95 | } 96 | wg.Wait() 97 | 98 | return err 99 | } 100 | 101 | func DownloadSocatBin(targetDir string) error { 102 | if exists, err := fs.PathExists(targetDir); err != nil { 103 | return err 104 | } else if !exists { 105 | if err := os.MkdirAll(targetDir, 0755); err != nil { 106 | return err 107 | } 108 | } 109 | inCachePath := path.Join(targetDir, path.Base(staticSocatUrl)) 110 | 111 | if exists, err := fs.PathExists(inCachePath); err != nil { 112 | return err 113 | } else if !exists { 114 | log.Printf("downloading %s", path.Base(inCachePath)) 115 | if err := Download(staticSocatUrl, inCachePath); err != nil { 116 | return errors.Wrapf(err, "error downloading %s", staticSocatUrl) 117 | } 118 | } 119 | return nil 120 | } 121 | -------------------------------------------------------------------------------- /pkg/bootstrap/util.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 Kinvolk GmbH 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package bootstrap 18 | 19 | import ( 20 | "fmt" 21 | "log" 22 | "os" 23 | "path/filepath" 24 | "syscall" 25 | "unsafe" 26 | ) 27 | 28 | const ( 29 | FsMagicAUFS = 0x61756673 // https://goo.gl/CBwx43 30 | FsMagicECRYPTFS = 0xF15F // https://goo.gl/4akUXJ 31 | FsMagicZFS = 0x2FC12FC1 // https://goo.gl/xTvzO5 32 | ) 33 | 34 | // PathSupportsOverlay checks whether the given path is compatible with OverlayFS. 35 | // This method also calls isOverlayfsAvailable(). 36 | // It returns error if OverlayFS is not supported. 37 | // - taken from https://github.com/rkt/rkt/blob/master/common/common.go 38 | func PathSupportsOverlay(path string) error { 39 | if err := ensureOverlayfs(); err != nil { 40 | return err 41 | } 42 | if !isOverlayfsAvailable() { 43 | return fmt.Errorf("overlayfs is not available") 44 | } 45 | 46 | if err := os.MkdirAll(path, 0755); err != nil { 47 | return fmt.Errorf("cannot create directory %q", path) 48 | } 49 | 50 | var data syscall.Statfs_t 51 | if err := syscall.Statfs(path, &data); err != nil { 52 | return fmt.Errorf("cannot statfs %q", path) 53 | } 54 | 55 | switch data.Type { 56 | case FsMagicAUFS: 57 | return fmt.Errorf("unsupported filesystem: aufs") 58 | case FsMagicECRYPTFS: 59 | return fmt.Errorf("unsupported filesystem: ecryptfs") 60 | case FsMagicZFS: 61 | return fmt.Errorf("unsupported filesystem: zfs") 62 | } 63 | 64 | dir, err := os.OpenFile(path, syscall.O_RDONLY|syscall.O_DIRECTORY, 0755) 65 | if err != nil { 66 | return fmt.Errorf("cannot open %q", path) 67 | } 68 | defer dir.Close() 69 | 70 | buf := make([]byte, 4096) 71 | // ReadDirent forwards to the raw syscall getdents(3), 72 | // passing the buffer size. 73 | n, err := syscall.ReadDirent(int(dir.Fd()), buf) 74 | if err != nil { 75 | return fmt.Errorf("cannot read directory %q", path) 76 | } 77 | 78 | offset := 0 79 | for offset < n { 80 | // offset overflow cannot happen, because Reclen 81 | // is being maintained by getdents(3), considering the buffer size. 82 | dirent := (*syscall.Dirent)(unsafe.Pointer(&buf[offset])) 83 | offset += int(dirent.Reclen) 84 | 85 | if dirent.Ino == 0 { // File absent in directory. 86 | continue 87 | } 88 | 89 | if dirent.Type == syscall.DT_UNKNOWN { 90 | return fmt.Errorf("unsupported filesystem: missing d_type support") 91 | } 92 | } 93 | 94 | return nil 95 | } 96 | 97 | func isSameFilesystem(a, b *syscall.Statfs_t) bool { 98 | return a.Fsid == b.Fsid 99 | } 100 | 101 | func checkMountpoint(dir string) error { 102 | sfs1 := &syscall.Statfs_t{} 103 | if err := syscall.Statfs(dir, sfs1); err != nil { 104 | return fmt.Errorf("error calling statfs on %q: %v", dir, err) 105 | } 106 | sfs2 := &syscall.Statfs_t{} 107 | if err := syscall.Statfs(filepath.Dir(dir), sfs2); err != nil { 108 | return fmt.Errorf("error calling statfs on %q: %v", dir, err) 109 | } 110 | if isSameFilesystem(sfs1, sfs2) { 111 | return fmt.Errorf("%q is not a mount point", dir) 112 | } 113 | 114 | return nil 115 | } 116 | 117 | // get free space of volume mounted on volPath (in bytes) 118 | func getVolFreeSpace(volPath string) (uint64, error) { 119 | var stat syscall.Statfs_t 120 | 121 | if err := syscall.Statfs(volPath, &stat); err != nil { 122 | log.Printf("statfs error: %v\n", err) 123 | return 0, err 124 | } 125 | 126 | freeSpace := stat.Bavail * uint64(stat.Bsize) 127 | 128 | return freeSpace, nil 129 | } 130 | 131 | // get allocated size of file (in bytes) 132 | func getAllocatedFileSize(filename string) (int64, error) { 133 | fi, err := os.Stat(filename) 134 | if err != nil { 135 | return 0, err 136 | } 137 | 138 | stat_t, ok := fi.Sys().(*syscall.Stat_t) 139 | if !ok { 140 | return 0, fmt.Errorf("cannot determine allocated filesize") 141 | } 142 | 143 | // stat(2) returns allocated filesize in blocks, each of which is 144 | // a fixed 512 bytes 145 | return (stat_t.Blocks * 512), nil 146 | } 147 | -------------------------------------------------------------------------------- /pkg/cache/cache.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | type Cache struct { 4 | dir string 5 | } 6 | 7 | func New(dir string) (*Cache, error) { 8 | return &Cache{ 9 | dir: dir, 10 | }, nil 11 | } 12 | 13 | func (c *Cache) Dir() string { 14 | return c.dir 15 | } 16 | -------------------------------------------------------------------------------- /pkg/cnispawn/netns.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 Kinvolk GmbH 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package cnispawn 18 | 19 | import ( 20 | "fmt" 21 | "io/ioutil" 22 | "os" 23 | "os/exec" 24 | "path" 25 | "strings" 26 | 27 | "github.com/containernetworking/plugins/pkg/ns" 28 | "github.com/kinvolk/kube-spawn/pkg/bootstrap" 29 | ) 30 | 31 | type CniNetns struct { 32 | netns ns.NetNS 33 | } 34 | 35 | func NewCniNetns(cniPluginDir string) (*CniNetns, error) { 36 | var err error 37 | 38 | netns, err := ns.NewNS() 39 | if err != nil { 40 | return nil, err 41 | } 42 | 43 | sNetnsPath := strings.Split(netns.Path(), "/") 44 | containerId := sNetnsPath[len(sNetnsPath)-1] 45 | 46 | cniBridgePluginPath := path.Join(cniPluginDir, "bridge") 47 | 48 | // CNI-specific environment variables must appear before other ones 49 | // obtained from os.Environ(), so that they can override default ones. 50 | var env []string 51 | env = append(env, "CNI_COMMAND=ADD") 52 | env = append(env, fmt.Sprintf("CNI_CONTAINERID=%s", containerId)) 53 | env = append(env, fmt.Sprintf("CNI_NETNS=%s", netns.Path())) 54 | env = append(env, "CNI_IFNAME=eth0") 55 | env = append(env, fmt.Sprintf("CNI_PATH=%s", cniPluginDir)) 56 | env = append(env, os.Environ()...) 57 | 58 | c := exec.Cmd{ 59 | Path: cniBridgePluginPath, 60 | Args: nil, 61 | Env: env, 62 | Stdout: os.Stdout, 63 | Stderr: os.Stderr, 64 | } 65 | 66 | stdin, err := c.StdinPipe() 67 | if err != nil { 68 | return nil, err 69 | } 70 | 71 | netconfig, err := ioutil.ReadFile(bootstrap.NspawnNetPath) 72 | if err != nil { 73 | return nil, err 74 | } 75 | 76 | if _, err := stdin.Write(netconfig); err != nil { 77 | return nil, err 78 | } 79 | stdin.Close() 80 | 81 | if err := c.Run(); err != nil { 82 | return nil, err 83 | } 84 | 85 | return &CniNetns{ 86 | netns: netns, 87 | }, nil 88 | } 89 | 90 | func (c *CniNetns) Set() error { 91 | return c.netns.Set() 92 | } 93 | 94 | func (c *CniNetns) Close() error { 95 | return c.netns.Close() 96 | } 97 | -------------------------------------------------------------------------------- /pkg/cnispawn/spawn.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 Kinvolk GmbH 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package cnispawn 18 | 19 | import ( 20 | "os" 21 | "os/exec" 22 | "runtime" 23 | "syscall" 24 | ) 25 | 26 | func Spawn(cniPluginDir string, nspawnArgs []string) error { 27 | runtime.LockOSThread() 28 | 29 | cniNetns, err := NewCniNetns(cniPluginDir) 30 | if err != nil { 31 | return err 32 | } 33 | 34 | if err := cniNetns.Set(); err != nil { 35 | return err 36 | } 37 | defer cniNetns.Close() 38 | 39 | systemdNspawnPath := os.Getenv("SYSTEMD_NSPAWN_PATH") 40 | 41 | if systemdNspawnPath == "" { 42 | systemdNspawnPath, err = exec.LookPath("systemd-nspawn") 43 | if err != nil { 44 | return err 45 | } 46 | } 47 | 48 | args := []string{ 49 | systemdNspawnPath, 50 | "--capability=cap_audit_control,cap_audit_read,cap_audit_write,cap_audit_control,cap_block_suspend,cap_chown,cap_dac_override,cap_dac_read_search,cap_fowner,cap_fsetid,cap_ipc_lock,cap_ipc_owner,cap_kill,cap_lease,cap_linux_immutable,cap_mac_admin,cap_mac_override,cap_mknod,cap_net_admin,cap_net_bind_service,cap_net_broadcast,cap_net_raw,cap_setgid,cap_setfcap,cap_setpcap,cap_setuid,cap_sys_admin,cap_sys_boot,cap_sys_chroot,cap_sys_module,cap_sys_nice,cap_sys_pacct,cap_sys_ptrace,cap_sys_rawio,cap_sys_resource,cap_sys_time,cap_sys_tty_config,cap_syslog,cap_wake_alarm", 51 | "--system-call-filter=@keyring", 52 | "--private-users=false", 53 | "--bind=/sys/kernel/security", 54 | "--bind=/sys/fs/cgroup", 55 | "--bind-ro=/boot", 56 | "--bind-ro=/lib/modules", 57 | "--boot", 58 | "--notify-ready=yes", 59 | "--keep-unit", 60 | } 61 | 62 | args = append(args, nspawnArgs...) 63 | 64 | env := os.Environ() 65 | env = append(env, "SYSTEMD_NSPAWN_MOUNT_RW=1") 66 | env = append(env, "SYSTEMD_NSPAWN_API_VFS_WRITABLE=1") 67 | env = append(env, "SYSTEMD_NSPAWN_USE_CGNS=0") 68 | 69 | _, err = syscall.ForkExec(systemdNspawnPath, args, &syscall.ProcAttr{ 70 | Dir: "", 71 | Env: env, 72 | Files: []uintptr{}, 73 | Sys: &syscall.SysProcAttr{}, 74 | }) 75 | return err 76 | } 77 | -------------------------------------------------------------------------------- /pkg/machinectl/machinectl.go: -------------------------------------------------------------------------------- 1 | package machinectl 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "fmt" 7 | "io" 8 | "net" 9 | "os/exec" 10 | "regexp" 11 | "strings" 12 | ) 13 | 14 | type Machine struct { 15 | Name string 16 | IP string 17 | } 18 | 19 | type Image struct { 20 | Name string 21 | } 22 | 23 | func cleanIP(ip string) (string, error) { 24 | trimmed := ip 25 | for _, s := range []string{"...", "…"} { 26 | trimmed = strings.TrimSuffix(trimmed, s) 27 | } 28 | 29 | parsedIP := net.ParseIP(trimmed) 30 | if parsedIP == nil { 31 | return "", fmt.Errorf("invalid IP %q", trimmed) 32 | } 33 | 34 | return parsedIP.String(), nil 35 | } 36 | 37 | func List() ([]Machine, error) { 38 | var machines []Machine 39 | out, err := exec.Command("machinectl", "list", "--no-legend").Output() 40 | if err != nil { 41 | return nil, err 42 | } 43 | scanner := bufio.NewScanner(bytes.NewReader(out)) 44 | for scanner.Scan() { 45 | // Example `machinectl list --no-legend` output: 46 | // kube-spawn-default-worker-fpllng container systemd-nspawn coreos 1478.0.0 10.22.0.130... 47 | line := strings.Fields(scanner.Text()) 48 | if len(line) < 6 { 49 | return nil, fmt.Errorf("got unexpected output from `machinectl list --no-legend`: %s", line) 50 | } 51 | 52 | ip, err := cleanIP(line[5]) 53 | if err != nil { 54 | return nil, err 55 | } 56 | 57 | machine := Machine{ 58 | Name: strings.TrimSpace(line[0]), 59 | IP: ip, 60 | } 61 | machines = append(machines, machine) 62 | } 63 | return machines, nil 64 | } 65 | 66 | func ListByRegexp(expStr string) ([]Machine, error) { 67 | machines, err := List() 68 | if err != nil { 69 | return nil, err 70 | } 71 | exp, err := regexp.Compile(expStr) 72 | if err != nil { 73 | return nil, err 74 | } 75 | var matching []Machine 76 | for _, machine := range machines { 77 | if exp.MatchString(machine.Name) { 78 | matching = append(matching, machine) 79 | } 80 | } 81 | return matching, nil 82 | } 83 | 84 | func ListImages() ([]Image, error) { 85 | var images []Image 86 | out, err := exec.Command("machinectl", "list-images", "--no-legend").Output() 87 | if err != nil { 88 | return nil, err 89 | } 90 | scanner := bufio.NewScanner(bytes.NewReader(out)) 91 | for scanner.Scan() { 92 | // Example `machinectl list-images --no-legend` output: 93 | // kube-spawn-default-worker-zyyios raw no 1.4G n/a Fri 2018-01-26 10:54:43 CET 94 | line := strings.Fields(scanner.Text()) 95 | if len(line) < 1 { 96 | return nil, fmt.Errorf("got unexpected output from `machinectl list-images --no-legend`: %s", line) 97 | } 98 | image := Image{ 99 | Name: strings.TrimSpace(line[0]), 100 | } 101 | images = append(images, image) 102 | } 103 | return images, nil 104 | } 105 | 106 | func ListImagesByRegexp(expStr string) ([]Image, error) { 107 | images, err := ListImages() 108 | if err != nil { 109 | return nil, err 110 | } 111 | exp, err := regexp.Compile(expStr) 112 | if err != nil { 113 | return nil, err 114 | } 115 | var matching []Image 116 | for _, image := range images { 117 | if exp.MatchString(image.Name) { 118 | matching = append(matching, image) 119 | } 120 | } 121 | return matching, nil 122 | } 123 | 124 | func RunCommand(stdout, stderr io.Writer, opts, cmd, machine string, args ...string) ([]byte, error) { 125 | mPath, err := exec.LookPath("machinectl") 126 | if err != nil { 127 | return nil, err 128 | } 129 | cmdArgs := []string{mPath} 130 | if opts != "" { 131 | cmdArgs = append(cmdArgs, opts) 132 | } 133 | cmdArgs = append(cmdArgs, cmd) 134 | cmdArgs = append(cmdArgs, machine) 135 | 136 | run := exec.Cmd{ 137 | Path: mPath, 138 | Args: cmdArgs, 139 | Stdout: stdout, 140 | Stderr: stderr, 141 | } 142 | run.Args = append(run.Args, args...) 143 | 144 | var buf []byte 145 | 146 | if stdout != nil { 147 | err = run.Run() 148 | } else { 149 | buf, err = run.Output() 150 | } 151 | if err != nil { 152 | if exitErr, ok := err.(*exec.ExitError); ok { 153 | return nil, fmt.Errorf("%q failed: %s", strings.Join(run.Args, " "), exitErr.Stderr) 154 | } 155 | return nil, fmt.Errorf("%q failed: %s", strings.Join(run.Args, " "), err) 156 | } 157 | return buf, nil 158 | } 159 | 160 | func Exec(machine string, cmd ...string) error { 161 | _, err := RunCommand(nil, nil, "", "shell", machine, cmd...) 162 | return err 163 | } 164 | 165 | func Clone(base, dest string) error { 166 | _, err := RunCommand(nil, nil, "", "clone", base, dest) 167 | return err 168 | } 169 | 170 | func Poweroff(machine string) error { 171 | _, err := RunCommand(nil, nil, "", "poweroff", machine) 172 | return err 173 | } 174 | 175 | func Terminate(machine string) error { 176 | _, err := RunCommand(nil, nil, "", "terminate", machine) 177 | return err 178 | } 179 | 180 | func Remove(image string) error { 181 | _, err := RunCommand(nil, nil, "", "remove", image) 182 | return err 183 | } 184 | 185 | func IsRunning(machine string) bool { 186 | check := exec.Command("systemctl", "--machine", machine, "status", "basic.target", "--state=running") 187 | if err := check.Run(); err != nil { 188 | return false 189 | } 190 | return check.ProcessState.Success() 191 | } 192 | 193 | func ImageExists(image string) bool { 194 | _, err := RunCommand(nil, nil, "", "show-image", image) 195 | return err == nil 196 | } 197 | -------------------------------------------------------------------------------- /pkg/multiprint/multiprint.go: -------------------------------------------------------------------------------- 1 | package multiprint 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "context" 7 | "fmt" 8 | "strings" 9 | ) 10 | 11 | type message struct { 12 | prefix string 13 | value []byte 14 | } 15 | 16 | type Multiprint struct { 17 | ctx context.Context 18 | messageChan chan message 19 | } 20 | 21 | type Writer struct { 22 | ctx context.Context 23 | messageChan chan message 24 | prefix string 25 | cancelled bool 26 | } 27 | 28 | func New(ctx context.Context) *Multiprint { 29 | return &Multiprint{ 30 | ctx: ctx, 31 | messageChan: make(chan message), 32 | } 33 | } 34 | 35 | func (m *Multiprint) RunPrintLoop() { 36 | go func() { 37 | var previousPrefix, prefix string 38 | for { 39 | select { 40 | case <-m.ctx.Done(): 41 | return 42 | case message, ok := <-m.messageChan: 43 | if !ok { 44 | return 45 | } 46 | if previousPrefix != message.prefix { 47 | previousPrefix = message.prefix 48 | prefix = message.prefix 49 | } 50 | scanner := bufio.NewScanner(bytes.NewBuffer(message.value)) 51 | for scanner.Scan() { 52 | text := strings.TrimSpace(scanner.Text()) 53 | if text == "" { 54 | continue 55 | } 56 | fmt.Printf("%s%s\n", prefix, text) 57 | prefix = strings.Repeat(" ", len(prefix)) 58 | } 59 | } 60 | } 61 | }() 62 | } 63 | 64 | func (m *Multiprint) NewWriter(prefix string) *Writer { 65 | writer := &Writer{ 66 | ctx: m.ctx, 67 | messageChan: m.messageChan, 68 | prefix: prefix, 69 | } 70 | go func() { 71 | select { 72 | case <-writer.ctx.Done(): 73 | writer.cancelled = true 74 | } 75 | }() 76 | return writer 77 | } 78 | 79 | func (w *Writer) Write(p []byte) (n int, err error) { 80 | if w.cancelled { 81 | return 0, fmt.Errorf("writer was cancelled") 82 | } 83 | w.messageChan <- message{prefix: w.prefix, value: p} 84 | return len(p), nil 85 | } 86 | -------------------------------------------------------------------------------- /pkg/nspawntool/run.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 Kinvolk GmbH 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package nspawntool 18 | 19 | import ( 20 | "encoding/json" 21 | "fmt" 22 | "io/ioutil" 23 | "os" 24 | "os/exec" 25 | "path" 26 | "time" 27 | 28 | cnitypes "github.com/containernetworking/cni/pkg/types" 29 | cniversion "github.com/containernetworking/cni/pkg/version" 30 | "github.com/pkg/errors" 31 | 32 | "github.com/kinvolk/kube-spawn/pkg/machinectl" 33 | ) 34 | 35 | func Run(baseImageName, lowerRootPath, upperRootPath, machineName, cniPluginDir string) error { 36 | if machinectl.IsRunning(machineName) { 37 | return errors.Errorf("a machine with name %q is running already", machineName) 38 | } 39 | 40 | if err := machinectl.Clone(baseImageName, machineName); err != nil { 41 | return errors.Wrap(err, "error cloning image") 42 | } 43 | 44 | if err := os.MkdirAll(lowerRootPath, 0755); err != nil { 45 | return err 46 | } 47 | 48 | if err := os.MkdirAll(upperRootPath, 0755); err != nil { 49 | return err 50 | } 51 | 52 | // Create all directories which will be overlay mounts (see below) 53 | // Otherwise systemd-nspawn will fail: 54 | // `overlayfs: failed to resolve '/var/lib/kube-spawn/...'` 55 | if err := os.MkdirAll(path.Join(upperRootPath, "etc"), 0755); err != nil { 56 | return err 57 | } 58 | if err := os.MkdirAll(path.Join(upperRootPath, "opt"), 0755); err != nil { 59 | return err 60 | } 61 | if err := os.MkdirAll(path.Join(upperRootPath, "usr/bin"), 0755); err != nil { 62 | return err 63 | } 64 | 65 | // Create all directories that will be bind mounted 66 | bindmountDirs := []string{ 67 | "/var/lib/docker", 68 | "/var/lib/rktlet", 69 | "/var/lib/kubelet", 70 | } 71 | for _, d := range bindmountDirs { 72 | if err := os.MkdirAll(path.Join(upperRootPath, d), 0755); err != nil { 73 | return err 74 | } 75 | } 76 | 77 | // Invocation of systemd-nspawn is done in the following steps. 78 | // 79 | // 1. "kube-spawn start" calls systemd-run to make use of transient scope. 80 | // This is necessary to avoid dealing with an additional unit file. 81 | // 2. The transient scope calls "kube-spawn cni-spawn", a wrapper for 82 | // dealing with the network namespace for CNI. Note that only the options 83 | // before `--` are interpreted by cni-spawn. 84 | // 3. cni-spawn actually calls systemd-nspawn. Note that only the options 85 | // after `--`, which are given below for systemd-run, are interpreted by 86 | // systemd-nspawn. 87 | kubeSpawnExec, err := os.Executable() 88 | if err != nil { 89 | kubeSpawnExec = "kube-spawn" 90 | } 91 | 92 | var systemdRunExec string 93 | if systemdRunExec, err = exec.LookPath("systemd-run"); err != nil { 94 | return fmt.Errorf("systemd-run not installed: %s", err) 95 | } 96 | 97 | args := []string{ 98 | "--scope", 99 | "--property=DevicePolicy=auto", 100 | kubeSpawnExec, 101 | "cni-spawn", 102 | "--cni-plugin-dir", cniPluginDir, 103 | "--", 104 | "--machine", machineName, 105 | optionsOverlay("--overlay", "/etc", lowerRootPath, upperRootPath), 106 | optionsOverlay("--overlay", "/opt", lowerRootPath, upperRootPath), 107 | optionsOverlay("--overlay", "/usr/bin", lowerRootPath, upperRootPath), 108 | } 109 | 110 | for _, d := range bindmountDirs { 111 | args = append(args, fmt.Sprintf("--bind=%s:%s", path.Join(upperRootPath, d), d)) 112 | } 113 | 114 | c := &exec.Cmd{ 115 | Path: systemdRunExec, 116 | Args: append([]string{systemdRunExec}, args...), 117 | } 118 | c.Stderr = os.Stderr 119 | 120 | stdout, err := c.StdoutPipe() 121 | if err != nil { 122 | return errors.Wrap(err, "error creating stdout pipe") 123 | } 124 | defer stdout.Close() 125 | 126 | if err := c.Start(); err != nil { 127 | return errors.Wrapf(err, "error running %s cnispawn: %v", systemdRunExec, args) 128 | } 129 | 130 | cniDataJSON, err := ioutil.ReadAll(stdout) 131 | if err != nil { 132 | return errors.Wrap(err, "error reading cni data from stdin") 133 | } 134 | 135 | if _, err := cniversion.NewResult(cniversion.Current(), cniDataJSON); err != nil { 136 | return errors.Wrapf(err, "unable to parse CNI data %q", cniDataJSON) 137 | } 138 | 139 | if err := c.Wait(); err != nil { 140 | var cniError cnitypes.Error 141 | if err := json.Unmarshal(cniDataJSON, &cniError); err != nil { 142 | return errors.Wrapf(err, "error unmarshaling CNI error %q", cniDataJSON) 143 | } 144 | return errors.Wrap(&cniError, "error running cnispawn") 145 | } 146 | 147 | return waitMachinesRunning(machineName) 148 | } 149 | 150 | func waitMachinesRunning(machineName string) error { 151 | for retries := 0; retries <= 30; retries++ { 152 | if machinectl.IsRunning(machineName) { 153 | return nil 154 | } 155 | time.Sleep(2 * time.Second) 156 | } 157 | return errors.Errorf("timeout waiting for %q to start", machineName) 158 | } 159 | 160 | func optionsOverlay(prefix, targetDir, lower, upper string) string { 161 | return fmt.Sprintf("%s=+%s:%s:%s:%s", prefix, targetDir, path.Join(lower, targetDir), path.Join(upper, targetDir), targetDir) 162 | } 163 | -------------------------------------------------------------------------------- /pkg/utils/fs/fs.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 Kinvolk GmbH 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package fs 18 | 19 | import ( 20 | "bytes" 21 | "io" 22 | "os" 23 | "path/filepath" 24 | 25 | "github.com/pkg/errors" 26 | ) 27 | 28 | func PathExists(path string) (bool, error) { 29 | _, err := os.Stat(path) 30 | if err == nil { 31 | return true, nil 32 | } 33 | if os.IsNotExist(err) { 34 | return false, nil 35 | } 36 | return true, err 37 | } 38 | 39 | func CreateFileFromReader(path string, reader io.Reader) error { 40 | dir := filepath.Dir(path) 41 | if err := os.MkdirAll(dir, os.FileMode(0755)); err != nil { 42 | return errors.Wrapf(err, "error creating directory %q", dir) 43 | } 44 | f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0755) 45 | if err != nil { 46 | return errors.Wrapf(err, "error creating %q", path) 47 | } 48 | defer f.Close() 49 | if _, err := io.Copy(f, reader); err != nil { 50 | return errors.Wrapf(err, "error writing %q", path) 51 | } 52 | return nil 53 | } 54 | 55 | func CreateFileFromString(path string, content string) error { 56 | buf := bytes.NewBuffer([]byte(content)) 57 | return CreateFileFromReader(path, buf) 58 | } 59 | 60 | func CopyFile(src, dst string) error { 61 | f, err := os.OpenFile(src, os.O_RDONLY, 0755) 62 | if err != nil { 63 | return err 64 | } 65 | defer f.Close() 66 | return CreateFileFromReader(dst, f) 67 | } 68 | -------------------------------------------------------------------------------- /pkg/utils/hash.go: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright 2018 Kinvolk GmbH 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | package utils 18 | 19 | import ( 20 | "crypto/sha1" 21 | "encoding/base64" 22 | "io/ioutil" 23 | "strings" 24 | 25 | "github.com/pkg/errors" 26 | ) 27 | 28 | func VerifySha1(binFilePath, checksumPath string) error { 29 | outB, err := ioutil.ReadFile(binFilePath) 30 | if err != nil { 31 | return errors.Wrapf(err, "error reading file %s", binFilePath) 32 | } 33 | 34 | outC, err := ioutil.ReadFile(checksumPath) 35 | if err != nil { 36 | return errors.Wrapf(err, "error reading file %s", checksumPath) 37 | } 38 | 39 | hasher := sha1.New() 40 | if _, err := hasher.Write(outB); err != nil { 41 | return errors.Wrapf(err, "error reading for hash from %s", binFilePath) 42 | } 43 | 44 | hashSha := base64.URLEncoding.EncodeToString(hasher.Sum(nil)) 45 | 46 | if strings.TrimSpace(hashSha) != strings.TrimSpace(string(outC)) { 47 | return errors.Wrapf(err, "error verifying checksum for file %s", binFilePath) 48 | } 49 | 50 | return nil 51 | } 52 | -------------------------------------------------------------------------------- /pkg/utils/terminal.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 Kinvolk GmbH 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package utils 18 | 19 | import ( 20 | "syscall" 21 | "unsafe" 22 | 23 | "golang.org/x/sys/unix" 24 | ) 25 | 26 | // IsTerminal returns true if the given file descriptor is a terminal. 27 | func IsTerminal(fd uintptr) bool { 28 | var termios syscall.Termios 29 | _, _, err := unix.Syscall(unix.SYS_IOCTL, fd, uintptr(syscall.TCGETS), uintptr(unsafe.Pointer(&termios))) 30 | return err == 0 31 | } 32 | -------------------------------------------------------------------------------- /scripts/vagrant-mod-env.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Run it with env variable $VUSER set to a customized user, other than the 4 | # default user "vagrant". For example on Ubuntu VM on Vagrant: 5 | # 6 | # $ sudo VUSER=ubuntu ./vagrant-mod-env.sh 7 | 8 | set -eo pipefail 9 | 10 | if [ ${EUID} -ne 0 ]; then 11 | echo "This script must be run as root" 12 | exit 1 13 | fi 14 | 15 | if [ "${VUSER}" == "" ]; then 16 | VUSER=vagrant 17 | fi 18 | 19 | set -u 20 | 21 | HOME=/home/${VUSER} 22 | 23 | echo 'Modifying environment' 24 | chmod +x ${HOME}/build.sh 25 | 26 | # setenforce always returns 1 when selinux is disabled. 27 | # we should ignore the error and continue. 28 | /usr/sbin/setenforce 0 || true 29 | 30 | # Run iptables to allow CNI traffic by default. 31 | iptables -C FORWARD -i cni0 -j ACCEPT 2>/dev/null || iptables -I FORWARD 1 -i cni0 -j ACCEPT 32 | 33 | # Note that especially on Debian systems, it's not sufficient to add 34 | # a single iptables rule, because the FORWARD chain's policy is still DROP. 35 | iptables -P FORWARD ACCEPT 36 | sysctl -w net.ipv4.ip_forward=1 37 | 38 | modprobe overlay 39 | modprobe nf_conntrack 40 | 41 | NF_HASHSIZE=/sys/module/nf_conntrack/parameters/hashsize 42 | 43 | [ -f ${NF_HASHSIZE} ] && echo "131072" > ${NF_HASHSIZE} 44 | 45 | # systemd-nspawn containers are not able to resolve DNS, if systemd-resolved 46 | # is running on the host. 47 | # As workaround, we need to disable stub listener of systemd-resolved for now. 48 | # We also need to explicitly set nameserver to an external one, as 49 | # /etc/resolv.conf is a symlink that points to 50 | # /run/systemd/resolve/stub-resolv.conf created by systemd-resolved. 51 | # This is hacky, but it's at least necessary for systemd v234, the default 52 | # version on Ubuntu 17.10. 53 | sudo sed -i -e 's/^#*.*DNSStubListener=.*$/DNSStubListener=no/' /etc/systemd/resolved.conf 54 | sudo sed -i -e 's/nameserver 127.0.0.53/nameserver 8.8.8.8/' /etc/resolv.conf 55 | systemctl is-active systemd-resolved >& /dev/null && sudo systemctl stop systemd-resolved || true 56 | systemctl is-enabled systemd-resolved >& /dev/null && sudo systemctl disable systemd-resolved || true 57 | -------------------------------------------------------------------------------- /scripts/vagrant-setup-env.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -eo pipefail 4 | 5 | echo 'Setting up correct env. variables' 6 | echo "export GOPATH=$GOPATH" >> "$HOME/.bash_profile" 7 | echo "export PATH=$PATH:$GOPATH/bin:/usr/local/go/bin" >> "$HOME/.bash_profile" 8 | echo "export KUBECONFIG=/var/lib/kube-spawn/clusters/default/admin.kubeconfig" >> "$HOME/.bash_profile" 9 | 10 | # shellcheck disable=SC1090 11 | source ~/.bash_profile 12 | 13 | # -u must be set after "source ~/.bash_profile" to avoid errors like 14 | # "PS1: unbound variable" 15 | set -u 16 | 17 | echo 'Writing build.sh' 18 | 19 | if [[ ! -f $HOME/build.sh ]]; then 20 | cat >>"$HOME/build.sh" <<-EOF 21 | #!/bin/bash 22 | set -xeo pipefail 23 | 24 | export PATH=$PATH:/usr/lib/go-1.12/bin 25 | 26 | cd $GOPATH/src/github.com/kinvolk/kube-spawn 27 | 28 | GO111MODULE=off go get -u github.com/containernetworking/plugins/plugins/... 29 | 30 | DOCKERIZED=n make all 31 | 32 | if ! sudo machinectl show-image flatcar; then 33 | sudo machinectl pull-raw --verify=no https://alpha.release.flatcar-linux.net/amd64-usr/current/flatcar_developer_container.bin.bz2 flatcar && rm /var/lib/machines/.raw-https* 34 | fi 35 | 36 | test -d /var/lib/kube-spawn/clusters/default || sudo GOPATH=$GOPATH ./kube-spawn create --cni-plugin-dir=$GOPATH/bin 37 | sudo GOPATH=$GOPATH ./kube-spawn start --cni-plugin-dir=$GOPATH/bin --nodes=2 && (rm /var/lib/machines/.raw-https* || true) 38 | 39 | if [ "\$KUBESPAWN_REDIRECT_TRAFFIC" == "true" ]; then 40 | # Redirect traffic from the VM to kube-apiserver inside container 41 | APISERVER_IP_PORT=\$(grep server /var/lib/kube-spawn/clusters/default/admin.kubeconfig | awk '{print \$2;}' | perl -pe 's/(https|http):\/\///g') 42 | APISERVER_IP=\$(echo \$APISERVER_IP_PORT | perl -pe 's/:\d*$//g') 43 | APISERVER_PORT=\$(echo \$APISERVER_IP_PORT | perl -pe 's/^[\d.]+://g') 44 | echo "0.0.0.0 \$APISERVER_PORT \$APISERVER_IP \$APISERVER_PORT" | sudo tee /etc/rinetd.conf > /dev/null 45 | sudo systemctl enable rinetd 46 | sudo systemctl start rinetd 47 | 48 | # Generate kubeconfig 49 | cd /home/vagrant 50 | VAGRANT_IP=\$(ip addr show eth0 | grep "inet\\b" | awk '{print \$2}' | cut -d/ -f1) 51 | cp /var/lib/kube-spawn/clusters/default/admin.kubeconfig . 52 | perl -pi.back -e "s/\$APISERVER_IP/\$VAGRANT_IP/g;" admin.kubeconfig 53 | perl -pi.back -e "s/certificate-authority-data.*/insecure-skip-tls-verify: true/g;" admin.kubeconfig 54 | fi 55 | EOF 56 | fi 57 | 58 | if [[ ! -f /usr/local/bin/kubectl ]]; then 59 | KUBERNETES_VERSION=$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt) 60 | sudo curl -Lo /usr/local/bin/kubectl https://storage.googleapis.com/kubernetes-release/release/${KUBERNETES_VERSION}/bin/linux/amd64/kubectl 61 | sudo chmod +x /usr/local/bin/kubectl 62 | fi 63 | -------------------------------------------------------------------------------- /vagrant-all.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | 5 | export KUBESPAWN_AUTOBUILD="true" 6 | export KUBESPAWN_DISTRO=${KUBESPAWN_DISTRO:-fedora} 7 | export KUBESPAWN_REDIRECT_TRAFFIC="true" 8 | 9 | KUBESPAWN_PROVIDER=${KUBESPAWN_PROVIDER:-virtualbox} 10 | KUBESPAWN_VAGRANT_EXTRA_FLAGS=${KUBESPAWN_VAGRANT_EXTRA_FLAGS:-} 11 | 12 | vagrant up $KUBESPAWN_DISTRO --provider=$KUBESPAWN_PROVIDER ${KUBESPAWN_VAGRANT_EXTRA_FLAGS} 13 | 14 | ./vagrant-fetch-kubeconfig.sh 15 | 16 | export KUBECONFIG=$(pwd)/kubeconfig 17 | -------------------------------------------------------------------------------- /vagrant-fetch-kubeconfig.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -eo pipefail 4 | 5 | KUBESPAWN_DISTRO=${KUBESPAWN_DISTRO:-fedora} 6 | 7 | # The following command sometimes exists with status 1 when using vagrant-libvirt, 8 | # despite the fact that ssh config was generated successfully. 9 | vagrant ssh-config $KUBESPAWN_DISTRO > $(pwd)/.ssh_config || true 10 | scp -F $(pwd)/.ssh_config $(vagrant status | awk '/running/{print $1;}'):/home/vagrant/kubeconfig . 11 | --------------------------------------------------------------------------------