├── .gitignore ├── LICENSE ├── Setup-Environment.ps1 ├── isolated └── vagrant-k8s-build-latest │ ├── Vagrantfile │ ├── builder.sh │ ├── readme.md │ └── ubuntu.sh ├── readme.md ├── vagrant-k8s-kubeadm-vanilla ├── 00-ubuntu.sh ├── 01-kubernetes.sh ├── 02-master.sh ├── 02-worker.sh ├── Vagrantfile └── readme.md ├── vagrant-k8s-manual ├── 00-ubuntu.sh ├── 00-windows.ps1 ├── 01-builder.sh ├── 01-kubernetes.sh ├── 01-routes.ps1 ├── 02-master.sh ├── 02-worker.sh ├── Vagrantfile └── readme.md ├── vagrant-synced └── readme.md └── vagrant-windows-hyperv ├── Vagrantfile ├── provision.ps1 └── readme.md /.gitignore: -------------------------------------------------------------------------------- 1 | .vagrant 2 | /vagrant-synced/* 3 | !/vagrant-synced/readme.md 4 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Robert Morse 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Setup-Environment.ps1: -------------------------------------------------------------------------------- 1 | If (-NOT ([Security.Principal.WindowsPrincipal] [Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole(` 2 | [Security.Principal.WindowsBuiltInRole] "Administrator")) 3 | { 4 | Write-Warning "You do not have Administrator rights to run this script!`nPlease re-run this script as an Administrator!" 5 | Break 6 | } 7 | if ((Get-WindowsOptionalFeature -Online -FeatureName:Microsoft-Hyper-V).State -ne "Enabled") 8 | { 9 | Write-Warning "You will need to restart after Hyper-V is enabled" 10 | Enable-WindowsOptionalFeature -Online -FeatureName:Microsoft-Hyper-V -All 11 | } 12 | if((Get-VMSwitch -SwitchType External).Count -lt 1) 13 | { 14 | Write-Warning "You need at least one external vSwitch. Please create an External vSwitch." 15 | } 16 | 17 | if((Get-Command choco).Count -ne 1) 18 | { 19 | Write-Warning "Chocolatey (choco) should be installed" 20 | Write-Host "Installing Chocolatey" 21 | iex ((new-object net.webclient).DownloadString('http://bit.ly/psChocInstall')) 22 | 23 | } 24 | else { 25 | if((Get-Command choco).Count -ne 1) 26 | { 27 | Write-Warning "Vagrant should be installed" 28 | Write-Host "Installing applications from Chocolatey" 29 | choco install vagrant -y 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /isolated/vagrant-k8s-build-latest/Vagrantfile: -------------------------------------------------------------------------------- 1 | $VirtualSwitchName = "External" 2 | 3 | LINUX_IMAGE = "kmm/ubuntu-xenial64" 4 | 5 | Vagrant.configure("2") do |config| 6 | config.vm.define "builder" do |subconfig| 7 | subconfig.vm.box = LINUX_IMAGE 8 | subconfig.vm.hostname = "builder" 9 | subconfig.vm.network :public_network, bridge: $VirtualSwitchName 10 | subconfig.vm.synced_folder "../../vagrant-synced/", "/vagrant", type: "smb" 11 | subconfig.vm.provision "shell", path: "ubuntu.sh" 12 | subconfig.vm.provision "shell", path: "builder.sh" 13 | subconfig.vm.provider "hyperv" do |h| 14 | h.enable_virtualization_extensions = true 15 | h.differencing_disk = true 16 | h.cpus = 4 17 | h.memory = 1024 18 | h.maxmemory = 4096 19 | h.vmname = "builder" 20 | end 21 | end 22 | end 23 | -------------------------------------------------------------------------------- /isolated/vagrant-k8s-build-latest/builder.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | ##################################################################################### 4 | #Install necessary components 5 | sudo apt-get update 6 | sudo apt-get install -y curl git build-essential docker.io conntrack 7 | #add vagrant user to docker 8 | sudo usermod -a -G docker vagrant 9 | ##################################################################################### 10 | 11 | 12 | 13 | ##################################################################################### 14 | # Make directories 15 | DIST_DIR="/vagrant/kube-win/" 16 | KUBE_DIR="/vagrant/kube/" 17 | if [ -d ${DIST_DIR} ]; 18 | then (echo "Directory exists: ${DIST_DIR}"); 19 | else (mkdir -p ${DIST_DIR}); 20 | fi 21 | 22 | if [ -d ${KUBE_DIR} ]; 23 | then (echo "Directory exists: ${KUBE_DIR}"); 24 | else (mkdir -p ${KUBE_DIR}); 25 | fi 26 | ##################################################################################### 27 | 28 | 29 | 30 | ##################################################################################### 31 | # Clone k8s guide 32 | SRC_DIR="/vagrant/k8s-guide/" 33 | BRANCH="k8s-guide" 34 | REPO_URL="https://github.com/Microsoft/SDN" 35 | 36 | 37 | if [ -d $SRC_DIR ]; 38 | then ( 39 | cd $SRC_DIR && git checkout $BRANCH && git pull 40 | ); 41 | else ( 42 | git clone --progress $REPO_URL $SRC_DIR && cd $SRC_DIR && git checkout $BRANCH 43 | ); 44 | fi 45 | 46 | cp -rf ${SRC_DIR}/Kubernetes/linux/* /vagrant/kube/ 47 | cp -rf ${SRC_DIR}/Kubernetes/windows/* /vagrant/kube-win/ 48 | ##################################################################################### 49 | 50 | 51 | 52 | ##################################################################################### 53 | # Build Windows Binaries 54 | SRC_DIR="/home/vagrant/src/k8s/" 55 | BRANCH="release-1.9" 56 | REPO_URL="https://github.com/kubernetes/kubernetes.git" 57 | 58 | if [ -d ${DIST_DIR} ]; 59 | then (echo "Directory exists: ${DIST_DIR}"); 60 | else (mkdir -p ${DIST_DIR}); 61 | fi 62 | 63 | if [ -d $SRC_DIR ]; 64 | then ( 65 | cd $SRC_DIR && git checkout $BRANCH && git pull 66 | ); 67 | else ( 68 | git clone --progress $REPO_URL $SRC_DIR && cd $SRC_DIR && git checkout $BRANCH 69 | ); 70 | fi 71 | 72 | cd $SRC_DIR 73 | build/run.sh make WHAT=cmd/kubelet KUBE_BUILD_PLATFORMS=linux/amd64 74 | build/run.sh make WHAT=cmd/kubelet KUBE_BUILD_PLATFORMS=windows/amd64 75 | build/run.sh make WHAT=cmd/kube-proxy KUBE_BUILD_PLATFORMS=windows/amd64 76 | cp _output/dockerized/bin/windows/amd64/kube*.exe ${DIST_DIR} 77 | 78 | ls -lah ${DIST_DIR}kube*.exe 79 | ##################################################################################### 80 | 81 | -------------------------------------------------------------------------------- /isolated/vagrant-k8s-build-latest/readme.md: -------------------------------------------------------------------------------- 1 | # About this box 2 | 3 | - This box pulls and creates resources that other boxes depend on: 4 | - Compiled binaries for Windows 5 | - Scripts for Windows and Linux from SDN repo -------------------------------------------------------------------------------- /isolated/vagrant-k8s-build-latest/ubuntu.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | echo "Starting bootstrap" 3 | 4 | systemctl stop apt-daily.service 5 | systemctl kill --kill-who=all apt-daily.service 6 | 7 | # wait until `apt-get updated` has been killed 8 | while ! (systemctl list-units --all apt-daily.service | fgrep -q dead) 9 | do 10 | echo "apt-daily still running" 11 | sleep 1; 12 | done -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # Kubernetes v1.9 with Windows Workers - Vagrant Multi-Machine Setup # 2 | 3 | ## Goal: ## 4 | This guide will help you stand up a local Kubernetes cluster in Hyper-V VMs through Vagrant. 5 | You should be able to use this guide to have a working cluster with 2 Windows nodes within about 3 hours (as short as 80 minutes given my experience). 6 | 7 | This is based on [GitHub docs from Microsoft](https://github.com/Microsoft/SDN/blob/k8s-guide/Kubernetes/HOWTO-on-prem.md) and you should at least read that before diving in here. 8 | 9 | All directories should have a README to convey its purpose 10 | 11 | ## Current status ## 12 | 13 | - Focused on `./vagrant-k8s-manual/` work 14 | - Stands up a working Windows worker cluster (deployments can be created) 15 | - Leverages Vagrant 2.0 and Hyper-V 16 | - `k-builder` builds windows binaries and places them and other resources in `./vagrant-synced/` for subsequent Vagrant instances to leverage 17 | - `k-m1` preps an instance with control plane running 18 | - Using v1.9.0-beta.2 precompiled/downloaded binaries for master, and branch release-1.9 for building Windows binaries 19 | - Windows worker nodes `k-w-w1` and `k-w-w2` ready to join the cluster 20 | - You can use `kubectl` on your system to connect to the master by copying the config from `./vagrant-synced/kube/config` to `~/.kube/config` 21 | 22 | ## Usage ## 23 | 24 | - Ensure requirements below are met 25 | - Update Vagrantfile at `./vagrant-k8s-manual/Vagrantfile` 26 | - Set the Cluster CIDR you want to use. I chose 10.4.0.0/16 as a unique network that won't collide with other networks. You want to choose a range that overlaps with the external IPs that the nodes will get so they have routing to each other by default 27 | - Open Powershell Administrator prompt 28 | - CD to repo location 29 | - CD to `./vagrant-k8s-manual/` 30 | ``` 31 | # do you need windows binaries? if you already have exe files for kubectl, kubelet, kube-proxy, place them in `../vagrant-synced/kube-win/` 32 | vagrant up k-builder 33 | # enter credentials that can access `../vagrant-synced/` as an SMB share. Administrator rights required. 34 | # wait approximately 30 minutes for Windows binaries to be compiled and placed in `../vagrant-synced/kube-win/` 35 | # you will likely build windows binaries once if at all 36 | vagrant up 37 | # wait approximately 10 minutes for k-m1 to complete 38 | # at this point you can check the dashboard via kubectl proxy below if you're interested 39 | # wait approximately 10 minutes for k-w-w1 to come up, with majority of time for WinRM copy of kube*.exe to node 40 | # wait approximately 10 minutes for k-w-w2 to come up for the same reason 41 | ``` 42 | Login to each `k-w-w1` and `k-w-w2` (via `vagrant rdp k-w-w1` or Hyper-V Manager) and run via Powershell (due to TODO in 00-windows.ps1 provisioning script) 43 | ``` 44 | #NOTE: (Replace with your chosen ClusterCIDR) 45 | $ClusterCIDR="10.4.0.0/16" 46 | 47 | Start-Job -Name kubelet {c:\k\start-kubelet.ps1 -clusterCIDR $ClusterCIDR *> c:\k\kubelet-logs.txt} 48 | Start-Job -Name kubeproxy {c:\k\start-kubeproxy.ps1 *> c:\k\kubeproxy-logs.txt} 49 | ``` 50 | Continue with kubectl to view the cluster info 51 | ``` 52 | cp ../vagrant-synced/kube/config ~/.kube/config 53 | kubectl cluster-info 54 | kubectl proxy 55 | # navigate web browser to http://localhost:8001/api/v1/namespaces/kube-system/services/kubernetes-dashboard/proxy/#!/node?namespace=_all 56 | ``` 57 | 58 | ## Other directories ## 59 | 60 | - Previous work on `./vagrant-k8s-kubeadm-vanilla/` was used for testing v1.8 kubeadm to standup a master and join a Linux worker node 61 | - This creates a functioning master in `vagrant up` 62 | - Manual step to grab the `kubeadm join` command and run from the worker 63 | 64 | ## Requirements ## 65 | 66 | - RAM: 5GB RAM for Master, 2GB RAM for each Windows worker 67 | - Vagrant 2.0 68 | - Hyper-V 69 | - Vagrant box created for Windows Server 1709 with Containers feature and Docker installed per [Vagrant/Packer Box instructions](https://github.com/StefanScherer/packer-windows) or similar 70 | - Box generation takes roughly 1 hr, which includes caching the 1709 Docker images 71 | ``` 72 | packer build --only hyperv-iso -var 'hyperv_switchname=External' -var 'iso_url=c:/images/en_windows_server_version_1709_x64_dvd_100090904.iso' .\windows_server_1709_docker.json 73 | vagrant box add windows_server_1709_docker_hyperv.box --name WindowsServer1709Docker 74 | ``` 75 | - Internet connectivity for connecting to GitHub, and also download Kubernetes bits 76 | 77 | **Note:** `./Setup-Environment.ps1` may or may not help to install the Requirements listed above 78 | 79 | ## Known issues ## 80 | 81 | - Windows nodes do not join automatically as part of `vagrant up`. This is due to some issue with **Start-Job** running in the context of Vagrant and then disconnecting, perhaps. 82 | - Windows nodes do not report CPU or RAM metrics 83 | - Vagrant SMB synced folder to Windows nodes does not work, and the file provisioner to copy files is slow (~5 minutes to copy 250MB) 84 | - SMB synced folder does not accept parameters for some reason. This means typing in the username/password for SMB sync a couple minutes into the standup of each Linux instance 85 | - kube-dns on the master gets into a crash loop and is likely due to a configuration error or could be related to https://github.com/kubernetes/kubernetes/issues/56902 86 | -------------------------------------------------------------------------------- /vagrant-k8s-kubeadm-vanilla/00-ubuntu.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | echo "Starting bootstrap" 3 | 4 | systemctl stop apt-daily.service 5 | systemctl kill --kill-who=all apt-daily.service 6 | 7 | # wait until `apt-get updated` has been killed 8 | while ! (systemctl list-units --all apt-daily.service | fgrep -q dead) 9 | do 10 | echo "apt-daily still running" 11 | sleep 1; 12 | done 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /vagrant-k8s-kubeadm-vanilla/01-kubernetes.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | apt-get update && apt-get install -y curl apt-transport-https 4 | curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - 5 | cat </etc/apt/sources.list.d/docker.list 6 | deb https://download.docker.com/linux/$(lsb_release -si | tr '[:upper:]' '[:lower:]') $(lsb_release -cs) stable 7 | EOF 8 | apt-get update && apt-get install -y docker-ce=$(apt-cache madison docker-ce | grep 17.03 | head -1 | awk '{print $3}') 9 | 10 | 11 | #cat << EOF > /etc/docker/daemon.json 12 | # { 13 | # "exec-opts": ["native.cgroupdriver=systemd"] 14 | # } 15 | #EOF 16 | #sudo systemctl restart docker 17 | 18 | cat /proc/swaps 19 | echo "disabling swap" 20 | awk '/swap/{$0="#"$0} 1' /etc/fstab >/etc/fstab.tmp && mv /etc/fstab.tmp /etc/fstab 21 | sudo swapoff -a 22 | sudo sysctl vm.swappiness=0 23 | cat /proc/swaps 24 | 25 | 26 | apt-get update && apt-get install -y apt-transport-https 27 | curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - 28 | cat </etc/apt/sources.list.d/kubernetes.list 29 | deb http://apt.kubernetes.io/ kubernetes-xenial main 30 | EOF 31 | 32 | apt-get update 33 | apt-get install -y kubelet kubeadm kubectl 34 | 35 | systemctl status kubelet 36 | journalctl -xeu kubelet 37 | 38 | cat /proc/swaps 39 | sudo swapoff -a 40 | cat /proc/swaps 41 | 42 | 43 | # #For after 44 | # mkdir -p $HOME/.kube 45 | # sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config 46 | # sudo chown $(id -u):$(id -g) $HOME/.kube/config 47 | # source <(kubectl completion bash) 48 | # source <(kubeadm completion bash) 49 | # 50 | # 51 | 52 | 53 | # kubeadm join --token f9af47.a217eadf051789fc 10.4.128.254:6443 --discovery-token-ca-cert-hash sha256:463a498dd399781da87b6e08cba694e95ac3fe97fb8c713891b7b9b970a25d1c -------------------------------------------------------------------------------- /vagrant-k8s-kubeadm-vanilla/02-master.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | kubeadm init --pod-network-cidr=10.244.0.0/16 #flannel CIDR 3 | 4 | mkdir -p $HOME/.kube 5 | sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config 6 | sudo chown $(id -u):$(id -g) $HOME/.kube/config 7 | 8 | kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/v0.9.0/Documentation/kube-flannel.yml 9 | kubectl taint nodes --all node-role.kubernetes.io/master 10 | 11 | kubectl get all --all-namespaces -------------------------------------------------------------------------------- /vagrant-k8s-kubeadm-vanilla/02-worker.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | echo "worker command depends on master output..." 3 | echo "use 'kubeadm join' command output from master..." 4 | 5 | #kubeadm join --token 9bac71.89182ffe636488e6 10.4.128.166:6443 --discovery-token-ca-cert-hash sha256:d77b66d50b7da8a93b4696f1b1562ee35fecd4fe521de74ab13be5becb311643 -------------------------------------------------------------------------------- /vagrant-k8s-kubeadm-vanilla/Vagrantfile: -------------------------------------------------------------------------------- 1 | $VirtualSwitchName = "External" 2 | 3 | LINUX_IMAGE = "kmm/ubuntu-xenial64" 4 | 5 | Vagrant.configure("2") do |config| 6 | config.vm.define "k-master" do |subconfig| 7 | subconfig.vm.box = LINUX_IMAGE 8 | subconfig.vm.hostname = "k-master" 9 | subconfig.vm.network :public_network, bridge: $VirtualSwitchName 10 | subconfig.vm.synced_folder ".", "/vagrant", disabled: true 11 | subconfig.vm.provision "shell", path: "00-ubuntu.sh" 12 | subconfig.vm.provision "shell", path: "01-kubernetes.sh" 13 | subconfig.vm.provision "shell", path: "02-master.sh" 14 | subconfig.vm.provider "hyperv" do |h| 15 | h.enable_virtualization_extensions = true 16 | h.differencing_disk = true 17 | h.cpus = 4 18 | h.memory = 2048 19 | h.maxmemory = 4096 20 | h.vmname = "k-master" 21 | end 22 | end 23 | config.vm.define "k-worker-linux" do |subconfig| 24 | subconfig.vm.box = LINUX_IMAGE 25 | subconfig.vm.hostname = "k-worker-linux" 26 | subconfig.vm.network :public_network, bridge: $VirtualSwitchName 27 | subconfig.vm.synced_folder ".", "/vagrant", disabled: true 28 | subconfig.vm.provision "shell", path: "00-ubuntu.sh" 29 | subconfig.vm.provision "shell", path: "01-kubernetes.sh" 30 | subconfig.vm.provision "shell", path: "02-worker.sh" 31 | subconfig.vm.provider "hyperv" do |h| 32 | h.enable_virtualization_extensions = true 33 | h.differencing_disk = true 34 | h.cpus = 4 35 | h.memory = 1024 36 | h.maxmemory = 4096 37 | h.vmname = "k-worker-linux" 38 | end 39 | end 40 | 41 | end 42 | -------------------------------------------------------------------------------- /vagrant-k8s-kubeadm-vanilla/readme.md: -------------------------------------------------------------------------------- 1 | # Usage instructions 2 | 3 | - This will standup a 2 Ubuntu Linux instances: one Master, one Worker 4 | - Pulls k8s binaries that are published to http://apt.kubernetes.io/ 5 | - Master is schedulable, with Flannel 6 | - Worker needs you to SSH and run a `kubeadm join` command -------------------------------------------------------------------------------- /vagrant-k8s-manual/00-ubuntu.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | echo "Starting bootstrap" 3 | 4 | systemctl stop apt-daily.service 5 | systemctl kill --kill-who=all apt-daily.service 6 | 7 | # wait until `apt-get updated` has been killed 8 | while ! (systemctl list-units --all apt-daily.service | fgrep -q dead) 9 | do 10 | echo "apt-daily still running" 11 | sleep 1; 12 | done 13 | 14 | sudo apt-get update 15 | sudo apt-get install -y git conntrack curl 16 | 17 | ##################################################################################### 18 | # Make directories 19 | DIST_DIR="/vagrant/kube-win/" 20 | KUBE_DIR="/vagrant/kube/" 21 | if [ -d ${DIST_DIR} ]; 22 | then (echo "Directory exists: ${DIST_DIR}"); 23 | else (mkdir -p ${DIST_DIR}); 24 | fi 25 | 26 | if [ -d ${KUBE_DIR} ]; 27 | then (echo "Directory exists: ${KUBE_DIR}"); 28 | else (mkdir -p ${KUBE_DIR}); 29 | fi 30 | ##################################################################################### 31 | 32 | ##################################################################################### 33 | # Clone k8s guide 34 | SRC_DIR="/vagrant/k8s-guide/" 35 | BRANCH="prenetworkcleanup " 36 | REPO_URL="https://github.com/rjmorse/SDN" 37 | 38 | 39 | if [ -d $SRC_DIR ]; 40 | then ( 41 | cd $SRC_DIR && git checkout $BRANCH && git pull 42 | ); 43 | else ( 44 | git clone --progress $REPO_URL $SRC_DIR && cd $SRC_DIR && git checkout $BRANCH 45 | ); 46 | fi 47 | 48 | cp -rf ${SRC_DIR}/Kubernetes/linux/* /vagrant/kube/ 49 | cp -rf ${SRC_DIR}/Kubernetes/windows/* /vagrant/kube-win/ 50 | ##################################################################################### 51 | -------------------------------------------------------------------------------- /vagrant-k8s-manual/00-windows.ps1: -------------------------------------------------------------------------------- 1 | Param( 2 | $ClusterCIDR="192.168.0.0/16" 3 | ) 4 | "...Info about this box..." 5 | Get-WmiObject -Class Win32_OperatingSystem | ForEach-Object -MemberName Caption 6 | (Get-ItemProperty -Path c:\windows\system32\hal.dll).VersionInfo.FileVersion 7 | "...End info..." 8 | 9 | cd c:/k/ 10 | ls c:/k/ 11 | 12 | docker system info 13 | docker images 14 | docker pull microsoft/windowsservercore:1709 15 | docker tag microsoft/windowsservercore:1709 microsoft/windowsservercore:latest 16 | docker build -t kubeletwin/pause . 17 | 18 | #This file should have been created by the master, placed in the synced folder, and then copied to c:/k upon Vagrant provisioning 19 | Get-Item c:/k/config 20 | 21 | #Change to userspace until bugfix merged from https://github.com/kubernetes/kubernetes/pull/56529 22 | "Before:" 23 | (Get-Content c:\k\start-kubeproxy.ps1) 24 | #(Get-Content c:\k\start-kubeproxy.ps1).replace('--proxy-mode=kernelspace ', '--proxy-mode=userspace ') | Set-Content c:\k\start-kubeproxy.ps1 25 | "After:" 26 | (Get-Content c:\k\start-kubeproxy.ps1) 27 | 28 | "TODO: Starting kubelet and joining cluster" 29 | "Check logs at c:\k\kubelet-logs.txt or c:\k\kubeproxy-logs.txt on this node" 30 | 31 | #TODO: these create jobs, but do not actually register. Running manually from Powershell works fine 32 | #Start-Job -Name kubelet {c:\k\start-kubelet.ps1 -clusterCIDR $ClusterCIDR *> c:\k\kubelet-logs.txt} 33 | #Start-Job -Name kubeproxy {c:\k\start-kubeproxy.ps1 *> c:\k\kubeproxy-logs.txt} 34 | 35 | -------------------------------------------------------------------------------- /vagrant-k8s-manual/01-builder.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | ##################################################################################### 4 | #Install necessary components 5 | sudo apt-get update 6 | sudo apt-get install -y build-essential docker.io 7 | #add vagrant user to docker 8 | sudo usermod -a -G docker vagrant 9 | ##################################################################################### 10 | 11 | 12 | 13 | ##################################################################################### 14 | # Build Windows Binaries 15 | SRC_DIR="/home/vagrant/src/k8s/" 16 | BRANCH="release-1.9" 17 | REPO_URL="https://github.com/kubernetes/kubernetes.git" 18 | 19 | DIST_DIR="/vagrant/kube-win/" 20 | 21 | if [ -d ${DIST_DIR} ]; 22 | then (echo "Directory exists: ${DIST_DIR}"); 23 | else (mkdir -p ${DIST_DIR}); 24 | fi 25 | 26 | if [ -d $SRC_DIR ]; 27 | then ( 28 | cd $SRC_DIR && git checkout $BRANCH && git pull 29 | ); 30 | else ( 31 | git clone --progress --depth 1 $REPO_URL $SRC_DIR -b $BRANCH && cd $SRC_DIR 32 | ); 33 | fi 34 | 35 | cd $SRC_DIR 36 | build/run.sh make WHAT=cmd/kubelet KUBE_BUILD_PLATFORMS=linux/amd64 37 | build/run.sh make WHAT=cmd/kubelet KUBE_BUILD_PLATFORMS=windows/amd64 38 | build/run.sh make WHAT=cmd/kubectl KUBE_BUILD_PLATFORMS=windows/amd64 39 | build/run.sh make WHAT=cmd/kube-proxy KUBE_BUILD_PLATFORMS=windows/amd64 40 | cp _output/dockerized/bin/windows/amd64/kube*.exe ${DIST_DIR} 41 | 42 | ls -lah ${DIST_DIR}kube*.exe 43 | ##################################################################################### 44 | 45 | -------------------------------------------------------------------------------- /vagrant-k8s-manual/01-kubernetes.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | apt-get update && apt-get install -y curl apt-transport-https 4 | curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - 5 | cat </etc/apt/sources.list.d/docker.list 6 | deb https://download.docker.com/linux/$(lsb_release -si | tr '[:upper:]' '[:lower:]') $(lsb_release -cs) stable 7 | EOF 8 | apt-get update && apt-get install -y docker-ce=$(apt-cache madison docker-ce | grep 17.03 | head -1 | awk '{print $3}') 9 | 10 | cat /proc/swaps 11 | echo "disabling swap" 12 | awk '/swap/{$0="#"$0} 1' /etc/fstab >/etc/fstab.tmp && mv /etc/fstab.tmp /etc/fstab 13 | sudo swapoff -a 14 | sudo sysctl vm.swappiness=0 15 | cat /proc/swaps 16 | 17 | cd $HOME 18 | #install linux binaries per https://github.com/Microsoft/SDN/blob/k8s-guide/Kubernetes/HOWTO-on-prem.md#installing-the-linux-binaries 19 | wget --quiet -O kubernetes.tar.gz https://github.com/kubernetes/kubernetes/releases/download/v1.9.0-beta.2/kubernetes.tar.gz 20 | tar -xzf kubernetes.tar.gz #removed -v for verbose 21 | cd kubernetes/cluster 22 | # follow the prompts from this command: 23 | ./get-kube-binaries.sh 24 | cd ../server 25 | tar -xzf kubernetes-server-linux-amd64.tar.gz #removed -v for verbose 26 | cd kubernetes/server/bin 27 | ls -lah 28 | 29 | # TODO: check that this PATH is not creating a problem 30 | # changed from $HOME to /vagrant. Have to mkdir since the bin isn't there regardless 31 | mkdir -p /vagrant/kube/bin/ 32 | cp hyperkube kubectl /vagrant/kube/bin/ 33 | PATH="/vagrant/kube/bin/:$PATH" 34 | 35 | # #Get CNI plugins per https://github.com/Microsoft/SDN/blob/k8s-guide/Kubernetes/HOWTO-on-prem.md#install-cni-plugins 36 | # Changed from $HOME to /vagrant 37 | DOWNLOAD_DIR="/vagrant/kube/cni-plugins" 38 | CNI_BIN="/opt/cni/bin/" 39 | mkdir ${DOWNLOAD_DIR} 40 | cd $DOWNLOAD_DIR 41 | curl --silent -L $(curl -s https://api.github.com/repos/containernetworking/plugins/releases/latest | grep browser_download_url | grep 'amd64.*tgz' | head -n 1 | cut -d '"' -f 4) -o cni-plugins-amd64.tgz 42 | tar -xzf cni-plugins-amd64.tgz #removed -v for verbose 43 | sudo mkdir -p ${CNI_BIN} 44 | # This command doesn't work 45 | #sudo cp -r !(*.tgz) ${CNI_BIN} 46 | sudo cp -r `ls | egrep -v '^.*tgz$'` ${CNI_BIN} 47 | ls ${CNI_BIN} 48 | 49 | -------------------------------------------------------------------------------- /vagrant-k8s-manual/01-routes.ps1: -------------------------------------------------------------------------------- 1 | Param( 2 | [parameter(Mandatory = $true)] [string] $ClusterCIDR, 3 | [parameter(Mandatory = $true)] [int] $counter 4 | ) 5 | 6 | $mip = gc c:\k\masterip 7 | c:\k\AddRoutes.ps1 -MasterIp $mip -Gateway "$ClusterCIDR.$counter.2" -------------------------------------------------------------------------------- /vagrant-k8s-manual/02-master.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | FULL_CLUSTER=$1 3 | CLUSTER=$2 4 | 5 | cd /vagrant/kube 6 | MASTER_IP=$(ifconfig eth0 | sed -En 's/127.0.0.1//;s/.*inet (addr:)?(([0-9]*\.){3}[0-9]*).*/\2/p') 7 | echo $MASTER_IP | tee /vagrant/kube/masterip /vagrant/kube-win/masterip 8 | 9 | cd /vagrant/kube/certs 10 | ./generate-certs.sh $MASTER_IP 11 | 12 | cd /vagrant/kube/manifest 13 | ./generate.py $MASTER_IP --cluster-cidr $FULL_CLUSTER 14 | rm ./generate.py 15 | 16 | cp -R /vagrant/* /root 17 | 18 | cd /vagrant/kube 19 | ./configure-kubectl.sh $MASTER_IP 20 | 21 | cd /vagrant/kube 22 | sudo cp ~/.kube/config /vagrant/kube/config 23 | sudo cp ~/.kube/config /vagrant/kube-win/config 24 | 25 | mkdir -p /root/kube/kubelet/ 26 | sudo cp ~/.kube/config /root/kube/kubelet/config #critical this is here 27 | 28 | cd /vagrant/kube 29 | ls -lah 30 | echo "now running:" 31 | echo "cd /vagrant/kube" 32 | echo "./start-kubelet.sh $CLUSTER &> /vagrant/master-kubelet-logs.txt &disown" 33 | echo "./start-kubeproxy.sh $CLUSTER &> /vagrant/master-kubeproxy-logs.txt &disown" 34 | echo "./generate-routes.sh $CLUSTER &> /vagrant/master-routes.txt &disown" 35 | 36 | cd /vagrant/kube 37 | ./start-kubelet.sh $CLUSTER &> /vagrant/master-kubelet-logs.txt &disown 38 | ./start-kubeproxy.sh $CLUSTER &> /vagrant/master-kubeproxy-logs.txt &disown 39 | ./generate-routes.sh $CLUSTER &> /vagrant/master-routes.txt &disown 40 | 41 | docker ps 42 | docker images 43 | echo "waiting 10s before next update" 44 | sleep 10 45 | docker ps 46 | docker images 47 | tail -n 10 /vagrant/master-kubelet-logs.txt 48 | 49 | echo "done. continuing..." -------------------------------------------------------------------------------- /vagrant-k8s-manual/02-worker.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | -------------------------------------------------------------------------------- /vagrant-k8s-manual/Vagrantfile: -------------------------------------------------------------------------------- 1 | $VirtualSwitchName = "External" 2 | 3 | LINUX_IMAGE = "generic/ubuntu1604" 4 | 5 | IMAGE = "WindowsServer1709Docker" 6 | ClusterCIDR = "10.4.0.0/16" 7 | ClusterCIDRShort = "10.4" 8 | 9 | Vagrant.configure("2") do |config| 10 | config.vm.define "k-builder", autostart: false do |subconfig| 11 | subconfig.vm.box = LINUX_IMAGE 12 | subconfig.vm.hostname = "k-builder" 13 | subconfig.vm.network :public_network, bridge: $VirtualSwitchName 14 | subconfig.vm.synced_folder "../vagrant-synced/", "/vagrant", type: "smb" 15 | subconfig.vm.provision "shell", path: "00-ubuntu.sh" 16 | subconfig.vm.provision "shell", path: "01-builder.sh" 17 | subconfig.vm.provider "hyperv" do |h| 18 | h.enable_virtualization_extensions = true 19 | h.differencing_disk = true 20 | h.cpus = 4 21 | h.memory = 1024 22 | h.maxmemory = 4096 23 | h.vmname = "k-builder" 24 | end 25 | end 26 | config.vm.define "k-m1" do |subconfig| 27 | subconfig.vm.box = LINUX_IMAGE 28 | subconfig.vm.hostname = "k-m1" 29 | subconfig.vm.network :public_network, bridge: $VirtualSwitchName 30 | subconfig.vm.synced_folder "../vagrant-synced/", "/vagrant", type: "smb" 31 | subconfig.vm.provision "shell", path: "00-ubuntu.sh" 32 | subconfig.vm.provision "shell", path: "01-kubernetes.sh" 33 | subconfig.vm.provision "shell", path: "02-master.sh", :args => [ClusterCIDR, ClusterCIDRShort] 34 | subconfig.vm.provider "hyperv" do |h| 35 | h.enable_virtualization_extensions = true 36 | h.differencing_disk = true 37 | h.cpus = 4 38 | h.memory = 5000 39 | h.maxmemory = 5000 40 | h.vmname = "k-m1" 41 | end 42 | end 43 | # Not currently using Linux worker, and untested 44 | config.vm.define "k-l-w1", autostart: false do |subconfig| 45 | subconfig.vm.box = LINUX_IMAGE 46 | subconfig.vm.hostname = "k-l-w1" 47 | subconfig.vm.network :public_network, bridge: $VirtualSwitchName 48 | subconfig.vm.synced_folder "../vagrant-synced/kube", "/vagrant", type: "smb" 49 | subconfig.vm.provision "shell", path: "00-ubuntu.sh" 50 | subconfig.vm.provision "shell", path: "01-kubernetes.sh" 51 | subconfig.vm.provision "shell", path: "02-worker.sh" 52 | subconfig.vm.provider "hyperv" do |h| 53 | h.enable_virtualization_extensions = true 54 | h.differencing_disk = true 55 | h.cpus = 4 56 | h.memory = 1024 57 | h.maxmemory = 4096 58 | h.vmname = "k-l-w1" 59 | end 60 | end 61 | # TODO: add routes to Linux pods from Windows; depends on HNS config currently in **start-kubelet.ps1** 62 | # Error output below if **start-kubelet.ps1** has not be ran successfully 63 | # ==> k-w-w1: Running provisioner: shell... 64 | # k-w-w1: Running: ./01-routes.ps1 as c:\tmp\vagrant-shell.ps1 65 | # k-w-w1: Add-RouteToPodCIDR - available nodes linux 10.4.0.0/24 66 | # k-w-w1: Adding route for Remote Pod CIDR 10.4.0.0/24, GW 10.4.0.1, for node type linux 67 | # k-w-w1: new-netroute : Invalid parameter InterfaceAlias vEthernet (cbr0) 68 | # k-w-w1: At C:\k\AddRoutes.ps1:27 char:13 69 | # k-w-w1: + new-netroute -InterfaceAlias "$nicName" -DestinationPrefi ... 70 | # k-w-w1: + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 71 | # k-w-w1: + CategoryInfo : InvalidArgument: (MSFT_NetRoute:ROOT/StandardCimv2/MSFT_NetRoute) [New-NetRoute], CimExc 72 | # k-w-w1: eption 73 | # k-w-w1: + FullyQualifiedErrorId : Windows System Error 87,New-NetRoute 74 | # k-w-w1: C:\k\AddRoutes.ps1 : Do you have a virtual adapter configured? Couldn't find one! 75 | # k-w-w1: At C:\tmp\vagrant-shell.ps1:7 char:1 76 | # k-w-w1: + c:\k\AddRoutes.ps1 -MasterIp $mip -Gateway "$ClusterCIDR.$counter.2" 77 | # k-w-w1: + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 78 | # k-w-w1: + CategoryInfo : NotSpecified: (:) [Write-Error], WriteErrorException 79 | # k-w-w1: + FullyQualifiedErrorId : Microsoft.PowerShell.Commands.WriteErrorException,AddRoutes.ps1 80 | (1..2).each do |i| 81 | config.vm.define "k-w-w#{i}" do |subconfig| 82 | subconfig.vm.box = IMAGE 83 | subconfig.vm.communicator = "winrm" 84 | subconfig.vm.hostname = "k-w-w#{i}" 85 | subconfig.vm.network :public_network, bridge: $VirtualSwitchName 86 | subconfig.vm.synced_folder ".", "/vagrant", disabled: true 87 | #subconfig.vm.synced_folder "../vagrant-synced/kube-win", "c:/k/", type: "smb" #experienced issues with permissions, perhaps user error though 88 | subconfig.vm.provision "file", source: "../vagrant-synced/kube-win", destination: "c:/k" 89 | subconfig.vm.provision "shell", path: "./00-windows.ps1", :args => ["-ClusterCIDR", ClusterCIDR], privileged: true 90 | #subconfig.vm.provision "shell", path: "./01-routes.ps1", :args => ["-ClusterCIDR", ClusterCIDRShort, "-counter", "#{i}"], privileged: true 91 | subconfig.vm.provider "hyperv" do |h| 92 | h.enable_virtualization_extensions = true 93 | h.differencing_disk = true 94 | h.cpus = 4 95 | h.memory = 2048 96 | h.maxmemory = 2048 97 | h.vmname = "k-w-w#{i}" 98 | end 99 | end 100 | end 101 | end 102 | -------------------------------------------------------------------------------- /vagrant-k8s-manual/readme.md: -------------------------------------------------------------------------------- 1 | # Usage instructions 2 | 3 | - See instructions at root of repo -------------------------------------------------------------------------------- /vagrant-synced/readme.md: -------------------------------------------------------------------------------- 1 | # About this folder 2 | 3 | The contents here are dynamically generated and used by vagrant boxes. Generally it includes: 4 | 5 | - `./kube/` - Linux related binaries and scripts 6 | - `./kube-win/` - Windows related binaries and scripts -------------------------------------------------------------------------------- /vagrant-windows-hyperv/Vagrantfile: -------------------------------------------------------------------------------- 1 | $VirtualSwitchName = "External" 2 | 3 | IMAGE = "WindowsServer1709-Containers" 4 | 5 | Vagrant.configure("2") do |config| 6 | config.vm.define "k-worker1" do |subconfig| 7 | subconfig.vm.box = IMAGE 8 | subconfig.vm.communicator = "winrm" 9 | subconfig.vm.hostname = "k-worker1" 10 | subconfig.vm.network :public_network, bridge: $VirtualSwitchName 11 | #subconfig.vm.synced_folder "../vagrant-synced/kube-win", "c:/k/", type: "smb" 12 | subconfig.vm.provision "file", source: "../vagrant-synced/kube-win/*", destination: "c:/k/" 13 | subconfig.vm.provision "shell", path: "./provision.ps1", privileged: true 14 | subconfig.vm.provider "hyperv" do |h| 15 | h.enable_virtualization_extensions = true 16 | h.differencing_disk = true 17 | h.cpus = 4 18 | h.memory = 1024 19 | h.maxmemory = 4096 20 | h.vmname = "k-worker1" 21 | end 22 | end 23 | end 24 | -------------------------------------------------------------------------------- /vagrant-windows-hyperv/provision.ps1: -------------------------------------------------------------------------------- 1 | docker images -------------------------------------------------------------------------------- /vagrant-windows-hyperv/readme.md: -------------------------------------------------------------------------------- 1 | ## About 2 | 3 | - This is simply a test of the box for `WindowsServer1709-Containers` and mounting resource directory via SMB 4 | - This is currently working fine --------------------------------------------------------------------------------