├── config ├── .gitignore ├── encryption-config.yaml ├── worker-0-crio.service ├── worker-1-crio.service ├── worker-2-crio.service ├── controller-0-kube-scheduler.service ├── controller-1-kube-scheduler.service ├── controller-2-kube-scheduler.service ├── worker-0-10-bridge.conf ├── worker-1-10-bridge.conf ├── worker-2-10-bridge.conf ├── kube-proxy.service ├── worker-0-kubelet-config ├── worker-1-kubelet-config ├── worker-2-kubelet-config ├── worker-0-kubelet.service ├── worker-1-kubelet.service ├── worker-2-kubelet.service ├── controller-0-kube-controller-manager.service ├── controller-1-kube-controller-manager.service ├── controller-2-kube-controller-manager.service ├── controller-0-etcd.service ├── controller-1-etcd.service ├── controller-2-etcd.service ├── controller-0-kube-apiserver.service ├── controller-1-kube-apiserver.service ├── controller-2-kube-apiserver.service └── registries.conf ├── tools └── .gitignore ├── certificates ├── .gitignore ├── ca-csr.json ├── ca-config.json ├── admin-csr.json ├── kubernetes-csr.json ├── worker-0-csr.json ├── worker-1-csr.json ├── worker-2-csr.json ├── kube-proxy-csr.json ├── kube-controller-manager-csr.json └── kube-scheduler-csr.json ├── .gitignore ├── scripts ├── generate-ca ├── distclean ├── generate-admin-cert ├── generate-kube-proxy-cert ├── generate-certs ├── generate-kube-schduler-cert ├── versions.bash ├── generate-service-files ├── generate-kube-controller-manager-cert ├── generate-worker-certs ├── generate-kubernetes-cert ├── vagrant-setup-hosts-file.bash ├── generate-cni-config ├── generate-scheduler-service-file ├── configure-kubectl-on-host ├── vagrant-setup-routes.bash ├── generate-kubelet-config-file ├── setup-etcd ├── setup ├── generate-kubeconfig-kube-proxy ├── generate-kubeconfig-scheduler ├── generate-kubelet-service-file ├── generate-kubeconfig-worker ├── generate-kubeconfig-controller-manager ├── generate-controller-manager-service-file ├── setup-kubelet-api-cluster-role ├── generate-etcd-service-files ├── install-tools ├── setup-worker-services ├── download-tools ├── setup-controller-services ├── vagrant-setup-haproxy.bash ├── generate-apiserver-service-file └── setup-traefik ├── manifests ├── nginx-ingress.yaml ├── nginx.yaml └── coredns.yaml ├── Vagrantfile ├── README.md └── LICENSE.txt /config/.gitignore: -------------------------------------------------------------------------------- 1 | *.kubeconfig 2 | -------------------------------------------------------------------------------- /tools/.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | !.gitignore 3 | -------------------------------------------------------------------------------- /certificates/.gitignore: -------------------------------------------------------------------------------- 1 | *.csr 2 | *.pem 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | .*.swp 3 | *console.log 4 | .vagrant/ 5 | -------------------------------------------------------------------------------- /scripts/generate-ca: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | 5 | readonly dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 6 | 7 | pushd "${dir}/../certificates" 8 | trap 'popd' EXIT 9 | 10 | cfssl gencert -initca ca-csr.json | cfssljson -bare ca 11 | -------------------------------------------------------------------------------- /certificates/ca-csr.json: -------------------------------------------------------------------------------- 1 | { 2 | "CN": "Kubernetes", 3 | "key": { 4 | "algo": "rsa", 5 | "size": 2048 6 | }, 7 | "names": [ 8 | { 9 | "C": "US", 10 | "L": "Portland", 11 | "O": "Kubernetes", 12 | "OU": "CA", 13 | "ST": "Oregon" 14 | } 15 | ] 16 | } 17 | -------------------------------------------------------------------------------- /scripts/distclean: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | 5 | readonly dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 6 | 7 | pushd "${dir}/../" 8 | trap 'popd' EXIT 9 | 10 | rm -vfr tools/* 11 | 12 | rm -vf certificates/*pem 13 | rm -vf certificates/*csr 14 | 15 | rm -vf config/*kubeconfig 16 | -------------------------------------------------------------------------------- /certificates/ca-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "signing": { 3 | "default": { 4 | "expiry": "8760h" 5 | }, 6 | "profiles": { 7 | "kubernetes": { 8 | "usages": ["signing", "key encipherment", "server auth", "client auth"], 9 | "expiry": "8760h" 10 | } 11 | } 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /config/encryption-config.yaml: -------------------------------------------------------------------------------- 1 | kind: EncryptionConfig 2 | apiVersion: v1 3 | resources: 4 | - resources: 5 | - secrets 6 | providers: 7 | - aescbc: 8 | keys: 9 | - name: key1 10 | secret: ZM8yphFEHXpsg9PHPwMSTQayRZi6EIS+FOeWSuLBQ28= 11 | - identity: {} 12 | -------------------------------------------------------------------------------- /certificates/admin-csr.json: -------------------------------------------------------------------------------- 1 | { 2 | "CN": "admin", 3 | "key": { 4 | "algo": "rsa", 5 | "size": 2048 6 | }, 7 | "names": [ 8 | { 9 | "C": "US", 10 | "L": "Portland", 11 | "O": "system:masters", 12 | "OU": "Kubernetes The Hard Way", 13 | "ST": "Oregon" 14 | } 15 | ] 16 | } 17 | -------------------------------------------------------------------------------- /certificates/kubernetes-csr.json: -------------------------------------------------------------------------------- 1 | { 2 | "CN": "kubernetes", 3 | "key": { 4 | "algo": "rsa", 5 | "size": 2048 6 | }, 7 | "names": [ 8 | { 9 | "C": "US", 10 | "L": "Portland", 11 | "O": "Kubernetes", 12 | "OU": "Kubernetes The Hard Way", 13 | "ST": "Oregon" 14 | } 15 | ] 16 | } 17 | -------------------------------------------------------------------------------- /certificates/worker-0-csr.json: -------------------------------------------------------------------------------- 1 | { 2 | "CN": "system:node:worker-0", 3 | "key": { 4 | "algo": "rsa", 5 | "size": 2048 6 | }, 7 | "names": [ 8 | { 9 | "C": "US", 10 | "L": "Portland", 11 | "O": "system:nodes", 12 | "OU": "Kubernetes The Hard Way", 13 | "ST": "Oregon" 14 | } 15 | ] 16 | } 17 | -------------------------------------------------------------------------------- /certificates/worker-1-csr.json: -------------------------------------------------------------------------------- 1 | { 2 | "CN": "system:node:worker-1", 3 | "key": { 4 | "algo": "rsa", 5 | "size": 2048 6 | }, 7 | "names": [ 8 | { 9 | "C": "US", 10 | "L": "Portland", 11 | "O": "system:nodes", 12 | "OU": "Kubernetes The Hard Way", 13 | "ST": "Oregon" 14 | } 15 | ] 16 | } 17 | -------------------------------------------------------------------------------- /certificates/worker-2-csr.json: -------------------------------------------------------------------------------- 1 | { 2 | "CN": "system:node:worker-2", 3 | "key": { 4 | "algo": "rsa", 5 | "size": 2048 6 | }, 7 | "names": [ 8 | { 9 | "C": "US", 10 | "L": "Portland", 11 | "O": "system:nodes", 12 | "OU": "Kubernetes The Hard Way", 13 | "ST": "Oregon" 14 | } 15 | ] 16 | } 17 | -------------------------------------------------------------------------------- /certificates/kube-proxy-csr.json: -------------------------------------------------------------------------------- 1 | { 2 | "CN": "system:kube-proxy", 3 | "key": { 4 | "algo": "rsa", 5 | "size": 2048 6 | }, 7 | "names": [ 8 | { 9 | "C": "US", 10 | "L": "Portland", 11 | "O": "system:node-proxier", 12 | "OU": "Kubernetes The Hard Way", 13 | "ST": "Oregon" 14 | } 15 | ] 16 | } 17 | -------------------------------------------------------------------------------- /config/worker-0-crio.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=CRI-O daemon 3 | Documentation=https://github.com/kubernetes-incubator/cri-o 4 | 5 | [Service] 6 | ExecStart=/usr/local/bin/crio --stream-address 192.168.199.20 --runtime /usr/local/bin/runc --registry docker.io 7 | Restart=always 8 | RestartSec=10s 9 | 10 | [Install] 11 | WantedBy=multi-user.target 12 | -------------------------------------------------------------------------------- /config/worker-1-crio.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=CRI-O daemon 3 | Documentation=https://github.com/kubernetes-incubator/cri-o 4 | 5 | [Service] 6 | ExecStart=/usr/local/bin/crio --stream-address 192.168.199.21 --runtime /usr/local/bin/runc --registry docker.io 7 | Restart=always 8 | RestartSec=10s 9 | 10 | [Install] 11 | WantedBy=multi-user.target 12 | -------------------------------------------------------------------------------- /config/worker-2-crio.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=CRI-O daemon 3 | Documentation=https://github.com/kubernetes-incubator/cri-o 4 | 5 | [Service] 6 | ExecStart=/usr/local/bin/crio --stream-address 192.168.199.22 --runtime /usr/local/bin/runc --registry docker.io 7 | Restart=always 8 | RestartSec=10s 9 | 10 | [Install] 11 | WantedBy=multi-user.target 12 | -------------------------------------------------------------------------------- /manifests/nginx-ingress.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: Ingress 3 | metadata: 4 | name: nginx 5 | namespace: default 6 | annotations: 7 | kubernetes.io/ingress.class: traefik 8 | spec: 9 | rules: 10 | - host: nginx.kthw 11 | http: 12 | paths: 13 | - backend: 14 | serviceName: nginx 15 | servicePort: 8080 16 | -------------------------------------------------------------------------------- /scripts/generate-admin-cert: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | 5 | readonly dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 6 | 7 | pushd "${dir}/../certificates" 8 | trap 'popd' EXIT 9 | 10 | cfssl gencert \ 11 | -ca=ca.pem \ 12 | -ca-key=ca-key.pem \ 13 | -config=ca-config.json \ 14 | -profile=kubernetes \ 15 | admin-csr.json | cfssljson -bare admin 16 | -------------------------------------------------------------------------------- /certificates/kube-controller-manager-csr.json: -------------------------------------------------------------------------------- 1 | { 2 | "CN": "system:kube-controller-manager", 3 | "key": { 4 | "algo": "rsa", 5 | "size": 2048 6 | }, 7 | "names": [ 8 | { 9 | "C": "US", 10 | "L": "Portland", 11 | "O": "system:kube-controller-manager", 12 | "OU": "Kubernetes The Hard Way", 13 | "ST": "Oregon" 14 | } 15 | ] 16 | } 17 | -------------------------------------------------------------------------------- /certificates/kube-scheduler-csr.json: -------------------------------------------------------------------------------- 1 | { 2 | "CN": "system:kube-scheduler", 3 | "key": { 4 | "algo": "rsa", 5 | "size": 2048 6 | }, 7 | "names": [ 8 | { 9 | "C": "US", 10 | "L": "Portland", 11 | "O": "system:kube-scheduler", 12 | "OU": "Kubernetes The Hard Way", 13 | "ST": "Oregon" 14 | } 15 | ] 16 | } 17 | -------------------------------------------------------------------------------- /config/controller-0-kube-scheduler.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Kubernetes Scheduler 3 | Documentation=https://github.com/GoogleCloudPlatform/kubernetes 4 | 5 | [Service] 6 | ExecStart=/usr/local/bin/kube-scheduler \ 7 | --leader-elect=true \ 8 | --master=http://127.0.0.1:8080 \ 9 | --v=2 10 | Restart=on-failure 11 | RestartSec=5 12 | 13 | [Install] 14 | WantedBy=multi-user.target 15 | -------------------------------------------------------------------------------- /config/controller-1-kube-scheduler.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Kubernetes Scheduler 3 | Documentation=https://github.com/GoogleCloudPlatform/kubernetes 4 | 5 | [Service] 6 | ExecStart=/usr/local/bin/kube-scheduler \ 7 | --leader-elect=true \ 8 | --master=http://127.0.0.1:8080 \ 9 | --v=2 10 | Restart=on-failure 11 | RestartSec=5 12 | 13 | [Install] 14 | WantedBy=multi-user.target 15 | -------------------------------------------------------------------------------- /config/controller-2-kube-scheduler.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Kubernetes Scheduler 3 | Documentation=https://github.com/GoogleCloudPlatform/kubernetes 4 | 5 | [Service] 6 | ExecStart=/usr/local/bin/kube-scheduler \ 7 | --leader-elect=true \ 8 | --master=http://127.0.0.1:8080 \ 9 | --v=2 10 | Restart=on-failure 11 | RestartSec=5 12 | 13 | [Install] 14 | WantedBy=multi-user.target 15 | -------------------------------------------------------------------------------- /scripts/generate-kube-proxy-cert: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | 5 | readonly dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 6 | 7 | pushd "${dir}/../certificates" 8 | trap 'popd' EXIT 9 | 10 | cfssl gencert \ 11 | -ca=ca.pem \ 12 | -ca-key=ca-key.pem \ 13 | -config=ca-config.json \ 14 | -profile=kubernetes \ 15 | kube-proxy-csr.json | cfssljson -bare kube-proxy 16 | -------------------------------------------------------------------------------- /config/worker-0-10-bridge.conf: -------------------------------------------------------------------------------- 1 | { 2 | "cniVersion": "0.3.1", 3 | "name": "crio-bridge", 4 | "type": "bridge", 5 | "bridge": "cnio0", 6 | "isGateway": true, 7 | "ipMasq": true, 8 | "ipam": { 9 | "type": "host-local", 10 | "ranges": [ 11 | [{"subnet": "10.20.0.0/16"}] 12 | ], 13 | "routes": [{"dst": "0.0.0.0/0"}] 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /config/worker-1-10-bridge.conf: -------------------------------------------------------------------------------- 1 | { 2 | "cniVersion": "0.3.1", 3 | "name": "crio-bridge", 4 | "type": "bridge", 5 | "bridge": "cnio0", 6 | "isGateway": true, 7 | "ipMasq": true, 8 | "ipam": { 9 | "type": "host-local", 10 | "ranges": [ 11 | [{"subnet": "10.21.0.0/16"}] 12 | ], 13 | "routes": [{"dst": "0.0.0.0/0"}] 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /config/worker-2-10-bridge.conf: -------------------------------------------------------------------------------- 1 | { 2 | "cniVersion": "0.3.1", 3 | "name": "crio-bridge", 4 | "type": "bridge", 5 | "bridge": "cnio0", 6 | "isGateway": true, 7 | "ipMasq": true, 8 | "ipam": { 9 | "type": "host-local", 10 | "ranges": [ 11 | [{"subnet": "10.22.0.0/16"}] 12 | ], 13 | "routes": [{"dst": "0.0.0.0/0"}] 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /scripts/generate-certs: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | 5 | readonly dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 6 | 7 | pushd "${dir}" 8 | trap 'popd' EXIT 9 | 10 | ./generate-ca 11 | ./generate-admin-cert 12 | ./generate-worker-certs 13 | ./generate-kube-proxy-cert 14 | ./generate-kubernetes-cert 15 | ./generate-kube-controller-manager-cert 16 | ./generate-kube-schduler-cert 17 | -------------------------------------------------------------------------------- /scripts/generate-kube-schduler-cert: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | 5 | readonly dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 6 | 7 | pushd "${dir}/../certificates" 8 | trap 'popd' EXIT 9 | 10 | cfssl gencert \ 11 | -ca=ca.pem \ 12 | -ca-key=ca-key.pem \ 13 | -config=ca-config.json \ 14 | -profile=kubernetes \ 15 | kube-scheduler-csr.json | cfssljson -bare kube-scheduler 16 | -------------------------------------------------------------------------------- /scripts/versions.bash: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # https://storage.googleapis.com/kubernetes-release/release/stable.txt 4 | readonly k8s_version="v1.20.2" 5 | readonly etcd_version="v3.4.14" 6 | readonly cfssl_version="1.5.0" 7 | readonly traefik_version="v2.4.0" 8 | readonly crio_version="v1.19.1" 9 | readonly conmon_version="v2.0.25" 10 | readonly cni_plugins_version="v0.9.0" 11 | readonly runc_version="v1.0.0-rc92" 12 | -------------------------------------------------------------------------------- /config/kube-proxy.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Kubernetes Kube Proxy 3 | Documentation=https://github.com/GoogleCloudPlatform/kubernetes 4 | 5 | [Service] 6 | ExecStart=/usr/local/bin/kube-proxy \ 7 | --cluster-cidr=10.200.0.0/16 \ 8 | --kubeconfig=/var/lib/kube-proxy/kubeconfig \ 9 | --proxy-mode=iptables \ 10 | --v=2 11 | Restart=on-failure 12 | RestartSec=5 13 | 14 | [Install] 15 | WantedBy=multi-user.target 16 | -------------------------------------------------------------------------------- /scripts/generate-service-files: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | 5 | readonly dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 6 | 7 | pushd "${dir}" 8 | trap 'popd' EXIT 9 | 10 | ./generate-apiserver-service-file 11 | ./generate-controller-manager-service-file 12 | ./generate-etcd-service-files 13 | 14 | ./generate-kubelet-config-file 15 | ./generate-kubelet-service-file 16 | ./generate-scheduler-service-file 17 | -------------------------------------------------------------------------------- /scripts/generate-kube-controller-manager-cert: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | 5 | readonly dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 6 | 7 | pushd "${dir}/../certificates" 8 | trap 'popd' EXIT 9 | 10 | cfssl gencert \ 11 | -ca=ca.pem \ 12 | -ca-key=ca-key.pem \ 13 | -config=ca-config.json \ 14 | -profile=kubernetes \ 15 | kube-controller-manager-csr.json | cfssljson -bare kube-controller-manager 16 | -------------------------------------------------------------------------------- /scripts/generate-worker-certs: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | 5 | readonly dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 6 | 7 | pushd "${dir}/../certificates" 8 | trap 'popd' EXIT 9 | 10 | for i in {0..2}; do 11 | cfssl gencert \ 12 | -ca=ca.pem \ 13 | -ca-key=ca-key.pem \ 14 | -config=ca-config.json \ 15 | -hostname="worker-${i},192.168.199.2$((i))" \ 16 | -profile=kubernetes \ 17 | "worker-${i}-csr.json" | cfssljson -bare "worker-${i}" 18 | done 19 | -------------------------------------------------------------------------------- /scripts/generate-kubernetes-cert: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | 5 | readonly dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 6 | 7 | pushd "${dir}/../certificates" 8 | trap 'popd' EXIT 9 | 10 | cfssl gencert \ 11 | -ca=ca.pem \ 12 | -ca-key=ca-key.pem \ 13 | -config=ca-config.json \ 14 | -hostname=10.32.0.1,192.168.199.40,192.168.199.10,192.168.199.11,192.168.199.12,127.0.0.1,kubernetes.default \ 15 | -profile=kubernetes \ 16 | kubernetes-csr.json | cfssljson -bare kubernetes 17 | -------------------------------------------------------------------------------- /scripts/vagrant-setup-hosts-file.bash: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | 5 | cat <"${dir}/../config/worker-${i}-10-bridge.conf" <"${dir}/../config/controller-${i}-kube-scheduler.service" <"${dir}/../config/worker-${i}-kubelet-config" <"${dir}/../config/worker-${i}-kubelet.service" <"${dir}/../config/controller-${i}-kube-controller-manager.service" <"${dir}/../config/controller-${i}-etcd.service" < "scripts/vagrant-setup-haproxy.bash" 18 | 19 | c.vm.provider "virtualbox" do |vb| 20 | end 21 | end 22 | 23 | (0..2).each do |n| 24 | config.vm.define "controller-#{n}" do |c| 25 | c.vm.hostname = "controller-#{n}" 26 | c.vm.network "private_network", ip: "192.168.199.1#{n}" 27 | 28 | c.vm.provision :shell, :path => "scripts/vagrant-setup-hosts-file.bash" 29 | 30 | c.vm.provider "virtualbox" do |vb| 31 | vb.memory = "750" 32 | end 33 | end 34 | end 35 | 36 | (0..2).each do |n| 37 | config.vm.define "worker-#{n}" do |c| 38 | c.vm.hostname = "worker-#{n}" 39 | c.vm.network "private_network", ip: "192.168.199.2#{n}" 40 | 41 | c.vm.provision :shell, :path => "scripts/vagrant-setup-routes.bash" 42 | c.vm.provision :shell, :path => "scripts/vagrant-setup-hosts-file.bash" 43 | end 44 | end 45 | 46 | config.vm.define "traefik-0", autostart: false do |c| 47 | c.vm.hostname = "traefik-0" 48 | c.vm.network "private_network", ip: "192.168.199.30" 49 | 50 | c.vm.provision :shell, :path => "scripts/vagrant-setup-routes.bash" 51 | end 52 | end 53 | -------------------------------------------------------------------------------- /scripts/setup-worker-services: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | 5 | readonly dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 6 | 7 | pushd "${dir}/../" 8 | trap 'popd' EXIT 9 | 10 | for i in {0..2}; do 11 | cat <<'EOF' | vagrant ssh "worker-${i}" -- sudo bash 12 | set -euo pipefail 13 | 14 | apt-get update 15 | apt-get install -y socat libgpgme11 make conntrack 16 | 17 | mkdir -p \ 18 | /etc/containers \ 19 | /etc/cni/net.d \ 20 | /var/lib/kubelet \ 21 | /var/lib/kube-proxy \ 22 | /var/lib/kubernetes \ 23 | 24 | make -C /vagrant/tools/crio all 25 | 26 | cp /vagrant/tools/{kube-proxy,kubelet,kubectl} /usr/local/bin/ 27 | 28 | # Override the CNI shipped with CRIO. 29 | cp "/vagrant/config/$(hostname)-10-bridge.conf" /etc/cni/net.d/10-bridge.conf 30 | 31 | # Create a registries configuration, so that container images can be pulled successfully. 32 | cp /vagrant/config/registries.conf /etc/containers/ 33 | cp /vagrant/config/kube-proxy.kubeconfig /var/lib/kube-proxy/kubeconfig 34 | 35 | cp /vagrant/certificates/ca.pem /var/lib/kubernetes/ 36 | cp "/vagrant/certificates/$(hostname).pem" "/vagrant/certificates/$(hostname)-key.pem" /var/lib/kubelet 37 | cp "/vagrant/config/$(hostname).kubeconfig" /var/lib/kubelet/kubeconfig 38 | cp /vagrant/config/$(hostname)-kubelet-config /var/lib/kubelet/kubelet-config.yaml 39 | 40 | cp "/vagrant/config/$(hostname)-kubelet.service" /etc/systemd/system/kubelet.service 41 | cp /vagrant/config/kube-proxy.service /etc/systemd/system/ 42 | 43 | systemctl daemon-reload 44 | services="crio kubelet kube-proxy" 45 | systemctl enable ${services} 46 | systemctl start ${services} 47 | systemctl status --no-pager ${services} 48 | EOF 49 | done 50 | -------------------------------------------------------------------------------- /config/controller-0-kube-apiserver.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Kubernetes API Server 3 | Documentation=https://github.com/GoogleCloudPlatform/kubernetes 4 | 5 | [Service] 6 | ExecStart=/usr/local/bin/kube-apiserver \ 7 | --advertise-address=192.168.199.10 \ 8 | --allow-privileged=true \ 9 | --apiserver-count=3 \ 10 | --audit-log-maxage=30 \ 11 | --audit-log-maxbackup=3 \ 12 | --audit-log-maxsize=100 \ 13 | --audit-log-path=/var/log/audit.log \ 14 | --authorization-mode=Node,RBAC \ 15 | --bind-address=0.0.0.0 \ 16 | --client-ca-file=/var/lib/kubernetes/ca.pem \ 17 | --enable-admission-plugins=NamespaceLifecycle,NodeRestriction,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota \ 18 | --etcd-cafile=/var/lib/kubernetes/ca.pem \ 19 | --etcd-certfile=/var/lib/kubernetes/kubernetes.pem \ 20 | --etcd-keyfile=/var/lib/kubernetes/kubernetes-key.pem \ 21 | --etcd-servers=https://192.168.199.10:2379,https://192.168.199.11:2379,https://192.168.199.12:2379 \ 22 | --event-ttl=1h \ 23 | --encryption-provider-config=/var/lib/kubernetes/encryption-config.yaml \ 24 | --kubelet-certificate-authority=/var/lib/kubernetes/ca.pem \ 25 | --kubelet-client-certificate=/var/lib/kubernetes/kubernetes.pem \ 26 | --kubelet-client-key=/var/lib/kubernetes/kubernetes-key.pem \ 27 | --kubelet-https=true \ 28 | --runtime-config=api/all=true \ 29 | --service-account-key-file=/var/lib/kubernetes/ca-key.pem \ 30 | --service-cluster-ip-range=10.32.0.0/24 \ 31 | --service-node-port-range=30000-32767 \ 32 | --tls-cert-file=/var/lib/kubernetes/kubernetes.pem \ 33 | --tls-private-key-file=/var/lib/kubernetes/kubernetes-key.pem \ 34 | --v=2 35 | Restart=on-failure 36 | RestartSec=5 37 | 38 | [Install] 39 | WantedBy=multi-user.target 40 | -------------------------------------------------------------------------------- /config/controller-1-kube-apiserver.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Kubernetes API Server 3 | Documentation=https://github.com/GoogleCloudPlatform/kubernetes 4 | 5 | [Service] 6 | ExecStart=/usr/local/bin/kube-apiserver \ 7 | --advertise-address=192.168.199.11 \ 8 | --allow-privileged=true \ 9 | --apiserver-count=3 \ 10 | --audit-log-maxage=30 \ 11 | --audit-log-maxbackup=3 \ 12 | --audit-log-maxsize=100 \ 13 | --audit-log-path=/var/log/audit.log \ 14 | --authorization-mode=Node,RBAC \ 15 | --bind-address=0.0.0.0 \ 16 | --client-ca-file=/var/lib/kubernetes/ca.pem \ 17 | --enable-admission-plugins=NamespaceLifecycle,NodeRestriction,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota \ 18 | --etcd-cafile=/var/lib/kubernetes/ca.pem \ 19 | --etcd-certfile=/var/lib/kubernetes/kubernetes.pem \ 20 | --etcd-keyfile=/var/lib/kubernetes/kubernetes-key.pem \ 21 | --etcd-servers=https://192.168.199.10:2379,https://192.168.199.11:2379,https://192.168.199.12:2379 \ 22 | --event-ttl=1h \ 23 | --encryption-provider-config=/var/lib/kubernetes/encryption-config.yaml \ 24 | --kubelet-certificate-authority=/var/lib/kubernetes/ca.pem \ 25 | --kubelet-client-certificate=/var/lib/kubernetes/kubernetes.pem \ 26 | --kubelet-client-key=/var/lib/kubernetes/kubernetes-key.pem \ 27 | --kubelet-https=true \ 28 | --runtime-config=api/all=true \ 29 | --service-account-key-file=/var/lib/kubernetes/ca-key.pem \ 30 | --service-cluster-ip-range=10.32.0.0/24 \ 31 | --service-node-port-range=30000-32767 \ 32 | --tls-cert-file=/var/lib/kubernetes/kubernetes.pem \ 33 | --tls-private-key-file=/var/lib/kubernetes/kubernetes-key.pem \ 34 | --v=2 35 | Restart=on-failure 36 | RestartSec=5 37 | 38 | [Install] 39 | WantedBy=multi-user.target 40 | -------------------------------------------------------------------------------- /config/controller-2-kube-apiserver.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Kubernetes API Server 3 | Documentation=https://github.com/GoogleCloudPlatform/kubernetes 4 | 5 | [Service] 6 | ExecStart=/usr/local/bin/kube-apiserver \ 7 | --advertise-address=192.168.199.12 \ 8 | --allow-privileged=true \ 9 | --apiserver-count=3 \ 10 | --audit-log-maxage=30 \ 11 | --audit-log-maxbackup=3 \ 12 | --audit-log-maxsize=100 \ 13 | --audit-log-path=/var/log/audit.log \ 14 | --authorization-mode=Node,RBAC \ 15 | --bind-address=0.0.0.0 \ 16 | --client-ca-file=/var/lib/kubernetes/ca.pem \ 17 | --enable-admission-plugins=NamespaceLifecycle,NodeRestriction,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota \ 18 | --etcd-cafile=/var/lib/kubernetes/ca.pem \ 19 | --etcd-certfile=/var/lib/kubernetes/kubernetes.pem \ 20 | --etcd-keyfile=/var/lib/kubernetes/kubernetes-key.pem \ 21 | --etcd-servers=https://192.168.199.10:2379,https://192.168.199.11:2379,https://192.168.199.12:2379 \ 22 | --event-ttl=1h \ 23 | --encryption-provider-config=/var/lib/kubernetes/encryption-config.yaml \ 24 | --kubelet-certificate-authority=/var/lib/kubernetes/ca.pem \ 25 | --kubelet-client-certificate=/var/lib/kubernetes/kubernetes.pem \ 26 | --kubelet-client-key=/var/lib/kubernetes/kubernetes-key.pem \ 27 | --kubelet-https=true \ 28 | --runtime-config=api/all=true \ 29 | --service-account-key-file=/var/lib/kubernetes/ca-key.pem \ 30 | --service-cluster-ip-range=10.32.0.0/24 \ 31 | --service-node-port-range=30000-32767 \ 32 | --tls-cert-file=/var/lib/kubernetes/kubernetes.pem \ 33 | --tls-private-key-file=/var/lib/kubernetes/kubernetes-key.pem \ 34 | --v=2 35 | Restart=on-failure 36 | RestartSec=5 37 | 38 | [Install] 39 | WantedBy=multi-user.target 40 | -------------------------------------------------------------------------------- /scripts/download-tools: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | 5 | readonly dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 6 | 7 | pushd "${dir}/../tools/" 8 | trap 'popd' EXIT 9 | 10 | echo "Downloading tools ..." 11 | 12 | # shellcheck source=versions.bash 13 | source "${dir}/versions.bash" 14 | 15 | curl -sSL \ 16 | -O "https://storage.googleapis.com/kubernetes-release/release/${k8s_version}/bin/linux/amd64/kube-apiserver" \ 17 | -O "https://storage.googleapis.com/kubernetes-release/release/${k8s_version}/bin/linux/amd64/kube-controller-manager" \ 18 | -O "https://storage.googleapis.com/kubernetes-release/release/${k8s_version}/bin/linux/amd64/kube-scheduler" \ 19 | -O "https://storage.googleapis.com/kubernetes-release/release/${k8s_version}/bin/linux/amd64/kube-proxy" \ 20 | -O "https://storage.googleapis.com/kubernetes-release/release/${k8s_version}/bin/linux/amd64/kubelet" \ 21 | -O "https://storage.googleapis.com/kubernetes-release/release/${k8s_version}/bin/linux/amd64/kubectl" 22 | 23 | curl -sSL \ 24 | -O "https://github.com/coreos/etcd/releases/download/${etcd_version}/etcd-${etcd_version}-linux-amd64.tar.gz" 25 | 26 | tar -xf "etcd-${etcd_version}-linux-amd64.tar.gz" 27 | 28 | curl -sSL \ 29 | -O "https://github.com/cri-o/cri-o/releases/download/${crio_version}/crio-${crio_version}.tar.gz" \ 30 | 31 | tar -xf "crio-${crio_version}.tar.gz" 32 | mv "crio-${crio_version}" crio 33 | 34 | curl -sSL \ 35 | -O "https://github.com/containous/traefik/releases/download/${traefik_version}/traefik_linux-amd64" 36 | 37 | mv traefik_linux-amd64 traefik 38 | 39 | chmod +x \ 40 | kube-apiserver \ 41 | kube-controller-manager \ 42 | kube-scheduler \ 43 | kube-proxy \ 44 | kubelet \ 45 | kubectl \ 46 | traefik 47 | -------------------------------------------------------------------------------- /scripts/setup-controller-services: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | 5 | readonly dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 6 | 7 | pushd "${dir}/../" 8 | trap 'popd' EXIT 9 | 10 | for i in {0..2}; do 11 | cat <<'EOF' | vagrant ssh "controller-${i}" -- sudo bash 12 | 13 | mkdir -p /var/lib/kubernetes 14 | 15 | cp \ 16 | /vagrant/certificates/ca.pem \ 17 | /vagrant/certificates/ca-key.pem \ 18 | /vagrant/certificates/kubernetes-key.pem \ 19 | /vagrant/certificates/kubernetes.pem \ 20 | /vagrant/config/encryption-config.yaml \ 21 | /vagrant/config/kube-controller-manager.kubeconfig \ 22 | /vagrant/config/kube-scheduler.kubeconfig \ 23 | /var/lib/kubernetes/ 24 | 25 | cp \ 26 | /vagrant/tools/kube-apiserver \ 27 | /vagrant/tools/kube-controller-manager \ 28 | /vagrant/tools/kube-scheduler \ 29 | /vagrant/tools/kubectl \ 30 | /usr/local/bin/ 31 | 32 | cp \ 33 | "/vagrant/config/$(hostname)-kube-apiserver.service" \ 34 | /etc/systemd/system/kube-apiserver.service 35 | cp \ 36 | "/vagrant/config/$(hostname)-kube-scheduler.service" \ 37 | /etc/systemd/system/kube-scheduler.service 38 | cp \ 39 | "/vagrant/config/$(hostname)-kube-controller-manager.service" \ 40 | /etc/systemd/system/kube-controller-manager.service 41 | 42 | systemctl daemon-reload 43 | services="kube-apiserver kube-controller-manager kube-scheduler" 44 | systemctl enable ${services} 45 | systemctl start ${services} 46 | systemctl status --no-pager ${services} 47 | EOF 48 | 49 | until vagrant ssh "controller-${i}" -- curl --ipv4 --fail --silent --max-time 3 -k "https://localhost:6443/healthz" &>/dev/null; do 50 | echo "Waiting for kube-apiserver to become ready in controller-${i}..." 51 | sleep 3 52 | done 53 | 54 | done 55 | -------------------------------------------------------------------------------- /scripts/vagrant-setup-haproxy.bash: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | 5 | apt-get update 6 | apt-get install -y haproxy 7 | 8 | cat >/etc/haproxy/haproxy.cfg <"${dir}/../config/controller-${i}-kube-apiserver.service" </etc/traefik.toml </etc/systemd/system/traefik.service < 53s v1.20.2 183 | worker-1 Ready 33s v1.20.2 184 | worker-2 Ready 11s v1.20.2 185 | ``` 186 | 187 | ## Using the cluster 188 | 189 | ### Setup DNS add-on 190 | 191 | Deploy the DNS add-on and verify it's working: 192 | 193 | ``` 194 | kubectl apply -f ./manifests/coredns.yaml 195 | [...] 196 | kubectl get pods -l k8s-app=coredns -n kube-system 197 | [...] 198 | kubectl run busybox --image=busybox:1.28 --command -- sleep 3600 199 | [...] 200 | kubectl exec -ti busybox -- nslookup kubernetes 201 | ``` 202 | 203 | ### Smoke tests 204 | 205 | ```console 206 | $ kubectl create -f ./manifests/nginx.yaml 207 | deployment "nginx" created 208 | service "nginx" created 209 | ``` 210 | 211 | ```console 212 | $ NODE_PORT=$(kubectl get svc nginx --output=jsonpath='{range .spec.ports[0]}{.nodePort}') 213 | $ for i in {0..2}; do curl -sS 192.168.199.2${i}:${NODE_PORT} | awk '/

/{gsub("<[/]*h1>", ""); print $0}'; done 214 | Welcome to nginx! 215 | Welcome to nginx! 216 | Welcome to nginx! 217 | ``` 218 | 219 | ### Connect to services from host 220 | 221 | `10.32.0.0/24` is the IP range for services. In order to connect to a service 222 | from the host, one of the worker nodes (with `kube-proxy`) must be used as a 223 | gateway. Example: 224 | 225 | 226 | ``` 227 | # On Linux 228 | sudo route add -net 10.32.0.0/24 gw 192.168.199.22 229 | 230 | # On macOS 231 | sudo route -n add -net 10.32.0.0/24 192.168.199.22 232 | ``` 233 | 234 | ### Use [Traefik](https://traefik.io/) loadbalancer 235 | 236 | ``` 237 | ./scripts/setup-traefik 238 | [...] 239 | curl 192.168.199.30 240 | 404 page not found 241 | ``` 242 | 243 | To test traefik is actually doing its job, you can create an ingress rule 244 | for the nginx service that you created above: 245 | 246 | ``` 247 | kubectl apply -f ./manifests/nginx-ingress.yaml 248 | echo "192.168.199.30 nginx.kthw" | sudo tee -a /etc/hosts 249 | curl nginx.kthw 250 | 251 | [...] 252 | ``` 253 | 254 | ## Contributing 255 | 256 | Contributions are welcome: KTHW Vagrant is meant to be a learning 257 | project and testbed for aspiring Kubernetes operators and CKAs 258 | ([Certified Kubernetes Administrator](https://www.cncf.io/certification/cka/)). 259 | 260 | If you want to contribute code or updates, look for the label 261 | [good first issue](https://github.com/kinvolk/kubernetes-the-hard-way-vagrant/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22). 262 | 263 | ## Pitfalls 264 | 265 | ### Error loading config file "/var/log": read /var/log: is a directory 266 | 267 | On OSX, `KUBECONFIG` apparently needs to be set explicitly. `~/.kube/config` 268 | is a good place and the default on Linux. 269 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | --------------------------------------------------------------------------------