├── .gitignore ├── kube-manifests ├── kube-ui-svc.yaml ├── kube-dns-svc.yaml ├── kube-ui-rc.yaml └── kube-dns-rc.yaml ├── cert-gen ├── init-ca.sh └── init-ssl.sh ├── README.md ├── worker.yaml └── master.yaml /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | -------------------------------------------------------------------------------- /kube-manifests/kube-ui-svc.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: kube-ui 5 | namespace: kube-system 6 | labels: 7 | k8s-app: kube-ui 8 | kubernetes.io/cluster-service: "true" 9 | kubernetes.io/name: "KubeUI" 10 | spec: 11 | selector: 12 | k8s-app: kube-ui 13 | ports: 14 | - port: 80 15 | targetPort: 8080 -------------------------------------------------------------------------------- /kube-manifests/kube-dns-svc.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: "v1" 3 | kind: "Service" 4 | metadata: 5 | name: "kube-dns" 6 | namespace: "kube-system" 7 | labels: 8 | k8s-app: "kube-dns" 9 | kubernetes.io/name: "KubeDNS" 10 | kubernetes.io/cluster-service: "true" 11 | spec: 12 | clusterIP: "${DNS_SERVICE_IP}" 13 | ports: 14 | - protocol: "UDP" 15 | name: "dns" 16 | port: 53 17 | - protocol: "TCP" 18 | name: "dns-tcp" 19 | port: 53 20 | selector: 21 | k8s-app: "kube-dns" 22 | -------------------------------------------------------------------------------- /cert-gen/init-ca.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | # define location of openssl binary manually since running this 4 | # script under Vagrant fails on some systems without it 5 | OPENSSL=/usr/bin/openssl 6 | 7 | function usage { 8 | echo "USAGE: $0 " 9 | echo " example: $0 ./ssl/ca.pem" 10 | } 11 | 12 | if [ -z "$1" ]; then 13 | usage 14 | exit 1 15 | fi 16 | 17 | OUTDIR="$1" 18 | 19 | if [ ! -d $OUTDIR ]; then 20 | echo "ERROR: output directory does not exist: $OUTDIR" 21 | exit 1 22 | fi 23 | 24 | OUTFILE="$OUTDIR/ca.pem" 25 | 26 | if [ -f "$OUTFILE" ];then 27 | exit 0 28 | fi 29 | 30 | # establish cluster CA and self-sign a cert 31 | $OPENSSL genrsa -out "$OUTDIR/ca-key.pem" 2048 32 | $OPENSSL req -x509 -new -nodes -key "$OUTDIR/ca-key.pem" -days 10000 -out "$OUTFILE" -subj "/CN=kube-ca" 33 | 34 | -------------------------------------------------------------------------------- /kube-manifests/kube-ui-rc.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ReplicationController 3 | metadata: 4 | name: kube-ui-v4 5 | namespace: kube-system 6 | labels: 7 | k8s-app: kube-ui 8 | version: v4 9 | kubernetes.io/cluster-service: "true" 10 | spec: 11 | replicas: 1 12 | selector: 13 | k8s-app: kube-ui 14 | version: v4 15 | template: 16 | metadata: 17 | labels: 18 | k8s-app: kube-ui 19 | version: v4 20 | kubernetes.io/cluster-service: "true" 21 | spec: 22 | containers: 23 | - name: kube-ui 24 | image: gcr.io/google_containers/kube-ui:v4 25 | resources: 26 | limits: 27 | cpu: 100m 28 | memory: 50Mi 29 | ports: 30 | - containerPort: 8080 31 | livenessProbe: 32 | httpGet: 33 | path: / 34 | port: 8080 35 | initialDelaySeconds: 30 36 | timeoutSeconds: 5 -------------------------------------------------------------------------------- /cert-gen/init-ssl.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | # define location of openssl binary manually since running this 4 | # script under Vagrant fails on some systems without it 5 | OPENSSL=/usr/bin/openssl 6 | 7 | function usage { 8 | echo "USAGE: $0 [SAN,SAN,SAN]" 9 | echo " example: $0 ./ssl/ worker kube-worker IP.1=127.0.0.1,IP.2=10.0.0.1" 10 | } 11 | 12 | if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then 13 | usage 14 | exit 1 15 | fi 16 | 17 | OUTDIR="$1" 18 | CERTBASE="$2" 19 | CN="$3" 20 | SANS="$4" 21 | 22 | if [ ! -d $OUTDIR ]; then 23 | echo "ERROR: output directory does not exist: $OUTDIR" 24 | exit 1 25 | fi 26 | 27 | OUTFILE="$OUTDIR/$CN.tar" 28 | 29 | if [ -f "$OUTFILE" ];then 30 | exit 0 31 | fi 32 | 33 | CNF_TEMPLATE=" 34 | [req] 35 | req_extensions = v3_req 36 | distinguished_name = req_distinguished_name 37 | 38 | [req_distinguished_name] 39 | 40 | [ v3_req ] 41 | basicConstraints = CA:FALSE 42 | keyUsage = nonRepudiation, digitalSignature, keyEncipherment 43 | subjectAltName = @alt_names 44 | 45 | [alt_names] 46 | DNS.1 = kubernetes 47 | DNS.2 = kubernetes.default 48 | " 49 | echo "Generating SSL artifacts in $OUTDIR" 50 | 51 | 52 | CONFIGFILE="$OUTDIR/$CERTBASE-req.cnf" 53 | CAFILE="$OUTDIR/ca.pem" 54 | CAKEYFILE="$OUTDIR/ca-key.pem" 55 | KEYFILE="$OUTDIR/$CERTBASE-key.pem" 56 | CSRFILE="$OUTDIR/$CERTBASE.csr" 57 | PEMFILE="$OUTDIR/$CERTBASE.pem" 58 | 59 | CONTENTS="${CAFILE} ${KEYFILE} ${PEMFILE}" 60 | 61 | 62 | # Add SANs to openssl config 63 | echo "$CNF_TEMPLATE$(echo $SANS | tr ',' '\n')" > "$CONFIGFILE" 64 | 65 | $OPENSSL genrsa -out "$KEYFILE" 2048 66 | $OPENSSL req -new -key "$KEYFILE" -out "$CSRFILE" -subj "/CN=$CN" -config "$CONFIGFILE" 67 | $OPENSSL x509 -req -in "$CSRFILE" -CA "$CAFILE" -CAkey "$CAKEYFILE" -CAcreateserial -out "$PEMFILE" -days 365 -extensions v3_req -extfile "$CONFIGFILE" 68 | 69 | tar -cf $OUTFILE -C $OUTDIR $(for f in $CONTENTS;do printf "$(basename $f) ";done) 70 | 71 | echo "Bundled SSL artifacts into $OUTFILE" 72 | echo "$CONTENTS" 73 | -------------------------------------------------------------------------------- /kube-manifests/kube-dns-rc.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ReplicationController 3 | metadata: 4 | name: kube-dns-v10 5 | namespace: kube-system 6 | labels: 7 | k8s-app: kube-dns 8 | version: v10 9 | kubernetes.io/cluster-service: "true" 10 | spec: 11 | replicas: 2 12 | selector: 13 | k8s-app: kube-dns 14 | version: v10 15 | template: 16 | metadata: 17 | labels: 18 | k8s-app: kube-dns 19 | version: v10 20 | kubernetes.io/cluster-service: "true" 21 | spec: 22 | containers: 23 | - name: etcd 24 | image: gcr.io/google_containers/etcd:2.0.9 25 | resources: 26 | # keep request = limit to keep this container in guaranteed class 27 | limits: 28 | cpu: 100m 29 | memory: 50Mi 30 | requests: 31 | cpu: 100m 32 | memory: 50Mi 33 | command: 34 | - /usr/local/bin/etcd 35 | - -data-dir 36 | - /var/etcd/data 37 | - -listen-client-urls 38 | - http://127.0.0.1:2379,http://127.0.0.1:4001 39 | - -advertise-client-urls 40 | - http://127.0.0.1:2379,http://127.0.0.1:4001 41 | - -initial-cluster-token 42 | - skydns-etcd 43 | volumeMounts: 44 | - name: etcd-storage 45 | mountPath: /var/etcd/data 46 | - name: kube2sky 47 | image: gcr.io/google_containers/kube2sky:1.12 48 | resources: 49 | # keep request = limit to keep this container in guaranteed class 50 | limits: 51 | cpu: 100m 52 | memory: 50Mi 53 | requests: 54 | cpu: 100m 55 | memory: 50Mi 56 | args: 57 | # command = "/kube2sky" 58 | - -domain=cluster.local 59 | - name: skydns 60 | image: gcr.io/google_containers/skydns:2015-10-13-8c72f8c 61 | resources: 62 | # keep request = limit to keep this container in guaranteed class 63 | limits: 64 | cpu: 100m 65 | memory: 50Mi 66 | requests: 67 | cpu: 100m 68 | memory: 50Mi 69 | args: 70 | # command = "/skydns" 71 | - -machines=http://127.0.0.1:4001 72 | - -addr=0.0.0.0:53 73 | - -ns-rotate=false 74 | - "-domain=cluster.local" 75 | ports: 76 | - containerPort: 53 77 | name: dns 78 | protocol: UDP 79 | - containerPort: 53 80 | name: dns-tcp 81 | protocol: TCP 82 | livenessProbe: 83 | httpGet: 84 | path: /healthz 85 | port: 8080 86 | scheme: HTTP 87 | initialDelaySeconds: 30 88 | timeoutSeconds: 5 89 | readinessProbe: 90 | httpGet: 91 | path: /healthz 92 | port: 8080 93 | scheme: HTTP 94 | initialDelaySeconds: 1 95 | timeoutSeconds: 5 96 | - name: healthz 97 | image: gcr.io/google_containers/exechealthz:1.0 98 | resources: 99 | # keep request = limit to keep this container in guaranteed class 100 | limits: 101 | cpu: 10m 102 | memory: 20Mi 103 | requests: 104 | cpu: 10m 105 | memory: 20Mi 106 | args: 107 | - -cmd=nslookup kubernetes.default.svc.cluster.local 127.0.0.1 >/dev/null 108 | - -port=8080 109 | ports: 110 | - containerPort: 8080 111 | protocol: TCP 112 | volumes: 113 | - name: etcd-storage 114 | emptyDir: {} 115 | dnsPolicy: Default # Don't use cluster DNS. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # kubernetes-small-cluster 2 | Kubernetes small cluster build for running on single NIC baremetal systems such as [Intel NUC](www.intel.com/content/www/us/en/nuc/overview.html) or [Gigabyte Brix](http://www.gigabyte.us/brix/2015brix/) 3 | 4 | Based on https://github.com/coreos/coreos-kubernetes 5 | 6 | ## Why? 7 | 8 | This is used for building small clusters based on spare servers, or in this case Intel NUC units, without incurring compute charges from cloud providers. 9 | 10 | ## Create configuration: 11 | 12 | Customize build parameters in `build-data.sh` 13 | 14 | ``` 15 | ./build-data.sh 16 | ``` 17 | 18 | ## CoreOS installation 19 | 20 | You can use the install method of choice, but for simplicity we are using a USB baremetal install: 21 | 22 | Mount the USB drive: 23 | 24 | ``` 25 | mount /dev/sdb1 /mnt 26 | ``` 27 | 28 | Download CoreOS, update permissions, and install on USB drive: 29 | 30 | ``` 31 | wget https://raw.githubusercontent.com/coreos/init/master/bin/coreos-install 32 | chmod +x coreos-install 33 | sudo ./coreos-install -d /dev/sda -v 899.1.0 -c /mnt/user-data- 34 | ``` 35 | 36 | Reboot system and choose to boot from USB drive. 37 | 38 | ## Configure kubectl 39 | 40 | ``` 41 | kubectl config set-cluster nuc --server=https://10.10.10.10:443 --certificate-authority=${PWD}/ssl/ca.pem 42 | kubectl config set-credentials nuc-admin --certificate-authority=${PWD}/ssl/ca.pem --client-key=${PWD}/ssl/admin-key.pem --client-certificate=${PWD}/ssl/admin.pem 43 | kubectl config set-context nuc --cluster=nuc --user=nuc-admin 44 | kubectl config use-context nuc 45 | ``` 46 | 47 | ``` 48 | $ kubectl get nodes 49 | NAME LABELS STATUS AGE 50 | 10.10.10.10 kubernetes.io/hostname=10.10.10.10 Ready 1h 51 | 10.10.10.11 kubernetes.io/hostname=10.10.10.11 Ready 1h 52 | 10.10.10.12 kubernetes.io/hostname=10.10.10.12 Ready 1h 53 | ``` 54 | 55 | ## Client certificate installation 56 | 57 | To access the apiserver url (https://10.10.10.10) you'll need a client certificate. Without one you'll see this: 58 | ``` 59 | $ curl https://10.10.10.10/ -v 60 | * Trying 10.10.10.10... 61 | * Connected to 10.10.10.10 (10.10.10.10) port 443 (#0) 62 | * WARNING: using IP address, SNI is being disabled by the OS. 63 | * TLS 1.2 connection using TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 64 | * Server certificate: kube-controller 65 | * Server certificate: kube-ca 66 | > GET / HTTP/1.1 67 | > Host: 10.10.10.10 68 | > User-Agent: curl/7.43.0 69 | > Accept: */* 70 | > 71 | < HTTP/1.1 401 Unauthorized 72 | < Content-Type: text/plain; charset=utf-8 73 | < Date: Mon, 11 Jan 2016 18:16:31 GMT 74 | < Content-Length: 13 75 | < 76 | Unauthorized 77 | * Connection #0 to host 10.10.10.10 left intact 78 | ``` 79 | 80 | ``` 81 | curl https://10.10.10.10/ -E ssl/worker.p12: --cacert ssl/ca.pem 82 | { 83 | "paths": [ 84 | "/api", 85 | "/api/v1", 86 | "/apis", 87 | "/apis/extensions", 88 | "/apis/extensions/v1beta1", 89 | "/healthz", 90 | "/healthz/ping", 91 | "/logs/", 92 | "/metrics", 93 | "/resetMetrics", 94 | "/swagger-ui/", 95 | "/swaggerapi/", 96 | "/ui/", 97 | "/version" 98 | ] 99 | } 100 | ``` 101 | 102 | To fix this issue you need to install the generated certificate `worker.p12` and `ca.pem` located in the ssl directory. 103 | 104 | ## Addon installation: 105 | ``` 106 | kubectl create -f kube-manifests/kube-dns-rc.yaml 107 | kubectl create -f kube-manifests/kube-dns-svc.yaml 108 | kubectl create -f kube-manifests/kube-ui-rc.yaml 109 | kubectl create -f kube-manifests/kube-ui-svc.yaml 110 | ``` 111 | 112 | Now you'll be able to access the Kubernetes UI located in `https://10.10.10.10/api/v1/proxy/namespaces/kube-system/services/kube-ui/#/dashboard/` 113 | 114 | -------------------------------------------------------------------------------- /worker.yaml: -------------------------------------------------------------------------------- 1 | #!cloud-config # https://coreos.com/os/docs/latest/cloud-config.html 2 | --- 3 | 4 | hostname: ${NODE_NAME} 5 | 6 | users: 7 | - name: "dockerwho" 8 | passwd: "110uOUNBSWaybBvhQ6JcTNyC/bV37Aj2LnZQi0zRf1v9KTP6KNdKQ4clZLgzzCoqU168lOTudZHQcRiDp+a2cxY1zQbzLQ4QGgFFiPSE2NpSX/4JrTHViIQMhZ2sAePdfrXHsJVJvfgV299XqqrW8KVlflmjRkezefK0kF0wekg=" 9 | groups: 10 | - "sudo" 11 | - "docker" 12 | 13 | ssh_authorized_keys: 14 | - ${PUB_RSA} 15 | 16 | coreos: 17 | update: 18 | group: alpha 19 | reboot-strategy: off 20 | etcd2: 21 | name: ${NODE_NAME} 22 | advertise-client-urls: "http://${NODE_IP}:2379" 23 | initial-cluster: "${ETCD_ENDPOINTS}" 24 | initial-cluster-token: k8s-etcd-cluster 25 | initial-cluster-state: existing 26 | initial-advertise-peer-urls: "http://${NODE_IP}:2380" 27 | listen-peer-urls: "http://0.0.0.0:2380" 28 | listen-client-urls: "http://0.0.0.0:2379" 29 | units: 30 | - name: 10-static.network 31 | content: | 32 | [Match] 33 | Name=${NET_IFACE} 34 | 35 | [Network] 36 | Address=${NODE_IP}/24 37 | Gateway=${NODE_GATEWAY} 38 | DNS=8.8.8.8 39 | - name: etcd.service 40 | mask: true 41 | - name: etcd2.service 42 | command: start 43 | - name: flanneld.service 44 | command: start 45 | - name: docker-tcp.socket 46 | command: start 47 | enable: true 48 | content: | 49 | [Unit] 50 | Description=Docker Socket for the API 51 | 52 | [Socket] 53 | ListenStream=2375 54 | Service=docker.service 55 | BindIPv6Only=both 56 | 57 | [Install] 58 | WantedBy=sockets.target 59 | - name: docker.service 60 | drop-ins: 61 | - name: 10-require-flannel.conf 62 | content: | 63 | [Unit] 64 | Requires=flanneld.service 65 | After=flanneld.service 66 | - name: 50-change-options.conf 67 | content: | 68 | [Service] 69 | Environment='DOCKER_OPTS=-s=overlay --insecure-registry="0.0.0.0/0" --iptables=false --log-level=warn' 70 | - name: kubelet.service 71 | command: start 72 | enable: true 73 | content: | 74 | [Service] 75 | ExecStartPre=/usr/bin/mkdir -p /etc/kubernetes/manifests 76 | ExecStartPre=/opt/bin/download-k8s-binary kubelet 77 | ExecStart=/opt/bin/kubelet \ 78 | --api-servers=${MASTER_URL} \ 79 | --register-node=true \ 80 | --allow-privileged=true \ 81 | --config=/etc/kubernetes/manifests \ 82 | --hostname-override=${NODE_IP} \ 83 | --cluster-dns=${DNS_SERVICE_IP} \ 84 | --cluster-domain=cluster.local \ 85 | --cadvisor-port=4194 \ 86 | --healthz-bind-address=0.0.0.0 \ 87 | --max-pods=300 \ 88 | --healthz-port=10248 \ 89 | --kubeconfig=/etc/kubernetes/worker-kubeconfig.yaml \ 90 | --tls-cert-file=/etc/kubernetes/ssl/worker.pem \ 91 | --tls-private-key-file=/etc/kubernetes/ssl/worker-key.pem \ 92 | --host_network_sources="*" 93 | Restart=always 94 | RestartSec=10 95 | 96 | [Install] 97 | WantedBy=multi-user.target 98 | - name: disable-transparent-huge-pages.service 99 | command: start 100 | content: | 101 | [Unit] 102 | Description=Disable Transparent Huge Pages 103 | 104 | [Service] 105 | Type=oneshot 106 | ExecStart=/bin/sh -c "until (echo 'Waiting for sys mount...' && cat /sys/kernel/mm/transparent_hugepage/enabled >/dev/null 2>&1); do sleep 1; done" 107 | ExecStart=/bin/sh -c "echo never | tee /sys/kernel/mm/transparent_hugepage/enabled >/dev/null 2>&1" 108 | ExecStart=/bin/sh -c "echo never | tee /sys/kernel/mm/transparent_hugepage/defrag >/dev/null 2>&1" 109 | write_files: 110 | - path: /etc/kubernetes/manifests/kube-proxy.yaml 111 | permissions: "0644" 112 | owner: "root" 113 | content: | 114 | apiVersion: v1 115 | kind: Pod 116 | metadata: 117 | name: kube-proxy 118 | namespace: kube-system 119 | spec: 120 | hostNetwork: true 121 | containers: 122 | - name: kube-proxy 123 | image: ${HYPERKUBE_IMAGE}:${K8S_VER} 124 | command: 125 | - /hyperkube 126 | - proxy 127 | - --master=https://${NODE_START_IP} 128 | - --kubeconfig=/etc/kubernetes/worker-kubeconfig.yaml 129 | - --proxy-mode=iptables 130 | securityContext: 131 | privileged: true 132 | volumeMounts: 133 | - mountPath: /etc/ssl/certs 134 | name: "ssl-certs" 135 | - mountPath: /etc/kubernetes/worker-kubeconfig.yaml 136 | name: "kubeconfig" 137 | readOnly: true 138 | - mountPath: /etc/kubernetes/ssl 139 | name: "etc-kube-ssl" 140 | readOnly: true 141 | volumes: 142 | - name: "ssl-certs" 143 | hostPath: 144 | path: "/usr/share/ca-certificates" 145 | - name: "kubeconfig" 146 | hostPath: 147 | path: "/etc/kubernetes/worker-kubeconfig.yaml" 148 | - name: "etc-kube-ssl" 149 | hostPath: 150 | path: "/etc/kubernetes/ssl" 151 | - path: /etc/kubernetes/ssl/ca.pem 152 | permissions: "0644" 153 | owner: "root" 154 | encoding: "base64" 155 | content: | 156 | ${CA_PEM} 157 | - path: /etc/kubernetes/ssl/worker-key.pem 158 | permissions: "0644" 159 | owner: "root" 160 | encoding: "base64" 161 | content: | 162 | ${WORKER_KEY_PEM} 163 | - path: /etc/kubernetes/ssl/worker.pem 164 | permissions: "0644" 165 | owner: "root" 166 | encoding: "base64" 167 | content: | 168 | ${WORKER_PEM} 169 | - path: /etc/kubernetes/worker-kubeconfig.yaml 170 | permissions: "0644" 171 | owner: "root" 172 | content: | 173 | apiVersion: v1 174 | kind: Config 175 | clusters: 176 | - name: local 177 | cluster: 178 | certificate-authority: /etc/kubernetes/ssl/ca.pem 179 | users: 180 | - name: kubelet 181 | user: 182 | client-certificate: /etc/kubernetes/ssl/worker.pem 183 | client-key: /etc/kubernetes/ssl/worker-key.pem 184 | contexts: 185 | - context: 186 | cluster: local 187 | user: kubelet 188 | name: kubelet-context 189 | current-context: kubelet-context 190 | - path: /etc/systemd/coredump.conf 191 | content: | 192 | [Coredump] 193 | Storage=none 194 | - path: /etc/profile.d/nse-function.sh 195 | permissions: '0755' 196 | owner: core 197 | content: | 198 | function nse() { 199 | docker exec -it $1 bash 200 | } 201 | - path: /home/core/.toolboxrc 202 | owner: core 203 | content: | 204 | TOOLBOX_DOCKER_IMAGE=alpine 205 | TOOLBOX_DOCKER_TAG=3.2 206 | TOOLBOX_USER=root 207 | - path: /opt/bin/download-k8s-binary 208 | permissions: '0755' 209 | content: | 210 | #!/usr/bin/env bash 211 | FILE=$1 212 | if [ ! -f /opt/bin/$FILE ]; then 213 | curl -sSL -o /opt/bin/$FILE https://storage.googleapis.com/kubernetes-release/release/${K8S_VER}/bin/linux/amd64/$FILE 214 | chmod +x /opt/bin/$FILE 215 | else 216 | # we check the version of the binary 217 | INSTALLED_VERSION=$(/opt/bin/$FILE --version=true) 218 | MATCH=$(echo "$INSTALLED_VERSION" | grep -c "${K8S_VER}") 219 | if [ $MATCH -eq 0 ]; then 220 | # the version is different 221 | curl -sSL -o /opt/bin/$FILE https://storage.googleapis.com/kubernetes-release/release/${K8S_VER}/bin/linux/amd64/$FILE 222 | chmod +x /opt/bin/$FILE 223 | fi 224 | fi 225 | 226 | - path: /opt/bin/wupiao 227 | permissions: '0755' 228 | owner: core 229 | content: | 230 | #!/usr/bin/env bash 231 | # [w]ait [u]ntil [p]ort [i]s [a]ctually [o]pen 232 | [ -n "$1" ] && \ 233 | until curl -o /dev/null -sIf http://${1}; do \ 234 | sleep 1 && echo .; 235 | done; 236 | exit $? 237 | manage_etc_hosts: "localhost" 238 | -------------------------------------------------------------------------------- /master.yaml: -------------------------------------------------------------------------------- 1 | #!cloud-config # https://coreos.com/os/docs/latest/cloud-config.html 2 | --- 3 | 4 | hostname: ${NODE_NAME} 5 | 6 | users: 7 | - name: "dockerwho" 8 | passwd: "110uOUNBSWaybBvhQ6JcTNyC/bV37Aj2LnZQi0zRf1v9KTP6KNdKQ4clZLgzzCoqU168lOTudZHQcRiDp+a2cxY1zQbzLQ4QGgFFiPSE2NpSX/4JrTHViIQMhZ2sAePdfrXHsJVJvfgV299XqqrW8KVlflmjRkezefK0kF0wekg=" 9 | groups: 10 | - "sudo" 11 | - "docker" 12 | 13 | ssh_authorized_keys: 14 | - ${PUB_RSA} 15 | 16 | coreos: 17 | update: 18 | group: alpha 19 | reboot-strategy: off 20 | etcd2: 21 | name: ${NODE_NAME} 22 | advertise-client-urls: "http://${NODE_IP}:2379" 23 | initial-cluster: "${ETCD_ENDPOINTS}" 24 | initial-cluster-token: k8s-etcd-cluster 25 | initial-cluster-state: new 26 | initial-advertise-peer-urls: "http://${NODE_IP}:2380" 27 | listen-peer-urls: "http://0.0.0.0:2380" 28 | listen-client-urls: "http://0.0.0.0:2379" 29 | units: 30 | - name: 10-static.network 31 | content: | 32 | [Match] 33 | Name=${NET_IFACE} 34 | 35 | [Network] 36 | Address=${NODE_IP}/24 37 | Gateway=${NODE_GATEWAY} 38 | DNS=8.8.8.8 39 | - name: etcd.service 40 | mask: true 41 | - name: etcd2.service 42 | command: start 43 | - name: flanneld.service 44 | command: start 45 | drop-ins: 46 | - name: 50-network-config.conf 47 | content: | 48 | [Unit] 49 | Requires=etcd2.service 50 | [Service] 51 | ExecStartPre=/usr/bin/etcdctl set /coreos.com/network/config '{"Network":"${POD_NETWORK}","SubnetLen":20,"Backend": {"Type": "vxlan"} }' 52 | - name: docker.socket 53 | command: start 54 | - name: docker-tcp.socket 55 | command: start 56 | enable: true 57 | content: | 58 | [Unit] 59 | Description=Docker Socket for the API 60 | 61 | [Socket] 62 | ListenStream=2375 63 | Service=docker.service 64 | BindIPv6Only=both 65 | 66 | [Install] 67 | WantedBy=sockets.target 68 | - name: docker.service 69 | drop-ins: 70 | - name: 10-require-flannel.conf 71 | content: | 72 | [Unit] 73 | Requires=flanneld.service 74 | After=flanneld.service 75 | - name: 50-change-options.conf 76 | content: | 77 | [Service] 78 | Environment='DOCKER_OPTS=-s=overlay --insecure-registry="0.0.0.0/0" --iptables=false --log-level=warn' 79 | - name: kubelet.service 80 | command: start 81 | enable: true 82 | content: | 83 | [Service] 84 | ExecStartPre=/usr/bin/mkdir -p /etc/kubernetes/manifests 85 | ExecStartPre=/opt/bin/download-k8s-binary kubelet 86 | ExecStart=/opt/bin/kubelet \ 87 | --api-servers=http://127.0.0.1:8080 \ 88 | --register-node=true \ 89 | --allow-privileged=true \ 90 | --config=/etc/kubernetes/manifests \ 91 | --hostname-override=${NODE_IP} \ 92 | --cluster-dns=${DNS_SERVICE_IP} \ 93 | --cluster-domain=cluster.local \ 94 | --cadvisor-port=4194 \ 95 | --healthz-bind-address=0.0.0.0 \ 96 | --max-pods=300 \ 97 | --healthz-port=10248 \ 98 | --host_network_sources="*" \ 99 | --register-node=false 100 | Restart=always 101 | RestartSec=10 102 | 103 | [Install] 104 | WantedBy=multi-user.target 105 | - name: k8s-manifest.service 106 | command: start 107 | content: | 108 | [Service] 109 | Type=oneshot 110 | RemainAfterExit=yes 111 | ExecStartPre=/usr/bin/mkdir -p /etc/kubernetes 112 | ExecStart=/usr/bin/tar -xpvf /etc/kubernetes/manifests.tar.gz -C /etc/kubernetes 113 | - name: disable-transparent-huge-pages.service 114 | command: start 115 | content: | 116 | [Unit] 117 | Description=Disable Transparent Huge Pages 118 | 119 | [Service] 120 | Type=oneshot 121 | ExecStart=/bin/sh -c "until (echo 'Waiting for sys mount...' && cat /sys/kernel/mm/transparent_hugepage/enabled >/dev/null 2>&1); do sleep 1; done" 122 | ExecStart=/bin/sh -c "echo never | tee /sys/kernel/mm/transparent_hugepage/enabled >/dev/null 2>&1" 123 | ExecStart=/bin/sh -c "echo never | tee /sys/kernel/mm/transparent_hugepage/defrag >/dev/null 2>&1" 124 | - name: wait-apiserver.service 125 | command: start 126 | enable: true 127 | content: | 128 | [Unit] 129 | Description=Wait for apiserver 130 | 131 | [Service] 132 | Type=oneshot 133 | RemainAfterExit=yes 134 | ExecStartPre=/opt/bin/wupiao 127.0.0.1:8080 135 | ExecStart=/bin/sh -c 'curl --silent -XPOST -d @/etc/kubernetes/manifests/kube-system.json http://127.0.0.1:8080/api/v1/namespaces' 136 | write_files: 137 | - path: /etc/kubernetes/manifests/kube-apiserver.yaml 138 | permissions: '0644' 139 | content: | 140 | apiVersion: v1 141 | kind: Pod 142 | metadata: 143 | name: kube-apiserver 144 | namespace: kube-system 145 | spec: 146 | hostNetwork: true 147 | containers: 148 | - name: kube-apiserver 149 | image: ${HYPERKUBE_IMAGE}:${K8S_VER} 150 | command: 151 | - /hyperkube 152 | - apiserver 153 | - --advertise-address=${NODE_START_IP} 154 | - --bind-address=0.0.0.0 155 | - --etcd-servers=${ETCD_URL} 156 | - --allow-privileged=true 157 | - --service-cluster-ip-range=${SERVICE_IP_RANGE} 158 | - --secure-port=443 159 | - --admission-control=NamespaceLifecycle,NamespaceExists,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota 160 | - --tls-cert-file=/etc/kubernetes/ssl/apiserver.pem 161 | - --tls-private-key-file=/etc/kubernetes/ssl/apiserver-key.pem 162 | - --client-ca-file=/etc/kubernetes/ssl/ca.pem 163 | - --service-account-key-file=/etc/kubernetes/ssl/apiserver-key.pem 164 | - --runtime-config=extensions/v1beta1=true,extensions/v1beta1/daemonsets=true,extensions/v1beta1/thirdpartyresource=true 165 | ports: 166 | - containerPort: 443 167 | hostPort: 443 168 | name: https 169 | - containerPort: 8080 170 | hostPort: 8080 171 | name: local 172 | volumeMounts: 173 | - mountPath: /etc/kubernetes/ssl 174 | name: ssl-certs-kubernetes 175 | readOnly: true 176 | - mountPath: /etc/ssl/certs 177 | name: ssl-certs-host 178 | readOnly: true 179 | volumes: 180 | - hostPath: 181 | path: /etc/kubernetes/ssl 182 | name: ssl-certs-kubernetes 183 | - hostPath: 184 | path: /usr/share/ca-certificates 185 | name: ssl-certs-host 186 | - path: /etc/kubernetes/manifests/kube-controller-manager.yaml 187 | permissions: '0644' 188 | content: | 189 | apiVersion: v1 190 | kind: Pod 191 | metadata: 192 | name: kube-controller-manager 193 | namespace: kube-system 194 | spec: 195 | containers: 196 | - name: kube-controller-manager 197 | image: ${HYPERKUBE_IMAGE}:${K8S_VER} 198 | command: 199 | - /hyperkube 200 | - controller-manager 201 | - --master=http://127.0.0.1:8080 202 | - --service-account-private-key-file=/etc/kubernetes/ssl/apiserver-key.pem 203 | - --root-ca-file=/etc/kubernetes/ssl/ca.pem 204 | livenessProbe: 205 | httpGet: 206 | host: 127.0.0.1 207 | path: /healthz 208 | port: 10252 209 | initialDelaySeconds: 15 210 | timeoutSeconds: 1 211 | volumeMounts: 212 | - mountPath: /etc/kubernetes/ssl 213 | name: ssl-certs-kubernetes 214 | readOnly: true 215 | - mountPath: /etc/ssl/certs 216 | name: ssl-certs-host 217 | readOnly: true 218 | hostNetwork: true 219 | volumes: 220 | - hostPath: 221 | path: /etc/kubernetes/ssl 222 | name: ssl-certs-kubernetes 223 | - hostPath: 224 | path: /usr/share/ca-certificates 225 | name: ssl-certs-host 226 | - path: /etc/kubernetes/manifests/kube-podmaster.yaml 227 | permissions: '0644' 228 | content: | 229 | apiVersion: v1 230 | kind: Pod 231 | metadata: 232 | name: kube-podmaster 233 | namespace: kube-system 234 | spec: 235 | hostNetwork: true 236 | containers: 237 | - name: scheduler-elector 238 | image: gcr.io/google_containers/podmaster:1.1 239 | command: 240 | - /podmaster 241 | - --etcd-servers=${ETCD_URL} 242 | - --key=scheduler 243 | - --whoami=${NODE_IP} 244 | - --source-file=/src/manifests/kube-scheduler.yaml 245 | - --dest-file=/dst/manifests/kube-scheduler.yaml 246 | volumeMounts: 247 | - mountPath: /src/manifests 248 | name: manifest-src 249 | readOnly: true 250 | - mountPath: /dst/manifests 251 | name: manifest-dst 252 | - name: controller-manager-elector 253 | image: gcr.io/google_containers/podmaster:1.1 254 | command: 255 | - /podmaster 256 | - --etcd-servers=${ETCD_URL} 257 | - --key=controller 258 | - --whoami=${NODE_IP} 259 | - --source-file=/src/manifests/kube-controller-manager.yaml 260 | - --dest-file=/dst/manifests/kube-controller-manager.yaml 261 | terminationMessagePath: /dev/termination-log 262 | volumeMounts: 263 | - mountPath: /src/manifests 264 | name: manifest-src 265 | readOnly: true 266 | - mountPath: /dst/manifests 267 | name: manifest-dst 268 | volumes: 269 | - hostPath: 270 | path: /srv/kubernetes/manifests 271 | name: manifest-src 272 | - hostPath: 273 | path: /etc/kubernetes/manifests 274 | name: manifest-dst 275 | - path: /etc/kubernetes/manifests/kube-scheduler.yaml 276 | permissions: '0644' 277 | content: | 278 | apiVersion: v1 279 | kind: Pod 280 | metadata: 281 | name: kube-scheduler 282 | namespace: kube-system 283 | spec: 284 | hostNetwork: true 285 | containers: 286 | - name: kube-scheduler 287 | image: ${HYPERKUBE_IMAGE}:${K8S_VER} 288 | command: 289 | - /hyperkube 290 | - scheduler 291 | - --master=http://127.0.0.1:8080 292 | livenessProbe: 293 | httpGet: 294 | host: 127.0.0.1 295 | path: /healthz 296 | port: 10251 297 | initialDelaySeconds: 15 298 | timeoutSeconds: 1 299 | - path: /etc/kubernetes/manifests/kube-system.json 300 | permissions: '0644' 301 | content: | 302 | { 303 | "apiVersion": "v1", 304 | "kind": "Namespace", 305 | "metadata": { 306 | "name": "kube-system" 307 | } 308 | } 309 | - path: /etc/kubernetes/ssl/ca.pem 310 | permissions: "0644" 311 | owner: "root" 312 | encoding: "base64" 313 | content: | 314 | ${CA_PEM} 315 | - path: /etc/kubernetes/ssl/apiserver-key.pem 316 | permissions: "0644" 317 | owner: "root" 318 | encoding: "base64" 319 | content: | 320 | ${APISERVER_KEY_PEM} 321 | - path: /etc/kubernetes/ssl/apiserver.pem 322 | permissions: "0644" 323 | owner: "root" 324 | encoding: "base64" 325 | content: | 326 | ${APISERVER_PEM} 327 | - path: /etc/kubernetes/manifests.tar.gz 328 | permissions: "0644" 329 | owner: "root" 330 | encoding: "base64" 331 | content: | 332 | ${MANIFESTS} 333 | - path: /etc/systemd/coredump.conf 334 | content: | 335 | [Coredump] 336 | Storage=none 337 | - path: /etc/profile.d/nse-function.sh 338 | permissions: '0755' 339 | owner: core 340 | content: | 341 | function nse() { 342 | docker exec -it $1 bash 343 | } 344 | - path: /home/core/.toolboxrc 345 | owner: core 346 | content: | 347 | TOOLBOX_DOCKER_IMAGE=alpine 348 | TOOLBOX_DOCKER_TAG=3.2 349 | TOOLBOX_USER=root 350 | - path: /opt/bin/download-k8s-binary 351 | permissions: '0755' 352 | content: | 353 | #!/bin/bash 354 | FILE=$1 355 | if [ ! -f /opt/bin/$FILE ]; then 356 | curl -sSL -o /opt/bin/$FILE https://storage.googleapis.com/kubernetes-release/release/${K8S_VER}/bin/linux/amd64/$FILE 357 | chmod +x /opt/bin/$FILE 358 | else 359 | # we check the version of the binary 360 | INSTALLED_VERSION=$(/opt/bin/$FILE --version=true) 361 | MATCH=$(echo "$INSTALLED_VERSION" | grep -c "${K8S_VER}") 362 | if [ $MATCH -eq 0 ]; then 363 | # the version is different 364 | curl -sSL -o /opt/bin/$FILE https://storage.googleapis.com/kubernetes-release/release/${K8S_VER}/bin/linux/amd64/$FILE 365 | chmod +x /opt/bin/$FILE 366 | fi 367 | fi 368 | - path: /opt/bin/wupiao 369 | permissions: '0755' 370 | owner: core 371 | content: | 372 | #!/usr/bin/env bash 373 | # [w]ait [u]ntil [p]ort [i]s [a]ctually [o]pen 374 | [ -n "$1" ] && \ 375 | until curl -o /dev/null -sIf http://$1; do \ 376 | sleep 1 && echo .; 377 | done; 378 | exit $? 379 | manage_etc_hosts: "localhost" 380 | --------------------------------------------------------------------------------