├── README.md ├── cloud-config-master.yml ├── cloud-config-node.yml └── examples ├── nginx.pod.json ├── nginx.rc.json └── nginx.srv.json /README.md: -------------------------------------------------------------------------------- 1 | # kubernetes-coreos-do 2 | Walkthrough for setup Kubernetes on CoreOS on DigitalOcean. 3 | 4 | ### Create Master Instance 5 | 1. Create a new droplet via DO interface, and choose following options 6 | 1. Droplet Hostname: master 7 | 1. Select Size: Any 8 | 1. Select Region: Any with private networking support 9 | 1. Available Settings: "Private Networking", "Enable User Data" 10 | 1. Put [cloud-config](https://github.com/SergeyCherepanov/kubernetes-coreos-do/blob/master/cloud-config-master.yml) to user data textarea 11 | 1. Select Image: CoreOS (stable) 12 | 1. Choose you SSH key 13 | 1. Press a "Create a Droplet" button 14 | 15 | ### Prepare cloud-config for node (minion) Instance: 16 | When the droplet will be created, login to system: 17 | 18 | ssh core@%MASTER_DROPLET_EXTERNAL_IP% 19 | 20 | Take and remember internal ip address (Note: if you use other cloud provider, network interface may be different from eth1, if is true, you also have to change it in all cloud-configs): 21 | 22 | echo $(ip addr | grep inet | grep eth1 | grep global | awk '{print $2}' | sed -e 's/\/.*$//') 23 | 24 | Replace all %MASTER_IP% placeholders in node [cloud-config](https://github.com/SergeyCherepanov/kubernetes-coreos-do/blob/master/cloud-config-node.yml) to the master ip. 25 | 26 | ### Create node (minion) Instance 27 | Create a new droplet via DO interface, and choose following options 28 | 29 | 1. Droplet Hostname: node-001 30 | 1. Select Size: Anyone you want 31 | 1. Select Region: Must be same as the master node 32 | 1. Available Settings: "Private Networking", "Enable User Data" 33 | 1. Put your changed cloud-config with MASTER IP to user data textarea 34 | 1. Select Image: CoreOS (stable) 35 | 1. Choose you SSH key 36 | 1. Press a "Create a Droplet" button 37 | 38 | ### Completion 39 | 40 | Login to your master instance: 41 | 42 | ssh core@%MASTER_DROPLET_EXTERNAL_IP% 43 | 44 | Download Kubectl: 45 | 46 | sudo /usr/bin/wget -N -P /opt/bin https://storage.googleapis.com/kubernetes-release/release/v0.15.0/bin/linux/amd64/kubectl 47 | sudo chmod +x /opt/bin/kubectl 48 | 49 | Check whether the all your nodes registered in Kubernetes 50 | 51 | /opt/bin/kubectl get nodes 52 | -------------------------------------------------------------------------------- /cloud-config-master.yml: -------------------------------------------------------------------------------- 1 | #cloud-config 2 | --- 3 | write_files: 4 | - path: /opt/etc/kubernetes-version 5 | permissions: 0644 6 | owner: root 7 | content: "v0.18.2" # change it to actual release 8 | - path: /opt/bin/downloader.sh 9 | permissions: 0744 10 | owner: root 11 | content: | 12 | #!/usr/bin/bash 13 | version=$(cat /opt/etc/kubernetes-version) 14 | if [ ! -z ${1} ] && [ ! -z ${2} ]; then 15 | if [ ! -d ${1} ]; then 16 | /usr/bin/mkdir -p ${1} 17 | fi 18 | /usr/bin/wget -N -P ${1} https://storage.googleapis.com/kubernetes-release/release/${version}/bin/linux/amd64/${2} 19 | /usr/bin/chmod +x ${1}/${2} 20 | fi 21 | - path: /opt/bin/waiter.sh 22 | owner: root 23 | content: | 24 | #!/usr/bin/bash 25 | until curl http://127.0.0.1:4001/v2/machines; do sleep 2; done 26 | - path: /opt/bin/etcd.sh 27 | owner: root 28 | content: | 29 | #!/usr/bin/bash 30 | ip=$(ip addr | grep inet | grep eth1 | grep global | awk '{print $2}' | sed -e 's/\/.*$//') 31 | /usr/bin/etcd \ 32 | --name ${ip} \ 33 | --addr ${ip}:4001 \ 34 | --bind-addr 0.0.0.0 \ 35 | --cluster-active-size 1 \ 36 | --data-dir /var/lib/etcd \ 37 | --http-read-timeout 86400 \ 38 | --peer-addr ${ip}:7001 \ 39 | --snapshot true 40 | coreos: 41 | fleet: 42 | etcd-servers: http://localhost:4001 43 | metadata: "role=master" 44 | flannel: 45 | interface: eth1 46 | units: 47 | - name: setup-network-environment.service 48 | command: start 49 | content: | 50 | [Unit] 51 | Description=Setup Network Environment 52 | Documentation=https://github.com/kelseyhightower/setup-network-environment 53 | Requires=network-online.target 54 | After=network-online.target 55 | 56 | [Service] 57 | ExecStartPre=-/usr/bin/mkdir -p /opt/bin 58 | ExecStartPre=/usr/bin/wget -N -P /opt/bin https://storage.googleapis.com/k8s/setup-network-environment 59 | ExecStartPre=/usr/bin/chmod +x /opt/bin/setup-network-environment 60 | ExecStart=/opt/bin/setup-network-environment 61 | RemainAfterExit=yes 62 | Type=oneshot 63 | - name: etcd.service 64 | command: start 65 | content: | 66 | [Unit] 67 | Description=etcd 68 | Requires=setup-network-environment.service 69 | After=setup-network-environment.service 70 | 71 | [Service] 72 | EnvironmentFile=/etc/network-environment 73 | User=etcd 74 | PermissionsStartOnly=true 75 | ExecStartPre=/usr/bin/chmod +x /opt/bin/etcd.sh 76 | ExecStart=/opt/bin/etcd.sh 77 | Restart=always 78 | RestartSec=10s 79 | - name: fleet.service 80 | command: start 81 | - name: etcd-waiter.service 82 | command: start 83 | content: | 84 | [Unit] 85 | Description=etcd waiter 86 | Wants=network-online.target 87 | Wants=etcd.service 88 | After=etcd.service 89 | After=network-online.target 90 | Before=flanneld.service 91 | 92 | [Service] 93 | ExecStartPre=/usr/bin/chmod +x /opt/bin/waiter.sh 94 | ExecStart=/usr/bin/bash /opt/bin/waiter.sh 95 | RemainAfterExit=true 96 | Type=oneshot 97 | - name: flanneld.service 98 | command: start 99 | drop-ins: 100 | - name: 50-network-config.conf 101 | content: | 102 | [Unit] 103 | Requires=etcd.service 104 | [Service] 105 | ExecStartPre=/usr/bin/etcdctl set /coreos.com/network/config '{"Network":"10.244.0.0/16", "Backend": {"Type": "vxlan"}}' 106 | - name: docker-cache.service 107 | command: start 108 | content: | 109 | [Unit] 110 | Description=Docker cache proxy 111 | Requires=early-docker.service 112 | After=early-docker.service 113 | Before=early-docker.target 114 | 115 | [Service] 116 | Restart=always 117 | TimeoutStartSec=0 118 | RestartSec=5 119 | Environment="TMPDIR=/var/tmp/" 120 | Environment="DOCKER_HOST=unix:///var/run/early-docker.sock" 121 | ExecStartPre=-/usr/bin/docker kill docker-registry 122 | ExecStartPre=-/usr/bin/docker rm docker-registry 123 | ExecStartPre=/usr/bin/docker pull quay.io/devops/docker-registry:latest 124 | # GUNICORN_OPTS is an workaround for 125 | # https://github.com/docker/docker-registry/issues/892 126 | ExecStart=/usr/bin/docker run --rm --net host --name docker-registry \ 127 | -e STANDALONE=false \ 128 | -e GUNICORN_OPTS=[--preload] \ 129 | -e MIRROR_SOURCE=https://registry-1.docker.io \ 130 | -e MIRROR_SOURCE_INDEX=https://index.docker.io \ 131 | -e MIRROR_TAGS_CACHE_TTL=1800 \ 132 | quay.io/devops/docker-registry:latest 133 | - name: docker.service 134 | drop-ins: 135 | - name: 51-docker-mirror.conf 136 | content: | 137 | [Unit] 138 | # making sure that docker-cache is up and that flanneld finished 139 | # startup, otherwise containers won't land in flannel's network... 140 | Requires=docker-cache.service flanneld.service 141 | After=docker-cache.service flanneld.service 142 | [Service] 143 | Environment=DOCKER_OPTS='--registry-mirror=http://$private_ipv4:5000' 144 | - name: kube-apiserver.service 145 | command: start 146 | content: | 147 | [Unit] 148 | Description=Kubernetes API Server 149 | Documentation=https://github.com/GoogleCloudPlatform/kubernetes 150 | Requires=etcd.service 151 | After=etcd.service 152 | 153 | [Service] 154 | ExecStartPre=/opt/bin/downloader.sh /opt/bin kube-apiserver 155 | ExecStart=/opt/bin/kube-apiserver \ 156 | --address=0.0.0.0 \ 157 | --port=8080 \ 158 | --portal_net=10.100.0.0/16 \ 159 | --etcd_servers=http://127.0.0.1:4001 \ 160 | --public_address_override=$private_ipv4 \ 161 | --logtostderr=true 162 | Restart=always 163 | RestartSec=10 164 | - name: kube-controller-manager.service 165 | command: start 166 | content: | 167 | [Unit] 168 | Description=Kubernetes Controller Manager 169 | Documentation=https://github.com/GoogleCloudPlatform/kubernetes 170 | Requires=kube-apiserver.service 171 | After=kube-apiserver.service 172 | 173 | [Service] 174 | ExecStartPre=/opt/bin/downloader.sh /opt/bin kube-controller-manager 175 | ExecStart=/opt/bin/kube-controller-manager \ 176 | --master=127.0.0.1:8080 \ 177 | --logtostderr=true 178 | Restart=always 179 | RestartSec=10 180 | - name: kube-scheduler.service 181 | command: start 182 | content: | 183 | [Unit] 184 | Description=Kubernetes Scheduler 185 | Documentation=https://github.com/GoogleCloudPlatform/kubernetes 186 | Requires=kube-apiserver.service 187 | After=kube-apiserver.service 188 | 189 | [Service] 190 | ExecStartPre=/opt/bin/downloader.sh /opt/bin kube-scheduler 191 | ExecStart=/opt/bin/kube-scheduler --master=127.0.0.1:8080 192 | Restart=always 193 | RestartSec=10 194 | - name: kube-register.service 195 | command: start 196 | content: | 197 | [Unit] 198 | Description=Kubernetes Registration Service 199 | Documentation=https://github.com/kelseyhightower/kube-register 200 | Requires=kube-apiserver.service 201 | After=kube-apiserver.service 202 | Requires=fleet.service 203 | After=fleet.service 204 | 205 | [Service] 206 | ExecStartPre=/usr/bin/wget -N -P /opt/bin https://storage.googleapis.com/k8s/kube-register 207 | ExecStartPre=/usr/bin/chmod +x /opt/bin/kube-register 208 | ExecStart=/opt/bin/kube-register \ 209 | --metadata=role=node \ 210 | --fleet-endpoint=unix:///var/run/fleet.sock \ 211 | --api-endpoint=http://127.0.0.1:8080 212 | Restart=always 213 | RestartSec=10 214 | update: 215 | group: stable 216 | reboot-strategy: off 217 | -------------------------------------------------------------------------------- /cloud-config-node.yml: -------------------------------------------------------------------------------- 1 | #cloud-config 2 | --- 3 | write_files: 4 | - path: /opt/etc/kubernetes-version 5 | permissions: 0644 6 | owner: root 7 | content: "v0.18.2" # change it to actual release 8 | - path: /opt/bin/downloader.sh 9 | permissions: 0744 10 | owner: root 11 | content: | 12 | #!/usr/bin/bash 13 | version=$(cat /opt/etc/kubernetes-version) 14 | if [ ! -z ${1} ] && [ ! -z ${2} ]; then 15 | if [ ! -d ${1} ]; then 16 | /usr/bin/mkdir -p ${1} 17 | fi 18 | /usr/bin/wget -N -P ${1} https://storage.googleapis.com/kubernetes-release/release/${version}/bin/linux/amd64/${2} 19 | /usr/bin/chmod +x ${1}/${2} 20 | fi 21 | - path: /opt/bin/kubelet-service.sh 22 | owner: root 23 | permissions: 0744 24 | content: | 25 | #!/usr/bin/bash 26 | ip=$(ip addr | grep inet | grep eth1 | grep global | awk '{print $2}' | sed -e 's/\/.*$//') 27 | /opt/bin/kubelet \ 28 | --address=0.0.0.0 \ 29 | --port=10250 \ 30 | --hostname_override=${ip} \ 31 | --api_servers=%MASTER_IP%:8080 32 | - path: "/opt/bin/register-node.sh" 33 | owner: root 34 | permissions: 0744 35 | content: | 36 | #!/bin/bash 37 | ip=$(ip addr | grep inet | grep eth1 | grep global | awk '{print $2}' | sed -e 's/\/.*$//') 38 | name=$(hostname) 39 | curl -X POST -d @- http://${1}/api/v1beta1/nodes << EOF 40 | { 41 | "id": "${ip}", 42 | "kind": "Minion", 43 | "apiVersion": "v1beta1", 44 | "resources": { 45 | "capacity": { 46 | "cpu": 1000, 47 | "memory": 1073741824 48 | } 49 | }, 50 | "externalID": "${ip}", 51 | "labels": { 52 | "name": "${name}" 53 | } 54 | } 55 | EOF 56 | coreos: 57 | fleet: 58 | etcd-servers: http://%MASTER_IP%:4001 59 | metadata: "role=node" 60 | flannel: 61 | interface: eth1 62 | etcd_endpoints: http://%MASTER_IP%:4001 63 | units: 64 | - name: etcd.service 65 | mask: true 66 | - name: fleet.service 67 | command: start 68 | - name: flanneld.service 69 | command: start 70 | drop-ins: 71 | - name: 50-network-config.conf 72 | content: | 73 | [Unit] 74 | Requires=network-online.target 75 | After=network-online.target 76 | - name: setup-network-environment.service 77 | command: start 78 | content: | 79 | [Unit] 80 | Description=Setup Network Environment 81 | Documentation=https://github.com/kelseyhightower/setup-network-environment 82 | Requires=network-online.target 83 | After=network-online.target 84 | 85 | [Service] 86 | ExecStartPre=/usr/bin/mkdir -p /opt/bin 87 | ExecStartPre=/usr/bin/wget -N -P /opt/bin https://storage.googleapis.com/k8s/setup-network-environment 88 | ExecStartPre=/usr/bin/chmod +x /opt/bin/setup-network-environment 89 | ExecStart=/opt/bin/setup-network-environment 90 | RemainAfterExit=yes 91 | Type=oneshot 92 | - name: docker.service 93 | command: start 94 | drop-ins: 95 | - name: 51-docker-mirror.conf 96 | content: | 97 | [Unit] 98 | # making sure that flanneld finished startup, otherwise containers 99 | # won't land in flannel's network... 100 | Requires=flanneld.service 101 | After=flanneld.service 102 | [Service] 103 | Environment=DOCKER_OPTS='--registry-mirror=http://%MASTER_IP%:5000' 104 | - name: kube-proxy.service 105 | command: start 106 | content: | 107 | [Unit] 108 | Description=Kubernetes Proxy 109 | Documentation=https://github.com/GoogleCloudPlatform/kubernetes 110 | Requires=setup-network-environment.service 111 | After=setup-network-environment.service 112 | 113 | [Service] 114 | ExecStartPre=/opt/bin/downloader.sh /opt/bin kube-proxy 115 | ExecStartPre=/usr/bin/chmod +x /opt/bin/kube-proxy 116 | ExecStart=/opt/bin/kube-proxy \ 117 | --master=%MASTER_IP%:8080 \ 118 | --logtostderr=true 119 | Restart=always 120 | RestartSec=10 121 | - name: kube-kubelet.service 122 | command: start 123 | content: | 124 | [Unit] 125 | Description=Kubernetes Kubelet 126 | Documentation=https://github.com/GoogleCloudPlatform/kubernetes 127 | Requires=setup-network-environment.service 128 | After=setup-network-environment.service 129 | 130 | [Service] 131 | EnvironmentFile=/etc/network-environment 132 | ExecStartPre=/opt/bin/downloader.sh /opt/bin kubelet 133 | ExecStart=/opt/bin/kubelet-service.sh 134 | Restart=always 135 | RestartSec=10 136 | - name: register-node.service 137 | command: start 138 | content: | 139 | [Unit] 140 | Description=Register Node on Master 141 | Requires=kube-kubelet.service 142 | After=kube-kubelet.service 143 | 144 | [Service] 145 | ExecStart=/opt/bin/register-node.sh %MASTER_IP%:8080 146 | RemainAfterExit=yes 147 | Type=oneshot 148 | update: 149 | group: stable 150 | reboot-strategy: off 151 | -------------------------------------------------------------------------------- /examples/nginx.pod.json: -------------------------------------------------------------------------------- 1 | { 2 | "kind": "Pod", 3 | "apiVersion": "v1beta3", 4 | "metadata": { 5 | "name": "pod-nginx-01", 6 | "labels": { 7 | "name": "pod-nginx" 8 | } 9 | }, 10 | "spec": { 11 | "containers": [{ 12 | "name": "pod-nginx-01", 13 | "image": "dockerfile/nginx", 14 | "ports": [{ 15 | "containerPort": 80 16 | }], 17 | "livenessProbe": { 18 | "enabled": true, 19 | "type": "http", 20 | "initialDelaySeconds": 30, 21 | "httpGet": { 22 | "path": "/", 23 | "port": "80" 24 | } 25 | } 26 | }] 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /examples/nginx.rc.json: -------------------------------------------------------------------------------- 1 | { 2 | 3 | "kind": "ReplicationController", 4 | "apiVersion": "v1beta3", 5 | "metadata": { 6 | "name": "nginx-controller", 7 | "labels": { 8 | "name": "rc-nginx" 9 | } 10 | }, 11 | "spec": { 12 | "replicas": 1, 13 | "selector": {"name": "pod-nginx"}, 14 | "template": { 15 | "metadata": { 16 | "labels": { 17 | "name": "pod-nginx" 18 | } 19 | }, 20 | "spec": { 21 | "containers": [{ 22 | "name": "nginx", 23 | "image": "dockerfile/nginx", 24 | "ports": [{ 25 | "name": "http-port", 26 | "containerPort": 80 27 | }] 28 | }] 29 | } 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /examples/nginx.srv.json: -------------------------------------------------------------------------------- 1 | { 2 | "kind": "Service", 3 | "apiVersion": "v1beta3", 4 | "metadata": { 5 | "name": "service-nginx", 6 | "labels": { 7 | "name": "service-nginx" 8 | } 9 | }, 10 | "spec": { 11 | "selector": { 12 | "name": "pod-nginx" 13 | }, 14 | "ports": [{"port":80, "targetPort": "http-port"}], 15 | "publicIPs":["%NODE_PUBLIC_IP%"] 16 | } 17 | } 18 | --------------------------------------------------------------------------------