├── .gitignore ├── README.md ├── config ├── ca │ ├── ca-config.json │ └── ca-csr.json └── env ├── mk_credentials ├── packer ├── Makefile ├── README.md ├── base.json ├── files │ ├── etc │ │ ├── bash_completion.d │ │ │ └── aliases.sh │ │ ├── rc.local │ │ └── tinc │ │ │ └── default │ │ │ ├── tinc-down │ │ │ └── tinc-up │ ├── lib │ │ └── systemd │ │ │ └── system │ │ │ ├── docker.service │ │ │ ├── etcd.service │ │ │ ├── k8s-apiserver.service │ │ │ ├── k8s-controller-manager.service │ │ │ ├── k8s-kubelet.service │ │ │ ├── k8s-proxy.service │ │ │ ├── k8s-scheduler.service │ │ │ ├── node_exporter.service │ │ │ ├── tinc@.service │ │ │ └── torusd.service │ └── usr │ │ └── libexec │ │ ├── etcd-remove-self │ │ └── kubernetes │ │ └── kubelet-plugins │ │ └── volume │ │ └── exec │ │ ├── 5pi.de~do-volume │ │ └── do-volume │ │ └── coreos.com~torus │ │ └── torus └── install.sh └── tf ├── README.md ├── configure.sh ├── id_rsa.pub ├── main.tf ├── terraform └── upgrade /.gitignore: -------------------------------------------------------------------------------- 1 | id_rsa 2 | rsa_key.priv 3 | server-key.pem 4 | ca-key.pem 5 | tf/terraform.tfstate 6 | tf/terraform.tfstate.backup 7 | config/generated/ 8 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # 5pi.de Infrastructure 2 | *$15/month kubernetes cluster* 3 | 4 | This is how I deploy Kubernetes to DigitalOcean. 5 | This aims to be generic enough to also deploy your infrastructure but there 6 | might be some specifics. The biggest limitation right now is that all servers 7 | are both master and minion which is discouraged for any larger infrastructure. 8 | 9 | While it's possible that this will become a generic way to setup any kind of 10 | Kubernetes infrastructures, it's not a top priority. If interested in a 11 | deployment on AWS, have a look at [kops](https://github.com/kubernetes/kops) 12 | which is more active developed. 13 | 14 | ## Overview 15 | Terraform deploys the infrastructure by setting up network, spinning up servers 16 | with an image build by Packer. 17 | 18 | This repository contains [packer](/packer) and [terraform](/tf) configuration. 19 | The directory [config](/config) contains configuration that is shared between 20 | the image and terraform templates. See the individual directories for details. 21 | 22 | The infrastructure is designed to be immutable. All state is intended to be kept 23 | on DigitalOcean volumes and all change to the host require a new image and 24 | replacement of all instances. 25 | 26 | ## Deploying a new stack 27 | First you might want to edit `config/env` to customize: 28 | 29 | - `REGION`: The DigitalOcean region your cluster should run in 30 | - `DOMAIN`: The Domain used for the cluster (see [tf](/tf) for details) 31 | - `SERVERS`: Number of servers 32 | - `SERVER_SIZE`: Size of servers 33 | - `IP_INT_PREFIX`: Prefix to use for internal private network (tinc) 34 | - `CA_FILE`: The full path (on the servers) for your TLS CA cert file 35 | 36 | You might also want to edit `config/ca/ca-csr.json`. 37 | 38 | Then run `mk_credentials` to create TLS CA and keys in `config/generated/`. 39 | 40 | Now the image can be build by running `make -C packer`. After finishing, it 41 | should print image id which is used in the next step. 42 | 43 | Enter the `tf/` directory and save a ssh public key to `id_rsa.pub`. This key 44 | will be allowed to ssh into the servers. 45 | 46 | A DigitalOcean API token is required for running Packer and Terraform as well as 47 | for attaching the DigitalOcean Volumes. This scripts expect it in `~/.do-token`. 48 | 49 | Now the stack can be spun up by running: 50 | 51 | ``` 52 | ./terraform apply -var cluster_state=new -var image=image-id-from-last-step 53 | ``` 54 | 55 | ## Updating a stack 56 | Since the servers are immutable, configuration shouldn't be changed on the 57 | systems directly. Instead a new image should be built. Once the build finished, 58 | the stack can be updated by running this in `tf/`: 59 | 60 | ``` 61 | ./upgrade apply -var image=image-id-from-build 62 | ``` 63 | 64 | This is a small wrapper around terraform to apply changes to the cluster one 65 | server at a time. It removes a server from the cluster gracefully and waits for 66 | a replacement to come up and successfully join the cluster. 67 | -------------------------------------------------------------------------------- /config/ca/ca-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "signing": { 3 | "default": { 4 | "expiry": "43800h" 5 | }, 6 | "profiles": { 7 | "server": { 8 | "expiry": "43800h", 9 | "usages": [ 10 | "signing", 11 | "key encipherment", 12 | "server auth" 13 | ] 14 | }, 15 | "client": { 16 | "expiry": "43800h", 17 | "usages": [ 18 | "signing", 19 | "key encipherment", 20 | "client auth" 21 | ] 22 | }, 23 | "peer": { 24 | "expiry": "43800h", 25 | "usages": [ 26 | "signing", 27 | "key encipherment", 28 | "server auth", 29 | "client auth" 30 | ] 31 | } 32 | } 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /config/ca/ca-csr.json: -------------------------------------------------------------------------------- 1 | { 2 | "CN": "Cluster CA", 3 | "key": { 4 | "algo": "ecdsa", 5 | "size": 256 6 | }, 7 | "names": [ 8 | { 9 | "C": "DE", 10 | "ST": "Berlin", 11 | "L": "Berlin" 12 | } 13 | ] 14 | } 15 | 16 | -------------------------------------------------------------------------------- /config/env: -------------------------------------------------------------------------------- 1 | REGION=fra1 2 | DOMAIN=int.5pi.de 3 | SERVERS=3 4 | SERVER_SIZE=512mb 5 | IP_INT_PREFIX=10.130 6 | CA_FILE=/etc/ssl/5pi-ca.pem 7 | -------------------------------------------------------------------------------- /mk_credentials: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | . config/env 4 | 5 | CONFIG=config 6 | if [[ "$#" -gt 0 ]]; then 7 | CONFIG="$1" 8 | fi 9 | 10 | mkdir -p "$CONFIG/generated" 11 | for ((i=0;i&2 10 | exit 1 11 | fi 12 | 13 | $ETCDCTL member remove "$ID" 14 | -------------------------------------------------------------------------------- /packer/files/usr/libexec/kubernetes/kubelet-plugins/volume/exec/5pi.de~do-volume/do-volume: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | exec 3>&1 # Preserve stdout 4 | exec > "/tmp/do-volume.$(id -gn).log" 2>&1 5 | 6 | DEV_PREFIX="/dev/disk/by-id/scsi-0DO_Volume_" 7 | DO_API="https://api.digitalocean.com" 8 | DO_TOKEN= 9 | 10 | CURL="curl -LSsf" 11 | DROPLET_ID=$($CURL "http://169.254.169.254/metadata/v1/id") 12 | REGION=$($CURL "http://169.254.169.254/metadata/v1/region") 13 | ACTION= 14 | 15 | ocean() { 16 | local action=$1 17 | local path=$2 18 | shift 2 19 | args=$(echo "$@" | sed 's/\([^ ][^=]*\)=\([^ ]*\)/"\1": "\2",/g;s/,$//') 20 | $CURL -X "$action" "$DO_API/$path" \ 21 | -H 'Content-Type: application/json' \ 22 | -H "Authorization: Bearer $DO_TOKEN" \ 23 | -d "{ $args }" 2>&1 # If curl suceeds, output is only stdout. If fails, only stderr 24 | } 25 | 26 | fatal() { 27 | local msg="$1" 28 | local device="${2:-}" 29 | [ -t 2 ] && echo "Fatal: $msg" >&2 30 | echo '{ "status": "Failure", "message": "'$msg'", "device": "'$device'" }' >&3 31 | exit 1 32 | } 33 | 34 | success() { 35 | local device="$1" 36 | local msg="${2:-}" 37 | [ -t 2 ] && echo "Success: $msg" >&2 38 | echo '{ "status": "Success", "message": "'$msg'", "device": "'$device'" }' >&3 39 | exit 0 40 | } 41 | 42 | init_vol() { 43 | if ! which jq > /dev/null; then 44 | fatal "jq required but not found in path" >&2 45 | fi 46 | success "" 47 | } 48 | 49 | attach_vol() { 50 | local name=$(echo "$1" | jq -r .volume) 51 | 52 | attached_id=$(ocean GET "v2/volumes?name=$name®ion=$REGION" \ 53 | | jq -r '.volumes[0].droplet_ids[0]') \ 54 | || fatal "Couldn't retrieve volume" 55 | 56 | if [[ "$attached_id" != "null" ]]; then 57 | if [[ "$attached_id" -eq "$DROPLET_ID" ]]; then 58 | success "${DEV_PREFIX}$name" "already attached" 59 | fi 60 | fatal "Volume already attached to $attached_id" 61 | fi 62 | 63 | response=$(ocean POST "v2/volumes/actions" \ 64 | "type=attach" \ 65 | "droplet_id=$DROPLET_ID" \ 66 | "volume_name=$name" \ 67 | "region=$REGION" \ 68 | ) || fatal "Couldn't attach volume: $response" 69 | success "${DEV_PREFIX}$name" 70 | } 71 | 72 | device_to_name() { 73 | local device="$1" 74 | if [[ $device != ${DEV_PREFIX}* ]]; then 75 | for f in ${DEV_PREFIX}*; do 76 | t=$(readlink -f "$f") 77 | if [[ "$t" == "$device" ]]; then 78 | device="$f" 79 | break 80 | fi 81 | done 82 | fi 83 | echo "${device#${DEV_PREFIX}}" 84 | } 85 | 86 | detach_vol() { 87 | local device="$1" 88 | local name=$(device_to_name "$device") 89 | response=$(ocean POST "v2/volumes/actions" \ 90 | "type=detach" \ 91 | "droplet_id=$DROPLET_ID" \ 92 | "volume_name=$name" \ 93 | "region=$REGION" 94 | ) || fatal "Couldn't detach volume: $response" 95 | success "$device" 96 | } 97 | 98 | mount_vol() { 99 | local mnt="$1" 100 | local device="$2" 101 | mkdir -p "$mnt" 102 | mount "$device" "$mnt" || fatal "Couldn't mount $device on $mnt" 103 | success "$device" 104 | } 105 | 106 | unmount_vol() { 107 | local mnt="$1" 108 | local device=$(grep "$mnt" /etc/mtab|cut -d' ' -f1) 109 | if [[ -z "$device" ]]; then 110 | success "$device" "Volume not mounted" 111 | fi 112 | umount "$mnt" || fatal "Couldn't unmount $mnt" 113 | success "$device" 114 | } 115 | 116 | create_vol() { 117 | local name="$1" 118 | local size="$2" 119 | shift 2 120 | local comment="$@" 121 | ocean POST "v2/volumes" \ 122 | "name=$name" \ 123 | "size_gigabytes=$size" \ 124 | "comment=$comment" \ 125 | "region=$REGION" \ 126 | || fatal "Couldn't create volume" 127 | } 128 | 129 | main() { 130 | if [ "$#" -lt 1 ]; then 131 | fatal "Missing argument. Syntax: $0 create|init|attach|detach|mount|unmount" 132 | fi 133 | ACTION=$1 134 | shift 135 | 136 | DO_TOKEN=$(cat /etc/do.token 2>/dev/null) \ 137 | || fatal "/etc/do.token missing or unreadable" 138 | 139 | if [ -n "${DEBUG:-}" ]; then 140 | CURL="curl -Lf" 141 | set -x 142 | fi 143 | 144 | case "$ACTION" in 145 | init) 146 | init_vol "$@" 147 | ;; 148 | attach) 149 | [ "$#" -lt 1 ] && fatal "Missing argument. Syntax $0 attach " 150 | attach_vol "$@" 151 | ;; 152 | detach) 153 | [ "$#" -lt 1 ] && fatal "Missing argument. Syntax $0 detach " 154 | detach_vol "$@" 155 | ;; 156 | mount) 157 | [ "$#" -lt 2 ] && fatal "Missing argument. Syntax $0 mount " 158 | mount_vol "$@" 159 | ;; 160 | unmount) 161 | [ "$#" -lt 1 ] && fatal "Missing argument. Syntax $0 unmount " 162 | unmount_vol "$@" 163 | ;; 164 | create) 165 | [ "$#" -lt 2 ] && fatal "Missing argument. Syntax $0 create [description...]" 166 | create_vol "$@" 167 | ;; 168 | *) 169 | fatal "Invalid action $ACTION" 170 | ;; 171 | esac 172 | } 173 | 174 | main "$@" 175 | -------------------------------------------------------------------------------- /packer/files/usr/libexec/kubernetes/kubelet-plugins/volume/exec/coreos.com~torus/torus: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | exec /usr/bin/torusblk \ 4 | -C https://`hostname`:2379 \ 5 | --etcd-ca-file ${CA_FILE} \ 6 | --etcd-cert-file /etc/ssl/server.pem \ 7 | --etcd-key-file /etc/ssl/server-key.pem \ 8 | "$@" 9 | -------------------------------------------------------------------------------- /packer/install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | ETCD_VERSION=3.1.1 4 | KUB_VERSION=1.4.9 5 | NODE_EXPORTER_VERSION=0.14.0-rc.1 6 | 7 | ETCD_URL="https://github.com/coreos/etcd/releases/download/v${ETCD_VERSION}/etcd-v${ETCD_VERSION}-linux-amd64.tar.gz" 8 | KUB_URL="https://github.com/kubernetes/kubernetes/releases/download/v${KUB_VERSION}/kubernetes.tar.gz" 9 | NODE_EXPORTER_URL="https://github.com/prometheus/node_exporter/releases/download/v${NODE_EXPORTER_VERSION}/node_exporter-${NODE_EXPORTER_VERSION}.linux-amd64.tar.gz" 10 | 11 | cat < /etc/buildinfo 12 | REVISION="$REVISION" 13 | BRANCH="$BRANCH" 14 | USER="$BUILD_USER" 15 | DATE="$(date -R)" 16 | EOF 17 | 18 | cat < /etc/apt/apt.conf.d/local 19 | Dpkg::Options { 20 | "--force-confdef"; 21 | "--force-confold"; 22 | } 23 | EOF 24 | export DEBIAN_FRONTEND=noninteractive 25 | 26 | sudo systemctl stop apt-daily 27 | 28 | # Setup docker repo 29 | apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D 30 | echo 'deb https://apt.dockerproject.org/repo ubuntu-xenial main' \ 31 | > /etc/apt/sources.list.d/docker.list 32 | 33 | apt-get -qy update 34 | apt-get -qy dist-upgrade 35 | 36 | # Remove packages 37 | apt-get -qy remove update-notifier-common 38 | 39 | # Install packages 40 | apt-get -qy install tinc docker-engine jq htop conntrack 41 | systemctl disable docker apt-daily 42 | 43 | # Configure tinc 44 | mkdir -p /etc/tinc/default/hosts 45 | cat < /etc/tinc/default/tinc.conf 46 | Name = \$HOST 47 | AddressFamily = ipv4 48 | Interface = tun0 49 | EOF 50 | 51 | for n in /tmp/config/generated/tinc/master*; do 52 | echo "ConnectTo = $(basename $n)" 53 | done >> /etc/tinc/default/tinc.conf 54 | 55 | . /tmp/config/env 56 | i=0 57 | for n in /tmp/config/generated/tinc/master*; do 58 | cat < /etc/tinc/default/hosts/master$i 59 | Address = master$i.$DOMAIN 60 | Subnet = $IP_INT_PREFIX.$i.0/24 61 | $(cat $n/rsa_key.pub) 62 | EOF 63 | let i++ || true 64 | done 65 | cp /tmp/config/generated/ca.pem ${CA_FILE} 66 | 67 | # Set docker options 68 | sed -i 's/^ExecStart=.*/& --storage-driver=overlay --iptables=false --ip-masq=false --bip ${IP_INT_PREFIX}.${INDEX}.1/' /lib/systemd/system/docker.service 69 | 70 | # Install etcd 71 | curl -L "$ETCD_URL" \ 72 | | tar -C /usr/bin -xzf - --strip-components=1 73 | 74 | # Install Kubernetes 75 | curl -L "$KUB_URL" \ 76 | | tar -C /tmp -xzf - kubernetes/server/kubernetes-server-linux-amd64.tar.gz 77 | tar -C /tmp -xzf /tmp/kubernetes/server/kubernetes-server-linux-amd64.tar.gz kubernetes/server/bin/hyperkube 78 | mv /tmp/kubernetes/server/bin/hyperkube /usr/bin 79 | chmod a+x /usr/bin/hyperkube 80 | ln -s hyperkube /usr/bin/kubectl 81 | 82 | # Install my patched Torus 83 | for b in torusblk torusctl torusd; do 84 | curl -L "https://github.com/discordianfish/torus/releases/download/v0.1.1-fish/$b.linux.amd64.gz" \ 85 | | zcat | install -m 755 /dev/stdin -o root -g root /usr/bin/$b 86 | done 87 | 88 | useradd -m -G docker k8s 89 | install -d -m 755 -o k8s -g k8s /etc/kubernetes 90 | openssl genrsa 2048 | install -m600 -ok8s /dev/stdin /etc/kubernetes/serviceaccount.key 91 | 92 | # Enable rc-local which sets up NAT 93 | systemctl enable rc-local.service 94 | 95 | # Install node-exporter 96 | curl -L "$NODE_EXPORTER_URL" \ 97 | | tar -C /usr/bin -xzf - --strip-components=1 98 | 99 | # Rsync stuff 100 | rsync -av --chown root:root /tmp/rootfs/ / 101 | -------------------------------------------------------------------------------- /tf/README.md: -------------------------------------------------------------------------------- 1 | # Terraform Deployment 2 | ## Prerequisits 3 | - `id\_rsa.pub`, a pub key the machine terraform runs on has access to 4 | - add private key to ssh agent 5 | - create domain 6 | 7 | ## Operations 8 | To deploy a new cluster, you need to run `terraform apply -var 9 | cluster_state=new` which will configure etcd in initial bootstrapping mode. 10 | 11 | The option `cluster_state=new` enables etcd bootstrapping and make it not wait 12 | for a fully formed cluster before continuing. This is required when deploying 13 | the cluster the first time. 14 | 15 | Since terraform doesn't provide flexible enough orchestration to support rolling 16 | upgrades, updates to the stack needs to be deployed by running `./upgrade`. 17 | -------------------------------------------------------------------------------- /tf/configure.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | exec > /tmp/configure.log 2>&1 3 | ETCDCTL_BASE="etcdctl \ 4 | --ca-file ${CA_FILE} \ 5 | --cert-file /etc/ssl/server.pem \ 6 | --key-file /etc/ssl/server-key.pem" 7 | ETCDCTL="$ETCDCTL_BASE --endpoints https://$(hostname):2379" 8 | 9 | # First fix permissions, no matter what. See hashicorp/terraform#8811 10 | chmod 640 /etc/ssl/server-key.pem 11 | chown :k8s /etc/ssl/server-key.pem 12 | set -euo pipefail 13 | . /etc/environment.tf 14 | 15 | # Add servers to /etc/hosts 16 | for ((i=0;i> /etc/hosts 19 | 20 | # Enable swap 21 | fallocate -l 4G /swapfile 22 | chmod 600 /swapfile 23 | mkswap /swapfile 24 | swapon /swapfile 25 | if ! grep /swapfile /etc/fstab; then 26 | echo '/swapfile none swap sw 0 0' >> /etc/fstab 27 | fi 28 | 29 | # Bring up tinc 30 | systemctl enable tinc@default 31 | systemctl start tinc@default 32 | 33 | # Calculate IP_INT 34 | IP_INT="${IP_INT_PREFIX}.${INDEX}.1" 35 | 36 | # Configuring etcd 37 | ETCD_SERVERS= 38 | ETCD_SERVERS_OTHER= 39 | CLUSTER= 40 | for ((i=0;i&2 57 | exit 1 58 | esac 59 | 60 | cat < /etc/environment.calc 61 | ETCD_OPTS='$ETCD_OPTS' 62 | ETCD_SERVERS='$ETCD_SERVERS' 63 | IP_INT='$IP_INT' 64 | EOF 65 | 66 | # Enabling services here, so they don't come up unconfigured 67 | for s in etcd k8s-apiserver k8s-controller-manager \ 68 | k8s-kubelet k8s-proxy k8s-scheduler docker node_exporter; do 69 | systemctl enable "$s" 70 | systemctl start "$s" --no-block 71 | done 72 | 73 | if [[ "$STATE" == "new" ]]; then 74 | exit 0 75 | fi 76 | 77 | # Add ourself to the existing cluster 78 | while ! $ETCDCTL_BASE --endpoints $ETCD_SERVERS_OTHER member add master$INDEX "https://$IP_INT:2380"; do 79 | echo "Waiting for existing cluster to be reachable" 80 | sleep 1 81 | done 82 | 83 | # Waiting for things to be ready 84 | if [ "$STATE" = "existing" ]; then 85 | while ! $ETCDCTL cluster-health; do 86 | echo "Waiting for cluster to become healthy" 87 | sleep 1 88 | done 89 | fi 90 | 91 | kubectl uncordon master$INDEX 92 | -------------------------------------------------------------------------------- /tf/id_rsa.pub: -------------------------------------------------------------------------------- 1 | ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDuyHbjFduAGOImmZS7vkPxhNeznlQ0b+iG6j73m7GSUsEzI2l+sdB6gj8GXVe+PmSEQXtJjC/wsTREbjIbHv1Lu/bGpHngzwFXA05rkOv7ai721w2QOZCzFsXF0Gw2jS32i8E5HtJDTnCQaZilQoC7UlAfVyG1QXHZaUsK7c3e4rwsuczenemiWtyf454ztCvMo+PJfE8HiKn68icxZqgXSNqhcw6qLDEvMipDJvHlb5cIz3ggw6NXD8e1yPXJiMPTrxDqqcJ2j2wp6vImbKtY7XDb1Mi+Y8SIGoRdRuCpnFEIWVrfZ/RTukqC2AdpVqUcwjP7BRGN+xyFq3v32UuD fish@xps 2 | -------------------------------------------------------------------------------- /tf/main.tf: -------------------------------------------------------------------------------- 1 | variable "domain" { 2 | type = "string" 3 | } 4 | 5 | variable "region" { 6 | type = "string" 7 | } 8 | 9 | variable "api_token" { 10 | type = "string" 11 | } 12 | 13 | variable "config" { 14 | type = "string" 15 | default = "../config" 16 | } 17 | 18 | variable "cluster_state" { 19 | type = "string" 20 | default = "existing" 21 | description = "Set this to 'new' for initial cluster creation" 22 | } 23 | 24 | variable "servers" {} 25 | 26 | variable "server_size" { 27 | type = "string" 28 | } 29 | 30 | variable "ip_int_prefix" { 31 | type = "string" 32 | } 33 | 34 | variable "image" { 35 | type = "string" 36 | } 37 | 38 | provider "digitalocean" { 39 | token = "${var.api_token}" 40 | } 41 | 42 | resource "digitalocean_ssh_key" "default" { 43 | name = "default" 44 | public_key = "${file("id_rsa.pub")}" 45 | } 46 | 47 | resource "digitalocean_floating_ip" "edge" { 48 | region = "${var.region}" 49 | lifecycle { 50 | ignore_changes = [ "droplet_id" ] 51 | } 52 | } 53 | 54 | output "edge" { 55 | value = "${digitalocean_floating_ip.edge.ip_address}" 56 | } 57 | 58 | resource "digitalocean_droplet" "master" { 59 | count = "${var.servers}" 60 | name = "master${count.index}" 61 | image = "${var.image}" 62 | region = "${var.region}" 63 | size = "${var.server_size}" 64 | private_networking = true 65 | ipv6 = true 66 | ssh_keys = ["${digitalocean_ssh_key.default.id}"] 67 | 68 | provisioner "file" { 69 | source = "configure.sh" 70 | destination = "/tmp/configure.sh" 71 | } 72 | 73 | provisioner "file" { 74 | source = "${var.config}/generated/tinc/master${count.index}/rsa_key.priv" 75 | destination = "/etc/tinc/default/rsa_key.priv" 76 | } 77 | 78 | provisioner "file" { 79 | source = "${var.config}/generated/master${count.index}.pem" 80 | destination = "/etc/ssl/server.pem" 81 | } 82 | 83 | provisioner "file" { 84 | source = "${var.config}/generated/master${count.index}-key.pem" 85 | destination = "/etc/ssl/server-key.pem" 86 | } 87 | 88 | provisioner "remote-exec" { 89 | inline = [ 90 | "cat < /etc/environment.tf", 91 | "DOMAIN=${var.domain}", 92 | "INDEX=${count.index}", 93 | "SERVERS=${var.servers}", 94 | "IP_INT_PREFIX=${var.ip_int_prefix}", 95 | "IP_PRIVATE=${self.ipv4_address_private}", 96 | "STATE=${var.cluster_state}", 97 | "TORUS_SIZE=20GiB", 98 | "EOF", 99 | "chmod a+x /tmp/configure.sh", 100 | "echo '${var.api_token}' | install -m 640 -g k8s /dev/stdin /etc/do.token", 101 | "exec /tmp/configure.sh", 102 | ] 103 | } 104 | } 105 | 106 | output "master_ips" { 107 | value = "${join(",",digitalocean_droplet.master.*.ipv4_address_public)}" 108 | } 109 | 110 | # masterX A record pointing to 'internal' IP, used for finding etcd peers 111 | resource "digitalocean_record" "master_a" { 112 | count = "${var.servers}" 113 | domain = "${var.domain}" 114 | type = "A" 115 | name = "master${count.index}" 116 | value = "${element(digitalocean_droplet.master.*.ipv4_address_private, count.index)}" 117 | } 118 | 119 | # edge A record pointing to main LB via floating IP 120 | resource "digitalocean_record" "edge_a" { 121 | count = "${var.servers}" 122 | domain = "${var.domain}" 123 | type = "A" 124 | name = "edge${count.index}" 125 | value = "${element(digitalocean_droplet.master.*.ipv4_address, count.index)}" 126 | } 127 | 128 | # *.edge A wildcard record pointing to LB via floating IP 129 | resource "digitalocean_record" "star_edge" { 130 | domain = "${var.domain}" 131 | type = "A" 132 | name = "*.edge" 133 | value = "${digitalocean_floating_ip.edge.ip_address}" 134 | } 135 | 136 | # edgeX A record pointing to the public IPs 137 | resource "digitalocean_record" "edge" { 138 | domain = "${var.domain}" 139 | type = "A" 140 | name = "edge" 141 | value = "${digitalocean_floating_ip.edge.ip_address}" 142 | } 143 | -------------------------------------------------------------------------------- /tf/terraform: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | . ../config/env 3 | 4 | export TF_VAR_region=$REGION 5 | export TF_VAR_domain=$DOMAIN 6 | export TF_VAR_api_token=$(cat ~/.do-token) 7 | export TF_VAR_servers=$SERVERS 8 | export TF_VAR_server_size=$SERVER_SIZE 9 | export TF_VAR_ip_int_prefix=$IP_INT_PREFIX 10 | 11 | terraform \ 12 | "$@" 13 | -------------------------------------------------------------------------------- /tf/upgrade: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | . ../config/env 4 | 5 | # It's safe to disable HostKeyChecking since we use no password and the input 6 | # isn't secret 7 | SSH="ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" 8 | 9 | tf_upgrade() { 10 | local i=$1 11 | shift 12 | # FIXME: This should probably use a systemd target 13 | if [[ -z "${SKIP_SHUTDOWN:-}" ]]; then 14 | $SSH "root@edge$i.$DOMAIN" "kubectl drain master$i && systemctl stop 'k8s*' etcd tinc@default" 15 | $SSH "root@edge$i.$DOMAIN" "shutdown -h now" || true # shutdown kills ssh, so ignoring error 16 | fi 17 | ./terraform "$@" \ 18 | -target "digitalocean_droplet.master[$i]" \ 19 | -target "digitalocean_record.master_a[$i]" 20 | } 21 | 22 | if [[ -n "${LIMIT:-}" ]]; then 23 | tf_upgrade "${LIMIT}" "$@" 24 | exit 0 25 | fi 26 | 27 | for ((i=0;i