├── README.md
├── bin
    ├── cleanup-gce
    └── cleanup-nomad
├── client-install.sh
├── hashistack.md
├── jobs
    ├── consul.nomad
    ├── fabio.nomad
    └── hashiapp.nomad
├── server-install.sh
└── vault
    └── hashiapp-policy.hcl


/README.md:
--------------------------------------------------------------------------------
  1 | # HashiConf EU 2016
  2 | 
  3 | ## Prerequisites
  4 | 
  5 | [Bootstrap the HashiStack on Google Compute Engine](hashistack.md)
  6 | 
  7 | Login into the controller node and checkout this repository.
  8 | 
  9 | ```
 10 | gcloud compute ssh ns-1
 11 | ```
 12 | 
 13 | ```
 14 | git clone https://github.com/kelseyhightower/hashiconf-eu-2016.git
 15 | ```
 16 | 
 17 | ```
 18 | cd hashiconf-eu-2016
 19 | ```
 20 | 
 21 | ### Create the Hashiapp Policy and Token
 22 | 
 23 | ```
 24 | vault policy-write hashiapp vault/hashiapp-policy.hcl
 25 | ```
 26 | 
 27 | ```
 28 | vault token-create \
 29 |   -policy="hashiapp" \
 30 |   -display-name="hashiapp"
 31 | ```
 32 | 
 33 | Edit `jobs/hashiapp.nomad` job
 34 | 
 35 | ```
 36 | env {
 37 |   VAULT_TOKEN = "HASHIAPP_TOKEN"
 38 |   VAULT_ADDR = "http://vault.service.consul:8200"
 39 |   HASHIAPP_DB_HOST = "CLOUD_SQL:3306"
 40 | }
 41 | ```
 42 | 
 43 | ### Create the Hashiapp Secret
 44 | 
 45 | ```
 46 | vault read mysql/creds/hashiapp
 47 | ```
 48 | 
 49 | ```
 50 | vault write secret/hashiapp jwtsecret=secret
 51 | ```
 52 | 
 53 | ## Service Discovery with Consul
 54 | 
 55 | ```
 56 | nomad plan jobs/consul.nomad
 57 | ```
 58 | 
 59 | ```
 60 | nomad run jobs/consul.nomad
 61 | ```
 62 | 
 63 | ```
 64 | nomad status consul
 65 | ```
 66 | 
 67 | ```
 68 | consul join nc-1 nc-2 nc-3 nc-4 nc-5
 69 | ```
 70 | 
 71 | ```
 72 | consul members
 73 | ```
 74 | 
 75 | ## Hashiapp Job
 76 | 
 77 | Submit the hashiapp service job.
 78 | 
 79 | ```
 80 | nomad run jobs/hashiapp.nomad
 81 | ```
 82 | 
 83 | ```
 84 | nomad status hashiapp
 85 | ```
 86 | 
 87 | #### Viewing Logs
 88 | 
 89 | ```
 90 | nomad fs -job hashiapp alloc/logs/hashiapp.stderr.0
 91 | nomad fs -job hashiapp alloc/logs/hashiapp.stdout.0
 92 | ```
 93 | 
 94 | ## Load Balancing with Fabio
 95 | 
 96 | run curl on a private intance
 97 | 
 98 | ```
 99 | nomad run jobs/fabio.nomad
100 | ```
101 | 
102 | ```
103 | nomad status fabio
104 | ```
105 | 
106 | #### Send Traffic
107 | 
108 | ```
109 | curl -H "Host: hashiapp.com" http://<loadbalancer-ip>:9999/version
110 | ```
111 | 
112 | ```
113 | while true; do curl -H "Host: hashiapp.com" http://<loadbalancer-ip>:9999/version; sleep 1; done
114 | ```
115 | 
116 | ### Scaling Up
117 | 
118 | Edit `jobs/hashiapp.nomad`
119 | 
120 | ```
121 | count = 5
122 | ```
123 | 
124 | ```
125 | nomad plan jobs/hashiapp.nomad
126 | ```
127 | 
128 | ```
129 | nomad run jobs/hashiapp.nomad
130 | ```
131 | 
132 | ### Rolling Upgrades
133 | 
134 | Edit `jobs/hashiapp.nomad`
135 | 
136 | ```
137 | source = "https://storage.googleapis.com/hashistack/hashiapp/v2.0.0/hashiapp"
138 | checksum = "sha256:372ddaeb9ac97a2eecd7dd3307bd32f8b0c188d47239f7ef6790609f9a157ca4"
139 | ```
140 | 
141 | ```
142 | nomad plan jobs/hashiapp.nomad
143 | ```
144 | 
145 | ```
146 | nomad run jobs/hashiapp.nomad
147 | ```
148 | 


--------------------------------------------------------------------------------
/bin/cleanup-gce:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | gcloud sql instances delete hashiapp -q
 4 | 
 5 | gcloud compute instances delete \
 6 |   ns-1 ns-2 ns-3 nc-1 nc-2 nc-3 nc-4 nc-5 -q
 7 | 
 8 | gcloud compute forwarding-rules delete hashistack -q
 9 | gcloud compute target-pools delete hashistack -q
10 | gcloud compute http-health-checks delete hashistack -q
11 | gcloud compute addresses delete hashistack -q
12 | gcloud compute firewall-rules delete fabio -q
13 | 


--------------------------------------------------------------------------------
/bin/cleanup-nomad:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | 
3 | nomad stop fabio
4 | nomad stop hashiapp
5 | nomad stop consul
6 | 


--------------------------------------------------------------------------------
/client-install.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | export IP_ADDRESS=$(curl -s -H "Metadata-Flavor: Google" \
 4 |   http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/0/ip)
 5 | 
 6 | apt-get update
 7 | apt-get install -y unzip dnsmasq
 8 | 
 9 | wget https://releases.hashicorp.com/nomad/0.4.1/nomad_0.4.1_linux_amd64.zip
10 | unzip nomad_0.4.1_linux_amd64.zip
11 | mv nomad /usr/local/bin/
12 | 
13 | mkdir -p /var/lib/nomad
14 | mkdir -p /etc/nomad
15 | 
16 | rm nomad_0.4.1_linux_amd64.zip
17 | 
18 | cat > client.hcl <<EOF
19 | addresses {
20 |     rpc  = "ADVERTISE_ADDR"
21 |     http = "ADVERTISE_ADDR"
22 | }
23 | 
24 | advertise {
25 |     http = "ADVERTISE_ADDR:4646"
26 |     rpc  = "ADVERTISE_ADDR:4647"
27 | }
28 | 
29 | data_dir  = "/var/lib/nomad"
30 | log_level = "DEBUG"
31 | 
32 | client {
33 |     enabled = true
34 |     servers = [
35 |       "ns-1", "ns-2", "ns-3"
36 |     ]
37 |     options {
38 |         "driver.raw_exec.enable" = "1"
39 |     }
40 | }
41 | EOF
42 | sed -i "s/ADVERTISE_ADDR/${IP_ADDRESS}/" client.hcl
43 | mv client.hcl /etc/nomad/client.hcl
44 | 
45 | cat > nomad.service <<'EOF'
46 | [Unit]
47 | Description=Nomad
48 | Documentation=https://nomadproject.io/docs/
49 | 
50 | [Service]
51 | ExecStart=/usr/local/bin/nomad agent -config /etc/nomad
52 | ExecReload=/bin/kill -HUP $MAINPID
53 | LimitNOFILE=65536
54 | 
55 | [Install]
56 | WantedBy=multi-user.target
57 | EOF
58 | mv nomad.service /etc/systemd/system/nomad.service
59 | 
60 | systemctl enable nomad
61 | systemctl start nomad
62 | 
63 | ## Setup dnsmasq
64 | 
65 | mkdir -p /etc/dnsmasq.d
66 | cat > /etc/dnsmasq.d/10-consul <<'EOF'
67 | server=/consul/127.0.0.1#8600
68 | EOF
69 | 
70 | systemctl enable dnsmasq
71 | systemctl start dnsmasq
72 | 


--------------------------------------------------------------------------------
/hashistack.md:
--------------------------------------------------------------------------------
  1 | # Bootstrap the HashiStack on Google Compute Engine
  2 | 
  3 | ## Provision MySQL
  4 | 
  5 | ```
  6 | gcloud sql instances create hashiapp \
  7 |   --tier db-n1-standard-1 \
  8 |   --activation-policy ALWAYS \
  9 |   --authorized-networks 0.0.0.0/0
 10 | ```
 11 | 
 12 | ```
 13 | gcloud sql instances set-root-password hashiapp \
 14 |   --password <password>
 15 | ```
 16 | 
 17 | ```
 18 | gcloud sql instances describe hashiapp
 19 | ```
 20 | ```
 21 | MYSQL_IPADDRESS=$(gcloud sql instances describe hashiapp \
 22 |   --format='value(ipAddresses[0].ipAddress)')
 23 | ```
 24 | 
 25 | ```
 26 | mysql -u root -h $MYSQL_IPADDRESS -p
 27 | ```
 28 | ```
 29 | Enter password: 
 30 | ```
 31 | 
 32 | ```
 33 | mysql> CREATE DATABASE hashiapp;
 34 | ```
 35 | 
 36 | ## Bootstrap a Nomad Cluster
 37 | 
 38 | This step will also install Nomad, Consul, and Vault.
 39 | 
 40 | ```
 41 | gcloud compute instances create ns-1 ns-2 ns-3 \
 42 |   --image-project ubuntu-os-cloud \
 43 |   --image-family ubuntu-1604-lts \
 44 |   --boot-disk-size 200GB \
 45 |   --machine-type n1-standard-2 \
 46 |   --can-ip-forward \
 47 |   --metadata-from-file startup-script=server-install.sh
 48 | ```
 49 | 
 50 | Complete the setup of the nomad cluser.
 51 | 
 52 | ```
 53 | gcloud compute ssh ns-1
 54 | ```
 55 | 
 56 | ```
 57 | nomad server-join ns-2 ns-3
 58 | ```
 59 | 
 60 | ```
 61 | nomad status
 62 | ```
 63 | 
 64 | ### Complete the setup of the consul cluster
 65 | 
 66 | ```
 67 | consul join ns-2 ns-3
 68 | ```
 69 | 
 70 | ```
 71 | consul members
 72 | ```
 73 | 
 74 | ### Complete the setup of the vault cluster
 75 | 
 76 | ```
 77 | export VAULT_ADDR=http://ns-1:8200
 78 | ```
 79 | 
 80 | ```
 81 | vault init
 82 | ```
 83 | ```
 84 | vault unseal
 85 | ```
 86 | ```
 87 | vault status
 88 | ```
 89 | ```
 90 | vault auth <root-token>
 91 | ```
 92 | 
 93 | Enable and configure the MySQL secret backend.
 94 | 
 95 | ```
 96 | vault mount mysql
 97 | ```
 98 | 
 99 | ```
100 | vault write mysql/config/connection \
101 | > connection_url="USERNAME:PASSWORD@tcp(HOST:PORT)/"
102 | ```
103 | 
104 | ```
105 | vault write mysql/roles/hashiapp \
106 |   sql="CREATE USER '{{name}}'@'%' IDENTIFIED BY '{{password}}';GRANT ALL PRIVILEGES ON hashiapp.* TO '{{name}}'@'%';"
107 | ```
108 | 
109 | ### Bootstrap Nomad Worker Nodes
110 | 
111 | ```
112 | gcloud compute instances create nc-1 nc-2 nc-3 nc-4 nc-5 \
113 |   --image-project ubuntu-os-cloud \
114 |   --image-family ubuntu-1604-lts \
115 |   --boot-disk-size 200GB \
116 |   --machine-type n1-standard-2 \
117 |   --can-ip-forward \
118 |   --metadata-from-file startup-script=client-install.sh
119 | ```
120 | 
121 | ```
122 | gcloud compute ssh ns-1
123 | ```
124 | 
125 | ```
126 | nomad node-status
127 | ```
128 | 
129 | ## Create L3 LoadBalancer
130 | 
131 | ```
132 | gcloud compute addresses create hashistack
133 | ```
134 | 
135 | ```
136 | gcloud compute http-health-checks create hashistack \
137 |   --port 9998 \
138 |   --request-path "/health"
139 | ```
140 | 
141 | ```
142 | gcloud compute target-pools create hashistack \
143 |   --health-check hashistack
144 | ```
145 | 
146 | ```
147 | gcloud compute target-pools add-instances hashistack \
148 |   --instances nc-1,nc-2,nc-3,nc-4,nc-5
149 | ```
150 | 
151 | ```
152 | gcloud compute addresses list
153 | ```
154 | 
155 | ```
156 | gcloud compute firewall-rules create fabio \
157 |   --allow tcp:9999 \
158 |   --source-ranges 0.0.0.0/0
159 | ```
160 | 
161 | ```
162 | gcloud compute firewall-rules create fabio-ui \
163 |   --allow tcp:9998 \
164 |   --source-ranges 0.0.0.0/0
165 | ```
166 | 
167 | ## Setup SSH Tunnel to Consul UI
168 | 
169 | ```
170 | gcloud compute ssh ns-1 \
171 |   --ssh-flag="-n" \
172 |   --ssh-flag="-N" \
173 |   --ssh-flag="-T" \
174 |   --ssh-flag="-L" \
175 |   --ssh-flag="8500:127.0.0.1:8500"
176 | ```
177 | 


--------------------------------------------------------------------------------
/jobs/consul.nomad:
--------------------------------------------------------------------------------
 1 | job "consul" {
 2 |   datacenters = ["dc1"]
 3 |   type = "system"
 4 |   update {
 5 |     stagger = "5s"
 6 |     max_parallel = 1
 7 |   }
 8 | 
 9 |   group "consul-agent" {
10 |     task "consul-agent" {
11 |       driver = "exec"
12 |       config {
13 |         command = "consul"
14 |         args = ["agent", "-data-dir", "/var/lib/consul"] 
15 |       }
16 | 
17 |       artifact {
18 |         source = "https://releases.hashicorp.com/consul/0.6.4/consul_0.6.4_linux_amd64.zip"
19 |         options {
20 |           checksum = "sha256:abdf0e1856292468e2c9971420d73b805e93888e006c76324ae39416edcf0627"
21 |         }
22 |       }
23 | 
24 |       resources {
25 |         cpu = 500
26 |         memory = 64
27 |         network {
28 |           mbits = 1
29 | 
30 |           port "server" {
31 |             static = 8300
32 |           }
33 |           port "serf_lan" {
34 |             static = 8301
35 |           }
36 |           port "serf_wan" {
37 |             static = 8302
38 |           }
39 |           port "rpc" {
40 |             static = 8400
41 |           }
42 |           port "http" {
43 |             static = 8500
44 |           }
45 |           port "dns" {
46 |             static = 8600
47 |           }
48 |         }
49 |       }
50 |     }
51 |   }
52 | }
53 | 


--------------------------------------------------------------------------------
/jobs/fabio.nomad:
--------------------------------------------------------------------------------
 1 | job "fabio" {
 2 |   datacenters = ["dc1"]
 3 |   type = "system"
 4 |   update {
 5 |     stagger = "5s"
 6 |     max_parallel = 1
 7 |   }
 8 | 
 9 |   group "fabio" {
10 |     task "fabio" {
11 |       driver = "exec"
12 |       config {
13 |         command = "fabio"
14 |       }
15 | 
16 |       artifact {
17 |         source = "https://storage.googleapis.com/hashistack/fabio/v1.2.1/fabio"
18 |         options {
19 |           checksum = "sha256:2eea2b36e10a1d5c00dc281542dcd36174d6288c6045b323767b47cfcf24f153"
20 |         }
21 |       }
22 | 
23 |       resources {
24 |         cpu = 500
25 |         memory = 64
26 |         network {
27 |           mbits = 1
28 | 
29 |           port "http" {
30 |             static = 9999
31 |           }
32 |           port "ui" {
33 |             static = 9998
34 |           }
35 |         }
36 |       }
37 |     }
38 |   }
39 | }
40 | 


--------------------------------------------------------------------------------
/jobs/hashiapp.nomad:
--------------------------------------------------------------------------------
 1 | job "hashiapp" {
 2 |   datacenters = ["dc1"]
 3 |   type = "service"
 4 | 
 5 |   update {
 6 |     stagger = "5s"
 7 |     max_parallel = 1
 8 |   }
 9 | 
10 |   group "hashiapp" {
11 |     count = 3
12 | 
13 |     task "hashiapp" {
14 |       driver = "exec"
15 |       config {
16 |         command = "hashiapp"
17 |       }
18 | 
19 |       env {
20 |         VAULT_TOKEN = ""
21 |         VAULT_ADDR = "http://vault.service.consul:8200"
22 |         HASHIAPP_DB_HOST = ""
23 |       }
24 | 
25 |       artifact {
26 |         source = "https://storage.googleapis.com/hashistack/hashiapp/v1.0.0/hashiapp"
27 |         options {
28 |           checksum = "sha256:d2127dd0356241819e4db5407284a6d100d800ebbf37b4b2b8e9aefc97f48636"
29 |         }
30 |       }
31 | 
32 |       resources {
33 |         cpu = 500
34 |         memory = 64
35 |         network {
36 |           mbits = 1
37 |           port "http" {}
38 |         }
39 |       }
40 | 
41 |       service {
42 |         name = "hashiapp"
43 |         tags = ["urlprefix-hashiapp.com/"]
44 |         port = "http"
45 |         check {
46 |           type = "http"
47 |           name = "healthz"
48 |           interval = "15s"
49 |           timeout = "5s"
50 |           path = "/healthz"
51 |         }
52 |       }
53 |     }
54 |   }
55 | }
56 | 


--------------------------------------------------------------------------------
/server-install.sh:
--------------------------------------------------------------------------------
  1 | #!/bin/bash
  2 | 
  3 | export IP_ADDRESS=$(curl -s -H "Metadata-Flavor: Google" \
  4 |   http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/0/ip)
  5 | 
  6 | apt-get update
  7 | apt-get install -y unzip dnsmasq
  8 | 
  9 | wget https://releases.hashicorp.com/nomad/0.4.1/nomad_0.4.1_linux_amd64.zip
 10 | unzip nomad_0.4.1_linux_amd64.zip
 11 | mv nomad /usr/local/bin/
 12 | 
 13 | mkdir -p /var/lib/nomad
 14 | mkdir -p /etc/nomad
 15 | 
 16 | rm nomad_0.4.1_linux_amd64.zip
 17 | 
 18 | cat > server.hcl <<EOF
 19 | addresses {
 20 |     rpc  = "ADVERTISE_ADDR"
 21 |     serf = "ADVERTISE_ADDR"
 22 | }
 23 | 
 24 | advertise {
 25 |     http = "ADVERTISE_ADDR:4646"
 26 |     rpc  = "ADVERTISE_ADDR:4647"
 27 |     serf = "ADVERTISE_ADDR:4648"
 28 | }
 29 | 
 30 | bind_addr = "0.0.0.0"
 31 | data_dir  = "/var/lib/nomad"
 32 | log_level = "DEBUG"
 33 | 
 34 | server {
 35 |     enabled = true
 36 |     bootstrap_expect = 3
 37 | }
 38 | EOF
 39 | sed -i "s/ADVERTISE_ADDR/${IP_ADDRESS}/" server.hcl
 40 | mv server.hcl /etc/nomad/server.hcl
 41 | 
 42 | cat > nomad.service <<'EOF'
 43 | [Unit]
 44 | Description=Nomad
 45 | Documentation=https://nomadproject.io/docs/
 46 | 
 47 | [Service]
 48 | ExecStart=/usr/local/bin/nomad agent -config /etc/nomad
 49 | ExecReload=/bin/kill -HUP $MAINPID
 50 | LimitNOFILE=65536
 51 | 
 52 | [Install]
 53 | WantedBy=multi-user.target
 54 | EOF
 55 | mv nomad.service /etc/systemd/system/nomad.service
 56 | 
 57 | systemctl enable nomad
 58 | systemctl start nomad
 59 | 
 60 | ## Setup consul
 61 | 
 62 | mkdir -p /var/lib/consul
 63 | 
 64 | wget https://releases.hashicorp.com/consul/0.6.4/consul_0.6.4_linux_amd64.zip
 65 | unzip consul_0.6.4_linux_amd64.zip
 66 | mv consul /usr/local/bin/consul
 67 | rm consul_0.6.4_linux_amd64.zip
 68 | 
 69 | cat > consul.service <<'EOF'
 70 | [Unit]
 71 | Description=consul
 72 | Documentation=https://consul.io/docs/
 73 | 
 74 | [Service]
 75 | ExecStart=/usr/local/bin/consul agent \
 76 |   -advertise=ADVERTISE_ADDR \
 77 |   -bind=0.0.0.0 \
 78 |   -bootstrap-expect=3 \
 79 |   -client=0.0.0.0 \
 80 |   -data-dir=/var/lib/consul \
 81 |   -server \
 82 |   -ui
 83 |   
 84 | ExecReload=/bin/kill -HUP $MAINPID
 85 | LimitNOFILE=65536
 86 | 
 87 | [Install]
 88 | WantedBy=multi-user.target
 89 | EOF
 90 | 
 91 | sed -i "s/ADVERTISE_ADDR/${IP_ADDRESS}/" consul.service
 92 | mv consul.service /etc/systemd/system/consul.service
 93 | systemctl enable consul
 94 | systemctl start consul
 95 | 
 96 | ## Setup Vault
 97 | 
 98 | wget https://releases.hashicorp.com/vault/0.6.1/vault_0.6.1_linux_amd64.zip
 99 | unzip vault_0.6.1_linux_amd64.zip
100 | mv vault /usr/local/bin/vault
101 | rm vault_0.6.1_linux_amd64.zip
102 | 
103 | mkdir -p /etc/vault
104 | 
105 | cat > /etc/vault/vault.hcl <<'EOF'
106 | backend "consul" {
107 |   advertise_addr = "http://ADVERTISE_ADDR:8200"
108 |   address = "127.0.0.1:8500"
109 |   path = "vault"
110 | }
111 | 
112 | listener "tcp" {
113 |   address = "ADVERTISE_ADDR:8200"
114 |   tls_disable = 1
115 | }
116 | EOF
117 | 
118 | sed -i "s/ADVERTISE_ADDR/${IP_ADDRESS}/" /etc/vault/vault.hcl
119 | 
120 | cat > /etc/systemd/system/vault.service <<'EOF'
121 | [Unit]
122 | Description=Vault
123 | Documentation=https://vaultproject.io/docs/
124 | 
125 | [Service]
126 | ExecStart=/usr/local/bin/vault server \
127 |   -config /etc/vault/vault.hcl
128 |   
129 | ExecReload=/bin/kill -HUP $MAINPID
130 | LimitNOFILE=65536
131 | 
132 | [Install]
133 | WantedBy=multi-user.target
134 | EOF
135 | 
136 | systemctl enable vault
137 | systemctl start vault
138 | 
139 | ## Setup dnsmasq
140 | 
141 | mkdir -p /etc/dnsmasq.d
142 | cat > /etc/dnsmasq.d/10-consul <<'EOF'
143 | server=/consul/127.0.0.1#8600
144 | EOF
145 | 
146 | systemctl enable dnsmasq
147 | systemctl start dnsmasq
148 | 


--------------------------------------------------------------------------------
/vault/hashiapp-policy.hcl:
--------------------------------------------------------------------------------
 1 | path "secret/hashiapp" {
 2 |   capabilities = ["read", "list"]
 3 | }
 4 | 
 5 | path "mysql/creds/hashiapp" {
 6 |   capabilities = ["read", "list"]
 7 | }
 8 | 
 9 | path "sys/renew/*" {
10 |   capabilities = ["update"]
11 | }
12 | 


--------------------------------------------------------------------------------