├── .gitignore ├── LICENSE ├── Makefile ├── README.md ├── mesos ├── cloud-config-agent.yml ├── cloud-config-bastion.yml └── cloud-config-master.yml ├── nomad ├── certs-config.sh ├── certs │ ├── ca-csr.json │ └── cfssl.json ├── cloud-config-agent.yml ├── cloud-config-bastion.yml ├── cloud-config-master.yml └── sleeping-beauty.hcl ├── terraform ├── agents.tf ├── bastion.tf ├── common.tf ├── masters.tf ├── network.tf └── vars.tf └── test.sh /.gitignore: -------------------------------------------------------------------------------- 1 | dhparam.pem 2 | _tmp 3 | 4 | # terraform ignores 5 | .terraform/ 6 | terraform.tfstate* 7 | .terraform.tfstate* 8 | 9 | # cert files 10 | *.pem 11 | *.csr 12 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Jessie Frazelle 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | this software and associated documentation files (the "Software"), to deal in 7 | the Software without restriction, including without limitation the rights to 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software is furnished to do so, 10 | subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 17 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 18 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 19 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 20 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | SHELL := /bin/bash 2 | 3 | null := 4 | space := ${null} ${null} 5 | ${space} := ${space} # ${ } is a space. 6 | comma := , 7 | define newline 8 | \n 9 | endef 10 | 11 | # if this session isn't interactive, then we don't want to allocate a 12 | # TTY, which would fail, but if it is interactive, we do want to attach 13 | # so that the user can send e.g. ^C through. 14 | INTERACTIVE := $(shell [ -t 0 ] && echo 1 || echo 0) 15 | ifeq ($(INTERACTIVE), 1) 16 | DOCKER_FLAGS += -t 17 | endif 18 | 19 | check_defined = \ 20 | $(strip $(foreach 1,$1, \ 21 | $(call __check_defined,$1,$(strip $(value 2))))) 22 | __check_defined = \ 23 | $(if $(value $1),, \ 24 | $(error Undefined $1$(if $2, ($2))$(if $(value @), \ 25 | required by target `$@'))) 26 | 27 | CLIENT_ID := ${AZURE_CLIENT_ID} 28 | CLIENT_SECRET := ${AZURE_CLIENT_SECRET} 29 | TENANT_ID := ${AZURE_TENANT_ID} 30 | SUBSCRIPTION_ID := ${AZURE_SUBSCRIPTION_ID} 31 | 32 | PREFIX := jessfraz 33 | LOCATION := westus2 34 | 35 | MASTER_COUNT := 5 36 | AGENT_COUNT := 18 37 | 38 | TMPDIR:=$(CURDIR)/_tmp 39 | 40 | .PHONY: ips 41 | ips: 42 | $(foreach NUM,$(shell [[ $(MASTER_COUNT) == 0 ]] || seq 5 1 $$(( $(MASTER_COUNT) + 4))),$(call get_master_ips,$(NUM))) 43 | @echo "Master IPs: $(MASTER_IPS)" 44 | $(foreach NUM,$(shell [[ $(MASTER_COUNT) == 0 ]] || seq 5 1 $$(( $(MASTER_COUNT) + 4))),$(call get_zookeeper_config_ips,$(NUM),$(shell expr $(NUM) - 4))) 45 | 46 | # Define the function to populate the MASTER_IPS variable with the 47 | # corresponding IPs of the master private_ips. 48 | # This assumes you are using three different regions and your terraform files 49 | # set the cidr ranges to 10.0.0.0/16 10.1.0.0/16 and 10.2.0.0/16 50 | # # @param number Number of the master. 51 | define get_master_ips 52 | $(eval MASTER_IPS := $(MASTER_IPS) 10.0.0.$(NUM) 10.1.0.$(NUM) 10.2.0.$(NUM)) 53 | endef 54 | 55 | define get_zookeeper_config_ips 56 | $(eval ZOOKEEPER_CONFIG_IPS := $(ZOOKEEPER_CONFIG_IPS) server.$(2)=10.0.0.$(1):2888:3888) 57 | endef 58 | 59 | .PHONY: test 60 | test: shellcheck ## Runs all the tests. 61 | 62 | .PHONY: shellcheck 63 | shellcheck: ## Run shellcheck on all scripts in the repository. 64 | docker run --rm -i $(DOCKER_FLAGS) \ 65 | --name configs-shellcheck \ 66 | -v $(CURDIR):/usr/src:ro \ 67 | --workdir /usr/src \ 68 | r.j3ss.co/shellcheck ./test.sh 69 | 70 | TERRAFORM_FLAGS = -var "client_id=$(CLIENT_ID)" \ 71 | -var "client_secret=$(CLIENT_SECRET)" \ 72 | -var "tenant_id=$(TENANT_ID)" \ 73 | -var "subscription_id=$(SUBSCRIPTION_ID)" \ 74 | -var "prefix=$(PREFIX)" \ 75 | -var "location=$(LOCATION)" \ 76 | -var "master_count=$(MASTER_COUNT)" \ 77 | -var "agent_count="$(AGENT_COUNT) 78 | 79 | MESOS_TERRAFORM_FLAGS = -v "cloud_config_master=../_tmp/mesos/cloud-config-master.yml" \ 80 | -v "cloud_config_bastion=../_tmp/mesos/cloud-config-bastion.yml" \ 81 | -v "cloud_config_agent=../_tmp/mesos/cloud-config-agent.yml" 82 | 83 | TERRAFORM_DIR=$(CURDIR)/terraform 84 | 85 | MESOS_TMPDIR=$(TMPDIR)/mesos 86 | .PHONY: mesos-config 87 | mesos-config: clean ips $(MESOS_TMPDIR) $(MESOS_TMPDIR)/cloud-config-master.yml $(MESOS_TMPDIR)/cloud-config-agent.yml $(MESOS_TMPDIR)/cloud-config-bastion.yml 88 | 89 | $(MESOS_TMPDIR): 90 | mkdir -p $(MESOS_TMPDIR) 91 | 92 | $(MESOS_TMPDIR)/cloud-config-master.yml: 93 | sed "s#ZOOKEEPER_MASTER_IPS#$(subst ${space},:2181${comma},$(MASTER_IPS)):2181#g" $(CURDIR)/mesos/cloud-config-master.yml > $@ 94 | sed -i "s#ZOOKEEPER_CONFIG_MASTER_IPS#$(subst ${space},${newline} ,$(ZOOKEEPER_CONFIG_IPS))#g" $@ 95 | 96 | $(MESOS_TMPDIR)/cloud-config-agent.yml: 97 | sed "s#ZOOKEEPER_MASTER_IPS#$(subst ${space},:2181${comma},$(MASTER_IPS)):2181#g" $(CURDIR)/mesos/cloud-config-agent.yml > $@ 98 | 99 | $(MESOS_TMPDIR)/cloud-config-bastion.yml: 100 | sed "s#ZOOKEEPER_MASTER_IPS#$(subst ${space},:2181${comma},$(MASTER_IPS)):2181#g" $(CURDIR)/mesos/cloud-config-bastion.yml > $@ 101 | 102 | .PHONY: mesos-init 103 | mesos-init: 104 | @:$(call check_defined, CLIENT_ID, Azure Client ID) 105 | @:$(call check_defined, CLIENT_SECRET, Azure Client Secret) 106 | @:$(call check_defined, TENANT_ID, Azure Tenant ID) 107 | @:$(call check_defined, SUBSCRIPTION_ID, Azure Subscription ID) 108 | cd $(TERRAFORM_DIR) && terraform init \ 109 | -var "orchestrator=mesos" \ 110 | $(MESOS_TERRAFORM_FLAGS) \ 111 | $(TERRAFORM_FLAGS) 112 | 113 | .PHONY: mesos-apply 114 | mesos-apply: mesos-init mesos-config ## Run terraform apply for mesos. 115 | cd $(TERRAFORM_DIR) && terraform apply \ 116 | -var "orchestrator=mesos" \ 117 | $(MESOS_TERRAFORM_FLAGS) \ 118 | $(TERRAFORM_FLAGS) 119 | 120 | .PHONY: mesos-destroy 121 | mesos-destroy: mesos-init ## Run terraform destroy for mesos. 122 | cd $(TERRAFORM_DIR) && terraform destroy \ 123 | -var "orchestrator=mesos" \ 124 | $(MESOS_TERRAFORM_FLAGS) \ 125 | $(TERRAFORM_FLAGS) 126 | 127 | .PHONY: nomad-init 128 | nomad-init: 129 | @:$(call check_defined, CLIENT_ID, Azure Client ID) 130 | @:$(call check_defined, CLIENT_SECRET, Azure Client Secret) 131 | @:$(call check_defined, TENANT_ID, Azure Tenant ID) 132 | @:$(call check_defined, SUBSCRIPTION_ID, Azure Subscription ID) 133 | cd $(TERRAFORM_DIR) && terraform init \ 134 | -var "orchestrator=nomad" \ 135 | $(TERRAFORM_FLAGS) 136 | 137 | .PHONY: nomad-apply 138 | nomad-apply: nomad-init nomad-config certs-config ## Run terraform apply for nomad. 139 | cd $(TERRAFORM_DIR) && terraform apply \ 140 | -var "orchestrator=nomad" \ 141 | $(TERRAFORM_FLAGS) 142 | 143 | .PHONY: nomad-destroy 144 | nomad-destroy: nomad-init ## Run terraform destroy for nomad. 145 | cd $(TERRAFORM_DIR) && terraform destroy \ 146 | -var "orchestrator=nomad" \ 147 | $(TERRAFORM_FLAGS) 148 | 149 | NOMAD_TMPDIR=$(TMPDIR)/nomad 150 | CONSUL_GOSSIP_ENCRYPTION_SECRET=$(shell docker run --rm r.j3ss.co/consul keygen) 151 | NOMAD_GOSSIP_ENCRYPTION_SECRET=$(shell docker run --rm r.j3ss.co/nomad operator keygen) 152 | .PHONY: nomad-config 153 | nomad-config: clean ips $(NOMAD_TMPDIR) $(NOMAD_TMPDIR)/cloud-config-master.yml $(NOMAD_TMPDIR)/cloud-config-agent.yml $(NOMAD_TMPDIR)/cloud-config-bastion.yml 154 | 155 | $(NOMAD_TMPDIR): 156 | mkdir -p $(NOMAD_TMPDIR) 157 | 158 | $(NOMAD_TMPDIR)/cloud-config-master.yml: 159 | sed "s#CONSUL_GOSSIP_ENCRYPTION_SECRET#$(CONSUL_GOSSIP_ENCRYPTION_SECRET)#g" $(CURDIR)/nomad/cloud-config-master.yml > $@ 160 | sed -i "s#NOMAD_GOSSIP_ENCRYPTION_SECRET#$(NOMAD_GOSSIP_ENCRYPTION_SECRET)#g" $@ 161 | sed -i "s#COMMA_SEPARATED_MASTER_IPS#$(subst ${space},${comma},$(MASTER_IPS))#g" $@ 162 | sed -i "s#NOMAD_MASTER_IPS#\"$(subst ${space},\"${comma} \",$(MASTER_IPS))\"#g" $@ 163 | 164 | $(NOMAD_TMPDIR)/cloud-config-agent.yml: 165 | sed "s#CONSUL_GOSSIP_ENCRYPTION_SECRET#$(CONSUL_GOSSIP_ENCRYPTION_SECRET)#g" $(CURDIR)/nomad/cloud-config-agent.yml > $@ 166 | sed -i "s#NOMAD_GOSSIP_ENCRYPTION_SECRET#$(NOMAD_GOSSIP_ENCRYPTION_SECRET)#g" $@ 167 | sed -i "s#COMMA_SEPARATED_MASTER_IPS#$(subst ${space},${comma},$(MASTER_IPS))#g" $@ 168 | sed -i "s#NOMAD_MASTER_IPS#\"$(subst ${space},\"${comma} \",$(MASTER_IPS))\"#g" $@ 169 | 170 | $(NOMAD_TMPDIR)/cloud-config-bastion.yml: 171 | sed "s#CONSUL_GOSSIP_ENCRYPTION_SECRET#$(CONSUL_GOSSIP_ENCRYPTION_SECRET)#g" $(CURDIR)/nomad/cloud-config-bastion.yml > $@ 172 | sed -i "s#NOMAD_GOSSIP_ENCRYPTION_SECRET#$(NOMAD_GOSSIP_ENCRYPTION_SECRET)#g" $@ 173 | sed -i "s#COMMA_SEPARATED_MASTER_IPS#$(subst ${space},${comma},$(MASTER_IPS))#g" $@ 174 | sed -i "s#NOMAD_MASTER_IPS#\"$(subst ${space},\"${comma} \",$(MASTER_IPS))\"#g" $@ 175 | 176 | CERTDIR=$(CURDIR)/nomad/certs 177 | 178 | DOCKER_CFSSL=docker run --rm -i -v $(CERTDIR):$(CERTDIR) -w $(CERTDIR) 179 | CFSSL_CMD=$(DOCKER_CFSSL) r.j3ss.co/cfssl 180 | CFSSLJSON_CMD=$(DOCKER_CFSSL) --entrypoint cfssljson r.j3ss.co/cfssl 181 | 182 | .PHONY: consul-certs 183 | consul-certs: 184 | # generate a private CA certificate (consul-ca.pem) and key (consul-ca-key.pem) 185 | $(CFSSL_CMD) gencert -initca $(CERTDIR)/ca-csr.json | $(CFSSLJSON_CMD) -bare consul-ca 186 | # generate a certificate for all the Consul servers in a specific region (global) 187 | echo '{"key":{"algo":"rsa","size":2048}}' | $(CFSSL_CMD) gencert \ 188 | -ca=consul-ca.pem -ca-key=consul-ca-key.pem -config=cfssl.json \ 189 | -hostname="server.global.consul,localhost,127.0.0.1,10.0.0.5" - | \ 190 | $(CFSSLJSON_CMD) -bare consul-server 191 | # generate a certificate for all the Consul clients in a specific region (global) 192 | echo '{"key":{"algo":"rsa","size":2048}}' | $(CFSSL_CMD) gencert \ 193 | -ca=consul-ca.pem -ca-key=consul-ca-key.pem -config=cfssl.json \ 194 | -hostname="client.global.consul,localhost,127.0.0.1,10.0.0.5" - | \ 195 | $(CFSSLJSON_CMD) -bare consul-client 196 | # generate a certificate for the cli 197 | echo '{"key":{"algo":"rsa","size":2048}}' | $(CFSSL_CMD) gencert \ 198 | -ca=consul-ca.pem -ca-key=consul-ca-key.pem -profile=client - | \ 199 | $(CFSSLJSON_CMD) -bare consul-cli 200 | 201 | .PHONY: nomad-certs 202 | nomad-certs: 203 | # generate a private CA certificate (cnomad-ca.pem) and key (nomad-ca-key.pem) 204 | $(CFSSL_CMD) gencert -initca $(CERTDIR)/ca-csr.json | $(CFSSLJSON_CMD) -bare nomad-ca 205 | # generate a certificate for all the Nomad servers in a specific region (global) 206 | echo '{"key":{"algo":"rsa","size":2048}}' | $(CFSSL_CMD) gencert \ 207 | -ca=nomad-ca.pem -ca-key=nomad-ca-key.pem -config=cfssl.json \ 208 | -hostname="server.global.nomad,localhost,127.0.0.1,10.0.0.5" - | \ 209 | $(CFSSLJSON_CMD) -bare nomad-server 210 | # generate a certificate for all the Nomad clients in a specific region (global) 211 | echo '{"key":{"algo":"rsa","size":2048}}' | $(CFSSL_CMD) gencert \ 212 | -ca=nomad-ca.pem -ca-key=nomad-ca-key.pem -config=cfssl.json \ 213 | -hostname="client.global.nomad,localhost,127.0.0.1,10.0.0.5" - | \ 214 | $(CFSSLJSON_CMD) -bare nomad-client 215 | # generate a certificate for the cli 216 | echo '{"key":{"algo":"rsa","size":2048}}' | $(CFSSL_CMD) gencert \ 217 | -ca=nomad-ca.pem -ca-key=nomad-ca-key.pem -profile=client - | \ 218 | $(CFSSLJSON_CMD) -bare nomad-cli 219 | 220 | .PHONY: certs-config 221 | certs-config: consul-certs nomad-certs 222 | CERTDIR=$(CERTDIR) NOMAD_TMPDIR=$(NOMAD_TMPDIR) ./nomad/certs-config.sh 223 | 224 | .PHONY: update 225 | update: update-terraform ## Run all update targets. 226 | 227 | TERRAFORM_BINARY:=$(shell which terraform || echo "/usr/local/bin/terraform") 228 | TMP_TERRAFORM_BINARY:=/tmp/terraform 229 | .PHONY: update-terraform 230 | update-terraform: ## Update terraform binary locally from the docker container. 231 | @echo "Updating terraform binary..." 232 | $(shell docker run --rm --entrypoint bash r.j3ss.co/terraform -c "cd \$\$$(dirname \$\$$(which terraform)) && tar -Pc terraform" | tar -xvC $(dir $(TMP_TERRAFORM_BINARY)) > /dev/null) 233 | sudo mv $(TMP_TERRAFORM_BINARY) $(TERRAFORM_BINARY) 234 | sudo chmod +x $(TERRAFORM_BINARY) 235 | @echo "Update terraform binary: $(TERRAFORM_BINARY)" 236 | @terraform version 237 | 238 | .PHONY: clean 239 | clean: ## Cleans up any unneeded files. 240 | $(RM) -r $(TMPDIR) 241 | sudo $(RM) $(CERTDIR)/*.pem 242 | sudo $(RM) $(CERTDIR)/*.csr 243 | 244 | .PHONY: help 245 | help: 246 | @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' 247 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # azure-terraform-cluster 2 | 3 | Scripts to create a minimal mesos or nomad cluster on Azure using terraform. 4 | 5 | **Table of Contents** 6 | 7 | 8 | * [Overview](README.md#overview) 9 | * [Nomad](README.md#nomad) 10 | * [Mesos](README.md#mesos) 11 | * [Using the Makefile](README.md#using-the-makefile) 12 | * [Spinning up a cluster](README.md#spinning-up-a-cluster) 13 | * [Azure credentials setup](README.md#azure-credentials-setup) 14 | * [Creating a service principal](README.md#creating-a-service-principal) 15 | 16 | ## Overview 17 | 18 | This creates `5` master and `10` agents in a mesos or nomad cluster. 19 | You can change the number of masters with `MASTER_COUNT` and the number of 20 | agents with `AGENT_COUNT`. 21 | 22 | It also creates a "jumpbox" or ["bastion host"](https://en.wikipedia.org/wiki/Bastion_host) 23 | since all the masters and agents are not publicly accessible. 24 | 25 | If you want to ssh into the internal nodes you must first go through the 26 | bastion on node. 27 | 28 | The username on the nodes is `vmuser`. 29 | 30 | The base image for all the virtual machines 31 | is [CoreOS Container Linux](https://coreos.com/os/docs/latest/). 32 | 33 | The cloud-config.yml files defines the servers running on each of the hosts. 34 | The hosts are designed to be super minimal. This is done via the 35 | [CoreOS Cloud Configuration](https://coreos.com/os/docs/latest/cloud-config.html). 36 | 37 | All internal IPs are in the block 10.x.0.x. 38 | 39 | These are multi-region so `westus2` is 10.1.0.x and so on and so forth for the 40 | other regions. 41 | 42 | So the first 5 in the block are the masters: `10.x.0.5-9`. And the agents 43 | follow after starting at `10.x.0.10`. 44 | 45 | The IPs get populated programmatically in the terraform files. But they are 46 | also generated and substituted into the `cloud-config*` files via the Makefile. 47 | 48 | 49 | ### Nomad 50 | 51 | On each server there are helpers in the `bashrc` so that you can run `nomad` 52 | and `consul` commands from the host. This just uses a container under the hood 53 | and you can see the alias with `type nomad`. 54 | 55 | Both consul and nomad are setup with TLS which is generated by the Makefile 56 | when the cluster is created. 57 | 58 | On the **bastion server** we run: 59 | 60 | - Nomad Dashboard: **This is opened on port 8080 by default so you will want to 61 | change that if you want your cluster to be secure.** 62 | This is only done so it is easy to demo. 63 | 64 | On the **masters** we run: 65 | 66 | - Consul 67 | - Nomad Server 68 | 69 | On the **agents** we run: 70 | 71 | - Nomad Agent 72 | 73 | ### Mesos 74 | 75 | On the **bastion server** we run: 76 | 77 | - [Mesos Marathon](https://mesosphere.github.io/marathon/): **This is opened on 78 | port 8080 by default so you will want to change that if you want your cluster 79 | to be secure.** This is only done so it is easy to demo. 80 | 81 | On the **masters** we run: 82 | 83 | - Mesos Master 84 | - Zookeeper 85 | 86 | 87 | On the **agents** we run: 88 | 89 | - Mesos Agent 90 | 91 | 92 | ## Using the `Makefile` 93 | 94 | You will need to set the following environment variables: 95 | 96 | - `AZURE_CLIENT_ID` 97 | - `AZURE_CLIENT_SECRET` 98 | - `AZURE_TENANT_ID` 99 | - `AZURE_SUBSCRIPTION_ID` 100 | 101 | See [creating a service principal](#creating-a-service-principal) on how to get 102 | these values. 103 | 104 | ```console 105 | $ make help 106 | mesos-apply Run terraform apply for mesos. 107 | mesos-destroy Run terraform destroy for mesos. 108 | nomad-apply Run terraform apply for nomad. 109 | nomad-destroy Run terraform destroy for nomad. 110 | shellcheck Run shellcheck on all scripts in the repository. 111 | test Runs all the tests. 112 | update-terraform Update terraform binary locally from the docker container. 113 | update Run all update targets. 114 | ``` 115 | 116 | ### Spinning up a cluster 117 | 118 | This is as simple as: 119 | 120 | ```console 121 | $ AZURE_CLIENT_ID=0000 AZURE_CLIENT_SECRET=0000 AZURE_TENANT_ID=0000 AZURE_SUBSCRIPTION_ID=0000 \ 122 | make az-apply 123 | ``` 124 | 125 | ## Azure credentials setup 126 | 127 | You need a service principal in order to use the `Makefile`. 128 | 129 | ### Creating a service principal 130 | 131 | ```console 132 | $ az ad sp create-for-rbac --role="Contributor" --scopes="/subscriptions/SUBSCRIPTION_ID" 133 | ``` 134 | 135 | The command will output the following: 136 | 137 | ```json 138 | { 139 | "appId": "00000000-0000-0000-0000-000000000000", 140 | "displayName": "azure-cli-2017-06-05-10-41-15", 141 | "name": "http://azure-cli-2017-06-05-10-41-15", 142 | "password": "0000-0000-0000-0000-000000000000", 143 | "tenant": "00000000-0000-0000-0000-000000000000" 144 | } 145 | ``` 146 | 147 | These values map to the `Makefile` variables like so: 148 | 149 | - `appId` is the `AZURE_CLIENT_ID` defined above 150 | - `password` is the `AZURE_CLIENT_SECRET` defined above 151 | - `tenant` is the `AZURE_TENANT_ID` defined above 152 | 153 | **Reference docs:** 154 | 155 | - `terraform` docs on setting up authentication: 156 | [here](https://www.terraform.io/docs/providers/azurerm/authenticating_via_service_principal.html). 157 | 158 | -------------------------------------------------------------------------------- /mesos/cloud-config-agent.yml: -------------------------------------------------------------------------------- 1 | #cloud-config 2 | 3 | --- 4 | coreos: 5 | etcd: 6 | discovery: "https://discovery.etcd.io/1d77508dc31cf372d517ed2e96e8f5cf" 7 | advertise-client-urls: "http://$private_ipv4:2379" 8 | initial-advertise-peer-urls: "http://$private_ipv4:2380" 9 | listen_client_urls: "http://0.0.0.0:2379" 10 | listen_peer_urls: "http://$private_ipv4:2380" 11 | units: 12 | - name: mesos_executors.slice 13 | command: start 14 | enable: true 15 | content: | 16 | [Unit] 17 | Description=Mesos Executors Slice 18 | - name: mesos-agent.service 19 | command: start 20 | enable: true 21 | content: | 22 | [Unit] 23 | Description=Mesos Agent 24 | After=mesos-master.service 25 | Requires=docker.service mesos_executors.slice 26 | 27 | [Service] 28 | Restart=on-failure 29 | RestartSec=20 30 | TimeoutStartSec=0 31 | ExecStartPre=-/usr/bin/docker rm -f mesos-agent 32 | ExecStartPre=-/usr/bin/docker pull mesosphere/mesos-agent:1.7.0 33 | ExecStart=/usr/bin/sh -c "/usr/bin/docker run \ 34 | --name=mesos-agent \ 35 | --net=host \ 36 | --privileged \ 37 | -v /sys/fs/cgroup:/sys/fs/cgroup \ 38 | -v /tmp:/tmp \ 39 | -v /run:/run \ 40 | -v /usr/bin/docker:/usr/bin/docker:ro \ 41 | -v /var/run/docker.sock:/var/run/docker.sock \ 42 | -p 5051:5051 \ 43 | -e MESOS_IP=$private_ipv4 \ 44 | -e MESOS_HOSTNAME=$private_ipv4 \ 45 | -e MESOS_MASTER=zk://ZOOKEEPER_MASTER_IPS/mesos \ 46 | -e MESOS_LOG_DIR=/var/log/mesos/agent \ 47 | -e MESOS_WORK_DIR=/var/lib/mesos/agent \ 48 | -e MESOS_EXECUTOR_REGISTRATION_TIMEOUT=5mins \ 49 | mesosphere/mesos-agent:1.7.0 \ 50 | --containerizers=docker,mesos \ 51 | --image_providers=docker \ 52 | --isolation=filesystem/linux,docker/runtime \ 53 | --executor_shutdown_grace_period=60secs \ 54 | --docker_stop_timeout=50secs" 55 | ExecStop=/usr/bin/docker stop mesos-agent 56 | 57 | [Install] 58 | WantedBy=multi-user.target 59 | -------------------------------------------------------------------------------- /mesos/cloud-config-bastion.yml: -------------------------------------------------------------------------------- 1 | #cloud-config 2 | 3 | --- 4 | coreos: 5 | etcd: 6 | discovery: "https://discovery.etcd.io/1d77508dc31cf372d517ed2e96e8f5cf" 7 | advertise-client-urls: "http://$private_ipv4:2379" 8 | initial-advertise-peer-urls: "http://$private_ipv4:2380" 9 | listen_client_urls: "http://0.0.0.0:2379" 10 | listen_peer_urls: "http://$private_ipv4:2380" 11 | units: 12 | - name: marathon.service 13 | command: start 14 | enable: true 15 | content: |- 16 | [Unit] 17 | Description=Marathon 18 | Requires=docker.service 19 | 20 | [Service] 21 | Restart=on-failure 22 | RestartSec=20 23 | TimeoutStartSec=0 24 | ExecStartPre=-/usr/bin/docker rm -f marathon 25 | ExecStartPre=-/usr/bin/docker pull mesosphere/marathon:v1.7.166 26 | ExecStart=/usr/bin/sh -c "/usr/bin/docker run \ 27 | --name marathon \ 28 | -e LIBPROCESS_PORT=9090 \ 29 | --net host \ 30 | mesosphere/marathon:v1.7.166 \ 31 | --master zk://ZOOKEEPER_MASTER_IPS/mesos \ 32 | --zk zk://ZOOKEEPER_MASTER_IPS/marathon \ 33 | --checkpoint \ 34 | --hostname $private_ipv4" 35 | ExecStop=/usr/bin/docker stop marathon 36 | 37 | [Install] 38 | WantedBy=multi-user.target 39 | -------------------------------------------------------------------------------- /mesos/cloud-config-master.yml: -------------------------------------------------------------------------------- 1 | #cloud-config 2 | 3 | --- 4 | coreos: 5 | etcd: 6 | discovery: "https://discovery.etcd.io/1d77508dc31cf372d517ed2e96e8f5cf" 7 | advertise-client-urls: "http://$private_ipv4:2379" 8 | initial-advertise-peer-urls: "http://$private_ipv4:2380" 9 | listen_client_urls: "http://0.0.0.0:2379" 10 | listen_peer_urls: "http://$private_ipv4:2380" 11 | units: 12 | - name: zookeeper.service 13 | command: start 14 | enable: true 15 | content: | 16 | [Unit] 17 | Description=Zookeeper 18 | After=docker.service 19 | Requires=docker.service 20 | 21 | [Service] 22 | Restart=on-failure 23 | RestartSec=20 24 | TimeoutStartSec=0 25 | ExecStartPre=-/usr/bin/docker rm -f zookeeper 26 | ExecStartPre=-/usr/bin/docker pull r.j3ss.co/zookeeper:3.4 27 | ExecStart=/usr/bin/sh -c "/usr/bin/docker run \ 28 | --net=host \ 29 | --name=zookeeper \ 30 | -v /opt/zookeeper/conf/zoo.cfg:/opt/zookeeper/conf/zoo.cfg \ 31 | -v /var/lib/zookeeper:/var/lib/zookeeper \ 32 | r.j3ss.co/zookeeper:3.4" 33 | ExecStop=/usr/bin/docker stop zookeeper 34 | 35 | [Install] 36 | WantedBy=multi-user.target 37 | - name: mesos-master.service 38 | command: start 39 | enable: true 40 | content: | 41 | [Unit] 42 | Description=Mesos Master 43 | After=zookeeper.service 44 | Requires=docker.service 45 | 46 | [Service] 47 | Restart=on-failure 48 | RestartSec=20 49 | TimeoutStartSec=0 50 | ExecStartPre=-/usr/bin/docker rm -f mesos-master 51 | ExecStartPre=-/usr/bin/docker pull mesosphere/mesos-master:1.7.0 52 | ExecStart=/usr/bin/sh -c "/usr/bin/docker run \ 53 | --name=mesos-master \ 54 | --net=host \ 55 | -e MESOS_IP=$private_ipv4 \ 56 | -e MESOS_HOSTNAME=$private_ipv4 \ 57 | -e MESOS_CLUSTER=mesos-cluster \ 58 | -e MESOS_ZK=zk://ZOOKEEPER_MASTER_IPS/mesos \ 59 | -e MESOS_LOG_DIR=/var/log/mesos/master \ 60 | -e MESOS_WORK_DIR=/var/lib/mesos/master \ 61 | -e MESOS_QUORUM=2 \ 62 | mesosphere/mesos-master:1.7.0" 63 | ExecStop=/usr/bin/docker stop mesos-master 64 | 65 | [Install] 66 | WantedBy=multi-user.target 67 | write_files: 68 | - path: "/opt/zookeeper/conf/zoo.cfg" 69 | permissions: "0644" 70 | owner: "root" 71 | content: | 72 | # The number of milliseconds of each tick 73 | tickTime=2000 74 | # The number of ticks that the initial 75 | # synchronization phase can take 76 | initLimit=90 77 | # The number of ticks that can pass between 78 | # sending a request and getting an acknowledgement 79 | syncLimit=30 80 | # the directory where the snapshot is stored. 81 | # do not use /tmp for storage, /tmp here is just 82 | # example sakes. 83 | dataDir=/var/lib/zookeeper 84 | # the port at which the clients will connect 85 | clientPort=2181 86 | # the maximum number of client connections. 87 | # increase this if you need to handle more clients 88 | #maxClientCnxns=60 89 | # 90 | # Be sure to read the maintenance section of the 91 | # administrator guide before turning on autopurge. 92 | # 93 | # http://zookeeper.apache.org/doc/current/zookeeperAdmin.html#sc_maintenance 94 | # 95 | # The number of snapshots to retain in dataDir 96 | #autopurge.snapRetainCount=3 97 | # Purge task interval in hours 98 | # Set to "0" to disable auto purge feature 99 | #autopurge.purgeInterval=1 100 | ZOOKEEPER_CONFIG_MASTER_IPS 101 | - path: "/var/lib/zookeeper/myid" 102 | permissions: "0644" 103 | owner: "root" 104 | content: $private_ipv4 105 | -------------------------------------------------------------------------------- /nomad/certs-config.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | set -o pipefail 4 | 5 | # Make sure we have a certdir. 6 | # This is filled by the Makefile. 7 | if [[ -z "$CERTDIR" ]]; then 8 | echo "Set the CERTDIR env var." 9 | exit 1 10 | fi 11 | 12 | # Make sure we have the tmpdir set as well. 13 | # This is filled by the Makefile. 14 | if [[ -z "$NOMAD_TMPDIR" ]]; then 15 | echo "Set the NOMAD_TMPDIR env var." 16 | exit 1 17 | fi 18 | 19 | # Chown the certs since docker will own them as root. 20 | sudo chown -R "${USER}:${USER}" "$CERTDIR" 21 | 22 | # Create variables to hold base64+gzip encoded values of the files. 23 | CONSUL_CA=$(gzip -c "${CERTDIR}/consul-ca.pem" | base64 -w0) 24 | 25 | CONSUL_SERVER_KEY=$(sudo gzip -c "${CERTDIR}/consul-server-key.pem" | base64 -w0) 26 | CONSUL_SERVER_CERT=$(gzip -c "${CERTDIR}/consul-server.pem" | base64 -w0) 27 | 28 | CONSUL_CLI_KEY=$(sudo gzip -c "${CERTDIR}/consul-cli-key.pem" | base64 -w0) 29 | CONSUL_CLI_CERT=$(gzip -c "${CERTDIR}/consul-cli.pem" | base64 -w0) 30 | 31 | NOMAD_CA=$(gzip -c "${CERTDIR}/nomad-ca.pem" | base64 -w0) 32 | 33 | NOMAD_SERVER_KEY=$(sudo gzip -c "${CERTDIR}/nomad-server-key.pem" | base64 -w0) 34 | NOMAD_SERVER_CERT=$(gzip -c "${CERTDIR}/nomad-server.pem" | base64 -w0) 35 | 36 | NOMAD_CLIENT_KEY=$(sudo gzip -c "${CERTDIR}/nomad-client-key.pem" | base64 -w0) 37 | NOMAD_CLIENT_CERT=$(gzip -c "${CERTDIR}/nomad-client.pem" | base64 -w0) 38 | 39 | NOMAD_CLI_KEY=$(sudo gzip -c "${CERTDIR}/nomad-cli-key.pem" | base64 -w0) 40 | NOMAD_CLI_CERT=$(gzip -c "${CERTDIR}/nomad-cli.pem" | base64 -w0) 41 | 42 | # Add the certs to the bastion config. 43 | cat <<-EOF >> "${NOMAD_TMPDIR}/cloud-config-bastion.yml" 44 | - path: "/etc/consul/certs/ca.pem" 45 | permissions: "0644" 46 | owner: "root" 47 | encoding: "gzip+base64" 48 | content: | 49 | ${CONSUL_CA} 50 | - path: "/etc/consul/certs/cli-key.pem" 51 | permissions: "0644" 52 | owner: "root" 53 | encoding: "gzip+base64" 54 | content: | 55 | ${CONSUL_CLI_KEY} 56 | - path: "/etc/consul/certs/cli.pem" 57 | permissions: "0644" 58 | owner: "root" 59 | encoding: "gzip+base64" 60 | content: | 61 | ${CONSUL_CLI_CERT} 62 | - path: "/etc/nomad/certs/ca.pem" 63 | permissions: "0644" 64 | owner: "root" 65 | encoding: "gzip+base64" 66 | content: | 67 | ${NOMAD_CA} 68 | - path: "/etc/nomad/certs/cli-key.pem" 69 | permissions: "0644" 70 | owner: "root" 71 | encoding: "gzip+base64" 72 | content: | 73 | ${NOMAD_CLI_KEY} 74 | - path: "/etc/nomad/certs/cli.pem" 75 | permissions: "0644" 76 | owner: "root" 77 | encoding: "gzip+base64" 78 | content: | 79 | ${NOMAD_CLI_CERT} 80 | - path: "/etc/nomad/certs/client-key.pem" 81 | permissions: "0644" 82 | owner: "root" 83 | encoding: "gzip+base64" 84 | content: | 85 | ${NOMAD_CLIENT_KEY} 86 | - path: "/etc/nomad/certs/client.pem" 87 | permissions: "0644" 88 | owner: "root" 89 | encoding: "gzip+base64" 90 | content: | 91 | ${NOMAD_CLIENT_CERT} 92 | EOF 93 | 94 | # Add the certs to the master config. 95 | cat <<-EOF >> "${NOMAD_TMPDIR}/cloud-config-master.yml" 96 | - path: "/etc/consul/certs/ca.pem" 97 | permissions: "0644" 98 | owner: "root" 99 | encoding: "gzip+base64" 100 | content: | 101 | ${CONSUL_CA} 102 | - path: "/etc/consul/certs/cli-key.pem" 103 | permissions: "0644" 104 | owner: "root" 105 | encoding: "gzip+base64" 106 | content: | 107 | ${CONSUL_CLI_KEY} 108 | - path: "/etc/consul/certs/cli.pem" 109 | permissions: "0644" 110 | owner: "root" 111 | encoding: "gzip+base64" 112 | content: | 113 | ${CONSUL_CLI_CERT} 114 | - path: "/etc/consul/certs/server-key.pem" 115 | permissions: "0644" 116 | owner: "root" 117 | encoding: "gzip+base64" 118 | content: | 119 | ${CONSUL_SERVER_KEY} 120 | - path: "/etc/consul/certs/server.pem" 121 | permissions: "0644" 122 | owner: "root" 123 | encoding: "gzip+base64" 124 | content: | 125 | ${CONSUL_SERVER_CERT} 126 | - path: "/etc/nomad/certs/ca.pem" 127 | permissions: "0644" 128 | owner: "root" 129 | encoding: "gzip+base64" 130 | content: | 131 | ${NOMAD_CA} 132 | - path: "/etc/nomad/certs/cli-key.pem" 133 | permissions: "0644" 134 | owner: "root" 135 | encoding: "gzip+base64" 136 | content: | 137 | ${NOMAD_CLI_KEY} 138 | - path: "/etc/nomad/certs/cli.pem" 139 | permissions: "0644" 140 | owner: "root" 141 | encoding: "gzip+base64" 142 | content: | 143 | ${NOMAD_CLI_CERT} 144 | - path: "/etc/nomad/certs/server-key.pem" 145 | permissions: "0644" 146 | owner: "root" 147 | encoding: "gzip+base64" 148 | content: | 149 | ${NOMAD_SERVER_KEY} 150 | - path: "/etc/nomad/certs/server.pem" 151 | permissions: "0644" 152 | owner: "root" 153 | encoding: "gzip+base64" 154 | content: | 155 | ${NOMAD_SERVER_CERT} 156 | EOF 157 | 158 | # Add the certs to the agent config. 159 | cat <<-EOF >> "${NOMAD_TMPDIR}/cloud-config-agent.yml" 160 | - path: "/etc/consul/certs/ca.pem" 161 | permissions: "0644" 162 | owner: "root" 163 | encoding: "gzip+base64" 164 | content: | 165 | ${CONSUL_CA} 166 | - path: "/etc/consul/certs/cli-key.pem" 167 | permissions: "0644" 168 | owner: "root" 169 | encoding: "gzip+base64" 170 | content: | 171 | ${CONSUL_CLI_KEY} 172 | - path: "/etc/consul/certs/cli.pem" 173 | permissions: "0644" 174 | owner: "root" 175 | encoding: "gzip+base64" 176 | content: | 177 | ${CONSUL_CLI_CERT} 178 | - path: "/etc/nomad/certs/ca.pem" 179 | permissions: "0644" 180 | owner: "root" 181 | encoding: "gzip+base64" 182 | content: | 183 | ${NOMAD_CA} 184 | - path: "/etc/nomad/certs/cli-key.pem" 185 | permissions: "0644" 186 | owner: "root" 187 | encoding: "gzip+base64" 188 | content: | 189 | ${NOMAD_CLI_KEY} 190 | - path: "/etc/nomad/certs/cli.pem" 191 | permissions: "0644" 192 | owner: "root" 193 | encoding: "gzip+base64" 194 | content: | 195 | ${NOMAD_CLI_CERT} 196 | - path: "/etc/nomad/certs/client-key.pem" 197 | permissions: "0644" 198 | owner: "root" 199 | encoding: "gzip+base64" 200 | content: | 201 | ${NOMAD_CLIENT_KEY} 202 | - path: "/etc/nomad/certs/client.pem" 203 | permissions: "0644" 204 | owner: "root" 205 | encoding: "gzip+base64" 206 | content: | 207 | ${NOMAD_CLIENT_CERT} 208 | EOF 209 | -------------------------------------------------------------------------------- /nomad/certs/ca-csr.json: -------------------------------------------------------------------------------- 1 | { 2 | "CN": "nomad-cluster.io", 3 | "hosts": [ 4 | "nomad-cluster.io", 5 | "www.nomad-cluster.io" 6 | ], 7 | "key": { 8 | "algo": "rsa", 9 | "size": 2048 10 | }, 11 | "names": [ 12 | { 13 | "C": "US", 14 | "ST": "CA", 15 | "L": "San Francisco" 16 | } 17 | ] 18 | } 19 | 20 | -------------------------------------------------------------------------------- /nomad/certs/cfssl.json: -------------------------------------------------------------------------------- 1 | { 2 | "signing": { 3 | "default": { 4 | "expiry": "87600h", 5 | "usages": [ 6 | "signing", 7 | "key encipherment", 8 | "server auth", 9 | "client auth" 10 | ] 11 | } 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /nomad/cloud-config-agent.yml: -------------------------------------------------------------------------------- 1 | #cloud-config 2 | 3 | --- 4 | coreos: 5 | etcd: 6 | discovery: "https://discovery.etcd.io/1d77508dc31cf372d517ed2e96e8f5cf" 7 | advertise-client-urls: "http://$private_ipv4:2379" 8 | initial-advertise-peer-urls: "http://$private_ipv4:2380" 9 | listen_client_urls: "http://0.0.0.0:2379" 10 | listen_peer_urls: "http://$private_ipv4:2380" 11 | units: 12 | units: 13 | - name: iptables-firewall.service 14 | content: | 15 | [Unit] 16 | Description=Iptables Firewall 17 | Before=docker.service 18 | 19 | [Service] 20 | StandardOutput=console 21 | Restart=on-failure 22 | RestartSec=20 23 | TimeoutStartSec=0 24 | Environment=NOMAD_HOSTS=COMMA_SEPARATED_MASTER_IPS 25 | Environment=NOMAD_CIDRS=10.0.0.0/16,10.1.0.0/16,10.2.0.0/16 26 | ExecStart=/bin/bash /etc/scripts/iptables-firewall.sh 27 | 28 | 29 | [Install] 30 | WantedBy=multi-user.target 31 | - name: nomad.service 32 | command: start 33 | enable: true 34 | content: | 35 | [Unit] 36 | Description=Nomad Agent 37 | After=docker.service 38 | Requires=docker.service 39 | 40 | [Service] 41 | Restart=on-failure 42 | RestartSec=20 43 | TimeoutStartSec=0 44 | ExecStartPre=-/usr/bin/docker rm -f nomad 45 | ExecStartPre=-/usr/bin/docker pull r.j3ss.co/nomad:latest 46 | ExecStart=/usr/bin/sh -c "/usr/bin/docker run \ 47 | -e SERVICE_IGNORE=true \ 48 | --net=host \ 49 | --privileged \ 50 | -v /sys/fs/cgroup:/sys/fs/cgroup \ 51 | -v /tmp:/tmp \ 52 | -v /run:/run \ 53 | -v /usr/bin/docker:/usr/bin/docker:ro \ 54 | -v /var/run/docker.sock:/var/run/docker.sock \ 55 | -v /etc/nomad/certs:/etc/nomad/certs:ro \ 56 | --volume /etc/nomad.d:/etc/nomad.d \ 57 | --name=nomad \ 58 | r.j3ss.co/nomad:latest \ 59 | agent -config=/etc/nomad.d/client.hcl" 60 | ExecStop=/usr/bin/docker stop nomad 61 | 62 | [Install] 63 | WantedBy=multi-user.target 64 | write_files: 65 | - path: "/home/vmuser/.bashrc" 66 | permissions: "0777" 67 | owner: "vmuser" 68 | content: | 69 | # /etc/skel/.bashrc 70 | # 71 | # This file is sourced by all *interactive* bash shells on startup, 72 | # including some apparently interactive shells such as scp and rcp 73 | # that can't tolerate any output. So make sure this doesn't display 74 | # anything or bad things will happen ! 75 | 76 | 77 | # Test for an interactive shell. There is no need to set anything 78 | # past this point for scp and rcp, and it's important to refrain from 79 | # outputting anything in those cases. 80 | if [[ $- != *i* ]] ; then 81 | # Shell is non-interactive. Be done now! 82 | return 83 | fi 84 | 85 | consul(){ 86 | sudo docker run --rm -it --net host \ 87 | -e CONSUL_HTTP_ADDR=https://10.0.0.5:8501 \ 88 | -e CONSUL_CACERT=/etc/consul/certs/ca.pem \ 89 | -e CONSUL_CLIENT_CERT=/etc/consul/certs/cli.pem \ 90 | -e CONSUL_CLIENT_KEY=/etc/consul/certs/cli-key.pem \ 91 | -v /etc/consul/certs:/etc/consul/certs:ro \ 92 | r.j3ss.co/consul $@ 93 | } 94 | 95 | nomad(){ 96 | sudo docker run --rm -it --net host \ 97 | -e NOMAD_ADDR=https://127.0.0.1:4646 \ 98 | -e NOMAD_CACERT=/etc/nomad/certs/ca.pem \ 99 | -e NOMAD_CLIENT_CERT=/etc/nomad/certs/cli.pem \ 100 | -e NOMAD_CLIENT_KEY=/etc/nomad/certs/cli-key.pem \ 101 | -v /etc/nomad/certs:/etc/nomad/certs:ro \ 102 | -v $(pwd):/usr/src \ 103 | --workdir /usr/src \ 104 | r.j3ss.co/nomad $@ 105 | } 106 | - path: "/etc/scripts/iptables-firewall.sh" 107 | permissions: "0744" 108 | owner: "root" 109 | content: | 110 | #!/bin/bash 111 | set -e 112 | set -o pipefail 113 | set -x 114 | 115 | # iptables rules are processed in the order in which they are. 116 | # If there is a match for a rule no other rules will be processed for that IP packet in your case. 117 | 118 | 119 | # Allow all established connections. 120 | iptables -A OUTPUT -m conntrack --ctstate ESTABLISHED -j ACCEPT 121 | 122 | # Drop all invalid packets. 123 | iptables -A INPUT -m conntrack --ctstate INVALID -j DROP 124 | 125 | # Block outbound traffic to cloud metadata. 126 | iptables -A OUTPUT -d 169.254.169.254 -m comment --comment "Block traffic to cloud metadata" -j DROP 127 | 128 | # Accept on localhost 129 | iptables -A INPUT -i lo -m comment --comment "Accept input on localhost" -j ACCEPT 130 | iptables -A OUTPUT -o lo -m comment --comment "Accept output on localhsot" -j ACCEPT 131 | 132 | # Iterate over the nomad masters. 133 | if [ -n "${NOMAD_HOSTS+1}" ]; then 134 | IFS=',' read -r -a nhosts <<< "$NOMAD_HOSTS" 135 | for host in "${nhosts[@]}"; do 136 | 137 | # Add the nomad masters as allowed source IPs. 138 | iptables -A INPUT -s "$host" -m comment --comment "Allowing incoming connections from nomad master $host" -j ACCEPT 139 | 140 | # Allow outgoing connections from this IP to the other nomad servers. 141 | # Ports come from: https://www.nomadproject.io/docs/configuration/index.html#rpc-2 142 | iptables -A OUTPUT -p tcp -d "$host" --dport 4646 -m comment --comment "Allowing allow outgoing connections to nomad master $host:4646" -j ACCEPT 143 | iptables -A OUTPUT -p tcp -d "$host" --dport 4647 -m comment --comment "Allowing allow outgoing connections to nomad master $host:4647" -j ACCEPT 144 | iptables -A OUTPUT -p tcp -d "$host" --dport 4648 -m comment --comment "Allowing allow outgoing connections to nomad master $host:4648" -j ACCEPT 145 | 146 | done 147 | fi 148 | 149 | our_ip=$private_ipv4 150 | 151 | # Block all outbound traffic to the other CIDRS in our range. 152 | if [ -n "${NOMAD_CIDRS+1}" ]; then 153 | IFS=',' read -r -a ncidrs <<< "$NOMAD_CIDRS" 154 | for cidr in "${ncidrs[@]}"; do 155 | 156 | # Skip ourself, to test this remove everything after the last . 157 | if [[ "$cidr" =~ ${our_ip%.*} ]]; then 158 | continue 159 | fi 160 | 161 | iptables -A OUTPUT -d "$cidr" -m comment --comment "Block traffic to nomad cidr $cidr" -j DROP 162 | 163 | done 164 | fi 165 | 166 | # Accept ssh 167 | iptables -A INPUT -p tcp --dport 22 -m comment --comment "Accept ssh" -j ACCEPT 168 | 169 | # Allow ping to work 170 | iptables -A INPUT -p icmp -m comment --comment "Allow ping to work as expected" -j ACCEPT 171 | 172 | # Set default chain policies 173 | iptables -P OUTPUT ACCEPT 174 | iptables -P INPUT DROP 175 | iptables -P FORWARD DROP 176 | 177 | set +x 178 | - path: "/etc/nomad.d/client.hcl" 179 | permissions: "0644" 180 | owner: "root" 181 | content: | 182 | data_dir = "/etc/nomad.d" 183 | 184 | client { 185 | enabled = true 186 | servers = [NOMAD_MASTER_IPS] 187 | 188 | options = { 189 | "driver.whitelist" = "docker" 190 | } 191 | } 192 | 193 | tls { 194 | http = true 195 | rpc = true 196 | 197 | ca_file = "/etc/nomad/certs/ca.pem" 198 | cert_file = "/etc/nomad/certs/client.pem" 199 | key_file = "/etc/nomad/certs/client-key.pem" 200 | 201 | #verify_server_hostname = true 202 | verify_https_client = true 203 | } 204 | -------------------------------------------------------------------------------- /nomad/cloud-config-bastion.yml: -------------------------------------------------------------------------------- 1 | #cloud-config 2 | 3 | --- 4 | coreos: 5 | etcd: 6 | discovery: "https://discovery.etcd.io/1d77508dc31cf372d517ed2e96e8f5cf" 7 | advertise-client-urls: "http://$private_ipv4:2379" 8 | initial-advertise-peer-urls: "http://$private_ipv4:2380" 9 | listen_client_urls: "http://0.0.0.0:2379" 10 | listen_peer_urls: "http://$private_ipv4:2380" 11 | units: 12 | - name: nomad.service 13 | command: start 14 | enable: true 15 | content: | 16 | [Unit] 17 | Description=Nomad Agent 18 | After=docker.service 19 | Requires=docker.service 20 | 21 | [Service] 22 | Restart=on-failure 23 | RestartSec=20 24 | TimeoutStartSec=0 25 | ExecStartPre=-/usr/bin/docker rm -f nomad 26 | ExecStartPre=-/usr/bin/docker pull r.j3ss.co/nomad:latest 27 | ExecStart=/usr/bin/sh -c "/usr/bin/docker run \ 28 | -e SERVICE_IGNORE=true \ 29 | -p 8080:4646 \ 30 | --hostname bastion \ 31 | -v /etc/nomad/certs:/etc/nomad/certs:ro \ 32 | --volume /etc/nomad.d:/etc/nomad.d \ 33 | --name=nomad \ 34 | r.j3ss.co/nomad:latest \ 35 | agent -config=/etc/nomad.d/client.hcl" 36 | ExecStop=/usr/bin/docker stop nomad 37 | 38 | [Install] 39 | WantedBy=multi-user.target 40 | write_files: 41 | - path: "/home/vmuser/.bashrc" 42 | permissions: "0777" 43 | owner: "vmuser" 44 | content: | 45 | # /etc/skel/.bashrc 46 | # 47 | # This file is sourced by all *interactive* bash shells on startup, 48 | # including some apparently interactive shells such as scp and rcp 49 | # that can't tolerate any output. So make sure this doesn't display 50 | # anything or bad things will happen ! 51 | 52 | 53 | # Test for an interactive shell. There is no need to set anything 54 | # past this point for scp and rcp, and it's important to refrain from 55 | # outputting anything in those cases. 56 | if [[ $- != *i* ]] ; then 57 | # Shell is non-interactive. Be done now! 58 | return 59 | fi 60 | 61 | consul(){ 62 | sudo docker run --rm -i --net host \ 63 | -e CONSUL_HTTP_ADDR=https://10.0.0.5:8501 \ 64 | -e CONSUL_CACERT=/etc/consul/certs/ca.pem \ 65 | -e CONSUL_CLIENT_CERT=/etc/consul/certs/cli.pem \ 66 | -e CONSUL_CLIENT_KEY=/etc/consul/certs/cli-key.pem \ 67 | -v /etc/consul/certs:/etc/consul/certs:ro \ 68 | r.j3ss.co/consul $@ 69 | } 70 | 71 | nomad(){ 72 | sudo docker run --rm -i --net host \ 73 | -e NOMAD_ADDR=https://10.0.0.5:4646 \ 74 | -e NOMAD_CACERT=/etc/nomad/certs/ca.pem \ 75 | -e NOMAD_CLIENT_CERT=/etc/nomad/certs/cli.pem \ 76 | -e NOMAD_CLIENT_KEY=/etc/nomad/certs/cli-key.pem \ 77 | -v /etc/nomad/certs:/etc/nomad/certs:ro \ 78 | -v $(pwd):/usr/src \ 79 | --workdir /usr/src \ 80 | r.j3ss.co/nomad $@ 81 | } 82 | - path: "/etc/nomad.d/client.hcl" 83 | permissions: "0644" 84 | owner: "root" 85 | content: | 86 | data_dir = "/etc/nomad.d" 87 | 88 | client { 89 | enabled = true 90 | servers = [NOMAD_MASTER_IPS] 91 | 92 | # turn off the default driver. 93 | #"driver.raw_exec.enable" = "0" 94 | 95 | reserved { 96 | cpu = 1 97 | memory = 1 98 | disk = 1 99 | } 100 | } 101 | 102 | tls { 103 | http = false 104 | rpc = true 105 | 106 | ca_file = "/etc/nomad/certs/ca.pem" 107 | cert_file = "/etc/nomad/certs/client.pem" 108 | key_file = "/etc/nomad/certs/client-key.pem" 109 | 110 | #verify_server_hostname = true 111 | #verify_https_client = true 112 | } 113 | -------------------------------------------------------------------------------- /nomad/cloud-config-master.yml: -------------------------------------------------------------------------------- 1 | #cloud-config 2 | 3 | --- 4 | coreos: 5 | etcd: 6 | discovery: "https://discovery.etcd.io/1d77508dc31cf372d517ed2e96e8f5cf" 7 | advertise-client-urls: "http://$private_ipv4:2379" 8 | initial-advertise-peer-urls: "http://$private_ipv4:2380" 9 | listen_client_urls: "http://0.0.0.0:2379" 10 | listen_peer_urls: "http://$private_ipv4:2380" 11 | units: 12 | - name: consul.service 13 | command: start 14 | enable: true 15 | content: | 16 | [Unit] 17 | Description=Consul 18 | After=docker.service 19 | Requires=docker.service 20 | Wants=consul-announce.service 21 | Before=consul-announce.service 22 | 23 | [Service] 24 | Restart=on-failure 25 | RestartSec=20 26 | TimeoutStartSec=0 27 | ExecStartPre=-/usr/bin/docker rm -f consul 28 | ExecStartPre=-/usr/bin/docker pull r.j3ss.co/consul:latest 29 | ExecStart=/usr/bin/sh -c "/usr/bin/docker run \ 30 | -e SERVICE_IGNORE=true \ 31 | --net=host \ 32 | --volume /etc/consul.d:/etc/consul.d \ 33 | --volume /etc/consul/certs:/etc/consul/certs:ro \ 34 | --volume /opt/consul:/opt/consul \ 35 | --name=consul \ 36 | r.j3ss.co/consul:latest \ 37 | agent \ 38 | -bootstrap-expect 4 \ 39 | -config-dir /etc/consul.d \ 40 | -advertise $private_ipv4" 41 | ExecStop=/usr/bin/docker stop consul 42 | 43 | [Install] 44 | WantedBy=multi-user.target 45 | - name: consul-announce.service 46 | command: start 47 | content: | 48 | [Unit] 49 | Description=Consul Announcer 50 | PartOf=consul.service 51 | After=consul.service 52 | 53 | [Service] 54 | StandardOutput=console 55 | Restart=on-failure 56 | RestartSec=20 57 | Environment=NOMAD_HOSTS=COMMA_SEPARATED_MASTER_IPS 58 | ExecStart=/bin/bash /etc/consul/join.sh 59 | - name: nomad.service 60 | command: start 61 | enable: true 62 | content: | 63 | [Unit] 64 | Description=Nomad Server 65 | After=docker.service consul.service consul-announce.service 66 | Requires=docker.service consul.service 67 | 68 | [Service] 69 | Restart=on-failure 70 | RestartSec=20 71 | TimeoutStartSec=0 72 | ExecStartPre=-/usr/bin/docker rm -f nomad 73 | ExecStartPre=-/usr/bin/docker pull r.j3ss.co/nomad:latest 74 | ExecStart=/usr/bin/sh -c "/usr/bin/docker run \ 75 | -e SERVICE_IGNORE=true \ 76 | --net=host \ 77 | --volume /etc/nomad.d:/etc/nomad.d \ 78 | --volume /etc/consul/certs:/etc/consul/certs:ro \ 79 | --volume /etc/nomad/certs:/etc/nomad/certs:ro \ 80 | --name=nomad \ 81 | r.j3ss.co/nomad:latest \ 82 | agent -config=/etc/nomad.d/server.hcl" 83 | ExecStop=/usr/bin/docker stop nomad 84 | 85 | [Install] 86 | WantedBy=multi-user.target 87 | write_files: 88 | - path: "/home/vmuser/.bashrc" 89 | permissions: "0777" 90 | owner: "vmuser" 91 | content: | 92 | # /etc/skel/.bashrc 93 | # 94 | # This file is sourced by all *interactive* bash shells on startup, 95 | # including some apparently interactive shells such as scp and rcp 96 | # that can't tolerate any output. So make sure this doesn't display 97 | # anything or bad things will happen ! 98 | 99 | 100 | # Test for an interactive shell. There is no need to set anything 101 | # past this point for scp and rcp, and it's important to refrain from 102 | # outputting anything in those cases. 103 | if [[ $- != *i* ]] ; then 104 | # Shell is non-interactive. Be done now! 105 | return 106 | fi 107 | 108 | consul(){ 109 | sudo docker run --rm -it --net host \ 110 | -e CONSUL_HTTP_ADDR=https://127.0.0.1:8501 \ 111 | -e CONSUL_CACERT=/etc/consul/certs/ca.pem \ 112 | -e CONSUL_CLIENT_CERT=/etc/consul/certs/cli.pem \ 113 | -e CONSUL_CLIENT_KEY=/etc/consul/certs/cli-key.pem \ 114 | -v /etc/consul/certs:/etc/consul/certs:ro \ 115 | r.j3ss.co/consul $@ 116 | } 117 | 118 | nomad(){ 119 | sudo docker run --rm -it --net host \ 120 | -e NOMAD_ADDR=https://127.0.0.1:4646 \ 121 | -e NOMAD_CACERT=/etc/nomad/certs/ca.pem \ 122 | -e NOMAD_CLIENT_CERT=/etc/nomad/certs/cli.pem \ 123 | -e NOMAD_CLIENT_KEY=/etc/nomad/certs/cli-key.pem \ 124 | -v /etc/nomad/certs:/etc/nomad/certs:ro \ 125 | -v $(pwd):/usr/src \ 126 | --workdir /usr/src \ 127 | r.j3ss.co/nomad $@ 128 | } 129 | - path: "/etc/nomad.d/server.hcl" 130 | permissions: "0644" 131 | owner: "root" 132 | content: | 133 | data_dir = "/etc/nomad.d" 134 | 135 | server { 136 | enabled = true 137 | bootstrap_expect = 4 138 | 139 | encrypt = "NOMAD_GOSSIP_ENCRYPTION_SECRET" 140 | } 141 | 142 | consul { 143 | address = "127.0.0.1:8501" 144 | ca_file = "/etc/consul/certs/ca.pem" 145 | cert_file = "/etc/consul/certs/server.pem" 146 | key_file = "/etc/consul/certs/server-key.pem" 147 | ssl = true 148 | } 149 | 150 | tls { 151 | http = true 152 | rpc = true 153 | 154 | ca_file = "/etc/nomad/certs/ca.pem" 155 | cert_file = "/etc/nomad/certs/server.pem" 156 | key_file = "/etc/nomad/certs/server-key.pem" 157 | 158 | #verify_server_hostname = true 159 | verify_https_client = true 160 | } 161 | - path: "/etc/consul.d/config.json" 162 | permissions: "0644" 163 | owner: "root" 164 | content: | 165 | { 166 | "datacenter": "dc1", 167 | "data_dir": "/opt/consul", 168 | "log_level": "INFO", 169 | "server": true, 170 | "encrypt": "CONSUL_GOSSIP_ENCRYPTION_SECRET", 171 | "verify_outgoing": true, 172 | "verify_incoming": true, 173 | "verify_incoming_rpc": true, 174 | "verify_incoming_https": true, 175 | "addresses": { 176 | "https": "0.0.0.0" 177 | }, 178 | "ports": { 179 | "http": -1, 180 | "https": 8501, 181 | "grpc": 8502 182 | }, 183 | "key_file": "/etc/consul/certs/server-key.pem", 184 | "cert_file": "/etc/consul/certs/server.pem", 185 | "ca_file": "/etc/consul/certs/ca.pem" 186 | } 187 | - path: "/etc/consul/join.sh" 188 | permissions: "0744" 189 | owner: "root" 190 | content: | 191 | #!/bin/bash 192 | set -e 193 | set -o pipefail 194 | set -x 195 | 196 | # Add the hosts. 197 | if [ -n "${NOMAD_HOSTS+1}" ]; then 198 | IFS=',' read -r -a nhosts <<< "$NOMAD_HOSTS" 199 | for host in "${nhosts[@]}"; do 200 | /usr/bin/docker run --rm --net host \ 201 | -e CONSUL_HTTP_ADDR=https://127.0.0.1:8501 \ 202 | -e CONSUL_CACERT=/etc/consul/certs/ca.pem \ 203 | -e CONSUL_CLIENT_CERT=/etc/consul/certs/cli.pem \ 204 | -e CONSUL_CLIENT_KEY=/etc/consul/certs/cli-key.pem \ 205 | -v /etc/consul/certs:/etc/consul/certs:ro \ 206 | r.j3ss.co/consul:latest join "$host" || true 207 | done 208 | fi 209 | 210 | set +x 211 | -------------------------------------------------------------------------------- /nomad/sleeping-beauty.hcl: -------------------------------------------------------------------------------- 1 | job "sleeping-beauty" { 2 | # Specify this job should run in the region named "us". Regions 3 | # are defined by the Nomad servers' configuration. 4 | region = "global" 5 | 6 | # Spread the tasks in these datacenters based on the Nomad server' configuration. 7 | datacenters = ["dc1"] 8 | 9 | # Run this job as a "service" type. Each job type has different 10 | # properties. See the documentation below for more examples. 11 | type = "batch" 12 | 13 | # A group defines a series of tasks that should be co-located 14 | # on the same client (host). All tasks within a group will be 15 | # placed on the same host. 16 | group "sb-group" { 17 | # Specify the number of these tasks we want. 18 | count = 1000000 19 | 20 | # Create an individual task (unit of work). This particular 21 | # task utilizes a Docker container to front a web application. 22 | task "sb-task" { 23 | # Specify the driver to be "docker". Nomad supports 24 | # multiple drivers. 25 | driver = "docker" 26 | 27 | # Configuration is specific to each driver. 28 | config { 29 | image = "tianon/sleeping-beauty" 30 | } 31 | 32 | # Specify the maximum resources required to run the task, 33 | # include CPU, memory, and bandwidth. 34 | resources { 35 | cpu = 20 # MHz 36 | memory = 10 # MB 37 | } 38 | } 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /terraform/agents.tf: -------------------------------------------------------------------------------- 1 | ###################### 2 | # Agent VM Scale Set 3 | ###################### 4 | resource "azurerm_virtual_machine_scale_set" "agent" { 5 | depends_on = ["azurerm_virtual_machine.master", "azurerm_virtual_machine.bastion"] 6 | 7 | count = "${length(var.locations)}" 8 | 9 | name = "agent" 10 | location = "${element(azurerm_resource_group.rg.*.location, count.index)}" 11 | resource_group_name = "${element(azurerm_resource_group.rg.*.name, count.index)}" 12 | 13 | upgrade_policy_mode = "Manual" 14 | 15 | single_placement_group = false 16 | 17 | sku { 18 | name = "${var.agent_vmsize}" 19 | tier = "Standard" 20 | capacity = "${var.agent_count}" 21 | } 22 | 23 | network_profile { 24 | name = "networkprofile" 25 | primary = true 26 | 27 | ip_configuration { 28 | name = "ipconfig" 29 | subnet_id = "${element(azurerm_subnet.subnet.*.id, count.index)}" 30 | } 31 | } 32 | 33 | storage_profile_image_reference { 34 | publisher = "${var.image_publisher}" 35 | offer = "${var.image_publisher}" 36 | sku = "${var.image_sku}" 37 | version = "${var.image_version}" 38 | } 39 | 40 | storage_profile_os_disk { 41 | name = "" 42 | managed_disk_type = "Standard_LRS" 43 | caching = "ReadWrite" 44 | create_option = "FromImage" 45 | } 46 | 47 | storage_profile_data_disk { 48 | lun = 0 49 | caching = "ReadWrite" 50 | create_option = "Empty" 51 | disk_size_gb = 500 52 | } 53 | 54 | os_profile { 55 | computer_name_prefix = "${element(azurerm_resource_group.rg.*.name, count.index)}-agent" 56 | admin_username = "${var.username}" 57 | custom_data = "${file(var.cloud_config_agent)}" 58 | } 59 | 60 | os_profile_linux_config { 61 | disable_password_authentication = true 62 | 63 | ssh_keys { 64 | path = "/home/${var.username}/.ssh/authorized_keys" 65 | key_data = "${file(var.public_key_path)}" 66 | } 67 | } 68 | 69 | tags { 70 | orchestrator = "${var.orchestrator}" 71 | type = "agent" 72 | datacenter = "${element(azurerm_resource_group.rg.*.location, count.index)}" 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /terraform/bastion.tf: -------------------------------------------------------------------------------- 1 | # 2 | # Bastion Host 3 | # 4 | resource "azurerm_public_ip" "bastion_public_ip" { 5 | name = "bastion-public_ip" 6 | resource_group_name = "${azurerm_resource_group.rg.0.name}" 7 | location = "${azurerm_resource_group.rg.0.location}" 8 | public_ip_address_allocation = "Static" 9 | domain_name_label = "${azurerm_resource_group.rg.0.name}-bastion" 10 | 11 | tags { 12 | orchestrator = "${var.orchestrator}" 13 | type = "bastion" 14 | datacenter = "${azurerm_resource_group.rg.0.location}" 15 | } 16 | } 17 | 18 | resource "azurerm_network_security_group" "bastion_nsg" { 19 | name = "bastion-nsg" 20 | location = "${azurerm_resource_group.rg.0.location}" 21 | resource_group_name = "${azurerm_resource_group.rg.0.name}" 22 | 23 | security_rule { 24 | name = "allow_SSH_in_all" 25 | description = "Allow SSH in from all locations" 26 | priority = 100 27 | direction = "Inbound" 28 | access = "Allow" 29 | protocol = "Tcp" 30 | source_port_range = "*" 31 | destination_port_range = "22" 32 | source_address_prefix = "*" 33 | destination_address_prefix = "*" 34 | } 35 | 36 | security_rule { 37 | name = "allow_HTTP_in_all" 38 | description = "Allow HTTP in from all locations" 39 | priority = 101 40 | direction = "Inbound" 41 | access = "Allow" 42 | protocol = "Tcp" 43 | source_port_range = "*" 44 | destination_port_range = "8080" 45 | source_address_prefix = "*" 46 | destination_address_prefix = "*" 47 | } 48 | 49 | tags { 50 | orchestrator = "${var.orchestrator}" 51 | type = "bastion" 52 | datacenter = "${azurerm_resource_group.rg.0.location}" 53 | } 54 | } 55 | 56 | resource "azurerm_network_interface" "bastion_nic" { 57 | name = "bastion-nic" 58 | location = "${azurerm_resource_group.rg.0.location}" 59 | resource_group_name = "${azurerm_resource_group.rg.0.name}" 60 | network_security_group_id = "${azurerm_network_security_group.bastion_nsg.id}" 61 | 62 | ip_configuration { 63 | name = "ipconfig" 64 | subnet_id = "${azurerm_subnet.subnet.0.id}" 65 | public_ip_address_id = "${azurerm_public_ip.bastion_public_ip.id}" 66 | private_ip_address_allocation = "Static" 67 | private_ip_address = "10.0.0.4" 68 | } 69 | 70 | tags { 71 | orchestrator = "${var.orchestrator}" 72 | type = "bastion" 73 | datacenter = "${azurerm_resource_group.rg.0.location}" 74 | } 75 | } 76 | 77 | resource "azurerm_virtual_machine" "bastion" { 78 | name = "bastion" 79 | location = "${azurerm_resource_group.rg.0.location}" 80 | resource_group_name = "${azurerm_resource_group.rg.0.name}" 81 | vm_size = "${var.master_vmsize}" 82 | network_interface_ids = ["${azurerm_network_interface.bastion_nic.id}"] 83 | delete_os_disk_on_termination = true 84 | delete_data_disks_on_termination = true 85 | 86 | connection { 87 | type = "ssh" 88 | host = "${azurerm_public_ip.bastion_public_ip.fqdn}" 89 | user = "${var.username}" 90 | agent = true 91 | } 92 | 93 | os_profile { 94 | computer_name = "${azurerm_resource_group.rg.0.name}-bastion" 95 | admin_username = "${var.username}" 96 | custom_data = "${file(var.cloud_config_bastion)}" 97 | } 98 | 99 | os_profile_linux_config { 100 | disable_password_authentication = true 101 | 102 | ssh_keys { 103 | path = "/home/${var.username}/.ssh/authorized_keys" 104 | key_data = "${file(var.public_key_path)}" 105 | } 106 | } 107 | 108 | storage_image_reference { 109 | publisher = "${var.image_publisher}" 110 | offer = "${var.image_publisher}" 111 | sku = "${var.image_sku}" 112 | version = "${var.image_version}" 113 | } 114 | 115 | storage_os_disk { 116 | name = "bastion-osdisk" 117 | managed_disk_type = "StandardSSD_LRS" 118 | caching = "ReadWrite" 119 | create_option = "FromImage" 120 | } 121 | 122 | provisioner "file" { 123 | source = "../nomad/sleeping-beauty.hcl" 124 | destination = "/home/${var.username}/sleeping-beauty.hcl" 125 | } 126 | 127 | tags { 128 | orchestrator = "${var.orchestrator}" 129 | type = "bastion" 130 | datacenter = "${azurerm_resource_group.rg.0.location}" 131 | } 132 | } 133 | -------------------------------------------------------------------------------- /terraform/common.tf: -------------------------------------------------------------------------------- 1 | resource "azurerm_resource_group" "rg" { 2 | count = "${length(var.locations)}" 3 | 4 | name = "${var.prefix}-${var.orchestrator}-${var.rg}-${element(var.locations, count.index)}" 5 | location = "${element(var.locations, count.index)}" 6 | 7 | tags { 8 | orchestrator = "${var.orchestrator}" 9 | datacenter = "${element(var.locations, count.index)}" 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /terraform/masters.tf: -------------------------------------------------------------------------------- 1 | resource "azurerm_network_interface" "master-nic" { 2 | count = "${var.master_count * length(var.locations)}" 3 | 4 | name = "master-nic${count.index - (floor(count.index / var.master_count) * var.master_count)}" 5 | location = "${element(azurerm_resource_group.rg.*.location, floor(count.index / var.master_count))}" 6 | resource_group_name = "${element(azurerm_resource_group.rg.*.name, floor(count.index / var.master_count))}" 7 | 8 | ip_configuration { 9 | name = "ipconfig" 10 | subnet_id = "${element(azurerm_subnet.subnet.*.id, floor(count.index / var.master_count))}" 11 | private_ip_address_allocation = "Static" 12 | 13 | # This expands to 10.0.0.0/16 for the first region and 10.1.0.0/16 for the second region... 14 | # And then the IP being accessed is the number of the master for that location: 15 | # 10.0.0.5 16 | # 10.0.0.6 17 | # ... 18 | # 10.1.0.5 19 | # 10.1.0.6 20 | # ... 21 | private_ip_address = "${cidrhost(format("10.%d.0.0/16", floor(count.index / var.master_count)), count.index - (floor(count.index / var.master_count) * var.master_count) + 5)}" 22 | } 23 | 24 | tags { 25 | orchestrator = "${var.orchestrator}" 26 | type = "master" 27 | datacenter = "${element(azurerm_resource_group.rg.*.location, floor(count.index / var.master_count))}" 28 | } 29 | } 30 | 31 | ###################### 32 | # Master VM 33 | ###################### 34 | resource "azurerm_virtual_machine" "master" { 35 | count = "${var.master_count * length(var.locations)}" 36 | 37 | name = "master${count.index - (floor(count.index / var.master_count) * var.master_count)}" 38 | location = "${element(azurerm_resource_group.rg.*.location, floor(count.index / var.master_count))}" 39 | resource_group_name = "${element(azurerm_resource_group.rg.*.name, floor(count.index / var.master_count))}" 40 | vm_size = "${var.master_vmsize}" 41 | network_interface_ids = ["${element(azurerm_network_interface.master-nic.*.id, count.index)}"] 42 | 43 | delete_os_disk_on_termination = true 44 | delete_data_disks_on_termination = true 45 | 46 | connection { 47 | type = "ssh" 48 | bastion_host = "${azurerm_public_ip.bastion_public_ip.fqdn}" 49 | bastion_user = "${var.username}" 50 | host = "${element(azurerm_network_interface.master-nic.*.ip_configuration.0.private_ip_addresses, count.index)}" 51 | user = "${var.username}" 52 | agent = true 53 | } 54 | 55 | storage_image_reference { 56 | publisher = "${var.image_publisher}" 57 | offer = "${var.image_publisher}" 58 | sku = "${var.image_sku}" 59 | version = "${var.image_version}" 60 | } 61 | 62 | storage_os_disk { 63 | name = "master-osdisk${count.index - (floor(count.index / var.master_count) * var.master_count)}" 64 | managed_disk_type = "StandardSSD_LRS" 65 | caching = "ReadWrite" 66 | create_option = "FromImage" 67 | } 68 | 69 | os_profile { 70 | computer_name = "${element(azurerm_resource_group.rg.*.name, floor(count.index / var.master_count))}-master${count.index - (floor(count.index / var.master_count) * var.master_count)}" 71 | admin_username = "${var.username}" 72 | custom_data = "${file(var.cloud_config_master)}" 73 | } 74 | 75 | os_profile_linux_config { 76 | disable_password_authentication = true 77 | 78 | ssh_keys { 79 | path = "/home/${var.username}/.ssh/authorized_keys" 80 | key_data = "${file(var.public_key_path)}" 81 | } 82 | } 83 | 84 | tags { 85 | orchestrator = "${var.orchestrator}" 86 | type = "master" 87 | datacenter = "${element(azurerm_resource_group.rg.*.location, floor(count.index / var.master_count))}" 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /terraform/network.tf: -------------------------------------------------------------------------------- 1 | resource "azurerm_virtual_network" "vnet" { 2 | count = "${length(var.locations)}" 3 | 4 | name = "vnet-${count.index}" 5 | resource_group_name = "${element(azurerm_resource_group.rg.*.name, count.index)}" 6 | address_space = ["${element(var.vnet_address_space, count.index)}"] 7 | location = "${element(azurerm_resource_group.rg.*.location, count.index)}" 8 | 9 | tags { 10 | orchestrator = "${var.orchestrator}" 11 | datacenter = "${element(azurerm_resource_group.rg.*.location, count.index)}" 12 | } 13 | } 14 | 15 | resource "azurerm_subnet" "subnet" { 16 | count = "${length(var.locations)}" 17 | 18 | name = "subnet-${count.index}" 19 | resource_group_name = "${element(azurerm_resource_group.rg.*.name, count.index)}" 20 | virtual_network_name = "${element(azurerm_virtual_network.vnet.*.name, count.index)}" 21 | address_prefix = "${element(var.vnet_address_space, count.index)}" 22 | } 23 | 24 | # Enable global peering between the virtual networks. 25 | 26 | # TODO: generate the below with math. 27 | 28 | /*resource "azurerm_virtual_network_peering" "peering" { 29 | count = "${length(var.locations) * 2}" 30 | 31 | name = "peering-to-${var.locations[floor(count.index / 2)]}" 32 | resource_group_name = "${element(azurerm_resource_group.rg.*.name, 0)}" 33 | virtual_network_name = "${element(azurerm_virtual_network.vnet.*.name, 0)}" 34 | remote_virtual_network_id = "${element(azurerm_virtual_network.vnet.*.id, 1)}" 35 | allow_virtual_network_access = true 36 | allow_forwarded_traffic = true 37 | 38 | # `allow_gateway_transit` must be set to false for vnet global peering. 39 | allow_gateway_transit = false 40 | }*/ 41 | 42 | resource "azurerm_virtual_network_peering" "peering0to1" { 43 | name = "peering-to-${var.locations[1]}" 44 | resource_group_name = "${element(azurerm_resource_group.rg.*.name, 0)}" 45 | virtual_network_name = "${element(azurerm_virtual_network.vnet.*.name, 0)}" 46 | remote_virtual_network_id = "${element(azurerm_virtual_network.vnet.*.id, 1)}" 47 | allow_virtual_network_access = true 48 | allow_forwarded_traffic = true 49 | 50 | # `allow_gateway_transit` must be set to false for vnet global peering. 51 | allow_gateway_transit = false 52 | } 53 | 54 | resource "azurerm_virtual_network_peering" "peering1to0" { 55 | name = "peering-to-${var.locations[0]}" 56 | resource_group_name = "${element(azurerm_resource_group.rg.*.name,1)}" 57 | virtual_network_name = "${element(azurerm_virtual_network.vnet.*.name, 1)}" 58 | remote_virtual_network_id = "${element(azurerm_virtual_network.vnet.*.id, 0)}" 59 | allow_virtual_network_access = true 60 | allow_forwarded_traffic = true 61 | 62 | # `allow_gateway_transit` must be set to false for vnet global peering. 63 | allow_gateway_transit = false 64 | } 65 | 66 | resource "azurerm_virtual_network_peering" "peering0to2" { 67 | name = "peering-to-${var.locations[2]}" 68 | resource_group_name = "${element(azurerm_resource_group.rg.*.name, 0)}" 69 | virtual_network_name = "${element(azurerm_virtual_network.vnet.*.name, 0)}" 70 | remote_virtual_network_id = "${element(azurerm_virtual_network.vnet.*.id, 2)}" 71 | allow_virtual_network_access = true 72 | allow_forwarded_traffic = true 73 | 74 | # `allow_gateway_transit` must be set to false for vnet global peering. 75 | allow_gateway_transit = false 76 | } 77 | 78 | resource "azurerm_virtual_network_peering" "peering2to0" { 79 | name = "peering-to-${var.locations[0]}" 80 | resource_group_name = "${element(azurerm_resource_group.rg.*.name,2)}" 81 | virtual_network_name = "${element(azurerm_virtual_network.vnet.*.name, 2)}" 82 | remote_virtual_network_id = "${element(azurerm_virtual_network.vnet.*.id, 0)}" 83 | allow_virtual_network_access = true 84 | allow_forwarded_traffic = true 85 | 86 | # `allow_gateway_transit` must be set to false for vnet global peering. 87 | allow_gateway_transit = false 88 | } 89 | 90 | resource "azurerm_virtual_network_peering" "peering1to2" { 91 | name = "peering-to-${var.locations[2]}" 92 | resource_group_name = "${element(azurerm_resource_group.rg.*.name, 1)}" 93 | virtual_network_name = "${element(azurerm_virtual_network.vnet.*.name, 1)}" 94 | remote_virtual_network_id = "${element(azurerm_virtual_network.vnet.*.id, 2)}" 95 | allow_virtual_network_access = true 96 | allow_forwarded_traffic = true 97 | 98 | # `allow_gateway_transit` must be set to false for vnet global peering. 99 | allow_gateway_transit = false 100 | } 101 | 102 | resource "azurerm_virtual_network_peering" "peering2to1" { 103 | name = "peering-to-${var.locations[1]}" 104 | resource_group_name = "${element(azurerm_resource_group.rg.*.name,2)}" 105 | virtual_network_name = "${element(azurerm_virtual_network.vnet.*.name, 2)}" 106 | remote_virtual_network_id = "${element(azurerm_virtual_network.vnet.*.id, 1)}" 107 | allow_virtual_network_access = true 108 | allow_forwarded_traffic = true 109 | 110 | # `allow_gateway_transit` must be set to false for vnet global peering. 111 | allow_gateway_transit = false 112 | } 113 | -------------------------------------------------------------------------------- /terraform/vars.tf: -------------------------------------------------------------------------------- 1 | variable "prefix" {} 2 | 3 | variable "location" { 4 | description = "The location/region where the resources are created. Changing this forces a new resource to be created." 5 | default = "westus2" 6 | } 7 | 8 | variable "locations" { 9 | default = [ 10 | "westus2", 11 | "eastus", 12 | "eastus2", 13 | ] 14 | } 15 | 16 | variable "vnet_address_space" { 17 | default = [ 18 | "10.0.0.0/16", 19 | "10.1.0.0/16", 20 | "10.2.0.0/16", 21 | ] 22 | } 23 | 24 | variable "rg" { 25 | description = "The name of the resource group in which to create the resources." 26 | default = "cluster" 27 | } 28 | 29 | variable "orchestrator" { 30 | description = "The orchestrator for the cluster." 31 | default = "nomad" 32 | } 33 | 34 | variable "master_count" { 35 | description = "Number of master nodes to create." 36 | default = 5 37 | } 38 | 39 | variable "agent_count" { 40 | description = "Number of master nodes to create." 41 | default = 18 42 | } 43 | 44 | variable "master_vmsize" { 45 | description = "Specifies the size of the virtual machine for the masters." 46 | default = "Standard_DS3_v2" 47 | } 48 | 49 | variable "agent_vmsize" { 50 | description = "Specifies the size of the virtual machine for the agents." 51 | default = "Standard_DS3_v2" 52 | } 53 | 54 | variable "username" { 55 | description = "The system administrator user name." 56 | default = "vmuser" 57 | } 58 | 59 | variable "public_key_path" { 60 | description = "Path to your SSH Public Key" 61 | default = "~/.azure/ssh_key" 62 | } 63 | 64 | # This file is generated and populated with certs from the Makefile at runtime. 65 | variable "cloud_config_master" { 66 | default = "../_tmp/nomad/cloud-config-master.yml" 67 | } 68 | 69 | # This file is generated and populated with certs from the Makefile at runtime. 70 | variable "cloud_config_agent" { 71 | default = "../_tmp/nomad/cloud-config-agent.yml" 72 | } 73 | 74 | # This file is generated and populated with certs from the Makefile at runtime. 75 | variable "cloud_config_bastion" { 76 | default = "../_tmp/nomad/cloud-config-bastion.yml" 77 | } 78 | 79 | variable "image_publisher" { 80 | description = "Virtual machine image publisher." 81 | default = "CoreOS" 82 | } 83 | 84 | variable "image_sku" { 85 | description = "Virtual machine image sku." 86 | default = "Stable" 87 | } 88 | 89 | variable "image_version" { 90 | description = "Virtual machine image version." 91 | default = "latest" 92 | } 93 | 94 | variable "client_id" {} 95 | 96 | variable "client_secret" {} 97 | 98 | variable "tenant_id" {} 99 | 100 | variable "subscription_id" {} 101 | 102 | provider "azurerm" { 103 | client_id = "${var.client_id}" 104 | client_secret = "${var.client_secret}" 105 | tenant_id = "${var.tenant_id}" 106 | subscription_id = "${var.subscription_id}" 107 | } 108 | -------------------------------------------------------------------------------- /test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | set -o pipefail 4 | 5 | ERRORS=() 6 | 7 | # find all executables and run `shellcheck` 8 | for f in $(find . -type f -not -iwholename '*.git*' | sort -u); do 9 | if file "$f" | grep --quiet shell; then 10 | { 11 | shellcheck "$f" && echo "[OK]: sucessfully linted $f" 12 | } || { 13 | # add to errors 14 | ERRORS+=("$f") 15 | } 16 | fi 17 | done 18 | 19 | if [ ${#ERRORS[@]} -eq 0 ]; then 20 | echo "No errors, hooray" 21 | else 22 | echo "These files failed shellcheck: ${ERRORS[*]}" 23 | exit 1 24 | fi 25 | --------------------------------------------------------------------------------