├── .gitignore ├── .terraform.lock.hcl ├── README.md ├── hcp.tf ├── nomad.tf ├── outputs.tf ├── providers.tf ├── user-data-client.sh ├── user-data-server.sh ├── variables.tf ├── vault-config.tf └── vpc.tf /.gitignore: -------------------------------------------------------------------------------- 1 | # Local .terraform directories 2 | **/.terraform/* 3 | 4 | # .tfstate files 5 | *.tfstate 6 | *.tfstate.* 7 | 8 | # Crash log files 9 | crash.log 10 | 11 | # Ignore any .tfvars files that are generated automatically for each Terraform run. Most 12 | # .tfvars files are managed as part of configuration and so should be included in 13 | # version control. 14 | # 15 | # example.tfvars 16 | 17 | # Ignore override files as they are usually used to override resources locally and so 18 | # are not checked in 19 | override.tf 20 | override.tf.json 21 | *_override.tf 22 | *_override.tf.json 23 | 24 | # Include override files you do wish to add to version control using negated pattern 25 | # 26 | # !example_override.tf 27 | 28 | # Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan 29 | # example: *tfplan* 30 | */.terraform.lock.hcl 31 | vars.sh 32 | -------------------------------------------------------------------------------- /.terraform.lock.hcl: -------------------------------------------------------------------------------- 1 | # This file is maintained automatically by "terraform init". 2 | # Manual edits may be lost in future updates. 3 | 4 | provider "registry.terraform.io/hashicorp/aws" { 5 | version = "4.24.0" 6 | constraints = ">= 3.63.0" 7 | hashes = [ 8 | "h1:+wJiCQKLEgY8a1ILFbebLgFX6q2xPJVV9wWp7eU2dsI=", 9 | "zh:3b58916e93cab4249bef6fcf6fb2ae3bbf0cb67a876e669205e1f785ffce88a4", 10 | "zh:5a51329c4d91ecdc2879a7d4acbc1dfd521ca6cd9a64f0d6f8c01d99a23fc98d", 11 | "zh:5c65414467db9b4bbf2f83fb1188543d1015514bab8a2336b38fcccb507fc7ca", 12 | "zh:65fc1514f0f1a06463b70694add57589c31debba625d78e25a9434e521a7a290", 13 | "zh:71b357f85d47cdb806df850b950193abae7ed14201edeba184be4c1672631f50", 14 | "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", 15 | "zh:a1a89a7fb35fa6160963dae13861033493bd5f3e6bc5fd18a0fd745a066378be", 16 | "zh:a9482369470168f3830a4a688506426769e1beb09fbdae25633acc508c0a9457", 17 | "zh:bf93cb9d15a822bbb0510d3333f763d3d117ca56da350a30ff769049c6851b4c", 18 | "zh:c17a405fe50bb16947b189a30e2c6e5983105023fa0c45bb57fb5e63232b316d", 19 | "zh:d0c2a0bec642444fd2eb1ecc13e5716bcfe30c80aae5622c8a5692b7af143a57", 20 | "zh:dd469fa460f4ce8ebd6a107babf13b1aebee9b2e274f216155f62c23df67c228", 21 | ] 22 | } 23 | 24 | provider "registry.terraform.io/hashicorp/hcp" { 25 | version = "0.37.0" 26 | hashes = [ 27 | "h1:TCRA4FTsN9F0tn/MlUiGu1LR7XPLQ7aLeCb6AI1wgIk=", 28 | "zh:0fa82a384b25a58b65523e0ea4768fa1212b1f5cfc0c9379d31162454fedcc9d", 29 | "zh:1d19dcb701571d84ce59d7f77153c16ee78fb8b84759cd3db0aaee80e2b4fd2b", 30 | "zh:2d418a6a4ec94ca00a4e486e7a23ba9d9d88d8fc7b7a0613148b96987a26ad8d", 31 | "zh:4ab0bc44d23f2f19e22e2fca33317ffed2d660521c7877062c25b69e19939618", 32 | "zh:4ea99dc05748e719902b3f8f5941f498f44ac3930ca8ee94bea5119cf212fb89", 33 | "zh:5d1af6ce978a315907c2ace567b825350fe084028ef02906b9c1ca54bd5345a3", 34 | "zh:5d2887f495b6850b93a082315961e27c7a89280d1c2c4d29c888e5b4480438be", 35 | "zh:8e5280dbd3530ba8a13eae4e0a23f369e0afd10909ee27368f6d71da0b547706", 36 | "zh:97c6a45ca40406a799467f1bcc90fcb7ff7429ad0412701f21176e4a6c82fa03", 37 | "zh:ba3d9cf3975f92af4cb3836cc9566c4812f4292090d6fae3b15d7116fe0b540d", 38 | "zh:bd7056974ea14889487e12b49864a805a74143b0486c3f2334d803bc17a64064", 39 | "zh:eb699ade9c5ff2cd8d8fab29d9ff8440387f6c6adea5a884e885f11d3e62ba54", 40 | ] 41 | } 42 | 43 | provider "registry.terraform.io/hashicorp/http" { 44 | version = "3.0.1" 45 | hashes = [ 46 | "h1:wEnSFj3SX3kPMQAv8o66FzFAc/G4CD5j4U0w0sxtQDY=", 47 | "zh:3b161998147d8cc3986a1580ddb065009ab628747424934cbcb9d221783541f8", 48 | "zh:62c78b565cde08d8e3b98e8138cd8e46b50fdc2ddc560ac1f62b5646ce8e9b1f", 49 | "zh:69ba560cd6360a285e83e1c220ab140d3119371850756ff2ed0abe39d362ea49", 50 | "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", 51 | "zh:95f38aebfa176a3424a329bc0f2e958bcf5a1f98d91dee21a436ca670fb2d570", 52 | "zh:97eae729eb859948201d4393761f5c1a7ffe84046473527f65163f062d9af5d9", 53 | "zh:b42de839114707e2fcfdf5ebf3a89129e5e17ebb5f84651c5775daecd776dc3b", 54 | "zh:c47fa93605b8378504008534e0057e295d209a2128553c7b1bcc4fc7f6efafa2", 55 | "zh:d9d4fe5143f80c1ccf22b055f069445ab7470942bb46027dadda8f3bc62d2780", 56 | "zh:f051820764c50f4736d21e40d9b13a1ffde678748a9e6e1ef22a26adf27db9bf", 57 | "zh:f67c9b73998fce13e94623be9b7afe89b30e3e6d34b504f765a344b11b8808b8", 58 | "zh:f7d255dac5a73d30c7e629699fdf064decf705cd701d29e2120cef7bf0fb1d7f", 59 | ] 60 | } 61 | 62 | provider "registry.terraform.io/hashicorp/vault" { 63 | version = "3.8.0" 64 | constraints = "3.8.0" 65 | hashes = [ 66 | "h1:T4UoPuJbAio9ZML6vpNV4bIe5Tp6f5UFGTRBtkEv/vM=", 67 | "zh:2c807352fd061f31d2972f131b74ab2e2c47031760a9f18b6f4b4a699d384969", 68 | "zh:3c5d6334c367c41d570f0eb226be0dfbdb31034669b8914b509f145a279c2bfa", 69 | "zh:4ce3887e53cc9536bfd500fac09caaab93084ed145532a521826a5093e7f8dd7", 70 | "zh:6990eac4216fb8d7fcbe0a483cc1c6a077d0e970db84fb1c0b9032158b555c0e", 71 | "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", 72 | "zh:939576f814ee4406131bdd3564cee041b05176d2e0a0b55e8081019348125e76", 73 | "zh:a0545395bd6039f7c9998113ada4334717eb1c74fee4ece7da1d4f3e6d5ef7ba", 74 | "zh:a086e5e4fdadcb0492f48074047954cc6c437b9ee57d9ec7ba850fb7cb5455a8", 75 | "zh:c997156a7c23fa06304d7e22cfd64407e9ed69237c5780d20026521ce2be478d", 76 | "zh:d47ad773cf50d703450cf301872cbc33938712a5ae491dfebf77611e1bcb0237", 77 | "zh:d95de02ccc23416e2eefb689c94046a5dcb4c65ab96cebc61838c5b1ef70e1d3", 78 | "zh:f166c7ed64c12978c4296d477ca508df82791648e6e9ff523268c1d361493851", 79 | ] 80 | } 81 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Demo for Nomad cluster backed by HCP Consul and Vault 2 | 3 | It will provision VPC, HVN, connect it and deploy Consul, Vault in HCP and Nomad in AWS 4 | 5 | Just `terraform apply` in the root directory, it will provision everything 6 | 7 | # Disclaimer 8 | 9 | It'll spin non free resources 10 | 11 | ## Variables 12 | 13 | AWS and HCP credentials are needed: 14 | 15 | - HCP_CLIENT_ID 16 | - HCP_CLIENT_SECRET 17 | - AWS_ACCESS_KEY_ID 18 | - AWS_SECRET_ACCESS_KEY 19 | 20 | -------------------------------------------------------------------------------- /hcp.tf: -------------------------------------------------------------------------------- 1 | resource "hcp_hvn" "demo_hcp_hvn" { 2 | hvn_id = "demo-hvn" 3 | cloud_provider = "aws" 4 | region = var.region 5 | } 6 | 7 | resource "hcp_aws_network_peering" "peer" { 8 | hvn_id = hcp_hvn.demo_hcp_hvn.hvn_id 9 | peering_id = "nomad" 10 | peer_vpc_id = module.vpc.vpc_id 11 | peer_account_id = module.vpc.vpc_owner_id 12 | peer_vpc_region = var.region 13 | } 14 | 15 | resource "hcp_hvn_route" "hvn-to-vpc" { 16 | hvn_link = hcp_hvn.demo_hcp_hvn.self_link 17 | hvn_route_id = "hvn-to-vpc" 18 | destination_cidr = module.vpc.vpc_cidr_block 19 | target_link = hcp_aws_network_peering.peer.self_link 20 | } 21 | 22 | resource "aws_vpc_peering_connection_accepter" "peer" { 23 | vpc_peering_connection_id = hcp_aws_network_peering.peer.provider_peering_id 24 | auto_accept = true 25 | accepter { 26 | allow_remote_vpc_dns_resolution = true 27 | } 28 | } 29 | 30 | resource "aws_route" "hvn-peering" { 31 | route_table_id = module.vpc.public_route_table_ids[0] 32 | destination_cidr_block = hcp_hvn.demo_hcp_hvn.cidr_block 33 | vpc_peering_connection_id = aws_vpc_peering_connection_accepter.peer.id 34 | } 35 | 36 | resource "hcp_vault_cluster" "demo_hcp_vault" { 37 | hvn_id = hcp_hvn.demo_hcp_hvn.hvn_id 38 | cluster_id = "demo-vault" 39 | public_endpoint = true 40 | } 41 | 42 | resource "hcp_vault_cluster_admin_token" "demo_hcp_vault_token" { 43 | cluster_id = hcp_vault_cluster.demo_hcp_vault.cluster_id 44 | } 45 | 46 | resource "hcp_consul_cluster" "demo_hcp_consul" { 47 | hvn_id = hcp_hvn.demo_hcp_hvn.hvn_id 48 | cluster_id = "demo-consul" 49 | tier = "development" 50 | public_endpoint = true 51 | } 52 | -------------------------------------------------------------------------------- /nomad.tf: -------------------------------------------------------------------------------- 1 | data "aws_ami" "base" { 2 | most_recent = true 3 | 4 | # If we change the AWS Account in which test are run, update this value. 5 | owners = ["099720109477"] 6 | 7 | filter { 8 | name = "virtualization-type" 9 | values = ["hvm"] 10 | } 11 | 12 | filter { 13 | name = "name" 14 | values = ["ubuntu/images/*ubuntu-jammy-22.04-amd64-server-*"] 15 | } 16 | } 17 | 18 | 19 | #--------------------------------------------------------------------------------------------------------------------- 20 | # DEPLOY THE SERVER NODES 21 | # --------------------------------------------------------------------------------------------------------------------- 22 | 23 | resource "aws_security_group" "demo" { 24 | name = "HCP Enabled Nomad Demo" 25 | vpc_id = module.vpc.vpc_id 26 | ingress { 27 | description = "Allow All" 28 | from_port = 0 29 | to_port = 0 30 | protocol = "all" 31 | cidr_blocks = ["0.0.0.0/0"] 32 | ipv6_cidr_blocks = ["::/0"] 33 | } 34 | egress { 35 | description = "Allow All" 36 | from_port = 0 37 | to_port = 0 38 | protocol = "all" 39 | cidr_blocks = ["0.0.0.0/0"] 40 | ipv6_cidr_blocks = ["::/0"] 41 | } 42 | tags = { 43 | Name = "HCP Enabled Nomad Demo" 44 | } 45 | } 46 | 47 | resource "aws_launch_template" "nomad-servers" { 48 | name = "HCP_Enabled_Nomad_Servers" 49 | 50 | block_device_mappings { 51 | device_name = "/dev/sda1" 52 | 53 | ebs { 54 | volume_size = 16 55 | } 56 | } 57 | # TODO: add IAM role 58 | # iam_instance_profile { 59 | # name = "test" 60 | # } 61 | 62 | image_id = data.aws_ami.base.image_id 63 | 64 | instance_initiated_shutdown_behavior = "terminate" 65 | 66 | instance_type = "t3.medium" 67 | 68 | metadata_options { 69 | http_endpoint = "enabled" 70 | http_tokens = "required" 71 | http_put_response_hop_limit = 1 72 | instance_metadata_tags = "enabled" 73 | } 74 | 75 | monitoring { 76 | enabled = true 77 | } 78 | 79 | vpc_security_group_ids = [aws_security_group.demo.id] 80 | 81 | tag_specifications { 82 | resource_type = "instance" 83 | 84 | tags = { 85 | Name = "Nomad-server" 86 | } 87 | } 88 | 89 | user_data = base64encode(templatefile("user-data-server.sh", { 90 | nomad_region = var.region, 91 | nomad_datacenter = var.cluster_name, 92 | consul_ca_file = base64decode(hcp_consul_cluster.demo_hcp_consul.consul_ca_file), 93 | consul_gossip_encrypt_key = jsondecode(base64decode(hcp_consul_cluster.demo_hcp_consul.consul_config_file)).encrypt, 94 | consul_acl_token = hcp_consul_cluster.demo_hcp_consul.consul_root_token_secret_id, 95 | vault_endpoint = hcp_vault_cluster.demo_hcp_vault.vault_private_endpoint_url, 96 | vault_token = vault_token.nomad_server.client_token 97 | })) 98 | 99 | } 100 | 101 | resource "aws_autoscaling_group" "nomad-servers" { 102 | name = "HCP Enabled Nomad servers" 103 | vpc_zone_identifier = module.vpc.public_subnets 104 | 105 | desired_capacity = 3 106 | max_size = 3 107 | min_size = 3 108 | 109 | launch_template { 110 | id = aws_launch_template.nomad-servers.id 111 | version = "$Latest" 112 | } 113 | } 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | #------------------------ 122 | 123 | 124 | 125 | resource "aws_launch_template" "nomad-clients" { 126 | name = "HCP_Enabled_Nomad_Clients" 127 | 128 | block_device_mappings { 129 | device_name = "/dev/sda1" 130 | 131 | ebs { 132 | volume_size = 16 133 | } 134 | } 135 | # TODO: add IAM role 136 | # iam_instance_profile { 137 | # name = "test" 138 | # } 139 | 140 | image_id = data.aws_ami.base.image_id 141 | 142 | instance_initiated_shutdown_behavior = "terminate" 143 | 144 | instance_type = "t3.medium" 145 | 146 | metadata_options { 147 | http_endpoint = "enabled" 148 | http_tokens = "required" 149 | http_put_response_hop_limit = 1 150 | instance_metadata_tags = "enabled" 151 | } 152 | 153 | monitoring { 154 | enabled = true 155 | } 156 | 157 | vpc_security_group_ids = [aws_security_group.demo.id] 158 | 159 | tag_specifications { 160 | resource_type = "instance" 161 | 162 | tags = { 163 | Name = "Nomad-Client" 164 | } 165 | } 166 | 167 | user_data = base64encode(templatefile("user-data-client.sh", { 168 | nomad_region = var.region, 169 | nomad_datacenter = var.cluster_name, 170 | consul_ca_file = base64decode(hcp_consul_cluster.demo_hcp_consul.consul_ca_file), 171 | consul_gossip_encrypt_key = jsondecode(base64decode(hcp_consul_cluster.demo_hcp_consul.consul_config_file)).encrypt, 172 | consul_acl_token = hcp_consul_cluster.demo_hcp_consul.consul_root_token_secret_id, 173 | vault_endpoint = hcp_vault_cluster.demo_hcp_vault.vault_private_endpoint_url, 174 | })) 175 | 176 | } 177 | 178 | resource "aws_autoscaling_group" "nomad-clients" { 179 | name = "HCP Enabled Nomad Clients" 180 | vpc_zone_identifier = module.vpc.public_subnets 181 | 182 | desired_capacity = 3 183 | max_size = 3 184 | min_size = 3 185 | 186 | launch_template { 187 | id = aws_launch_template.nomad-clients.id 188 | version = "$Latest" 189 | } 190 | } 191 | 192 | 193 | # module "clients" { 194 | # source = "hashicorp/consul/aws//modules/consul-cluster" 195 | # version = "0.8.6" 196 | # # insert the 14 required variables here 197 | # allowed_inbound_cidr_blocks = ["0.0.0.0/0"] 198 | # ami_id = data.aws_ami.base.image_id 199 | # spot_price = var.spot_price 200 | # vpc_id = module.vpc.vpc_id 201 | # ssh_key_name = var.ssh_key_name 202 | 203 | # # end of required variables 204 | 205 | # cluster_name = "${var.cluster_name}-client" 206 | # cluster_size = var.num_servers 207 | # instance_type = "t3.medium" 208 | # root_volume_size = 16 209 | 210 | # # The EC2 Instances will use these tags to automatically discover each othe r and form a cluster 211 | # cluster_tag_key = var.cluster_tag_key 212 | # cluster_tag_value = var.cluster_name 213 | 214 | 215 | # user_data = templatefile("user-data-client.sh", { 216 | # nomad_region = var.region, 217 | # nomad_datacenter = var.cluster_name, 218 | # consul_ca_file = base64decode(hcp_consul_cluster.demo_hcp_consul.consul_ca_file), 219 | # consul_gossip_encrypt_key = jsondecode(base64decode(hcp_consul_cluster.demo_hcp_consul.consul_config_file)).encrypt, 220 | # consul_acl_token = hcp_consul_cluster.demo_hcp_consul.consul_root_token_secret_id, 221 | # vault_endpoint = hcp_vault_cluster.demo_hcp_vault.vault_private_endpoint_url 222 | # }) 223 | 224 | # subnet_ids = module.vpc.public_subnets 225 | 226 | # # To make testing easier, we allow Consul and SSH requests from any IP address here but in a production 227 | # # deployment, we strongly recommend you limit this to the IP address ranges of known, trusted servers inside your VPC. 228 | # allowed_ssh_cidr_blocks = ["0.0.0.0/0"] 229 | 230 | # } 231 | -------------------------------------------------------------------------------- /outputs.tf: -------------------------------------------------------------------------------- 1 | # VPC 2 | 3 | output "cidr_block" { 4 | value = module.vpc.vpc_cidr_block 5 | } 6 | 7 | output "vpc_id" { 8 | value = module.vpc.vpc_id 9 | } 10 | 11 | output "public_subnets" { 12 | value = module.vpc.public_subnets 13 | } 14 | 15 | output "public_route_table_ids" { 16 | value = module.vpc.public_route_table_ids 17 | } 18 | 19 | output "vpc_owner_id" { 20 | value = module.vpc.vpc_owner_id 21 | } 22 | 23 | 24 | # HVN 25 | 26 | output "hvn_cidr_block" { 27 | value = hcp_hvn.demo_hcp_hvn.cidr_block 28 | } 29 | 30 | output "consul_ca_file" { 31 | value = hcp_consul_cluster.demo_hcp_consul.consul_ca_file 32 | } 33 | 34 | output "consul_gossip_encrypt_key" { 35 | value = jsondecode(base64decode(hcp_consul_cluster.demo_hcp_consul.consul_config_file)).encrypt 36 | } 37 | 38 | output "consul_root_token_secret_id" { 39 | value = hcp_consul_cluster.demo_hcp_consul.consul_root_token_secret_id 40 | sensitive = true 41 | } 42 | 43 | output "vault_private_endpoint_url" { 44 | value = hcp_vault_cluster.demo_hcp_vault.vault_private_endpoint_url 45 | } 46 | 47 | output "vault_public_endpoint_url" { 48 | value = hcp_vault_cluster.demo_hcp_vault.vault_public_endpoint_url 49 | } 50 | 51 | output "vault_admin_token" { 52 | value = hcp_vault_cluster_admin_token.demo_hcp_vault_token.token 53 | sensitive = true 54 | } 55 | -------------------------------------------------------------------------------- /providers.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | hcp = { 4 | source = "hashicorp/hcp" 5 | version = "0.37.0" 6 | } 7 | vault = { 8 | source = "hashicorp/vault" 9 | version = "3.8.0" 10 | } 11 | aws = { 12 | source = "hashicorp/aws" 13 | version = "4.24.0" 14 | } 15 | } 16 | } 17 | 18 | provider "hcp" { 19 | # Configuration options 20 | } 21 | 22 | provider "aws" { 23 | region = var.region 24 | } 25 | 26 | 27 | provider "vault" { 28 | address = hcp_vault_cluster.demo_hcp_vault.vault_public_endpoint_url 29 | namespace = "admin" 30 | token = hcp_vault_cluster_admin_token.demo_hcp_vault_token.token 31 | } 32 | -------------------------------------------------------------------------------- /user-data-client.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # This script is meant to be run in the User Data of each EC2 Instance while it's booting. 3 | set -e 4 | set -x 5 | 6 | export TERM=xterm-256color 7 | export DEBIAN_FRONTEND=noninteractive 8 | 9 | apt-get update 10 | apt-get install -y \ 11 | apt-transport-https \ 12 | ca-certificates \ 13 | curl \ 14 | gnupg-agent \ 15 | software-properties-common \ 16 | jq \ 17 | unzip 18 | 19 | curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - 20 | 21 | add-apt-repository \ 22 | "deb [arch=amd64] https://download.docker.com/linux/ubuntu \ 23 | $(lsb_release -cs) \ 24 | stable" 25 | 26 | apt-get update 27 | apt-get install -y docker-ce docker-ce-cli containerd.io 28 | 29 | 30 | ##### 31 | # Configure resolving 32 | ##### 33 | 34 | echo "Determining local IP address" 35 | LOCAL_IPV4=$(ec2metadata --local-ipv4) 36 | 37 | 38 | mkdir -p /etc/systemd/resolved.conf.d 39 | cat << EOSDRCF >/etc/systemd/resolved.conf.d/consul.conf 40 | # Enable forward lookup of the 'consul' domain: 41 | [Resolve] 42 | Cache=no 43 | DNS=127.0.0.1:8600 44 | Domains=~.consul 45 | EOSDRCF 46 | 47 | cat << EOSDRLF >/etc/systemd/resolved.conf.d/listen.conf 48 | # Enable listener on private ip: 49 | [Resolve] 50 | DNSStubListenerExtra=$${LOCAL_IPV4} 51 | EOSDRLF 52 | 53 | systemctl restart systemd-resolved.service 54 | 55 | cat << EODDJ >/etc/docker/daemon.json 56 | { 57 | "dns": ["$${LOCAL_IPV4}"], 58 | "dns-search": ["service.consul"] 59 | } 60 | EODDJ 61 | 62 | systemctl restart docker.service 63 | 64 | 65 | ###### 66 | # Install Consul and Nomad 67 | ###### 68 | 69 | echo "Checking latest Consul and Nomad versions..." 70 | CHECKPOINT_URL="https://checkpoint-api.hashicorp.com/v1/check" 71 | CONSUL_VERSION=$(curl -s "$${CHECKPOINT_URL}"/consul | jq -r .current_version) 72 | NOMAD_VERSION=$(curl -s "$${CHECKPOINT_URL}"/nomad | jq -r .current_version) 73 | 74 | cd /tmp/ 75 | 76 | echo "Fetching Consul version $${CONSUL_VERSION} ..." 77 | curl -s https://releases.hashicorp.com/consul/$${CONSUL_VERSION}/consul_$${CONSUL_VERSION}_linux_amd64.zip -o consul.zip 78 | echo "Installing Consul version $${CONSUL_VERSION} ..." 79 | unzip consul.zip 80 | chmod +x consul 81 | mv consul /usr/local/bin/consul 82 | 83 | echo "Fetching Nomad version $${NOMAD_VERSION} ..." 84 | curl -s https://releases.hashicorp.com/nomad/$${NOMAD_VERSION}/nomad_$${NOMAD_VERSION}_linux_amd64.zip -o nomad.zip 85 | echo "Installing Nomad version $${NOMAD_VERSION} ..." 86 | unzip nomad.zip 87 | chmod +x nomad 88 | mv nomad /usr/local/bin/nomad 89 | 90 | ######## 91 | # Consul config 92 | ######## 93 | 94 | mkdir -p /etc/consul.d 95 | mkdir -p /var/lib/consul 96 | cat << EOCCF >/etc/consul.d/client.hcl 97 | "acl" = { 98 | "default_policy" = "deny" 99 | 100 | "down_policy" = "async-cache" 101 | 102 | "enabled" = true 103 | } 104 | 105 | "auto_encrypt" = { 106 | "tls" = true 107 | } 108 | 109 | "ca_file" = "/var/lib/consul/ca.pem" 110 | 111 | "datacenter" = "demo-consul" 112 | 113 | "encrypt" = "${consul_gossip_encrypt_key}" 114 | 115 | "encrypt_verify_incoming" = true 116 | 117 | "encrypt_verify_outgoing" = true 118 | 119 | "log_level" = "INFO" 120 | 121 | "retry_join" = ["demo-consul.private.consul.6db67239-e33c-447f-be31-72943a3b3533.aws.hashicorp.cloud"] 122 | 123 | "server" = false 124 | 125 | "ui" = true 126 | 127 | "verify_outgoing" = true 128 | 129 | advertise_addr = "{{ GetPrivateIP }}" 130 | client_addr = "0.0.0.0" 131 | data_dir = "/var/lib/consul" 132 | EOCCF 133 | 134 | cat << EOCACF >/etc/consul.d/acl.hcl 135 | acl = { 136 | tokens = { 137 | agent = "${consul_acl_token}" 138 | } 139 | } 140 | EOCACF 141 | 142 | cat << EOCCA >/var/lib/consul/ca.pem 143 | ${consul_ca_file} 144 | EOCCA 145 | 146 | 147 | cat << EOCSU >/etc/systemd/system/consul.service 148 | [Unit] 149 | Description="HashiCorp Consul - A service mesh solution" 150 | Documentation=https://www.consul.io/ 151 | Requires=network-online.target 152 | After=network-online.target 153 | 154 | [Service] 155 | Type=notify 156 | ExecStart=/usr/local/bin/consul agent -config-dir=/etc/consul.d/ 157 | ExecReload=/bin/kill --signal HUP $MAINPID 158 | KillMode=process 159 | KillSignal=SIGTERM 160 | Restart=on-failure 161 | LimitNOFILE=65536 162 | 163 | [Install] 164 | WantedBy=multi-user.target 165 | EOCSU 166 | 167 | ########## 168 | # Nomad config 169 | ########## 170 | 171 | cat << EONEF >/etc/default/nomad 172 | CONSUL_HTTP_TOKEN=${consul_acl_token} 173 | VAULT_NAMESPACE=admin 174 | EONEF 175 | 176 | mkdir -p /etc/nomad.d/ 177 | cat << EONCF >/etc/nomad.d/client.hcl 178 | bind_addr = "0.0.0.0" 179 | region = "${nomad_region}" 180 | datacenter = "${nomad_datacenter}" 181 | data_dir = "/var/lib/nomad/" 182 | log_level = "DEBUG" 183 | leave_on_interrupt = true 184 | leave_on_terminate = true 185 | client { 186 | enabled = true 187 | } 188 | EONCF 189 | 190 | cat << EONVCF >/etc/nomad.d/vault.hcl 191 | vault { 192 | enabled = true 193 | address = "https://${vault_endpoint}:8200" 194 | } 195 | EONVCF 196 | 197 | cat << EONSU >/etc/systemd/system/nomad.service 198 | [Unit] 199 | Description=nomad agent 200 | Requires=network-online.target consul.service 201 | After=network-online.target consul.service 202 | [Service] 203 | LimitNOFILE=65536 204 | Restart=on-failure 205 | EnvironmentFile=/etc/default/nomad 206 | ExecStart=/usr/local/bin/nomad agent -config /etc/nomad.d 207 | KillSignal=SIGINT 208 | RestartSec=5s 209 | [Install] 210 | WantedBy=multi-user.target 211 | EONSU 212 | 213 | systemctl daemon-reload 214 | systemctl start consul 215 | systemctl start nomad 216 | -------------------------------------------------------------------------------- /user-data-server.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # This script is meant to be run in the User Data of each EC2 Instance while it's booting. 3 | set -e 4 | set -x 5 | 6 | export TERM=xterm-256color 7 | export DEBIAN_FRONTEND=noninteractive 8 | 9 | apt-get update 10 | apt-get install -y \ 11 | apt-transport-https \ 12 | ca-certificates \ 13 | curl \ 14 | gnupg-agent \ 15 | software-properties-common \ 16 | jq \ 17 | unzip 18 | 19 | echo "Checking latest Consul and Nomad versions..." 20 | CHECKPOINT_URL="https://checkpoint-api.hashicorp.com/v1/check" 21 | CONSUL_VERSION=$(curl -s "$${CHECKPOINT_URL}"/consul | jq -r .current_version) 22 | NOMAD_VERSION=$(curl -s "$${CHECKPOINT_URL}"/nomad | jq -r .current_version) 23 | 24 | cd /tmp/ 25 | 26 | echo "Fetching Consul version $${CONSUL_VERSION} ..." 27 | curl -s https://releases.hashicorp.com/consul/$${CONSUL_VERSION}/consul_$${CONSUL_VERSION}_linux_amd64.zip -o consul.zip 28 | echo "Installing Consul version $${CONSUL_VERSION} ..." 29 | unzip consul.zip 30 | chmod +x consul 31 | mv consul /usr/local/bin/consul 32 | 33 | echo "Fetching Nomad version $${NOMAD_VERSION} ..." 34 | curl -s https://releases.hashicorp.com/nomad/$${NOMAD_VERSION}/nomad_$${NOMAD_VERSION}_linux_amd64.zip -o nomad.zip 35 | echo "Installing Nomad version $${NOMAD_VERSION} ..." 36 | unzip nomad.zip 37 | chmod +x nomad 38 | mv nomad /usr/local/bin/nomad 39 | 40 | ######## 41 | # Consul config 42 | ######## 43 | 44 | mkdir -p /etc/consul.d 45 | mkdir -p /var/lib/consul 46 | cat << EOCCF >/etc/consul.d/client.hcl 47 | "acl" = { 48 | "default_policy" = "deny" 49 | 50 | "down_policy" = "async-cache" 51 | 52 | "enabled" = true 53 | } 54 | 55 | "auto_encrypt" = { 56 | "tls" = true 57 | } 58 | 59 | "ca_file" = "/var/lib/consul/ca.pem" 60 | 61 | "datacenter" = "demo-consul" 62 | 63 | "encrypt" = "${consul_gossip_encrypt_key}" 64 | 65 | "encrypt_verify_incoming" = true 66 | 67 | "encrypt_verify_outgoing" = true 68 | 69 | "log_level" = "INFO" 70 | 71 | "retry_join" = ["demo-consul.private.consul.6db67239-e33c-447f-be31-72943a3b3533.aws.hashicorp.cloud"] 72 | 73 | "server" = false 74 | 75 | "ui" = true 76 | 77 | "verify_outgoing" = true 78 | 79 | advertise_addr = "{{ GetPrivateIP }}" 80 | client_addr = "0.0.0.0" 81 | data_dir = "/var/lib/consul" 82 | EOCCF 83 | 84 | cat << EOCACF >/etc/consul.d/acl.hcl 85 | acl = { 86 | tokens = { 87 | agent = "${consul_acl_token}" 88 | } 89 | } 90 | EOCACF 91 | 92 | cat << EOCCA >/var/lib/consul/ca.pem 93 | ${consul_ca_file} 94 | EOCCA 95 | 96 | 97 | cat << EOCSU >/etc/systemd/system/consul.service 98 | [Unit] 99 | Description="HashiCorp Consul - A service mesh solution" 100 | Documentation=https://www.consul.io/ 101 | Requires=network-online.target 102 | After=network-online.target 103 | 104 | [Service] 105 | Type=notify 106 | ExecStart=/usr/local/bin/consul agent -config-dir=/etc/consul.d/ 107 | ExecReload=/bin/kill --signal HUP $MAINPID 108 | KillMode=process 109 | KillSignal=SIGTERM 110 | Restart=on-failure 111 | LimitNOFILE=65536 112 | 113 | [Install] 114 | WantedBy=multi-user.target 115 | EOCSU 116 | 117 | ########## 118 | # Nomad config 119 | ########## 120 | 121 | cat << EONEF >/etc/default/nomad 122 | CONSUL_HTTP_TOKEN=${consul_acl_token} 123 | VAULT_NAMESPACE=admin 124 | VAULT_TOKEN=${vault_token} 125 | EONEF 126 | 127 | mkdir -p /etc/nomad.d/ 128 | cat << EONCF >/etc/nomad.d/server.hcl 129 | bind_addr = "0.0.0.0" 130 | region = "${nomad_region}" 131 | datacenter = "${nomad_datacenter}" 132 | data_dir = "/var/lib/nomad/" 133 | log_level = "DEBUG" 134 | leave_on_interrupt = true 135 | leave_on_terminate = true 136 | server { 137 | enabled = true 138 | bootstrap_expect = 3 139 | } 140 | EONCF 141 | 142 | cat << EONVF >/etc/nomad.d/vault.hcl 143 | vault { 144 | enabled = true 145 | address = "https://${vault_endpoint}:8200" 146 | create_from_role = "nomad-cluster" 147 | } 148 | EONVF 149 | 150 | cat << EONSU >/etc/systemd/system/nomad.service 151 | [Unit] 152 | Description=nomad agent 153 | Requires=network-online.target consul.service 154 | After=network-online.target consul.service 155 | [Service] 156 | LimitNOFILE=65536 157 | Restart=on-failure 158 | EnvironmentFile=/etc/default/nomad 159 | ExecStart=/usr/local/bin/nomad agent -config /etc/nomad.d 160 | KillSignal=SIGINT 161 | RestartSec=5s 162 | [Install] 163 | WantedBy=multi-user.target 164 | EONSU 165 | 166 | systemctl daemon-reload 167 | systemctl start consul 168 | systemctl start nomad 169 | -------------------------------------------------------------------------------- /variables.tf: -------------------------------------------------------------------------------- 1 | # VPC # 2 | 3 | 4 | variable "region" { 5 | description = "The region of the VPC, HCP HVN and Vault & Consul clusters" 6 | type = string 7 | default = "eu-central-1" 8 | } 9 | 10 | variable "name" { 11 | type = string 12 | default = "hcpenablednomad" 13 | } 14 | variable "public_subnets" { 15 | type = list(any) 16 | default = [ 17 | "10.0.20.0/24", 18 | "10.0.21.0/24", 19 | "10.0.22.0/24", 20 | ] 21 | } 22 | variable "cidr" { 23 | default = "10.0.0.0/16" 24 | } 25 | 26 | # Compute 27 | 28 | variable "ami_id" { 29 | description = "The ID of the AMI to run in the cluster. This should be an AMI built from the Packer template under examples/consul-ami/consul.json. To keep this example simple, we run the same AMI on both server and client nodes, but in real-world usage, your client nodes would also run your apps. If the default value is used, Terraform will look up the latest AMI build automatically." 30 | type = string 31 | default = null 32 | } 33 | 34 | variable "cluster_name" { 35 | description = "What to name the Consul cluster and all of its associated resources" 36 | type = string 37 | default = "hc_demo" 38 | } 39 | 40 | variable "num_servers" { 41 | description = "The number of Server nodes to deploy. We strongly recommend using 3 or 5." 42 | type = number 43 | default = 3 44 | } 45 | 46 | variable "num_clients" { 47 | description = "The number of client nodes to deploy. You typically run the Consul client alongside your apps, so set this value to however many Instances make sense for your app code." 48 | type = number 49 | default = 3 50 | } 51 | 52 | variable "cluster_tag_key" { 53 | description = "The tag the EC2 Instances will look for to automatically discover each other and form a cluster." 54 | type = string 55 | default = "hc_demo-servers" 56 | } 57 | 58 | variable "ssh_key_name" { 59 | description = "The name of an EC2 Key Pair that can be used to SSH to the EC2 Instances in this cluster. Set to an empty string to not associate a Key Pair." 60 | type = string 61 | default = null 62 | } 63 | 64 | variable "vpc_id" { 65 | description = "The ID of the VPC in which the nodes will be deployed. Uses default VPC if not supplied." 66 | type = string 67 | default = null 68 | } 69 | 70 | variable "spot_price" { 71 | description = "The maximum hourly price to pay for EC2 Spot Instances." 72 | type = number 73 | default = null 74 | } 75 | 76 | variable "allowed_inbound_cidr_blocks" { 77 | description = "A list of CIDR-formatted IP address ranges from which the EC2 Instances will allow connections to Consul" 78 | type = list(string) 79 | default = ["0.0.0.0/0"] 80 | } 81 | 82 | variable "nomad_datacenter" { 83 | type = string 84 | description = "Nomad Datacenter" 85 | default = "demo" 86 | } 87 | -------------------------------------------------------------------------------- /vault-config.tf: -------------------------------------------------------------------------------- 1 | data "http" "nomad_server_policy" { 2 | url = "https://nomadproject.io/data/vault/nomad-server-policy.hcl" 3 | } 4 | 5 | resource "vault_policy" "nomad-server" { 6 | name = "nomad-server" 7 | policy = data.http.nomad_server_policy.response_body 8 | depends_on = [ 9 | hcp_vault_cluster.demo_hcp_vault 10 | ] 11 | } 12 | 13 | resource "vault_token_auth_backend_role" "nomad-cluster" { 14 | role_name = "nomad-cluster" 15 | disallowed_policies = ["nomad-server"] 16 | orphan = true 17 | token_period = "259200" 18 | renewable = true 19 | token_explicit_max_ttl = 0 20 | depends_on = [ 21 | hcp_vault_cluster.demo_hcp_vault 22 | ] 23 | } 24 | 25 | resource "vault_token" "nomad_server" { 26 | policies = ["nomad-server"] 27 | renewable = true 28 | ttl = "72h" 29 | no_parent = true 30 | depends_on = [ 31 | hcp_vault_cluster.demo_hcp_vault 32 | ] 33 | } 34 | 35 | output "nomad_server_vault_token" { 36 | value = vault_token.nomad_server.client_token 37 | sensitive = true 38 | depends_on = [ 39 | hcp_vault_cluster.demo_hcp_vault 40 | ] 41 | } 42 | -------------------------------------------------------------------------------- /vpc.tf: -------------------------------------------------------------------------------- 1 | module "vpc" { 2 | source = "terraform-aws-modules/vpc/aws" 3 | name = var.name 4 | cidr = var.cidr 5 | 6 | azs = slice(data.aws_availability_zones.available.*.names[0], 0, 3) 7 | public_subnets = var.public_subnets 8 | public_subnet_tags = { 9 | Name = "${var.name}-public" 10 | } 11 | enable_dns_hostnames = true 12 | enable_ipv6 = true 13 | assign_ipv6_address_on_creation = true 14 | public_subnet_ipv6_prefixes = [0, 1, 2] 15 | 16 | tags = { 17 | Terraform = "true" 18 | Name = var.name 19 | } 20 | } 21 | 22 | data "aws_availability_zones" "available" { 23 | state = "available" 24 | } 25 | --------------------------------------------------------------------------------