├── cloud ├── aws │ ├── .gitignore │ ├── terraform │ │ ├── control │ │ │ ├── variables.tf │ │ │ ├── terraform.tfvars.sample │ │ │ ├── main.tf │ │ │ ├── outputs.tf │ │ │ └── .terraform.lock.hcl │ │ └── modules │ │ │ └── aws-hashistack │ │ │ ├── templates │ │ │ ├── user-data-client.sh │ │ │ ├── user-data-server.sh │ │ │ └── aws_autoscaler.nomad.tpl │ │ │ ├── instances.tf │ │ │ ├── templates.tf │ │ │ ├── outputs.tf │ │ │ ├── asg.tf │ │ │ ├── elb.tf │ │ │ ├── iam.tf │ │ │ ├── variables.tf │ │ │ └── sg.tf │ ├── packer │ │ └── aws-packer.pkr.hcl │ └── README.md ├── gcp │ ├── terraform │ │ ├── .gitignore │ │ ├── control │ │ │ ├── terraform.tfvars.sample │ │ │ ├── variables.tf │ │ │ ├── main.tf │ │ │ ├── output.tf │ │ │ └── .terraform.lock.hcl │ │ └── modules │ │ │ └── gcp-hashistack │ │ │ ├── provider.tf │ │ │ ├── project.tf │ │ │ ├── address.tf │ │ │ ├── templates │ │ │ ├── user-data-client.sh │ │ │ ├── user-data-server.sh │ │ │ └── gcp_autoscaler.nomad.tpl │ │ │ ├── output.tf │ │ │ ├── locals.tf │ │ │ ├── autoscaler.tf │ │ │ ├── image.tf │ │ │ ├── network.tf │ │ │ ├── vm.tf │ │ │ ├── firewall.tf │ │ │ ├── instance_group.tf │ │ │ ├── lb.tf │ │ │ └── variables.tf │ ├── packer │ │ └── gcp-packer.pkr.hcl │ └── README.md ├── azure │ ├── terraform │ │ ├── .gitignore │ │ ├── modules │ │ │ └── azure-hashistack │ │ │ │ ├── rg.tf │ │ │ │ ├── templates │ │ │ │ ├── user-data-client.sh │ │ │ │ ├── user-data-server.sh │ │ │ │ └── azure_autoscaler.nomad.tpl │ │ │ │ ├── ssh.tf │ │ │ │ ├── network.tf │ │ │ │ ├── image.tf │ │ │ │ ├── templates.tf │ │ │ │ ├── outputs.tf │ │ │ │ ├── variables.tf │ │ │ │ ├── clients.tf │ │ │ │ ├── servers_lb.tf │ │ │ │ ├── servers.tf │ │ │ │ ├── sg.tf │ │ │ │ └── clients_lb.tf │ │ └── control │ │ │ ├── main.tf │ │ │ ├── outputs.tf │ │ │ └── .terraform.lock.hcl │ ├── packer │ │ └── azure-packer.pkr.hcl │ └── README.md └── shared │ ├── packer │ ├── config │ │ ├── 99-default.dnsmasq.aws │ │ ├── 99-default.dnsmasq.azure │ │ ├── 99-default.dnsmasq.gcp │ │ ├── 10-consul.dnsmasq │ │ ├── consul_client.hcl │ │ ├── nomad.hcl │ │ ├── consul.hcl │ │ ├── consul_aws.service │ │ ├── consul_azure.service │ │ ├── consul_gcp.service │ │ ├── nomad.service │ │ └── nomad_client.hcl │ └── scripts │ │ ├── net.sh │ │ ├── client.sh │ │ ├── setup.sh │ │ └── server.sh │ └── terraform │ └── modules │ └── shared-nomad-jobs │ ├── variables.tf │ ├── version.tf │ ├── jobs.tf │ └── files │ ├── grafana.nomad.tpl │ ├── prometheus.nomad │ ├── webapp.nomad │ └── traefik.nomad ├── .github └── CODEOWNERS ├── vagrant ├── horizontal-app-scaling │ ├── images │ │ ├── demo_dashboard.png │ │ └── autoscaler_dashboard.png │ ├── files │ │ └── nomad.hcl │ ├── jobs │ │ ├── loki.nomad │ │ ├── prometheus.nomad │ │ ├── webapp.nomad │ │ ├── grafana.nomad │ │ ├── haproxy.nomad │ │ └── autoscaler.nomad │ ├── Vagrantfile │ └── README.md ├── shared │ ├── consul.hcl │ ├── consul.service │ └── nomad.service └── dynamic-app-sizing │ ├── files │ └── nomad.hcl │ ├── jobs │ ├── das-load-test.nomad │ ├── prometheus.nomad │ ├── das-autoscaler.nomad │ └── example.nomad │ ├── Vagrantfile │ └── README.md ├── .gitignore └── README.md /cloud/aws/.gitignore: -------------------------------------------------------------------------------- 1 | aws_autoscaler.nomad 2 | -------------------------------------------------------------------------------- /cloud/gcp/terraform/.gitignore: -------------------------------------------------------------------------------- 1 | gcp_autoscaler.nomad 2 | -------------------------------------------------------------------------------- /cloud/azure/terraform/.gitignore: -------------------------------------------------------------------------------- 1 | azure-hashistack.pem 2 | azure_autoscaler.nomad 3 | -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | # Blanket ownership of all PRs. 2 | * @cgbaker @jazzyfresh @jrasell @lgfa29 3 | -------------------------------------------------------------------------------- /cloud/gcp/terraform/control/terraform.tfvars.sample: -------------------------------------------------------------------------------- 1 | org_id = "" 2 | billing_account = "" 3 | -------------------------------------------------------------------------------- /cloud/shared/packer/config/99-default.dnsmasq.aws: -------------------------------------------------------------------------------- 1 | # 99-default.dnsmasq 2 | server=169.254.169.253 3 | -------------------------------------------------------------------------------- /cloud/shared/packer/config/99-default.dnsmasq.azure: -------------------------------------------------------------------------------- 1 | # 99-default.dnsmasq 2 | server=168.63.129.16 3 | -------------------------------------------------------------------------------- /cloud/shared/packer/config/99-default.dnsmasq.gcp: -------------------------------------------------------------------------------- 1 | # 99-default.dnsmasq 2 | server=169.254.169.254 3 | -------------------------------------------------------------------------------- /cloud/shared/packer/config/10-consul.dnsmasq: -------------------------------------------------------------------------------- 1 | # Enable forward lookup of the 'consul' domain: 2 | server=/consul/127.0.0.1#8600 3 | -------------------------------------------------------------------------------- /cloud/gcp/terraform/modules/gcp-hashistack/provider.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | google = { 4 | version = "=3.54.0" 5 | } 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /cloud/shared/terraform/modules/shared-nomad-jobs/variables.tf: -------------------------------------------------------------------------------- 1 | variable "nomad_addr" { 2 | description = "The Nomad API HTTP address." 3 | type = string 4 | } 5 | -------------------------------------------------------------------------------- /vagrant/horizontal-app-scaling/images/demo_dashboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zen/nomad-autoscaler-demos/main/vagrant/horizontal-app-scaling/images/demo_dashboard.png -------------------------------------------------------------------------------- /vagrant/horizontal-app-scaling/images/autoscaler_dashboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zen/nomad-autoscaler-demos/main/vagrant/horizontal-app-scaling/images/autoscaler_dashboard.png -------------------------------------------------------------------------------- /cloud/gcp/terraform/modules/gcp-hashistack/project.tf: -------------------------------------------------------------------------------- 1 | resource "google_project_service" "compute" { 2 | service = "compute.googleapis.com" 3 | disable_on_destroy = false 4 | } 5 | -------------------------------------------------------------------------------- /cloud/azure/terraform/modules/azure-hashistack/rg.tf: -------------------------------------------------------------------------------- 1 | resource "random_pet" "server" {} 2 | 3 | resource "azurerm_resource_group" "hashistack" { 4 | name = "hashistack-${random_pet.server.id}" 5 | location = var.location 6 | } 7 | -------------------------------------------------------------------------------- /cloud/gcp/terraform/modules/gcp-hashistack/address.tf: -------------------------------------------------------------------------------- 1 | resource "google_compute_address" "nomad_server" { 2 | name = local.server_stack_name 3 | } 4 | 5 | resource "google_compute_address" "nomad_client" { 6 | name = local.client_stack_name 7 | } 8 | -------------------------------------------------------------------------------- /cloud/shared/terraform/modules/shared-nomad-jobs/version.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_version = ">= 0.13" 3 | required_providers { 4 | nomad = { 5 | source = "hashicorp/nomad" 6 | version = ">= 1.4.6" 7 | } 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /cloud/shared/packer/config/consul_client.hcl: -------------------------------------------------------------------------------- 1 | advertise_addr = "IP_ADDRESS" 2 | 3 | bind_addr = "0.0.0.0" 4 | 5 | client_addr = "0.0.0.0" 6 | 7 | data_dir = "/opt/consul/data" 8 | 9 | log_level = "INFO" 10 | 11 | retry_join = ["RETRY_JOIN"] 12 | 13 | ui = true 14 | -------------------------------------------------------------------------------- /cloud/aws/terraform/control/variables.tf: -------------------------------------------------------------------------------- 1 | variable "owner_name" {} 2 | variable "owner_email" {} 3 | variable "region" {} 4 | variable "availability_zones" {} 5 | variable "ami" {} 6 | variable "key_name" {} 7 | 8 | variable "stack_name" { 9 | default = "hashistack" 10 | } 11 | -------------------------------------------------------------------------------- /cloud/aws/terraform/control/terraform.tfvars.sample: -------------------------------------------------------------------------------- 1 | region = "us-east-1" 2 | availability_zones = ["us-east-1a"] 3 | ami = "ami-0bd21458ecf89f85e" 4 | key_name = "my-dev-ec2-keypair" 5 | owner_name = "alovelace" 6 | owner_email = "alovelace@example.com" 7 | -------------------------------------------------------------------------------- /vagrant/shared/consul.hcl: -------------------------------------------------------------------------------- 1 | datacenter = "dc1" 2 | advertise_addr = "127.0.0.1" 3 | client_addr = "0.0.0.0" 4 | data_dir = "/opt/consul" 5 | server = true 6 | bootstrap_expect = 1 7 | ui = true 8 | 9 | telemetry { 10 | prometheus_retention_time = "30s" 11 | } 12 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # ignore Vagrant related files 2 | *.log 3 | .vagrant 4 | 5 | # ignore GoLand IDE 6 | .idea 7 | 8 | # Terraform state files. 9 | *.tfstate 10 | *.tfstate.backup 11 | *.tfstate.*.backup 12 | .terraform.tfstate.lock.info 13 | 14 | # Terraform module and provider directory. 15 | .terraform 16 | 17 | # Terraform vars file 18 | terraform.tfvars 19 | -------------------------------------------------------------------------------- /cloud/shared/packer/scripts/net.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | function net_getInterfaceAddress() { 4 | ip -4 address show "${1}" | awk '/inet / { print $2 }' | cut -d/ -f1 5 | } 6 | 7 | function net_getDefaultRouteAddress() { 8 | # Default route IP address (seems to be a good way to get host ip) 9 | ip -4 route get 1.1.1.1 | grep -oP 'src \K\S+' 10 | } 11 | -------------------------------------------------------------------------------- /cloud/shared/packer/config/nomad.hcl: -------------------------------------------------------------------------------- 1 | data_dir = "/opt/nomad/data" 2 | bind_addr = "0.0.0.0" 3 | log_level = "DEBUG" 4 | 5 | telemetry { 6 | publish_allocation_metrics = true 7 | publish_node_metrics = true 8 | prometheus_metrics = true 9 | } 10 | 11 | server { 12 | enabled = true 13 | bootstrap_expect = SERVER_COUNT 14 | } 15 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Nomad Autoscaler Demos 2 | 3 | This repository hosts a series of demos for the [Nomad Autoscaler](https://github.com/hashicorp/nomad-autoscaler). 4 | 5 | ## List of demos 6 | 7 | * [Horizontal Application Scaling](./vagrant/horizontal-app-scaling) 8 | * [Horizontal Cluster Scaling](./cloud) 9 | * [Dynamic Application Sizing](./vagrant/dynamic-app-sizing) 10 | -------------------------------------------------------------------------------- /cloud/shared/packer/config/consul.hcl: -------------------------------------------------------------------------------- 1 | advertise_addr = "IP_ADDRESS" 2 | 3 | bind_addr = "0.0.0.0" 4 | 5 | bootstrap_expect = SERVER_COUNT 6 | 7 | client_addr = "0.0.0.0" 8 | 9 | data_dir = "/opt/consul/data" 10 | 11 | log_level = "INFO" 12 | 13 | retry_join = ["RETRY_JOIN"] 14 | 15 | server = true 16 | 17 | ui = true 18 | 19 | service { 20 | name = "consul" 21 | } 22 | -------------------------------------------------------------------------------- /cloud/aws/terraform/modules/aws-hashistack/templates/user-data-client.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | exec > >(sudo tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1 6 | sudo chmod +x /ops/scripts/client.sh 7 | sudo bash -c "NOMAD_BINARY=${nomad_binary} CONSUL_BINARY=${consul_binary} /ops/scripts/client.sh \"aws\" \"${retry_join}\" \"${node_class}\"" 8 | rm -rf /ops/ 9 | -------------------------------------------------------------------------------- /cloud/gcp/terraform/modules/gcp-hashistack/templates/user-data-client.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | exec > >(sudo tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1 6 | sudo chmod +x /ops/scripts/client.sh 7 | sudo bash -c "NOMAD_BINARY=${nomad_binary} CONSUL_BINARY=${consul_binary} /ops/scripts/client.sh \"gcp\" \"${retry_join}\" \"${node_class}\"" 8 | rm -rf /ops/ 9 | -------------------------------------------------------------------------------- /cloud/aws/terraform/modules/aws-hashistack/templates/user-data-server.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | exec > >(sudo tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1 6 | sudo chmod +x /ops/scripts/server.sh 7 | sudo bash -c "NOMAD_BINARY=${nomad_binary} CONSUL_BINARY=${consul_binary} /ops/scripts/server.sh \"aws\" \"${server_count}\" \"${retry_join}\"" 8 | rm -rf /ops/ 9 | -------------------------------------------------------------------------------- /cloud/gcp/terraform/modules/gcp-hashistack/templates/user-data-server.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | exec > >(sudo tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1 6 | sudo chmod +x /ops/scripts/server.sh 7 | sudo bash -c "NOMAD_BINARY=${nomad_binary} CONSUL_BINARY=${consul_binary} /ops/scripts/server.sh \"gcp\" \"${server_count}\" \"${retry_join}\"" 8 | rm -rf /ops/ 9 | -------------------------------------------------------------------------------- /cloud/azure/terraform/modules/azure-hashistack/templates/user-data-client.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | exec > >(sudo tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1 6 | sudo chmod +x /ops/scripts/client.sh 7 | sudo bash -c "NOMAD_BINARY=${nomad_binary} CONSUL_BINARY=${consul_binary} /ops/scripts/client.sh \"azure\" \"${retry_join}\" \"${node_class}\"" 8 | rm -rf /ops/ 9 | -------------------------------------------------------------------------------- /cloud/azure/terraform/modules/azure-hashistack/templates/user-data-server.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | exec > >(sudo tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1 6 | sudo chmod +x /ops/scripts/server.sh 7 | sudo bash -c "NOMAD_BINARY=${nomad_binary} CONSUL_BINARY=${consul_binary} /ops/scripts/server.sh \"azure\" \"${server_count}\" \"${retry_join}\"" 8 | rm -rf /ops/ 9 | -------------------------------------------------------------------------------- /cloud/azure/terraform/modules/azure-hashistack/ssh.tf: -------------------------------------------------------------------------------- 1 | resource "tls_private_key" "main" { 2 | algorithm = "RSA" 3 | } 4 | 5 | resource "null_resource" "main" { 6 | provisioner "local-exec" { 7 | command = "echo \"${tls_private_key.main.private_key_pem}\" > azure-hashistack.pem" 8 | } 9 | 10 | provisioner "local-exec" { 11 | command = "chmod 600 azure-hashistack.pem" 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /cloud/shared/packer/config/consul_aws.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Consul Agent 3 | Requires=network-online.target 4 | After=network-online.target 5 | 6 | [Service] 7 | Restart=on-failure 8 | ExecStart=/usr/local/bin/consul agent -config-dir="/etc/consul.d" 9 | ExecReload=/bin/kill -HUP $MAINPID 10 | KillSignal=SIGTERM 11 | User=root 12 | Group=root 13 | 14 | [Install] 15 | WantedBy=multi-user.target 16 | -------------------------------------------------------------------------------- /cloud/shared/packer/config/consul_azure.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Consul Agent 3 | Requires=network-online.target 4 | After=network-online.target 5 | 6 | [Service] 7 | Restart=on-failure 8 | ExecStart=/usr/local/bin/consul agent -config-dir="/etc/consul.d" 9 | ExecReload=/bin/kill -HUP $MAINPID 10 | KillSignal=SIGTERM 11 | User=root 12 | Group=root 13 | 14 | [Install] 15 | WantedBy=multi-user.target 16 | -------------------------------------------------------------------------------- /cloud/shared/packer/config/consul_gcp.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Consul Agent 3 | Requires=network-online.target 4 | After=network-online.target 5 | 6 | [Service] 7 | Restart=on-failure 8 | ExecStart=/usr/local/bin/consul agent -config-dir="/etc/consul.d" 9 | ExecReload=/bin/kill -HUP $MAINPID 10 | KillSignal=SIGTERM 11 | User=root 12 | Group=root 13 | 14 | [Install] 15 | WantedBy=multi-user.target 16 | -------------------------------------------------------------------------------- /cloud/shared/packer/config/nomad.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Nomad Agent 3 | Requires=network-online.target 4 | Wants=consul.service 5 | After=consul.service 6 | 7 | [Service] 8 | Restart=on-failure 9 | ExecStart=/usr/local/bin/nomad agent -config="/etc/nomad.d/nomad.hcl" 10 | ExecReload=/bin/kill -HUP $MAINPID 11 | KillSignal=SIGTERM 12 | User=root 13 | Group=root 14 | 15 | [Install] 16 | WantedBy=multi-user.target 17 | -------------------------------------------------------------------------------- /cloud/gcp/terraform/modules/gcp-hashistack/output.tf: -------------------------------------------------------------------------------- 1 | output "google_project_id" { 2 | value = var.project_id 3 | } 4 | 5 | output "nomad_addr" { 6 | value = "http://${google_compute_address.nomad_server.address}:4646" 7 | } 8 | 9 | output "consul_addr" { 10 | value = "http://${google_compute_address.nomad_server.address}:8500" 11 | } 12 | 13 | output "client_public_ip_addr" { 14 | value = google_compute_address.nomad_client.address 15 | } 16 | -------------------------------------------------------------------------------- /vagrant/shared/consul.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description="HashiCorp Consul - A service mesh solution" 3 | Documentation=https://www.consul.io/ 4 | Requires=network-online.target 5 | After=network-online.target 6 | ConditionFileNotEmpty=/etc/consul.d/consul.hcl 7 | 8 | [Service] 9 | Type=simple 10 | ExecStart=/usr/local/bin/consul agent -config-dir=/etc/consul.d/ 11 | ExecReload=/usr/local/bin/consul reload 12 | KillMode=process 13 | Restart=on-failure 14 | LimitNOFILE=65536 15 | 16 | [Install] 17 | WantedBy=multi-user.target 18 | -------------------------------------------------------------------------------- /cloud/gcp/terraform/modules/gcp-hashistack/locals.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | zone_id = format("%s-%s", var.region, var.zone) 3 | client_regional_mig = var.client_mig_type == "regional" 4 | client_mig_name = local.client_regional_mig ? google_compute_region_instance_group_manager.nomad_client[0].name : google_compute_instance_group_manager.nomad_client[0].name 5 | stack_name = var.stack_name 6 | client_stack_name = "${var.stack_name}-nomad-client" 7 | server_stack_name = "${var.stack_name}-nomad-server" 8 | } 9 | -------------------------------------------------------------------------------- /vagrant/shared/nomad.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Nomad 3 | Documentation=https://nomadproject.io/docs/ 4 | Wants=network-online.target 5 | After=network-online.target 6 | 7 | [Service] 8 | ExecReload=/bin/kill -HUP $MAINPID 9 | ExecStart=/usr/local/bin/nomad agent -config /etc/nomad.d 10 | KillMode=process 11 | KillSignal=SIGINT 12 | LimitNOFILE=infinity 13 | LimitNPROC=infinity 14 | Restart=on-failure 15 | RestartSec=2 16 | StartLimitBurst=3 17 | StartLimitIntervalSec=10 18 | TasksMax=infinity 19 | 20 | [Install] 21 | WantedBy=multi-user.target 22 | -------------------------------------------------------------------------------- /cloud/shared/packer/config/nomad_client.hcl: -------------------------------------------------------------------------------- 1 | data_dir = "/opt/nomad/data" 2 | bind_addr = "0.0.0.0" 3 | log_level = "DEBUG" 4 | 5 | telemetry { 6 | publish_allocation_metrics = true 7 | publish_node_metrics = true 8 | prometheus_metrics = true 9 | } 10 | 11 | client { 12 | enabled = true 13 | node_class = NODE_CLASS 14 | 15 | options { 16 | "driver.raw_exec.enable" = "1" 17 | "docker.privileged.enabled" = "true" 18 | } 19 | } 20 | 21 | vault { 22 | enabled = true 23 | address = "http://active.vault.service.consul:8200" 24 | } 25 | -------------------------------------------------------------------------------- /vagrant/horizontal-app-scaling/files/nomad.hcl: -------------------------------------------------------------------------------- 1 | datacenter = "dc1" 2 | 3 | data_dir = "/opt/nomad" 4 | 5 | server { 6 | enabled = true 7 | bootstrap_expect = 1 8 | } 9 | 10 | client { 11 | enabled = true 12 | 13 | host_volume "grafana" { 14 | path = "/opt/nomad-volumes/grafana" 15 | } 16 | } 17 | 18 | plugin "docker" { 19 | config { 20 | volumes { 21 | enabled = true 22 | } 23 | } 24 | } 25 | 26 | telemetry { 27 | publish_allocation_metrics = true 28 | publish_node_metrics = true 29 | prometheus_metrics = true 30 | } 31 | -------------------------------------------------------------------------------- /cloud/azure/terraform/modules/azure-hashistack/network.tf: -------------------------------------------------------------------------------- 1 | resource "azurerm_virtual_network" "primary" { 2 | name = "virtual-network" 3 | address_space = ["10.0.0.0/16"] 4 | location = azurerm_resource_group.hashistack.location 5 | resource_group_name = azurerm_resource_group.hashistack.name 6 | } 7 | 8 | resource "azurerm_subnet" "primary" { 9 | name = "subnet" 10 | resource_group_name = azurerm_resource_group.hashistack.name 11 | virtual_network_name = azurerm_virtual_network.primary.name 12 | address_prefixes = ["10.0.2.0/24"] 13 | } 14 | -------------------------------------------------------------------------------- /cloud/gcp/terraform/control/variables.tf: -------------------------------------------------------------------------------- 1 | variable "org_id" { 2 | description = "The Google Cloud Platform organization where resources will be created." 3 | type = string 4 | } 5 | 6 | variable "billing_account" { 7 | description = "The billing account that will be linked to the project." 8 | type = string 9 | } 10 | 11 | variable "region" { 12 | description = "The region where resources will be created." 13 | type = string 14 | default = "us-central1" 15 | } 16 | 17 | variable "zone" { 18 | description = "The zone where resources will be created." 19 | type = string 20 | default = "a" 21 | } 22 | -------------------------------------------------------------------------------- /vagrant/dynamic-app-sizing/files/nomad.hcl: -------------------------------------------------------------------------------- 1 | // Setup some default parameters, including data_dir which is required. 2 | datacenter = "dc1" 3 | data_dir = "/opt/nomad" 4 | 5 | // Enable to Nomad agent to run in server, only expecting itself as a server in 6 | // the raft pool. 7 | server { 8 | enabled = true 9 | bootstrap_expect = 1 10 | } 11 | 12 | // Enable the Nomad agent to run in client mode. 13 | client { 14 | enabled = true 15 | } 16 | 17 | // Enable allocation and node metrics telemetry as well as expose them via the 18 | // API in Prometheus format. 19 | telemetry { 20 | publish_allocation_metrics = true 21 | publish_node_metrics = true 22 | prometheus_metrics = true 23 | } 24 | -------------------------------------------------------------------------------- /cloud/gcp/terraform/modules/gcp-hashistack/autoscaler.tf: -------------------------------------------------------------------------------- 1 | data "template_file" "nomad_autoscaler_jobspec" { 2 | template = file("${path.module}/templates/gcp_autoscaler.nomad.tpl") 3 | 4 | vars = { 5 | nomad_autoscaler_image = var.nomad_autoscaler_image 6 | project = var.project_id 7 | region = var.region 8 | zone = local.zone_id 9 | mig_type = var.client_mig_type 10 | mig_name = local.client_mig_name 11 | } 12 | } 13 | 14 | resource "null_resource" "nomad_autoscaler_jobspec" { 15 | provisioner "local-exec" { 16 | command = "echo '${data.template_file.nomad_autoscaler_jobspec.rendered}' > gcp_autoscaler.nomad" 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /cloud/azure/terraform/control/main.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_version = ">= 0.12" 3 | 4 | required_providers { 5 | azurerm = { 6 | version = "2.32.0" 7 | } 8 | } 9 | } 10 | 11 | provider "nomad" { 12 | address = module.hashistack_cluster.nomad_addr 13 | } 14 | 15 | provider "azurerm" { 16 | features {} 17 | } 18 | 19 | module "my_ip_address" { 20 | source = "matti/resource/shell" 21 | 22 | command = "curl https://ipinfo.io/ip" 23 | } 24 | 25 | module "hashistack_cluster" { 26 | source = "../modules/azure-hashistack" 27 | 28 | allowlist_ip = ["${module.my_ip_address.stdout}/32"] 29 | } 30 | 31 | module "hashistack_jobs" { 32 | source = "../../../shared/terraform/modules/shared-nomad-jobs" 33 | depends_on = [module.hashistack_cluster] 34 | 35 | nomad_addr = module.hashistack_cluster.nomad_addr 36 | } 37 | -------------------------------------------------------------------------------- /cloud/gcp/terraform/modules/gcp-hashistack/image.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | hashistack_image_project_id = var.hashistack_image_project_id != "" ? var.hashistack_image_project_id : var.project_id 3 | } 4 | 5 | resource "null_resource" "packer_build" { 6 | count = var.build_hashistack_image ? 1 : 0 7 | depends_on = [google_project_service.compute] 8 | 9 | provisioner "local-exec" { 10 | command = <` command within your terminal or use the UI. 10 | 11 | You can test the integrity of the cluster by running: 12 | 13 | $ consul members 14 | $ nomad server members 15 | $ nomad node status 16 | 17 | The Nomad UI can be accessed at ${module.hashistack_cluster.nomad_addr}/ui 18 | The Consul UI can be accessed at ${module.hashistack_cluster.consul_addr}/ui 19 | Grafana dashbaord can be accessed at http://${module.hashistack_cluster.client_public_ip_addr}:3000/d/AQphTqmMk/demo?orgId=1&refresh=5s 20 | Traefik can be accessed at http://${module.hashistack_cluster.client_public_ip_addr}:8081 21 | Prometheus can be accessed at http://${module.hashistack_cluster.client_public_ip_addr}:9090 22 | Webapp can be accessed at http://${module.hashistack_cluster.client_public_ip_addr}:80 23 | 24 | CLI environment variables: 25 | export NOMAD_CLIENT_DNS=http://${module.hashistack_cluster.client_public_ip_addr} 26 | export NOMAD_ADDR=${module.hashistack_cluster.nomad_addr} 27 | 28 | CONFIGURATION 29 | } 30 | -------------------------------------------------------------------------------- /cloud/azure/packer/azure-packer.pkr.hcl: -------------------------------------------------------------------------------- 1 | variable "client_id" {} 2 | variable "client_secret" {} 3 | variable "resource_group" {} 4 | variable "subscription_id" {} 5 | variable "location" { default = "East US" } 6 | variable "image_name" { default = "hashistack" } 7 | 8 | source "azure-arm" "hashistack" { 9 | azure_tags = { 10 | Product = "Hashistack" 11 | } 12 | client_id = "${var.client_id}" 13 | client_secret = "${var.client_secret}" 14 | image_offer = "UbuntuServer" 15 | image_publisher = "Canonical" 16 | image_sku = "18.04-LTS" 17 | location = "${var.location}" 18 | managed_image_name = "${var.image_name}" 19 | managed_image_resource_group_name = "${var.resource_group}" 20 | os_type = "Linux" 21 | ssh_username = "packer" 22 | subscription_id = "${var.subscription_id}" 23 | } 24 | 25 | build { 26 | sources = [ 27 | "source.azure-arm.hashistack" 28 | ] 29 | 30 | provisioner "shell" { 31 | inline = [ 32 | "sudo mkdir -p /ops", 33 | "sudo chmod 777 /ops" 34 | ] 35 | } 36 | 37 | provisioner "file" { 38 | source = "../../shared/packer/" 39 | destination = "/ops" 40 | } 41 | 42 | provisioner "shell" { 43 | script = "../../shared/packer/scripts/setup.sh" 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /cloud/azure/terraform/modules/azure-hashistack/templates.tf: -------------------------------------------------------------------------------- 1 | data "azurerm_subscription" "main" {} 2 | 3 | data "template_file" "user_data_server" { 4 | template = file("${path.module}/templates/user-data-server.sh") 5 | 6 | vars = { 7 | server_count = var.server_count 8 | retry_join = "provider=azure tag_name=ConsulAutoJoin tag_value=auto-join subscription_id=${data.azurerm_subscription.main.subscription_id}" 9 | consul_binary = var.consul_binary 10 | nomad_binary = var.nomad_binary 11 | } 12 | } 13 | 14 | data "template_file" "user_data_client" { 15 | template = file("${path.module}/templates/user-data-client.sh") 16 | 17 | vars = { 18 | retry_join = "provider=azure tag_name=ConsulAutoJoin tag_value=auto-join subscription_id=${data.azurerm_subscription.main.subscription_id}" 19 | consul_binary = var.consul_binary 20 | nomad_binary = var.nomad_binary 21 | node_class = "hashistack" 22 | } 23 | } 24 | 25 | data "template_file" "nomad_autoscaler_jobspec" { 26 | template = file("${path.module}/templates/azure_autoscaler.nomad.tpl") 27 | 28 | vars = { 29 | nomad_autoscaler_image = var.nomad_autoscaler_image 30 | subscription_id = data.azurerm_subscription.main.subscription_id 31 | resource_group = azurerm_resource_group.hashistack.name 32 | } 33 | } 34 | 35 | resource "null_resource" "nomad_autoscaler_jobspec" { 36 | provisioner "local-exec" { 37 | command = "echo '${data.template_file.nomad_autoscaler_jobspec.rendered}' > azure_autoscaler.nomad" 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /cloud/azure/terraform/modules/azure-hashistack/outputs.tf: -------------------------------------------------------------------------------- 1 | output "server_public_ips" { 2 | value = azurerm_linux_virtual_machine.servers.*.public_ip_address 3 | } 4 | 5 | output "server_private_ips" { 6 | value = azurerm_linux_virtual_machine.servers.*.private_ip_address 7 | } 8 | 9 | output "server_addresses" { 10 | value = join("\n", formatlist( 11 | " * instance %v - Public: %v, Private: %v", 12 | azurerm_linux_virtual_machine.servers.*.name, 13 | azurerm_linux_virtual_machine.servers.*.public_ip_address, 14 | azurerm_linux_virtual_machine.servers.*.private_ip_address 15 | )) 16 | } 17 | 18 | output "server_lb_id" { 19 | value = azurerm_lb.servers.id 20 | } 21 | 22 | output "server_lb_public_ip" { 23 | value = azurerm_public_ip.servers_lb.ip_address 24 | } 25 | 26 | output "clients_lb_id" { 27 | value = azurerm_lb.clients.id 28 | } 29 | 30 | output "clients_lb_public_ip" { 31 | value = azurerm_public_ip.clients_lb.ip_address 32 | } 33 | 34 | output "nomad_addr" { 35 | value = "http://${azurerm_public_ip.servers_lb.ip_address}:4646" 36 | 37 | depends_on = [ 38 | azurerm_linux_virtual_machine.servers, 39 | azurerm_linux_virtual_machine_scale_set.clients, 40 | ] 41 | } 42 | 43 | output "consul_addr" { 44 | value = "http://${azurerm_public_ip.servers_lb.ip_address}:8500" 45 | 46 | depends_on = [ 47 | azurerm_linux_virtual_machine.servers, 48 | azurerm_linux_virtual_machine_scale_set.clients, 49 | ] 50 | } 51 | 52 | output "client_vmss_id" { 53 | value = azurerm_linux_virtual_machine_scale_set.clients.id 54 | } 55 | 56 | output "client_vmss_name" { 57 | value = azurerm_linux_virtual_machine_scale_set.clients.name 58 | } 59 | -------------------------------------------------------------------------------- /cloud/aws/terraform/modules/aws-hashistack/outputs.tf: -------------------------------------------------------------------------------- 1 | output "server_tag_name" { 2 | value = aws_instance.nomad_server.*.tags.Name 3 | } 4 | 5 | output "server_public_ips" { 6 | value = aws_instance.nomad_server.*.public_ip 7 | } 8 | 9 | output "server_private_ips" { 10 | value = aws_instance.nomad_server.*.private_ip 11 | } 12 | 13 | output "server_addresses" { 14 | value = join("\n", formatlist(" * instance %v - Public: %v, Private: %v", aws_instance.nomad_server.*.tags.Name, aws_instance.nomad_server.*.public_ip, aws_instance.nomad_server.*.private_ip)) 15 | } 16 | 17 | output "server_elb_dns" { 18 | value = aws_elb.nomad_server.dns_name 19 | } 20 | 21 | output "server_elb_dns_zone_id" { 22 | value = aws_elb.nomad_server.zone_id 23 | } 24 | 25 | output "client_elb_dns" { 26 | value = aws_elb.nomad_client.dns_name 27 | } 28 | 29 | output "client_elb_dns_zone_id" { 30 | value = aws_elb.nomad_client.zone_id 31 | } 32 | 33 | output "nomad_addr" { 34 | value = "http://${aws_elb.nomad_server.dns_name}:4646" 35 | } 36 | 37 | output "consul_addr" { 38 | value = "http://${aws_elb.nomad_server.dns_name}:8500" 39 | } 40 | 41 | output "hosts_file" { 42 | value = join("\n", concat( 43 | formatlist(" %-16s %v.hs", aws_instance.nomad_server.*.public_ip, aws_instance.nomad_server.*.tags.Name) 44 | )) 45 | } 46 | 47 | output "client_asg_arn" { 48 | value = aws_autoscaling_group.nomad_client.arn 49 | } 50 | 51 | output "client_asg_name" { 52 | value = aws_autoscaling_group.nomad_client.name 53 | } 54 | 55 | output "ssh_file" { 56 | value = join("\n", concat( 57 | formatlist("Host %v.hs\n User ubuntu\n HostName %v\n", aws_instance.nomad_server.*.tags.Name, aws_instance.nomad_server.*.public_dns) 58 | )) 59 | } 60 | -------------------------------------------------------------------------------- /cloud/aws/terraform/modules/aws-hashistack/asg.tf: -------------------------------------------------------------------------------- 1 | resource "aws_launch_template" "nomad_client" { 2 | name_prefix = "nomad-client" 3 | image_id = var.ami 4 | instance_type = var.client_instance_type 5 | key_name = var.key_name 6 | vpc_security_group_ids = [aws_security_group.primary.id] 7 | user_data = base64encode(data.template_file.user_data_client.rendered) 8 | 9 | iam_instance_profile { 10 | name = aws_iam_instance_profile.nomad_client.name 11 | } 12 | 13 | tag_specifications { 14 | resource_type = "instance" 15 | tags = { 16 | Name = "${var.stack_name}-client" 17 | ConsulAutoJoin = "auto-join" 18 | } 19 | } 20 | 21 | block_device_mappings { 22 | device_name = "/dev/xvdd" 23 | ebs { 24 | volume_type = "gp2" 25 | volume_size = "50" 26 | delete_on_termination = "true" 27 | } 28 | } 29 | } 30 | 31 | resource "aws_autoscaling_group" "nomad_client" { 32 | name = "${var.stack_name}-nomad_client" 33 | availability_zones = var.availability_zones 34 | desired_capacity = var.client_count 35 | min_size = 0 36 | max_size = 10 37 | depends_on = [aws_instance.nomad_server] 38 | load_balancers = [aws_elb.nomad_client.name] 39 | 40 | launch_template { 41 | id = aws_launch_template.nomad_client.id 42 | version = "$Latest" 43 | } 44 | 45 | tag { 46 | key = "OwnerName" 47 | value = var.owner_name 48 | propagate_at_launch = true 49 | } 50 | tag { 51 | key = "OwnerEmail" 52 | value = var.owner_email 53 | propagate_at_launch = true 54 | } 55 | } -------------------------------------------------------------------------------- /cloud/aws/terraform/modules/aws-hashistack/elb.tf: -------------------------------------------------------------------------------- 1 | resource "aws_elb" "nomad_server" { 2 | name = "${var.stack_name}-nomad-server" 3 | availability_zones = distinct(aws_instance.nomad_server.*.availability_zone) 4 | internal = false 5 | instances = aws_instance.nomad_server.*.id 6 | idle_timeout = 360 7 | 8 | listener { 9 | instance_port = 4646 10 | instance_protocol = "http" 11 | lb_port = 4646 12 | lb_protocol = "http" 13 | } 14 | listener { 15 | instance_port = 8500 16 | instance_protocol = "http" 17 | lb_port = 8500 18 | lb_protocol = "http" 19 | } 20 | security_groups = [aws_security_group.server_lb.id] 21 | } 22 | 23 | resource "aws_elb" "nomad_client" { 24 | name = "${var.stack_name}-nomad-client" 25 | availability_zones = var.availability_zones 26 | internal = false 27 | listener { 28 | instance_port = 80 29 | instance_protocol = "http" 30 | lb_port = 80 31 | lb_protocol = "http" 32 | } 33 | listener { 34 | instance_port = 9090 35 | instance_protocol = "http" 36 | lb_port = 9090 37 | lb_protocol = "http" 38 | } 39 | listener { 40 | instance_port = 3000 41 | instance_protocol = "http" 42 | lb_port = 3000 43 | lb_protocol = "http" 44 | } 45 | listener { 46 | instance_port = 8081 47 | instance_protocol = "http" 48 | lb_port = 8081 49 | lb_protocol = "http" 50 | } 51 | 52 | health_check { 53 | healthy_threshold = 8 54 | unhealthy_threshold = 2 55 | timeout = 3 56 | target = "TCP:8081" 57 | interval = 30 58 | } 59 | 60 | security_groups = [aws_security_group.client_lb.id] 61 | } 62 | -------------------------------------------------------------------------------- /vagrant/horizontal-app-scaling/jobs/loki.nomad: -------------------------------------------------------------------------------- 1 | job "loki" { 2 | datacenters = ["dc1"] 3 | 4 | group "loki" { 5 | count = 1 6 | 7 | network { 8 | port "loki" {} 9 | } 10 | 11 | task "loki" { 12 | driver = "docker" 13 | 14 | config { 15 | image = "grafana/loki:2.1.0" 16 | ports = ["loki"] 17 | 18 | args = [ 19 | "--config.file=/etc/loki/config/loki.yml", 20 | ] 21 | 22 | volumes = [ 23 | "local/config:/etc/loki/config", 24 | ] 25 | } 26 | 27 | template { 28 | data = <&1 | grep 'HTTP/1.1 200 OK'` ]]; then 30 | curl -L $CONSUL_BINARY > consul.zip 31 | sudo unzip -o consul.zip -d /usr/local/bin 32 | sudo chmod 0755 /usr/local/bin/consul 33 | sudo chown root:root /usr/local/bin/consul 34 | fi 35 | 36 | sudo systemctl start consul.service 37 | sleep 10 38 | 39 | # Nomad 40 | 41 | ## Replace existing Nomad binary if remote file exists 42 | if [[ `wget -S --spider $NOMAD_BINARY 2>&1 | grep 'HTTP/1.1 200 OK'` ]]; then 43 | curl -L $NOMAD_BINARY > nomad.zip 44 | sudo unzip -o nomad.zip -d /usr/local/bin 45 | sudo chmod 0755 /usr/local/bin/nomad 46 | sudo chown root:root /usr/local/bin/nomad 47 | fi 48 | 49 | sed -i "s/NODE_CLASS/\"$NODE_CLASS\"/g" $CONFIGDIR/nomad_client.hcl 50 | sudo cp $CONFIGDIR/nomad_client.hcl $NOMADCONFIGDIR/nomad.hcl 51 | sudo cp $CONFIGDIR/nomad.service /etc/systemd/system/nomad.service 52 | 53 | sudo systemctl start nomad.service 54 | sleep 10 55 | export NOMAD_ADDR=http://$IP_ADDRESS:4646 56 | 57 | # Add hostname to /etc/hosts 58 | echo "127.0.0.1 $(hostname)" | sudo tee --append /etc/hosts 59 | 60 | # dnsmasq config 61 | echo "DNSStubListener=no" | sudo tee -a /etc/systemd/resolved.conf 62 | sudo cp /ops/config/10-consul.dnsmasq /etc/dnsmasq.d/10-consul 63 | sudo cp /ops/config/99-default.dnsmasq.$CLOUD /etc/dnsmasq.d/99-default 64 | sudo mv /etc/resolv.conf /etc/resolv.conf.orig 65 | grep -v "nameserver" /etc/resolv.conf.orig | grep -v -e"^#" | grep -v -e '^$' | sudo tee /etc/resolv.conf 66 | echo "nameserver 127.0.0.1" | sudo tee -a /etc/resolv.conf 67 | sudo systemctl restart systemd-resolved 68 | sudo systemctl restart dnsmasq 69 | 70 | # Add Docker bridge network IP to /etc/resolv.conf (at the top) 71 | echo "nameserver $DOCKER_BRIDGE_IP_ADDRESS" | sudo tee /etc/resolv.conf.new 72 | cat /etc/resolv.conf | sudo tee --append /etc/resolv.conf.new 73 | sudo mv /etc/resolv.conf.new /etc/resolv.conf 74 | 75 | # Set env vars for tool CLIs 76 | echo "export VAULT_ADDR=http://$IP_ADDRESS:8200" | sudo tee --append /home/$HOME_DIR/.bashrc 77 | echo "export NOMAD_ADDR=http://$IP_ADDRESS:4646" | sudo tee --append /home/$HOME_DIR/.bashrc 78 | echo "export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/jre" | sudo tee --append /home/$HOME_DIR/.bashrc 79 | -------------------------------------------------------------------------------- /cloud/shared/packer/scripts/setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | echo "Waiting for cloud-init to update /etc/apt/sources.list" 6 | timeout 180 /bin/bash -c \ 7 | 'until stat /var/lib/cloud/instance/boot-finished 2>/dev/null; do echo waiting ...; sleep 1; done' 8 | 9 | # Disable interactive apt prompts 10 | export DEBIAN_FRONTEND=noninteractive 11 | echo 'debconf debconf/frontend select Noninteractive' | sudo debconf-set-selections 12 | 13 | cd /ops 14 | 15 | # Dependencies 16 | sudo apt-get update 17 | 18 | sudo apt-get install -y software-properties-common unzip tree redis-tools jq curl tmux dnsmasq 19 | 20 | CONFIGDIR=/ops/config 21 | 22 | CONSULVERSION=$(curl -s https://checkpoint-api.hashicorp.com/v1/check/consul | jq -r '.current_version') 23 | CONSULDOWNLOAD=https://releases.hashicorp.com/consul/${CONSULVERSION}/consul_${CONSULVERSION}_linux_amd64.zip 24 | CONSULCONFIGDIR=/etc/consul.d 25 | CONSULDIR=/opt/consul 26 | 27 | NOMADVERSION=$(curl -s https://checkpoint-api.hashicorp.com/v1/check/nomad | jq -r '.current_version') 28 | NOMADDOWNLOAD=https://releases.hashicorp.com/nomad/${NOMADVERSION}/nomad_${NOMADVERSION}_linux_amd64.zip 29 | NOMADCONFIGDIR=/etc/nomad.d 30 | NOMADDIR=/opt/nomad 31 | 32 | CNIVERSION=0.8.5 33 | CNIDOWNLOAD=https://github.com/containernetworking/plugins/releases/download/v${CNIVERSION}/cni-plugins-linux-amd64-v${CNIVERSION}.tgz 34 | CNIDIR=/opt/cni 35 | 36 | # Disable the firewall 37 | sudo ufw disable || echo "ufw not installed" 38 | 39 | # Consul 40 | curl -sL -o consul.zip ${CONSULDOWNLOAD} 41 | 42 | ## Install 43 | sudo unzip consul.zip -d /usr/local/bin 44 | sudo chmod 0755 /usr/local/bin/consul 45 | sudo chown root:root /usr/local/bin/consul 46 | 47 | ## Configure 48 | sudo mkdir -p ${CONSULCONFIGDIR} 49 | sudo chmod 755 ${CONSULCONFIGDIR} 50 | sudo mkdir -p ${CONSULDIR} 51 | sudo chmod 755 ${CONSULDIR} 52 | 53 | # Nomad 54 | curl -sL -o nomad.zip ${NOMADDOWNLOAD} 55 | 56 | ## Install 57 | sudo unzip nomad.zip -d /usr/local/bin 58 | sudo chmod 0755 /usr/local/bin/nomad 59 | sudo chown root:root /usr/local/bin/nomad 60 | 61 | ## Configure 62 | sudo mkdir -p ${NOMADCONFIGDIR} 63 | sudo chmod 755 ${NOMADCONFIGDIR} 64 | sudo mkdir -p ${NOMADDIR} 65 | sudo chmod 755 ${NOMADDIR} 66 | 67 | # Docker 68 | distro=$(lsb_release -si | tr '[:upper:]' '[:lower:]') 69 | sudo apt-get install -y apt-transport-https ca-certificates gnupg2 70 | curl -fsSL https://download.docker.com/linux/debian/gpg | sudo APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 apt-key add - 71 | sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/${distro} $(lsb_release -cs) stable" 72 | sudo apt-get update 73 | sudo apt-get install -y docker-ce 74 | 75 | # Java 76 | sudo add-apt-repository -y ppa:openjdk-r/ppa 77 | sudo apt-get update 78 | sudo apt-get install -y openjdk-8-jdk 79 | JAVA_HOME=$(readlink -f /usr/bin/java | sed "s:bin/java::") 80 | 81 | # CNI plugins 82 | curl -sL -o cni-plugins.tgz ${CNIDOWNLOAD} 83 | sudo mkdir -p ${CNIDIR}/bin 84 | sudo tar -C ${CNIDIR}/bin -xzf cni-plugins.tgz 85 | 86 | echo 'debconf debconf/frontend select Dialog' | sudo debconf-set-selections 87 | -------------------------------------------------------------------------------- /cloud/aws/terraform/modules/aws-hashistack/variables.tf: -------------------------------------------------------------------------------- 1 | variable "stack_name" { 2 | description = "The name to prefix onto resources." 3 | type = string 4 | default = "hashistack" 5 | } 6 | 7 | variable "owner_name" { 8 | description = "Your name so resources can be easily assigned." 9 | type = string 10 | } 11 | 12 | variable "owner_email" { 13 | description = "Your email so you can be contacted about resources." 14 | type = string 15 | } 16 | 17 | variable "region" { 18 | description = "The AWS region to deploy into." 19 | type = string 20 | default = "us-east-1" 21 | } 22 | 23 | variable "availability_zones" { 24 | description = "The AWS region AZs to deploy into." 25 | type = list(string) 26 | default = ["us-east-1a"] 27 | } 28 | 29 | variable "vpc_id" { 30 | description = "The AWS VPC to use for resources. If left empty, the default will be used." 31 | type = string 32 | default = "" 33 | } 34 | 35 | variable "ami" { 36 | description = "The AMI to use, preferably built by the supplied Packer scripts." 37 | type = string 38 | } 39 | 40 | variable "key_name" { 41 | description = "The EC2 key pair to use for EC2 instance SSH access." 42 | type = string 43 | } 44 | 45 | variable "server_instance_type" { 46 | description = "The EC2 instance type to launch for Nomad servers." 47 | type = string 48 | default = "t3.small" 49 | } 50 | 51 | variable "server_count" { 52 | description = "The number of Nomad servers to run." 53 | type = number 54 | default = 1 55 | } 56 | 57 | variable "client_instance_type" { 58 | description = "The EC2 instance type to launch for Nomad clients." 59 | type = string 60 | default = "t3.small" 61 | } 62 | variable "client_count" { 63 | description = "The number of Nomad clients to run." 64 | type = number 65 | default = 1 66 | } 67 | 68 | variable "root_block_device_size" { 69 | description = "The number of GB to assign as a block device on instances." 70 | type = number 71 | default = 16 72 | } 73 | 74 | variable "retry_join" { 75 | description = "The retry join configuration to use." 76 | type = string 77 | default = "provider=aws tag_key=ConsulAutoJoin tag_value=auto-join" 78 | } 79 | 80 | variable "nomad_binary" { 81 | description = "The URL to download a custom Nomad binary if desired." 82 | type = string 83 | default = "none" 84 | } 85 | 86 | variable "consul_binary" { 87 | description = "The URL to download a custom Consul binary if desired." 88 | type = string 89 | default = "none" 90 | } 91 | 92 | variable "allowlist_ip" { 93 | description = "A list of IP address to grant access via the LBs." 94 | type = list(string) 95 | default = ["0.0.0.0/0"] 96 | } 97 | 98 | variable "nomad_autoscaler_image" { 99 | description = "The Docker image to use for the Nomad Autoscaler job." 100 | type = string 101 | default = "hashicorp/nomad-autoscaler:0.3.0" 102 | } 103 | -------------------------------------------------------------------------------- /vagrant/dynamic-app-sizing/jobs/das-autoscaler.nomad: -------------------------------------------------------------------------------- 1 | job "das-autoscaler" { 2 | datacenters = ["dc1"] 3 | 4 | group "autoscaler" { 5 | count = 1 6 | 7 | network { 8 | port "http" {} 9 | } 10 | 11 | task "autoscaler" { 12 | driver = "docker" 13 | 14 | config { 15 | image = "hashicorp/nomad-autoscaler-enterprise:0.3.0" 16 | command = "nomad-autoscaler" 17 | ports = ["http"] 18 | 19 | args = [ 20 | "agent", 21 | "-config", 22 | "local/autoscaler.hcl", 23 | "-http-bind-address", 24 | "0.0.0.0", 25 | "-http-bind-port", 26 | "${NOMAD_PORT_http}", 27 | ] 28 | } 29 | 30 | template { 31 | destination = "local/autoscaler.hcl" 32 | 33 | data = <&1 | grep 'HTTP/1.1 200 OK'` ]]; then 25 | curl -L $CONSUL_BINARY > consul.zip 26 | sudo unzip -o consul.zip -d /usr/local/bin 27 | sudo chmod 0755 /usr/local/bin/consul 28 | sudo chown root:root /usr/local/bin/consul 29 | fi 30 | 31 | sed -i "s/IP_ADDRESS/$IP_ADDRESS/g" $CONFIGDIR/consul.hcl 32 | sed -i "s/SERVER_COUNT/$SERVER_COUNT/g" $CONFIGDIR/consul.hcl 33 | sed -i "s/RETRY_JOIN/$RETRY_JOIN/g" $CONFIGDIR/consul.hcl 34 | sudo cp $CONFIGDIR/consul.hcl $CONSULCONFIGDIR 35 | sudo cp $CONFIGDIR/consul_$CLOUD.service /etc/systemd/system/consul.service 36 | 37 | sudo systemctl start consul.service 38 | sleep 10 39 | export CONSUL_HTTP_ADDR=$IP_ADDRESS:8500 40 | export CONSUL_RPC_ADDR=$IP_ADDRESS:8400 41 | 42 | # Nomad 43 | 44 | ## Replace existing Nomad binary if remote file exists 45 | if [[ `wget -S --spider $NOMAD_BINARY 2>&1 | grep 'HTTP/1.1 200 OK'` ]]; then 46 | curl -L $NOMAD_BINARY > nomad.zip 47 | sudo unzip -o nomad.zip -d /usr/local/bin 48 | sudo chmod 0755 /usr/local/bin/nomad 49 | sudo chown root:root /usr/local/bin/nomad 50 | fi 51 | 52 | sed -i "s/SERVER_COUNT/$SERVER_COUNT/g" $CONFIGDIR/nomad.hcl 53 | sudo cp $CONFIGDIR/nomad.hcl $NOMADCONFIGDIR 54 | sudo cp $CONFIGDIR/nomad.service /etc/systemd/system/nomad.service 55 | 56 | sudo systemctl start nomad.service 57 | sleep 10 58 | export NOMAD_ADDR=http://$IP_ADDRESS:4646 59 | 60 | # Add hostname to /etc/hosts 61 | echo "127.0.0.1 $(hostname)" | sudo tee --append /etc/hosts 62 | 63 | # dnsmasq config 64 | echo "DNSStubListener=no" | sudo tee -a /etc/systemd/resolved.conf 65 | sudo cp /ops/config/10-consul.dnsmasq /etc/dnsmasq.d/10-consul 66 | sudo cp /ops/config/99-default.dnsmasq.$CLOUD /etc/dnsmasq.d/99-default 67 | sudo mv /etc/resolv.conf /etc/resolv.conf.orig 68 | grep -v "nameserver" /etc/resolv.conf.orig | grep -v -e"^#" | grep -v -e '^$' | sudo tee /etc/resolv.conf 69 | echo "nameserver 127.0.0.1" | sudo tee -a /etc/resolv.conf 70 | sudo systemctl restart systemd-resolved 71 | sudo systemctl restart dnsmasq 72 | 73 | # Add Docker bridge network IP to /etc/resolv.conf (at the top) 74 | echo "nameserver $DOCKER_BRIDGE_IP_ADDRESS" | sudo tee /etc/resolv.conf.new 75 | cat /etc/resolv.conf | sudo tee --append /etc/resolv.conf.new 76 | sudo mv /etc/resolv.conf.new /etc/resolv.conf 77 | 78 | 79 | # Set env vars for tool CLIs 80 | echo "export CONSUL_RPC_ADDR=$IP_ADDRESS:8400" | sudo tee --append /home/$HOME_DIR/.bashrc 81 | echo "export CONSUL_HTTP_ADDR=$IP_ADDRESS:8500" | sudo tee --append /home/$HOME_DIR/.bashrc 82 | echo "export NOMAD_ADDR=http://$IP_ADDRESS:4646" | sudo tee --append /home/$HOME_DIR/.bashrc 83 | echo "export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/jre" | sudo tee --append /home/$HOME_DIR/.bashrc 84 | 85 | -------------------------------------------------------------------------------- /cloud/azure/terraform/modules/azure-hashistack/templates/azure_autoscaler.nomad.tpl: -------------------------------------------------------------------------------- 1 | job "autoscaler" { 2 | datacenters = ["dc1"] 3 | 4 | group "autoscaler" { 5 | count = 1 6 | 7 | network { 8 | port "http" {} 9 | } 10 | 11 | task "autoscaler" { 12 | driver = "docker" 13 | 14 | config { 15 | image = "${nomad_autoscaler_image}" 16 | command = "nomad-autoscaler" 17 | 18 | args = [ 19 | "agent", 20 | "-config", 21 | "$${NOMAD_TASK_DIR}/config.hcl", 22 | "-http-bind-address", 23 | "0.0.0.0", 24 | "-http-bind-port", 25 | "$${NOMAD_PORT_http}", 26 | "-policy-dir", 27 | "$${NOMAD_TASK_DIR}/policies/", 28 | ] 29 | 30 | ports = ["http"] 31 | } 32 | 33 | template { 34 | data = <[a-zA-Z0-9_-]+).*source=(?P[a-zA-Z0-9_-]+).*strategy=(?P[a-zA-Z0-9_-]+).*target=(?P[a-zA-Z0-9_-]+).*Group:(?P[a-zA-Z0-9]+).*Job:(?P[a-zA-Z0-9_-]+).*Namespace:(?P[a-zA-Z0-9_-]+)' 135 | - labels: 136 | policy_id: 137 | source: 138 | strategy: 139 | target: 140 | group: 141 | job: 142 | namespace: 143 | EOH 144 | 145 | destination = "local/promtail.yaml" 146 | } 147 | 148 | resources { 149 | cpu = 50 150 | memory = 32 151 | } 152 | 153 | service { 154 | name = "promtail" 155 | port = "promtail" 156 | 157 | check { 158 | type = "http" 159 | path = "/ready" 160 | interval = "10s" 161 | timeout = "2s" 162 | } 163 | } 164 | } 165 | } 166 | } 167 | -------------------------------------------------------------------------------- /vagrant/horizontal-app-scaling/Vagrantfile: -------------------------------------------------------------------------------- 1 | # -*- mode: ruby -*- 2 | # vi: set ft=ruby : 3 | 4 | Vagrant.configure("2") do |config| 5 | config.vm.box = "ubuntu/focal64" 6 | 7 | # Expose ports to the host. 8 | config.vm.network "forwarded_port", guest: 4646, host: 4646, host_ip: "127.0.0.1" # Nomad 9 | config.vm.network "forwarded_port", guest: 8500, host: 8500, host_ip: "127.0.0.1" # Consul UI 10 | config.vm.network "forwarded_port", guest: 8080, host: 8080, host_ip: "127.0.0.1" # Nomad Autoscaler health check 11 | config.vm.network "forwarded_port", guest: 8000, host: 8000, host_ip: "127.0.0.1" # Demo webapp 12 | config.vm.network "forwarded_port", guest: 1936, host: 1936, host_ip: "127.0.0.1" # HAProxy stats 13 | config.vm.network "forwarded_port", guest: 9090, host: 9090, host_ip: "127.0.0.1" # Prometheus 14 | config.vm.network "forwarded_port", guest: 3000, host: 3000, host_ip: "127.0.0.1" # Grafana 15 | 16 | # Share current directory with jobs and configuration files with the VM. Add 17 | # the shared files in a sub-dir. 18 | config.vm.synced_folder "./", "/home/vagrant/nomad-autoscaler" 19 | config.vm.synced_folder "../shared", "/home/vagrant/nomad-autoscaler-shared" 20 | 21 | # VM configuration. 22 | config.vm.provider "virtualbox" do |vb| 23 | vb.customize [ "modifyvm", :id, "--uartmode1", "file", File::NULL ] # https://bugs.launchpad.net/cloud-images/+bug/1874453 24 | vb.memory = "4096" 25 | vb.cpus = 2 26 | end 27 | 28 | # Provision demo dependencies. 29 | # - Downloads and install Nomad, Consul and Docker 30 | # Only runs when the VM is created. 31 | config.vm.provision "deps", type: "shell", inline: <<-SHELL 32 | 33 | mkdir /tmp/downloads 34 | 35 | # Install dependencies. 36 | apt-get update 37 | apt-get install -y \ 38 | apt-transport-https \ 39 | ca-certificates \ 40 | curl \ 41 | gnupg-agent \ 42 | jq \ 43 | software-properties-common \ 44 | zip 45 | 46 | nomad_version=$(curl -s https://checkpoint-api.hashicorp.com/v1/check/nomad | jq -r '.current_version') 47 | consul_version=$(curl -s https://checkpoint-api.hashicorp.com/v1/check/consul | jq -r '.current_version') 48 | 49 | # Download and install Docker. 50 | curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - 51 | add-apt-repository \ 52 | "deb [arch=amd64] https://download.docker.com/linux/ubuntu \ 53 | $(lsb_release -cs) \ 54 | stable" 55 | apt-get update 56 | apt-get install -y \ 57 | docker-ce \ 58 | docker-ce-cli \ 59 | containerd.io 60 | docker run hello-world 61 | usermod -aG docker vagrant 62 | 63 | # Install Hey. 64 | apt-get install -y hey 65 | 66 | # Download and install Nomad and Consul. 67 | pushd /tmp/downloads 68 | curl --silent --show-error --remote-name-all \ 69 | https://releases.hashicorp.com/nomad/${nomad_version}/nomad_${nomad_version}_linux_amd64.zip \ 70 | https://releases.hashicorp.com/consul/${consul_version}/consul_${consul_version}_linux_amd64.zip 71 | unzip nomad_${nomad_version}_linux_amd64.zip 72 | unzip consul_${consul_version}_linux_amd64.zip 73 | mkdir -p /opt/hashicorp/bin 74 | mv nomad consul /opt/hashicorp/bin 75 | chmod +x /opt/hashicorp/bin/{nomad,consul} 76 | ln -s /opt/hashicorp/bin/{nomad,consul} /usr/local/bin 77 | popd 78 | 79 | rm -fr /tmp/downloads 80 | SHELL 81 | 82 | # Setup demo dependencies. 83 | # - Create daemons for Nomad and Consul 84 | # Runs everytime the VM starts. 85 | config.vm.provision "app:setup", type: "shell", run: "always", inline: <<-SHELL 86 | # Create paths for Nomad host volumes. 87 | mkdir -p /opt/nomad-volumes 88 | pushd /opt/nomad-volumes 89 | mkdir -p grafana 90 | chown 472:472 grafana 91 | popd 92 | 93 | # Configure the Nomad and Consul daemons. 94 | pushd /home/vagrant/nomad-autoscaler-shared 95 | for t in consul nomad; do 96 | cp ${t}.service /etc/systemd/system/ 97 | mkdir -p /etc/${t}.d 98 | done 99 | popd 100 | 101 | # Copy across the config files. 102 | cp /home/vagrant/nomad-autoscaler-shared/consul.hcl /etc/consul.d/ 103 | cp /home/vagrant/nomad-autoscaler/files/nomad.hcl /etc/nomad.d/ 104 | 105 | # Enable and start the daemons 106 | systemctl enable consul 107 | systemctl start consul 108 | sudo systemctl enable nomad 109 | sudo systemctl start nomad 110 | SHELL 111 | 112 | end 113 | -------------------------------------------------------------------------------- /cloud/aws/terraform/modules/aws-hashistack/templates/aws_autoscaler.nomad.tpl: -------------------------------------------------------------------------------- 1 | job "autoscaler" { 2 | datacenters = ["dc1"] 3 | 4 | group "autoscaler" { 5 | count = 1 6 | 7 | network { 8 | port "http" {} 9 | } 10 | 11 | task "autoscaler" { 12 | driver = "docker" 13 | 14 | config { 15 | image = "${nomad_autoscaler_image}" 16 | command = "nomad-autoscaler" 17 | 18 | args = [ 19 | "agent", 20 | "-config", 21 | "$${NOMAD_TASK_DIR}/config.hcl", 22 | "-http-bind-address", 23 | "0.0.0.0", 24 | "-http-bind-port", 25 | "$${NOMAD_PORT_http}", 26 | "-policy-dir", 27 | "$${NOMAD_TASK_DIR}/policies/", 28 | ] 29 | 30 | ports = ["http"] 31 | } 32 | 33 | template { 34 | data = <' \ 16 | aws-packer.pkr.hcl 17 | ``` 18 | 19 | You can then navigate to the Terraform AWS environment you will be using to build the 20 | infrastructure components. 21 | ``` 22 | $ cd ../terraform/control 23 | ``` 24 | 25 | In order for Terraform to run correctly you'll need to provide the appropriate variables within a 26 | file named `terraform.tfvars`. You can rename the `terraform.tfvars.sample` file and then update the 27 | variables. The most important are the `ami` which is the AMI ID you built in the previous step and 28 | the `key_name` which is your EC2 key-pair for authenticating to the instances via SSH. 29 | 30 | You can then run the Terraform commands, carefully observing the output of each. 31 | ``` 32 | $ terraform init 33 | $ terraform plan 34 | $ terraform apply --auto-approve 35 | ``` 36 | 37 | Once the Terraform apply finishes, a number of useful pieces of information should be output to 38 | your console. These include URLs to deployed resources as well as a semi-populated Nomad Autoscaler 39 | config. 40 | ``` 41 | Apply complete! Resources: 23 added, 0 changed, 0 destroyed. 42 | 43 | Outputs: 44 | 45 | ip_addresses = 46 | Server IPs: 47 | * instance hashistack-server-1 - Public: 52.47.76.15, Private: 172.31.18.240 48 | 49 | To connect, add your private key and SSH into any client or server with 50 | `ssh ubuntu@PUBLIC_IP`. You can test the integrity of the cluster by running: 51 | 52 | $ consul members 53 | $ nomad server members 54 | $ nomad node status 55 | 56 | The Nomad UI can be accessed at http://hashistack-nomad-server-1436431625.eu-west-3.elb.amazonaws.com:4646/ui 57 | The Consul UI can be accessed at http://hashistack-nomad-server-1436431625.eu-west-3.elb.amazonaws.com:8500/ui 58 | Grafana can be accessed at http://hashistack-nomad-client-167906299.eu-west-3.elb.amazonaws.com:3000/d/AQphTqmMk/demo?orgId=1&refresh=5s 59 | Traefik can be accessed at http://hashistack-nomad-client-167906299.eu-west-3.elb.amazonaws.com:8081 60 | Prometheus can be accessed at http://hashistack-nomad-client-167906299.eu-west-3.elb.amazonaws.com:9090 61 | Webapp can be accessed at http://hashistack-nomad-client-167906299.eu-west-3.elb.amazonaws.com:80 62 | 63 | CLI environment variables: 64 | export NOMAD_CLIENT_DNS=http://hashistack-nomad-client-167906299.eu-west-3.elb.amazonaws.com 65 | export NOMAD_ADDR=http://hashistack-nomad-server-1436431625.eu-west-3.elb.amazonaws.com:4646 66 | export CONSUL_HTTP_ADDR=http://hashistack-nomad-server-1436431625.eu-west-3.elb.amazonaws.com:8500 67 | ``` 68 | 69 | You can visit the URLs and explore what has been created. This will include registration of a 70 | number of Nomad jobs which provide metrics and dashboarding as well as a demo application and 71 | routing provided by Traefik. It may take a few seconds for all the applications to start, so if 72 | any of the URLs doesn't load the first time, wait a little bit and give it a try again. 73 | 74 | Please also copy the export commands and run these in the terminal where the rest of the demo will 75 | be run. 76 | 77 | The application is pre-configured with a scaling policy, you can view this by opening the job file 78 | or calling the Nomad API. The application scales based on the average number of active connections, 79 | and we are targeting an average of 10 per instance of our app. 80 | ``` 81 | curl $NOMAD_ADDR/v1/scaling/policies 82 | ``` 83 | 84 | ## Run the Autoscaler 85 | The Autoscaler is not triggered automatically. This provides the opportunity to look through the 86 | jobfile to understand it better before deploying. The most important parts of the `aws_autoscaler.nomad` 87 | file are the template sections. The first defines our agent config where we configure the 88 | `prometheus`, `aws-asg` and `target-value` plugins. The second is where we define our cluster 89 | scaling policy and write this to a local directory for reading. Once you have an understanding of 90 | the job file, submit it to the Nomad cluster ensuring the `NOMAD_ADDR` env var has been exported. 91 | ``` 92 | $ nomad run aws_autoscaler.nomad 93 | ``` 94 | 95 | If you wish, in another terminal window you can export the `NOMAD_ADDR` env var and then follow 96 | the Nomad Autoscaler logs. 97 | ``` 98 | $ nomad logs -stderr -f 99 | ``` 100 | 101 | You can now return to the [demo instrunctions](../README.md#the-demo). 102 | 103 | ## Post Demo Steps 104 | The AMI is built outside of Terraform's control and therefore needs to be deregistered. This can be 105 | performed via the AWS console or via the [AWS CLI](https://aws.amazon.com/cli/). 106 | 107 | ``` 108 | $ aws ec2 deregister-image --image-id 109 | ``` 110 | -------------------------------------------------------------------------------- /cloud/gcp/README.md: -------------------------------------------------------------------------------- 1 | # Google Cloud Platform 2 | In order to set up some basic GCP resources for the demo, we will utilise the 3 | [gcloud CLI tool](https://cloud.google.com/sdk/gcloud/). Please ensure this is installed before 4 | continuing. 5 | 6 | ### Environment Setup 7 | The first step is to authenticate with Google Cloud: 8 | 9 | ```shellsession 10 | $ gcloud auth login 11 | ``` 12 | 13 | Next, you will need to pick which organization and billing account to use. Take a note of their 14 | IDs, we will use them in the next step: 15 | 16 | ```shellsession 17 | $ gcloud organizations list 18 | DISPLAY_NAME ID DIRECTORY_CUSTOMER_ID 19 | org1 1111111111111 ZZZZZZZZZ 20 | org2 222222222222 ZZZZZZZZZ 21 | 22 | $ gcloud alpha billing accounts list 23 | ACCOUNT_ID NAME OPEN MASTER_ACCOUNT_ID 24 | 111111-AAAAAA-222222 Account 1 True AAAAAA-BBBBBB-CCCCCC 25 | BBBBBB-333333-DDDDDD Account 2 True AAAAAA-BBBBBB-CCCCCC 26 | ``` 27 | 28 | # Running Terraform 29 | The Terraform setup handles creating all GCP resources as well as the compute images via 30 | [Packer](https://www.packer.io/). In particular all resources will be placed into a newly created 31 | project for isolation. 32 | 33 | ```shellsession 34 | $ cd ./terraform/control 35 | ``` 36 | 37 | Make a copy of the `terraform.tfvars.sample` file and call it `terraform.tfvars`: 38 | 39 | ```shellsession 40 | $ cp terraform.tfvars.sample terraform.tfvars 41 | ``` 42 | 43 | Fill in the IDs for the organization and billing account that were retrieved before: 44 | 45 | ```hcl 46 | org_id = "1111111111111" 47 | billing_account = "BBBBBB-333333-DDDDDD" 48 | ``` 49 | 50 | We are now ready to initialize and run Terraform: 51 | 52 | ```shellsession 53 | $ terraform init 54 | $ terraform plan 55 | $ terraform apply --auto-approve 56 | ``` 57 | 58 | You may see this error message: 59 | 60 | ``` 61 | Error: Error creating Network: googleapi: Error 403: Compute Engine API has not been used in project 603655938425 before or it is disabled. Enable it by visiting https://console.developers.google.com/apis/api/compute.googleapis.com/overview?project=603655938425 then retry. If you enabled this API recently, wait a few minutes for the action to propagate to our systems and retry., accessNotConfigured 62 | ``` 63 | 64 | As the message indicates, it may take a few minutes for the Compute API to be enabled in the new 65 | project. Please wait and try again. 66 | 67 | Once the Terraform apply finishes, a number of useful pieces of information should be output to 68 | your console. These include some commands to run, URLs to deployed resources as well as a 69 | populated Nomad Autoscaler job. 70 | 71 | ``` 72 | You can set the gcloud project setting for CLI use with `gcloud config set project 73 | projects/hashistack-big-sponge`, otherwise you will need to set the `--project` 74 | flag on each command. 75 | 76 | To connect to any instance running within the environment you can use the 77 | `gcloud compute ssh ubuntu@` command within your terminal or use the UI. 78 | 79 | You can test the integrity of the cluster by running: 80 | 81 | $ consul members 82 | $ nomad server members 83 | $ nomad node status 84 | 85 | The Nomad UI can be accessed at http://34.121.186.59:4646/ui 86 | The Consul UI can be accessed at http://34.121.186.59:8500/ui 87 | Grafana dashbaord can be accessed at http://35.224.138.121:3000/d/AQphTqmMk/demo?orgId=1&refresh=5s 88 | Traefik can be accessed at http://35.224.138.121:8081 89 | Prometheus can be accessed at http://35.224.138.121:9090 90 | Webapp can be accessed at http://35.224.138.121:80 91 | 92 | CLI environment variables: 93 | export NOMAD_CLIENT_DNS=http://35.224.138.121 94 | export NOMAD_ADDR=http://34.121.186.59:4646 95 | ``` 96 | 97 | You can visit the URLs and explore what has been created. This will include registration of a 98 | number of Nomad jobs which provide metrics and dashboarding as well as a demo application and 99 | routing provided by Traefik. It may take a few seconds for all the applications to start, so if 100 | any of the URLs doesn't load the first time, wait a little bit and give it a try again. 101 | 102 | Please also copy the export commands and run these in the terminal where the rest of the demo will 103 | be run. 104 | 105 | The application is pre-configured with a scaling policy, you can view this by opening the job file 106 | or calling the Nomad API. The application scales based on the average number of active connections, 107 | and we are targeting an average of 10 connections per instance of our app. 108 | 109 | ```shellsession 110 | $ curl $NOMAD_ADDR/v1/scaling/policies 111 | ``` 112 | 113 | ## Run the Autoscaler 114 | The Autoscaler is not triggered automatically. This provides the opportunity to look through the 115 | jobfile to understand it better before deploying. The most important parts of the `gcp_autoscaler.nomad` 116 | file are the template sections. The first defines our agent config where we configure the 117 | `prometheus`, `gce-mig` and `target-value` plugins. The second is where we define our cluster 118 | scaling policy and write this to a local directory for reading. Once you have an understanding of 119 | the job file, submit it to the Nomad cluster ensuring the `NOMAD_ADDR` env var has been exported. 120 | 121 | ```shellsession 122 | $ nomad run gcp_autoscaler.nomad 123 | ``` 124 | 125 | If you wish, in another terminal window you can export the `NOMAD_ADDR` env var and then follow 126 | the Nomad Autoscaler logs. 127 | 128 | ```shellsession 129 | $ nomad logs -stderr -f 130 | ``` 131 | 132 | You can now return to the [demo instrunctions](../README.md#the-demo). 133 | -------------------------------------------------------------------------------- /cloud/gcp/terraform/control/.terraform.lock.hcl: -------------------------------------------------------------------------------- 1 | # This file is maintained automatically by "terraform init". 2 | # Manual edits may be lost in future updates. 3 | 4 | provider "registry.terraform.io/hashicorp/google" { 5 | version = "3.54.0" 6 | constraints = "3.54.0" 7 | hashes = [ 8 | "h1:mW/ml1ho3L1KqlXQWLdJf4GTKfhq97PL6GulOCYhEEw=", 9 | "zh:211dcbc641712250c1903f1ac031968aad1baba62c3cc350ffc6b42317613075", 10 | "zh:934f5afa3bc46bb6f4941c4a003178613e8566e8b6cedaf160312b5456fd0822", 11 | "zh:ac02478b4280cfe7064924bff2a653d9b3a010d0b3c6cf1b232ec7b0be00a2d3", 12 | "zh:b1738d6811849f81aac0a117cf9f2025e28385bd7a96652f4ef33b3402db2a36", 13 | "zh:b81e9699184fbff1fc8d08371989fabf89f8eedf2ec8531183e0c74cfd38bd11", 14 | "zh:ca5e59925bf3174892364c9e125886870bd51e3acdd1152861d9ed2c198e0957", 15 | "zh:cff24b8d85f420dccc099d29968856807d2f2e5ba47a80160d13bf5d78c1e9be", 16 | "zh:dc5d8d989dffed103b8ad897868d3b2efd33316f08c83b61a3a9ec5bb8c1cbee", 17 | "zh:e78677deaad77d0e36866fb0b25fab7f01bae4f812942c6c1fa0bf8eb02edb1a", 18 | "zh:e9d94ddfb1ad845a005098f1644e1fb5e5e77e3fc67142859de5818023b775b7", 19 | ] 20 | } 21 | 22 | provider "registry.terraform.io/hashicorp/local" { 23 | version = "2.1.0" 24 | hashes = [ 25 | "h1:KfieWtVyGWwplSoLIB5usKAUnrIkDQBkWaR5TI+4WYg=", 26 | "zh:0f1ec65101fa35050978d483d6e8916664b7556800348456ff3d09454ac1eae2", 27 | "zh:36e42ac19f5d68467aacf07e6adcf83c7486f2e5b5f4339e9671f68525fc87ab", 28 | "zh:6db9db2a1819e77b1642ec3b5e95042b202aee8151a0256d289f2e141bf3ceb3", 29 | "zh:719dfd97bb9ddce99f7d741260b8ece2682b363735c764cac83303f02386075a", 30 | "zh:7598bb86e0378fd97eaa04638c1a4c75f960f62f69d3662e6d80ffa5a89847fe", 31 | "zh:ad0a188b52517fec9eca393f1e2c9daea362b33ae2eb38a857b6b09949a727c1", 32 | "zh:c46846c8df66a13fee6eff7dc5d528a7f868ae0dcf92d79deaac73cc297ed20c", 33 | "zh:dc1a20a2eec12095d04bf6da5321f535351a594a636912361db20eb2a707ccc4", 34 | "zh:e57ab4771a9d999401f6badd8b018558357d3cbdf3d33cc0c4f83e818ca8e94b", 35 | "zh:ebdcde208072b4b0f8d305ebf2bfdc62c926e0717599dcf8ec2fd8c5845031c3", 36 | "zh:ef34c52b68933bedd0868a13ccfd59ff1c820f299760b3c02e008dc95e2ece91", 37 | ] 38 | } 39 | 40 | provider "registry.terraform.io/hashicorp/nomad" { 41 | version = "1.4.13" 42 | constraints = ">= 1.4.6" 43 | hashes = [ 44 | "h1:RpQGI8KThXs/t6/FZSWDogYe6AnGVpzHLU1ithLR8hU=", 45 | "zh:2406f842b2c5c70fceb5715b4dfc3ffe47630072816ecfb047b7f11169cd04fb", 46 | "zh:318575fecf8c77ea9d9c99198ec2df9d8c35ad5a3d53674d92dc6bdce5598d4d", 47 | "zh:3379c8466e0ba8f865ac2f0d909879e088e02559f527b0a45f6d9790fc7a13b5", 48 | "zh:6209427c15d6bb1ff327613d8e261758f0c1abf5d8045b2fe985d6546333b4bc", 49 | "zh:8c159fe5a9c2e12f831ac3e847ec9007e42d116ba4b8adc53c93998446d0e36d", 50 | "zh:90bc5ea082ff0400b698df4f9d70ad82d8f85d21653b341c229a477aba196bf5", 51 | "zh:a0c9c7fe2a0f024365a0e94894d074f61ab5f0db89092eeb538ba9b12ff0b9b9", 52 | "zh:b35293b9fbacca3a3ef772658d977ddc7061c94e4b460623b184293e8fc8ebb4", 53 | "zh:c5fbd8c0639a9b421f92f29268707ac6b16ae008b477256f4aac89d7f14c2f1d", 54 | "zh:d4a8cfcb867fc24ab400340a07c06a62e913317d2d20961c0b6a4f4578af6cb5", 55 | ] 56 | } 57 | 58 | provider "registry.terraform.io/hashicorp/null" { 59 | version = "3.0.0" 60 | hashes = [ 61 | "h1:V1tzrSG6t3e7zWvUwRbGbhsWU2Jd/anrJpOl9XM+R/8=", 62 | "zh:05fb7eab469324c97e9b73a61d2ece6f91de4e9b493e573bfeda0f2077bc3a4c", 63 | "zh:1688aa91885a395c4ae67636d411475d0b831e422e005dcf02eedacaafac3bb4", 64 | "zh:24a0b1292e3a474f57c483a7a4512d797e041bc9c2fbaac42fe12e86a7fb5a3c", 65 | "zh:2fc951bd0d1b9b23427acc93be09b6909d72871e464088171da60fbee4fdde03", 66 | "zh:6db825759425599a326385a68acc6be2d9ba0d7d6ef587191d0cdc6daef9ac63", 67 | "zh:85985763d02618993c32c294072cc6ec51f1692b803cb506fcfedca9d40eaec9", 68 | "zh:a53186599c57058be1509f904da512342cfdc5d808efdaf02dec15f0f3cb039a", 69 | "zh:c2e07b49b6efa676bdc7b00c06333ea1792a983a5720f9e2233db27323d2707c", 70 | "zh:cdc8fe1096103cf5374751e2e8408ec4abd2eb67d5a1c5151fe2c7ecfd525bef", 71 | "zh:dbdef21df0c012b0d08776f3d4f34eb0f2f229adfde07ff252a119e52c0f65b7", 72 | ] 73 | } 74 | 75 | provider "registry.terraform.io/hashicorp/random" { 76 | version = "3.0.1" 77 | hashes = [ 78 | "h1:0QaSbRBgBi8vI/8IRwec1INdOqBxXbgsSFElx1O4k4g=", 79 | "zh:0d4f683868324af056a9eb2b06306feef7c202c88dbbe6a4ad7517146a22fb50", 80 | "zh:4824b3c7914b77d41dfe90f6f333c7ac9860afb83e2a344d91fbe46e5dfbec26", 81 | "zh:4b82e43712f3cf0d0cbc95b2cbcd409ba8f0dc7848fdfb7c13633c27468ed04a", 82 | "zh:78b3a2b860c3ebc973a794000015f5946eb59b82705d701d487475406b2612f1", 83 | "zh:88bc65197bd74ff408d147b32f0045372ae3a3f2a2fdd7f734f315d988c0e4a2", 84 | "zh:91bd3c9f625f177f3a5d641a64e54d4b4540cb071070ecda060a8261fb6eb2ef", 85 | "zh:a6818842b28d800f784e0c93284ff602b0c4022f407e4750da03f50b853a9a2c", 86 | "zh:c4a1a2b52abd05687e6cfded4a789dcd7b43e7a746e4d02dd1055370cf9a994d", 87 | "zh:cf65041bf12fc3bde709c1d267dbe94142bc05adcabc4feb17da3b12249132ac", 88 | "zh:e385e00e7425dda9d30b74ab4ffa4636f4b8eb23918c0b763f0ffab84ece0c5c", 89 | ] 90 | } 91 | 92 | provider "registry.terraform.io/hashicorp/template" { 93 | version = "2.2.0" 94 | hashes = [ 95 | "h1:0wlehNaxBX7GJQnPfQwTNvvAf38Jm0Nv7ssKGMaG6Og=", 96 | "zh:01702196f0a0492ec07917db7aaa595843d8f171dc195f4c988d2ffca2a06386", 97 | "zh:09aae3da826ba3d7df69efeb25d146a1de0d03e951d35019a0f80e4f58c89b53", 98 | "zh:09ba83c0625b6fe0a954da6fbd0c355ac0b7f07f86c91a2a97849140fea49603", 99 | "zh:0e3a6c8e16f17f19010accd0844187d524580d9fdb0731f675ffcf4afba03d16", 100 | "zh:45f2c594b6f2f34ea663704cc72048b212fe7d16fb4cfd959365fa997228a776", 101 | "zh:77ea3e5a0446784d77114b5e851c970a3dde1e08fa6de38210b8385d7605d451", 102 | "zh:8a154388f3708e3df5a69122a23bdfaf760a523788a5081976b3d5616f7d30ae", 103 | "zh:992843002f2db5a11e626b3fc23dc0c87ad3729b3b3cff08e32ffb3df97edbde", 104 | "zh:ad906f4cebd3ec5e43d5cd6dc8f4c5c9cc3b33d2243c89c5fc18f97f7277b51d", 105 | "zh:c979425ddb256511137ecd093e23283234da0154b7fa8b21c2687182d9aea8b2", 106 | ] 107 | } 108 | -------------------------------------------------------------------------------- /cloud/aws/terraform/control/.terraform.lock.hcl: -------------------------------------------------------------------------------- 1 | # This file is maintained automatically by "terraform init". 2 | # Manual edits may be lost in future updates. 3 | 4 | provider "registry.terraform.io/hashicorp/aws" { 5 | version = "2.70.0" 6 | constraints = "~> 2.65" 7 | hashes = [ 8 | "h1:mM6eIaG1Gcrk47TveViXBO9YjY6nDaGukbED2bdo8Mk=", 9 | "zh:01a5f351146434b418f9ff8d8cc956ddc801110f1cc8b139e01be2ff8c544605", 10 | "zh:1ec08abbaf09e3e0547511d48f77a1e2c89face2d55886b23f643011c76cb247", 11 | "zh:606d134fef7c1357c9d155aadbee6826bc22bc0115b6291d483bc1444291c3e1", 12 | "zh:67e31a71a5ecbbc96a1a6708c9cc300bbfe921c322320cdbb95b9002026387e1", 13 | "zh:75aa59ae6f0834ed7142c81569182a658e4c22724a34db5d10f7545857d8db0c", 14 | "zh:76880f29fca7a0a3ff1caef31d245af2fb12a40709d67262e099bc22d039a51d", 15 | "zh:aaeaf97ffc1f76714e68bc0242c7407484c783d604584c04ad0b267b6812b6dc", 16 | "zh:ae1f88d19cc85b2e9b6ef71994134d55ef7830fd02f1f3c58c0b3f2b90e8b337", 17 | "zh:b155bdda487461e7b3d6e3a8d5ce5c887a047e4d983512e81e2c8266009f2a1f", 18 | "zh:ba394a7c391a26c4a91da63ad680e83bde0bc1ecc0a0856e26e9d62a4e77c408", 19 | "zh:e243c9d91feb0979638f28eb26f89ebadc179c57a2bd299b5729fb52bd1902f2", 20 | "zh:f6c05e20d9a3fba76ca5f47206dde35e5b43b6821c6cbf57186164ce27ba9f15", 21 | ] 22 | } 23 | 24 | provider "registry.terraform.io/hashicorp/local" { 25 | version = "2.1.0" 26 | hashes = [ 27 | "h1:KfieWtVyGWwplSoLIB5usKAUnrIkDQBkWaR5TI+4WYg=", 28 | "zh:0f1ec65101fa35050978d483d6e8916664b7556800348456ff3d09454ac1eae2", 29 | "zh:36e42ac19f5d68467aacf07e6adcf83c7486f2e5b5f4339e9671f68525fc87ab", 30 | "zh:6db9db2a1819e77b1642ec3b5e95042b202aee8151a0256d289f2e141bf3ceb3", 31 | "zh:719dfd97bb9ddce99f7d741260b8ece2682b363735c764cac83303f02386075a", 32 | "zh:7598bb86e0378fd97eaa04638c1a4c75f960f62f69d3662e6d80ffa5a89847fe", 33 | "zh:ad0a188b52517fec9eca393f1e2c9daea362b33ae2eb38a857b6b09949a727c1", 34 | "zh:c46846c8df66a13fee6eff7dc5d528a7f868ae0dcf92d79deaac73cc297ed20c", 35 | "zh:dc1a20a2eec12095d04bf6da5321f535351a594a636912361db20eb2a707ccc4", 36 | "zh:e57ab4771a9d999401f6badd8b018558357d3cbdf3d33cc0c4f83e818ca8e94b", 37 | "zh:ebdcde208072b4b0f8d305ebf2bfdc62c926e0717599dcf8ec2fd8c5845031c3", 38 | "zh:ef34c52b68933bedd0868a13ccfd59ff1c820f299760b3c02e008dc95e2ece91", 39 | ] 40 | } 41 | 42 | provider "registry.terraform.io/hashicorp/nomad" { 43 | version = "1.4.13" 44 | constraints = ">= 1.4.6" 45 | hashes = [ 46 | "h1:RpQGI8KThXs/t6/FZSWDogYe6AnGVpzHLU1ithLR8hU=", 47 | "zh:2406f842b2c5c70fceb5715b4dfc3ffe47630072816ecfb047b7f11169cd04fb", 48 | "zh:318575fecf8c77ea9d9c99198ec2df9d8c35ad5a3d53674d92dc6bdce5598d4d", 49 | "zh:3379c8466e0ba8f865ac2f0d909879e088e02559f527b0a45f6d9790fc7a13b5", 50 | "zh:6209427c15d6bb1ff327613d8e261758f0c1abf5d8045b2fe985d6546333b4bc", 51 | "zh:8c159fe5a9c2e12f831ac3e847ec9007e42d116ba4b8adc53c93998446d0e36d", 52 | "zh:90bc5ea082ff0400b698df4f9d70ad82d8f85d21653b341c229a477aba196bf5", 53 | "zh:a0c9c7fe2a0f024365a0e94894d074f61ab5f0db89092eeb538ba9b12ff0b9b9", 54 | "zh:b35293b9fbacca3a3ef772658d977ddc7061c94e4b460623b184293e8fc8ebb4", 55 | "zh:c5fbd8c0639a9b421f92f29268707ac6b16ae008b477256f4aac89d7f14c2f1d", 56 | "zh:d4a8cfcb867fc24ab400340a07c06a62e913317d2d20961c0b6a4f4578af6cb5", 57 | ] 58 | } 59 | 60 | provider "registry.terraform.io/hashicorp/null" { 61 | version = "3.1.0" 62 | hashes = [ 63 | "h1:xhbHC6in3nQryvTQBWKxebi3inG5OCgHgc4fRxL0ymc=", 64 | "zh:02a1675fd8de126a00460942aaae242e65ca3380b5bb192e8773ef3da9073fd2", 65 | "zh:53e30545ff8926a8e30ad30648991ca8b93b6fa496272cd23b26763c8ee84515", 66 | "zh:5f9200bf708913621d0f6514179d89700e9aa3097c77dac730e8ba6e5901d521", 67 | "zh:9ebf4d9704faba06b3ec7242c773c0fbfe12d62db7d00356d4f55385fc69bfb2", 68 | "zh:a6576c81adc70326e4e1c999c04ad9ca37113a6e925aefab4765e5a5198efa7e", 69 | "zh:a8a42d13346347aff6c63a37cda9b2c6aa5cc384a55b2fe6d6adfa390e609c53", 70 | "zh:c797744d08a5307d50210e0454f91ca4d1c7621c68740441cf4579390452321d", 71 | "zh:cecb6a304046df34c11229f20a80b24b1603960b794d68361a67c5efe58e62b8", 72 | "zh:e1371aa1e502000d9974cfaff5be4cfa02f47b17400005a16f14d2ef30dc2a70", 73 | "zh:fc39cc1fe71234a0b0369d5c5c7f876c71b956d23d7d6f518289737a001ba69b", 74 | "zh:fea4227271ebf7d9e2b61b89ce2328c7262acd9fd190e1fd6d15a591abfa848e", 75 | ] 76 | } 77 | 78 | provider "registry.terraform.io/hashicorp/random" { 79 | version = "3.1.0" 80 | hashes = [ 81 | "h1:rKYu5ZUbXwrLG1w81k7H3nce/Ys6yAxXhWcbtk36HjY=", 82 | "zh:2bbb3339f0643b5daa07480ef4397bd23a79963cc364cdfbb4e86354cb7725bc", 83 | "zh:3cd456047805bf639fbf2c761b1848880ea703a054f76db51852008b11008626", 84 | "zh:4f251b0eda5bb5e3dc26ea4400dba200018213654b69b4a5f96abee815b4f5ff", 85 | "zh:7011332745ea061e517fe1319bd6c75054a314155cb2c1199a5b01fe1889a7e2", 86 | "zh:738ed82858317ccc246691c8b85995bc125ac3b4143043219bd0437adc56c992", 87 | "zh:7dbe52fac7bb21227acd7529b487511c91f4107db9cc4414f50d04ffc3cab427", 88 | "zh:a3a9251fb15f93e4cfc1789800fc2d7414bbc18944ad4c5c98f466e6477c42bc", 89 | "zh:a543ec1a3a8c20635cf374110bd2f87c07374cf2c50617eee2c669b3ceeeaa9f", 90 | "zh:d9ab41d556a48bd7059f0810cf020500635bfc696c9fc3adab5ea8915c1d886b", 91 | "zh:d9e13427a7d011dbd654e591b0337e6074eef8c3b9bb11b2e39eaaf257044fd7", 92 | "zh:f7605bd1437752114baf601bdf6931debe6dc6bfe3006eb7e9bb9080931dca8a", 93 | ] 94 | } 95 | 96 | provider "registry.terraform.io/hashicorp/template" { 97 | version = "2.2.0" 98 | hashes = [ 99 | "h1:0wlehNaxBX7GJQnPfQwTNvvAf38Jm0Nv7ssKGMaG6Og=", 100 | "zh:01702196f0a0492ec07917db7aaa595843d8f171dc195f4c988d2ffca2a06386", 101 | "zh:09aae3da826ba3d7df69efeb25d146a1de0d03e951d35019a0f80e4f58c89b53", 102 | "zh:09ba83c0625b6fe0a954da6fbd0c355ac0b7f07f86c91a2a97849140fea49603", 103 | "zh:0e3a6c8e16f17f19010accd0844187d524580d9fdb0731f675ffcf4afba03d16", 104 | "zh:45f2c594b6f2f34ea663704cc72048b212fe7d16fb4cfd959365fa997228a776", 105 | "zh:77ea3e5a0446784d77114b5e851c970a3dde1e08fa6de38210b8385d7605d451", 106 | "zh:8a154388f3708e3df5a69122a23bdfaf760a523788a5081976b3d5616f7d30ae", 107 | "zh:992843002f2db5a11e626b3fc23dc0c87ad3729b3b3cff08e32ffb3df97edbde", 108 | "zh:ad906f4cebd3ec5e43d5cd6dc8f4c5c9cc3b33d2243c89c5fc18f97f7277b51d", 109 | "zh:c979425ddb256511137ecd093e23283234da0154b7fa8b21c2687182d9aea8b2", 110 | ] 111 | } 112 | -------------------------------------------------------------------------------- /cloud/azure/terraform/modules/azure-hashistack/clients_lb.tf: -------------------------------------------------------------------------------- 1 | resource "azurerm_public_ip" "clients_lb" { 2 | name = "clients-lb-ip" 3 | location = azurerm_resource_group.hashistack.location 4 | resource_group_name = azurerm_resource_group.hashistack.name 5 | allocation_method = "Static" 6 | } 7 | 8 | resource "azurerm_lb" "clients" { 9 | name = "clients-lb" 10 | location = azurerm_resource_group.hashistack.location 11 | resource_group_name = azurerm_resource_group.hashistack.name 12 | 13 | frontend_ip_configuration { 14 | name = "PublicIPAddress" 15 | public_ip_address_id = azurerm_public_ip.clients_lb.id 16 | } 17 | } 18 | 19 | resource "azurerm_lb_backend_address_pool" "clients_lb" { 20 | name = "clients-lb-backend" 21 | resource_group_name = azurerm_resource_group.hashistack.name 22 | loadbalancer_id = azurerm_lb.clients.id 23 | } 24 | 25 | # Nomad rule 26 | resource "azurerm_lb_probe" "clients_nomad" { 27 | name = "clients-nomad-lb-probe" 28 | resource_group_name = azurerm_resource_group.hashistack.name 29 | loadbalancer_id = azurerm_lb.clients.id 30 | port = 4646 31 | } 32 | 33 | resource "azurerm_lb_rule" "clients_nomad" { 34 | name = "clients-nomad-lb-rule" 35 | resource_group_name = azurerm_resource_group.hashistack.name 36 | loadbalancer_id = azurerm_lb.clients.id 37 | backend_address_pool_id = azurerm_lb_backend_address_pool.clients_lb.id 38 | probe_id = azurerm_lb_probe.clients_nomad.id 39 | protocol = "Tcp" 40 | frontend_port = 4646 41 | backend_port = 4646 42 | frontend_ip_configuration_name = "PublicIPAddress" 43 | } 44 | 45 | # Consul rule 46 | resource "azurerm_lb_probe" "clients_consul" { 47 | name = "clients-consul-lb-probe" 48 | resource_group_name = azurerm_resource_group.hashistack.name 49 | loadbalancer_id = azurerm_lb.clients.id 50 | port = 8500 51 | } 52 | 53 | resource "azurerm_lb_rule" "clients_consul" { 54 | name = "clients-consul-lb-rule" 55 | resource_group_name = azurerm_resource_group.hashistack.name 56 | loadbalancer_id = azurerm_lb.clients.id 57 | backend_address_pool_id = azurerm_lb_backend_address_pool.clients_lb.id 58 | probe_id = azurerm_lb_probe.clients_consul.id 59 | protocol = "Tcp" 60 | frontend_port = 8500 61 | backend_port = 8500 62 | frontend_ip_configuration_name = "PublicIPAddress" 63 | } 64 | 65 | # Grafana rule 66 | resource "azurerm_lb_probe" "clients_grafana" { 67 | name = "clients-grafana-lb-probe" 68 | resource_group_name = azurerm_resource_group.hashistack.name 69 | loadbalancer_id = azurerm_lb.clients.id 70 | port = 3000 71 | } 72 | 73 | resource "azurerm_lb_rule" "clients_grafana" { 74 | name = "clients-grafana-lb-rule" 75 | resource_group_name = azurerm_resource_group.hashistack.name 76 | loadbalancer_id = azurerm_lb.clients.id 77 | backend_address_pool_id = azurerm_lb_backend_address_pool.clients_lb.id 78 | probe_id = azurerm_lb_probe.clients_grafana.id 79 | protocol = "Tcp" 80 | frontend_port = 3000 81 | backend_port = 3000 82 | frontend_ip_configuration_name = "PublicIPAddress" 83 | } 84 | 85 | # Prometheus rule 86 | resource "azurerm_lb_probe" "clients_prometheus" { 87 | name = "clients-prometheus-lb-probe" 88 | resource_group_name = azurerm_resource_group.hashistack.name 89 | loadbalancer_id = azurerm_lb.clients.id 90 | port = 9090 91 | } 92 | 93 | resource "azurerm_lb_rule" "clients_prometheus" { 94 | name = "clients-prometheus-lb-rule" 95 | resource_group_name = azurerm_resource_group.hashistack.name 96 | loadbalancer_id = azurerm_lb.clients.id 97 | backend_address_pool_id = azurerm_lb_backend_address_pool.clients_lb.id 98 | probe_id = azurerm_lb_probe.clients_prometheus.id 99 | protocol = "Tcp" 100 | frontend_port = 9090 101 | backend_port = 9090 102 | frontend_ip_configuration_name = "PublicIPAddress" 103 | } 104 | 105 | # Traefik rule 106 | resource "azurerm_lb_probe" "clients_traefik" { 107 | name = "clients-traefik-lb-probe" 108 | resource_group_name = azurerm_resource_group.hashistack.name 109 | loadbalancer_id = azurerm_lb.clients.id 110 | port = 8081 111 | } 112 | 113 | resource "azurerm_lb_rule" "clients_traefik" { 114 | name = "clients-traefik-lb-rule" 115 | resource_group_name = azurerm_resource_group.hashistack.name 116 | loadbalancer_id = azurerm_lb.clients.id 117 | backend_address_pool_id = azurerm_lb_backend_address_pool.clients_lb.id 118 | probe_id = azurerm_lb_probe.clients_traefik.id 119 | protocol = "Tcp" 120 | frontend_port = 8081 121 | backend_port = 8081 122 | frontend_ip_configuration_name = "PublicIPAddress" 123 | } 124 | 125 | # HTTP rule 126 | resource "azurerm_lb_probe" "clients_http" { 127 | name = "clients-http-lb-probe" 128 | resource_group_name = azurerm_resource_group.hashistack.name 129 | loadbalancer_id = azurerm_lb.clients.id 130 | port = 80 131 | } 132 | 133 | resource "azurerm_lb_rule" "clients_http" { 134 | name = "clients-http-lb-rule" 135 | resource_group_name = azurerm_resource_group.hashistack.name 136 | loadbalancer_id = azurerm_lb.clients.id 137 | backend_address_pool_id = azurerm_lb_backend_address_pool.clients_lb.id 138 | probe_id = azurerm_lb_probe.clients_http.id 139 | protocol = "Tcp" 140 | frontend_port = 80 141 | backend_port = 80 142 | frontend_ip_configuration_name = "PublicIPAddress" 143 | } 144 | -------------------------------------------------------------------------------- /cloud/azure/README.md: -------------------------------------------------------------------------------- 1 | ### Microsoft Azure 2 | Some of the steps below will require that you have the 3 | [Azure CLI installed](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli) 4 | in your machine. 5 | 6 | ### Environment setup 7 | The first step is to login using the CLI: 8 | 9 | ```shellsession 10 | $ az login 11 | [ 12 | { 13 | "cloudName": "AzureCloud", 14 | "id": "", 15 | "isDefault": true, 16 | "name": "Free Trial", 17 | "state": "Enabled", 18 | "tenantId": "", 19 | "user": { 20 | "name": "user@example.com", 21 | "type": "user" 22 | } 23 | } 24 | ``` 25 | 26 | Take a note of the values for `` and `` end export 27 | them as environment variables: 28 | 29 | ```shellsession 30 | $ export ARM_SUBSCRIPTION_ID= 31 | $ export ARM_TENANT_ID= 32 | ``` 33 | 34 | Next, create an application ID and password that will be used to run Terraform: 35 | 36 | ```shellsession 37 | $ az ad sp create-for-rbac --role="Owner" --scopes="/subscriptions/$ARM_SUBSCRIPTION_ID" 38 | { 39 | "appId": "", 40 | "displayName": "azure-cli-...", 41 | "name": "http://azure-cli-...", 42 | "password": "", 43 | "tenant": "" 44 | } 45 | ``` 46 | 47 | Export the values for `` and `` as environment 48 | variables as well: 49 | 50 | ```shellsession 51 | $ export ARM_CLIENT_ID= 52 | $ export ARM_CLIENT_SECRET= 53 | ``` 54 | 55 | # Running Terraform 56 | Navigate to the Terraform control folder and execute the Terraform 57 | configuration to deploy the demo infrastructure: 58 | 59 | ```shellsession 60 | $ cd ./terraform/control 61 | $ terraform init 62 | $ terraform plan 63 | $ terraform apply --auto-approve 64 | ``` 65 | 66 | Once the Terraform apply finishes, a number of useful pieces of information 67 | should be output to your console. These include URLs to deployed resources as 68 | well as a semi-populated Nomad Autoscaler config. 69 | 70 | ``` 71 | ip_addresses = 72 | Server IPs: 73 | * instance server-1 - Public: 52.188.111.20, Private: 10.0.2.4 74 | 75 | 76 | To connect, add your private key and SSH into any client or server with 77 | `ssh -i azure-hashistack.pem -o IdentitiesOnly=yes ubuntu@PUBLIC_IP`. 78 | You can test the integrity of the cluster by running: 79 | 80 | $ consul members 81 | $ nomad server members 82 | $ nomad node status 83 | 84 | The Nomad UI can be accessed at http://52.249.185.10:4646/ui 85 | The Consul UI can be accessed at http://52.249.185.10:8500/ui 86 | Grafana dashbaord can be accessed at http://52.249.187.190:3000/d/AQphTqmMk/demo?orgId=1&refresh=5s 87 | Traefik can be accessed at http://52.249.187.190:8081 88 | Prometheus can be accessed at http://52.249.187.190:9090 89 | Webapp can be accessed at http://52.249.187.190:80 90 | 91 | CLI environment variables: 92 | export NOMAD_CLIENT_DNS=http://52.249.187.190 93 | export NOMAD_ADDR=http://52.249.185.10:4646 94 | ``` 95 | 96 | You can visit the URLs and explore what has been created. This will include 97 | registration of a number of Nomad jobs which provide metrics and dashboarding 98 | as well as a demo application and routing provided by Traefik. It may take a 99 | few seconds for all the applications to start, so if any of the URLs doesn't 100 | load the first time, wait a little bit and give it a try again. 101 | 102 | Please also copy the export commands and run these in the terminal where the 103 | rest of the demo will be run. 104 | 105 | The application is pre-configured with a scaling policy, you can view this by 106 | opening the job file or calling the Nomad API. The application scales based on 107 | the average number of active connections, and we are targeting an average of 10 108 | per instance of our app. 109 | ``` 110 | curl $NOMAD_ADDR/v1/scaling/policies 111 | ``` 112 | 113 | ## Run the Autoscaler 114 | The Autoscaler is not triggered automatically. This provides the opportunity to 115 | look through the jobfile to understand it better before deploying. The most 116 | important parts of the `azure_autoscaler.nomad` file are the template sections. 117 | The first defines our agent config where we configure the `prometheus`, 118 | `azure-vmss` and `target-value` plugins. 119 | 120 | ```hcl 121 | template { 122 | data = < 208 | ``` 209 | 210 | You can now return to the [demo instrunctions](../README.md#the-demo). 211 | -------------------------------------------------------------------------------- /vagrant/dynamic-app-sizing/README.md: -------------------------------------------------------------------------------- 1 | # Dynamic Application Sizing demo 2 | This Vagrantfile and associated configuration and job files provide a basic demo for running and 3 | using the Dynamic Application Sizing feature of Nomad Enterprise and Nomad Autoscaler Enterprise. 4 | The demo will enable Dynamic Application Sizing on an example web application. 5 | 6 | ## Requirements 7 | To use the Vagrant environment, first install Vagrant following these [instructions](https://www.vagrantup.com/docs/installation/). 8 | You will also need a virtualization tool, such as [VirtualBox](https://www.virtualbox.org/). 9 | 10 | ## Start the Virtual Machine 11 | Vagrant is used to provision a virtual machine which will run a Nomad server and client for scheduling 12 | work. To create the virtual machine, you should run the `vagrant up` command. 13 | ```shellsession 14 | $ vagrant up 15 | ``` 16 | This will take a few minutes as the base Ubuntu box must be downloaded and provisioned with both Docker 17 | and Nomad. Once this completes, you should see this output. 18 | ```shellsession 19 | Bringing machine 'default' up with 'virtualbox' provider... 20 | ==> default: Importing base box 'ubuntu/bionic64'... 21 | ... 22 | ==> default: Running provisioner: deps (shell)... 23 | ``` 24 | At this point the Vagrant box is running and ready to go. 25 | 26 | ## Start the Nomad Base Jobs 27 | In order to start the Nomad jobs you should ssh onto the vagrant machine using the `vagrant ssh` 28 | command and navigate to the jobs directory. 29 | ```shellsession 30 | $ vagrant ssh 31 | $ cd /home/vagrant/nomad-autoscaler/jobs/ 32 | ``` 33 | 34 | The demo uses [Prometheus](https://prometheus.io/) to provide historical metrics on Nomad job resources. 35 | This should be started using the `nomad run` command. Once the application has started, the Prometheus 36 | UI can be viewed from your workstation by browsing to http://127.0.0.1:9090. 37 | ```shellsession 38 | $ nomad run prometheus.nomad 39 | ``` 40 | 41 | You can check the status of the job and the allocation using the `nomad status` command. 42 | ```shellsession 43 | $ nomad status prometheus 44 | $ nomad status 45 | ``` 46 | 47 | The example application can now be started, again using the `nomad run` command. The job specification 48 | includes scaling policies within the task stanza, which enables the task for CPU and Memory based 49 | dynamic sizing. 50 | ```shellsession 51 | $ nomad run example.nomad 52 | ``` 53 | 54 | With Prometheus and the example job running, you can view the raw data used by the Nomad Autoscaler 55 | to make recommendations at any time at the following endpoints: 56 | * [allocation CPU resource usage](http://localhost:9090/graph?g0.range_input=30m&g0.expr=nomad_client_allocs_cpu_total_ticks%7Bexported_job%3D%22example%22%2Ctask%3D%22redis%22%2Ctask_group%3D%22cache%22%7D&g0.tab=0) 57 | * [allocation memory resource usage](http://localhost:9090/graph?g0.range_input=30m&g0.expr=nomad_client_allocs_memory_usage%7Bexported_job%3D%22example%22%2Ctask%3D%22redis%22%2Ctask_group%3D%22cache%22%7D%2F1024%2F1024&g0.tab=0) 58 | 59 | ## Start the Nomad Autoscaler 60 | The Nomad Autoscaler Enterprise binary is available on the Vagrant machine. In order to run the binary 61 | you can use the provided `das-autoscaler.nomad` job. 62 | ```shellsession 63 | $ nomad run das-autoscaler.nomad 64 | ``` 65 | 66 | To see what's the Autoscaler is doing, you can tail its logs in the [Nomad UI](http://localhost:4646) 67 | or via the CLI with the command: 68 | ```shellsession 69 | $ nomad alloc logs -f -stderr 70 | 2020-10-19T22:37:11.531Z [INFO] agent.plugin_manager: successfully launched and dispensed plugin: plugin_name=nomad-target 71 | 2020-10-19T22:37:11.531Z [INFO] agent.plugin_manager: successfully launched and dispensed plugin: plugin_name=app-sizing-nomad 72 | 2020-10-19T22:37:11.531Z [INFO] agent.plugin_manager: successfully launched and dispensed plugin: plugin_name=nomad-apm 73 | 2020-10-19T22:37:11.531Z [INFO] agent.plugin_manager: successfully launched and dispensed plugin: plugin_name=prometheus 74 | 2020-10-19T22:37:11.531Z [INFO] agent.plugin_manager: successfully launched and dispensed plugin: plugin_name=target-value 75 | 2020-10-19T22:37:11.531Z [INFO] agent.plugin_manager: successfully launched and dispensed plugin: plugin_name=app-sizing-avg 76 | 2020-10-19T22:37:11.531Z [INFO] agent.plugin_manager: successfully launched and dispensed plugin: plugin_name=app-sizing-max 77 | 2020-10-19T22:37:11.531Z [INFO] agent.plugin_manager: successfully launched and dispensed plugin: plugin_name=app-sizing-percentile 78 | 2020-10-19T22:37:11.533Z [INFO] agent.http_server: server now listening for connections: address=127.0.0.1:8080 79 | 2020-10-19T22:37:11.533Z [DEBUG] nomad_policy_source: starting policy blocking query watcher 80 | 2020-10-19T22:37:11.533Z [INFO] policy_eval.worker: starting worker: id=6af2f4b3-52d8-1466-eda0-c47516e5396a queue=vertical_mem 81 | 2020-10-19T22:37:11.534Z [DEBUG] policy_eval.broker: dequeue eval: queue=vertical_mem 82 | 2020-10-19T22:37:11.534Z [DEBUG] policy_eval.broker: waiting for eval: queue=vertical_mem 83 | 2020-10-19T22:37:11.534Z [INFO] policy_eval.worker: starting worker: id=4e4c01d8-da8a-9a6c-59ca-bb524691ec14 queue=vertical_cpu 84 | ... 85 | ``` 86 | 87 | ## Check for Dynamic Application Sizing Recommendations 88 | Once the Nomad Autoscaler is running, it will take about 10 seconds until the first recommendation is 89 | calculated and submitted to Nomad. You can navigate to the [Nomad Optimize](http://localhost:4646/ui/optimize) 90 | UI page to view this. Feel free to accept or reject them. 91 | 92 | ### Generate Application Load 93 | You can cause a load increase on the example application by using the `redis-benchmark` CLI to interact 94 | with the Redis instances. 95 | 96 | This CLI is wrapped in a Nomad dispatch batch job. First, register the batch job in Nomad: 97 | ```shellsession 98 | $ nomad run das-load-test.nomad 99 | Job registration successful 100 | ``` 101 | 102 | And then run it with the command: 103 | ```shellsession 104 | $ nomad job dispatch das-load-test 105 | Dispatched Job ID = das-load-test/dispatch-1603150078-4ce64dd2 106 | Evaluation ID = 003c42fb 107 | 108 | ==> Monitoring evaluation "003c42fb" 109 | Evaluation triggered by job "das-load-test/dispatch-1603150078-4ce64dd2" 110 | Allocation "1e6c9ae1" created: node "96dfe679", group "redis-benchmark" 111 | Evaluation status changed: "pending" -> "complete" 112 | ==> Evaluation "003c42fb" finished with status "complete" 113 | ``` 114 | 115 | The increase in load should result in new recommendation being availble within Nomad. You can again 116 | navigate to the [Nomad Optimize](http://localhost:4646/ui/optimize) page to view and interact with 117 | them. 118 | 119 | If you leave the jobs running with no extra load, you will receive recommendations that slowly 120 | decrease resource limits since there's no more load in the system. 121 | 122 | ## Demo End 123 | Congratulations, you have run through the Dynamic Application Sizing Nomad Autoscaler Vagrant demo. 124 | In order to destroy the created virtual machine, close all SSH connection and then issue a `vagrant destroy -f` 125 | command. 126 | ``` 127 | $ vagrant destroy -f 128 | ``` 129 | -------------------------------------------------------------------------------- /cloud/azure/terraform/control/.terraform.lock.hcl: -------------------------------------------------------------------------------- 1 | # This file is maintained automatically by "terraform init". 2 | # Manual edits may be lost in future updates. 3 | 4 | provider "registry.terraform.io/hashicorp/azurerm" { 5 | version = "2.32.0" 6 | constraints = "2.32.0" 7 | hashes = [ 8 | "h1:MUqXaC1Qg8IkHzkI4wDVRCC3Fnm0CC3Bu/nJkGF/BnI=", 9 | "zh:3203d4a775b40a275b739a6c76e02de2180b6f590edbe437cb1bcf0f85d1df7c", 10 | "zh:3d0b8337309cdcb8119887e69ad415aa6e581747ea02cde21ed05d7c45b00880", 11 | "zh:4cda189d8f53c78c9cbacc7281a841457886cc40b2bdb1303482ab00e3762f89", 12 | "zh:4eb9a1b3222c16d0e4060f3dca946737f364bf2a75f47cd66b78d6a290dfc309", 13 | "zh:8deecf6a0b9df92b75b7cda7987c7795ae80610b8cab041b46c0953616063cc1", 14 | "zh:9ebdd4490bd1e1a96e7ea5f0335c273687112ce396cb465bee7330341499296e", 15 | "zh:e0d305d1ab8638ef59d48a9f5036623c8c296102d226b89ed6b57215133ef971", 16 | "zh:e1d0f2e6214f69ecf9ea2e28986c9b8060581b1f43837781c90a036198f81012", 17 | "zh:f425d57544f3826bf2d75d0163b826808cc1f09889f4a719bd1ae23dbd406ac1", 18 | "zh:f82f2116ac814acc51457938c899daf604a5579d3a22014be92cafa9f0615fca", 19 | ] 20 | } 21 | 22 | provider "registry.terraform.io/hashicorp/local" { 23 | version = "2.1.0" 24 | hashes = [ 25 | "h1:KfieWtVyGWwplSoLIB5usKAUnrIkDQBkWaR5TI+4WYg=", 26 | "zh:0f1ec65101fa35050978d483d6e8916664b7556800348456ff3d09454ac1eae2", 27 | "zh:36e42ac19f5d68467aacf07e6adcf83c7486f2e5b5f4339e9671f68525fc87ab", 28 | "zh:6db9db2a1819e77b1642ec3b5e95042b202aee8151a0256d289f2e141bf3ceb3", 29 | "zh:719dfd97bb9ddce99f7d741260b8ece2682b363735c764cac83303f02386075a", 30 | "zh:7598bb86e0378fd97eaa04638c1a4c75f960f62f69d3662e6d80ffa5a89847fe", 31 | "zh:ad0a188b52517fec9eca393f1e2c9daea362b33ae2eb38a857b6b09949a727c1", 32 | "zh:c46846c8df66a13fee6eff7dc5d528a7f868ae0dcf92d79deaac73cc297ed20c", 33 | "zh:dc1a20a2eec12095d04bf6da5321f535351a594a636912361db20eb2a707ccc4", 34 | "zh:e57ab4771a9d999401f6badd8b018558357d3cbdf3d33cc0c4f83e818ca8e94b", 35 | "zh:ebdcde208072b4b0f8d305ebf2bfdc62c926e0717599dcf8ec2fd8c5845031c3", 36 | "zh:ef34c52b68933bedd0868a13ccfd59ff1c820f299760b3c02e008dc95e2ece91", 37 | ] 38 | } 39 | 40 | provider "registry.terraform.io/hashicorp/nomad" { 41 | version = "1.4.13" 42 | constraints = ">= 1.4.6" 43 | hashes = [ 44 | "h1:RpQGI8KThXs/t6/FZSWDogYe6AnGVpzHLU1ithLR8hU=", 45 | "zh:2406f842b2c5c70fceb5715b4dfc3ffe47630072816ecfb047b7f11169cd04fb", 46 | "zh:318575fecf8c77ea9d9c99198ec2df9d8c35ad5a3d53674d92dc6bdce5598d4d", 47 | "zh:3379c8466e0ba8f865ac2f0d909879e088e02559f527b0a45f6d9790fc7a13b5", 48 | "zh:6209427c15d6bb1ff327613d8e261758f0c1abf5d8045b2fe985d6546333b4bc", 49 | "zh:8c159fe5a9c2e12f831ac3e847ec9007e42d116ba4b8adc53c93998446d0e36d", 50 | "zh:90bc5ea082ff0400b698df4f9d70ad82d8f85d21653b341c229a477aba196bf5", 51 | "zh:a0c9c7fe2a0f024365a0e94894d074f61ab5f0db89092eeb538ba9b12ff0b9b9", 52 | "zh:b35293b9fbacca3a3ef772658d977ddc7061c94e4b460623b184293e8fc8ebb4", 53 | "zh:c5fbd8c0639a9b421f92f29268707ac6b16ae008b477256f4aac89d7f14c2f1d", 54 | "zh:d4a8cfcb867fc24ab400340a07c06a62e913317d2d20961c0b6a4f4578af6cb5", 55 | ] 56 | } 57 | 58 | provider "registry.terraform.io/hashicorp/null" { 59 | version = "3.1.0" 60 | hashes = [ 61 | "h1:xhbHC6in3nQryvTQBWKxebi3inG5OCgHgc4fRxL0ymc=", 62 | "zh:02a1675fd8de126a00460942aaae242e65ca3380b5bb192e8773ef3da9073fd2", 63 | "zh:53e30545ff8926a8e30ad30648991ca8b93b6fa496272cd23b26763c8ee84515", 64 | "zh:5f9200bf708913621d0f6514179d89700e9aa3097c77dac730e8ba6e5901d521", 65 | "zh:9ebf4d9704faba06b3ec7242c773c0fbfe12d62db7d00356d4f55385fc69bfb2", 66 | "zh:a6576c81adc70326e4e1c999c04ad9ca37113a6e925aefab4765e5a5198efa7e", 67 | "zh:a8a42d13346347aff6c63a37cda9b2c6aa5cc384a55b2fe6d6adfa390e609c53", 68 | "zh:c797744d08a5307d50210e0454f91ca4d1c7621c68740441cf4579390452321d", 69 | "zh:cecb6a304046df34c11229f20a80b24b1603960b794d68361a67c5efe58e62b8", 70 | "zh:e1371aa1e502000d9974cfaff5be4cfa02f47b17400005a16f14d2ef30dc2a70", 71 | "zh:fc39cc1fe71234a0b0369d5c5c7f876c71b956d23d7d6f518289737a001ba69b", 72 | "zh:fea4227271ebf7d9e2b61b89ce2328c7262acd9fd190e1fd6d15a591abfa848e", 73 | ] 74 | } 75 | 76 | provider "registry.terraform.io/hashicorp/random" { 77 | version = "3.1.0" 78 | hashes = [ 79 | "h1:rKYu5ZUbXwrLG1w81k7H3nce/Ys6yAxXhWcbtk36HjY=", 80 | "zh:2bbb3339f0643b5daa07480ef4397bd23a79963cc364cdfbb4e86354cb7725bc", 81 | "zh:3cd456047805bf639fbf2c761b1848880ea703a054f76db51852008b11008626", 82 | "zh:4f251b0eda5bb5e3dc26ea4400dba200018213654b69b4a5f96abee815b4f5ff", 83 | "zh:7011332745ea061e517fe1319bd6c75054a314155cb2c1199a5b01fe1889a7e2", 84 | "zh:738ed82858317ccc246691c8b85995bc125ac3b4143043219bd0437adc56c992", 85 | "zh:7dbe52fac7bb21227acd7529b487511c91f4107db9cc4414f50d04ffc3cab427", 86 | "zh:a3a9251fb15f93e4cfc1789800fc2d7414bbc18944ad4c5c98f466e6477c42bc", 87 | "zh:a543ec1a3a8c20635cf374110bd2f87c07374cf2c50617eee2c669b3ceeeaa9f", 88 | "zh:d9ab41d556a48bd7059f0810cf020500635bfc696c9fc3adab5ea8915c1d886b", 89 | "zh:d9e13427a7d011dbd654e591b0337e6074eef8c3b9bb11b2e39eaaf257044fd7", 90 | "zh:f7605bd1437752114baf601bdf6931debe6dc6bfe3006eb7e9bb9080931dca8a", 91 | ] 92 | } 93 | 94 | provider "registry.terraform.io/hashicorp/template" { 95 | version = "2.2.0" 96 | hashes = [ 97 | "h1:0wlehNaxBX7GJQnPfQwTNvvAf38Jm0Nv7ssKGMaG6Og=", 98 | "zh:01702196f0a0492ec07917db7aaa595843d8f171dc195f4c988d2ffca2a06386", 99 | "zh:09aae3da826ba3d7df69efeb25d146a1de0d03e951d35019a0f80e4f58c89b53", 100 | "zh:09ba83c0625b6fe0a954da6fbd0c355ac0b7f07f86c91a2a97849140fea49603", 101 | "zh:0e3a6c8e16f17f19010accd0844187d524580d9fdb0731f675ffcf4afba03d16", 102 | "zh:45f2c594b6f2f34ea663704cc72048b212fe7d16fb4cfd959365fa997228a776", 103 | "zh:77ea3e5a0446784d77114b5e851c970a3dde1e08fa6de38210b8385d7605d451", 104 | "zh:8a154388f3708e3df5a69122a23bdfaf760a523788a5081976b3d5616f7d30ae", 105 | "zh:992843002f2db5a11e626b3fc23dc0c87ad3729b3b3cff08e32ffb3df97edbde", 106 | "zh:ad906f4cebd3ec5e43d5cd6dc8f4c5c9cc3b33d2243c89c5fc18f97f7277b51d", 107 | "zh:c979425ddb256511137ecd093e23283234da0154b7fa8b21c2687182d9aea8b2", 108 | ] 109 | } 110 | 111 | provider "registry.terraform.io/hashicorp/tls" { 112 | version = "3.1.0" 113 | hashes = [ 114 | "h1:XTU9f6sGMZHOT8r/+LWCz2BZOPH127FBTPjMMEAAu1U=", 115 | "zh:3d46616b41fea215566f4a957b6d3a1aa43f1f75c26776d72a98bdba79439db6", 116 | "zh:623a203817a6dafa86f1b4141b645159e07ec418c82fe40acd4d2a27543cbaa2", 117 | "zh:668217e78b210a6572e7b0ecb4134a6781cc4d738f4f5d09eb756085b082592e", 118 | "zh:95354df03710691773c8f50a32e31fca25f124b7f3d6078265fdf3c4e1384dca", 119 | "zh:9f97ab190380430d57392303e3f36f4f7835c74ea83276baa98d6b9a997c3698", 120 | "zh:a16f0bab665f8d933e95ca055b9c8d5707f1a0dd8c8ecca6c13091f40dc1e99d", 121 | "zh:be274d5008c24dc0d6540c19e22dbb31ee6bfdd0b2cddd4d97f3cd8a8d657841", 122 | "zh:d5faa9dce0a5fc9d26b2463cea5be35f8586ab75030e7fa4d4920cd73ee26989", 123 | "zh:e9b672210b7fb410780e7b429975adcc76dd557738ecc7c890ea18942eb321a5", 124 | "zh:eb1f8368573d2370605d6dbf60f9aaa5b64e55741d96b5fb026dbfe91de67c0d", 125 | "zh:fc1e12b713837b85daf6c3bb703d7795eaf1c5177aebae1afcf811dd7009f4b0", 126 | ] 127 | } 128 | -------------------------------------------------------------------------------- /vagrant/horizontal-app-scaling/README.md: -------------------------------------------------------------------------------- 1 | # Vagrant Nomad Autoscaler Demo 2 | This Vagrantfile and associated configuration and job files are meant to provide a basic demo for running and using the Nomad Autoscaler. The demo will enable autoscaling on an example web application, showing both scaling in and scaling out. 3 | 4 | ### Requirements 5 | To use the Vagrant environment, first install Vagrant following these [instructions](https://www.vagrantup.com/docs/installation/). You will also need a virtualization tool, such as [VirtualBox](https://www.virtualbox.org/). 6 | 7 | ## Start the Virtual Machine 8 | Vagrant is used to provision a virtual machine which will run a Nomad server and client for scheduling work. To create the virtual machine, you should run the `vagrant up` command. 9 | ``` 10 | $ vagrant up 11 | ``` 12 | This will take a few minutes as the base Ubuntu box must be downloaded and provisioned with both Docker and Nomad. Once this completes, you should see this output. 13 | ``` 14 | Bringing machine 'default' up with 'virtualbox' provider... 15 | ==> default: Importing base box 'ubuntu/bionic64'... 16 | ... 17 | ==> default: Running provisioner: deps (shell)... 18 | ``` 19 | At this point the Vagrant box is running and ready to go. 20 | 21 | ## Start the Nomad Base Jobs 22 | In order to start the Nomad jobs you should ssh onto the vagrant machine using the `vagrant ssh` command and navigate to the jobs directory. 23 | 24 | ``` 25 | $ vagrant ssh 26 | $ cd /home/vagrant/nomad-autoscaler/jobs/ 27 | ``` 28 | 29 | The demo uses [HAProxy](https://www.haproxy.org/) to provide ingress services. The job also includes a Prometheus exporter, which allows HAProxy metrics to be exported for consumption via Prometheus. This should be started using the `nomad run` command. 30 | ``` 31 | $ nomad run haproxy.nomad 32 | ``` 33 | 34 | You can check the status of the job and the allocation using the `nomad status` command. 35 | ``` 36 | $ nomad status haproxy 37 | $ nomad status 38 | ``` 39 | 40 | Prometheus can now be started, again using the `nomad run` command. The virtual machine is set to forward a number of ports to your localhost, including the Prometheus UI. This can be seen in a web browser at http://127.0.0.1:9090. 41 | ``` 42 | $ nomad run prometheus.nomad 43 | ``` 44 | 45 | To visualize the state of the system and better understand what actions are being taken by the Autoscaler, you can run the Loki and Grafana services which comes with a sample dashboard that can be accessed from your browser at [http://localhost:3000/d/8QlvShyZz/nomad-autoscaler-demo](http://localhost:3000/d/8QlvShyZz/nomad-autoscaler-demo?orgId=1&refresh=5s). 46 | 47 | ``` 48 | $ nomad run loki.nomad 49 | $ nomad run grafana.nomad 50 | ``` 51 | 52 | We will explore the details of each panel shortly. 53 | 54 | ## Start the Autoscaler Component Jobs 55 | First start the Nomad job which will be autoscaled. The `webapp.nomad` jobfile contains a scaling stanza which defines the key autoscaling parameters for a task group. There are a number of interesting key options to understand. 56 | - `enabled = false` is a parameter to allow operators to administratively disable scaling for a task group. 57 | - `source = "prometheus"` specifies that the Autoscaler will use the Prometheus APM plugin to retrieve metrics. 58 | - `query = "avg((haproxy.."` is the query that will be run against the APM and is expected to return a single value. 59 | - `strategy = { name = "target-value" }` defines the calculation strategy the Autoscaler will use, in this case we a targeting a value. 60 | 61 | Register the application to Nomad, so Nomad can start 3 allocations of the example web application. 62 | ``` 63 | $ nomad run webapp.nomad 64 | ``` 65 | 66 | The Autoscaler Nomad job can now be submitted. The Nomad Autoscaler does not require persistent state, but will store key data in-memory to reduce load on the Nomad API. 67 | ``` 68 | $ nomad run autoscaler.nomad 69 | ``` 70 | Check the logs of the Autoscaler to see that it has started up correctly. 71 | ``` 72 | $ nomad logs -stderr 73 | ``` 74 | 75 | ## Enable the Scaling Policy and Scale Down 76 | The submitted webapp job has a scaling stanza, but has scaling disabled. In order to enable the task group for scaling, you should edit the file and change the `enabled = false` line to read `enabled = true` within the scaling stanza. Once updated, preview what changes are to be made using the `nomad plan` command. 77 | ``` 78 | $ nomad plan webapp.nomad 79 | ``` 80 | Submit the updated version of the job, taking the index generated from the plan command. 81 | ``` 82 | $ nomad run -check-index webapp.nomad 83 | ``` 84 | The Autoscaler will now actively evaluate the example group of the webapp job, and will determine that the current count 3, is more than is needed to meet the required scaling target. 85 | ``` 86 | 2020-03-25T17:17:02.725Z [INFO] agent: reading policies: policy_storage=policystorage.Nomad 87 | 2020-03-25T17:17:02.726Z [INFO] agent: found 1 policies: policy_storage=policystorage.Nomad 88 | 2020-03-25T17:17:02.727Z [INFO] agent: fetching current count: policy_id=248f6157-ca37-f868-a0ab-cabbc67fec1d source=prometheus strategy=target-value target=local-nomad 89 | 2020-03-25T17:17:02.729Z [INFO] agent: querying APM: policy_id=248f6157-ca37-f868-a0ab-cabbc67fec1d source=prometheus strategy=target-value target=local-nomad 90 | 2020-03-25T17:17:02.731Z [INFO] agent: calculating new count: policy_id=248f6157-ca37-f868-a0ab-cabbc67fec1d source=prometheus strategy=target-value target=local-nomad 91 | 2020-03-25T17:17:02.731Z [INFO] agent: next count outside limits: policy_id=248f6157-ca37-f868-a0ab-cabbc67fec1d source=prometheus strategy=target-value target=local-nomad from=3 to=0 min=1 max=10 92 | 2020-03-25T17:17:02.731Z [INFO] agent: updated count to be within limits: policy_id=248f6157-ca37-f868-a0ab-cabbc67fec1d source=prometheus strategy=target-value target=local-nomad from=3 to=1 min=1 max=10 93 | 2020-03-25T17:17:02.731Z [INFO] agent: scaling target: policy_id=248f6157-ca37-f868-a0ab-cabbc67fec1d source=prometheus strategy=target-value target=local-nomad target_config="map[group:demo job_id:webapp]" from=3 to=1 reason="capping count to min value of 1" 94 | ``` 95 | The Autoscaler will never scale a job group past either the `min` or `max` parameters. This ensures applications maintain high availability even during minimal load, while also not over scaling due to problems such as misconfiguration or faulty metrics values. 96 | 97 | ## Generate Load and Scale Up 98 | In order to generate load, a tool called [hey](https://github.com/rakyll/hey) is installed and available on the virtual machine. It is recommended to create a second ssh connection to the virtual machine. Inside this terminal we can then generate load on the webapp service. 99 | ``` 100 | $ hey -z 1m -c 30 http://127.0.0.1:8000 101 | ``` 102 | 103 | The increase in load will be reflected through Prometheus metrics to the Autoscaler. Checking the Autoscaler logs, you should see messages indicating it has chosen to scale out the job due to the increase in load. 104 | ``` 105 | 2020-03-25T17:23:12.725Z [INFO] agent: reading policies: policy_storage=policystorage.Nomad 106 | 2020-03-25T17:23:12.727Z [INFO] agent: found 1 policies: policy_storage=policystorage.Nomad 107 | 2020-03-25T17:23:12.728Z [INFO] agent: fetching current count: policy_id=248f6157-ca37-f868-a0ab-cabbc67fec1d source=prometheus strategy=target-value target=local-nomad 108 | 2020-03-25T17:23:12.730Z [INFO] agent: querying APM: policy_id=248f6157-ca37-f868-a0ab-cabbc67fec1d source=prometheus strategy=target-value target=local-nomad 109 | 2020-03-25T17:23:12.735Z [INFO] agent: calculating new count: policy_id=248f6157-ca37-f868-a0ab-cabbc67fec1d source=prometheus strategy=target-value target=local-nomad 110 | 2020-03-25T17:23:12.735Z [INFO] agent: scaling target: policy_id=248f6157-ca37-f868-a0ab-cabbc67fec1d source=prometheus strategy=target-value target=local-nomad target_config="map[group:demo job_id:webapp]" from=1 to=2 reason="scaling up because factor is 1.500000" 111 | ``` 112 | 113 | ## Understanding the Demo Dashboard 114 | From the [demo dashboard](http://localhost:3000/d/8QlvShyZz/nomad-autoscaler-demo?orgId=1&refresh=5s) you will be able to see the actions taken by the Autoscaler: 115 | 116 | ![dashboard](images/demo_dashboard.png) 117 | 118 | The green shaded area in the middle of the dashboard is the `count` of the `webapp` task group that the Autoscaler will act on. The blue line is the total number of connections hitting our services and the purple line is the average number of connections per instance of `webapp`, and it's the metric we are monitoring. 119 | 120 | You can see that the `count` starts at 3, but drops to 1 after the Autoscaler is started because the average number of connections is at 0. Once we run the `hey` command and start to generate load into our system, the number of connections sharply increases, along with the average. 121 | 122 | The Autoscaler notices this spike and reacts a few seconds later by increasing the `count`. This will add more instances of our web app to handle the load and drop the average number of connections to our target value. Once the load is removed, the Autoscaler returns `count` to 1. The log entries for the scaling actions performed by the Autoscaler are automatically added to the graphs as annotations represented by the dashed light blue vertical lines. Hovering over them will display more details about the scaling event. 123 | 124 | The actual values you observe might be different, but the general idea of `count` reacting to the average number of connections should be noticeable. 125 | 126 | ## Understanding the Autoscaler Dashboard 127 | The [autoscaler dashboard](http://localhost:3000/d/23veuVIGk/nomad-autoscaler-telemetry?orgId=1) provides 128 | insights into the runtime performance of the Nomad Autoscaler using its 129 | [metrics API endpoint](https://www.nomadproject.io/docs/autoscaling/api#metrics-api): 130 | 131 | ![dashboard](images/autoscaler_dashboard.png) 132 | 133 | The top row of panels detail the plugin performance by invocation time. The invocation time is the 134 | time taken to perform a single RPC plugin call. 135 | 136 | The middle row provides useful business metrics about the Nomad Autoscaler, in particular the total 137 | time taken to perform a scaling evaluation, as well as the success and error rates when submitting 138 | scaling actions to the target. 139 | 140 | The bottom row provides insight into the Go runtime for the Nomad Autoscaler. It displays the most 141 | commonly tracked stats which indicate runtime performance. 142 | 143 | ## Demo End 144 | Congratulations, you have run through the Nomad Autoscaler Vagrant demo. In order to destroy the created virtual machine, close all SSH connection and then issue a `vagrant destroy -f` command. 145 | ``` 146 | $ vagrant destroy -f 147 | ``` 148 | --------------------------------------------------------------------------------