├── ansible └── playbooks │ ├── roles │ ├── vault │ │ ├── files │ │ │ ├── vault-nomad-workload-policy.hcl │ │ │ └── consul-vault-policy.hcl │ │ ├── vars │ │ │ └── main.yml │ │ ├── handlers │ │ │ └── main.yml │ │ ├── tasks │ │ │ ├── configure.yml │ │ │ ├── main.yml │ │ │ ├── consul_integration.yml │ │ │ └── acl_bootstrap.yml │ │ └── templates │ │ │ └── vault.hcl.j2 │ ├── traefik │ │ ├── vars │ │ │ └── main.yml │ │ ├── templates │ │ │ ├── traefik.env.j2 │ │ │ ├── traefik.yml.j2 │ │ │ └── traefik.service.j2 │ │ ├── handlers │ │ │ └── main.yml │ │ ├── tasks │ │ │ ├── main.yml │ │ │ └── configure.yml │ │ └── files │ │ │ └── dynamic_config.yml │ ├── nomad │ │ ├── vars │ │ │ └── main.yml │ │ ├── files │ │ │ ├── nomad-client-policy.hcl │ │ │ ├── nomad-deployer-policy.hcl │ │ │ ├── nomad-operator-policy.hcl │ │ │ └── consul-nomad-policy.hcl │ │ ├── handlers │ │ │ └── main.yml │ │ ├── tasks │ │ │ ├── configure.yml │ │ │ ├── main.yml │ │ │ ├── consul_integration.yml │ │ │ ├── acl_bootstrap.yml │ │ │ └── vault_integration.yml │ │ └── templates │ │ │ ├── vault-nomad-policy.hcl.j2 │ │ │ └── nomad.hcl.j2 │ └── consul │ │ ├── vars │ │ └── main.yml │ │ ├── files │ │ ├── consul-dns-policy.hcl │ │ └── consul-operator-policy.hcl │ │ ├── templates │ │ ├── consul-systemd-resolved.conf.j2 │ │ └── consul.hcl.j2 │ │ ├── handlers │ │ └── main.yml │ │ └── tasks │ │ ├── configure.yml │ │ ├── main.yml │ │ ├── dns.yml │ │ └── acl_bootstrap.yml │ ├── 03-vault.yml │ ├── 04-nomad.yml │ ├── 02-consul.yml │ ├── 05-traefik.yml │ └── 01-common.yml ├── .gitignore ├── terraform ├── inventory.tpl ├── variables.hcl.example ├── variables.tf └── main.tf ├── nomad ├── main.tf ├── csi-driver │ ├── main.tf │ ├── hcloud-csi-node.hcl │ └── hcloud-csi-controller.hcl ├── redis │ ├── main.tf │ └── redis.hcl └── postgres │ ├── main.tf │ └── postgres.hcl ├── LICENSE └── README.md /ansible/playbooks/roles/vault/files/vault-nomad-workload-policy.hcl: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /ansible/playbooks/roles/traefik/vars/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | traefik_config_dir: "/etc/traefik" 3 | -------------------------------------------------------------------------------- /ansible/playbooks/roles/traefik/templates/traefik.env.j2: -------------------------------------------------------------------------------- 1 | CF_DNS_API_TOKEN={{ cloudflare_dns_api_token }} 2 | -------------------------------------------------------------------------------- /ansible/playbooks/roles/nomad/vars/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | nomad_config_dir: "/etc/nomad.d" 3 | nomad_data_dir: "/opt/nomad" 4 | -------------------------------------------------------------------------------- /ansible/playbooks/roles/vault/vars/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | vault_config_dir: "/etc/vault.d" 3 | vault_data_dir: "/opt/vault" 4 | -------------------------------------------------------------------------------- /ansible/playbooks/03-vault.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Vault 3 | hosts: all 4 | become: true 5 | 6 | roles: 7 | - vault 8 | -------------------------------------------------------------------------------- /ansible/playbooks/04-nomad.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Nomad 3 | hosts: all 4 | become: true 5 | 6 | roles: 7 | - nomad 8 | -------------------------------------------------------------------------------- /ansible/playbooks/roles/consul/vars/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | consul_config_dir: "/etc/consul.d" 3 | consul_data_dir: "/opt/consul" 4 | -------------------------------------------------------------------------------- /ansible/playbooks/02-consul.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Consul 3 | hosts: all 4 | become: true 5 | 6 | roles: 7 | - consul 8 | -------------------------------------------------------------------------------- /ansible/playbooks/05-traefik.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Traefik 3 | hosts: all 4 | become: true 5 | 6 | roles: 7 | - traefik 8 | -------------------------------------------------------------------------------- /ansible/playbooks/roles/consul/files/consul-dns-policy.hcl: -------------------------------------------------------------------------------- 1 | node_prefix "" { 2 | policy = "read" 3 | } 4 | 5 | service_prefix "" { 6 | policy = "read" 7 | } 8 | -------------------------------------------------------------------------------- /ansible/playbooks/roles/vault/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: restart vault 3 | ansible.builtin.systemd: 4 | name: vault 5 | state: restarted 6 | -------------------------------------------------------------------------------- /ansible/playbooks/roles/traefik/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: restart traefik 3 | ansible.builtin.systemd: 4 | name: traefik 5 | state: restarted 6 | -------------------------------------------------------------------------------- /ansible/playbooks/roles/consul/templates/consul-systemd-resolved.conf.j2: -------------------------------------------------------------------------------- 1 | [Resolve] 2 | DNS=127.0.0.1:8600 3 | DNSSEC=false 4 | Domains=~consul 5 | DNSStubListenerExtra={{ docker_bridge_ip_address }} 6 | -------------------------------------------------------------------------------- /ansible/playbooks/roles/traefik/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Configure Traefik service 3 | ansible.builtin.import_tasks: configure.yml 4 | when: 5 | - ansible_hostname in groups['servers'] 6 | -------------------------------------------------------------------------------- /ansible/playbooks/roles/nomad/files/nomad-client-policy.hcl: -------------------------------------------------------------------------------- 1 | namespace "*" { 2 | policy = "write" 3 | } 4 | 5 | node { 6 | policy = "read" 7 | } 8 | 9 | agent { 10 | policy = "read" 11 | } 12 | -------------------------------------------------------------------------------- /ansible/playbooks/roles/nomad/files/nomad-deployer-policy.hcl: -------------------------------------------------------------------------------- 1 | namespace "default" { 2 | policy = "deny" 3 | 4 | variables { 5 | path "nomad/jobs/*" { 6 | capabilities = ["write"] 7 | } 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Terraform 2 | .terraform 3 | *.tfstate 4 | *.tfstate.backup 5 | .terraform.lock.hcl 6 | .terraform.tfstate.lock.info 7 | terraform.tfvars 8 | variables.hcl 9 | *.pem 10 | 11 | # Ansible 12 | inventory 13 | 14 | # Secrets 15 | *.token 16 | .env 17 | -------------------------------------------------------------------------------- /terraform/inventory.tpl: -------------------------------------------------------------------------------- 1 | [all:vars] 2 | ansible_user=root 3 | 4 | [servers] 5 | %{ for name, ip in servers ~} 6 | ${name} ansible_host=${ip} 7 | %{ endfor ~} 8 | 9 | [clients] 10 | %{ for name, ip in clients ~} 11 | ${name} ansible_host=${ip} 12 | %{ endfor ~} 13 | -------------------------------------------------------------------------------- /ansible/playbooks/roles/nomad/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: restart nomad 3 | ansible.builtin.systemd: 4 | name: nomad 5 | state: restarted 6 | 7 | - name: restart vault 8 | ansible.builtin.systemd: 9 | name: vault 10 | state: restarted 11 | -------------------------------------------------------------------------------- /ansible/playbooks/roles/vault/files/consul-vault-policy.hcl: -------------------------------------------------------------------------------- 1 | service "vault" { 2 | policy = "write" 3 | } 4 | 5 | key_prefix "vault/" { 6 | policy = "write" 7 | } 8 | 9 | agent_prefix "" { 10 | policy = "read" 11 | } 12 | 13 | session_prefix "" { 14 | policy = "write" 15 | } 16 | -------------------------------------------------------------------------------- /ansible/playbooks/roles/consul/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: restart consul 3 | ansible.builtin.systemd: 4 | name: consul 5 | state: restarted 6 | 7 | - name: restart systemd-resolved 8 | ansible.builtin.systemd: 9 | name: systemd-resolved 10 | state: restarted 11 | -------------------------------------------------------------------------------- /ansible/playbooks/roles/nomad/files/nomad-operator-policy.hcl: -------------------------------------------------------------------------------- 1 | namespace "*" { 2 | policy = "write" 3 | } 4 | 5 | node { 6 | policy = "write" 7 | } 8 | 9 | agent { 10 | policy = "write" 11 | } 12 | 13 | operator { 14 | policy = "write" 15 | } 16 | 17 | quota { 18 | policy = "write" 19 | } 20 | 21 | host_volume "*" { 22 | policy = "write" 23 | } 24 | -------------------------------------------------------------------------------- /terraform/variables.hcl.example: -------------------------------------------------------------------------------- 1 | # Terraform variables (all are required) 2 | location = "ash" 3 | network_zone = "us-east" 4 | 5 | # These variables are optional and have defaults 6 | # name = "nomad" 7 | # image = "ubuntu-22.04" 8 | # server_instance_type = "cpx11" 9 | # client_instance_type = "cpx11" 10 | # server_count = 3 11 | # client_count = 3 12 | -------------------------------------------------------------------------------- /ansible/playbooks/roles/nomad/files/consul-nomad-policy.hcl: -------------------------------------------------------------------------------- 1 | acl = "write" 2 | 3 | agent_prefix "" { 4 | policy = "write" 5 | } 6 | 7 | event_prefix "" { 8 | policy = "write" 9 | } 10 | 11 | key_prefix "" { 12 | policy = "write" 13 | } 14 | 15 | node_prefix "" { 16 | policy = "write" 17 | } 18 | 19 | query_prefix "" { 20 | policy = "write" 21 | } 22 | 23 | service_prefix "" { 24 | policy = "write" 25 | } 26 | -------------------------------------------------------------------------------- /ansible/playbooks/roles/consul/files/consul-operator-policy.hcl: -------------------------------------------------------------------------------- 1 | acl = "write" 2 | 3 | agent_prefix "" { 4 | policy = "write" 5 | } 6 | 7 | event_prefix "" { 8 | policy = "write" 9 | } 10 | 11 | key_prefix "" { 12 | policy = "write" 13 | } 14 | 15 | node_prefix "" { 16 | policy = "write" 17 | } 18 | 19 | query_prefix "" { 20 | policy = "write" 21 | } 22 | 23 | service_prefix "" { 24 | policy = "write" 25 | } 26 | -------------------------------------------------------------------------------- /ansible/playbooks/roles/nomad/tasks/configure.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Create Nomad configuration file 3 | ansible.builtin.template: 4 | src: nomad.hcl.j2 5 | dest: "{{ nomad_config_dir }}/nomad.hcl" 6 | notify: restart nomad 7 | 8 | - name: Start Nomad daemon 9 | ansible.builtin.systemd: 10 | name: nomad 11 | state: started 12 | enabled: true 13 | 14 | - name: Wait for Nomad to start 15 | ansible.builtin.wait_for: 16 | port: 4646 17 | timeout: 30 18 | -------------------------------------------------------------------------------- /ansible/playbooks/roles/vault/tasks/configure.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Create Vault configuration file 3 | ansible.builtin.template: 4 | src: vault.hcl.j2 5 | dest: "{{ vault_config_dir }}/vault.hcl" 6 | notify: restart vault 7 | 8 | - name: Start Vault daemon 9 | ansible.builtin.systemd: 10 | name: vault 11 | state: started 12 | enabled: true 13 | 14 | - name: Wait for Vault to start 15 | ansible.builtin.wait_for: 16 | port: 8200 17 | timeout: 30 18 | -------------------------------------------------------------------------------- /ansible/playbooks/roles/consul/tasks/configure.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Create Consul configuration file 3 | ansible.builtin.template: 4 | src: consul.hcl.j2 5 | dest: "{{ consul_config_dir }}/consul.hcl" 6 | notify: restart consul 7 | 8 | - name: Start Consul daemon 9 | ansible.builtin.systemd: 10 | name: consul 11 | state: started 12 | enabled: true 13 | 14 | - name: Wait for Consul to start 15 | ansible.builtin.wait_for: 16 | port: 8500 17 | timeout: 30 18 | -------------------------------------------------------------------------------- /ansible/playbooks/roles/vault/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Configure Consul integration 3 | ansible.builtin.import_tasks: consul_integration.yml 4 | when: 5 | - ansible_hostname in groups['servers'] 6 | delegate_to: "{{ groups['servers'][0] }}" 7 | run_once: true 8 | 9 | - name: Configure Vault service 10 | ansible.builtin.import_tasks: configure.yml 11 | when: 12 | - ansible_hostname in groups['servers'] 13 | 14 | - name: Bootstrap Vault ACL 15 | ansible.builtin.import_tasks: acl_bootstrap.yml 16 | when: 17 | - ansible_hostname in groups['servers'] 18 | delegate_to: "{{ groups['servers'][0] }}" 19 | run_once: true 20 | -------------------------------------------------------------------------------- /nomad/main.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | nomad = { 4 | source = "hashicorp/nomad" 5 | version = "~> 2.4.0" 6 | } 7 | 8 | vault = { 9 | source = "hashicorp/vault" 10 | version = "~> 4.6.0" 11 | } 12 | } 13 | } 14 | 15 | variable "hcloud_token" { 16 | type = string 17 | } 18 | 19 | module "csi_driver" { 20 | source = "./csi-driver" 21 | hcloud_token = var.hcloud_token 22 | } 23 | 24 | module "postgres" { 25 | depends_on = [module.csi_driver] 26 | source = "./postgres" 27 | } 28 | 29 | module "redis" { 30 | depends_on = [module.csi_driver] 31 | source = "./redis" 32 | } 33 | -------------------------------------------------------------------------------- /ansible/playbooks/roles/vault/templates/vault.hcl.j2: -------------------------------------------------------------------------------- 1 | ui = true 2 | api_addr = "{% raw %}http://{{ GetPrivateInterfaces | exclude \"name\" \"docker0\" | attr \"address\" }}:8200{% endraw %}" 3 | cluster_addr = "{% raw %}http://{{ GetPrivateInterfaces | exclude \"name\" \"docker0\" | attr \"address\" }}:8201{% endraw %}" 4 | disable_mlock = true 5 | 6 | storage "raft" { 7 | path = "/opt/vault/data" 8 | node_id = "{{ inventory_hostname }}" 9 | } 10 | 11 | listener "tcp" { 12 | address = "0.0.0.0:8200" 13 | tls_disable = true # TODO: TLS support 14 | } 15 | 16 | service_registration "consul" { 17 | address = "http://localhost:8500" 18 | token = "{{ consul_vault_token }}" 19 | } 20 | -------------------------------------------------------------------------------- /ansible/playbooks/roles/consul/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Configure Consul service 3 | ansible.builtin.import_tasks: configure.yml 4 | 5 | - name: Bootstrap Consul ACL 6 | ansible.builtin.import_tasks: acl_bootstrap.yml 7 | when: 8 | - ansible_hostname in groups['servers'] 9 | delegate_to: "{{ groups['servers'][0] }}" 10 | run_once: true 11 | 12 | - name: Update agent token in configuration 13 | ansible.builtin.replace: 14 | path: "{{ consul_config_dir }}/consul.hcl" 15 | regexp: '# default = ""' 16 | replace: 'default = "{{ consul_operator_token }}"' 17 | notify: restart consul 18 | 19 | - name: Configure DNS forwarding 20 | ansible.builtin.import_tasks: dns.yml 21 | -------------------------------------------------------------------------------- /nomad/csi-driver/main.tf: -------------------------------------------------------------------------------- 1 | variable "hcloud_token" { 2 | type = string 3 | } 4 | 5 | resource "vault_kv_secret_v2" "controller_hcloud_token" { 6 | mount = "secret" 7 | name = "default/hcloud-csi-controller" 8 | 9 | data_json = jsonencode({ 10 | HCLOUD_TOKEN = var.hcloud_token 11 | }) 12 | } 13 | 14 | resource "vault_kv_secret_v2" "node_hcloud_token" { 15 | mount = "secret" 16 | name = "default/hcloud-csi-node" 17 | 18 | data_json = jsonencode({ 19 | HCLOUD_TOKEN = var.hcloud_token 20 | }) 21 | } 22 | 23 | resource "nomad_job" "csi_controller" { 24 | jobspec = file("${path.module}/hcloud-csi-controller.hcl") 25 | } 26 | 27 | resource "nomad_job" "csi_node" { 28 | jobspec = file("${path.module}/hcloud-csi-node.hcl") 29 | } 30 | -------------------------------------------------------------------------------- /nomad/redis/main.tf: -------------------------------------------------------------------------------- 1 | data "nomad_plugin" "csi" { 2 | plugin_id = "csi.hetzner.cloud" 3 | wait_for_healthy = true 4 | } 5 | 6 | resource "nomad_csi_volume" "redis_volume" { 7 | depends_on = [data.nomad_plugin.csi] 8 | 9 | plugin_id = data.nomad_plugin.csi.plugin_id 10 | volume_id = "redis-vol" 11 | name = "redis-vol" 12 | namespace = "default" 13 | capacity_min = "10G" 14 | 15 | capability { 16 | access_mode = "single-node-writer" 17 | attachment_mode = "file-system" 18 | } 19 | 20 | mount_options { 21 | fs_type = "ext4" 22 | mount_flags = ["discard", "defaults"] 23 | } 24 | } 25 | 26 | resource "nomad_job" "redis" { 27 | depends_on = [nomad_csi_volume.redis_volume] 28 | 29 | jobspec = file("${path.module}/redis.hcl") 30 | } 31 | -------------------------------------------------------------------------------- /ansible/playbooks/roles/nomad/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Configure Consul integration 3 | ansible.builtin.import_tasks: consul_integration.yml 4 | when: 5 | - ansible_hostname in groups['servers'] 6 | delegate_to: "{{ groups['servers'][0] }}" 7 | run_once: true 8 | 9 | - name: Configure Nomad service 10 | ansible.builtin.import_tasks: configure.yml 11 | 12 | - name: Bootstrap Nomad ACL 13 | ansible.builtin.import_tasks: acl_bootstrap.yml 14 | when: 15 | - ansible_hostname in groups['servers'] 16 | delegate_to: "{{ groups['servers'][0] }}" 17 | run_once: true 18 | 19 | - name: Configure Vault integration 20 | ansible.builtin.import_tasks: vault_integration.yml 21 | when: 22 | - ansible_hostname in groups['servers'] 23 | delegate_to: "{{ groups['servers'][0] }}" 24 | run_once: true 25 | -------------------------------------------------------------------------------- /nomad/postgres/main.tf: -------------------------------------------------------------------------------- 1 | data "nomad_plugin" "csi" { 2 | plugin_id = "csi.hetzner.cloud" 3 | wait_for_healthy = true 4 | } 5 | 6 | resource "nomad_csi_volume" "postgres_volume" { 7 | depends_on = [data.nomad_plugin.csi] 8 | 9 | plugin_id = data.nomad_plugin.csi.plugin_id 10 | volume_id = "postgres-vol" 11 | name = "postgres-vol" 12 | namespace = "default" 13 | capacity_min = "10G" 14 | 15 | capability { 16 | access_mode = "single-node-writer" 17 | attachment_mode = "file-system" 18 | } 19 | 20 | mount_options { 21 | fs_type = "ext4" 22 | mount_flags = ["discard", "defaults"] 23 | } 24 | } 25 | 26 | resource "nomad_job" "postgres" { 27 | depends_on = [nomad_csi_volume.postgres_volume] 28 | 29 | jobspec = file("${path.module}/postgres.hcl") 30 | } 31 | -------------------------------------------------------------------------------- /ansible/playbooks/roles/consul/templates/consul.hcl.j2: -------------------------------------------------------------------------------- 1 | data_dir = "{{ consul_data_dir }}" 2 | retry_join = [{% for host in groups['servers'] %}{{ hostvars[host].ansible_host | to_json }}{% if not loop.last %}, {% endif %}{% endfor %}] 3 | advertise_addr = "{% raw %}{{ GetPrivateInterfaces | exclude \"name\" \"docker0\" | attr \"address\" }}{% endraw %}" 4 | 5 | {% if ansible_hostname in groups['servers'] %} 6 | ui = true 7 | server = true 8 | bootstrap_expect = {{ groups['servers'] | length }} 9 | client_addr = "0.0.0.0" 10 | {% endif %} 11 | 12 | acl { 13 | enabled = true 14 | default_policy = "deny" 15 | down_policy = "extend-cache" 16 | {% if ansible_hostname in groups['clients'] %} 17 | tokens { 18 | # default = "" 19 | } 20 | {% endif %} 21 | } 22 | 23 | service { 24 | name = "consul" 25 | } 26 | 27 | connect { 28 | enabled = true 29 | } 30 | 31 | ports { 32 | grpc = 8502 33 | } 34 | -------------------------------------------------------------------------------- /ansible/playbooks/roles/traefik/templates/traefik.yml.j2: -------------------------------------------------------------------------------- 1 | entryPoints: 2 | http: 3 | address: ":80" 4 | http: 5 | redirections: 6 | entryPoint: 7 | to: https 8 | scheme: https 9 | https: 10 | address: ":443" 11 | http: 12 | tls: 13 | certResolver: default 14 | postgres: 15 | address: ":5432" 16 | redis: 17 | address: ":6379" 18 | 19 | api: 20 | dashboard: true 21 | insecure: false 22 | 23 | certificatesResolvers: 24 | default: 25 | acme: 26 | email: "david@dmo.ooo" 27 | storage: "{{ traefik_config_dir }}/acme.json" 28 | dnsChallenge: 29 | provider: cloudflare 30 | 31 | providers: 32 | consulCatalog: 33 | prefix: "traefik" 34 | exposedByDefault: false 35 | endpoint: 36 | address: "consul.service.consul:8500" 37 | scheme: "http" 38 | 39 | file: 40 | filename: "{{ traefik_config_dir }}/dynamic_config.yml" 41 | watch: true 42 | -------------------------------------------------------------------------------- /ansible/playbooks/roles/nomad/templates/vault-nomad-policy.hcl.j2: -------------------------------------------------------------------------------- 1 | path "secret/data/{{ '{{' }}identity.entity.aliases.{{ nomad_vault_method_accessor }}.metadata.nomad_namespace{{ '}}' }}/{{ '{{' }}identity.entity.aliases.{{ nomad_vault_method_accessor }}.metadata.nomad_job_id{{ '}}' }}/*" { 2 | capabilities = ["read"] 3 | } 4 | 5 | path "secret/data/{{ '{{' }}identity.entity.aliases.{{ nomad_vault_method_accessor }}.metadata.nomad_namespace{{ '}}' }}/{{ '{{' }}identity.entity.aliases.{{ nomad_vault_method_accessor }}.metadata.nomad_job_id{{ '}}' }}" { 6 | capabilities = ["read"] 7 | } 8 | 9 | path "secret/data/{{ '{{' }}identity.entity.aliases.{{ nomad_vault_method_accessor }}.metadata.nomad_namespace{{ '}}' }}" { 10 | capabilities = ["read"] 11 | } 12 | 13 | path "secret/metadata/{{ '{{' }}identity.entity.aliases.{{ nomad_vault_method_accessor }}.metadata.nomad_namespace{{ '}}' }}/*" { 14 | capabilities = ["list"] 15 | } 16 | 17 | path "secret/metadata/*" { 18 | capabilities = ["list"] 19 | } 20 | -------------------------------------------------------------------------------- /ansible/playbooks/roles/consul/tasks/dns.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Configure systemd-resolved 3 | block: 4 | - name: Create systemd-resolved configuration directory 5 | ansible.builtin.file: 6 | path: "/etc/systemd/resolved.conf.d" 7 | state: directory 8 | mode: 0755 9 | 10 | - name: Get Docker bridge IP 11 | vars: 12 | format: "{% raw %}{{(index .IPAM.Config 0).Gateway}}{% endraw %}" 13 | ansible.builtin.command: docker network inspect bridge --format "{{ format }}" 14 | register: docker_bridge_ip_address_raw 15 | changed_when: false 16 | 17 | - name: Set Docker bridge IP fact 18 | ansible.builtin.set_fact: 19 | docker_bridge_ip_address: "{{ docker_bridge_ip_address_raw.stdout }}" 20 | 21 | - name: Create systemd-resolved configuration file 22 | ansible.builtin.template: 23 | src: consul-systemd-resolved.conf.j2 24 | dest: "/etc/systemd/resolved.conf.d/consul.conf" 25 | mode: 0644 26 | notify: restart systemd-resolved 27 | -------------------------------------------------------------------------------- /nomad/csi-driver/hcloud-csi-node.hcl: -------------------------------------------------------------------------------- 1 | job "hcloud-csi-node" { 2 | type = "system" 3 | 4 | group "node" { 5 | task "plugin" { 6 | driver = "docker" 7 | 8 | vault {} 9 | 10 | config { 11 | image = "hetznercloud/hcloud-csi-driver:v2.11.0" 12 | command = "bin/hcloud-csi-driver-node" 13 | privileged = true 14 | } 15 | 16 | env { 17 | CSI_ENDPOINT = "unix://csi/csi.sock" 18 | ENABLE_METRICS = true 19 | } 20 | 21 | template { 22 | data = <- 24 | consul acl policy create 25 | -name nomad 26 | -rules @{{ nomad_data_dir }}/consul-nomad-policy.hcl 27 | environment: 28 | CONSUL_HTTP_TOKEN: "{{ consul_operator_token }}" 29 | 30 | - name: Create Nomad's Consul token 31 | ansible.builtin.command: >- 32 | consul acl token create 33 | -format=json 34 | -description 'Nomad token' 35 | -policy-name nomad 36 | register: consul_nomad_token_raw 37 | environment: 38 | CONSUL_HTTP_TOKEN: "{{ consul_operator_token }}" 39 | 40 | - name: Set Nomad's Consul token fact 41 | ansible.builtin.set_fact: 42 | consul_nomad_token: "{{ (consul_nomad_token_raw.stdout | from_json).SecretID }}" 43 | -------------------------------------------------------------------------------- /ansible/playbooks/roles/vault/tasks/consul_integration.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Load Consul credentials 3 | block: 4 | - name: Read Consul operator token 5 | ansible.builtin.shell: "cat consul-operator.token" 6 | register: consul_acl_operator_output 7 | delegate_to: localhost 8 | become: false 9 | 10 | - name: Set Consul operator token fact 11 | ansible.builtin.set_fact: 12 | consul_operator_token: "{{ (consul_acl_operator_output.stdout | from_json).SecretID }}" 13 | 14 | - name: Configure Vault's Consul access 15 | block: 16 | - name: Deploy Vault's Consul policy 17 | ansible.builtin.copy: 18 | src: consul-vault-policy.hcl 19 | dest: "{{ vault_data_dir }}/consul-vault-policy.hcl" 20 | mode: 0644 21 | 22 | - name: Create Vault's Consul policy 23 | ansible.builtin.command: >- 24 | consul acl policy create 25 | -name vault 26 | -rules @{{ vault_data_dir }}/consul-vault-policy.hcl 27 | environment: 28 | CONSUL_HTTP_TOKEN: "{{ consul_operator_token }}" 29 | 30 | - name: Create Vault's Consul token 31 | ansible.builtin.command: >- 32 | consul acl token create 33 | -format=json 34 | -description 'Vault token' 35 | -policy-name vault 36 | register: consul_vault_token_raw 37 | environment: 38 | CONSUL_HTTP_TOKEN: "{{ consul_operator_token }}" 39 | 40 | - name: Set Vault's Consul token fact 41 | ansible.builtin.set_fact: 42 | consul_vault_token: "{{ (consul_vault_token_raw.stdout | from_json).SecretID }}" 43 | -------------------------------------------------------------------------------- /nomad/redis/redis.hcl: -------------------------------------------------------------------------------- 1 | job "redis" { 2 | type = "service" 3 | 4 | group "redis" { 5 | network { 6 | port "redis" { 7 | to = 6379 8 | } 9 | } 10 | 11 | volume "redis-volume" { 12 | type = "csi" 13 | read_only = false 14 | source = "redis-vol" 15 | attachment_mode = "file-system" 16 | access_mode = "single-node-writer" 17 | per_alloc = false 18 | } 19 | 20 | service { 21 | name = "redis" 22 | port = "redis" 23 | 24 | tags = [ 25 | "traefik.enable=true", 26 | "traefik.tcp.routers.redis.tls=true", 27 | "traefik.tcp.routers.redis.tls.certResolver=default", 28 | "traefik.tcp.routers.redis.entrypoints=redis", 29 | "traefik.tcp.routers.redis.rule=HostSNI(`redis.internal.example.com`)", 30 | ] 31 | } 32 | 33 | task "redis" { 34 | driver = "docker" 35 | 36 | vault {} 37 | 38 | template { 39 | data = <- 33 | consul acl policy create 34 | -name consul-operator 35 | -rules @{{ consul_data_dir }}/consul-operator-policy.hcl 36 | environment: 37 | CONSUL_HTTP_TOKEN: "{{ consul_acl_bootstrap }}" 38 | 39 | - name: Create Consul operator token 40 | ansible.builtin.command: >- 41 | consul acl token create 42 | -format=json 43 | -description 'Consul operator token' 44 | -policy-name consul-operator 45 | register: consul_operator_token_raw 46 | environment: 47 | CONSUL_HTTP_TOKEN: "{{ consul_acl_bootstrap }}" 48 | 49 | - name: Store operator token locally 50 | ansible.builtin.copy: 51 | dest: "consul-operator.token" 52 | content: "{{ consul_operator_token_raw.stdout }}" 53 | mode: 0644 54 | delegate_to: localhost 55 | become: false 56 | 57 | - name: Store operator token in facts 58 | ansible.builtin.set_fact: 59 | consul_operator_token: "{{ (consul_operator_token_raw.stdout | from_json).SecretID }}" 60 | 61 | - name: Configure DNS policy 62 | block: 63 | - name: Deploy DNS policy 64 | ansible.builtin.copy: 65 | src: consul-dns-policy.hcl 66 | dest: "{{ consul_data_dir }}/consul-dns-policy.hcl" 67 | mode: 0644 68 | 69 | - name: Create DNS policy 70 | ansible.builtin.command: >- 71 | consul acl policy create 72 | -name dns-lookup 73 | -rules @{{ consul_data_dir }}/consul-dns-policy.hcl 74 | environment: 75 | CONSUL_HTTP_TOKEN: "{{ consul_acl_bootstrap }}" 76 | 77 | - name: Update anonymous token DNS lookup policy 78 | ansible.builtin.command: >- 79 | consul acl token update 80 | -id anonymous 81 | -policy-name dns-lookup 82 | environment: 83 | CONSUL_HTTP_TOKEN: "{{ consul_acl_bootstrap }}" 84 | -------------------------------------------------------------------------------- /ansible/playbooks/roles/nomad/tasks/acl_bootstrap.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Bootstrap Nomad ACL 3 | block: 4 | - name: Attempt ACL bootstrap 5 | ansible.builtin.command: "nomad acl bootstrap -json" 6 | register: nomad_acl_bootstrap_raw 7 | failed_when: 8 | - nomad_acl_bootstrap_raw.rc != 0 9 | - "'ACL bootstrap already done' not in nomad_acl_bootstrap_raw.stderr" 10 | changed_when: nomad_acl_bootstrap_raw.rc == 0 11 | until: nomad_acl_bootstrap_raw is not failed 12 | retries: 3 13 | 14 | - name: Verify ACL bootstrap 15 | ansible.builtin.assert: 16 | that: "'ACL bootstrap already done' not in nomad_acl_bootstrap_raw.stderr" 17 | fail_msg: "{{ nomad_acl_bootstrap_raw.stderr }}" 18 | 19 | - name: Store bootstrap token locally 20 | ansible.builtin.copy: 21 | dest: "nomad-bootstrap.token" 22 | content: "{{ nomad_acl_bootstrap_raw.stdout }}" 23 | mode: 0644 24 | delegate_to: localhost 25 | become: false 26 | 27 | - name: Set bootstrap token fact 28 | ansible.builtin.set_fact: 29 | nomad_acl_bootstrap: "{{ nomad_acl_bootstrap_raw.stdout | from_json }}" 30 | 31 | - name: Configure operator policy 32 | block: 33 | - name: Deploy operator policy 34 | ansible.builtin.copy: 35 | src: nomad-operator-policy.hcl 36 | dest: "{{ nomad_data_dir }}/nomad-operator-policy.hcl" 37 | mode: 0644 38 | 39 | - name: Create operator policy 40 | ansible.builtin.command: >- 41 | nomad acl policy apply 42 | nomad-operator 43 | {{ nomad_data_dir }}/nomad-operator-policy.hcl 44 | environment: 45 | NOMAD_TOKEN: "{{ nomad_acl_bootstrap.SecretID }}" 46 | 47 | - name: Create Nomad operator token 48 | ansible.builtin.command: >- 49 | nomad acl token create 50 | -json 51 | -name nomad-operator 52 | -policy nomad-operator 53 | register: nomad_operator_token_raw 54 | environment: 55 | NOMAD_TOKEN: "{{ nomad_acl_bootstrap.SecretID }}" 56 | 57 | - name: Store Nomad operator token locally 58 | ansible.builtin.copy: 59 | dest: "nomad-operator.token" 60 | content: "{{ nomad_operator_token_raw.stdout }}" 61 | mode: 0644 62 | delegate_to: localhost 63 | become: false 64 | 65 | - name: Configure Nomad deployer policy 66 | block: 67 | - name: Deploy Nomad deployer policy 68 | ansible.builtin.copy: 69 | src: nomad-deployer-policy.hcl 70 | dest: "{{ nomad_data_dir }}/nomad-deployer-policy.hcl" 71 | mode: 0644 72 | 73 | - name: Create Nomad deployer policy 74 | ansible.builtin.command: >- 75 | nomad acl policy apply 76 | nomad-deployer 77 | {{ nomad_data_dir }}/nomad-deployer-policy.hcl 78 | environment: 79 | NOMAD_TOKEN: "{{ nomad_acl_bootstrap.SecretID }}" 80 | 81 | - name: Create Nomad deployer token 82 | ansible.builtin.command: >- 83 | nomad acl token create 84 | -json 85 | -name nomad-deployer 86 | -policy nomad-deployer 87 | register: nomad_deployer_token_raw 88 | environment: 89 | NOMAD_TOKEN: "{{ nomad_acl_bootstrap.SecretID }}" 90 | 91 | - name: Store Nomad deployer token locally 92 | ansible.builtin.copy: 93 | dest: "nomad-deployer.token" 94 | content: "{{ nomad_deployer_token_raw.stdout }}" 95 | mode: 0644 96 | delegate_to: localhost 97 | become: false 98 | -------------------------------------------------------------------------------- /terraform/main.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | hcloud = { 4 | source = "hetznercloud/hcloud" 5 | version = "~> 1.45.0" 6 | } 7 | } 8 | } 9 | 10 | resource "hcloud_firewall" "server_ingress" { 11 | name = "${var.name}-server-ingress" 12 | 13 | # SSH 14 | rule { 15 | direction = "in" 16 | protocol = "tcp" 17 | port = "22" 18 | 19 | source_ips = [ 20 | "0.0.0.0/0", 21 | "::/0", 22 | ] 23 | } 24 | 25 | # HTTP 26 | rule { 27 | direction = "in" 28 | protocol = "tcp" 29 | port = "80" 30 | 31 | source_ips = [ 32 | "0.0.0.0/0", 33 | "::/0", 34 | ] 35 | } 36 | 37 | # HTTPS 38 | rule { 39 | direction = "in" 40 | protocol = "tcp" 41 | port = "443" 42 | 43 | source_ips = [ 44 | "0.0.0.0/0", 45 | "::/0", 46 | ] 47 | } 48 | 49 | # Postgres 50 | rule { 51 | direction = "in" 52 | protocol = "tcp" 53 | port = "5432" 54 | 55 | source_ips = [ 56 | "0.0.0.0/0", 57 | "::/0", 58 | ] 59 | } 60 | 61 | # Redis 62 | rule { 63 | direction = "in" 64 | protocol = "tcp" 65 | port = "6379" 66 | 67 | source_ips = [ 68 | "0.0.0.0/0", 69 | "::/0", 70 | ] 71 | } 72 | } 73 | 74 | resource "hcloud_network" "hashistack_network" { 75 | name = "${var.name}-network" 76 | ip_range = "10.0.0.0/16" 77 | } 78 | 79 | resource "hcloud_network_subnet" "hashistack_subnet" { 80 | network_id = hcloud_network.hashistack_network.id 81 | type = "cloud" 82 | network_zone = var.network_zone 83 | ip_range = "10.0.2.0/24" 84 | } 85 | 86 | resource "tls_private_key" "pk" { 87 | algorithm = "RSA" 88 | rsa_bits = 4096 89 | } 90 | 91 | resource "hcloud_ssh_key" "nomad" { 92 | name = "${var.name}-hcloud-key-pair" 93 | public_key = tls_private_key.pk.public_key_openssh 94 | } 95 | 96 | resource "local_file" "nomad_key" { 97 | content = tls_private_key.pk.private_key_pem 98 | filename = "./nomad-hcloud-key-pair.pem" 99 | file_permission = "0400" 100 | } 101 | 102 | resource "hcloud_server" "server" { 103 | count = var.server_count 104 | name = "${var.name}-server-${count.index}" 105 | image = var.image 106 | location = var.location 107 | server_type = var.server_instance_type 108 | ssh_keys = [hcloud_ssh_key.nomad.id] 109 | depends_on = [hcloud_network_subnet.hashistack_subnet] 110 | firewall_ids = [hcloud_firewall.server_ingress.id] 111 | 112 | network { 113 | network_id = hcloud_network.hashistack_network.id 114 | ip = "10.0.2.${10 + (count.index + 1)}" 115 | } 116 | 117 | public_net { 118 | ipv4_enabled = true 119 | ipv6_enabled = true 120 | } 121 | } 122 | 123 | resource "hcloud_server" "client" { 124 | count = var.client_count 125 | name = "${var.name}-client-${count.index}" 126 | image = var.image 127 | location = var.location 128 | server_type = var.client_instance_type 129 | ssh_keys = [hcloud_ssh_key.nomad.id] 130 | depends_on = [hcloud_network_subnet.hashistack_subnet] 131 | 132 | network { 133 | network_id = hcloud_network.hashistack_network.id 134 | ip = "10.0.2.${10 + var.server_count + (count.index + 1)}" 135 | } 136 | 137 | public_net { 138 | ipv4_enabled = true 139 | ipv6_enabled = true 140 | } 141 | } 142 | 143 | resource "local_file" "ansible_inventory" { 144 | content = templatefile("inventory.tpl", { 145 | servers = tomap({ 146 | for server in hcloud_server.server : 147 | server.name => server.ipv4_address 148 | }) 149 | clients = tomap({ 150 | for client in hcloud_server.client : 151 | client.name => client.ipv4_address 152 | }) 153 | }) 154 | 155 | filename = "../ansible/inventory" 156 | } 157 | -------------------------------------------------------------------------------- /ansible/playbooks/roles/nomad/tasks/vault_integration.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Load Vault credentials 3 | block: 4 | - name: Read Vault's initialization data 5 | ansible.builtin.shell: "cat vault-init.token" 6 | register: vault_init_output 7 | delegate_to: localhost 8 | become: false 9 | 10 | - name: Set Vault's initialization data fact 11 | ansible.builtin.set_fact: 12 | vault_init: "{{ (vault_init_output.stdout | from_json) }}" 13 | 14 | - name: Unseal Vault 15 | ansible.builtin.command: "vault operator unseal {{ item }}" 16 | environment: 17 | VAULT_ADDR: "http://localhost:8200" 18 | loop: "{{ vault_init.unseal_keys_b64 }}" 19 | 20 | - name: Configure Nomad's Vault access 21 | block: 22 | - name: Enable JWT auth 23 | ansible.builtin.uri: 24 | url: "http://localhost:8200/v1/sys/auth/jwt-nomad" 25 | method: POST 26 | body_format: json 27 | body: 28 | type: "jwt" 29 | description: "JWT auth backend for Nomad" 30 | headers: 31 | X-Vault-Token: "{{ vault_init.root_token }}" 32 | status_code: 33 | - 200 34 | - 204 35 | 36 | - name: Create Nomad's Vault auth backend 37 | ansible.builtin.uri: 38 | url: "http://localhost:8200/v1/auth/jwt-nomad/config" 39 | method: POST 40 | body_format: json 41 | body: 42 | jwks_url: "http://nomad.service.consul:4646/.well-known/jwks.json" 43 | jwt_supported_algs: 44 | - "RS256" 45 | - "EdDSA" 46 | default_role: "nomad-workloads" 47 | headers: 48 | X-Vault-Token: "{{ vault_init.root_token }}" 49 | status_code: 50 | - 200 51 | - 204 52 | 53 | - name: Create Nomad's Vault auth role 54 | ansible.builtin.uri: 55 | url: "http://localhost:8200/v1/auth/jwt-nomad/role/nomad-workloads" 56 | method: POST 57 | body_format: json 58 | body: 59 | role_type: "jwt" 60 | bound_audiences: 61 | - "vault.io" 62 | user_claim: "/nomad_job_id" 63 | user_claim_json_pointer: true 64 | claim_mappings: 65 | nomad_namespace: nomad_namespace 66 | nomad_job_id: nomad_job_id 67 | nomad_group: nomad_group 68 | token_type: "service" 69 | token_policies: 70 | - "nomad-workloads" 71 | token_period: "30m" 72 | token_explicit_max_ttl: 0 73 | headers: 74 | X-Vault-Token: "{{ vault_init.root_token }}" 75 | status_code: 76 | - 200 77 | - 204 78 | 79 | - name: Load Nomad's Vault method's accessor ID 80 | block: 81 | - name: Read Nomad's Vault method's configuration 82 | ansible.builtin.uri: 83 | url: "http://localhost:8200/v1/sys/auth/jwt-nomad" 84 | method: GET 85 | headers: 86 | X-Vault-Token: "{{ vault_init.root_token }}" 87 | return_content: true 88 | register: nomad_vault_method_config_raw 89 | 90 | - name: Set Nomad's Vault method's accessor ID fact 91 | ansible.builtin.set_fact: 92 | nomad_vault_method_accessor: "{{ nomad_vault_method_config_raw.json.data.accessor }}" 93 | 94 | - name: Deploy Nomad's Vault policy 95 | ansible.builtin.template: 96 | src: vault-nomad-policy.hcl.j2 97 | dest: "{{ nomad_data_dir }}/vault-nomad-policy.hcl" 98 | mode: 0644 99 | 100 | - name: Create Nomad's Vault policy 101 | ansible.builtin.command: >- 102 | vault policy write 103 | nomad-workloads 104 | {{ nomad_data_dir }}/vault-nomad-policy.hcl 105 | environment: 106 | VAULT_ADDR: "http://localhost:8200" 107 | VAULT_TOKEN: "{{ vault_init.root_token }}" 108 | 109 | notify: restart vault 110 | --------------------------------------------------------------------------------