├── .gitignore ├── img ├── consul.png ├── cadvisor-target.png ├── prometheus-target.png ├── node-exporter-target.png └── removed-prometheus-target.png ├── nomad ├── server.hcl ├── client.hcl ├── consul.hcl ├── node-exporter.hcl ├── cadvisor.hcl └── prometheus.hcl ├── prometheus └── prometheus.yml ├── Vagrantfile ├── initialize.sh └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | .vagrant 2 | -------------------------------------------------------------------------------- /img/consul.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/visibilityspots/nomad-consul-prometheus/HEAD/img/consul.png -------------------------------------------------------------------------------- /img/cadvisor-target.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/visibilityspots/nomad-consul-prometheus/HEAD/img/cadvisor-target.png -------------------------------------------------------------------------------- /img/prometheus-target.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/visibilityspots/nomad-consul-prometheus/HEAD/img/prometheus-target.png -------------------------------------------------------------------------------- /img/node-exporter-target.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/visibilityspots/nomad-consul-prometheus/HEAD/img/node-exporter-target.png -------------------------------------------------------------------------------- /img/removed-prometheus-target.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/visibilityspots/nomad-consul-prometheus/HEAD/img/removed-prometheus-target.png -------------------------------------------------------------------------------- /nomad/server.hcl: -------------------------------------------------------------------------------- 1 | name = "nomad-server" 2 | data_dir = "/tmp/nomad-server" 3 | 4 | server { 5 | enabled = true 6 | bootstrap_expect = 1 7 | } 8 | -------------------------------------------------------------------------------- /prometheus/prometheus.yml: -------------------------------------------------------------------------------- 1 | scrape_configs: 2 | - job_name: 'self' 3 | consul_sd_configs: 4 | - server: '172.17.0.1:8500' 5 | services: [] 6 | relabel_configs: 7 | - source_labels: [__meta_consul_tags] 8 | regex: .*,metrics,.* 9 | action: keep 10 | - source_labels: [__meta_consul_service] 11 | target_label: job 12 | 13 | -------------------------------------------------------------------------------- /nomad/client.hcl: -------------------------------------------------------------------------------- 1 | name = "nomad-client" 2 | data_dir = "/tmp/nomad-client" 3 | 4 | client { 5 | enabled = true 6 | 7 | # For demo assume we are talking to server1. For production, 8 | # this should be like "nomad.service.consul:4647" and a system 9 | # like Consul used for service discovery. 10 | servers = ["127.0.0.1:4647"] 11 | alloc_dir = "/tmp/nomad-allocations" 12 | } 13 | 14 | plugin "docker" { 15 | config { 16 | # allow_caps = [ "CHOWN,DAC_OVERRIDE,FSETID,FOWNER,MKNOD,NET_RAW,SETGID,SETUID,SETFCAP,SETPCAP,NET_BIND_SERVICE,SYS_CHROOT,KILL,AUDIT_WRITE,NET_ADMIN" ] 17 | allow_caps = [ "ALL" ] 18 | } 19 | } 20 | 21 | ports { 22 | http = 5656 23 | } 24 | -------------------------------------------------------------------------------- /Vagrantfile: -------------------------------------------------------------------------------- 1 | # -*- mode: ruby -*- 2 | # vi: set ft=ruby : 3 | VAGRANTFILE_API_VERSION = "2" 4 | Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| 5 | 6 | if Vagrant.has_plugin?("vagrant-cachier") 7 | config.cache.scope = :box 8 | end 9 | 10 | config.vm.box = "centos/7" 11 | 12 | config.vm.provider :virtualbox do |virtualbox, override| 13 | virtualbox.customize ["modifyvm", :id, "--memory", 2048] 14 | end 15 | 16 | config.vm.provider :lxc do |lxc, override| 17 | override.vm.box = "visibilityspots/centos-7.x-minimal" 18 | end 19 | 20 | config.vm.hostname = "nomad" 21 | config.vm.synced_folder "nomad", "/opt/nomad", type: "rsync", rsync__chown: false 22 | config.vm.synced_folder "prometheus", "/opt/prometheus", type: "rsync", rsync__chown: false 23 | config.vm.provision "shell", path: "initialize.sh" 24 | config.vm.network "forwarded_port", guest: 9090, host: 9090 25 | config.vm.network "forwarded_port", guest: 8500, host: 8500 26 | end 27 | -------------------------------------------------------------------------------- /nomad/consul.hcl: -------------------------------------------------------------------------------- 1 | job "consul" { 2 | region = "global" 3 | datacenters = ["dc1"] 4 | type = "service" 5 | 6 | group "app" { 7 | count = 1 8 | 9 | restart { 10 | attempts = 3 11 | delay = "20s" 12 | mode = "delay" 13 | } 14 | 15 | task "consul" { 16 | driver = "docker" 17 | env = { 18 | CONSUL_BIND_INTERFACE="eth0" 19 | } 20 | config { 21 | image = "consul:1.4.4" 22 | force_pull = true 23 | network_mode = "host" 24 | logging { 25 | type = "journald" 26 | config { 27 | tag = "CONSUL" 28 | } 29 | } 30 | } 31 | resources { 32 | network { 33 | port "consul_dns" { 34 | static = 8600 35 | } 36 | } 37 | } 38 | service { 39 | name = "consul" 40 | 41 | check { 42 | type = "http" 43 | path = "/ui" 44 | interval = "10s" 45 | timeout = "2s" 46 | port = "8500" 47 | address_mode= "driver" 48 | } 49 | } 50 | 51 | } 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /nomad/node-exporter.hcl: -------------------------------------------------------------------------------- 1 | job "node-exporter" { 2 | region = "global" 3 | datacenters = ["dc1"] 4 | type = "service" 5 | 6 | group "app" { 7 | count = 1 8 | 9 | restart { 10 | attempts = 3 11 | delay = "20s" 12 | mode = "delay" 13 | } 14 | 15 | task "node-exporter" { 16 | driver = "docker" 17 | 18 | config { 19 | image = "prom/node-exporter:v0.17.0" 20 | force_pull = true 21 | volumes = [ 22 | "/proc:/host/proc", 23 | "/sys:/host/sys", 24 | "/:/rootfs" 25 | ] 26 | port_map { 27 | http = 9100 28 | } 29 | logging { 30 | type = "journald" 31 | config { 32 | tag = "NODE-EXPORTER" 33 | } 34 | } 35 | 36 | } 37 | 38 | service { 39 | name = "node-exporter" 40 | address_mode = "driver" 41 | tags = [ 42 | "metrics" 43 | ] 44 | port = "http" 45 | 46 | 47 | check { 48 | type = "http" 49 | path = "/metrics/" 50 | interval = "10s" 51 | timeout = "2s" 52 | } 53 | } 54 | 55 | resources { 56 | cpu = 50 57 | memory = 100 58 | 59 | network { 60 | port "http" { static = "9100" } 61 | } 62 | } 63 | } 64 | } 65 | } 66 | 67 | -------------------------------------------------------------------------------- /nomad/cadvisor.hcl: -------------------------------------------------------------------------------- 1 | job "cadvisor" { 2 | region = "global" 3 | datacenters = ["dc1"] 4 | type = "service" 5 | 6 | group "app" { 7 | count = 1 8 | 9 | restart { 10 | attempts = 3 11 | delay = "20s" 12 | mode = "delay" 13 | } 14 | 15 | task "cadvisor" { 16 | driver = "docker" 17 | 18 | config { 19 | image = "google/cadvisor:v0.33.0" 20 | force_pull = true 21 | volumes = [ 22 | "/:/rootfs:ro", 23 | "/var/run:/var/run:rw", 24 | "/sys:/sys:ro", 25 | "/var/lib/docker/:/var/lib/docker:ro", 26 | "/cgroup:/cgroup:ro" 27 | ] 28 | port_map { 29 | http = 8080 30 | } 31 | logging { 32 | type = "journald" 33 | config { 34 | tag = "CADVISOR" 35 | } 36 | } 37 | } 38 | 39 | service { 40 | address_mode = "driver" 41 | name = "cadvisor" 42 | tags = [ 43 | "metrics" 44 | ] 45 | port = "http" 46 | 47 | check { 48 | type = "http" 49 | path = "/metrics/" 50 | interval = "10s" 51 | timeout = "2s" 52 | } 53 | } 54 | 55 | resources { 56 | cpu = 50 57 | memory = 100 58 | 59 | network { 60 | port "http" { static = "8080" } 61 | } 62 | } 63 | } 64 | } 65 | } 66 | 67 | -------------------------------------------------------------------------------- /nomad/prometheus.hcl: -------------------------------------------------------------------------------- 1 | job "prometheus" { 2 | region = "global" 3 | datacenters = ["dc1"] 4 | type = "service" 5 | 6 | group "app" { 7 | count = 1 8 | 9 | restart { 10 | attempts = 3 11 | delay = "20s" 12 | mode = "delay" 13 | } 14 | 15 | task "prometheus" { 16 | driver = "docker" 17 | 18 | config { 19 | image = "prom/prometheus:v2.8.1" 20 | force_pull = true 21 | port_map = { 22 | http = 9090 23 | } 24 | volumes = [ 25 | "/opt/prometheus/:/etc/prometheus/" 26 | ] 27 | args = [ 28 | "--config.file=/etc/prometheus/prometheus.yml", 29 | "--storage.tsdb.path=/prometheus", 30 | "--web.console.libraries=/usr/share/prometheus/console_libraries", 31 | "--web.console.templates=/usr/share/prometheus/consoles", 32 | "--web.enable-admin-api" 33 | ] 34 | logging { 35 | type = "journald" 36 | config { 37 | tag = "PROMETHEUS" 38 | } 39 | } 40 | } 41 | 42 | service { 43 | name = "prometheus" 44 | address_mode = "driver" 45 | tags = [ 46 | "metrics" 47 | ] 48 | port = "http" 49 | 50 | check { 51 | type = "http" 52 | path = "/targets" 53 | interval = "10s" 54 | timeout = "2s" 55 | address_mode = "driver" 56 | } 57 | } 58 | 59 | resources { 60 | cpu = 50 61 | memory = 100 62 | 63 | network { 64 | port "http" { 65 | static = "9090" 66 | } 67 | } 68 | } 69 | } 70 | } 71 | } 72 | 73 | -------------------------------------------------------------------------------- /initialize.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | sudo yum install unzip -y 4 | 5 | NOMAD_VERSION=0.9.0 6 | 7 | echo "Fetching Nomad..." 8 | cd /tmp/ 9 | curl -sSL https://releases.hashicorp.com/nomad/${NOMAD_VERSION}/nomad_${NOMAD_VERSION}_linux_amd64.zip -o nomad.zip 10 | 11 | echo "Installing Nomad..." 12 | unzip nomad.zip 13 | sudo install nomad /usr/bin/nomad 14 | 15 | sudo mkdir -p /etc/nomad.d 16 | sudo chmod a+w /etc/nomad.d 17 | 18 | sudo yum install -y yum-utils device-mapper-persistent-data lvm2 bind-utils nmap 19 | 20 | sudo yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo 21 | sudo yum install -y docker-ce 22 | 23 | sudo systemctl enable docker.service 24 | sudo systemctl restart docker.service 25 | 26 | sudo usermod -aG docker vagrant 27 | 28 | ( 29 | cat <<-EOF 30 | [Unit] 31 | Description=Nomad 32 | Documentation=https://nomadproject.io/docs/ 33 | After=network-online.target 34 | Wants=network-online.target 35 | 36 | [Service] 37 | KillMode=process 38 | KillSignal=SIGINT 39 | ExecStart=/usr/bin/nomad agent -config /opt/nomad/server.hcl 40 | ExecReload=/bin/kill -HUP $MAINPID 41 | LimitNOFILE=65536 42 | 43 | [Install] 44 | WantedBy=multi-user.target 45 | EOF 46 | ) | sudo tee /usr/lib/systemd/system/nomad-server.service 47 | 48 | 49 | sudo systemctl enable nomad-server.service 50 | sudo systemctl start nomad-server.service 51 | 52 | ( 53 | cat <<-EOF 54 | [Unit] 55 | Description=Nomad 56 | Documentation=https://nomadproject.io/docs/ 57 | After=nomad-server.service 58 | Wants=nomad-server.service 59 | 60 | [Service] 61 | KillMode=process 62 | KillSignal=SIGINT 63 | ExecStart=/usr/bin/nomad agent -config /opt/nomad/client.hcl 64 | ExecReload=/bin/kill -HUP $MAINPID 65 | LimitNOFILE=65536 66 | 67 | [Install] 68 | WantedBy=multi-user.target 69 | EOF 70 | ) | sudo tee /usr/lib/systemd/system/nomad-client.service 71 | 72 | 73 | sudo systemctl enable nomad-client.service 74 | sudo systemctl start nomad-client.service 75 | 76 | for bin in cfssl cfssl-certinfo cfssljson 77 | do 78 | echo "Installing $bin..." 79 | curl -sSL https://pkg.cfssl.org/R1.2/${bin}_linux-amd64 > /tmp/${bin} 80 | sudo install /tmp/${bin} /usr/local/bin/${bin} 81 | done 82 | 83 | echo "Installing autocomplete..." 84 | nomad -autocomplete-install 85 | 86 | echo "Start consul container through nomad" 87 | nomad run /opt/nomad/consul.hcl 88 | 89 | echo "Start prometheus container through nomad" 90 | nomad run /opt/nomad/prometheus.hcl 91 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Nomad consul prometheus 2 | 3 | The aim of this project is to provide a development environment to show the dynamic configuration of [prometheus](https://prometheus.io) based on [consul](https://www.consul.io/). 4 | 5 | By using [nomad](https://www.nomadproject.io) to start and stop our services they are all registered into a central key value store with a specific tag we configured. 6 | 7 | Prometheus will use the key value store to collect the services with the 'metrics' tag to identify as an endpoint automatically. 8 | 9 | The following steps should make that clear; 10 | 11 | bring up the environment by using [vagrant](https://www.vagrantup.com) which will bring up a centos 7 virtualbox machine or lxc container. It will use the intialize.sh bash script to install both nomad and consul and also start a nomad job for our prometheus instance. 12 | 13 | The proved working vagrant providers used on an [ArchLinux](https://www.archlinux.org/) system are 14 | * [vagrant-lxc](https://github.com/fgrehm/vagrant-lxc) 15 | * [vagrant-libvirt](https://github.com/vagrant-libvirt/) 16 | * [virtualbox](https://www.virtualbox.org/) 17 | 18 | ```bash 19 | $ vagrant up --provider lxc 20 | OR 21 | $ vagrant up --provider libvirt 22 | OR 23 | $ vagrant up --provider virtualbox 24 | ``` 25 | 26 | Once it is finished, you should be able to connect to the vagrant environment through SSH and interact with Nomad: 27 | 28 | ```bash 29 | $ vagrant ssh 30 | [vagrant@nomad ~]$ nomad status 31 | ID Type Priority Status Submit Date 32 | consul service 50 running 2019-04-03T14:52:25Z 33 | prometheus service 50 running 2019-04-03T14:52:25Z 34 | ``` 35 | 36 | As you can see we have a consul container running next to the prometheus one. 37 | 38 | The consul interface is accessible through [http://localhost:8500](http://localhost:8500) 39 | 40 | ![](img/consul.png) 41 | 42 | When you browse to [http://localhost:9090/targets](http://localhost:9090/targets) you should be able to see the prometheus target configuration 43 | 44 | ![](img/prometheus-target.png) 45 | 46 | By starting a nomad job for one of the two exporters available as a nomad job you should see that it comes up in the target configuration after a few moments. 47 | 48 | ```bash 49 | [vagrant@nomad ~]$ nomad run /opt/nomad/node-exporter.hcl 50 | ==> Monitoring evaluation "310cc1b6" 51 | Evaluation triggered by job "node-exporter" 52 | Allocation "4cf3aff6" created: node "a52cf97d", group "app" 53 | Evaluation status changed: "pending" -> "complete" 54 | ==> Evaluation "310cc1b6" finished with status "complete" 55 | 56 | 57 | [vagrant@nomad ~]$ nomad status node-exporter 58 | ID = node-exporter 59 | Name = node-exporter 60 | Submit Date = 2018-06-04T20:24:13Z 61 | Type = service 62 | Priority = 50 63 | Datacenters = dc1 64 | Status = running 65 | Periodic = false 66 | Parameterized = false 67 | 68 | Summary 69 | Task Group Queued Starting Running Failed Complete Lost 70 | app 0 0 1 0 0 0 71 | 72 | Allocations 73 | ID Node ID Task Group Version Desired Status Created Modified 74 | 4cf3aff6 a52cf97d app 0 run running 29s ago 8s ago 75 | 76 | 77 | [vagrant@nomad ~]$ nomad alloc-status 4cf3aff6 78 | ID = 4cf3aff6 79 | Eval ID = 310cc1b6 80 | Name = node-exporter.app[0] 81 | Node ID = a52cf97d 82 | Job ID = node-exporter 83 | Job Version = 0 84 | Client Status = running 85 | Client Description = 86 | Desired Status = run 87 | Desired Description = 88 | Created = 52s ago 89 | Modified = 14s ago 90 | 91 | Task "node-exporter" is "running" 92 | Task Resources 93 | CPU Memory Disk IOPS Addresses 94 | 0/50 MHz 4.4 MiB/100 MiB 300 MiB 0 http: 127.0.0.1:9100 95 | 96 | Task Events: 97 | Started At = 2018-06-04T20:24:34Z 98 | Finished At = N/A 99 | Total Restarts = 0 100 | Last Restart = N/A 101 | 102 | Recent Events: 103 | Time Type Description 104 | 2018-06-04T20:24:34Z Started Task started by client 105 | 2018-06-04T20:24:13Z Driver Downloading image prom/node-exporter:v0.16.0 106 | 2018-06-04T20:24:13Z Task Setup Building Task Directory 107 | 2018-06-04T20:24:13Z Received Task received by client 108 | ``` 109 | 110 | If you are too fast it will be showing as unknown when refreshing a few times it should come up as UP 111 | 112 | ![](img/node-exporter-target.png) 113 | 114 | a second exporter can be started for example [cadvisor](https://github.com/google/cadvisor) 115 | 116 | ```bash 117 | [vagrant@nomad ~]$ nomad run /opt/nomad/cadvisor.hcl 118 | ==> Monitoring evaluation "d65d4fa8" 119 | Evaluation triggered by job "cadvisor" 120 | Allocation "f43d6b44" created: node "a52cf97d", group "app" 121 | Evaluation status changed: "pending" -> "complete" 122 | ==> Evaluation "d65d4fa8" finished with status "complete" 123 | ``` 124 | 125 | ![](img/cadvisor-target.png) 126 | 127 | When we now stop the node-exporter it should automatically be removed from the target configuration: 128 | 129 | ```bash 130 | [vagrant@nomad ~]$ nomad stop -purge node-exporter 131 | ==> Monitoring evaluation "7f683028" 132 | Evaluation triggered by job "node-exporter" 133 | Evaluation status changed: "pending" -> "complete" 134 | ==> Evaluation "7f683028" finished with status "complete" 135 | ``` 136 | 137 | ![](img/removed-prometheus-target.png) 138 | 139 | All the nomad jobs are sending the application logs from the docker container towards journalctl so you could follow the progress there too. 140 | --------------------------------------------------------------------------------