├── flask-prometheus ├── .gitignore ├── Dockerfile ├── docker-compose.yml ├── flask_prometheus │ └── __init__.py ├── main.py └── requirements.txt ├── grafana ├── dashboards │ └── node-dashboard.json ├── provisioning │ ├── dashboards.yaml │ └── datasource-prometheus.yaml └── scripts │ └── fill-memory.sh ├── kubernetes └── README.md ├── scripts ├── 1-install.sh ├── 2-node-exporter.sh ├── 3-install-grafana.sh ├── 4-install-alertmanager.sh ├── 5-install-mailserver.sh ├── 6-cloudwatch-exporter.sh ├── add-flask-app.sh ├── add-spring-boot-app.sh ├── install-consul-exporter.sh ├── install-docker.sh ├── mutual-tls.sh ├── register-service.sh ├── reverse-proxy-mutual-tls.sh ├── reverse-proxy.sh └── start-consul.sh ├── spring-boot-prometheus ├── Dockerfile ├── README.md ├── pom.xml └── src │ ├── main │ ├── java │ │ └── io │ │ │ └── in4it │ │ │ └── monitoring │ │ │ └── springboot │ │ │ ├── Application.java │ │ │ └── ApplicationController.java │ └── resources │ │ └── application.properties │ └── test │ └── java │ └── io │ └── in4it │ └── monitoring │ └── springboot │ └── ApplicationTests.java └── use-cases └── ec2-auto-discovery └── lab.txt /flask-prometheus/.gitignore: -------------------------------------------------------------------------------- 1 | */__pycache__/* 2 | */__pycache__ 3 | -------------------------------------------------------------------------------- /flask-prometheus/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3-alpine 2 | 3 | WORKDIR /usr/src/app 4 | 5 | COPY requirements.txt ./ 6 | RUN apk --update add --no-cache mariadb-dev gcc musl-dev && \ 7 | pip install --no-cache-dir -r requirements.txt && \ 8 | apk del gcc musl-dev 9 | 10 | RUN apk --update add bash && \ 11 | wget -q -O /wait-for-it.sh https://raw.githubusercontent.com/vishnubob/wait-for-it/master/wait-for-it.sh && \ 12 | chmod +x /wait-for-it.sh 13 | 14 | COPY . . 15 | 16 | EXPOSE 5000 17 | EXPOSE 8000 18 | 19 | CMD [ "python", "./main.py" ] 20 | -------------------------------------------------------------------------------- /flask-prometheus/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | services: 3 | flask-prometheus-example: 4 | ports: 5 | - "5000:5000" 6 | - "8000:8000" 7 | environment: 8 | MYSQL_HOST: mysql 9 | MYSQL_USER: root 10 | MYSQL_PASSWORD: app 11 | MYSQL_DB: app 12 | image: "wardviaene/flask-prometheus-example" 13 | depends_on: 14 | - mysql 15 | command: ["/wait-for-it.sh", "mysql:3306", "--", "python", "./main.py"] 16 | mysql: 17 | image: "mysql:5.7" 18 | environment: 19 | MYSQL_ROOT_PASSWORD: app 20 | MYSQL_DATABASE: app 21 | -------------------------------------------------------------------------------- /flask-prometheus/flask_prometheus/__init__.py: -------------------------------------------------------------------------------- 1 | # modified version of https://github.com/sbarratt/flask-prometheus 2 | 3 | import time 4 | 5 | import _mysql 6 | import os 7 | 8 | from prometheus_client import Counter, Histogram 9 | from prometheus_client import start_http_server 10 | from flask import request 11 | 12 | FLASK_REQUEST_LATENCY = Histogram('flask_request_latency_seconds', 'Flask Request Latency', 13 | ['method', 'endpoint']) 14 | FLASK_REQUEST_COUNT = Counter('flask_request_count', 'Flask Request Count', 15 | ['method', 'endpoint', 'http_status']) 16 | 17 | MYSQL_REQUEST_LATENCY = Histogram('mysql_query_latency_seconds', 'MYSQL Query Latency', 18 | ['query']) 19 | MYSQL_REQUEST_COUNT = Counter('mysql_query_count', 'Flask Request Count', 20 | ['query']) 21 | 22 | 23 | 24 | def before_request(): 25 | request.start_time = time.time() 26 | 27 | 28 | def after_request(response): 29 | request_latency = time.time() - request.start_time 30 | FLASK_REQUEST_LATENCY.labels(request.method, request.path).observe(request_latency) 31 | FLASK_REQUEST_COUNT.labels(request.method, request.path, response.status_code).inc() 32 | 33 | return response 34 | 35 | def monitor(app, port=8000, addr=''): 36 | app.before_request(before_request) 37 | app.after_request(after_request) 38 | start_http_server(port, addr) 39 | 40 | 41 | def mysql_fetchall(db, sql): 42 | # get start time 43 | start_time = time.time() 44 | # execute query 45 | cursor = db.cursor() 46 | cursor.execute(sql) 47 | data=cursor.fetchall() 48 | # log finish time 49 | query_latency = time.time() - start_time 50 | MYSQL_REQUEST_LATENCY.labels(sql[:50]).observe(query_latency) 51 | MYSQL_REQUEST_COUNT.labels(sql[:50]).inc() 52 | # return data 53 | return data 54 | -------------------------------------------------------------------------------- /flask-prometheus/main.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | import MySQLdb 4 | import os 5 | 6 | from flask import Flask 7 | from flask_prometheus import monitor, mysql_fetchall 8 | 9 | # flask 10 | app = Flask(__name__) 11 | 12 | # database 13 | db=MySQLdb.connect(host=os.getenv("MYSQL_HOST", "localhost"), 14 | user=os.getenv("MYSQL_USER", "root"), 15 | passwd=os.getenv("MYSQL_PASSWORD", ""), 16 | db=os.getenv("MYSQL_DB", "app")) 17 | 18 | # routing 19 | @app.route('/') 20 | def index(): 21 | return "Flask is up & running\n" 22 | @app.route('/query') 23 | def query(): 24 | res = mysql_fetchall(db, "select 1 as id, RAND()*10 as rand union select 2 as id, RAND()*100 as rand") 25 | output = "" 26 | for row in res: 27 | line = ",".join(str(s) for s in row) 28 | output += line+"\n" 29 | return output 30 | 31 | @app.route('/sleep') 32 | def sleep(): 33 | res = mysql_fetchall(db, "select SLEEP(RAND()*10) as sleeping") 34 | return "Done!" 35 | 36 | 37 | # monitoring 38 | monitor(app, port=8000) 39 | 40 | # run app 41 | app.run(host="0.0.0.0") 42 | -------------------------------------------------------------------------------- /flask-prometheus/requirements.txt: -------------------------------------------------------------------------------- 1 | mysqlclient 2 | flask 3 | prometheus_client 4 | -------------------------------------------------------------------------------- /grafana/dashboards/node-dashboard.json: -------------------------------------------------------------------------------- 1 | { 2 | "annotations": { 3 | "list": [ 4 | { 5 | "builtIn": 1, 6 | "datasource": "-- Grafana --", 7 | "enable": true, 8 | "hide": true, 9 | "iconColor": "rgba(0, 211, 255, 1)", 10 | "name": "Annotations & Alerts", 11 | "type": "dashboard" 12 | } 13 | ] 14 | }, 15 | "editable": true, 16 | "gnetId": null, 17 | "graphTooltip": 0, 18 | "id": 5, 19 | "links": [], 20 | "panels": [ 21 | { 22 | "aliasColors": {}, 23 | "bars": false, 24 | "dashLength": 10, 25 | "dashes": false, 26 | "datasource": null, 27 | "fill": 1, 28 | "gridPos": { 29 | "h": 9, 30 | "w": 12, 31 | "x": 0, 32 | "y": 0 33 | }, 34 | "id": 4, 35 | "legend": { 36 | "avg": false, 37 | "current": false, 38 | "max": false, 39 | "min": false, 40 | "show": true, 41 | "total": false, 42 | "values": false 43 | }, 44 | "lines": true, 45 | "linewidth": 1, 46 | "links": [], 47 | "nullPointMode": "null", 48 | "percentage": false, 49 | "pointradius": 5, 50 | "points": false, 51 | "renderer": "flot", 52 | "seriesOverrides": [], 53 | "spaceLength": 10, 54 | "stack": false, 55 | "steppedLine": false, 56 | "targets": [ 57 | { 58 | "expr": "sum( node_memory_MemAvailable_bytes ) by (instance) / 1024 / 1024 / 1024\n", 59 | "format": "time_series", 60 | "hide": false, 61 | "intervalFactor": 1, 62 | "refId": "A" 63 | } 64 | ], 65 | "thresholds": [], 66 | "timeFrom": null, 67 | "timeShift": null, 68 | "title": "Free Memory", 69 | "tooltip": { 70 | "shared": true, 71 | "sort": 0, 72 | "value_type": "individual" 73 | }, 74 | "type": "graph", 75 | "xaxis": { 76 | "buckets": null, 77 | "mode": "time", 78 | "name": null, 79 | "show": true, 80 | "values": [] 81 | }, 82 | "yaxes": [ 83 | { 84 | "decimals": null, 85 | "format": "short", 86 | "label": "Mem / GB", 87 | "logBase": 1, 88 | "max": null, 89 | "min": null, 90 | "show": true 91 | }, 92 | { 93 | "format": "short", 94 | "label": null, 95 | "logBase": 1, 96 | "max": null, 97 | "min": null, 98 | "show": true 99 | } 100 | ], 101 | "yaxis": { 102 | "align": false, 103 | "alignLevel": null 104 | } 105 | }, 106 | { 107 | "aliasColors": {}, 108 | "bars": false, 109 | "dashLength": 10, 110 | "dashes": false, 111 | "datasource": null, 112 | "fill": 1, 113 | "gridPos": { 114 | "h": 9, 115 | "w": 12, 116 | "x": 12, 117 | "y": 0 118 | }, 119 | "id": 2, 120 | "legend": { 121 | "avg": false, 122 | "current": false, 123 | "max": false, 124 | "min": false, 125 | "show": true, 126 | "total": false, 127 | "values": false 128 | }, 129 | "lines": true, 130 | "linewidth": 1, 131 | "links": [], 132 | "nullPointMode": "null", 133 | "percentage": false, 134 | "pointradius": 5, 135 | "points": false, 136 | "renderer": "flot", 137 | "seriesOverrides": [], 138 | "spaceLength": 10, 139 | "stack": false, 140 | "steppedLine": false, 141 | "targets": [ 142 | { 143 | "expr": "100 - (avg by (instance) (irate(node_cpu_seconds_total{job=\"node_exporter\",mode=\"idle\"}[5m])) * 100)\n", 144 | "format": "time_series", 145 | "hide": false, 146 | "intervalFactor": 1, 147 | "refId": "A" 148 | } 149 | ], 150 | "thresholds": [], 151 | "timeFrom": null, 152 | "timeShift": null, 153 | "title": "CPU Usage", 154 | "tooltip": { 155 | "shared": true, 156 | "sort": 0, 157 | "value_type": "individual" 158 | }, 159 | "type": "graph", 160 | "xaxis": { 161 | "buckets": null, 162 | "mode": "time", 163 | "name": null, 164 | "show": true, 165 | "values": [] 166 | }, 167 | "yaxes": [ 168 | { 169 | "decimals": null, 170 | "format": "short", 171 | "label": "CPU %", 172 | "logBase": 1, 173 | "max": "100", 174 | "min": "0", 175 | "show": true 176 | }, 177 | { 178 | "format": "short", 179 | "label": null, 180 | "logBase": 1, 181 | "max": null, 182 | "min": null, 183 | "show": true 184 | } 185 | ], 186 | "yaxis": { 187 | "align": false, 188 | "alignLevel": null 189 | } 190 | } 191 | ], 192 | "schemaVersion": 16, 193 | "style": "dark", 194 | "tags": [], 195 | "templating": { 196 | "list": [] 197 | }, 198 | "time": { 199 | "from": "now-6h", 200 | "to": "now" 201 | }, 202 | "timepicker": { 203 | "refresh_intervals": [ 204 | "5s", 205 | "10s", 206 | "30s", 207 | "1m", 208 | "5m", 209 | "15m", 210 | "30m", 211 | "1h", 212 | "2h", 213 | "1d" 214 | ], 215 | "time_options": [ 216 | "5m", 217 | "15m", 218 | "1h", 219 | "6h", 220 | "12h", 221 | "24h", 222 | "2d", 223 | "7d", 224 | "30d" 225 | ] 226 | }, 227 | "timezone": "", 228 | "title": "Node Statistics", 229 | "uid": "KchAVsdmk", 230 | "version": 3 231 | } 232 | 233 | -------------------------------------------------------------------------------- /grafana/provisioning/dashboards.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: 1 2 | providers: 3 | - name: 'default' 4 | orgId: 1 5 | folder: '' 6 | type: file 7 | options: 8 | path: /var/lib/grafana/dashboards 9 | -------------------------------------------------------------------------------- /grafana/provisioning/datasource-prometheus.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: 1 2 | datasources: 3 | - name: Prometheus 4 | type: prometheus 5 | orgId: 1 6 | url: http://localhost:9090 7 | access: proxy 8 | version: 1 9 | editable: false 10 | isDefault: true 11 | -------------------------------------------------------------------------------- /grafana/scripts/fill-memory.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | if [ ! -e /usr/bin/stress ] ; then 3 | apt-get install stress 4 | fi 5 | stress --vm-bytes $(awk '/MemAvailable/{printf "%d\n", $2 * 0.5;}' < /proc/meminfo)k --vm-keep -m 1 6 | -------------------------------------------------------------------------------- /kubernetes/README.md: -------------------------------------------------------------------------------- 1 | # Running Prometheus on Kubernetes 2 | 3 | # Install Helm 4 | ``` 5 | wget https://storage.googleapis.com/kubernetes-helm/helm-v2.11.0-linux-amd64.tar.gz 6 | tar -xzvf helm-v2.11.0-linux-amd64.tar.gz 7 | sudo mv linux-amd64/helm /usr/local/bin/helm 8 | kubectl create -f https://raw.githubusercontent.com/wardviaene/kubernetes-course/master/helm/helm-rbac.yaml 9 | helm init --service-account tiller 10 | ``` 11 | ## Start Prometheus (without storage) 12 | ``` 13 | helm install --name prometheus --set server.persistentVolume.enabled=false,alertmanager.persistentVolume.enabled=false stable/prometheus 14 | ``` 15 | 16 | ## Exposing prometheus port 17 | ``` 18 | export POD_NAME=$(kubectl get pods --namespace default -l "app=prometheus,component=server" -o jsonpath="{.items[0].metadata.name}") 19 | kubectl --namespace default port-forward $POD_NAME 9090 & 20 | socat TCP4-LISTEN:9091,fork TCP4:localhost:9090 & 21 | ``` 22 | -------------------------------------------------------------------------------- /scripts/1-install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | PROMETHEUS_VERSION="2.2.1" 3 | wget https://github.com/prometheus/prometheus/releases/download/v${PROMETHEUS_VERSION}/prometheus-${PROMETHEUS_VERSION}.linux-amd64.tar.gz 4 | tar -xzvf prometheus-${PROMETHEUS_VERSION}.linux-amd64.tar.gz 5 | cd prometheus-${PROMETHEUS_VERSION}.linux-amd64/ 6 | # if you just want to start prometheus as root 7 | #./prometheus --config.file=prometheus.yml 8 | 9 | # create user 10 | useradd --no-create-home --shell /bin/false prometheus 11 | 12 | # create directories 13 | mkdir -p /etc/prometheus 14 | mkdir -p /var/lib/prometheus 15 | 16 | # set ownership 17 | chown prometheus:prometheus /etc/prometheus 18 | chown prometheus:prometheus /var/lib/prometheus 19 | 20 | # copy binaries 21 | cp prometheus /usr/local/bin/ 22 | cp promtool /usr/local/bin/ 23 | 24 | chown prometheus:prometheus /usr/local/bin/prometheus 25 | chown prometheus:prometheus /usr/local/bin/promtool 26 | 27 | # copy config 28 | cp -r consoles /etc/prometheus 29 | cp -r console_libraries /etc/prometheus 30 | cp prometheus.yml /etc/prometheus/prometheus.yml 31 | 32 | chown -R prometheus:prometheus /etc/prometheus/consoles 33 | chown -R prometheus:prometheus /etc/prometheus/console_libraries 34 | 35 | # setup systemd 36 | echo '[Unit] 37 | Description=Prometheus 38 | Wants=network-online.target 39 | After=network-online.target 40 | 41 | [Service] 42 | User=prometheus 43 | Group=prometheus 44 | Type=simple 45 | ExecStart=/usr/local/bin/prometheus \ 46 | --config.file /etc/prometheus/prometheus.yml \ 47 | --storage.tsdb.path /var/lib/prometheus/ \ 48 | --web.console.templates=/etc/prometheus/consoles \ 49 | --web.console.libraries=/etc/prometheus/console_libraries 50 | 51 | [Install] 52 | WantedBy=multi-user.target' > /etc/systemd/system/prometheus.service 53 | 54 | systemctl daemon-reload 55 | systemctl enable prometheus 56 | systemctl start prometheus 57 | -------------------------------------------------------------------------------- /scripts/2-node-exporter.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | NODE_EXPORTER_VERSION="0.16.0" 3 | wget https://github.com/prometheus/node_exporter/releases/download/v${NODE_EXPORTER_VERSION}/node_exporter-${NODE_EXPORTER_VERSION}.linux-amd64.tar.gz 4 | tar -xzvf node_exporter-${NODE_EXPORTER_VERSION}.linux-amd64.tar.gz 5 | cd node_exporter-${NODE_EXPORTER_VERSION}.linux-amd64 6 | cp node_exporter /usr/local/bin 7 | 8 | # create user 9 | useradd --no-create-home --shell /bin/false node_exporter 10 | 11 | chown node_exporter:node_exporter /usr/local/bin/node_exporter 12 | 13 | echo '[Unit] 14 | Description=Node Exporter 15 | Wants=network-online.target 16 | After=network-online.target 17 | 18 | [Service] 19 | User=node_exporter 20 | Group=node_exporter 21 | Type=simple 22 | ExecStart=/usr/local/bin/node_exporter 23 | 24 | [Install] 25 | WantedBy=multi-user.target' > /etc/systemd/system/node_exporter.service 26 | 27 | # enable node_exporter in systemctl 28 | systemctl daemon-reload 29 | systemctl start node_exporter 30 | systemctl enable node_exporter 31 | 32 | 33 | echo "Setup complete. 34 | Add the following lines to /etc/prometheus/prometheus.yml: 35 | 36 | - job_name: 'node_exporter' 37 | scrape_interval: 5s 38 | static_configs: 39 | - targets: ['localhost:9100'] 40 | " 41 | 42 | -------------------------------------------------------------------------------- /scripts/3-install-grafana.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | echo 'deb https://packages.grafana.com/oss/deb stable main' >> /etc/apt/sources.list 3 | curl https://packages.grafana.com/gpg.key | sudo apt-key add - 4 | sudo apt-get update 5 | sudo apt-get -y install grafana 6 | 7 | systemctl daemon-reload 8 | systemctl start grafana-server 9 | systemctl enable grafana-server.service 10 | -------------------------------------------------------------------------------- /scripts/4-install-alertmanager.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ALERTMANAGER_VERSION="0.15.0" 3 | wget https://github.com/prometheus/alertmanager/releases/download/v${ALERTMANAGER_VERSION}/alertmanager-${ALERTMANAGER_VERSION}.linux-amd64.tar.gz 4 | tar xvzf alertmanager-${ALERTMANAGER_VERSION}.linux-amd64.tar.gz 5 | cd alertmanager-${ALERTMANAGER_VERSION}.linux-amd64/ 6 | # if you just want to start prometheus as root 7 | #./alertmanager --config.file=simple.yml 8 | 9 | # create user 10 | useradd --no-create-home --shell /bin/false alertmanager 11 | 12 | # create directories 13 | mkdir /etc/alertmanager 14 | mkdir /etc/alertmanager/template 15 | mkdir -p /var/lib/alertmanager/data 16 | 17 | # touch config file 18 | touch /etc/alertmanager/alertmanager.yml 19 | 20 | # set ownership 21 | chown -R alertmanager:alertmanager /etc/alertmanager 22 | chown -R alertmanager:alertmanager /var/lib/alertmanager 23 | 24 | # copy binaries 25 | cp alertmanager /usr/local/bin/ 26 | cp amtool /usr/local/bin/ 27 | 28 | # set ownership 29 | chown alertmanager:alertmanager /usr/local/bin/alertmanager 30 | chown alertmanager:alertmanager /usr/local/bin/amtool 31 | 32 | # setup systemd 33 | echo '[Unit] 34 | Description=Prometheus Alertmanager Service 35 | Wants=network-online.target 36 | After=network.target 37 | 38 | [Service] 39 | User=alertmanager 40 | Group=alertmanager 41 | Type=simple 42 | ExecStart=/usr/local/bin/alertmanager \ 43 | --config.file /etc/alertmanager/alertmanager.yml \ 44 | --storage.path /var/lib/alertmanager/data 45 | Restart=always 46 | 47 | [Install] 48 | WantedBy=multi-user.target' > /etc/systemd/system/alertmanager.service 49 | 50 | systemctl daemon-reload 51 | systemctl enable alertmanager 52 | systemctl start alertmanager 53 | 54 | # restart prometheus 55 | systemctl start prometheus 56 | 57 | 58 | echo "(1/2)Setup complete. 59 | Add the following lines and substitute with correct values to /etc/alertmanager/alertmanager.yml: 60 | 61 | global: 62 | smtp_smarthost: 'localhost:25' 63 | smtp_from: 'alertmanager@prometheus.com' 64 | smtp_auth_username: '' 65 | smtp_auth_password: '' 66 | smtp_require_tls: false 67 | 68 | templates: 69 | - '/etc/alertmanager/template/*.tmpl' 70 | 71 | route: 72 | repeat_interval: 1h 73 | receiver: operations-team 74 | 75 | receivers: 76 | - name: 'operations-team' 77 | email_configs: 78 | - to: 'operations-team+alerts@example.org' 79 | slack_configs: 80 | - api_url: https://hooks.slack.com/services/XXXXXX/XXXXXX/XXXXXX 81 | channel: '#prometheus-course' 82 | send_resolved: true 83 | " 84 | 85 | echo "(2/2)Setup complete. 86 | Alter the following config in /etc/prometheus/prometheus.yml: 87 | 88 | alerting: 89 | alertmanagers: 90 | - static_configs: 91 | - targets: 92 | - localhost:9093" 93 | 94 | -------------------------------------------------------------------------------- /scripts/5-install-mailserver.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # unattended postfix install 4 | debconf-set-selections <<< "postfix postfix/mailname string your.domain.com" 5 | debconf-set-selections <<< "postfix postfix/main_mailer_type string 'Internet Site'" 6 | apt-get install -y postfix 7 | apt-get install -y mailutils 8 | 9 | # set config to send-only and from localhost 10 | echo ' 11 | smtpd_banner = $myhostname ESMTP $mail_name (Ubuntu) 12 | biff = no 13 | append_dot_mydomain = no 14 | readme_directory = no 15 | smtpd_tls_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem 16 | smtpd_tls_key_file=/etc/ssl/private/ssl-cert-snakeoil.key 17 | smtpd_use_tls=yes 18 | smtpd_tls_session_cache_database = btree:${data_directory}/smtpd_scache 19 | smtp_tls_session_cache_database = btree:${data_directory}/smtp_scache 20 | smtpd_relay_restrictions = permit_mynetworks permit_sasl_authenticated defer_unauth_destination 21 | myhostname = prometheus.com 22 | alias_maps = hash:/etc/aliases 23 | alias_database = hash:/etc/aliases 24 | myorigin = /etc/mailname 25 | mydestination = $myhostname, localhost.$mydomain, $mydomain 26 | relayhost = 27 | mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128 28 | mailbox_size_limit = 0 29 | recipient_delimiter = + 30 | inet_interfaces = loopback-only 31 | inet_protocols = all 32 | ' > /etc/postfix/main.cf 33 | 34 | # restart postfix 35 | service postfix restart 36 | 37 | echo "(1/2)Setup complete. 38 | Add the following lines and substitute with correct values to /etc/alertmanager/alertmanager.yml: 39 | echo "You successfully installed the mail server" | mail -s "Prometheus mail server OK" 40 | If your mail isn't received check the syslog(tail -f /var/log/syslog) and the spam folder. 41 | " 42 | 43 | -------------------------------------------------------------------------------- /scripts/6-cloudwatch-exporter.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | CLOUDWATCH_EXPORTER_VERSION="0.4" 3 | wget -O /usr/local/bin/cloudwatch_exporter.jar http://search.maven.org/remotecontent?filepath=io/prometheus/cloudwatch/cloudwatch_exporter/${CLOUDWATCH_EXPORTER_VERSION}/cloudwatch_exporter-${CLOUDWATCH_EXPORTER_VERSION}-jar-with-dependencies.jar 4 | 5 | #install java 6 | apt-get install -y openjdk-9-jre-headless 7 | 8 | #create configuration directory 9 | mkdir -p /etc/cloudwatchexporter 10 | touch /etc/cloudwatchexporter/cloudwatchexporter.yml 11 | mkdir -p ~/.aws/ 12 | touch ~/.aws/credentials 13 | 14 | #aws credentail template 15 | echo '[default] 16 | aws_access_key_id=YOUR_ACCESS_KEY_ID 17 | aws_secret_access_key=YOUR_SECRET_ACCESS_KEY' >> ~/.aws/credentials 18 | 19 | echo '[Unit] 20 | Description=CLoudwatch Exporter 21 | Wants=network-online.target 22 | After=network-online.target 23 | 24 | [Service] 25 | User=root 26 | Group=root 27 | Type=simple 28 | ExecStart=/usr/bin/java -jar /usr/local/bin/cloudwatch_exporter.jar 9106 /etc/cloudwatchexporter/cloudwatchexporter.yml 29 | 30 | [Install] 31 | WantedBy=multi-user.target' > /etc/systemd/system/cloudwatch_exporter.service 32 | 33 | # enable node_exporter in systemctl 34 | systemctl daemon-reload 35 | systemctl enable cloudwatch_exporter 36 | 37 | 38 | echo "Installation complete. 39 | 40 | - Add your AWS keys to ~/.aws/credentials (IAM permissions needed: cloudwatch:ListMetrics and cloudwatch:GetMetricStatistics needed) 41 | 42 | - Add the following lines to /etc/prometheus/prometheus.yml: 43 | 44 | - job_name: 'cloudwatch_exporter' 45 | scrape_interval: 5s 46 | static_configs: 47 | - targets: ['localhost:9106'] 48 | 49 | - Add the following lines to /etc/cloudwatchexporter/cloudwatchexporter.yml: 50 | 51 | --- 52 | region: eu-west-1 53 | metrics: 54 | - aws_namespace: AWS/ELB 55 | aws_metric_name: HealthyHostCount 56 | aws_dimensions: [AvailabilityZone, LoadBalancerName] 57 | aws_statistics: [Average] 58 | 59 | - run: systemctl start cloudwatch_exporter 60 | " 61 | -------------------------------------------------------------------------------- /scripts/add-flask-app.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | echo " - job_name: 'flask_app' 3 | scrape_interval: 5s 4 | static_configs: 5 | - targets: ['localhost:8000']" >> /etc/prometheus/prometheus.yml 6 | 7 | -------------------------------------------------------------------------------- /scripts/add-spring-boot-app.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | echo " - job_name: 'spring_boot_demo_app' 3 | scrape_interval: 5s 4 | metrics_path: '/actuator/prometheus' 5 | static_configs: 6 | - targets: ['localhost:8080']" >> /etc/prometheus/prometheus.yml 7 | 8 | -------------------------------------------------------------------------------- /scripts/install-consul-exporter.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | CONSUL_EXPORTER_VERSION="0.3.0" 3 | wget https://github.com/prometheus/consul_exporter/releases/download/v${CONSUL_EXPORTER_VERSION}/consul_exporter-${CONSUL_EXPORTER_VERSION}.linux-amd64.tar.gz 4 | tar -xzvf consul_exporter-${CONSUL_EXPORTER_VERSION}.linux-amd64.tar.gz 5 | cd consul_exporter-${CONSUL_EXPORTER_VERSION}.linux-amd64 6 | cp consul_exporter /usr/local/bin 7 | 8 | # create user 9 | useradd --no-create-home --shell /bin/false consul_exporter 10 | 11 | chown consul_exporter:consul_exporter /usr/local/bin/consul_exporter 12 | 13 | echo '[Unit] 14 | Description=Consul Exporter 15 | Wants=network-online.target 16 | After=network-online.target 17 | 18 | [Service] 19 | User=consul_exporter 20 | Group=consul_exporter 21 | Type=simple 22 | ExecStart=/usr/local/bin/consul_exporter 23 | 24 | [Install] 25 | WantedBy=multi-user.target' > /etc/systemd/system/consul_exporter.service 26 | 27 | # enable consul_exporter in systemctl 28 | systemctl daemon-reload 29 | systemctl start consul_exporter 30 | systemctl enable consul_exporter 31 | 32 | 33 | echo "Setup complete. 34 | Add the following lines to /etc/prometheus/prometheus.yml: 35 | 36 | - job_name: 'consul_exporter' 37 | scrape_interval: 5s 38 | static_configs: 39 | - targets: ['localhost:9107'] 40 | " 41 | -------------------------------------------------------------------------------- /scripts/install-docker.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # this script installs docker and docker-compose 3 | 4 | curl -fsSL get.docker.com -o get-docker.sh 5 | sh get-docker.sh 6 | sudo curl -L https://github.com/docker/compose/releases/download/1.21.2/docker-compose-$(uname -s)-$(uname -m) -o /usr/local/bin/docker-compose 7 | chmod +x /usr/local/bin/docker-compose 8 | -------------------------------------------------------------------------------- /scripts/mutual-tls.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -ex 4 | 5 | TARGET_IP="138.68.135.9" 6 | 7 | echo ' 8 | # From http://apetec.com/support/GenerateSAN-CSR.htm 9 | 10 | [req] 11 | distinguished_name = req_distinguished_name 12 | req_extensions = v3_req 13 | 14 | [req_distinguished_name] 15 | countryName = Country Name (2 letter code) 16 | countryName_default = US 17 | stateOrProvinceName = State or Province Name (full name) 18 | stateOrProvinceName_default = MN 19 | localityName = Locality Name (eg, city) 20 | localityName_default = Minneapolis 21 | organizationalUnitName = Organizational Unit Name (eg, section) 22 | organizationalUnitName_default = Domain Control Validated 23 | commonName = Internet Widgits Ltd 24 | commonName_max = 64 25 | 26 | [ v3_req ] 27 | # Extensions to add to a certificate request 28 | basicConstraints = CA:FALSE 29 | extendedKeyUsage = clientAuth,serverAuth 30 | subjectAltName = @alt_names 31 | 32 | [alt_names]' > openssl-${TARGET_IP}.cnf 33 | 34 | echo -en "IP.1 = ${TARGET_IP}\n" >> openssl-${TARGET_IP}.cnf 35 | 36 | 37 | # create CA 38 | openssl genrsa -out ca.key 4096 39 | chmod 400 ca.key 40 | openssl req -new -x509 -sha256 -days 3650 -key ca.key -out ca.crt -subj "/CN=prometheus-ca.example.com" 41 | chmod 644 ca.crt 42 | 43 | # Create target key 44 | openssl genrsa -out target.key 2048 45 | chmod 400 target.key 46 | openssl req -new -key target.key -sha256 -out target.csr -config openssl-${TARGET_IP}.cnf -subj "/CN=prometheus-target.example.com" 47 | 48 | openssl x509 -req -days 365 -sha256 -in target.csr -CA ca.crt -CAkey ca.key -set_serial 1 -out target.crt -extensions v3_req -extfile openssl-${TARGET_IP}.cnf 49 | chmod 444 target.crt 50 | 51 | # Create client key for prometheus server 52 | openssl genrsa -out client.key 2048 53 | openssl req -new -key client.key -out client.csr -subj "/CN=prometheus.example.com" 54 | openssl x509 -req -days 365 -sha256 -in client.csr -CA ca.crt -CAkey ca.key -set_serial 2 -out client.crt 55 | 56 | mv ca.crt /etc/ssl/certs/prometheus-ca.crt 57 | mv ca.key /etc/ssl/private/prometheus-ca.key 58 | mv client.key /etc/prometheus/prometheus.key 59 | chown prometheus:prometheus /etc/prometheus/prometheus.key 60 | mv client.crt /etc/ssl/certs/prometheus.crt 61 | 62 | echo 'Add the following lines to /etc/prometheus/prometheus.yml:' 63 | echo " - job_name: 'node_exporter_ssl' 64 | scrape_interval: 5s 65 | scheme: https 66 | tls_config: 67 | ca_file: /etc/ssl/certs/prometheus-ca.crt 68 | cert_file: /etc/ssl/certs/prometheus.crt 69 | key_file: /etc/prometheus/prometheus.key 70 | static_configs: 71 | - targets: ['${TARGET_IP}:443']" 72 | 73 | -------------------------------------------------------------------------------- /scripts/register-service.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | IP=$(curl -s ifconfig.co) 3 | curl -X PUT -d '{ 4 | "ID": "python-flask", 5 | "Name": "python-flask", 6 | "Address": "'${IP}'", 7 | "Port": 8000, 8 | "Check": { 9 | "Name": "HTTP check", 10 | "Interval": "30s", 11 | "HTTP": "http://'${IP}':5000/" 12 | } 13 | }' http://localhost:8500/v1/agent/service/register 14 | 15 | -------------------------------------------------------------------------------- /scripts/reverse-proxy-mutual-tls.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | mv target.crt /etc/ssl/certs/target.crt 6 | mv target.key /etc/ssl/private/target.key 7 | mv prometheus-ca.crt /etc/ssl/certs/prometheus-ca.crt 8 | 9 | HOST="localhost" 10 | PORT="9100" 11 | 12 | # run script as root or with sudo 13 | 14 | # install nginx and openssl 15 | apt -y install nginx openssl 16 | 17 | echo 'server { 18 | listen 443; 19 | ssl on; 20 | ssl_certificate /etc/ssl/certs/target.crt; 21 | ssl_certificate_key /etc/ssl/private/target.key; 22 | ssl_client_certificate /etc/ssl/certs/prometheus-ca.crt; 23 | ssl_verify_client on; 24 | location / { 25 | proxy_pass http://'${HOST}':'${PORT}'/; 26 | } 27 | }' > /etc/nginx/sites-enabled/node-exporter 28 | 29 | systemctl enable nginx 30 | systemctl restart nginx 31 | 32 | EXTERNAL_IP=$(curl -s ifconfig.co) 33 | echo "Reverse proxy with mutual tls enabled on https://${EXTERNAL_IP}" 34 | 35 | -------------------------------------------------------------------------------- /scripts/reverse-proxy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | HOST="localhost" 4 | PORT="9090" 5 | 6 | # run script as root or with sudo 7 | 8 | # install nginx and openssl 9 | apt -y install nginx openssl apache2-utils 10 | 11 | # generate ssl certificate (host prometheus.example.com) 12 | openssl req -x509 -newkey rsa:4096 -keyout key.pem -out cert.pem -days 365 -subj '/CN=prometheus.example.com' -nodes 13 | mv key.pem /etc/ssl/private/nginx.pem 14 | chmod 600 /etc/ssl/private/nginx.pem 15 | mv cert.pem /etc/ssl/certs/nginx.pem 16 | 17 | echo 'server { 18 | listen 443; 19 | ssl on; 20 | ssl_certificate /etc/ssl/certs/nginx.pem; 21 | ssl_certificate_key /etc/ssl/private/nginx.pem; 22 | location / { 23 | proxy_pass http://'${HOST}':'${PORT}'/; 24 | auth_basic "Prometheus"; 25 | auth_basic_user_file /etc/nginx/.htpasswd; 26 | } 27 | }' > /etc/nginx/sites-enabled/prometheus 28 | 29 | systemctl enable nginx 30 | systemctl restart nginx 31 | 32 | EXTERNAL_IP=$(curl -s ifconfig.co) 33 | echo "Reverse proxy enabled on https://${EXTERNAL_IP}" 34 | -------------------------------------------------------------------------------- /scripts/start-consul.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker run --name consul -d --net=host -v /consul/data:/consul/data \ 3 | -e CONSUL_LOCAL_CONFIG='{ 4 | "datacenter":"eu-west", 5 | "server":true, 6 | "enable_debug":true 7 | }' consul:1.2.1 agent -server -bind 127.0.0.1 -bootstrap-expect=1 8 | 9 | echo "Setup complete. 10 | Add the following lines to /etc/prometheus/prometheus.yml: 11 | 12 | - job_name: 'consul' 13 | consul_sd_configs: 14 | - server: '127.0.0.1:8500' 15 | 16 | relabel_configs: 17 | - source_labels: ['__meta_consul_service'] 18 | regex: '(.*)' 19 | target_label: 'job' 20 | replacement: '\$1' 21 | - source_labels: ['__meta_consul_node'] 22 | regex: '(.*)' 23 | target_label: 'instance' 24 | replacement: '\$1' 25 | - source_labels: ['__meta_consul_tags'] 26 | regex: ',(dev|production|canary),' 27 | target_label: 'group' 28 | replacement: '\$1' 29 | " 30 | -------------------------------------------------------------------------------- /spring-boot-prometheus/Dockerfile: -------------------------------------------------------------------------------- 1 | #Builder container 2 | FROM maven:3.6.3-jdk-11-openj9 as builder 3 | WORKDIR /app 4 | COPY . /app 5 | RUN mvn package 6 | 7 | #Runtime container 8 | FROM openjdk:11.0.7-slim 9 | WORKDIR /app 10 | COPY --from=builder /app/target/in4it-springboot-demo.jar . 11 | CMD ["java", "-jar", "/app/in4it-springboot-demo.jar"] 12 | -------------------------------------------------------------------------------- /spring-boot-prometheus/README.md: -------------------------------------------------------------------------------- 1 | # Spring Boot demo app 2 | 3 | This is a demo application used in the Prometheus Udemy training, This demonstrates the use of spring boot 2 with micrometer and prometheus to generate app metrics and custom metrics. 4 | 5 | ## Demo 6 | 7 | ### Build 8 | `docker build -t spring-demo-app .` 9 | 10 | ### Run 11 | `docker run --rm -p 8080:8080 --name spring-demo-app spring-demo-app` 12 | 13 | ### Oneliner 14 | Build and Run the container via `docker build -t spring-demo-app . && docker run --rm -p 8080:8080 --name spring-demo-app spring-demo-app` 15 | 16 | ## Access 17 | ### App 18 | `http://localhost:8080/api/demo` 19 | `http://localhost:8080/api/delayed/demo` 20 | 21 | ### Prometheus metrics 22 | `http://localhost:8080/actuator/prometheus` 23 | 24 | 25 | ## Queries 26 | 27 | ### Requests/s globally 28 | `rate(http_server_requests_seconds_count[5m])` 29 | 30 | ### Requests/s for the demo app 31 | `rate(http_server_requests_seconds_count{uri="/api/demo",status="200"}[5m])` 32 | 33 | ### Custom metric 34 | `runCounter_total{application="prometheus-demo",job="spring_boot_demo_app"}` 35 | 36 | ### Requests duration for the delayed demo app 37 | `rate(http_server_requests_seconds_sum{uri="/api/delayed/demo",status="200"}[2m])` 38 | -------------------------------------------------------------------------------- /spring-boot-prometheus/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | io.in4it.monitoring.springboot 7 | in4it-java-prometheus-demo 8 | 0.0.1-SNAPSHOT 9 | jar 10 | 11 | in4it-java-prometheus-demo 12 | Demo of Prometheus for Spring Boot 13 | 14 | 15 | org.springframework.boot 16 | spring-boot-starter-parent 17 | 2.2.6.RELEASE 18 | 19 | 20 | 21 | 22 | UTF-8 23 | UTF-8 24 | 25 | 26 | 27 | 28 | org.springframework.boot 29 | spring-boot-starter-webflux 30 | 31 | 32 | 33 | org.springframework.boot 34 | spring-boot-starter-actuator 35 | 36 | 37 | io.micrometer 38 | micrometer-registry-prometheus 39 | runtime 40 | 41 | 42 | 43 | org.springframework.boot 44 | spring-boot-starter-test 45 | test 46 | 47 | 48 | 49 | 50 | 51 | in4it-springboot-demo 52 | 53 | 54 | org.springframework.boot 55 | spring-boot-maven-plugin 56 | 57 | 58 | 59 | 60 | 61 | -------------------------------------------------------------------------------- /spring-boot-prometheus/src/main/java/io/in4it/monitoring/springboot/Application.java: -------------------------------------------------------------------------------- 1 | package io.in4it.monitoring.springboot; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | 6 | @SpringBootApplication 7 | public class Application { 8 | 9 | public static void main(String[] args) { 10 | 11 | SpringApplication.run(Application.class, args); 12 | 13 | } 14 | 15 | } 16 | -------------------------------------------------------------------------------- /spring-boot-prometheus/src/main/java/io/in4it/monitoring/springboot/ApplicationController.java: -------------------------------------------------------------------------------- 1 | package io.in4it.monitoring.springboot; 2 | 3 | import org.slf4j.Logger; 4 | 5 | import io.micrometer.core.annotation.Timed; 6 | import io.micrometer.core.instrument.Counter; 7 | import io.micrometer.core.instrument.Metrics; 8 | import org.springframework.web.bind.annotation.GetMapping; 9 | import org.springframework.web.bind.annotation.RestController; 10 | 11 | import java.util.HashMap; 12 | import java.util.Map; 13 | import java.util.Random; 14 | 15 | @RestController 16 | public class ApplicationController { 17 | 18 | private static final Logger log = org.slf4j.LoggerFactory.getLogger(ApplicationController.class); 19 | private Counter runCounter = Metrics.counter("runCounter"); 20 | private Random random = new Random(); 21 | 22 | @GetMapping("/api/demo") 23 | @Timed 24 | // creates a Timer time series named http_server_requests which by default contains dimensions for the HTTP status of the response, HTTP method, exception type if the request fails, and the pre-variable substitution parameterized endpoint URI. 25 | public String apiUse() throws InterruptedException { 26 | runCounter.increment(); 27 | log.info("Hello world app accessed on /api/demo"); 28 | return "Hello world"; 29 | } 30 | 31 | @GetMapping("/api/delayed/demo") 32 | @Timed 33 | public String apiCountedUse() throws InterruptedException { 34 | int delay = (int) Math.sqrt(random.nextInt(400*200)); 35 | log.info("/api/delayed/demo called, waiting for: {}", delay); 36 | Thread.sleep(delay); 37 | return "Done waiting, waited: " + delay + " ms"; 38 | } 39 | 40 | } 41 | -------------------------------------------------------------------------------- /spring-boot-prometheus/src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | # To have support out of the box for dashboard: https://grafana.com/dashboards/4701 2 | management.metrics.tags.application=prometheus-demo 3 | 4 | management.endpoints.enabled-by-default=true 5 | management.endpoints.web.exposure.include=actuator,prometheus,health 6 | -------------------------------------------------------------------------------- /spring-boot-prometheus/src/test/java/io/in4it/monitoring/springboot/ApplicationTests.java: -------------------------------------------------------------------------------- 1 | package io.in4it.monitoring.springboot; 2 | 3 | import org.junit.Test; 4 | import org.junit.runner.RunWith; 5 | import org.springframework.boot.test.context.SpringBootTest; 6 | import org.springframework.test.context.junit4.SpringRunner; 7 | 8 | @RunWith(SpringRunner.class) 9 | @SpringBootTest 10 | public class ApplicationTests { 11 | 12 | @Test 13 | public void contextLoads() { 14 | } 15 | 16 | } 17 | -------------------------------------------------------------------------------- /use-cases/ec2-auto-discovery/lab.txt: -------------------------------------------------------------------------------- 1 | IAM role: 2 | AmazonEC2ReadOnlyAccess 3 | 4 | User data script: 5 | #!/usr/bin/env bash 6 | wget -O - https://raw.githubusercontent.com/in4it/prometheus-course/master/scripts/2-node-exporter.sh | bash 7 | 8 | Alter Prometheus config: 9 | /etc/prometheus/prometheus.yml 10 | 11 | - job_name: 'ec2_nodes' 12 | ec2_sd_configs: 13 | - region: eu-west-1 14 | access_key: PUT_THE_ACCESS_KEY_HERE 15 | secret_key: PUT_THE_SECRET_KEY_HERE 16 | port: 9100 17 | relabel_configs: 18 | - source_labels: [__meta_ec2_tag_Name] 19 | regex: Prometheus.* 20 | action: keep 21 | - source_labels: [__meta_ec2_public_ip] 22 | regex: '(.*)' 23 | target_label: __address__ 24 | replacement: '${1}:9100' 25 | 26 | Generate load: 27 | dd if=/dev/zero of=/dev/null 28 | 29 | Query grafana: 30 | 100 - (avg by (instance) (irate(node_cpu_seconds_total{job="ec2_nodes",mode="idle"}[5m])) * 100) --------------------------------------------------------------------------------