├── .gitignore ├── Dockerfile ├── README.md ├── alertmanager ├── config.yml └── data │ ├── nflog │ └── silences ├── app ├── main.py └── requirements.txt ├── docker-compose.yml ├── grafana ├── dashboards │ └── node-stat.json ├── datasources │ └── Prometheus.json └── setup.sh └── prometheus └── prometheus.yml /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | .vscode/ 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | .pytest_cache/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | local_settings.py 57 | db.sqlite3 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # Jupyter Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # SageMath parsed files 82 | *.sage.py 83 | 84 | # Environments 85 | .env 86 | .venv 87 | env/ 88 | venv/ 89 | ENV/ 90 | env.bak/ 91 | venv.bak/ 92 | 93 | # Spyder project settings 94 | .spyderproject 95 | .spyproject 96 | 97 | # Rope project settings 98 | .ropeproject 99 | 100 | # mkdocs documentation 101 | /site 102 | 103 | # mypy 104 | .mypy_cache/ 105 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.4-alpine 2 | ADD ./app /code 3 | WORKDIR /code 4 | RUN pip install -r requirements.txt 5 | CMD ["python", "main.py"] 6 | 7 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Prometheus tutorial 2 | 3 | This repository is meant to be a tutorial for Prometheus and Grafana. 4 | 5 | > Observation: To run, you have to install `Docker` and `Docker Compose`. 6 | 7 | To run, just use the command below: 8 | ``` 9 | docker-compose up --build 10 | ``` 11 | 12 | After that, you can access these services: 13 | - Grafana: http://localhost:3000 14 | - Prometheus: http://localhost:9090 15 | - Fake App: http://localhost:5000 -------------------------------------------------------------------------------- /alertmanager/config.yml: -------------------------------------------------------------------------------- 1 | route: 2 | repeat_interval: 2h 3 | receiver: email-1 4 | routes: 5 | - match: 6 | alertname: httpd_down 7 | receiver: email-1 8 | 9 | - match: 10 | alertname: nginx_down 11 | receiver: email-2 12 | 13 | receivers: 14 | - name: email-1 15 | email_configs: 16 | - to: 17 | from: 18 | smarthost: 19 | auth_username: "" 20 | auth_identity: "" 21 | auth_password: "" 22 | 23 | - name: email-2 24 | email_configs: 25 | - to: 26 | from: 27 | smarthost: 28 | auth_username: "" 29 | auth_identity: "" 30 | auth_password: "" 31 | -------------------------------------------------------------------------------- /alertmanager/data/nflog: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/evandroferreiras/prometheus_tutorial/0d6168fe71776f6e52c82cdcbea93f531e39e3be/alertmanager/data/nflog -------------------------------------------------------------------------------- /alertmanager/data/silences: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/evandroferreiras/prometheus_tutorial/0d6168fe71776f6e52c82cdcbea93f531e39e3be/alertmanager/data/silences -------------------------------------------------------------------------------- /app/main.py: -------------------------------------------------------------------------------- 1 | from prometheus_client import start_http_server, Summary, Counter, Histogram 2 | import random 3 | import time 4 | 5 | 6 | REQUEST_TIME_HIST = Histogram('request_latency_seconds', 'Histogram') 7 | 8 | 9 | COUNTER = Counter('requests_greater_than_half_sec_total', 10 | 'number of waits greater than half a second') 11 | 12 | 13 | @REQUEST_TIME_HIST.time() 14 | def process_request(t): 15 | """A dummy function that takes some time.""" 16 | time.sleep(t) 17 | 18 | 19 | if __name__ == '__main__': 20 | start_http_server(5000) 21 | while True: 22 | rnd = random.random() 23 | if rnd > 0.5: 24 | COUNTER.inc() 25 | process_request(rnd) 26 | -------------------------------------------------------------------------------- /app/requirements.txt: -------------------------------------------------------------------------------- 1 | prometheus-client -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | 3 | networks: 4 | monitor-net: 5 | driver: bridge 6 | 7 | volumes: 8 | prometheus_data: {} 9 | grafana_data: {} 10 | 11 | services: 12 | 13 | prometheus: 14 | image: prom/prometheus:latest 15 | container_name: prometheus 16 | volumes: 17 | - ./prometheus/:/etc/prometheus/ 18 | - prometheus_data:/prometheus 19 | command: 20 | - '--config.file=/etc/prometheus/prometheus.yml' 21 | - '--storage.tsdb.path=/prometheus' 22 | - '--web.console.libraries=/etc/prometheus/console_libraries' 23 | - '--web.console.templates=/etc/prometheus/consoles' 24 | - '--storage.tsdb.retention=200h' 25 | - '--web.enable-lifecycle' 26 | # restart: unless-stopped 27 | expose: 28 | - 9090 29 | ports: 30 | - "9090:9090" 31 | networks: 32 | - monitor-net 33 | 34 | nodeexporter: 35 | image: prom/node-exporter:latest 36 | container_name: nodeexporter 37 | user: root 38 | privileged: true 39 | volumes: 40 | - /proc:/host/proc:ro 41 | - /sys:/host/sys:ro 42 | - /:/rootfs:ro 43 | command: 44 | - '--path.procfs=/host/proc' 45 | - '--path.sysfs=/host/sys' 46 | - '--collector.filesystem.ignored-mount-points=^/(sys|proc|dev|host|etc)($$|/)' 47 | restart: unless-stopped 48 | expose: 49 | - 9100 50 | ports: 51 | - "9100:9100" 52 | networks: 53 | - monitor-net 54 | grafana: 55 | image: grafana/grafana:latest 56 | container_name: grafana 57 | volumes: 58 | - grafana_data:/var/lib/grafana 59 | - ./grafana/datasources:/etc/grafana/datasources 60 | - ./grafana/dashboards:/etc/grafana/dashboards 61 | - ./grafana/setup.sh:/setup.sh 62 | entrypoint: /setup.sh 63 | environment: 64 | - GF_SECURITY_ADMIN_USER=${ADMIN_USER:-admin} 65 | - GF_SECURITY_ADMIN_PASSWORD=${ADMIN_PASSWORD:-admin} 66 | - GF_USERS_ALLOW_SIGN_UP=false 67 | restart: unless-stopped 68 | expose: 69 | - 3000 70 | ports: 71 | - 3000:3000 72 | networks: 73 | - monitor-net 74 | web: 75 | build: . 76 | expose: 77 | - 3000 78 | ports: 79 | - "5000:5000" 80 | networks: 81 | - monitor-net -------------------------------------------------------------------------------- /grafana/dashboards/node-stat.json: -------------------------------------------------------------------------------- 1 | { 2 | "__inputs": [ 3 | { 4 | "name": "Prometheus", 5 | "label": "", 6 | "description": "", 7 | "type": "datasource", 8 | "pluginId": "prometheus", 9 | "pluginName": "Prometheus" 10 | } 11 | ], 12 | "__requires": [ 13 | { 14 | "type": "panel", 15 | "id": "graph", 16 | "name": "Graph", 17 | "version": "" 18 | }, 19 | { 20 | "type": "panel", 21 | "id": "table", 22 | "name": "Table", 23 | "version": "" 24 | }, 25 | { 26 | "type": "grafana", 27 | "id": "grafana", 28 | "name": "Grafana", 29 | "version": "3.1.1" 30 | }, 31 | { 32 | "type": "datasource", 33 | "id": "prometheus", 34 | "name": "Prometheus", 35 | "version": "1.0.0" 36 | } 37 | ], 38 | "id": null, 39 | "title": "Host Stats - Prometheus Node Exporter", 40 | "tags": [], 41 | "style": "dark", 42 | "timezone": "browser", 43 | "editable": true, 44 | "hideControls": false, 45 | "sharedCrosshair": false, 46 | "rows": [ 47 | { 48 | "collapse": false, 49 | "editable": true, 50 | "height": "250px", 51 | "panels": [ 52 | { 53 | "aliasColors": {}, 54 | "bars": false, 55 | "datasource": "prometheus", 56 | "editable": true, 57 | "error": false, 58 | "fill": 1, 59 | "grid": { 60 | "threshold1": null, 61 | "threshold1Color": "rgba(216, 200, 27, 0.27)", 62 | "threshold2": null, 63 | "threshold2Color": "rgba(234, 112, 112, 0.22)" 64 | }, 65 | "id": 1, 66 | "isNew": true, 67 | "legend": { 68 | "avg": false, 69 | "current": false, 70 | "max": false, 71 | "min": false, 72 | "show": true, 73 | "total": false, 74 | "values": false 75 | }, 76 | "lines": true, 77 | "linewidth": 2, 78 | "links": [], 79 | "nullPointMode": "connected", 80 | "percentage": false, 81 | "pointradius": 5, 82 | "points": false, 83 | "renderer": "flot", 84 | "seriesOverrides": [], 85 | "span": 6, 86 | "stack": true, 87 | "steppedLine": false, 88 | "targets": [ 89 | { 90 | "expr": "avg without (cpu)(irate(node_cpu{job=\"node\",instance=\"$instance\",mode!=\"idle\"}[5m]))", 91 | "interval": "", 92 | "intervalFactor": 2, 93 | "legendFormat": "{{mode}}", 94 | "metric": "node_cpu", 95 | "refId": "A", 96 | "step": 60 97 | } 98 | ], 99 | "timeFrom": null, 100 | "timeShift": null, 101 | "title": "CPU", 102 | "tooltip": { 103 | "shared": true, 104 | "value_type": "individual", 105 | "sort": 0, 106 | "msResolution": false 107 | }, 108 | "type": "graph", 109 | "yaxes": [ 110 | { 111 | "show": true, 112 | "min": null, 113 | "max": 1, 114 | "logBase": 1, 115 | "format": "percentunit" 116 | }, 117 | { 118 | "show": true, 119 | "min": null, 120 | "max": null, 121 | "logBase": 1, 122 | "format": "short" 123 | } 124 | ], 125 | "xaxis": { 126 | "show": true 127 | } 128 | }, 129 | { 130 | "aliasColors": {}, 131 | "bars": false, 132 | "datasource": "prometheus", 133 | "editable": true, 134 | "error": false, 135 | "fill": 1, 136 | "grid": { 137 | "threshold1": null, 138 | "threshold1Color": "rgba(216, 200, 27, 0.27)", 139 | "threshold2": null, 140 | "threshold2Color": "rgba(234, 112, 112, 0.22)" 141 | }, 142 | "id": 2, 143 | "isNew": true, 144 | "legend": { 145 | "alignAsTable": false, 146 | "avg": false, 147 | "current": false, 148 | "max": false, 149 | "min": false, 150 | "rightSide": false, 151 | "show": true, 152 | "total": false, 153 | "values": false 154 | }, 155 | "lines": true, 156 | "linewidth": 2, 157 | "links": [], 158 | "minSpan": null, 159 | "nullPointMode": "connected", 160 | "percentage": false, 161 | "pointradius": 5, 162 | "points": false, 163 | "renderer": "flot", 164 | "repeat": null, 165 | "seriesOverrides": [], 166 | "span": 6, 167 | "stack": true, 168 | "steppedLine": false, 169 | "targets": [ 170 | { 171 | "expr": "node_memory_MemTotal{job='node',instance='$instance'} - node_memory_MemFree{job='node',instance='$instance'} - node_memory_Buffers{job='node',instance='$instance'} - node_memory_Cached{job='node',instance='$instance'}", 172 | "intervalFactor": 2, 173 | "legendFormat": "Used", 174 | "refId": "A", 175 | "step": 60 176 | }, 177 | { 178 | "expr": "node_memory_Buffers{job='node',instance='$instance'}", 179 | "intervalFactor": 2, 180 | "legendFormat": "Buffers", 181 | "refId": "B", 182 | "step": 60 183 | }, 184 | { 185 | "expr": "node_memory_Cached{job='node',instance='$instance'}", 186 | "intervalFactor": 2, 187 | "legendFormat": "Cached", 188 | "refId": "D", 189 | "step": 60 190 | }, 191 | { 192 | "expr": "node_memory_MemFree{job='node',instance='$instance'}", 193 | "hide": false, 194 | "intervalFactor": 2, 195 | "legendFormat": "Free", 196 | "refId": "C", 197 | "step": 60 198 | } 199 | ], 200 | "timeFrom": null, 201 | "timeShift": null, 202 | "title": "Memory", 203 | "tooltip": { 204 | "shared": true, 205 | "value_type": "individual", 206 | "sort": 0, 207 | "msResolution": false 208 | }, 209 | "type": "graph", 210 | "yaxes": [ 211 | { 212 | "show": true, 213 | "min": 0, 214 | "max": null, 215 | "logBase": 1, 216 | "format": "bytes", 217 | "label": "" 218 | }, 219 | { 220 | "show": true, 221 | "min": null, 222 | "max": null, 223 | "logBase": 1, 224 | "format": "short" 225 | } 226 | ], 227 | "xaxis": { 228 | "show": true 229 | } 230 | } 231 | ], 232 | "title": "Row" 233 | }, 234 | { 235 | "collapse": false, 236 | "editable": true, 237 | "height": "250px", 238 | "panels": [ 239 | { 240 | "aliasColors": {}, 241 | "bars": false, 242 | "datasource": "prometheus", 243 | "editable": true, 244 | "error": false, 245 | "fill": 1, 246 | "grid": { 247 | "threshold1": null, 248 | "threshold1Color": "rgba(216, 200, 27, 0.27)", 249 | "threshold2": null, 250 | "threshold2Color": "rgba(234, 112, 112, 0.22)" 251 | }, 252 | "id": 3, 253 | "isNew": true, 254 | "legend": { 255 | "avg": false, 256 | "current": false, 257 | "max": false, 258 | "min": false, 259 | "show": true, 260 | "total": false, 261 | "values": false 262 | }, 263 | "lines": true, 264 | "linewidth": 2, 265 | "links": [], 266 | "nullPointMode": "connected", 267 | "percentage": false, 268 | "pointradius": 5, 269 | "points": false, 270 | "renderer": "flot", 271 | "seriesOverrides": [], 272 | "span": 4, 273 | "stack": false, 274 | "steppedLine": false, 275 | "targets": [ 276 | { 277 | "expr": "irate(node_disk_io_time_ms{job='node',instance='$instance',device!~'^(md\\\\d+$|dm-)'}[5m]) / 1000", 278 | "intervalFactor": 2, 279 | "legendFormat": "{{device}}", 280 | "refId": "A", 281 | "step": 120 282 | } 283 | ], 284 | "timeFrom": null, 285 | "timeShift": null, 286 | "title": "Disk I/O Utilisation", 287 | "tooltip": { 288 | "shared": true, 289 | "value_type": "cumulative", 290 | "sort": 0, 291 | "msResolution": false 292 | }, 293 | "type": "graph", 294 | "yaxes": [ 295 | { 296 | "show": true, 297 | "min": null, 298 | "max": 1, 299 | "logBase": 1, 300 | "format": "percentunit" 301 | }, 302 | { 303 | "show": true, 304 | "min": null, 305 | "max": null, 306 | "logBase": 1, 307 | "format": "short" 308 | } 309 | ], 310 | "xaxis": { 311 | "show": true 312 | } 313 | }, 314 | { 315 | "aliasColors": {}, 316 | "bars": false, 317 | "datasource": "prometheus", 318 | "editable": true, 319 | "error": false, 320 | "fill": 1, 321 | "grid": { 322 | "threshold1": null, 323 | "threshold1Color": "rgba(216, 200, 27, 0.27)", 324 | "threshold2": null, 325 | "threshold2Color": "rgba(234, 112, 112, 0.22)" 326 | }, 327 | "id": 4, 328 | "isNew": true, 329 | "legend": { 330 | "avg": false, 331 | "current": false, 332 | "max": false, 333 | "min": false, 334 | "show": true, 335 | "total": false, 336 | "values": false 337 | }, 338 | "lines": true, 339 | "linewidth": 2, 340 | "links": [], 341 | "nullPointMode": "connected", 342 | "percentage": false, 343 | "pointradius": 5, 344 | "points": false, 345 | "renderer": "flot", 346 | "seriesOverrides": [], 347 | "span": 4, 348 | "stack": false, 349 | "steppedLine": false, 350 | "targets": [ 351 | { 352 | "expr": "1 - node_filesystem_free{job='node',instance='$instance',fstype!='rootfs',mountpoint!~'/(run|var).*',mountpoint!=''} / node_filesystem_size{job='node',instance='$instance'}", 353 | "interval": "", 354 | "intervalFactor": 2, 355 | "legendFormat": "{{mountpoint}}", 356 | "refId": "A", 357 | "step": 120 358 | } 359 | ], 360 | "timeFrom": null, 361 | "timeShift": null, 362 | "title": "Filesystem Fullness", 363 | "tooltip": { 364 | "shared": true, 365 | "value_type": "cumulative", 366 | "sort": 0, 367 | "msResolution": false 368 | }, 369 | "type": "graph", 370 | "yaxes": [ 371 | { 372 | "show": true, 373 | "min": 0, 374 | "max": 1, 375 | "logBase": 1, 376 | "format": "percentunit" 377 | }, 378 | { 379 | "show": true, 380 | "min": null, 381 | "max": null, 382 | "logBase": 1, 383 | "format": "short" 384 | } 385 | ], 386 | "xaxis": { 387 | "show": true 388 | } 389 | }, 390 | { 391 | "columns": [ 392 | { 393 | "text": "Current", 394 | "value": "current" 395 | } 396 | ], 397 | "editable": true, 398 | "error": false, 399 | "fontSize": "100%", 400 | "hideTimeOverride": true, 401 | "id": 5, 402 | "isNew": true, 403 | "links": [], 404 | "pageSize": null, 405 | "scroll": true, 406 | "showHeader": true, 407 | "sort": { 408 | "col": 0, 409 | "desc": true 410 | }, 411 | "span": 4, 412 | "styles": [ 413 | { 414 | "dateFormat": "YYYY-MM-DD HH:mm:ss", 415 | "pattern": "Time", 416 | "type": "date" 417 | }, 418 | { 419 | "colorMode": null, 420 | "colors": [ 421 | "rgba(245, 54, 54, 0.9)", 422 | "rgba(237, 129, 40, 0.89)", 423 | "rgba(50, 172, 45, 0.97)" 424 | ], 425 | "decimals": 0, 426 | "pattern": "/.*/", 427 | "thresholds": [], 428 | "type": "number", 429 | "unit": "s" 430 | } 431 | ], 432 | "targets": [ 433 | { 434 | "expr": "(node_filesystem_size{job='node',instance='$instance'} - node_filesystem_free{job='node',instance='$instance'}) / deriv(node_filesystem_free{job='node',instance='$instance',fstype!='rootfs',mountpoint!~'/(run|var).*',mountpoint!=''}[3d]) > 0", 435 | "interval": "", 436 | "intervalFactor": 2, 437 | "legendFormat": "{{mountpoint}}", 438 | "refId": "A", 439 | "step": 20 440 | } 441 | ], 442 | "timeFrom": "1h", 443 | "timeShift": null, 444 | "title": "Filesystem Fill Up Time", 445 | "transform": "timeseries_aggregations", 446 | "type": "table" 447 | } 448 | ], 449 | "title": "New row" 450 | } 451 | ], 452 | "time": { 453 | "from": "now-6h", 454 | "to": "now" 455 | }, 456 | "timepicker": { 457 | "now": true, 458 | "refresh_intervals": [ 459 | "5s", 460 | "10s", 461 | "30s", 462 | "1m", 463 | "5m", 464 | "15m", 465 | "30m", 466 | "1h", 467 | "2h", 468 | "1d" 469 | ], 470 | "time_options": [ 471 | "5m", 472 | "15m", 473 | "1h", 474 | "6h", 475 | "12h", 476 | "24h", 477 | "2d", 478 | "7d", 479 | "30d" 480 | ] 481 | }, 482 | "templating": { 483 | "list": [ 484 | { 485 | "allFormat": "glob", 486 | "current": {}, 487 | "datasource": "prometheus", 488 | "hideLabel": false, 489 | "includeAll": false, 490 | "label": "Machine", 491 | "multi": false, 492 | "multiFormat": "glob", 493 | "name": "instance", 494 | "options": [], 495 | "query": "up{job=\"node\"}", 496 | "refresh": 1, 497 | "regex": ".*instance=\"(.*?)\".*", 498 | "type": "query", 499 | "hide": 0 500 | } 501 | ] 502 | }, 503 | "annotations": { 504 | "list": [] 505 | }, 506 | "schemaVersion": 12, 507 | "version": 12, 508 | "links": [], 509 | "gnetId": 718, 510 | "description": "Basic host stats: CPU, Memory Usage, Disk Utilisation, Filesystem usage and Predicted time to filesystems filling" 511 | } 512 | -------------------------------------------------------------------------------- /grafana/datasources/Prometheus.json: -------------------------------------------------------------------------------- 1 | { 2 | "name":"prometheus", 3 | "type":"prometheus", 4 | "url":"http://prometheus:9090", 5 | "access":"proxy", 6 | "basicAuth":false 7 | } 8 | -------------------------------------------------------------------------------- /grafana/setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Taken from https://github.com/grafana/grafana-docker/issues/74 4 | 5 | # Script to configure grafana datasources and dashboards. 6 | # Intended to be run before grafana entrypoint... 7 | # Image: grafana/grafana:4.1.2 8 | # ENTRYPOINT [\"/run.sh\"]" 9 | 10 | GRAFANA_URL=${GRAFANA_URL:-http://$GF_SECURITY_ADMIN_USER:$GF_SECURITY_ADMIN_PASSWORD@localhost:3000} 11 | #GRAFANA_URL=http://grafana-plain.k8s.playground1.aws.ad.zopa.com 12 | DATASOURCES_PATH=${DATASOURCES_PATH:-/etc/grafana/datasources} 13 | DASHBOARDS_PATH=${DASHBOARDS_PATH:-/etc/grafana/dashboards} 14 | 15 | # Generic function to call the Vault API 16 | grafana_api() { 17 | local verb=$1 18 | local url=$2 19 | local params=$3 20 | local bodyfile=$4 21 | local response 22 | local cmd 23 | 24 | cmd="curl -L -s --fail -H \"Accept: application/json\" -H \"Content-Type: application/json\" -X ${verb} -k ${GRAFANA_URL}${url}" 25 | [[ -n "${params}" ]] && cmd="${cmd} -d \"${params}\"" 26 | [[ -n "${bodyfile}" ]] && cmd="${cmd} --data @${bodyfile}" 27 | echo "Running ${cmd}" 28 | eval ${cmd} || return 1 29 | return 0 30 | } 31 | 32 | wait_for_api() { 33 | while ! grafana_api GET /api/user/preferences 34 | do 35 | sleep 5 36 | done 37 | } 38 | 39 | install_datasources() { 40 | local datasource 41 | 42 | for datasource in ${DATASOURCES_PATH}/*.json 43 | do 44 | if [[ -f "${datasource}" ]]; then 45 | echo "Installing datasource ${datasource}" 46 | if grafana_api POST /api/datasources "" "${datasource}"; then 47 | echo "installed ok" 48 | else 49 | echo "install failed" 50 | fi 51 | fi 52 | done 53 | } 54 | 55 | install_dashboards() { 56 | local dashboard 57 | 58 | for dashboard in ${DASHBOARDS_PATH}/*.json 59 | do 60 | if [[ -f "${dashboard}" ]]; then 61 | echo "Installing dashboard ${dashboard}" 62 | 63 | echo "{\"dashboard\": `cat $dashboard`}" > "${dashboard}.wrapped" 64 | 65 | if grafana_api POST /api/dashboards/db "" "${dashboard}.wrapped"; then 66 | echo "installed ok" 67 | else 68 | echo "install failed" 69 | fi 70 | 71 | rm "${dashboard}.wrapped" 72 | fi 73 | done 74 | } 75 | 76 | configure_grafana() { 77 | wait_for_api 78 | install_datasources 79 | install_dashboards 80 | } 81 | 82 | echo "Running configure_grafana in the background..." 83 | configure_grafana & 84 | /run.sh 85 | exit 0 -------------------------------------------------------------------------------- /prometheus/prometheus.yml: -------------------------------------------------------------------------------- 1 | global: 2 | scrape_interval: 5s 3 | evaluation_interval: 5s 4 | 5 | external_labels: 6 | monitor: 'prometheus-grafana-exporter' 7 | 8 | # Alertmanager configuration 9 | alerting: 10 | alertmanagers: 11 | - static_configs: 12 | - targets: 13 | - alertmanager:9093 14 | 15 | rule_files: 16 | - "alert.rules" 17 | 18 | scrape_configs: 19 | 20 | - job_name: 'prometheus' 21 | scrape_interval: 10s 22 | static_configs: 23 | - targets: ['localhost:9090'] 24 | 25 | - job_name: 'node' 26 | metrics_path: /metrics 27 | scrape_interval: 10s 28 | static_configs: 29 | - targets: ['nodeexporter:9100'] 30 | - job_name: 'app' 31 | metrics_path: / 32 | static_configs: 33 | - targets: ['web:5000'] 34 | 35 | - job_name: 'httpd' 36 | metrics_path: /probe 37 | params: 38 | module: [http_2xx] 39 | static_configs: 40 | - targets: 41 | - http://httpd:80 42 | relabel_configs: 43 | - source_labels: [__address__] 44 | target_label: __param_target 45 | - source_labels: [__param_target] 46 | target_label: instance 47 | - target_label: __address__ 48 | replacement: blackbox:9115 49 | 50 | - job_name: 'nginx' 51 | metrics_path: /probe 52 | params: 53 | module: [http_2xx] 54 | static_configs: 55 | - targets: 56 | - http://nginx:80 57 | relabel_configs: 58 | - source_labels: [__address__] 59 | target_label: __param_target 60 | - source_labels: [__param_target] 61 | target_label: instance 62 | - target_label: __address__ 63 | replacement: blackbox:9115 --------------------------------------------------------------------------------