├── .gitattributes ├── .gitignore ├── LICENSE ├── Makefile ├── README.md ├── assets ├── grafana.png ├── mkt_loki_logs.png ├── mktxp_1.png ├── mktxp_2.png ├── mktxp_3.png └── prometheus.png ├── blackbox └── blackbox.yml ├── docker-compose.yml ├── grafana └── provisioning │ ├── dashboards │ ├── mikrotik.yml │ ├── mikrotik │ │ ├── mikrotik_loki_logs.json │ │ └── mikrotik_mktxp_exporter.json │ ├── system.yml │ └── system │ │ ├── system_grafana_internals.json │ │ └── system_prometheus_stats.json │ └── datasources │ ├── loki.yaml │ └── prometheus.yaml ├── loki ├── loki-config-2.x.yml └── loki-config.yml ├── mktxp ├── _mktxp.conf └── mktxp.conf ├── prometheus └── prometheus.yml ├── promtail ├── promtail-config-fs.yml └── promtail-config.yml └── syslog-ng ├── logs └── .gitkeep ├── syslog-ng-fs.conf └── syslog-ng.conf /.gitattributes: -------------------------------------------------------------------------------- 1 | *.yml linguist-language=YAML 2 | *.json linguist-language=JSON 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | 131 | # PyCharm IDE 132 | .idea/ 133 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Gennadiy 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .DEFAULT_GOAL := run 2 | 3 | clean: 4 | docker compose down 5 | docker rm -f $(docker ps -a -q) 6 | docker volume rm "$(docker volume ls -q)" 7 | 8 | run: 9 | docker compose up 10 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## Описание 2 | 3 | Этот проект представляет собой готовый стек для мониторинга устройств Mikrotik через API на основе Docker, который использует Grafana, Prometheus, [Blackbox Exporter](https://github.com/prometheus/blackbox_exporter) и [MKTXP Exporter](https://github.com/akpw/mktxp). Так же он включает в себя централизованную обработку журналов Mikrotik на основе предварительно настроенного стека [syslog-ng](https://www.syslog-ng.com/) / [promtail](https://grafana.com/docs/loki/latest/clients/promtail/) / [Loki](https://grafana.com/docs/loki/latest). 4 | 5 | ## Функции 6 | - Мониторинг работы системы 7 | - утилизация диска 8 | - загрузка процессора 9 | - утилизация памяти 10 | - внешний IP-адрес (IPv4 и IPv6) 11 | - аптайм системы 12 | - температура 13 | - контроль напряжения 14 | - Мониторинг интерфейса 15 | - трафик (bit/s) 16 | - трафик (packets/s) 17 | - пропускная способность канала 18 | - Мониторинг latency 19 | - настраиваемые пинги ICMP и/или UDP 20 | - потеря пакетов 21 | - DHCP-мониторинг 22 | - активная аренда 23 | - MAC-адреса, имена хостов, ip-адреса 24 | - Мониторинг сети 25 | - маршруты 26 | - ошибки интерфейсов 27 | - состояния интерфейов 28 | - cостояние PoE 29 | - BGP 30 | - Мониторинг сессии BGP (настраивается индивидуально) 31 | - Мониторинг Firewall'а 32 | - rules traffic 33 | - logged rules traffic 34 | - Ipv4 & IPv6 35 | - активные пользователи 36 | - Мониторинг беспроводных интерфейсов 37 | - noise floor - уровень шума 38 | - TxCCQ 39 | - устройства клиентов 40 | - количество клиентов 41 | - трафик 42 | - signal strength - сила сигнала 43 | - signal to noise ratio - отношение сигнал/шум 44 | - Netwatch 45 | - Мониторинг CAPsMAN 46 | - remote CAPS 47 | - registrations 48 | - clients 49 | - frequencies 50 | - signal strength 51 | - traffic 52 | ## Требования 53 | - Роутер Mikrotik под управлением RouterOS 7.x.x 54 | - Ubuntu Server 24.04 (протестировано мной) 55 | ## Демо изображения дашбордов 56 | 57 | **Mikrotik MKTXP Monitoring** 58 | 59 | ![mktxp_1](./assets/mktxp_1.png) 60 | 61 |
Показать больше изображений 62 | 63 | ![mktxp_2](./assets/mktxp_2.png) 64 | ![mktxp_3](./assets/mktxp_3.png) 65 | **Mikrotik Loki Logs** 66 | ![mkt_loki_logs](./assets/mkt_loki_logs.png) 67 | **Grafana Internals** 68 | ![grafana](./assets/grafana.png) 69 | **Prometheus 2.0 Stats** 70 | ![prometheus](./assets/prometheus.png) 71 | 72 |
73 | 74 | ## Установка и начало работы 75 | 76 | ### Подготовка Mikrotik 77 | 78 | Первым делом нужно подготовить наш роутер. 79 | 80 | Создаем группу которая будет иметь read-only доступ к API 81 | 82 | ```bash 83 | /user group add name=prometheus policy=api,read 84 | ``` 85 | 86 | Создаем пользователя в этой группе: 87 | 88 | ```bash 89 | /user add name=prometheus group=prometheus password=prometheus_user_password 90 | ``` 91 | 92 | Для для получения и обработки логов с нескольких устройств Mikrotik RouterOS в централизованном месте, нам нужно настроить наши устройства Mikrotik для отправки своих логов на указанный целевой сервер логов. 93 | 94 | Настроим `logging action` (замените XX.XX.XX.XX на IP-адрес вашего сервера): 95 | 96 | ```bash 97 | /system logging action 98 | set remote bsd-syslog=yes name=remote remote=XX.XX.XX.XX remote-port=514 src-address=0.0.0.0 syslog-facility=local0 syslog-severity=auto target=remote 99 | ``` 100 | Далее изменяем соответствующие разделы логов для использования `logging action`: 101 | 102 | ```bash 103 | /system logging 104 | set 0 action=remote prefix=:Info 105 | set 1 action=remote prefix=:Error 106 | set 2 action=remote prefix=:Warning 107 | set 3 action=remote prefix=:Critical 108 | 109 | add action=remote disabled=no prefix=:Firewall topics=firewall 110 | add action=remote disabled=no prefix=:Account topics=account 111 | add action=remote disabled=no prefix=:Caps topics=caps 112 | add action=remote disabled=no prefix=:Wireles topics=wireless 113 | ``` 114 | 115 | При необходимости вы можете расширить приведенный выше список, следуя [документации](https://help.mikrotik.com/docs/display/ROS/Log) RouterOS, предоставленной Mikrotik. 116 | 117 | ### Подготовка сервера 118 | 119 | Устанавливаем Docker 120 | 121 | ```bash 122 | bash <(curl -sSL https://get.docker.com) 123 | ``` 124 | 125 | Добавляем текущего пользователя в группу `docker` 126 | 127 | ```bash 128 | sudo usermod -aG docker ${USER} 129 | ``` 130 | 131 | После выполнения команды выйдите из системы и войдите снова, чтобы изменения вступили в силу. Или используйте команду: 132 | 133 | ```bash 134 | newgrp docker 135 | ``` 136 | 137 | Клонируем репозиторий и переходим в директорию `mkt_monitoring`: 138 | 139 | ```bash 140 | git clone https://github.com/metgen/mkt_monitoring.git 141 | cd mkt_monitoring 142 | ``` 143 | 144 | ### Конфигурация экспортера MKTXP 145 | 146 | Отредактируем основной файл конфигурации mktxp, добавив IP-адрес вашего устройства Mikrotik и информацию для аутентификации: 147 | 148 | ```bash 149 | nano mktxp/mktxp.conf 150 | ``` 151 | 152 | >Вы можете добавить в мониторинг несколько устройств Mikrotik. Просто добавьте его в `mktxp/mktxp.conf`как предыдущее. 153 | 154 | ### Настраиваем мониторинг задержек 155 | 156 | В этом проекте используется Blackbox Exporter для измерения задержек сети. По умолчанию имеет три цели: 157 | 158 | - 1.1.1.1 (Cloudflare DNS) 159 | - 8.8.8.8 (Google DNS) 160 | - 77.88.8.8 (Yandex DNS) 161 | 162 | Их можно изменить в файле конфигурации Prometheus: 163 | 164 | ```bash 165 | nano prometheus/prometheus.yml 166 | ``` 167 | 168 | ### Запускаем docker-compose 169 | 170 | Теперь все должно быть готово и можно переходить к запкуску контейнеров: 171 | 172 | ```bash 173 | docker-compose up -d 174 | ``` 175 | Как только контейнеры будут запущены, откройте в своем веб-браузере Grafana по адресу `http://server_ip:3003` 176 | 177 | Логин и пароль по умолчанию `admin`:`admin` 178 | 179 | ## Ресурсы 180 | 181 | Проект собран на основе двух репозиториев: 182 | 183 | - [akpw/mktxp-stack](https://github.com/akpw/mktxp-stack) 184 | - [M0r13n/mikrotik_monitoring](https://github.com/M0r13n/mikrotik_monitoring) 185 | 186 | ## Почему используется API вместо SNMP для мониторинга? 187 | 188 | Мониторинг на основе SNMP очень медленный. Обходы по SNMP медленны и нагружают процессор. API работает намного быстрее и меньше нагружает процессор отслеживаемого устройства. 189 | 190 | Кроме того, API предлагает большую гибкость. Любую команду можно выполнить в RouterOS через API. Таким образом можно собирать сложные метрики. 191 | -------------------------------------------------------------------------------- /assets/grafana.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/metgen/mkt_monitoring/9934a9549a8bbffe3a4734553512801e5b4a4f8c/assets/grafana.png -------------------------------------------------------------------------------- /assets/mkt_loki_logs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/metgen/mkt_monitoring/9934a9549a8bbffe3a4734553512801e5b4a4f8c/assets/mkt_loki_logs.png -------------------------------------------------------------------------------- /assets/mktxp_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/metgen/mkt_monitoring/9934a9549a8bbffe3a4734553512801e5b4a4f8c/assets/mktxp_1.png -------------------------------------------------------------------------------- /assets/mktxp_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/metgen/mkt_monitoring/9934a9549a8bbffe3a4734553512801e5b4a4f8c/assets/mktxp_2.png -------------------------------------------------------------------------------- /assets/mktxp_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/metgen/mkt_monitoring/9934a9549a8bbffe3a4734553512801e5b4a4f8c/assets/mktxp_3.png -------------------------------------------------------------------------------- /assets/prometheus.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/metgen/mkt_monitoring/9934a9549a8bbffe3a4734553512801e5b4a4f8c/assets/prometheus.png -------------------------------------------------------------------------------- /blackbox/blackbox.yml: -------------------------------------------------------------------------------- 1 | modules: 2 | icmp_ttl5: 3 | prober: icmp 4 | timeout: 5s 5 | icmp: 6 | ttl: 10 -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | networks: 2 | monitoring: {} 3 | 4 | volumes: 5 | prometheus_storage: {} 6 | grafana_storage: {} 7 | 8 | services: 9 | mktxp: 10 | container_name: mktxp 11 | image: ghcr.io/akpw/mktxp:latest 12 | user: root 13 | volumes: 14 | - './mktxp/:/root/mktxp/' 15 | networks: 16 | monitoring: null 17 | restart: unless-stopped 18 | 19 | grafana: 20 | container_name: grafana 21 | environment: 22 | GF_SECURITY_ADMIN_USER: ${ADMIN_USER:-admin} 23 | GF_SECURITY_ADMIN_PASSWORD: ${ADMIN_PASSWORD:-admin} 24 | GF_USERS_ALLOW_SIGN_UP: "false" 25 | GF_USERS_DEFAULT_THEME: "dark" 26 | GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH: /etc/grafana/provisioning/dashboards/mikrotik/mikrotik_mktxp_exporter.json 27 | image: grafana/grafana:latest 28 | networks: 29 | monitoring: null 30 | ports: 31 | - protocol: tcp 32 | published: 3003 33 | target: 3000 34 | restart: unless-stopped 35 | volumes: 36 | - grafana_storage:/var/lib/grafana 37 | - ./grafana/provisioning/datasources:/etc/grafana/provisioning/datasources:ro 38 | - ./grafana/provisioning/dashboards:/etc/grafana/provisioning/dashboards:ro 39 | 40 | loki: 41 | command: -config.file=/etc/loki/loki-config.yml 42 | container_name: loki 43 | image: grafana/loki:latest 44 | networks: 45 | monitoring: null 46 | ports: 47 | - protocol: tcp 48 | published: 3100 49 | target: 3100 50 | restart: unless-stopped 51 | volumes: 52 | - ./loki/loki-config.yml:/etc/loki/loki-config.yml 53 | 54 | prometheus: 55 | command: 56 | - --config.file=/etc/prometheus/prometheus.yml 57 | - --storage.tsdb.path=/prometheus 58 | - --storage.tsdb.retention.time=1y 59 | - --web.enable-admin-api 60 | - --web.enable-lifecycle 61 | container_name: prometheus 62 | image: prom/prometheus:latest 63 | networks: 64 | monitoring: null 65 | ports: 66 | - published: 9090 67 | target: 9090 68 | restart: unless-stopped 69 | volumes: 70 | - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro 71 | - prometheus_storage:/prometheus 72 | 73 | promtail: 74 | command: -config.file=/etc/promtail/promtail-config.yml 75 | container_name: promtail 76 | image: grafana/promtail:latest 77 | networks: 78 | monitoring: null 79 | ports: 80 | - protocol: tcp 81 | published: 1514 82 | target: 1514 83 | - protocol: tcp 84 | published: 9080 85 | target: 9080 86 | restart: unless-stopped 87 | volumes: 88 | - ./promtail/promtail-config.yml:/etc/promtail/promtail-config.yml:ro 89 | 90 | syslog-ng: 91 | command: -edv 92 | container_name: syslog-ng 93 | depends_on: 94 | - promtail 95 | image: balabit/syslog-ng:latest 96 | networks: 97 | monitoring: null 98 | ports: 99 | - protocol: udp 100 | published: 514 101 | target: 514 102 | - protocol: tcp 103 | published: 601 104 | target: 601 105 | restart: unless-stopped 106 | volumes: 107 | - ./syslog-ng/syslog-ng.conf:/etc/syslog-ng/syslog-ng.conf:ro 108 | 109 | blackbox: 110 | image: quay.io/prometheus/blackbox-exporter:latest 111 | container_name: blackbox-exporter 112 | command: --config.file=/config/blackbox.yml 113 | volumes: 114 | - ./blackbox/blackbox.yml:/config/blackbox.yml 115 | networks: 116 | monitoring: null 117 | ports: 118 | - protocol: tcp 119 | published: 9115 120 | target: 9115 121 | restart: unless-stopped 122 | -------------------------------------------------------------------------------- /grafana/provisioning/dashboards/mikrotik.yml: -------------------------------------------------------------------------------- 1 | apiVersion: 1 2 | 3 | providers: 4 | - name: 'Mikrotik Dashboards' 5 | orgId: 1 6 | folder: 'Mikrotik' 7 | type: file 8 | disableDeletion: false 9 | editable: true 10 | updateIntervalSeconds: 10 11 | options: 12 | path: /etc/grafana/provisioning/dashboards/mikrotik 13 | -------------------------------------------------------------------------------- /grafana/provisioning/dashboards/mikrotik/mikrotik_loki_logs.json: -------------------------------------------------------------------------------- 1 | { 2 | "annotations": { 3 | "list": [ 4 | { 5 | "builtIn": 1, 6 | "datasource": { 7 | "type": "datasource", 8 | "uid": "grafana" 9 | }, 10 | "enable": true, 11 | "hide": true, 12 | "iconColor": "rgba(0, 211, 255, 1)", 13 | "name": "Annotations & Alerts", 14 | "target": { 15 | "limit": 100, 16 | "matchAny": false, 17 | "tags": [], 18 | "type": "dashboard" 19 | }, 20 | "type": "dashboard" 21 | } 22 | ] 23 | }, 24 | "description": "Mikrotik Loki Logs", 25 | "editable": true, 26 | "fiscalYearStartMonth": 0, 27 | "gnetId": 17139, 28 | "graphTooltip": 0, 29 | "id": 3, 30 | "links": [ 31 | { 32 | "asDropdown": false, 33 | "icon": "external link", 34 | "includeVars": false, 35 | "keepTime": false, 36 | "tags": [ 37 | "mikrotik", 38 | "mktxp" 39 | ], 40 | "targetBlank": false, 41 | "title": "MKTXP Exporter", 42 | "tooltip": "", 43 | "type": "dashboards", 44 | "url": "" 45 | }, 46 | { 47 | "asDropdown": true, 48 | "icon": "external link", 49 | "keepTime": false, 50 | "tags": [ 51 | "system" 52 | ], 53 | "title": "System Overview", 54 | "type": "dashboards" 55 | } 56 | ], 57 | "liveNow": false, 58 | "panels": [ 59 | { 60 | "datasource": { 61 | "type": "loki", 62 | "uid": "P8E80F9AEF21F6940" 63 | }, 64 | "description": "", 65 | "fieldConfig": { 66 | "defaults": { 67 | "color": { 68 | "mode": "thresholds" 69 | }, 70 | "mappings": [], 71 | "thresholds": { 72 | "mode": "absolute", 73 | "steps": [ 74 | { 75 | "color": "semi-dark-orange", 76 | "value": null 77 | } 78 | ] 79 | }, 80 | "unit": "short" 81 | }, 82 | "overrides": [] 83 | }, 84 | "gridPos": { 85 | "h": 7, 86 | "w": 9, 87 | "x": 0, 88 | "y": 0 89 | }, 90 | "id": 4, 91 | "interval": "$smooth", 92 | "options": { 93 | "colorMode": "value", 94 | "graphMode": "area", 95 | "justifyMode": "center", 96 | "orientation": "auto", 97 | "reduceOptions": { 98 | "calcs": [ 99 | "sum" 100 | ], 101 | "fields": "", 102 | "values": false 103 | }, 104 | "text": {}, 105 | "textMode": "auto" 106 | }, 107 | "pluginVersion": "9.3.2", 108 | "targets": [ 109 | { 110 | "datasource": { 111 | "type": "loki", 112 | "uid": "P8E80F9AEF21F6940" 113 | }, 114 | "editorMode": "code", 115 | "expr": "count_over_time({job=\"syslog\", routerboard=\"$routerboard\"} [$__interval] |~\"(?i)$topics\" |~\"(?i)$searchable_pattern\" !~\"(?i)$exclude\")", 116 | "legendFormat": "{{host}}", 117 | "queryType": "range", 118 | "refId": "A" 119 | } 120 | ], 121 | "title": "Log Counts By Topic", 122 | "transformations": [], 123 | "type": "stat" 124 | }, 125 | { 126 | "datasource": { 127 | "type": "loki", 128 | "uid": "P8E80F9AEF21F6940" 129 | }, 130 | "fieldConfig": { 131 | "defaults": { 132 | "color": { 133 | "mode": "palette-classic" 134 | }, 135 | "custom": { 136 | "axisCenteredZero": false, 137 | "axisColorMode": "text", 138 | "axisLabel": "", 139 | "axisPlacement": "auto", 140 | "barAlignment": 0, 141 | "drawStyle": "line", 142 | "fillOpacity": 30, 143 | "gradientMode": "opacity", 144 | "hideFrom": { 145 | "legend": false, 146 | "tooltip": false, 147 | "viz": false 148 | }, 149 | "lineInterpolation": "smooth", 150 | "lineWidth": 1, 151 | "pointSize": 5, 152 | "scaleDistribution": { 153 | "type": "linear" 154 | }, 155 | "showPoints": "never", 156 | "spanNulls": true, 157 | "stacking": { 158 | "group": "A", 159 | "mode": "normal" 160 | }, 161 | "thresholdsStyle": { 162 | "mode": "off" 163 | } 164 | }, 165 | "mappings": [], 166 | "thresholds": { 167 | "mode": "absolute", 168 | "steps": [ 169 | { 170 | "color": "green", 171 | "value": null 172 | }, 173 | { 174 | "color": "red", 175 | "value": 80 176 | } 177 | ] 178 | }, 179 | "unit": "short" 180 | }, 181 | "overrides": [] 182 | }, 183 | "gridPos": { 184 | "h": 7, 185 | "w": 15, 186 | "x": 9, 187 | "y": 0 188 | }, 189 | "id": 3, 190 | "interval": "$smooth", 191 | "options": { 192 | "legend": { 193 | "calcs": [ 194 | "mean", 195 | "sum" 196 | ], 197 | "displayMode": "table", 198 | "placement": "right", 199 | "showLegend": true 200 | }, 201 | "tooltip": { 202 | "mode": "none", 203 | "sort": "none" 204 | } 205 | }, 206 | "pluginVersion": "8.1.2", 207 | "targets": [ 208 | { 209 | "datasource": { 210 | "type": "loki", 211 | "uid": "P8E80F9AEF21F6940" 212 | }, 213 | "editorMode": "code", 214 | "expr": "count_over_time({job=\"syslog\", routerboard=\"$routerboard\"} [$__interval] |regexp \"(?P\\\\w+)\\\\: \" |~\"(?i)$topics\" |~\"(?i)$searchable_pattern\" !~\"(?i)$exclude\")", 215 | "legendFormat": "{{prefix}}", 216 | "queryType": "range", 217 | "refId": "A" 218 | } 219 | ], 220 | "title": "Log Counts By Topic Over Time", 221 | "transformations": [], 222 | "type": "timeseries" 223 | }, 224 | { 225 | "datasource": { 226 | "type": "loki", 227 | "uid": "P8E80F9AEF21F6940" 228 | }, 229 | "description": "", 230 | "gridPos": { 231 | "h": 23, 232 | "w": 24, 233 | "x": 0, 234 | "y": 7 235 | }, 236 | "id": 5, 237 | "options": { 238 | "dedupStrategy": "none", 239 | "enableLogDetails": true, 240 | "prettifyLogMessage": false, 241 | "showCommonLabels": false, 242 | "showLabels": false, 243 | "showTime": false, 244 | "sortOrder": "Descending", 245 | "wrapLogMessage": false 246 | }, 247 | "pluginVersion": "7.3.6", 248 | "targets": [ 249 | { 250 | "datasource": { 251 | "type": "loki", 252 | "uid": "P8E80F9AEF21F6940" 253 | }, 254 | "editorMode": "code", 255 | "expr": "{job=\"syslog\", routerboard=\"$routerboard\"} |~\"(?i)$topics\" |~\"(?i)$searchable_pattern\" !~\"(?i)$exclude\"", 256 | "legendFormat": "", 257 | "queryType": "range", 258 | "refId": "A" 259 | } 260 | ], 261 | "title": "Logs Lines By Topic", 262 | "type": "logs" 263 | } 264 | ], 265 | "refresh": "5s", 266 | "schemaVersion": 37, 267 | "style": "dark", 268 | "tags": [ 269 | "mikrotik", 270 | "loki" 271 | ], 272 | "templating": { 273 | "list": [ 274 | { 275 | "current": { 276 | "selected": false, 277 | "text": "MKT-GT", 278 | "value": "MKT-GT" 279 | }, 280 | "datasource": { 281 | "type": "loki", 282 | "uid": "P8E80F9AEF21F6940" 283 | }, 284 | "definition": "", 285 | "hide": 0, 286 | "includeAll": false, 287 | "label": "Routerboard", 288 | "multi": false, 289 | "name": "routerboard", 290 | "options": [], 291 | "query": { 292 | "label": "routerboard", 293 | "refId": "LokiVariableQueryEditor-VariableQuery", 294 | "stream": "", 295 | "type": 1 296 | }, 297 | "refresh": 1, 298 | "regex": "", 299 | "skipUrlSync": false, 300 | "sort": 0, 301 | "type": "query" 302 | }, 303 | { 304 | "current": { 305 | "selected": true, 306 | "text": [ 307 | "All" 308 | ], 309 | "value": [ 310 | "$__all" 311 | ] 312 | }, 313 | "hide": 0, 314 | "includeAll": true, 315 | "label": "Topics", 316 | "multi": true, 317 | "name": "topics", 318 | "options": [ 319 | { 320 | "selected": true, 321 | "text": "All", 322 | "value": "$__all" 323 | }, 324 | { 325 | "selected": false, 326 | "text": "account", 327 | "value": "account" 328 | }, 329 | { 330 | "selected": false, 331 | "text": "caps", 332 | "value": "caps" 333 | }, 334 | { 335 | "selected": false, 336 | "text": "critical", 337 | "value": "critical" 338 | }, 339 | { 340 | "selected": false, 341 | "text": "error", 342 | "value": "error" 343 | }, 344 | { 345 | "selected": false, 346 | "text": "firewall", 347 | "value": "firewall" 348 | }, 349 | { 350 | "selected": false, 351 | "text": "info", 352 | "value": "info" 353 | }, 354 | { 355 | "selected": false, 356 | "text": "warning", 357 | "value": "warning" 358 | }, 359 | { 360 | "selected": false, 361 | "text": "wireless", 362 | "value": "wireless" 363 | }, 364 | { 365 | "selected": false, 366 | "text": "dns", 367 | "value": "dns" 368 | }, 369 | { 370 | "selected": false, 371 | "text": "health", 372 | "value": "health" 373 | }, 374 | { 375 | "selected": false, 376 | "text": "dhcp", 377 | "value": "dhcp" 378 | }, 379 | { 380 | "selected": false, 381 | "text": "interface", 382 | "value": "interface" 383 | }, 384 | { 385 | "selected": false, 386 | "text": "ipsec", 387 | "value": "ipsec" 388 | }, 389 | { 390 | "selected": false, 391 | "text": "l2tp", 392 | "value": "l2tp" 393 | }, 394 | { 395 | "selected": false, 396 | "text": "ppp", 397 | "value": "ppp" 398 | }, 399 | { 400 | "selected": false, 401 | "text": "raw", 402 | "value": "raw" 403 | }, 404 | { 405 | "selected": false, 406 | "text": "ssh", 407 | "value": "ssh" 408 | }, 409 | { 410 | "selected": false, 411 | "text": "system", 412 | "value": "system" 413 | }, 414 | { 415 | "selected": false, 416 | "text": "ups", 417 | "value": "ups" 418 | } 419 | ], 420 | "query": "account,caps,critical,error,firewall,info,warning,wireless,dns,health,dhcp,interface,ipsec,l2tp,ppp,raw,ssh,system,ups", 421 | "queryValue": "", 422 | "skipUrlSync": false, 423 | "type": "custom" 424 | }, 425 | { 426 | "current": { 427 | "selected": true, 428 | "text": "", 429 | "value": "" 430 | }, 431 | "description": "Search (case insensitive)", 432 | "hide": 0, 433 | "label": "Search", 434 | "name": "searchable_pattern", 435 | "options": [ 436 | { 437 | "selected": true, 438 | "text": "", 439 | "value": "" 440 | } 441 | ], 442 | "query": "", 443 | "skipUrlSync": false, 444 | "type": "textbox" 445 | }, 446 | { 447 | "current": { 448 | "selected": false, 449 | "text": "None", 450 | "value": "None" 451 | }, 452 | "description": "Exclude pattern (case-insensitive, None includes everything)", 453 | "hide": 0, 454 | "includeAll": false, 455 | "label": "Exclude", 456 | "multi": false, 457 | "name": "exclude", 458 | "options": [ 459 | { 460 | "selected": true, 461 | "text": "None", 462 | "value": "None" 463 | } 464 | ], 465 | "query": "None,", 466 | "queryValue": "None", 467 | "skipUrlSync": false, 468 | "type": "custom" 469 | }, 470 | { 471 | "current": { 472 | "selected": false, 473 | "text": "5m", 474 | "value": "5m" 475 | }, 476 | "hide": 0, 477 | "includeAll": false, 478 | "label": "Smooth", 479 | "multi": false, 480 | "name": "smooth", 481 | "options": [ 482 | { 483 | "selected": false, 484 | "text": "30s", 485 | "value": "30s" 486 | }, 487 | { 488 | "selected": false, 489 | "text": "1m", 490 | "value": "1m" 491 | }, 492 | { 493 | "selected": false, 494 | "text": "2m", 495 | "value": "2m" 496 | }, 497 | { 498 | "selected": true, 499 | "text": "5m", 500 | "value": "5m" 501 | }, 502 | { 503 | "selected": false, 504 | "text": "10m", 505 | "value": "10m" 506 | }, 507 | { 508 | "selected": false, 509 | "text": "15m", 510 | "value": "15m" 511 | }, 512 | { 513 | "selected": false, 514 | "text": "30m", 515 | "value": "30m" 516 | }, 517 | { 518 | "selected": false, 519 | "text": "1h", 520 | "value": "1h" 521 | }, 522 | { 523 | "selected": false, 524 | "text": "2h", 525 | "value": "2h" 526 | } 527 | ], 528 | "query": "30s,1m,2m,5m,10m,15m,30m,1h,2h", 529 | "queryValue": "", 530 | "skipUrlSync": false, 531 | "type": "custom" 532 | } 533 | ] 534 | }, 535 | "time": { 536 | "from": "now-12h", 537 | "to": "now" 538 | }, 539 | "timepicker": {}, 540 | "timezone": "", 541 | "title": "Mikrotik Loki Logs", 542 | "uid": "awslix6doiakp-mkt-loki", 543 | "version": 1, 544 | "weekStart": "" 545 | } -------------------------------------------------------------------------------- /grafana/provisioning/dashboards/system.yml: -------------------------------------------------------------------------------- 1 | apiVersion: 1 2 | 3 | providers: 4 | - name: 'System Dashboards' 5 | orgId: 1 6 | folder: 'System' 7 | type: file 8 | disableDeletion: false 9 | editable: true 10 | updateIntervalSeconds: 10 11 | options: 12 | path: /etc/grafana/provisioning/dashboards/system 13 | -------------------------------------------------------------------------------- /grafana/provisioning/dashboards/system/system_grafana_internals.json: -------------------------------------------------------------------------------- 1 | { 2 | "annotations": { 3 | "list": [ 4 | { 5 | "builtIn": 1, 6 | "datasource": { 7 | "type": "datasource", 8 | "uid": "grafana" 9 | }, 10 | "enable": true, 11 | "hide": true, 12 | "iconColor": "rgba(0, 211, 255, 1)", 13 | "name": "Annotations & Alerts", 14 | "target": { 15 | "limit": 100, 16 | "matchAny": false, 17 | "tags": [], 18 | "type": "dashboard" 19 | }, 20 | "type": "dashboard" 21 | } 22 | ] 23 | }, 24 | "description": "Grafana Internal Metics Dashboards", 25 | "editable": true, 26 | "fiscalYearStartMonth": 0, 27 | "gnetId": 3590, 28 | "graphTooltip": 0, 29 | "id": 10, 30 | "links": [ 31 | { 32 | "asDropdown": true, 33 | "icon": "external link", 34 | "includeVars": false, 35 | "keepTime": false, 36 | "tags": [ 37 | "mikrotik" 38 | ], 39 | "targetBlank": false, 40 | "title": "Mikrotik", 41 | "tooltip": "", 42 | "type": "dashboards", 43 | "url": "" 44 | }, 45 | { 46 | "asDropdown": true, 47 | "icon": "external link", 48 | "includeVars": false, 49 | "keepTime": false, 50 | "tags": [ 51 | "system" 52 | ], 53 | "targetBlank": false, 54 | "title": "System Overview", 55 | "tooltip": "", 56 | "type": "dashboards", 57 | "url": "" 58 | } 59 | ], 60 | "liveNow": false, 61 | "panels": [ 62 | { 63 | "collapsed": false, 64 | "datasource": { 65 | "type": "datasource", 66 | "uid": "grafana" 67 | }, 68 | "gridPos": { 69 | "h": 1, 70 | "w": 24, 71 | "x": 0, 72 | "y": 0 73 | }, 74 | "id": 21, 75 | "panels": [], 76 | "targets": [ 77 | { 78 | "datasource": { 79 | "type": "datasource", 80 | "uid": "grafana" 81 | }, 82 | "refId": "A" 83 | } 84 | ], 85 | "title": "General Counters", 86 | "type": "row" 87 | }, 88 | { 89 | "datasource": { 90 | "type": "prometheus", 91 | "uid": "PBFA97CFB590B2093" 92 | }, 93 | "description": "start time of the process", 94 | "fieldConfig": { 95 | "defaults": { 96 | "color": { 97 | "mode": "thresholds" 98 | }, 99 | "mappings": [ 100 | { 101 | "options": { 102 | "match": "null", 103 | "result": { 104 | "text": "N/A" 105 | } 106 | }, 107 | "type": "special" 108 | } 109 | ], 110 | "thresholds": { 111 | "mode": "absolute", 112 | "steps": [ 113 | { 114 | "color": "green", 115 | "value": null 116 | }, 117 | { 118 | "color": "red", 119 | "value": 80 120 | } 121 | ] 122 | }, 123 | "unit": "dateTimeFromNow" 124 | }, 125 | "overrides": [] 126 | }, 127 | "gridPos": { 128 | "h": 7, 129 | "w": 6, 130 | "x": 0, 131 | "y": 1 132 | }, 133 | "id": 16, 134 | "links": [], 135 | "maxDataPoints": 100, 136 | "options": { 137 | "colorMode": "none", 138 | "graphMode": "none", 139 | "justifyMode": "auto", 140 | "orientation": "horizontal", 141 | "reduceOptions": { 142 | "calcs": [ 143 | "lastNotNull" 144 | ], 145 | "fields": "", 146 | "values": false 147 | }, 148 | "textMode": "auto" 149 | }, 150 | "pluginVersion": "9.2.0", 151 | "targets": [ 152 | { 153 | "datasource": { 154 | "type": "prometheus", 155 | "uid": "PBFA97CFB590B2093" 156 | }, 157 | "expr": "process_start_time_seconds{kubernetes_pod_name=\"$pod\"} * 1000", 158 | "format": "time_series", 159 | "intervalFactor": 2, 160 | "refId": "A" 161 | } 162 | ], 163 | "title": "Start time", 164 | "type": "stat" 165 | }, 166 | { 167 | "datasource": { 168 | "type": "prometheus", 169 | "uid": "PBFA97CFB590B2093" 170 | }, 171 | "description": "total amount of orgs", 172 | "fieldConfig": { 173 | "defaults": { 174 | "color": { 175 | "mode": "thresholds" 176 | }, 177 | "mappings": [ 178 | { 179 | "options": { 180 | "match": "null", 181 | "result": { 182 | "text": "N/A" 183 | } 184 | }, 185 | "type": "special" 186 | } 187 | ], 188 | "thresholds": { 189 | "mode": "absolute", 190 | "steps": [ 191 | { 192 | "color": "green", 193 | "value": null 194 | }, 195 | { 196 | "color": "red", 197 | "value": 80 198 | } 199 | ] 200 | }, 201 | "unit": "none" 202 | }, 203 | "overrides": [] 204 | }, 205 | "gridPos": { 206 | "h": 7, 207 | "w": 6, 208 | "x": 6, 209 | "y": 1 210 | }, 211 | "id": 2, 212 | "links": [], 213 | "maxDataPoints": 100, 214 | "options": { 215 | "colorMode": "none", 216 | "graphMode": "none", 217 | "justifyMode": "auto", 218 | "orientation": "horizontal", 219 | "reduceOptions": { 220 | "calcs": [ 221 | "mean" 222 | ], 223 | "fields": "", 224 | "values": false 225 | }, 226 | "textMode": "auto" 227 | }, 228 | "pluginVersion": "9.2.0", 229 | "targets": [ 230 | { 231 | "datasource": { 232 | "type": "prometheus", 233 | "uid": "PBFA97CFB590B2093" 234 | }, 235 | "expr": "grafana_stat_total_orgs{kubernetes_pod_name=\"$pod\"}", 236 | "format": "time_series", 237 | "intervalFactor": 2, 238 | "refId": "A" 239 | } 240 | ], 241 | "title": "Organisation Count", 242 | "type": "stat" 243 | }, 244 | { 245 | "datasource": { 246 | "type": "prometheus", 247 | "uid": "PBFA97CFB590B2093" 248 | }, 249 | "description": "total amount of users", 250 | "fieldConfig": { 251 | "defaults": { 252 | "color": { 253 | "mode": "thresholds" 254 | }, 255 | "mappings": [ 256 | { 257 | "options": { 258 | "match": "null", 259 | "result": { 260 | "text": "N/A" 261 | } 262 | }, 263 | "type": "special" 264 | } 265 | ], 266 | "thresholds": { 267 | "mode": "absolute", 268 | "steps": [ 269 | { 270 | "color": "green", 271 | "value": null 272 | }, 273 | { 274 | "color": "red", 275 | "value": 80 276 | } 277 | ] 278 | }, 279 | "unit": "none" 280 | }, 281 | "overrides": [] 282 | }, 283 | "gridPos": { 284 | "h": 7, 285 | "w": 6, 286 | "x": 12, 287 | "y": 1 288 | }, 289 | "id": 3, 290 | "links": [], 291 | "maxDataPoints": 100, 292 | "options": { 293 | "colorMode": "none", 294 | "graphMode": "none", 295 | "justifyMode": "auto", 296 | "orientation": "horizontal", 297 | "reduceOptions": { 298 | "calcs": [ 299 | "mean" 300 | ], 301 | "fields": "", 302 | "values": false 303 | }, 304 | "textMode": "auto" 305 | }, 306 | "pluginVersion": "9.2.0", 307 | "targets": [ 308 | { 309 | "datasource": { 310 | "type": "prometheus", 311 | "uid": "PBFA97CFB590B2093" 312 | }, 313 | "expr": "grafana_stat_total_users{kubernetes_pod_name=\"$pod\"}", 314 | "format": "time_series", 315 | "intervalFactor": 2, 316 | "refId": "A" 317 | } 318 | ], 319 | "title": "User Count", 320 | "type": "stat" 321 | }, 322 | { 323 | "datasource": { 324 | "type": "prometheus", 325 | "uid": "PBFA97CFB590B2093" 326 | }, 327 | "description": "total amount of dashboards", 328 | "fieldConfig": { 329 | "defaults": { 330 | "color": { 331 | "mode": "thresholds" 332 | }, 333 | "mappings": [ 334 | { 335 | "options": { 336 | "match": "null", 337 | "result": { 338 | "text": "N/A" 339 | } 340 | }, 341 | "type": "special" 342 | } 343 | ], 344 | "thresholds": { 345 | "mode": "absolute", 346 | "steps": [ 347 | { 348 | "color": "green", 349 | "value": null 350 | }, 351 | { 352 | "color": "red", 353 | "value": 80 354 | } 355 | ] 356 | }, 357 | "unit": "none" 358 | }, 359 | "overrides": [] 360 | }, 361 | "gridPos": { 362 | "h": 7, 363 | "w": 6, 364 | "x": 18, 365 | "y": 1 366 | }, 367 | "id": 5, 368 | "links": [], 369 | "maxDataPoints": 100, 370 | "options": { 371 | "colorMode": "none", 372 | "graphMode": "none", 373 | "justifyMode": "auto", 374 | "orientation": "horizontal", 375 | "reduceOptions": { 376 | "calcs": [ 377 | "mean" 378 | ], 379 | "fields": "", 380 | "values": false 381 | }, 382 | "textMode": "auto" 383 | }, 384 | "pluginVersion": "9.2.0", 385 | "targets": [ 386 | { 387 | "datasource": { 388 | "type": "prometheus", 389 | "uid": "PBFA97CFB590B2093" 390 | }, 391 | "expr": "grafana_stat_totals_dashboard{kubernetes_pod_name=\"$pod\"}", 392 | "format": "time_series", 393 | "intervalFactor": 2, 394 | "refId": "A" 395 | } 396 | ], 397 | "title": "Dashboard Count", 398 | "type": "stat" 399 | }, 400 | { 401 | "collapsed": false, 402 | "datasource": { 403 | "type": "datasource", 404 | "uid": "grafana" 405 | }, 406 | "gridPos": { 407 | "h": 1, 408 | "w": 24, 409 | "x": 0, 410 | "y": 8 411 | }, 412 | "id": 22, 413 | "panels": [], 414 | "targets": [ 415 | { 416 | "datasource": { 417 | "type": "datasource", 418 | "uid": "grafana" 419 | }, 420 | "refId": "A" 421 | } 422 | ], 423 | "title": "CPU, Memory and File Descriptor Stats", 424 | "type": "row" 425 | }, 426 | { 427 | "aliasColors": {}, 428 | "bars": false, 429 | "dashLength": 10, 430 | "dashes": false, 431 | "datasource": { 432 | "type": "prometheus", 433 | "uid": "PBFA97CFB590B2093" 434 | }, 435 | "description": "Average user and system CPU time spent in seconds.", 436 | "fill": 1, 437 | "fillGradient": 0, 438 | "gridPos": { 439 | "h": 7, 440 | "w": 8, 441 | "x": 0, 442 | "y": 9 443 | }, 444 | "hiddenSeries": false, 445 | "id": 14, 446 | "legend": { 447 | "alignAsTable": true, 448 | "avg": true, 449 | "current": true, 450 | "max": true, 451 | "min": true, 452 | "show": true, 453 | "total": false, 454 | "values": true 455 | }, 456 | "lines": true, 457 | "linewidth": 1, 458 | "links": [], 459 | "nullPointMode": "null", 460 | "options": { 461 | "alertThreshold": true 462 | }, 463 | "percentage": false, 464 | "pluginVersion": "9.2.0", 465 | "pointradius": 5, 466 | "points": false, 467 | "renderer": "flot", 468 | "seriesOverrides": [], 469 | "spaceLength": 10, 470 | "stack": false, 471 | "steppedLine": false, 472 | "targets": [ 473 | { 474 | "datasource": { 475 | "type": "prometheus", 476 | "uid": "PBFA97CFB590B2093" 477 | }, 478 | "expr": "avg(rate(process_cpu_seconds_total{kubernetes_pod_name=\"$pod\"}[5m]))", 479 | "format": "time_series", 480 | "intervalFactor": 2, 481 | "legendFormat": "CPU Time", 482 | "refId": "A" 483 | } 484 | ], 485 | "thresholds": [], 486 | "timeRegions": [], 487 | "title": "Average CPU Usage", 488 | "tooltip": { 489 | "shared": true, 490 | "sort": 0, 491 | "value_type": "individual" 492 | }, 493 | "type": "graph", 494 | "xaxis": { 495 | "mode": "time", 496 | "show": true, 497 | "values": [] 498 | }, 499 | "yaxes": [ 500 | { 501 | "format": "s", 502 | "logBase": 1, 503 | "show": true 504 | }, 505 | { 506 | "format": "short", 507 | "logBase": 1, 508 | "show": true 509 | } 510 | ], 511 | "yaxis": { 512 | "align": false 513 | } 514 | }, 515 | { 516 | "aliasColors": {}, 517 | "bars": false, 518 | "dashLength": 10, 519 | "dashes": false, 520 | "datasource": { 521 | "type": "prometheus", 522 | "uid": "PBFA97CFB590B2093" 523 | }, 524 | "description": "Virtual and Resident memory size in bytes, averages over 5 min interval", 525 | "fill": 1, 526 | "fillGradient": 0, 527 | "gridPos": { 528 | "h": 7, 529 | "w": 8, 530 | "x": 8, 531 | "y": 9 532 | }, 533 | "hiddenSeries": false, 534 | "id": 15, 535 | "legend": { 536 | "alignAsTable": true, 537 | "avg": true, 538 | "current": true, 539 | "max": true, 540 | "min": true, 541 | "show": true, 542 | "total": false, 543 | "values": true 544 | }, 545 | "lines": true, 546 | "linewidth": 1, 547 | "links": [], 548 | "nullPointMode": "null", 549 | "options": { 550 | "alertThreshold": true 551 | }, 552 | "percentage": false, 553 | "pluginVersion": "9.2.0", 554 | "pointradius": 5, 555 | "points": false, 556 | "renderer": "flot", 557 | "seriesOverrides": [], 558 | "spaceLength": 10, 559 | "stack": false, 560 | "steppedLine": false, 561 | "targets": [ 562 | { 563 | "datasource": { 564 | "type": "prometheus", 565 | "uid": "PBFA97CFB590B2093" 566 | }, 567 | "expr": "avg(rate(process_resident_memory_bytes{kubernetes_pod_name=\"$pod\"}[5m]))", 568 | "format": "time_series", 569 | "intervalFactor": 2, 570 | "legendFormat": "Resident Mem", 571 | "refId": "A" 572 | }, 573 | { 574 | "datasource": { 575 | "type": "prometheus", 576 | "uid": "PBFA97CFB590B2093" 577 | }, 578 | "expr": "avg(rate(process_virtual_memory_bytes{kubernetes_pod_name=\"$pod\"}[5m]))", 579 | "format": "time_series", 580 | "intervalFactor": 2, 581 | "legendFormat": "Virtual Mem", 582 | "refId": "B" 583 | } 584 | ], 585 | "thresholds": [], 586 | "timeRegions": [], 587 | "title": "Average Memory Usage", 588 | "tooltip": { 589 | "shared": true, 590 | "sort": 0, 591 | "value_type": "individual" 592 | }, 593 | "type": "graph", 594 | "xaxis": { 595 | "mode": "time", 596 | "show": true, 597 | "values": [] 598 | }, 599 | "yaxes": [ 600 | { 601 | "format": "decbytes", 602 | "logBase": 1, 603 | "show": true 604 | }, 605 | { 606 | "format": "short", 607 | "logBase": 1, 608 | "show": true 609 | } 610 | ], 611 | "yaxis": { 612 | "align": false 613 | } 614 | }, 615 | { 616 | "aliasColors": {}, 617 | "bars": false, 618 | "dashLength": 10, 619 | "dashes": false, 620 | "datasource": { 621 | "type": "prometheus", 622 | "uid": "PBFA97CFB590B2093" 623 | }, 624 | "description": "Number of open file descriptors", 625 | "fill": 1, 626 | "fillGradient": 0, 627 | "gridPos": { 628 | "h": 7, 629 | "w": 8, 630 | "x": 16, 631 | "y": 9 632 | }, 633 | "hiddenSeries": false, 634 | "id": 17, 635 | "legend": { 636 | "alignAsTable": true, 637 | "avg": true, 638 | "current": true, 639 | "max": true, 640 | "min": true, 641 | "show": true, 642 | "total": false, 643 | "values": true 644 | }, 645 | "lines": true, 646 | "linewidth": 1, 647 | "links": [], 648 | "nullPointMode": "null", 649 | "options": { 650 | "alertThreshold": true 651 | }, 652 | "percentage": false, 653 | "pluginVersion": "9.2.0", 654 | "pointradius": 5, 655 | "points": false, 656 | "renderer": "flot", 657 | "seriesOverrides": [], 658 | "spaceLength": 10, 659 | "stack": false, 660 | "steppedLine": false, 661 | "targets": [ 662 | { 663 | "datasource": { 664 | "type": "prometheus", 665 | "uid": "PBFA97CFB590B2093" 666 | }, 667 | "expr": "process_open_fds{kubernetes_pod_name=\"$pod\"}", 668 | "format": "time_series", 669 | "intervalFactor": 2, 670 | "legendFormat": "Open FD", 671 | "refId": "A" 672 | } 673 | ], 674 | "thresholds": [], 675 | "timeRegions": [], 676 | "title": "Open File Descriptors", 677 | "tooltip": { 678 | "shared": true, 679 | "sort": 0, 680 | "value_type": "individual" 681 | }, 682 | "type": "graph", 683 | "xaxis": { 684 | "mode": "time", 685 | "show": true, 686 | "values": [] 687 | }, 688 | "yaxes": [ 689 | { 690 | "format": "short", 691 | "logBase": 1, 692 | "show": true 693 | }, 694 | { 695 | "format": "short", 696 | "logBase": 1, 697 | "show": true 698 | } 699 | ], 700 | "yaxis": { 701 | "align": false 702 | } 703 | }, 704 | { 705 | "collapsed": false, 706 | "datasource": { 707 | "type": "datasource", 708 | "uid": "grafana" 709 | }, 710 | "gridPos": { 711 | "h": 1, 712 | "w": 24, 713 | "x": 0, 714 | "y": 16 715 | }, 716 | "id": 23, 717 | "panels": [], 718 | "targets": [ 719 | { 720 | "datasource": { 721 | "type": "datasource", 722 | "uid": "grafana" 723 | }, 724 | "refId": "A" 725 | } 726 | ], 727 | "title": "Responce Codes and Logins", 728 | "type": "row" 729 | }, 730 | { 731 | "aliasColors": {}, 732 | "bars": false, 733 | "dashLength": 10, 734 | "dashes": false, 735 | "datasource": { 736 | "type": "prometheus", 737 | "uid": "PBFA97CFB590B2093" 738 | }, 739 | "description": "http response status", 740 | "fill": 1, 741 | "fillGradient": 0, 742 | "gridPos": { 743 | "h": 7, 744 | "w": 8, 745 | "x": 0, 746 | "y": 17 747 | }, 748 | "hiddenSeries": false, 749 | "id": 18, 750 | "legend": { 751 | "alignAsTable": true, 752 | "avg": true, 753 | "current": true, 754 | "max": true, 755 | "min": true, 756 | "show": true, 757 | "total": false, 758 | "values": true 759 | }, 760 | "lines": true, 761 | "linewidth": 1, 762 | "links": [], 763 | "nullPointMode": "null", 764 | "options": { 765 | "alertThreshold": true 766 | }, 767 | "percentage": false, 768 | "pluginVersion": "9.2.0", 769 | "pointradius": 5, 770 | "points": false, 771 | "renderer": "flot", 772 | "seriesOverrides": [], 773 | "spaceLength": 10, 774 | "stack": false, 775 | "steppedLine": false, 776 | "targets": [ 777 | { 778 | "datasource": { 779 | "type": "prometheus", 780 | "uid": "PBFA97CFB590B2093" 781 | }, 782 | "expr": "grafana_api_response_status_total{kubernetes_pod_name=\"$pod\"}", 783 | "format": "time_series", 784 | "intervalFactor": 2, 785 | "legendFormat": "API {{ code }}", 786 | "refId": "A" 787 | }, 788 | { 789 | "datasource": { 790 | "type": "prometheus", 791 | "uid": "PBFA97CFB590B2093" 792 | }, 793 | "expr": "grafana_page_response_status_total{kubernetes_pod_name=\"$pod\"}", 794 | "format": "time_series", 795 | "intervalFactor": 2, 796 | "legendFormat": "Page {{ code }}", 797 | "refId": "B" 798 | }, 799 | { 800 | "datasource": { 801 | "type": "prometheus", 802 | "uid": "PBFA97CFB590B2093" 803 | }, 804 | "expr": "grafana_proxy_response_status_total{kubernetes_pod_name=\"$pod\"}", 805 | "format": "time_series", 806 | "intervalFactor": 2, 807 | "legendFormat": "Proxy {{ code }}", 808 | "refId": "C" 809 | } 810 | ], 811 | "thresholds": [], 812 | "timeRegions": [], 813 | "title": "Total Response Statuses", 814 | "tooltip": { 815 | "shared": true, 816 | "sort": 0, 817 | "value_type": "individual" 818 | }, 819 | "type": "graph", 820 | "xaxis": { 821 | "mode": "time", 822 | "show": true, 823 | "values": [] 824 | }, 825 | "yaxes": [ 826 | { 827 | "format": "short", 828 | "logBase": 1, 829 | "show": true 830 | }, 831 | { 832 | "format": "short", 833 | "logBase": 1, 834 | "show": true 835 | } 836 | ], 837 | "yaxis": { 838 | "align": false 839 | } 840 | }, 841 | { 842 | "aliasColors": {}, 843 | "bars": false, 844 | "dashLength": 10, 845 | "dashes": false, 846 | "datasource": { 847 | "type": "prometheus", 848 | "uid": "PBFA97CFB590B2093" 849 | }, 850 | "description": "http request counter", 851 | "fill": 1, 852 | "fillGradient": 0, 853 | "gridPos": { 854 | "h": 7, 855 | "w": 8, 856 | "x": 8, 857 | "y": 17 858 | }, 859 | "hiddenSeries": false, 860 | "id": 20, 861 | "legend": { 862 | "alignAsTable": true, 863 | "avg": true, 864 | "current": true, 865 | "max": true, 866 | "min": true, 867 | "show": true, 868 | "total": false, 869 | "values": true 870 | }, 871 | "lines": true, 872 | "linewidth": 1, 873 | "links": [], 874 | "nullPointMode": "null", 875 | "options": { 876 | "alertThreshold": true 877 | }, 878 | "percentage": false, 879 | "pluginVersion": "9.2.0", 880 | "pointradius": 5, 881 | "points": false, 882 | "renderer": "flot", 883 | "seriesOverrides": [], 884 | "spaceLength": 10, 885 | "stack": false, 886 | "steppedLine": false, 887 | "targets": [ 888 | { 889 | "datasource": { 890 | "type": "prometheus", 891 | "uid": "PBFA97CFB590B2093" 892 | }, 893 | "expr": "http_request_total{kubernetes_pod_name=\"$pod\"}", 894 | "format": "time_series", 895 | "intervalFactor": 2, 896 | "legendFormat": "{{handler}} - {{method}} - {{statuscode}}", 897 | "refId": "A" 898 | } 899 | ], 900 | "thresholds": [], 901 | "timeRegions": [], 902 | "title": "HTTP Req/Resp Details", 903 | "tooltip": { 904 | "shared": true, 905 | "sort": 0, 906 | "value_type": "individual" 907 | }, 908 | "type": "graph", 909 | "xaxis": { 910 | "mode": "time", 911 | "show": true, 912 | "values": [] 913 | }, 914 | "yaxes": [ 915 | { 916 | "format": "short", 917 | "logBase": 1, 918 | "show": true 919 | }, 920 | { 921 | "format": "short", 922 | "logBase": 1, 923 | "show": true 924 | } 925 | ], 926 | "yaxis": { 927 | "align": false 928 | } 929 | }, 930 | { 931 | "aliasColors": {}, 932 | "bars": false, 933 | "dashLength": 10, 934 | "dashes": false, 935 | "datasource": { 936 | "type": "prometheus", 937 | "uid": "PBFA97CFB590B2093" 938 | }, 939 | "description": "api login counters", 940 | "fill": 1, 941 | "fillGradient": 0, 942 | "gridPos": { 943 | "h": 7, 944 | "w": 8, 945 | "x": 16, 946 | "y": 17 947 | }, 948 | "hiddenSeries": false, 949 | "id": 19, 950 | "legend": { 951 | "alignAsTable": true, 952 | "avg": true, 953 | "current": true, 954 | "max": true, 955 | "min": true, 956 | "show": true, 957 | "total": false, 958 | "values": true 959 | }, 960 | "lines": true, 961 | "linewidth": 1, 962 | "links": [], 963 | "nullPointMode": "null", 964 | "options": { 965 | "alertThreshold": true 966 | }, 967 | "percentage": false, 968 | "pluginVersion": "9.2.0", 969 | "pointradius": 5, 970 | "points": false, 971 | "renderer": "flot", 972 | "seriesOverrides": [], 973 | "spaceLength": 10, 974 | "stack": false, 975 | "steppedLine": false, 976 | "targets": [ 977 | { 978 | "datasource": { 979 | "type": "prometheus", 980 | "uid": "PBFA97CFB590B2093" 981 | }, 982 | "expr": "grafana_api_login_post_total{kubernetes_pod_name=\"$pod\"}", 983 | "format": "time_series", 984 | "intervalFactor": 2, 985 | "legendFormat": "POST", 986 | "refId": "A" 987 | }, 988 | { 989 | "datasource": { 990 | "type": "prometheus", 991 | "uid": "PBFA97CFB590B2093" 992 | }, 993 | "expr": "grafana_api_login_oauth_total{kubernetes_pod_name=\"$pod\"}", 994 | "format": "time_series", 995 | "intervalFactor": 2, 996 | "legendFormat": "OAuth", 997 | "refId": "B" 998 | } 999 | ], 1000 | "thresholds": [], 1001 | "timeRegions": [], 1002 | "title": "Login Counts", 1003 | "tooltip": { 1004 | "shared": true, 1005 | "sort": 0, 1006 | "value_type": "individual" 1007 | }, 1008 | "type": "graph", 1009 | "xaxis": { 1010 | "mode": "time", 1011 | "show": true, 1012 | "values": [] 1013 | }, 1014 | "yaxes": [ 1015 | { 1016 | "format": "short", 1017 | "logBase": 1, 1018 | "show": true 1019 | }, 1020 | { 1021 | "format": "short", 1022 | "logBase": 1, 1023 | "show": true 1024 | } 1025 | ], 1026 | "yaxis": { 1027 | "align": false 1028 | } 1029 | }, 1030 | { 1031 | "collapsed": false, 1032 | "datasource": { 1033 | "type": "datasource", 1034 | "uid": "grafana" 1035 | }, 1036 | "gridPos": { 1037 | "h": 1, 1038 | "w": 24, 1039 | "x": 0, 1040 | "y": 24 1041 | }, 1042 | "id": 24, 1043 | "panels": [], 1044 | "targets": [ 1045 | { 1046 | "datasource": { 1047 | "type": "datasource", 1048 | "uid": "grafana" 1049 | }, 1050 | "refId": "A" 1051 | } 1052 | ], 1053 | "title": "Dataproxy Stats", 1054 | "type": "row" 1055 | }, 1056 | { 1057 | "aliasColors": {}, 1058 | "bars": false, 1059 | "dashLength": 10, 1060 | "dashes": false, 1061 | "datasource": { 1062 | "type": "prometheus", 1063 | "uid": "PBFA97CFB590B2093" 1064 | }, 1065 | "description": "summary for dataproxy request duration", 1066 | "fill": 1, 1067 | "fillGradient": 0, 1068 | "gridPos": { 1069 | "h": 7, 1070 | "w": 12, 1071 | "x": 0, 1072 | "y": 25 1073 | }, 1074 | "hiddenSeries": false, 1075 | "id": 6, 1076 | "legend": { 1077 | "alignAsTable": true, 1078 | "avg": true, 1079 | "current": true, 1080 | "max": true, 1081 | "min": true, 1082 | "show": true, 1083 | "total": false, 1084 | "values": true 1085 | }, 1086 | "lines": true, 1087 | "linewidth": 1, 1088 | "links": [], 1089 | "nullPointMode": "null", 1090 | "options": { 1091 | "alertThreshold": true 1092 | }, 1093 | "percentage": false, 1094 | "pluginVersion": "9.2.0", 1095 | "pointradius": 5, 1096 | "points": false, 1097 | "renderer": "flot", 1098 | "seriesOverrides": [], 1099 | "spaceLength": 10, 1100 | "stack": false, 1101 | "steppedLine": false, 1102 | "targets": [ 1103 | { 1104 | "datasource": { 1105 | "type": "prometheus", 1106 | "uid": "PBFA97CFB590B2093" 1107 | }, 1108 | "expr": "grafana_api_dataproxy_request_all_milliseconds{kubernetes_pod_name=\"$pod\", quantile=\"0.5\"}", 1109 | "format": "time_series", 1110 | "intervalFactor": 2, 1111 | "legendFormat": "0.5 ", 1112 | "refId": "A" 1113 | }, 1114 | { 1115 | "datasource": { 1116 | "type": "prometheus", 1117 | "uid": "PBFA97CFB590B2093" 1118 | }, 1119 | "expr": "grafana_api_dataproxy_request_all_milliseconds{kubernetes_pod_name=\"$pod\", quantile=\"0.9\"}", 1120 | "format": "time_series", 1121 | "intervalFactor": 2, 1122 | "legendFormat": "0.9", 1123 | "refId": "B" 1124 | }, 1125 | { 1126 | "datasource": { 1127 | "type": "prometheus", 1128 | "uid": "PBFA97CFB590B2093" 1129 | }, 1130 | "expr": "grafana_api_dataproxy_request_all_milliseconds{kubernetes_pod_name=\"$pod\", quantile=\"0.99\"}", 1131 | "format": "time_series", 1132 | "intervalFactor": 2, 1133 | "legendFormat": "0.99", 1134 | "refId": "C" 1135 | } 1136 | ], 1137 | "thresholds": [], 1138 | "timeRegions": [], 1139 | "title": "Dataproxy Request Quantiles", 1140 | "tooltip": { 1141 | "shared": true, 1142 | "sort": 0, 1143 | "value_type": "individual" 1144 | }, 1145 | "type": "graph", 1146 | "xaxis": { 1147 | "mode": "time", 1148 | "show": true, 1149 | "values": [] 1150 | }, 1151 | "yaxes": [ 1152 | { 1153 | "format": "ms", 1154 | "logBase": 1, 1155 | "show": true 1156 | }, 1157 | { 1158 | "format": "short", 1159 | "logBase": 1, 1160 | "show": true 1161 | } 1162 | ], 1163 | "yaxis": { 1164 | "align": false 1165 | } 1166 | }, 1167 | { 1168 | "aliasColors": {}, 1169 | "bars": false, 1170 | "dashLength": 10, 1171 | "dashes": false, 1172 | "datasource": { 1173 | "type": "prometheus", 1174 | "uid": "PBFA97CFB590B2093" 1175 | }, 1176 | "description": "average dataproxy request duration over 5 minutes", 1177 | "fill": 1, 1178 | "fillGradient": 0, 1179 | "gridPos": { 1180 | "h": 7, 1181 | "w": 12, 1182 | "x": 12, 1183 | "y": 25 1184 | }, 1185 | "hiddenSeries": false, 1186 | "id": 7, 1187 | "legend": { 1188 | "alignAsTable": true, 1189 | "avg": true, 1190 | "current": true, 1191 | "max": true, 1192 | "min": true, 1193 | "show": true, 1194 | "total": false, 1195 | "values": true 1196 | }, 1197 | "lines": true, 1198 | "linewidth": 1, 1199 | "links": [], 1200 | "nullPointMode": "null", 1201 | "options": { 1202 | "alertThreshold": true 1203 | }, 1204 | "percentage": false, 1205 | "pluginVersion": "9.2.0", 1206 | "pointradius": 5, 1207 | "points": false, 1208 | "renderer": "flot", 1209 | "seriesOverrides": [], 1210 | "spaceLength": 10, 1211 | "stack": false, 1212 | "steppedLine": false, 1213 | "targets": [ 1214 | { 1215 | "datasource": { 1216 | "type": "prometheus", 1217 | "uid": "PBFA97CFB590B2093" 1218 | }, 1219 | "expr": "rate(grafana_api_dataproxy_request_all_milliseconds_sum{kubernetes_pod_name=\"$pod\"}[5m]) / rate(grafana_api_dataproxy_request_all_milliseconds_count{kubernetes_pod_name=\"$pod\"}[5m])", 1220 | "format": "time_series", 1221 | "intervalFactor": 2, 1222 | "legendFormat": "Average Requests Duration", 1223 | "refId": "A" 1224 | } 1225 | ], 1226 | "thresholds": [], 1227 | "timeRegions": [], 1228 | "title": "Dataproxy Request Avg [5 min]", 1229 | "tooltip": { 1230 | "shared": true, 1231 | "sort": 0, 1232 | "value_type": "individual" 1233 | }, 1234 | "type": "graph", 1235 | "xaxis": { 1236 | "mode": "time", 1237 | "show": true, 1238 | "values": [] 1239 | }, 1240 | "yaxes": [ 1241 | { 1242 | "format": "ms", 1243 | "logBase": 1, 1244 | "show": true 1245 | }, 1246 | { 1247 | "format": "short", 1248 | "logBase": 1, 1249 | "show": true 1250 | } 1251 | ], 1252 | "yaxis": { 1253 | "align": false 1254 | } 1255 | }, 1256 | { 1257 | "collapsed": false, 1258 | "datasource": { 1259 | "type": "datasource", 1260 | "uid": "grafana" 1261 | }, 1262 | "gridPos": { 1263 | "h": 1, 1264 | "w": 24, 1265 | "x": 0, 1266 | "y": 32 1267 | }, 1268 | "id": 25, 1269 | "panels": [], 1270 | "targets": [ 1271 | { 1272 | "datasource": { 1273 | "type": "datasource", 1274 | "uid": "grafana" 1275 | }, 1276 | "refId": "A" 1277 | } 1278 | ], 1279 | "title": "Dashboard Get Stats", 1280 | "type": "row" 1281 | }, 1282 | { 1283 | "aliasColors": {}, 1284 | "bars": false, 1285 | "dashLength": 10, 1286 | "dashes": false, 1287 | "datasource": { 1288 | "type": "prometheus", 1289 | "uid": "PBFA97CFB590B2093" 1290 | }, 1291 | "description": "summary for dashboard get duration", 1292 | "fill": 1, 1293 | "fillGradient": 0, 1294 | "gridPos": { 1295 | "h": 7, 1296 | "w": 12, 1297 | "x": 0, 1298 | "y": 33 1299 | }, 1300 | "hiddenSeries": false, 1301 | "id": 8, 1302 | "legend": { 1303 | "alignAsTable": true, 1304 | "avg": true, 1305 | "current": true, 1306 | "max": true, 1307 | "min": true, 1308 | "show": true, 1309 | "total": false, 1310 | "values": true 1311 | }, 1312 | "lines": true, 1313 | "linewidth": 1, 1314 | "links": [], 1315 | "nullPointMode": "null", 1316 | "options": { 1317 | "alertThreshold": true 1318 | }, 1319 | "percentage": false, 1320 | "pluginVersion": "9.2.0", 1321 | "pointradius": 5, 1322 | "points": false, 1323 | "renderer": "flot", 1324 | "seriesOverrides": [], 1325 | "spaceLength": 10, 1326 | "stack": false, 1327 | "steppedLine": false, 1328 | "targets": [ 1329 | { 1330 | "datasource": { 1331 | "type": "prometheus", 1332 | "uid": "PBFA97CFB590B2093" 1333 | }, 1334 | "expr": "grafana_api_dashboard_get_milliseconds{kubernetes_pod_name=\"$pod\", quantile=\"0.5\"}", 1335 | "format": "time_series", 1336 | "intervalFactor": 2, 1337 | "legendFormat": "0.5", 1338 | "refId": "A" 1339 | }, 1340 | { 1341 | "datasource": { 1342 | "type": "prometheus", 1343 | "uid": "PBFA97CFB590B2093" 1344 | }, 1345 | "expr": "grafana_api_dashboard_get_milliseconds{kubernetes_pod_name=\"$pod\", quantile=\"0.9\"}", 1346 | "format": "time_series", 1347 | "intervalFactor": 2, 1348 | "legendFormat": "0.9", 1349 | "refId": "B" 1350 | }, 1351 | { 1352 | "datasource": { 1353 | "type": "prometheus", 1354 | "uid": "PBFA97CFB590B2093" 1355 | }, 1356 | "expr": "grafana_api_dashboard_get_milliseconds{kubernetes_pod_name=\"$pod\", quantile=\"0.99\"}", 1357 | "format": "time_series", 1358 | "intervalFactor": 2, 1359 | "legendFormat": "0.99", 1360 | "refId": "C" 1361 | } 1362 | ], 1363 | "thresholds": [], 1364 | "timeRegions": [], 1365 | "title": "Dashboard Get Duration Quantiles", 1366 | "tooltip": { 1367 | "shared": true, 1368 | "sort": 0, 1369 | "value_type": "individual" 1370 | }, 1371 | "type": "graph", 1372 | "xaxis": { 1373 | "mode": "time", 1374 | "show": true, 1375 | "values": [] 1376 | }, 1377 | "yaxes": [ 1378 | { 1379 | "format": "ms", 1380 | "logBase": 1, 1381 | "show": true 1382 | }, 1383 | { 1384 | "format": "short", 1385 | "logBase": 1, 1386 | "show": true 1387 | } 1388 | ], 1389 | "yaxis": { 1390 | "align": false 1391 | } 1392 | }, 1393 | { 1394 | "aliasColors": {}, 1395 | "bars": false, 1396 | "dashLength": 10, 1397 | "dashes": false, 1398 | "datasource": { 1399 | "type": "prometheus", 1400 | "uid": "PBFA97CFB590B2093" 1401 | }, 1402 | "description": "average dashboard get duration over 5 minutes", 1403 | "fill": 1, 1404 | "fillGradient": 0, 1405 | "gridPos": { 1406 | "h": 7, 1407 | "w": 12, 1408 | "x": 12, 1409 | "y": 33 1410 | }, 1411 | "hiddenSeries": false, 1412 | "id": 9, 1413 | "legend": { 1414 | "alignAsTable": true, 1415 | "avg": true, 1416 | "current": true, 1417 | "max": true, 1418 | "min": true, 1419 | "show": true, 1420 | "total": false, 1421 | "values": true 1422 | }, 1423 | "lines": true, 1424 | "linewidth": 1, 1425 | "links": [], 1426 | "nullPointMode": "null", 1427 | "options": { 1428 | "alertThreshold": true 1429 | }, 1430 | "percentage": false, 1431 | "pluginVersion": "9.2.0", 1432 | "pointradius": 5, 1433 | "points": false, 1434 | "renderer": "flot", 1435 | "seriesOverrides": [], 1436 | "spaceLength": 10, 1437 | "stack": false, 1438 | "steppedLine": false, 1439 | "targets": [ 1440 | { 1441 | "datasource": { 1442 | "type": "prometheus", 1443 | "uid": "PBFA97CFB590B2093" 1444 | }, 1445 | "expr": "rate(grafana_api_dashboard_get_milliseconds_sum{kubernetes_pod_name=\"$pod\"}[5m]) / rate(grafana_api_dashboard_get_milliseconds_count{kubernetes_pod_name=\"$pod\"}[5m])", 1446 | "format": "time_series", 1447 | "intervalFactor": 2, 1448 | "legendFormat": "0.5", 1449 | "refId": "A" 1450 | } 1451 | ], 1452 | "thresholds": [], 1453 | "timeRegions": [], 1454 | "title": "Dashboard Get Duration Avg [5 min]", 1455 | "tooltip": { 1456 | "shared": true, 1457 | "sort": 0, 1458 | "value_type": "individual" 1459 | }, 1460 | "type": "graph", 1461 | "xaxis": { 1462 | "mode": "time", 1463 | "show": true, 1464 | "values": [] 1465 | }, 1466 | "yaxes": [ 1467 | { 1468 | "format": "ms", 1469 | "logBase": 1, 1470 | "show": true 1471 | }, 1472 | { 1473 | "format": "short", 1474 | "logBase": 1, 1475 | "show": true 1476 | } 1477 | ], 1478 | "yaxis": { 1479 | "align": false 1480 | } 1481 | }, 1482 | { 1483 | "collapsed": false, 1484 | "datasource": { 1485 | "type": "datasource", 1486 | "uid": "grafana" 1487 | }, 1488 | "gridPos": { 1489 | "h": 1, 1490 | "w": 24, 1491 | "x": 0, 1492 | "y": 40 1493 | }, 1494 | "id": 26, 1495 | "panels": [], 1496 | "targets": [ 1497 | { 1498 | "datasource": { 1499 | "type": "datasource", 1500 | "uid": "grafana" 1501 | }, 1502 | "refId": "A" 1503 | } 1504 | ], 1505 | "title": "Dashboard Save Stats", 1506 | "type": "row" 1507 | }, 1508 | { 1509 | "aliasColors": {}, 1510 | "bars": false, 1511 | "dashLength": 10, 1512 | "dashes": false, 1513 | "datasource": { 1514 | "type": "prometheus", 1515 | "uid": "PBFA97CFB590B2093" 1516 | }, 1517 | "description": "summary for dashboard save duration", 1518 | "fill": 1, 1519 | "fillGradient": 0, 1520 | "gridPos": { 1521 | "h": 7, 1522 | "w": 12, 1523 | "x": 0, 1524 | "y": 41 1525 | }, 1526 | "hiddenSeries": false, 1527 | "id": 10, 1528 | "legend": { 1529 | "alignAsTable": true, 1530 | "avg": true, 1531 | "current": true, 1532 | "max": true, 1533 | "min": true, 1534 | "show": true, 1535 | "total": false, 1536 | "values": true 1537 | }, 1538 | "lines": true, 1539 | "linewidth": 1, 1540 | "links": [], 1541 | "nullPointMode": "null", 1542 | "options": { 1543 | "alertThreshold": true 1544 | }, 1545 | "percentage": false, 1546 | "pluginVersion": "9.2.0", 1547 | "pointradius": 5, 1548 | "points": false, 1549 | "renderer": "flot", 1550 | "seriesOverrides": [], 1551 | "spaceLength": 10, 1552 | "stack": false, 1553 | "steppedLine": false, 1554 | "targets": [ 1555 | { 1556 | "datasource": { 1557 | "type": "prometheus", 1558 | "uid": "PBFA97CFB590B2093" 1559 | }, 1560 | "expr": "grafana_api_dashboard_save_milliseconds{kubernetes_pod_name=\"$pod\", quantile=\"0.5\"}", 1561 | "format": "time_series", 1562 | "intervalFactor": 2, 1563 | "legendFormat": "0.5", 1564 | "refId": "A" 1565 | }, 1566 | { 1567 | "datasource": { 1568 | "type": "prometheus", 1569 | "uid": "PBFA97CFB590B2093" 1570 | }, 1571 | "expr": "grafana_api_dashboard_save_milliseconds{kubernetes_pod_name=\"$pod\", quantile=\"0.9\"}", 1572 | "format": "time_series", 1573 | "intervalFactor": 2, 1574 | "legendFormat": "0.9", 1575 | "refId": "B" 1576 | }, 1577 | { 1578 | "datasource": { 1579 | "type": "prometheus", 1580 | "uid": "PBFA97CFB590B2093" 1581 | }, 1582 | "expr": "grafana_api_dashboard_save_milliseconds{kubernetes_pod_name=\"$pod\", quantile=\"0.99\"}", 1583 | "format": "time_series", 1584 | "intervalFactor": 2, 1585 | "legendFormat": "0.99", 1586 | "refId": "C" 1587 | } 1588 | ], 1589 | "thresholds": [], 1590 | "timeRegions": [], 1591 | "title": "Dashboard Save Duration Quantiles", 1592 | "tooltip": { 1593 | "shared": true, 1594 | "sort": 0, 1595 | "value_type": "individual" 1596 | }, 1597 | "type": "graph", 1598 | "xaxis": { 1599 | "mode": "time", 1600 | "show": true, 1601 | "values": [] 1602 | }, 1603 | "yaxes": [ 1604 | { 1605 | "format": "ms", 1606 | "logBase": 1, 1607 | "show": true 1608 | }, 1609 | { 1610 | "format": "short", 1611 | "logBase": 1, 1612 | "show": true 1613 | } 1614 | ], 1615 | "yaxis": { 1616 | "align": false 1617 | } 1618 | }, 1619 | { 1620 | "aliasColors": {}, 1621 | "bars": false, 1622 | "dashLength": 10, 1623 | "dashes": false, 1624 | "datasource": { 1625 | "type": "prometheus", 1626 | "uid": "PBFA97CFB590B2093" 1627 | }, 1628 | "description": "average dashboard save duration over 5 minutes", 1629 | "fill": 1, 1630 | "fillGradient": 0, 1631 | "gridPos": { 1632 | "h": 7, 1633 | "w": 12, 1634 | "x": 12, 1635 | "y": 41 1636 | }, 1637 | "hiddenSeries": false, 1638 | "id": 11, 1639 | "legend": { 1640 | "alignAsTable": true, 1641 | "avg": true, 1642 | "current": true, 1643 | "max": true, 1644 | "min": true, 1645 | "show": true, 1646 | "total": false, 1647 | "values": true 1648 | }, 1649 | "lines": true, 1650 | "linewidth": 1, 1651 | "links": [], 1652 | "nullPointMode": "null", 1653 | "options": { 1654 | "alertThreshold": true 1655 | }, 1656 | "percentage": false, 1657 | "pluginVersion": "9.2.0", 1658 | "pointradius": 5, 1659 | "points": false, 1660 | "renderer": "flot", 1661 | "seriesOverrides": [], 1662 | "spaceLength": 10, 1663 | "stack": false, 1664 | "steppedLine": false, 1665 | "targets": [ 1666 | { 1667 | "datasource": { 1668 | "type": "prometheus", 1669 | "uid": "PBFA97CFB590B2093" 1670 | }, 1671 | "expr": "rate(grafana_api_dashboard_save_milliseconds_sum{kubernetes_pod_name=\"$pod\"}[5m]) / rate(grafana_api_dashboard_save_milliseconds_count{kubernetes_pod_name=\"$pod\"}[5m])", 1672 | "format": "time_series", 1673 | "intervalFactor": 2, 1674 | "legendFormat": "0.5", 1675 | "refId": "A" 1676 | } 1677 | ], 1678 | "thresholds": [], 1679 | "timeRegions": [], 1680 | "title": "Dashboard Save Duration Avg [5 min]", 1681 | "tooltip": { 1682 | "shared": true, 1683 | "sort": 0, 1684 | "value_type": "individual" 1685 | }, 1686 | "type": "graph", 1687 | "xaxis": { 1688 | "mode": "time", 1689 | "show": true, 1690 | "values": [] 1691 | }, 1692 | "yaxes": [ 1693 | { 1694 | "format": "ms", 1695 | "logBase": 1, 1696 | "show": true 1697 | }, 1698 | { 1699 | "format": "short", 1700 | "logBase": 1, 1701 | "show": true 1702 | } 1703 | ], 1704 | "yaxis": { 1705 | "align": false 1706 | } 1707 | }, 1708 | { 1709 | "collapsed": false, 1710 | "datasource": { 1711 | "type": "datasource", 1712 | "uid": "grafana" 1713 | }, 1714 | "gridPos": { 1715 | "h": 1, 1716 | "w": 24, 1717 | "x": 0, 1718 | "y": 48 1719 | }, 1720 | "id": 27, 1721 | "panels": [], 1722 | "targets": [ 1723 | { 1724 | "datasource": { 1725 | "type": "datasource", 1726 | "uid": "grafana" 1727 | }, 1728 | "refId": "A" 1729 | } 1730 | ], 1731 | "title": "Dashboard Search Stats", 1732 | "type": "row" 1733 | }, 1734 | { 1735 | "aliasColors": {}, 1736 | "bars": false, 1737 | "dashLength": 10, 1738 | "dashes": false, 1739 | "datasource": { 1740 | "type": "prometheus", 1741 | "uid": "PBFA97CFB590B2093" 1742 | }, 1743 | "description": "summary for dashboard search duration", 1744 | "fill": 1, 1745 | "fillGradient": 0, 1746 | "gridPos": { 1747 | "h": 7, 1748 | "w": 12, 1749 | "x": 0, 1750 | "y": 49 1751 | }, 1752 | "hiddenSeries": false, 1753 | "id": 12, 1754 | "legend": { 1755 | "alignAsTable": true, 1756 | "avg": true, 1757 | "current": true, 1758 | "max": true, 1759 | "min": true, 1760 | "show": true, 1761 | "total": false, 1762 | "values": true 1763 | }, 1764 | "lines": true, 1765 | "linewidth": 1, 1766 | "links": [], 1767 | "nullPointMode": "null", 1768 | "options": { 1769 | "alertThreshold": true 1770 | }, 1771 | "percentage": false, 1772 | "pluginVersion": "9.2.0", 1773 | "pointradius": 5, 1774 | "points": false, 1775 | "renderer": "flot", 1776 | "seriesOverrides": [], 1777 | "spaceLength": 10, 1778 | "stack": false, 1779 | "steppedLine": false, 1780 | "targets": [ 1781 | { 1782 | "datasource": { 1783 | "type": "prometheus", 1784 | "uid": "PBFA97CFB590B2093" 1785 | }, 1786 | "expr": "grafana_api_dashboard_search_milliseconds{kubernetes_pod_name=\"$pod\", quantile=\"0.5\"}", 1787 | "format": "time_series", 1788 | "intervalFactor": 2, 1789 | "legendFormat": "0.5", 1790 | "refId": "A" 1791 | }, 1792 | { 1793 | "datasource": { 1794 | "type": "prometheus", 1795 | "uid": "PBFA97CFB590B2093" 1796 | }, 1797 | "expr": "grafana_api_dashboard_search_milliseconds{kubernetes_pod_name=\"$pod\", quantile=\"0.9\"}", 1798 | "format": "time_series", 1799 | "intervalFactor": 2, 1800 | "legendFormat": "0.9", 1801 | "refId": "B" 1802 | }, 1803 | { 1804 | "datasource": { 1805 | "type": "prometheus", 1806 | "uid": "PBFA97CFB590B2093" 1807 | }, 1808 | "expr": "grafana_api_dashboard_search_milliseconds{kubernetes_pod_name=\"$pod\", quantile=\"0.99\"}", 1809 | "format": "time_series", 1810 | "intervalFactor": 2, 1811 | "legendFormat": "0.99", 1812 | "refId": "C" 1813 | } 1814 | ], 1815 | "thresholds": [], 1816 | "timeRegions": [], 1817 | "title": "Dashboard Search Duration Quantiles", 1818 | "tooltip": { 1819 | "shared": true, 1820 | "sort": 0, 1821 | "value_type": "individual" 1822 | }, 1823 | "type": "graph", 1824 | "xaxis": { 1825 | "mode": "time", 1826 | "show": true, 1827 | "values": [] 1828 | }, 1829 | "yaxes": [ 1830 | { 1831 | "format": "ms", 1832 | "logBase": 1, 1833 | "show": true 1834 | }, 1835 | { 1836 | "format": "short", 1837 | "logBase": 1, 1838 | "show": true 1839 | } 1840 | ], 1841 | "yaxis": { 1842 | "align": false 1843 | } 1844 | }, 1845 | { 1846 | "aliasColors": {}, 1847 | "bars": false, 1848 | "dashLength": 10, 1849 | "dashes": false, 1850 | "datasource": { 1851 | "type": "prometheus", 1852 | "uid": "PBFA97CFB590B2093" 1853 | }, 1854 | "description": "average dashboard search duration over 5 minutes", 1855 | "fill": 1, 1856 | "fillGradient": 0, 1857 | "gridPos": { 1858 | "h": 7, 1859 | "w": 12, 1860 | "x": 12, 1861 | "y": 49 1862 | }, 1863 | "hiddenSeries": false, 1864 | "id": 13, 1865 | "legend": { 1866 | "alignAsTable": true, 1867 | "avg": true, 1868 | "current": true, 1869 | "max": true, 1870 | "min": true, 1871 | "show": true, 1872 | "total": false, 1873 | "values": true 1874 | }, 1875 | "lines": true, 1876 | "linewidth": 1, 1877 | "links": [], 1878 | "nullPointMode": "null", 1879 | "options": { 1880 | "alertThreshold": true 1881 | }, 1882 | "percentage": false, 1883 | "pluginVersion": "9.2.0", 1884 | "pointradius": 5, 1885 | "points": false, 1886 | "renderer": "flot", 1887 | "seriesOverrides": [], 1888 | "spaceLength": 10, 1889 | "stack": false, 1890 | "steppedLine": false, 1891 | "targets": [ 1892 | { 1893 | "datasource": { 1894 | "type": "prometheus", 1895 | "uid": "PBFA97CFB590B2093" 1896 | }, 1897 | "expr": "rate(grafana_api_dashboard_search_milliseconds_sum{kubernetes_pod_name=\"$pod\"}[5m]) / rate(grafana_api_dashboard_search_milliseconds_count{kubernetes_pod_name=\"$pod\"}[5m])", 1898 | "format": "time_series", 1899 | "intervalFactor": 2, 1900 | "legendFormat": "0.5", 1901 | "refId": "A" 1902 | } 1903 | ], 1904 | "thresholds": [], 1905 | "timeRegions": [], 1906 | "title": "Dashboard Search Duration Avg [5 min]", 1907 | "tooltip": { 1908 | "shared": true, 1909 | "sort": 0, 1910 | "value_type": "individual" 1911 | }, 1912 | "type": "graph", 1913 | "xaxis": { 1914 | "mode": "time", 1915 | "show": true, 1916 | "values": [] 1917 | }, 1918 | "yaxes": [ 1919 | { 1920 | "format": "ms", 1921 | "logBase": 1, 1922 | "show": true 1923 | }, 1924 | { 1925 | "format": "short", 1926 | "logBase": 1, 1927 | "show": true 1928 | } 1929 | ], 1930 | "yaxis": { 1931 | "align": false 1932 | } 1933 | } 1934 | ], 1935 | "refresh": false, 1936 | "schemaVersion": 37, 1937 | "style": "dark", 1938 | "tags": [ 1939 | "grafana", 1940 | "system" 1941 | ], 1942 | "templating": { 1943 | "list": [ 1944 | { 1945 | "current": { 1946 | "isNone": true, 1947 | "selected": false, 1948 | "text": "None", 1949 | "value": "" 1950 | }, 1951 | "datasource": { 1952 | "type": "prometheus", 1953 | "uid": "PBFA97CFB590B2093" 1954 | }, 1955 | "definition": "", 1956 | "hide": 0, 1957 | "includeAll": false, 1958 | "label": "Pod", 1959 | "multi": false, 1960 | "name": "pod", 1961 | "options": [], 1962 | "query": { 1963 | "query": "label_values(grafana_info,kubernetes_pod_name)", 1964 | "refId": "Prometheus-pod-Variable-Query" 1965 | }, 1966 | "refresh": 2, 1967 | "regex": "", 1968 | "skipUrlSync": false, 1969 | "sort": 1, 1970 | "tagValuesQuery": "", 1971 | "tagsQuery": "", 1972 | "type": "query", 1973 | "useTags": false 1974 | }, 1975 | { 1976 | "current": { 1977 | "isNone": true, 1978 | "selected": false, 1979 | "text": "None", 1980 | "value": "" 1981 | }, 1982 | "datasource": { 1983 | "type": "prometheus", 1984 | "uid": "PBFA97CFB590B2093" 1985 | }, 1986 | "definition": "", 1987 | "hide": 0, 1988 | "includeAll": false, 1989 | "label": "Version", 1990 | "multi": false, 1991 | "name": "version", 1992 | "options": [], 1993 | "query": { 1994 | "query": "label_values(grafana_info,version)", 1995 | "refId": "Prometheus-version-Variable-Query" 1996 | }, 1997 | "refresh": 2, 1998 | "regex": "", 1999 | "skipUrlSync": false, 2000 | "sort": 1, 2001 | "tagValuesQuery": "", 2002 | "tagsQuery": "", 2003 | "type": "query", 2004 | "useTags": false 2005 | } 2006 | ] 2007 | }, 2008 | "time": { 2009 | "from": "now-15m", 2010 | "to": "now" 2011 | }, 2012 | "timepicker": { 2013 | "refresh_intervals": [ 2014 | "5s", 2015 | "10s", 2016 | "30s", 2017 | "1m", 2018 | "5m", 2019 | "15m", 2020 | "30m", 2021 | "1h", 2022 | "2h", 2023 | "1d" 2024 | ], 2025 | "time_options": [ 2026 | "5m", 2027 | "15m", 2028 | "1h", 2029 | "6h", 2030 | "12h", 2031 | "24h", 2032 | "2d", 2033 | "7d", 2034 | "30d" 2035 | ] 2036 | }, 2037 | "timezone": "browser", 2038 | "title": "Grafana Internals", 2039 | "uid": "-CrkUySVk", 2040 | "version": 4, 2041 | "weekStart": "" 2042 | } -------------------------------------------------------------------------------- /grafana/provisioning/dashboards/system/system_prometheus_stats.json: -------------------------------------------------------------------------------- 1 | { 2 | "annotations": { 3 | "list": [ 4 | { 5 | "builtIn": 1, 6 | "datasource": { 7 | "type": "datasource", 8 | "uid": "grafana" 9 | }, 10 | "enable": true, 11 | "hide": true, 12 | "iconColor": "rgba(0, 211, 255, 1)", 13 | "name": "Annotations & Alerts", 14 | "target": { 15 | "limit": 100, 16 | "matchAny": false, 17 | "tags": [], 18 | "type": "dashboard" 19 | }, 20 | "type": "dashboard" 21 | } 22 | ] 23 | }, 24 | "editable": true, 25 | "fiscalYearStartMonth": 0, 26 | "gnetId": 15489, 27 | "graphTooltip": 1, 28 | "id": 14, 29 | "links": [ 30 | { 31 | "asDropdown": true, 32 | "icon": "external link", 33 | "includeVars": false, 34 | "keepTime": false, 35 | "tags": [ 36 | "mikrotik" 37 | ], 38 | "targetBlank": false, 39 | "title": "Mikrotik", 40 | "tooltip": "", 41 | "type": "dashboards", 42 | "url": "" 43 | }, 44 | { 45 | "asDropdown": true, 46 | "icon": "external link", 47 | "includeVars": false, 48 | "keepTime": false, 49 | "tags": [ 50 | "system" 51 | ], 52 | "targetBlank": false, 53 | "title": "System", 54 | "tooltip": "", 55 | "type": "dashboards", 56 | "url": "" 57 | } 58 | ], 59 | "liveNow": false, 60 | "panels": [ 61 | { 62 | "aliasColors": { 63 | "prometheus": "#C15C17", 64 | "{instance=\"localhost:9090\",job=\"prometheus\"}": "#CCA300" 65 | }, 66 | "bars": false, 67 | "dashLength": 10, 68 | "dashes": false, 69 | "datasource": { 70 | "type": "prometheus", 71 | "uid": "PBFA97CFB590B2093" 72 | }, 73 | "editable": true, 74 | "error": false, 75 | "fill": 0, 76 | "fillGradient": 0, 77 | "grid": {}, 78 | "gridPos": { 79 | "h": 5, 80 | "w": 6, 81 | "x": 0, 82 | "y": 0 83 | }, 84 | "hiddenSeries": false, 85 | "id": 3, 86 | "legend": { 87 | "avg": false, 88 | "current": false, 89 | "max": false, 90 | "min": false, 91 | "show": true, 92 | "total": false, 93 | "values": false 94 | }, 95 | "lines": true, 96 | "linewidth": 1, 97 | "links": [], 98 | "nullPointMode": "connected", 99 | "options": { 100 | "alertThreshold": true 101 | }, 102 | "percentage": false, 103 | "pluginVersion": "9.2.0", 104 | "pointradius": 2, 105 | "points": false, 106 | "renderer": "flot", 107 | "seriesOverrides": [], 108 | "spaceLength": 10, 109 | "stack": false, 110 | "steppedLine": false, 111 | "targets": [ 112 | { 113 | "datasource": { 114 | "type": "prometheus", 115 | "uid": "PBFA97CFB590B2093" 116 | }, 117 | "expr": "sum(irate(prometheus_tsdb_head_samples_appended_total{job=\"prometheus\"}[5m]))", 118 | "format": "time_series", 119 | "hide": false, 120 | "interval": "", 121 | "intervalFactor": 2, 122 | "legendFormat": "samples", 123 | "metric": "", 124 | "refId": "A", 125 | "step": 20 126 | } 127 | ], 128 | "thresholds": [], 129 | "timeRegions": [], 130 | "title": "Samples Appended", 131 | "tooltip": { 132 | "shared": true, 133 | "sort": 0, 134 | "value_type": "cumulative" 135 | }, 136 | "type": "graph", 137 | "xaxis": { 138 | "mode": "time", 139 | "show": true, 140 | "values": [] 141 | }, 142 | "yaxes": [ 143 | { 144 | "format": "short", 145 | "logBase": 1, 146 | "min": "0", 147 | "show": true 148 | }, 149 | { 150 | "format": "short", 151 | "logBase": 1, 152 | "show": true 153 | } 154 | ], 155 | "yaxis": { 156 | "align": false 157 | } 158 | }, 159 | { 160 | "aliasColors": {}, 161 | "bars": false, 162 | "dashLength": 10, 163 | "dashes": false, 164 | "datasource": { 165 | "type": "prometheus", 166 | "uid": "PBFA97CFB590B2093" 167 | }, 168 | "editable": true, 169 | "error": false, 170 | "fill": 0, 171 | "fillGradient": 0, 172 | "grid": {}, 173 | "gridPos": { 174 | "h": 5, 175 | "w": 6, 176 | "x": 6, 177 | "y": 0 178 | }, 179 | "hiddenSeries": false, 180 | "id": 14, 181 | "legend": { 182 | "avg": false, 183 | "current": false, 184 | "max": false, 185 | "min": false, 186 | "show": true, 187 | "total": false, 188 | "values": false 189 | }, 190 | "lines": true, 191 | "linewidth": 1, 192 | "links": [], 193 | "nullPointMode": "connected", 194 | "options": { 195 | "alertThreshold": true 196 | }, 197 | "percentage": false, 198 | "pluginVersion": "9.2.0", 199 | "pointradius": 5, 200 | "points": false, 201 | "renderer": "flot", 202 | "seriesOverrides": [], 203 | "spaceLength": 10, 204 | "stack": false, 205 | "steppedLine": false, 206 | "targets": [ 207 | { 208 | "datasource": { 209 | "type": "prometheus", 210 | "uid": "PBFA97CFB590B2093" 211 | }, 212 | "expr": "topk(5, max(scrape_duration_seconds) by (job))", 213 | "format": "time_series", 214 | "interval": "", 215 | "intervalFactor": 2, 216 | "legendFormat": "{{job}}", 217 | "metric": "", 218 | "refId": "A", 219 | "step": 20 220 | } 221 | ], 222 | "thresholds": [], 223 | "timeRegions": [], 224 | "title": "Scrape Duration", 225 | "tooltip": { 226 | "shared": true, 227 | "sort": 0, 228 | "value_type": "cumulative" 229 | }, 230 | "type": "graph", 231 | "xaxis": { 232 | "mode": "time", 233 | "show": true, 234 | "values": [] 235 | }, 236 | "yaxes": [ 237 | { 238 | "format": "s", 239 | "logBase": 1, 240 | "show": true 241 | }, 242 | { 243 | "format": "short", 244 | "logBase": 1, 245 | "show": true 246 | } 247 | ], 248 | "yaxis": { 249 | "align": false 250 | } 251 | }, 252 | { 253 | "aliasColors": {}, 254 | "bars": false, 255 | "dashLength": 10, 256 | "dashes": false, 257 | "datasource": { 258 | "type": "prometheus", 259 | "uid": "PBFA97CFB590B2093" 260 | }, 261 | "description": "", 262 | "fill": 0, 263 | "fillGradient": 0, 264 | "gridPos": { 265 | "h": 5, 266 | "w": 6, 267 | "x": 12, 268 | "y": 0 269 | }, 270 | "hiddenSeries": false, 271 | "id": 16, 272 | "legend": { 273 | "avg": false, 274 | "current": false, 275 | "max": false, 276 | "min": false, 277 | "show": true, 278 | "total": false, 279 | "values": false 280 | }, 281 | "lines": true, 282 | "linewidth": 1, 283 | "links": [], 284 | "nullPointMode": "null", 285 | "options": { 286 | "alertThreshold": true 287 | }, 288 | "percentage": false, 289 | "pluginVersion": "9.2.0", 290 | "pointradius": 5, 291 | "points": false, 292 | "renderer": "flot", 293 | "seriesOverrides": [], 294 | "spaceLength": 10, 295 | "stack": false, 296 | "steppedLine": false, 297 | "targets": [ 298 | { 299 | "datasource": { 300 | "type": "prometheus", 301 | "uid": "PBFA97CFB590B2093" 302 | }, 303 | "expr": "sum(process_resident_memory_bytes{job=\"prometheus\"})", 304 | "format": "time_series", 305 | "hide": false, 306 | "interval": "", 307 | "intervalFactor": 2, 308 | "legendFormat": "p8s process resident memory", 309 | "refId": "D", 310 | "step": 20 311 | }, 312 | { 313 | "datasource": { 314 | "type": "prometheus", 315 | "uid": "PBFA97CFB590B2093" 316 | }, 317 | "expr": "process_virtual_memory_bytes{job=\"prometheus\"}", 318 | "format": "time_series", 319 | "hide": false, 320 | "intervalFactor": 2, 321 | "legendFormat": "virtual memory", 322 | "refId": "C", 323 | "step": 20 324 | } 325 | ], 326 | "thresholds": [], 327 | "timeRegions": [], 328 | "title": "Memory Profile", 329 | "tooltip": { 330 | "shared": true, 331 | "sort": 2, 332 | "value_type": "individual" 333 | }, 334 | "type": "graph", 335 | "xaxis": { 336 | "mode": "time", 337 | "show": true, 338 | "values": [] 339 | }, 340 | "yaxes": [ 341 | { 342 | "format": "bytes", 343 | "label": "", 344 | "logBase": 1, 345 | "min": "0", 346 | "show": true 347 | }, 348 | { 349 | "format": "short", 350 | "logBase": 1, 351 | "show": true 352 | } 353 | ], 354 | "yaxis": { 355 | "align": false 356 | } 357 | }, 358 | { 359 | "datasource": { 360 | "type": "prometheus", 361 | "uid": "PBFA97CFB590B2093" 362 | }, 363 | "fieldConfig": { 364 | "defaults": { 365 | "color": { 366 | "mode": "thresholds" 367 | }, 368 | "mappings": [ 369 | { 370 | "options": { 371 | "0": { 372 | "text": "None" 373 | } 374 | }, 375 | "type": "value" 376 | } 377 | ], 378 | "thresholds": { 379 | "mode": "absolute", 380 | "steps": [ 381 | { 382 | "color": "rgba(50, 172, 45, 0.97)", 383 | "value": null 384 | }, 385 | { 386 | "color": "rgba(237, 129, 40, 0.89)", 387 | "value": 0.1 388 | }, 389 | { 390 | "color": "rgba(245, 54, 54, 0.9)", 391 | "value": 1 392 | } 393 | ] 394 | }, 395 | "unit": "none" 396 | }, 397 | "overrides": [] 398 | }, 399 | "gridPos": { 400 | "h": 5, 401 | "w": 6, 402 | "x": 18, 403 | "y": 0 404 | }, 405 | "id": 37, 406 | "links": [], 407 | "maxDataPoints": 100, 408 | "options": { 409 | "colorMode": "value", 410 | "graphMode": "none", 411 | "justifyMode": "auto", 412 | "orientation": "horizontal", 413 | "reduceOptions": { 414 | "calcs": [ 415 | "max" 416 | ], 417 | "fields": "", 418 | "values": false 419 | }, 420 | "textMode": "auto" 421 | }, 422 | "pluginVersion": "9.2.0", 423 | "targets": [ 424 | { 425 | "datasource": { 426 | "type": "prometheus", 427 | "uid": "PBFA97CFB590B2093" 428 | }, 429 | "expr": "prometheus_tsdb_wal_corruptions_total{job=\"prometheus\"}", 430 | "format": "time_series", 431 | "intervalFactor": 2, 432 | "legendFormat": "", 433 | "refId": "A", 434 | "step": 60 435 | } 436 | ], 437 | "title": "WAL Corruptions", 438 | "type": "stat" 439 | }, 440 | { 441 | "aliasColors": {}, 442 | "bars": false, 443 | "dashLength": 10, 444 | "dashes": false, 445 | "datasource": { 446 | "type": "prometheus", 447 | "uid": "PBFA97CFB590B2093" 448 | }, 449 | "fill": 0, 450 | "fillGradient": 0, 451 | "gridPos": { 452 | "h": 5, 453 | "w": 6, 454 | "x": 0, 455 | "y": 5 456 | }, 457 | "hiddenSeries": false, 458 | "id": 29, 459 | "legend": { 460 | "avg": false, 461 | "current": false, 462 | "max": false, 463 | "min": false, 464 | "show": true, 465 | "total": false, 466 | "values": false 467 | }, 468 | "lines": true, 469 | "linewidth": 1, 470 | "links": [], 471 | "nullPointMode": "null", 472 | "options": { 473 | "alertThreshold": true 474 | }, 475 | "percentage": false, 476 | "pluginVersion": "9.2.0", 477 | "pointradius": 5, 478 | "points": false, 479 | "renderer": "flot", 480 | "seriesOverrides": [], 481 | "spaceLength": 10, 482 | "stack": false, 483 | "steppedLine": false, 484 | "targets": [ 485 | { 486 | "datasource": { 487 | "type": "prometheus", 488 | "uid": "PBFA97CFB590B2093" 489 | }, 490 | "expr": "sum(prometheus_tsdb_head_active_appenders{job=\"prometheus\"})", 491 | "format": "time_series", 492 | "interval": "", 493 | "intervalFactor": 2, 494 | "legendFormat": "active_appenders", 495 | "metric": "", 496 | "refId": "A", 497 | "step": 20 498 | }, 499 | { 500 | "datasource": { 501 | "type": "prometheus", 502 | "uid": "PBFA97CFB590B2093" 503 | }, 504 | "expr": "sum(process_open_fds{job=\"prometheus\"})", 505 | "format": "time_series", 506 | "interval": "", 507 | "intervalFactor": 2, 508 | "legendFormat": "open_fds", 509 | "refId": "B", 510 | "step": 20 511 | } 512 | ], 513 | "thresholds": [], 514 | "timeRegions": [], 515 | "title": "Active Appenders", 516 | "tooltip": { 517 | "shared": true, 518 | "sort": 0, 519 | "value_type": "individual" 520 | }, 521 | "type": "graph", 522 | "xaxis": { 523 | "mode": "time", 524 | "show": true, 525 | "values": [] 526 | }, 527 | "yaxes": [ 528 | { 529 | "format": "short", 530 | "logBase": 1, 531 | "show": true 532 | }, 533 | { 534 | "format": "short", 535 | "logBase": 1, 536 | "show": false 537 | } 538 | ], 539 | "yaxis": { 540 | "align": false 541 | } 542 | }, 543 | { 544 | "aliasColors": { 545 | "prometheus": "#F9BA8F", 546 | "{instance=\"localhost:9090\",interval=\"5s\",job=\"prometheus\"}": "#F9BA8F" 547 | }, 548 | "bars": false, 549 | "dashLength": 10, 550 | "dashes": false, 551 | "datasource": { 552 | "type": "prometheus", 553 | "uid": "PBFA97CFB590B2093" 554 | }, 555 | "editable": true, 556 | "error": false, 557 | "fill": 0, 558 | "fillGradient": 0, 559 | "grid": {}, 560 | "gridPos": { 561 | "h": 5, 562 | "w": 6, 563 | "x": 6, 564 | "y": 5 565 | }, 566 | "hiddenSeries": false, 567 | "id": 2, 568 | "legend": { 569 | "avg": false, 570 | "current": false, 571 | "max": false, 572 | "min": false, 573 | "show": true, 574 | "total": false, 575 | "values": false 576 | }, 577 | "lines": true, 578 | "linewidth": 1, 579 | "links": [], 580 | "nullPointMode": "connected", 581 | "options": { 582 | "alertThreshold": true 583 | }, 584 | "percentage": false, 585 | "pluginVersion": "9.2.0", 586 | "pointradius": 5, 587 | "points": false, 588 | "renderer": "flot", 589 | "seriesOverrides": [], 590 | "spaceLength": 10, 591 | "stack": false, 592 | "steppedLine": false, 593 | "targets": [ 594 | { 595 | "datasource": { 596 | "type": "prometheus", 597 | "uid": "PBFA97CFB590B2093" 598 | }, 599 | "expr": "prometheus_tsdb_blocks_loaded{job=\"prometheus\"}", 600 | "format": "time_series", 601 | "intervalFactor": 2, 602 | "legendFormat": "blocks", 603 | "refId": "A", 604 | "step": 20 605 | } 606 | ], 607 | "thresholds": [], 608 | "timeRegions": [], 609 | "title": "Blocks Loaded", 610 | "tooltip": { 611 | "shared": true, 612 | "sort": 0, 613 | "value_type": "cumulative" 614 | }, 615 | "type": "graph", 616 | "xaxis": { 617 | "mode": "time", 618 | "show": true, 619 | "values": [] 620 | }, 621 | "yaxes": [ 622 | { 623 | "format": "short", 624 | "logBase": 1, 625 | "show": true 626 | }, 627 | { 628 | "format": "short", 629 | "logBase": 1, 630 | "show": true 631 | } 632 | ], 633 | "yaxis": { 634 | "align": false 635 | } 636 | }, 637 | { 638 | "aliasColors": {}, 639 | "bars": false, 640 | "dashLength": 10, 641 | "dashes": false, 642 | "datasource": { 643 | "type": "prometheus", 644 | "uid": "PBFA97CFB590B2093" 645 | }, 646 | "description": "", 647 | "fill": 0, 648 | "fillGradient": 0, 649 | "gridPos": { 650 | "h": 5, 651 | "w": 6, 652 | "x": 12, 653 | "y": 5 654 | }, 655 | "hiddenSeries": false, 656 | "id": 33, 657 | "legend": { 658 | "avg": false, 659 | "current": false, 660 | "max": false, 661 | "min": false, 662 | "show": true, 663 | "total": false, 664 | "values": false 665 | }, 666 | "lines": true, 667 | "linewidth": 1, 668 | "links": [], 669 | "nullPointMode": "connected", 670 | "options": { 671 | "alertThreshold": true 672 | }, 673 | "percentage": false, 674 | "pluginVersion": "9.2.0", 675 | "pointradius": 5, 676 | "points": false, 677 | "renderer": "flot", 678 | "seriesOverrides": [], 679 | "spaceLength": 10, 680 | "stack": false, 681 | "steppedLine": false, 682 | "targets": [ 683 | { 684 | "datasource": { 685 | "type": "prometheus", 686 | "uid": "PBFA97CFB590B2093" 687 | }, 688 | "expr": "prometheus_tsdb_head_chunks{job=\"prometheus\"}", 689 | "format": "time_series", 690 | "interval": "", 691 | "intervalFactor": 2, 692 | "legendFormat": "chunks", 693 | "refId": "A", 694 | "step": 20 695 | } 696 | ], 697 | "thresholds": [], 698 | "timeRegions": [], 699 | "title": "Head Chunks", 700 | "tooltip": { 701 | "shared": true, 702 | "sort": 0, 703 | "value_type": "individual" 704 | }, 705 | "type": "graph", 706 | "xaxis": { 707 | "mode": "time", 708 | "show": true, 709 | "values": [] 710 | }, 711 | "yaxes": [ 712 | { 713 | "format": "short", 714 | "logBase": 1, 715 | "show": true 716 | }, 717 | { 718 | "format": "bytes", 719 | "label": "", 720 | "logBase": 1, 721 | "show": false 722 | } 723 | ], 724 | "yaxis": { 725 | "align": false 726 | } 727 | }, 728 | { 729 | "aliasColors": {}, 730 | "bars": false, 731 | "dashLength": 10, 732 | "dashes": false, 733 | "datasource": { 734 | "type": "prometheus", 735 | "uid": "PBFA97CFB590B2093" 736 | }, 737 | "fill": 1, 738 | "fillGradient": 0, 739 | "gridPos": { 740 | "h": 5, 741 | "w": 6, 742 | "x": 18, 743 | "y": 5 744 | }, 745 | "hiddenSeries": false, 746 | "id": 36, 747 | "legend": { 748 | "avg": false, 749 | "current": false, 750 | "max": false, 751 | "min": false, 752 | "show": true, 753 | "total": false, 754 | "values": false 755 | }, 756 | "lines": true, 757 | "linewidth": 1, 758 | "links": [], 759 | "nullPointMode": "null", 760 | "options": { 761 | "alertThreshold": true 762 | }, 763 | "percentage": false, 764 | "pluginVersion": "9.2.0", 765 | "pointradius": 5, 766 | "points": false, 767 | "renderer": "flot", 768 | "seriesOverrides": [ 769 | { 770 | "alias": "duration-p99", 771 | "yaxis": 2 772 | } 773 | ], 774 | "spaceLength": 10, 775 | "stack": false, 776 | "steppedLine": false, 777 | "targets": [ 778 | { 779 | "datasource": { 780 | "type": "prometheus", 781 | "uid": "PBFA97CFB590B2093" 782 | }, 783 | "expr": "prometheus_tsdb_head_gc_duration_seconds{job=\"prometheus\",quantile=\"0.99\"}", 784 | "format": "time_series", 785 | "intervalFactor": 2, 786 | "legendFormat": "duration-p99", 787 | "refId": "A", 788 | "step": 20 789 | }, 790 | { 791 | "datasource": { 792 | "type": "prometheus", 793 | "uid": "PBFA97CFB590B2093" 794 | }, 795 | "expr": "irate(prometheus_tsdb_head_gc_duration_seconds_count{job=\"prometheus\"}[5m])", 796 | "format": "time_series", 797 | "intervalFactor": 2, 798 | "legendFormat": "collections", 799 | "refId": "B", 800 | "step": 20 801 | } 802 | ], 803 | "thresholds": [], 804 | "timeRegions": [], 805 | "title": "Head Block GC Activity", 806 | "tooltip": { 807 | "shared": true, 808 | "sort": 0, 809 | "value_type": "individual" 810 | }, 811 | "type": "graph", 812 | "xaxis": { 813 | "mode": "time", 814 | "show": true, 815 | "values": [] 816 | }, 817 | "yaxes": [ 818 | { 819 | "format": "short", 820 | "logBase": 1, 821 | "min": "0", 822 | "show": true 823 | }, 824 | { 825 | "format": "s", 826 | "logBase": 1, 827 | "min": "0", 828 | "show": true 829 | } 830 | ], 831 | "yaxis": { 832 | "align": false 833 | } 834 | }, 835 | { 836 | "aliasColors": {}, 837 | "bars": false, 838 | "dashLength": 10, 839 | "dashes": false, 840 | "datasource": { 841 | "type": "prometheus", 842 | "uid": "PBFA97CFB590B2093" 843 | }, 844 | "description": "", 845 | "fill": 0, 846 | "fillGradient": 0, 847 | "gridPos": { 848 | "h": 5, 849 | "w": 8, 850 | "x": 0, 851 | "y": 10 852 | }, 853 | "hiddenSeries": false, 854 | "id": 20, 855 | "legend": { 856 | "avg": false, 857 | "current": false, 858 | "max": false, 859 | "min": false, 860 | "show": true, 861 | "total": false, 862 | "values": false 863 | }, 864 | "lines": true, 865 | "linewidth": 1, 866 | "links": [], 867 | "nullPointMode": "connected", 868 | "options": { 869 | "alertThreshold": true 870 | }, 871 | "percentage": false, 872 | "pluginVersion": "9.2.0", 873 | "pointradius": 5, 874 | "points": false, 875 | "renderer": "flot", 876 | "seriesOverrides": [ 877 | { 878 | "alias": "duration-p99", 879 | "yaxis": 2 880 | } 881 | ], 882 | "spaceLength": 10, 883 | "stack": false, 884 | "steppedLine": false, 885 | "targets": [ 886 | { 887 | "datasource": { 888 | "type": "prometheus", 889 | "uid": "PBFA97CFB590B2093" 890 | }, 891 | "expr": "histogram_quantile(0.99, sum(rate(prometheus_tsdb_compaction_duration_bucket{job=\"prometheus\"}[5m])) by (le))", 892 | "format": "time_series", 893 | "hide": false, 894 | "interval": "", 895 | "intervalFactor": 2, 896 | "legendFormat": "duration-{{p99}}", 897 | "refId": "A", 898 | "step": 20 899 | }, 900 | { 901 | "datasource": { 902 | "type": "prometheus", 903 | "uid": "PBFA97CFB590B2093" 904 | }, 905 | "expr": "irate(prometheus_tsdb_compactions_total{job=\"prometheus\"}[5m])", 906 | "format": "time_series", 907 | "intervalFactor": 2, 908 | "legendFormat": "compactions", 909 | "refId": "B", 910 | "step": 20 911 | }, 912 | { 913 | "datasource": { 914 | "type": "prometheus", 915 | "uid": "PBFA97CFB590B2093" 916 | }, 917 | "expr": "irate(prometheus_tsdb_compactions_failed_total{job=\"prometheus\"}[5m])", 918 | "format": "time_series", 919 | "intervalFactor": 2, 920 | "legendFormat": "failed", 921 | "refId": "C", 922 | "step": 20 923 | }, 924 | { 925 | "datasource": { 926 | "type": "prometheus", 927 | "uid": "PBFA97CFB590B2093" 928 | }, 929 | "expr": "irate(prometheus_tsdb_compactions_triggered_total{job=\"prometheus\"}[5m])", 930 | "format": "time_series", 931 | "intervalFactor": 2, 932 | "legendFormat": "triggered", 933 | "refId": "D", 934 | "step": 20 935 | } 936 | ], 937 | "thresholds": [], 938 | "timeRegions": [], 939 | "title": "Compaction Activity", 940 | "tooltip": { 941 | "shared": true, 942 | "sort": 0, 943 | "value_type": "individual" 944 | }, 945 | "type": "graph", 946 | "xaxis": { 947 | "mode": "time", 948 | "show": true, 949 | "values": [] 950 | }, 951 | "yaxes": [ 952 | { 953 | "format": "short", 954 | "logBase": 1, 955 | "min": "0", 956 | "show": true 957 | }, 958 | { 959 | "format": "s", 960 | "label": "", 961 | "logBase": 1, 962 | "min": "0", 963 | "show": true 964 | } 965 | ], 966 | "yaxis": { 967 | "align": false 968 | } 969 | }, 970 | { 971 | "aliasColors": {}, 972 | "bars": false, 973 | "dashLength": 10, 974 | "dashes": false, 975 | "datasource": { 976 | "type": "prometheus", 977 | "uid": "PBFA97CFB590B2093" 978 | }, 979 | "fill": 1, 980 | "fillGradient": 0, 981 | "gridPos": { 982 | "h": 5, 983 | "w": 8, 984 | "x": 8, 985 | "y": 10 986 | }, 987 | "hiddenSeries": false, 988 | "id": 32, 989 | "legend": { 990 | "avg": false, 991 | "current": false, 992 | "max": false, 993 | "min": false, 994 | "show": true, 995 | "total": false, 996 | "values": false 997 | }, 998 | "lines": true, 999 | "linewidth": 1, 1000 | "links": [], 1001 | "nullPointMode": "null", 1002 | "options": { 1003 | "alertThreshold": true 1004 | }, 1005 | "percentage": false, 1006 | "pluginVersion": "9.2.0", 1007 | "pointradius": 5, 1008 | "points": false, 1009 | "renderer": "flot", 1010 | "seriesOverrides": [], 1011 | "spaceLength": 10, 1012 | "stack": false, 1013 | "steppedLine": false, 1014 | "targets": [ 1015 | { 1016 | "datasource": { 1017 | "type": "prometheus", 1018 | "uid": "PBFA97CFB590B2093" 1019 | }, 1020 | "expr": "rate(prometheus_tsdb_reloads_total{job=\"prometheus\"}[5m])", 1021 | "format": "time_series", 1022 | "intervalFactor": 2, 1023 | "legendFormat": "reloads", 1024 | "refId": "A", 1025 | "step": 20 1026 | }, 1027 | { 1028 | "datasource": { 1029 | "type": "prometheus", 1030 | "uid": "PBFA97CFB590B2093" 1031 | }, 1032 | "expr": "rate(prometheus_tsdb_reloads_failures_total{job=\"prometheus\"}[5m])", 1033 | "format": "time_series", 1034 | "hide": false, 1035 | "intervalFactor": 2, 1036 | "legendFormat": "failures", 1037 | "refId": "B", 1038 | "step": 20 1039 | } 1040 | ], 1041 | "thresholds": [], 1042 | "timeRegions": [], 1043 | "title": "Reload Count", 1044 | "tooltip": { 1045 | "shared": true, 1046 | "sort": 0, 1047 | "value_type": "individual" 1048 | }, 1049 | "type": "graph", 1050 | "xaxis": { 1051 | "mode": "time", 1052 | "show": true, 1053 | "values": [] 1054 | }, 1055 | "yaxes": [ 1056 | { 1057 | "format": "short", 1058 | "logBase": 1, 1059 | "show": true 1060 | }, 1061 | { 1062 | "format": "short", 1063 | "logBase": 1, 1064 | "show": true 1065 | } 1066 | ], 1067 | "yaxis": { 1068 | "align": false 1069 | } 1070 | }, 1071 | { 1072 | "aliasColors": {}, 1073 | "bars": false, 1074 | "dashLength": 10, 1075 | "dashes": false, 1076 | "datasource": { 1077 | "type": "prometheus", 1078 | "uid": "PBFA97CFB590B2093" 1079 | }, 1080 | "fill": 0, 1081 | "fillGradient": 0, 1082 | "gridPos": { 1083 | "h": 5, 1084 | "w": 8, 1085 | "x": 16, 1086 | "y": 10 1087 | }, 1088 | "hiddenSeries": false, 1089 | "id": 38, 1090 | "legend": { 1091 | "avg": false, 1092 | "current": false, 1093 | "max": false, 1094 | "min": false, 1095 | "show": true, 1096 | "total": false, 1097 | "values": false 1098 | }, 1099 | "lines": true, 1100 | "linewidth": 1, 1101 | "links": [], 1102 | "nullPointMode": "null", 1103 | "options": { 1104 | "alertThreshold": true 1105 | }, 1106 | "percentage": false, 1107 | "pluginVersion": "9.2.0", 1108 | "pointradius": 5, 1109 | "points": false, 1110 | "renderer": "flot", 1111 | "seriesOverrides": [], 1112 | "spaceLength": 10, 1113 | "stack": false, 1114 | "steppedLine": false, 1115 | "targets": [ 1116 | { 1117 | "datasource": { 1118 | "type": "prometheus", 1119 | "uid": "PBFA97CFB590B2093" 1120 | }, 1121 | "expr": "prometheus_engine_query_duration_seconds{job=\"prometheus\", quantile=\"0.99\"}", 1122 | "format": "time_series", 1123 | "intervalFactor": 2, 1124 | "legendFormat": "{{slice}}_p99", 1125 | "refId": "A", 1126 | "step": 20 1127 | } 1128 | ], 1129 | "thresholds": [], 1130 | "timeRegions": [], 1131 | "title": "Query Durations", 1132 | "tooltip": { 1133 | "shared": true, 1134 | "sort": 0, 1135 | "value_type": "individual" 1136 | }, 1137 | "type": "graph", 1138 | "xaxis": { 1139 | "mode": "time", 1140 | "show": true, 1141 | "values": [] 1142 | }, 1143 | "yaxes": [ 1144 | { 1145 | "format": "short", 1146 | "logBase": 1, 1147 | "show": true 1148 | }, 1149 | { 1150 | "format": "short", 1151 | "logBase": 1, 1152 | "show": true 1153 | } 1154 | ], 1155 | "yaxis": { 1156 | "align": false 1157 | } 1158 | }, 1159 | { 1160 | "aliasColors": {}, 1161 | "bars": false, 1162 | "dashLength": 10, 1163 | "dashes": false, 1164 | "datasource": { 1165 | "type": "prometheus", 1166 | "uid": "PBFA97CFB590B2093" 1167 | }, 1168 | "editable": true, 1169 | "error": false, 1170 | "fill": 0, 1171 | "fillGradient": 0, 1172 | "grid": {}, 1173 | "gridPos": { 1174 | "h": 7, 1175 | "w": 12, 1176 | "x": 0, 1177 | "y": 15 1178 | }, 1179 | "hiddenSeries": false, 1180 | "id": 35, 1181 | "legend": { 1182 | "alignAsTable": false, 1183 | "avg": false, 1184 | "current": false, 1185 | "hideEmpty": true, 1186 | "max": false, 1187 | "min": false, 1188 | "show": true, 1189 | "total": false, 1190 | "values": false 1191 | }, 1192 | "lines": true, 1193 | "linewidth": 1, 1194 | "links": [], 1195 | "nullPointMode": "connected", 1196 | "options": { 1197 | "alertThreshold": true 1198 | }, 1199 | "percentage": false, 1200 | "pluginVersion": "9.2.0", 1201 | "pointradius": 5, 1202 | "points": false, 1203 | "renderer": "flot", 1204 | "seriesOverrides": [], 1205 | "spaceLength": 10, 1206 | "stack": false, 1207 | "steppedLine": false, 1208 | "targets": [ 1209 | { 1210 | "datasource": { 1211 | "type": "prometheus", 1212 | "uid": "PBFA97CFB590B2093" 1213 | }, 1214 | "expr": "max(prometheus_rule_group_duration_seconds{job=\"prometheus\"}) by (quantile)", 1215 | "format": "time_series", 1216 | "interval": "", 1217 | "intervalFactor": 2, 1218 | "legendFormat": "{{quantile}}", 1219 | "refId": "A", 1220 | "step": 10 1221 | } 1222 | ], 1223 | "thresholds": [], 1224 | "timeRegions": [], 1225 | "title": "Rule Group Eval Duration", 1226 | "tooltip": { 1227 | "shared": true, 1228 | "sort": 0, 1229 | "value_type": "cumulative" 1230 | }, 1231 | "type": "graph", 1232 | "xaxis": { 1233 | "mode": "time", 1234 | "show": true, 1235 | "values": [] 1236 | }, 1237 | "yaxes": [ 1238 | { 1239 | "format": "s", 1240 | "label": "", 1241 | "logBase": 1, 1242 | "show": true 1243 | }, 1244 | { 1245 | "format": "short", 1246 | "logBase": 1, 1247 | "show": true 1248 | } 1249 | ], 1250 | "yaxis": { 1251 | "align": false 1252 | } 1253 | }, 1254 | { 1255 | "aliasColors": {}, 1256 | "bars": false, 1257 | "dashLength": 10, 1258 | "dashes": false, 1259 | "datasource": { 1260 | "type": "prometheus", 1261 | "uid": "PBFA97CFB590B2093" 1262 | }, 1263 | "fill": 1, 1264 | "fillGradient": 0, 1265 | "gridPos": { 1266 | "h": 7, 1267 | "w": 12, 1268 | "x": 12, 1269 | "y": 15 1270 | }, 1271 | "hiddenSeries": false, 1272 | "id": 39, 1273 | "legend": { 1274 | "avg": false, 1275 | "current": false, 1276 | "max": false, 1277 | "min": false, 1278 | "show": true, 1279 | "total": false, 1280 | "values": false 1281 | }, 1282 | "lines": true, 1283 | "linewidth": 1, 1284 | "links": [], 1285 | "nullPointMode": "null", 1286 | "options": { 1287 | "alertThreshold": true 1288 | }, 1289 | "percentage": false, 1290 | "pluginVersion": "9.2.0", 1291 | "pointradius": 5, 1292 | "points": false, 1293 | "renderer": "flot", 1294 | "seriesOverrides": [], 1295 | "spaceLength": 10, 1296 | "stack": true, 1297 | "steppedLine": false, 1298 | "targets": [ 1299 | { 1300 | "datasource": { 1301 | "type": "prometheus", 1302 | "uid": "PBFA97CFB590B2093" 1303 | }, 1304 | "expr": "rate(prometheus_rule_group_iterations_missed_total{job=\"prometheus\"}[5m])", 1305 | "format": "time_series", 1306 | "intervalFactor": 2, 1307 | "legendFormat": "missed", 1308 | "refId": "B", 1309 | "step": 10 1310 | }, 1311 | { 1312 | "datasource": { 1313 | "type": "prometheus", 1314 | "uid": "PBFA97CFB590B2093" 1315 | }, 1316 | "expr": "rate(prometheus_rule_group_iterations_total{job=\"prometheus\"}[5m])", 1317 | "format": "time_series", 1318 | "intervalFactor": 2, 1319 | "legendFormat": "iterations", 1320 | "refId": "A", 1321 | "step": 10 1322 | } 1323 | ], 1324 | "thresholds": [], 1325 | "timeRegions": [], 1326 | "title": "Rule Group Eval Activity", 1327 | "tooltip": { 1328 | "shared": true, 1329 | "sort": 0, 1330 | "value_type": "individual" 1331 | }, 1332 | "type": "graph", 1333 | "xaxis": { 1334 | "mode": "time", 1335 | "show": true, 1336 | "values": [] 1337 | }, 1338 | "yaxes": [ 1339 | { 1340 | "format": "short", 1341 | "logBase": 1, 1342 | "show": true 1343 | }, 1344 | { 1345 | "format": "short", 1346 | "logBase": 1, 1347 | "show": true 1348 | } 1349 | ], 1350 | "yaxis": { 1351 | "align": false 1352 | } 1353 | }, 1354 | { 1355 | "datasource": { 1356 | "type": "datasource", 1357 | "uid": "grafana" 1358 | }, 1359 | "gridPos": { 1360 | "h": 3, 1361 | "w": 24, 1362 | "x": 0, 1363 | "y": 22 1364 | }, 1365 | "height": "1px", 1366 | "id": 171, 1367 | "links": [], 1368 | "options": { 1369 | "code": { 1370 | "language": "plaintext", 1371 | "showLineNumbers": false, 1372 | "showMiniMap": false 1373 | }, 1374 | "content": "
\n
\n \n
\n
\n
\n
\n
\n\n
", 1375 | "mode": "html" 1376 | }, 1377 | "pluginVersion": "9.2.0", 1378 | "targets": [ 1379 | { 1380 | "datasource": { 1381 | "type": "datasource", 1382 | "uid": "grafana" 1383 | }, 1384 | "refId": "A" 1385 | } 1386 | ], 1387 | "transparent": true, 1388 | "type": "text" 1389 | } 1390 | ], 1391 | "refresh": "1m", 1392 | "revision": "1.0", 1393 | "schemaVersion": 37, 1394 | "style": "dark", 1395 | "tags": [ 1396 | "prometheus", 1397 | "system" 1398 | ], 1399 | "templating": { 1400 | "list": [] 1401 | }, 1402 | "time": { 1403 | "from": "now-1h", 1404 | "to": "now" 1405 | }, 1406 | "timepicker": { 1407 | "now": true, 1408 | "refresh_intervals": [ 1409 | "5s", 1410 | "10s", 1411 | "30s", 1412 | "1m", 1413 | "5m", 1414 | "15m", 1415 | "30m", 1416 | "1h", 1417 | "2h", 1418 | "1d" 1419 | ], 1420 | "time_options": [ 1421 | "5m", 1422 | "15m", 1423 | "1h", 1424 | "6h", 1425 | "12h", 1426 | "24h", 1427 | "2d", 1428 | "7d", 1429 | "30d" 1430 | ] 1431 | }, 1432 | "timezone": "browser", 1433 | "title": "Prometheus 2.0 Stats", 1434 | "uid": "prometheus", 1435 | "version": 5, 1436 | "weekStart": "" 1437 | } -------------------------------------------------------------------------------- /grafana/provisioning/datasources/loki.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: 1 2 | datasources: 3 | - 4 | access: proxy 5 | basicAuth: false 6 | jsonData: 7 | maxLines: 1000 8 | name: Loki 9 | type: loki 10 | url: "http://loki:3100/" 11 | -------------------------------------------------------------------------------- /grafana/provisioning/datasources/prometheus.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: 1 2 | datasources: 3 | - 4 | access: proxy 5 | basicAuth: false 6 | name: Prometheus 7 | type: prometheus 8 | url: "http://prometheus:9090/" 9 | -------------------------------------------------------------------------------- /loki/loki-config-2.x.yml: -------------------------------------------------------------------------------- 1 | auth_enabled: false 2 | 3 | server: 4 | http_listen_port: 3100 5 | 6 | ingester: 7 | wal: 8 | enabled: false 9 | lifecycler: 10 | address: 127.0.0.1 11 | ring: 12 | kvstore: 13 | store: inmemory 14 | replication_factor: 1 15 | final_sleep: 0s 16 | chunk_idle_period: 1h # Any chunk not receiving new logs in this time will be flushed 17 | max_chunk_age: 1h # All chunks will be flushed when they hit this age, default is 1h 18 | chunk_target_size: 1048576 # Loki will attempt to build chunks up to 1.5MB, flushing first if chunk_idle_period or max_chunk_age is reached first 19 | chunk_retain_period: 30s # Must be greater than index read cache TTL if using an index cache (Default index read cache TTL is 5m) 20 | max_transfer_retries: 0 # Chunk transfers disabled 21 | 22 | schema_config: 23 | configs: 24 | - from: 2020-10-24 25 | store: boltdb-shipper 26 | object_store: filesystem 27 | schema: v11 28 | index: 29 | prefix: index_ 30 | period: 24h 31 | 32 | storage_config: 33 | boltdb_shipper: 34 | active_index_directory: /tmp/loki/boltdb-shipper-active 35 | cache_location: /tmp/loki/boltdb-shipper-cache 36 | cache_ttl: 24h # Can be increased for faster performance over longer query periods, uses more disk space 37 | shared_store: filesystem 38 | filesystem: 39 | directory: /tmp/loki/chunks 40 | 41 | compactor: 42 | working_directory: /tmp/loki/boltdb-shipper-compactor 43 | shared_store: filesystem 44 | 45 | limits_config: 46 | reject_old_samples: true 47 | reject_old_samples_max_age: 168h 48 | 49 | chunk_store_config: 50 | max_look_back_period: 0s 51 | 52 | table_manager: 53 | retention_deletes_enabled: false 54 | retention_period: 0s 55 | 56 | ruler: 57 | storage: 58 | type: local 59 | local: 60 | directory: /tmp/loki/rules 61 | rule_path: /tmp/loki/rules-temp 62 | alertmanager_url: http://localhost:9093 63 | ring: 64 | kvstore: 65 | store: inmemory 66 | enable_api: true 67 | -------------------------------------------------------------------------------- /loki/loki-config.yml: -------------------------------------------------------------------------------- 1 | auth_enabled: false 2 | 3 | server: 4 | http_listen_port: 3100 5 | 6 | common: 7 | ring: 8 | instance_addr: 127.0.0.1 9 | kvstore: 10 | store: inmemory 11 | replication_factor: 1 12 | path_prefix: /tmp/loki 13 | 14 | schema_config: 15 | configs: 16 | - from: 2020-05-15 17 | store: tsdb 18 | object_store: filesystem 19 | schema: v13 20 | index: 21 | prefix: index_ 22 | period: 24h 23 | 24 | storage_config: 25 | filesystem: 26 | directory: /tmp/loki/chunks -------------------------------------------------------------------------------- /mktxp/_mktxp.conf: -------------------------------------------------------------------------------- 1 | ## Copyright (c) 2020 Arseniy Kuznetsov 2 | ## 3 | ## This program is free software; you can redistribute it and/or 4 | ## modify it under the terms of the GNU General Public License 5 | ## as published by the Free Software Foundation; either version 2 6 | ## of the License, or (at your option) any later version. 7 | ## 8 | ## This program is distributed in the hope that it will be useful, 9 | ## but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | ## GNU General Public License for more details. 12 | 13 | 14 | [MKTXP] 15 | listen = '0.0.0.0:49090' # Space separated list of socket addresses to listen to, both IPV4 and IPV6 16 | socket_timeout = 5 17 | 18 | initial_delay_on_failure = 120 19 | max_delay_on_failure = 900 20 | delay_inc_div = 5 21 | 22 | bandwidth = True # Turns metrics bandwidth metrics collection on / off 23 | bandwidth_test_interval = 600 # Interval for collecting bandwidth metrics 24 | minimal_collect_interval = 5 # Minimal metric collection interval 25 | 26 | verbose_mode = False # Set it on for troubleshooting 27 | 28 | fetch_routers_in_parallel = False # Fetching metrics from multiple routers in parallel / sequentially 29 | max_worker_threads = 5 # Max number of worker threads that can fetch routers (parallel fetch only) 30 | max_scrape_duration = 30 # Max duration of individual routers' metrics collection (parallel fetch only) 31 | total_max_scrape_duration = 90 # Max overall duration of all metrics collection (parallel fetch only) 32 | 33 | compact_default_conf_values = False # Compact mktxp.conf, so only specific values are kept on the individual routers' level -------------------------------------------------------------------------------- /mktxp/mktxp.conf: -------------------------------------------------------------------------------- 1 | ## Copyright (c) 2020 Arseniy Kuznetsov 2 | ## 3 | ## This program is free software; you can redistribute it and/or 4 | ## modify it under the terms of the GNU General Public License 5 | ## as published by the Free Software Foundation; either version 2 6 | ## of the License, or (at your option) any later version. 7 | ## 8 | ## This program is distributed in the hope that it will be useful, 9 | ## but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | ## GNU General Public License for more details. 12 | 13 | 14 | [Sample-Router] 15 | # for specific configuration on the router level, change here the defaults values from below 16 | hostname = 192.168.88.1 17 | 18 | [default] 19 | # this affects configuration of all routers, unless overloaded on their specific levels 20 | 21 | enabled = True # turns metrics collection for this RouterOS device on / off 22 | hostname = localhost # RouterOS IP address 23 | port = 8728 # RouterOS IP Port 24 | 25 | username = username # RouterOS user, needs to have 'read' and 'api' permissions 26 | password = password 27 | 28 | use_ssl = False # enables connection via API-SSL servis 29 | no_ssl_certificate = False # enables API_SSL connect without router SSL certificate 30 | ssl_certificate_verify = False # turns SSL certificate verification on / off 31 | plaintext_login = True # for legacy RouterOS versions below 6.43 use False 32 | 33 | installed_packages = True # Installed packages 34 | dhcp = True # DHCP general metrics 35 | dhcp_lease = True # DHCP lease metrics 36 | 37 | connections = True # IP connections metrics 38 | connection_stats = True # Open IP connections metrics 39 | 40 | interface = True # Interfaces traffic metrics 41 | 42 | route = True # IPv4 Routes metrics 43 | pool = True # IPv4 Pool metrics 44 | firewall = True # IPv4 Firewall rules traffic metrics 45 | neighbor = True # IPv4 Reachable Neighbors 46 | dns = True # DNS stats 47 | 48 | ipv6_route = False # IPv6 Routes metrics 49 | ipv6_pool = False # IPv6 Pool metrics 50 | ipv6_firewall = False # IPv6 Firewall rules traffic metrics 51 | ipv6_neighbor = False # IPv6 Reachable Neighbors 52 | 53 | poe = True # POE metrics 54 | monitor = True # Interface monitor metrics 55 | netwatch = True # Netwatch metrics 56 | public_ip = True # Public IP metrics 57 | wireless = True # WLAN general metrics 58 | wireless_clients = True # WLAN clients metrics 59 | capsman = True # CAPsMAN general metrics 60 | capsman_clients = True # CAPsMAN clients metrics 61 | 62 | eoip = False # EoIP status metrics 63 | gre = False # GRE status metrics 64 | ipip = False # IPIP status metrics 65 | lte = False # LTE signal and status metrics (requires additional 'test' permission policy on RouterOS v6) 66 | ipsec = False # IPSec active peer metrics 67 | switch_port = True # Switch Port metrics 68 | 69 | kid_control_assigned = False # Allow Kid Control metrics for connected devices with assigned users 70 | kid_control_dynamic = False # Allow Kid Control metrics for all connected devices, including those without assigned user 71 | 72 | user = True # Active Users metrics 73 | queue = True # Queues metrics 74 | 75 | bgp = True # BGP sessions metrics 76 | routing_stats = True # Routing process stats 77 | certificate = False # Certificates metrics 78 | 79 | remote_dhcp_entry = None # An MKTXP entry to provide for remote DHCP info / resolution 80 | remote_capsman_entry = None # An MKTXP entry to provide for remote capsman info 81 | 82 | use_comments_over_names = True # when available, forces using comments over the interfaces names 83 | check_for_updates = True # check for available ROS updates 84 | -------------------------------------------------------------------------------- /prometheus/prometheus.yml: -------------------------------------------------------------------------------- 1 | # global config 2 | global: 3 | scrape_interval: 45s # Set the scrape interval to every 45 seconds. Default is every 1 minute. 4 | scrape_timeout: 30s 5 | evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute. 6 | external_labels: 7 | origin_prometheus: mktxp 8 | 9 | # Alertmanager configuration 10 | alerting: 11 | alertmanagers: 12 | - static_configs: 13 | - targets: 14 | # - alertmanager:9093 15 | 16 | # Load rules once and periodically evaluate them according to the global 'evaluation_interval'. 17 | rule_files: 18 | 19 | scrape_configs: 20 | 21 | - job_name: 'prometheus' 22 | static_configs: 23 | - targets: ['prometheus:9090'] 24 | 25 | - job_name: 'grafana' 26 | static_configs: 27 | - targets: ['grafana:3000'] 28 | 29 | - job_name: 'loki' 30 | static_configs: 31 | - targets: ['loki:3100'] 32 | 33 | - job_name: 'promtail' 34 | static_configs: 35 | - targets: ['promtail:9080'] 36 | 37 | - job_name: 'mktxp' 38 | static_configs: 39 | - targets: ['mktxp:49090'] 40 | 41 | - job_name: 'blackbox' 42 | scrape_interval: 1s 43 | metrics_path: /probe 44 | params: 45 | module: [icmp_ttl5] # use ICMP as the ping protocol 46 | static_configs: 47 | - targets: 48 | - 1.1.1.1 # Cloudflare DNS 49 | - 8.8.8.8 # Google DNS 50 | - 77.88.8.8 # Yandex DNS 51 | relabel_configs: 52 | - source_labels: [__address__] 53 | target_label: __param_target 54 | - source_labels: [__param_target] 55 | target_label: instance 56 | - target_label: __address__ 57 | replacement: blackbox:9115 # The blackbox exporter's real hostname:port. 58 | -------------------------------------------------------------------------------- /promtail/promtail-config-fs.yml: -------------------------------------------------------------------------------- 1 | server: 2 | http_listen_port: 9080 3 | grpc_listen_port: 0 4 | 5 | positions: 6 | filename: /tmp/positions.yaml 7 | 8 | clients: 9 | - url: http://loki:3100/loki/api/v1/push 10 | 11 | scrape_configs: 12 | 13 | - job_name: mktxp-syslog 14 | static_configs: 15 | - targets: 16 | - localhost 17 | labels: 18 | job: "syslog" 19 | __path__: /var/log/*.log 20 | pipeline_stages: 21 | - match: 22 | selector: '{job="syslog"}' 23 | stages: 24 | - regex: 25 | source: filename 26 | expression: "/var/log/(?P.+).log" 27 | - labels: 28 | routerboard: 29 | -------------------------------------------------------------------------------- /promtail/promtail-config.yml: -------------------------------------------------------------------------------- 1 | server: 2 | http_listen_port: 9080 3 | grpc_listen_port: 0 4 | 5 | positions: 6 | filename: /tmp/positions.yaml 7 | 8 | clients: 9 | - url: http://loki:3100/loki/api/v1/push 10 | 11 | scrape_configs: 12 | 13 | - job_name: syslog 14 | syslog: 15 | listen_address: 0.0.0.0:1514 16 | idle_timeout: 60s 17 | label_structured_data: yes 18 | labels: 19 | job: "syslog" 20 | relabel_configs: 21 | - source_labels: ['__syslog_message_hostname'] 22 | target_label: 'routerboard' 23 | - source_labels: ['__syslog_connection_hostname'] 24 | target_label: 'syslog_host' 25 | 26 | # - source_labels: ['__syslog_connection_ip_address'] 27 | # target_label: 'syslog_host_ip' 28 | # - source_labels: ['__syslog_message_severity'] 29 | # target_label: 'severity' 30 | # - source_labels: ['__syslog_message_facility'] 31 | # target_label: 'facility' 32 | -------------------------------------------------------------------------------- /syslog-ng/logs/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/metgen/mkt_monitoring/9934a9549a8bbffe3a4734553512801e5b4a4f8c/syslog-ng/logs/.gitkeep -------------------------------------------------------------------------------- /syslog-ng/syslog-ng-fs.conf: -------------------------------------------------------------------------------- 1 | ############################################################################# 2 | # Default syslog-ng.conf file which collects all local logs into a 3 | # single file called /var/log/messages tailored to container usage. 4 | # 5 | # The changes from the stock, default syslog-ng.conf file is that we've 6 | # dropped the system() source that is not needed and that we enabled network 7 | # connections using default-network-drivers(). Customize as needed and 8 | # override using the -v option to docker, such as: 9 | # 10 | # docker run ... -v "$PWD/syslog-ng.conf":/etc/syslog-ng/syslog-ng.conf 11 | # 12 | 13 | @version: 3.29 14 | @include "scl.conf" 15 | 16 | source s_local { 17 | internal(); 18 | }; 19 | 20 | source s_network { 21 | default-network-drivers( 22 | # NOTE: TLS support 23 | # 24 | # the default-network-drivers() source driver opens the TLS 25 | # enabled ports as well, however without an actual key/cert 26 | # pair they will not operate and syslog-ng would display a 27 | # warning at startup. 28 | # 29 | #tls(key-file("/path/to/ssl-private-key") cert-file("/path/to/ssl-cert")) 30 | ); 31 | }; 32 | 33 | destination d_netlog { file("/var/log/syslog-ng/${HOST}.log"); }; 34 | 35 | 36 | log { 37 | source(s_local); 38 | source(s_network); 39 | destination(d_netlog); 40 | }; 41 | 42 | 43 | -------------------------------------------------------------------------------- /syslog-ng/syslog-ng.conf: -------------------------------------------------------------------------------- 1 | ############################################################################# 2 | # Default syslog-ng.conf file which collects all local logs into a 3 | # single file called /var/log/messages tailored to container usage. 4 | # 5 | # The changes from the stock, default syslog-ng.conf file is that we've 6 | # dropped the system() source that is not needed and that we enabled network 7 | # connections using default-network-drivers(). Customize as needed and 8 | # override using the -v option to docker, such as: 9 | # 10 | # docker run ... -v "$PWD/syslog-ng.conf":/etc/syslog-ng/syslog-ng.conf 11 | # 12 | 13 | @version: 3.29 14 | @include "scl.conf" 15 | 16 | source s_local { 17 | internal(); 18 | }; 19 | 20 | source s_network { 21 | default-network-drivers( 22 | # NOTE: TLS support 23 | # 24 | # the default-network-drivers() source driver opens the TLS 25 | # enabled ports as well, however without an actual key/cert 26 | # pair they will not operate and syslog-ng would display a 27 | # warning at startup. 28 | # 29 | #tls(key-file("/path/to/ssl-private-key") cert-file("/path/to/ssl-cert")) 30 | ); 31 | }; 32 | 33 | destination d_loki { 34 | syslog("promtail" transport("tcp") port("1514")); 35 | }; 36 | 37 | log { 38 | source(s_local); 39 | source(s_network); 40 | destination(d_loki); 41 | }; 42 | 43 | --------------------------------------------------------------------------------