├── .gitattributes ├── .github └── workflows │ └── docker-image.yml ├── .gitignore ├── Dockerfile ├── LICENSE ├── README.md ├── grafana-screenshot.png └── prometheus_frigate_exporter.py /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | -------------------------------------------------------------------------------- /.github/workflows/docker-image.yml: -------------------------------------------------------------------------------- 1 | name: Docker Image CI 2 | 3 | on: 4 | push: 5 | tags: 6 | - 'v[0-9]+.[0-9]+.[0-9]+' 7 | jobs: 8 | docker: 9 | runs-on: ubuntu-latest 10 | steps: 11 | 12 | - name: Docker meta 13 | id: meta 14 | uses: docker/metadata-action@v5 15 | with: 16 | images: ${{ secrets.DOCKERHUB_USERNAME }}/prometheus-frigate-exporter 17 | tags: type=semver,pattern={{version}} 18 | 19 | - name: Set up QEMU 20 | uses: docker/setup-qemu-action@v3 21 | 22 | - name: Set up Docker Buildx 23 | uses: docker/setup-buildx-action@v3 24 | 25 | - name: Login to Docker Hub 26 | uses: docker/login-action@v3 27 | with: 28 | username: ${{ secrets.DOCKERHUB_USERNAME }} 29 | password: ${{ secrets.DOCKERHUB_TOKEN }} 30 | 31 | - name: Build and push 32 | uses: docker/build-push-action@v5 33 | with: 34 | push: true 35 | file: Dockerfile 36 | platforms: linux/amd64,linux/arm64 37 | tags: ${{ steps.meta.outputs.tags }} 38 | labels: ${{ steps.meta.outputs.labels }} -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | .vscode/launch.json 3 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine 2 | RUN apk update && apk add py3-prometheus-client 3 | COPY prometheus_frigate_exporter.py /var/python_scripts/prometheus_frigate_exporter.py 4 | CMD /usr/bin/python3 /var/python_scripts/prometheus_frigate_exporter.py $FRIGATE_STATS_URL 5 | 6 | # docker build -t rhysbailey/prometheus-frigate-exporter . 7 | # docker push rhysbailey/prometheus-frigate-exporter:latest -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Rhys Bailey 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Prometheus Frigate stats exporter 2 | 3 | This is a docker container that runs a Prometheus exporter for [Frigate](https://frigate.video/) stats. 4 | 5 | Tested with 0.13.0 and 0.14.0 Frigate docker images. 6 | 7 | Exports from Frigate API: 8 | 9 | - Inference Speed 10 | - CPU and MEM process stats 11 | - Camera, detection and skipped FPS 12 | - Camera audio stats 13 | - Storage total, used and free 14 | - Device Temperature (Coral temp) 15 | - Event counters for detected labels on each camera 16 | 17 | [Docker Hub](https://hub.docker.com/r/rhysbailey/prometheus-frigate-exporter) 18 | 19 | [GitHub](https://github.com/bairhys/prometheus-frigate-exporter) 20 | 21 | [Grafana Dashboard](https://grafana.com/grafana/dashboards/18226-frigate/) 22 | 23 | ![Grafana](https://raw.githubusercontent.com/bairhys/prometheus-frigate-exporter/main/grafana-screenshot.png) 24 | 25 | ## Run the exporter 26 | 27 | Modify the `FRIGATE_STATS_URL` environment variable below to point to your [Frigate API stats](https://docs.frigate.video/integrations/api#get-apistats) (replace `` with your Frigate docker container IP address). Then run the container: 28 | 29 | ```bash 30 | docker run \ 31 | -d \ 32 | --restart unless-stopped \ 33 | -p 9100:9100 \ 34 | -e "FRIGATE_STATS_URL=http://:5000/api/stats" \ 35 | --name prometheus_frigate_exporter \ 36 | rhysbailey/prometheus-frigate-exporter 37 | ``` 38 | 39 | The default internal exporter port can be modified with `-e "PORT=9100"` 40 | 41 | Metrics are available at http://localhost:9100/metrics 42 | 43 | If you want to export network bandwidth stats, include the section below in your Frigate config (see [here](https://docs.frigate.video/configuration/reference)): 44 | 45 | ```yml 46 | telemetry: 47 | stats: 48 | network_bandwidth: True 49 | ``` 50 | 51 | ### Setup Prometheus 52 | 53 | If you don't already have Prometheus set up to scrape the `prometheus-frigate-exporter` metrics, 54 | 55 | - create Prometheus config file `prometheus.yml` 56 | - copy example below into `prometheus.yml`, replacing `` with the IP address of your `prometheus_frigate_exporter` docker container. `` is likely the same IP address as your Frigate docker containers `` if running in the same docker instance 57 | ```yaml 58 | # my global config 59 | global: 60 | scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute. 61 | evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute. 62 | # scrape_timeout is set to the global default (10s). 63 | 64 | # A scrape configuration containing exactly one endpoint to scrape: 65 | # Here it's Prometheus itself. 66 | scrape_configs: 67 | # The job name is added as a label `job=` to any timeseries scraped from this config. 68 | - job_name: "prometheus" 69 | static_configs: 70 | - targets: ["localhost:9090"] 71 | 72 | - job_name: "prometheus_frigate_exporter" 73 | static_configs: 74 | - targets: [ 75 | ":9100" 76 | ] 77 | ``` 78 | 79 | - Run Prometheus docker container by replacing `/path/to/prometheus.yml` to point to the `prometheus.yml` just created 80 | 81 | ```bash 82 | docker run \ 83 | -d \ 84 | --restart unless-stopped \ 85 | -p 9090:9090 \ 86 | -v /path/to/prometheus.yml:/etc/prometheus/prometheus.yml \ 87 | prom/prometheus 88 | ``` 89 | 90 | To see if Prometheus is scraping the Frigate exporter, go to Prometheus targets page [http://your-prometheus-ip:9090/targets](http://your-prometheus-ip:9090/targets) and look for `UP` for `prometheus_frigate_exporter` job. 91 | 92 | ### Setup Grafana 93 | 94 | If you don't already have Grafana set up, 95 | 96 | - run Grafana 97 | 98 | ```bash 99 | docker run \ 100 | -d \ 101 | --restart unless-stopped \ 102 | -p 3000:3000 \ 103 | grafana/grafana-oss 104 | ``` 105 | 106 | - Go to Grafana [http://your-grafana-ip:3000](http://your-grafana-ip:3000) (might take a few minutes first run). Use admin:admin to log in 107 | - Go to [http://your-grafana-ip:3000/datasources](http://your-grafana-ip:3000/datasources) 108 | - add Prometheus datasource 109 | - Set Prometheus URL `http://:9090` 110 | - Click `Save and Test` to check if connected 111 | - Go to [http://your-grafana-ip:3000/dashboards](http://your-grafana-ip:3000/dashboards) 112 | - New -> Import 113 | - Enter in `Import via grafana.com`: `18226` (id can be found at [Grafana Dashboard](https://grafana.com/grafana/dashboards/18226-frigate/)) and click Load 114 | - Set the datasource as Prometheus instance set up before then click Import 115 | - Should now be able to see Frigate time series metrics in the Grafana dashboard 116 | 117 | ## Example metrics 118 | 119 | Metrics at `:9100` should look similar to this 120 | 121 | ```python 122 | # HELP python_gc_objects_collected_total Objects collected during gc 123 | # TYPE python_gc_objects_collected_total counter 124 | python_gc_objects_collected_total{generation="0"} 225.0 125 | python_gc_objects_collected_total{generation="1"} 156.0 126 | python_gc_objects_collected_total{generation="2"} 0.0 127 | # HELP python_gc_objects_uncollectable_total Uncollectable object found during GC 128 | # TYPE python_gc_objects_uncollectable_total counter 129 | python_gc_objects_uncollectable_total{generation="0"} 0.0 130 | python_gc_objects_uncollectable_total{generation="1"} 0.0 131 | python_gc_objects_uncollectable_total{generation="2"} 0.0 132 | # HELP python_gc_collections_total Number of times this generation was collected 133 | # TYPE python_gc_collections_total counter 134 | python_gc_collections_total{generation="0"} 42.0 135 | python_gc_collections_total{generation="1"} 3.0 136 | python_gc_collections_total{generation="2"} 0.0 137 | # HELP python_info Python platform information 138 | # TYPE python_info gauge 139 | python_info{implementation="CPython",major="3",minor="10",patchlevel="10",version="3.10.10"} 1.0 140 | # HELP process_virtual_memory_bytes Virtual memory size in bytes. 141 | # TYPE process_virtual_memory_bytes gauge 142 | process_virtual_memory_bytes 2.6222592e+07 143 | # HELP process_resident_memory_bytes Resident memory size in bytes. 144 | # TYPE process_resident_memory_bytes gauge 145 | process_resident_memory_bytes 1.9456e+07 146 | # HELP process_start_time_seconds Start time of the process since unix epoch in seconds. 147 | # TYPE process_start_time_seconds gauge 148 | process_start_time_seconds 1.67807825501e+09 149 | # HELP process_cpu_seconds_total Total user and system CPU time spent in seconds. 150 | # TYPE process_cpu_seconds_total counter 151 | process_cpu_seconds_total 0.19 152 | # HELP process_open_fds Number of open file descriptors. 153 | # TYPE process_open_fds gauge 154 | process_open_fds 6.0 155 | # HELP process_max_fds Maximum number of open file descriptors. 156 | # TYPE process_max_fds gauge 157 | process_max_fds 1.048576e+06 158 | # HELP frigate_camera_fps Frames per second being consumed from your camera. 159 | # TYPE frigate_camera_fps gauge 160 | frigate_camera_fps{camera_name="Camera1"} 25.0 161 | frigate_camera_fps{camera_name="Camera2"} 5.0 162 | frigate_camera_fps{camera_name="Camera3"} 5.0 163 | frigate_camera_fps{camera_name="Camera4"} 5.0 164 | # HELP frigate_detection_fps Number of times detection is run per second. 165 | # TYPE frigate_detection_fps gauge 166 | frigate_detection_fps{camera_name="Camera1"} 4.0 167 | frigate_detection_fps{camera_name="Camera2"} 5.0 168 | frigate_detection_fps{camera_name="Camera3"} 0.0 169 | frigate_detection_fps{camera_name="Camera4"} 0.0 170 | # HELP frigate_process_fps Frames per second being processed by frigate. 171 | # TYPE frigate_process_fps gauge 172 | frigate_process_fps{camera_name="Camera1"} 25.0 173 | frigate_process_fps{camera_name="Camera2"} 5.0 174 | frigate_process_fps{camera_name="Camera3"} 5.0 175 | frigate_process_fps{camera_name="Camera4"} 5.0 176 | # HELP frigate_skipped_fps Frames per second skip for processing by frigate. 177 | # TYPE frigate_skipped_fps gauge 178 | frigate_skipped_fps{camera_name="Camera1"} 0.0 179 | frigate_skipped_fps{camera_name="Camera2"} 0.0 180 | frigate_skipped_fps{camera_name="Camera3"} 0.0 181 | frigate_skipped_fps{camera_name="Camera4"} 0.0 182 | # HELP frigate_detection_enabled Detection enabled for camera 183 | # TYPE frigate_detection_enabled gauge 184 | frigate_detection_enabled{camera_name="Camera1"} 1.0 185 | frigate_detection_enabled{camera_name="Camera2"} 1.0 186 | frigate_detection_enabled{camera_name="Camera3"} 1.0 187 | frigate_detection_enabled{camera_name="Camera4"} 1.0 188 | # HELP frigate_detection_total_fps Sum of detection_fps across all cameras and detectors. 189 | # TYPE frigate_detection_total_fps gauge 190 | frigate_detection_total_fps 10.5 191 | # HELP frigate_detector_inference_speed_seconds Time spent running object detection in seconds. 192 | # TYPE frigate_detector_inference_speed_seconds gauge 193 | frigate_detector_inference_speed_seconds{name="ov"} 0.011 194 | # HELP frigate_detection_start Detector start time (unix timestamp) 195 | # TYPE frigate_detection_start gauge 196 | frigate_detection_start{name="ov"} 0.0 197 | # HELP frigate_cpu_usage_percent Process CPU usage % 198 | # TYPE frigate_cpu_usage_percent gauge 199 | frigate_cpu_usage_percent{name="Camera1",pid="296",process="ffmpeg",type="Camera"} 24.3 200 | frigate_cpu_usage_percent{name="Camera1",pid="289",process="capture",type="Camera"} 29.7 201 | frigate_cpu_usage_percent{name="Camera1",pid="285",process="detect",type="Camera"} 29.7 202 | frigate_cpu_usage_percent{name="Camera2",pid="556",process="ffmpeg",type="Camera"} 6.0 203 | frigate_cpu_usage_percent{name="Camera2",pid="292",process="capture",type="Camera"} 5.7 204 | frigate_cpu_usage_percent{name="Camera2",pid="286",process="detect",type="Camera"} 10.3 205 | frigate_cpu_usage_percent{name="Camera3",pid="309",process="ffmpeg",type="Camera"} 1.7 206 | frigate_cpu_usage_percent{name="Camera3",pid="295",process="capture",type="Camera"} 0.3 207 | frigate_cpu_usage_percent{name="Camera3",pid="287",process="detect",type="Camera"} 1.0 208 | frigate_cpu_usage_percent{name="Camera4",pid="310",process="ffmpeg",type="Camera"} 6.7 209 | frigate_cpu_usage_percent{name="Camera4",pid="299",process="capture",type="Camera"} 1.0 210 | frigate_cpu_usage_percent{name="Camera4",pid="288",process="detect",type="Camera"} 0.7 211 | frigate_cpu_usage_percent{name="ov",pid="280",process="detect",type="Detector"} 30.0 212 | frigate_cpu_usage_percent{pid="1"} 0.0 213 | frigate_cpu_usage_percent{pid="100"} 0.0 214 | frigate_cpu_usage_percent{pid="105"} 11.0 215 | frigate_cpu_usage_percent{pid="111"} 0.0 216 | frigate_cpu_usage_percent{pid="128"} 0.0 217 | frigate_cpu_usage_percent{pid="129"} 0.0 218 | frigate_cpu_usage_percent{pid="130"} 0.0 219 | frigate_cpu_usage_percent{pid="131"} 0.0 220 | frigate_cpu_usage_percent{pid="15"} 0.0 221 | frigate_cpu_usage_percent{pid="17"} 0.0 222 | frigate_cpu_usage_percent{pid="24"} 0.0 223 | frigate_cpu_usage_percent{pid="25"} 0.0 224 | frigate_cpu_usage_percent{pid="26"} 0.0 225 | frigate_cpu_usage_percent{pid="27"} 0.0 226 | frigate_cpu_usage_percent{pid="273"} 0.0 227 | frigate_cpu_usage_percent{pid="279"} 1.0 228 | frigate_cpu_usage_percent{pid="28"} 0.0 229 | frigate_cpu_usage_percent{pid="282"} 4.0 230 | frigate_cpu_usage_percent{pid="29"} 0.0 231 | frigate_cpu_usage_percent{pid="293"} 0.0 232 | frigate_cpu_usage_percent{pid="30"} 0.0 233 | frigate_cpu_usage_percent{pid="300"} 0.0 234 | frigate_cpu_usage_percent{pid="308"} 0.0 235 | frigate_cpu_usage_percent{pid="31"} 0.0 236 | frigate_cpu_usage_percent{pid="313"} 0.0 237 | frigate_cpu_usage_percent{pid="314"} 0.0 238 | frigate_cpu_usage_percent{pid="322030"} 0.0 239 | frigate_cpu_usage_percent{pid="322038"} 0.0 240 | frigate_cpu_usage_percent{pid="40"} 0.0 241 | frigate_cpu_usage_percent{pid="41"} 0.0 242 | frigate_cpu_usage_percent{pid="78"} 0.0 243 | frigate_cpu_usage_percent{pid="80"} 0.0 244 | frigate_cpu_usage_percent{pid="81"} 0.0 245 | # HELP frigate_mem_usage_percent Process memory usage % 246 | # TYPE frigate_mem_usage_percent gauge 247 | frigate_mem_usage_percent{name="Camera1",pid="296",process="ffmpeg",type="Camera"} 1.1 248 | frigate_mem_usage_percent{name="Camera1",pid="289",process="capture",type="Camera"} 0.6 249 | frigate_mem_usage_percent{name="Camera1",pid="285",process="detect",type="Camera"} 1.2 250 | frigate_mem_usage_percent{name="Camera2",pid="556",process="ffmpeg",type="Camera"} 0.7 251 | frigate_mem_usage_percent{name="Camera2",pid="292",process="capture",type="Camera"} 0.8 252 | frigate_mem_usage_percent{name="Camera2",pid="286",process="detect",type="Camera"} 1.2 253 | frigate_mem_usage_percent{name="Camera3",pid="309",process="ffmpeg",type="Camera"} 0.2 254 | frigate_mem_usage_percent{name="Camera3",pid="295",process="capture",type="Camera"} 0.5 255 | frigate_mem_usage_percent{name="Camera3",pid="287",process="detect",type="Camera"} 0.6 256 | frigate_mem_usage_percent{name="Camera4",pid="310",process="ffmpeg",type="Camera"} 0.1 257 | frigate_mem_usage_percent{name="Camera4",pid="299",process="capture",type="Camera"} 0.5 258 | frigate_mem_usage_percent{name="Camera4",pid="288",process="detect",type="Camera"} 0.6 259 | frigate_mem_usage_percent{name="ov",pid="280",process="detect",type="Detector"} 1.9 260 | frigate_mem_usage_percent{pid="1"} 0.0 261 | frigate_mem_usage_percent{pid="100"} 0.0 262 | frigate_mem_usage_percent{pid="105"} 5.0 263 | frigate_mem_usage_percent{pid="111"} 0.0 264 | frigate_mem_usage_percent{pid="128"} 0.0 265 | frigate_mem_usage_percent{pid="129"} 0.0 266 | frigate_mem_usage_percent{pid="130"} 0.0 267 | frigate_mem_usage_percent{pid="131"} 0.0 268 | frigate_mem_usage_percent{pid="15"} 0.0 269 | frigate_mem_usage_percent{pid="17"} 0.0 270 | frigate_mem_usage_percent{pid="24"} 0.0 271 | frigate_mem_usage_percent{pid="25"} 0.0 272 | frigate_mem_usage_percent{pid="26"} 0.0 273 | frigate_mem_usage_percent{pid="27"} 0.0 274 | frigate_mem_usage_percent{pid="273"} 0.0 275 | frigate_mem_usage_percent{pid="279"} 0.0 276 | frigate_mem_usage_percent{pid="28"} 0.0 277 | frigate_mem_usage_percent{pid="282"} 0.0 278 | frigate_mem_usage_percent{pid="29"} 0.0 279 | frigate_mem_usage_percent{pid="293"} 0.0 280 | frigate_mem_usage_percent{pid="30"} 0.0 281 | frigate_mem_usage_percent{pid="300"} 0.0 282 | frigate_mem_usage_percent{pid="308"} 0.0 283 | frigate_mem_usage_percent{pid="31"} 0.0 284 | frigate_mem_usage_percent{pid="313"} 0.0 285 | frigate_mem_usage_percent{pid="314"} 0.0 286 | frigate_mem_usage_percent{pid="322030"} 0.0 287 | frigate_mem_usage_percent{pid="322038"} 0.0 288 | frigate_mem_usage_percent{pid="40"} 0.0 289 | frigate_mem_usage_percent{pid="41"} 0.0 290 | frigate_mem_usage_percent{pid="78"} 0.0 291 | frigate_mem_usage_percent{pid="80"} 0.0 292 | frigate_mem_usage_percent{pid="81"} 0.0 293 | frigate_mem_usage_percent{pid="Tasks:"} 0.0 294 | # HELP frigate_gpu_usage_percent GPU utilisation % 295 | # TYPE frigate_gpu_usage_percent gauge 296 | frigate_gpu_usage_percent{gpu_name="intel-qsv"} 13.0 297 | # HELP frigate_gpu_mem_usage_percent GPU memory usage % 298 | # TYPE frigate_gpu_mem_usage_percent gauge 299 | # HELP frigate_service_info Frigate version info 300 | # TYPE frigate_service_info gauge 301 | frigate_service_info{latest_version="0.11.1",version="0.12.0-27a31e7"} 1.0 302 | # HELP frigate_service_uptime_seconds Uptime seconds 303 | # TYPE frigate_service_uptime_seconds gauge 304 | frigate_service_uptime_seconds 227029.0 305 | # HELP frigate_service_last_updated_timestamp Stats recorded time (unix timestamp) 306 | # TYPE frigate_service_last_updated_timestamp gauge 307 | frigate_service_last_updated_timestamp 1.678078264e+09 308 | # HELP frigate_storage_free_bytes Storage free bytes 309 | # TYPE frigate_storage_free_bytes gauge 310 | frigate_storage_free_bytes{storage="/dev/shm"} 2e+09 311 | frigate_storage_free_bytes{storage="/media/frigate/clips"} 2e+09 312 | frigate_storage_free_bytes{storage="/media/frigate/recordings"} 2e+09 313 | frigate_storage_free_bytes{storage="/tmp/cache"} 2e+09 314 | # HELP frigate_storage_mount_type_info Storage mount type 315 | # TYPE frigate_storage_mount_type_info gauge 316 | frigate_storage_mount_type_info{mount_type="tmpfs",storage="/dev/shm"} 1.0 317 | frigate_storage_mount_type_info{mount_type="ext4",storage="/media/frigate/clips"} 1.0 318 | frigate_storage_mount_type_info{mount_type="ext4",storage="/media/frigate/recordings"} 1.0 319 | frigate_storage_mount_type_info{mount_type="overlay",storage="/tmp/cache"} 1.0 320 | # HELP frigate_storage_total_bytes Storage total bytes 321 | # TYPE frigate_storage_total_bytes gauge 322 | frigate_storage_total_bytes{storage="/dev/shm"} 3e+09 323 | frigate_storage_total_bytes{storage="/media/frigate/clips"} 3e+09 324 | frigate_storage_total_bytes{storage="/media/frigate/recordings"} 3e+09 325 | frigate_storage_total_bytes{storage="/tmp/cache"} 3e+09 326 | # HELP frigate_storage_used_bytes Storage used bytes 327 | # TYPE frigate_storage_used_bytes gauge 328 | frigate_storage_used_bytes{storage="/dev/shm"} 1e+09 329 | frigate_storage_used_bytes{storage="/media/frigate/clips"} 1e+09 330 | frigate_storage_used_bytes{storage="/media/frigate/recordings"} 1e+09 331 | frigate_storage_used_bytes{storage="/tmp/cache"} 1e+09 332 | ``` 333 | -------------------------------------------------------------------------------- /grafana-screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bairhys/prometheus-frigate-exporter/8a5b45c3f853f1ce537c99e773baef629a2d68dd/grafana-screenshot.png -------------------------------------------------------------------------------- /prometheus_frigate_exporter.py: -------------------------------------------------------------------------------- 1 | import json 2 | import re 3 | import time 4 | import sys 5 | import logging 6 | import os 7 | from urllib.request import urlopen 8 | from urllib import error 9 | from prometheus_client.core import GaugeMetricFamily, InfoMetricFamily, CounterMetricFamily, REGISTRY 10 | from prometheus_client import start_http_server 11 | 12 | 13 | def add_metric(metric, label, stats, key, multiplier=1.0): 14 | try: 15 | string = str(stats[key]) 16 | value = float(re.findall(r'-?\d*\.?\d*', string)[0]) 17 | metric.add_metric(label, value * multiplier) 18 | except (KeyError, TypeError, IndexError, ValueError): 19 | pass 20 | 21 | 22 | class CustomCollector(object): 23 | 24 | def __init__(self, _url): 25 | self.stats_url = _url 26 | self.process_stats = {} 27 | self.previous_event_id = None 28 | self.previous_event_start_time = None 29 | self.all_events = {} 30 | 31 | def add_metric_process(self, metric, camera_stats, camera_name, pid_name, process_name, cpu_or_memory, process_type): 32 | try: 33 | pid = str(camera_stats[pid_name]) 34 | label_values = [pid, camera_name, process_name, process_type] 35 | try: 36 | # new frigate:0.13.0-beta3 stat 'cmdline' 37 | label_values.append(self.process_stats[pid]['cmdline']) 38 | except KeyError: 39 | pass 40 | metric.add_metric(label_values, self.process_stats[pid][cpu_or_memory]) 41 | del self.process_stats[pid][cpu_or_memory] 42 | except (KeyError, TypeError, IndexError): 43 | pass 44 | 45 | def collect(self): 46 | try: 47 | stats = json.loads(urlopen(self.stats_url).read()) 48 | except error.URLError as e: 49 | logging.error("URLError while opening Frigate stats url %s: %s", self.stats_url, e) 50 | return 51 | 52 | try: 53 | self.process_stats = stats['cpu_usages'] 54 | except KeyError: 55 | pass 56 | 57 | # process stats for cameras, detectors and other 58 | cpu_usages = GaugeMetricFamily('frigate_cpu_usage_percent', 'Process CPU usage %', 59 | labels=['pid', 'name', 'process', 'type', 'cmdline']) 60 | mem_usages = GaugeMetricFamily('frigate_mem_usage_percent', 'Process memory usage %', 61 | labels=['pid', 'name', 'process', 'type', 'cmdline']) 62 | 63 | # camera stats 64 | audio_dBFS = GaugeMetricFamily('frigate_audio_dBFS', 'Audio dBFS for camera', 65 | labels=['camera_name']) 66 | audio_rms = GaugeMetricFamily('frigate_audio_rms', 'Audio RMS for camera', 67 | labels=['camera_name']) 68 | camera_fps = GaugeMetricFamily('frigate_camera_fps', 'Frames per second being consumed from your camera.', 69 | labels=['camera_name']) 70 | detection_enabled = GaugeMetricFamily('frigate_detection_enabled', 'Detection enabled for camera', 71 | labels=['camera_name']) 72 | detection_fps = GaugeMetricFamily('frigate_detection_fps', 'Number of times detection is run per second.', 73 | labels=['camera_name']) 74 | process_fps = GaugeMetricFamily('frigate_process_fps', 'Frames per second being processed by frigate.', 75 | labels=['camera_name']) 76 | skipped_fps = GaugeMetricFamily('frigate_skipped_fps', 'Frames per second skip for processing by frigate.', 77 | labels=['camera_name']) 78 | 79 | # read camera stats assuming version < frigate:0.13.0-beta3 80 | cameras = stats 81 | try: 82 | # try to read camera stats in case >= frigate:0.13.0-beta3 83 | cameras = stats['cameras'] 84 | except KeyError: 85 | pass 86 | 87 | for camera_name, camera_stats in cameras.items(): 88 | add_metric(audio_dBFS, [camera_name], camera_stats, 'audio_dBFS') 89 | add_metric(audio_rms, [camera_name], camera_stats, 'audio_rms') 90 | add_metric(camera_fps, [camera_name], camera_stats, 'camera_fps') 91 | add_metric(detection_enabled, [camera_name], camera_stats, 'detection_enabled') 92 | add_metric(detection_fps, [camera_name], camera_stats, 'detection_fps') 93 | add_metric(process_fps, [camera_name], camera_stats, 'process_fps') 94 | add_metric(skipped_fps, [camera_name], camera_stats, 'skipped_fps') 95 | 96 | self.add_metric_process(cpu_usages, camera_stats, camera_name, 'ffmpeg_pid', 'ffmpeg', 'cpu', 'Camera') 97 | self.add_metric_process(cpu_usages, camera_stats, camera_name, 'capture_pid', 'capture', 'cpu', 'Camera') 98 | self.add_metric_process(cpu_usages, camera_stats, camera_name, 'pid', 'detect', 'cpu', 'Camera') 99 | 100 | self.add_metric_process(mem_usages, camera_stats, camera_name, 'ffmpeg_pid', 'ffmpeg', 'mem', 'Camera') 101 | self.add_metric_process(mem_usages, camera_stats, camera_name, 'capture_pid', 'capture', 'mem', 'Camera') 102 | self.add_metric_process(mem_usages, camera_stats, camera_name, 'pid', 'detect', 'mem', 'Camera') 103 | 104 | yield audio_dBFS 105 | yield audio_rms 106 | yield camera_fps 107 | yield detection_enabled 108 | yield detection_fps 109 | yield process_fps 110 | yield skipped_fps 111 | 112 | # bandwidth stats 113 | bandwidth_usages = GaugeMetricFamily('frigate_bandwidth_usages_kBps', 'bandwidth usages kilobytes per second', labels=['pid', 'name', 'process', 'cmdline']) 114 | 115 | try: 116 | for b_pid, b_stats in stats['bandwidth_usages'].items(): 117 | label = [b_pid] # pid label 118 | try: 119 | n = stats['cpu_usages'][b_pid]['cmdline'] 120 | for p_name, p_stats in stats['processes'].items(): 121 | if str(p_stats['pid']) == b_pid: 122 | n = p_name 123 | break 124 | 125 | # new frigate:0.13.0-beta3 stat 'cmdline' 126 | label.append(n) # name label 127 | label.append(stats['cpu_usages'][b_pid]['cmdline']) # process label 128 | label.append(stats['cpu_usages'][b_pid]['cmdline']) # cmdline label 129 | add_metric(bandwidth_usages, label, b_stats, 'bandwidth') 130 | except KeyError: 131 | pass 132 | except KeyError: 133 | pass 134 | 135 | yield bandwidth_usages 136 | 137 | # detector stats 138 | try: 139 | yield GaugeMetricFamily('frigate_detection_total_fps', 140 | 'Sum of detection_fps across all cameras and detectors.', 141 | value=stats['detection_fps']) 142 | except KeyError: 143 | pass 144 | 145 | detector_inference_speed = GaugeMetricFamily('frigate_detector_inference_speed_seconds', 146 | 'Time spent running object detection in seconds.', labels=['name']) 147 | 148 | detector_detection_start = GaugeMetricFamily('frigate_detection_start', 149 | 'Detector start time (unix timestamp)', 150 | labels=['name']) 151 | 152 | try: 153 | for detector_name, detector_stats in stats['detectors'].items(): 154 | add_metric(detector_inference_speed, [detector_name], detector_stats, 'inference_speed', 155 | 0.001) # ms to seconds 156 | add_metric(detector_detection_start, [detector_name], detector_stats, 'detection_start') 157 | self.add_metric_process(cpu_usages, stats['detectors'], detector_name, 'pid', 'detect', 'cpu', 158 | 'Detector') 159 | self.add_metric_process(mem_usages, stats['detectors'], detector_name, 'pid', 'detect', 'mem', 160 | 'Detector') 161 | except KeyError: 162 | pass 163 | 164 | yield detector_inference_speed 165 | yield detector_detection_start 166 | 167 | # detector process stats 168 | try: 169 | for detector_name, detector_stats in stats['detectors'].items(): 170 | p_pid = str(detector_stats['pid']) 171 | label = [p_pid] # pid label 172 | try: 173 | # new frigate:0.13.0-beta3 stat 'cmdline' 174 | label.append(detector_name) # name label 175 | label.append(detector_name) # process label 176 | label.append('detectors') # type label 177 | label.append(self.process_stats[p_pid]['cmdline']) # cmdline label 178 | add_metric(cpu_usages, label, self.process_stats[p_pid], 'cpu') 179 | add_metric(mem_usages, label, self.process_stats[p_pid], 'mem') 180 | del self.process_stats[p_pid] 181 | except KeyError: 182 | pass 183 | 184 | except KeyError: 185 | pass 186 | 187 | # other named process stats 188 | try: 189 | for process_name, process_stats in stats['processes'].items(): 190 | p_pid = str(process_stats['pid']) 191 | label = [p_pid] # pid label 192 | try: 193 | # new frigate:0.13.0-beta3 stat 'cmdline' 194 | label.append(process_name) # name label 195 | label.append(process_name) # process label 196 | label.append(process_name) # type label 197 | label.append(self.process_stats[p_pid]['cmdline']) # cmdline label 198 | add_metric(cpu_usages, label, self.process_stats[p_pid], 'cpu') 199 | add_metric(mem_usages, label, self.process_stats[p_pid], 'mem') 200 | del self.process_stats[p_pid] 201 | except KeyError: 202 | pass 203 | 204 | except KeyError: 205 | pass 206 | 207 | # remaining process stats 208 | try: 209 | for process_id, pid_stats in self.process_stats.items(): 210 | label = [process_id] # pid label 211 | try: 212 | # new frigate:0.13.0-beta3 stat 'cmdline' 213 | label.append(pid_stats['cmdline']) # name label 214 | label.append(pid_stats['cmdline']) # process label 215 | label.append('Other') # type label 216 | label.append(pid_stats['cmdline']) # cmdline label 217 | except KeyError: 218 | pass 219 | add_metric(cpu_usages, label, pid_stats, 'cpu') 220 | add_metric(mem_usages, label, pid_stats, 'mem') 221 | except KeyError: 222 | pass 223 | 224 | yield cpu_usages 225 | yield mem_usages 226 | 227 | # gpu stats 228 | gpu_usages = GaugeMetricFamily('frigate_gpu_usage_percent', 'GPU utilisation %', labels=['gpu_name']) 229 | gpu_mem_usages = GaugeMetricFamily('frigate_gpu_mem_usage_percent', 'GPU memory usage %', labels=['gpu_name']) 230 | 231 | try: 232 | for gpu_name, gpu_stats in stats['gpu_usages'].items(): 233 | add_metric(gpu_usages, [gpu_name], gpu_stats, 'gpu') 234 | add_metric(gpu_mem_usages, [gpu_name], gpu_stats, 'mem') 235 | except KeyError: 236 | pass 237 | 238 | yield gpu_usages 239 | yield gpu_mem_usages 240 | 241 | # service stats 242 | uptime_seconds = GaugeMetricFamily('frigate_service_uptime_seconds', 'Uptime seconds') 243 | last_updated_timestamp = GaugeMetricFamily('frigate_service_last_updated_timestamp', 244 | 'Stats recorded time (unix timestamp)') 245 | 246 | try: 247 | service_stats = stats['service'] 248 | add_metric(uptime_seconds, [''], service_stats, 'uptime') 249 | add_metric(last_updated_timestamp, [''], service_stats, 'last_updated') 250 | 251 | info = {'latest_version': stats['service']['latest_version'], 'version': stats['service']['version']} 252 | yield InfoMetricFamily('frigate_service', 'Frigate version info', value=info) 253 | 254 | except KeyError: 255 | pass 256 | 257 | yield uptime_seconds 258 | yield last_updated_timestamp 259 | 260 | temperatures = GaugeMetricFamily('frigate_device_temperature', 'Device Temperature', labels=['device']) 261 | try: 262 | for device_name in stats['service']['temperatures']: 263 | add_metric(temperatures, [device_name], stats['service']['temperatures'], device_name) 264 | except KeyError: 265 | pass 266 | 267 | yield temperatures 268 | 269 | storage_free = GaugeMetricFamily('frigate_storage_free_bytes', 'Storage free bytes', labels=['storage']) 270 | storage_mount_type = InfoMetricFamily('frigate_storage_mount_type', 'Storage mount type', labels=['mount_type', 'storage']) 271 | storage_total = GaugeMetricFamily('frigate_storage_total_bytes', 'Storage total bytes', labels=['storage']) 272 | storage_used = GaugeMetricFamily('frigate_storage_used_bytes', 'Storage used bytes', labels=['storage']) 273 | 274 | try: 275 | for storage_path, storage_stats in stats['service']['storage'].items(): 276 | add_metric(storage_free, [storage_path], storage_stats, 'free', 1e6) # MB to bytes 277 | add_metric(storage_total, [storage_path], storage_stats, 'total', 1e6) # MB to bytes 278 | add_metric(storage_used, [storage_path], storage_stats, 'used', 1e6) # MB to bytes 279 | storage_mount_type.add_metric(storage_path, {'mount_type': storage_stats['mount_type'], 'storage': storage_path}) 280 | except KeyError: 281 | pass 282 | 283 | yield storage_free 284 | yield storage_mount_type 285 | yield storage_total 286 | yield storage_used 287 | 288 | # count events 289 | events = [] 290 | try: 291 | # change url from stats to events 292 | events_url = self.stats_url.replace('stats', 'events') 293 | 294 | if self.previous_event_start_time: 295 | events_url = events_url + '?after=' + str(self.previous_event_start_time) 296 | 297 | events = json.loads(urlopen(events_url).read()) 298 | 299 | except error.URLError as e: 300 | logging.error("URLError while opening Frigate events url %s: %s", self.stats_url, e) 301 | return 302 | 303 | if len(events) > 0: 304 | # events[0] is newest event, last element is oldest, don't need to sort 305 | 306 | if not self.previous_event_id: 307 | # ignore all previous events on startup, prometheus might have already counted them 308 | self.previous_event_id = events[0]['id'] 309 | self.previous_event_start_time = int(events[0]['start_time']) 310 | 311 | for event in events: 312 | # break if event already counted 313 | if event['id'] == self.previous_event_id: 314 | break 315 | 316 | # break if event starts before previous event 317 | if event['start_time'] < self.previous_event_start_time: 318 | break 319 | 320 | # store counted events in a dict 321 | try: 322 | cam = self.all_events[event['camera']] 323 | try: 324 | cam[event['label']] += 1 325 | except KeyError: 326 | # create label dict if not exists 327 | cam.update({event['label']: 1 }) 328 | except KeyError: 329 | # create camera and label dict if not exists 330 | self.all_events.update({event['camera']: {event['label'] : 1} }) 331 | 332 | # don't recount events next time 333 | self.previous_event_id = events[0]['id'] 334 | self.previous_event_start_time = int(events[0]['start_time']) 335 | 336 | camera_events = CounterMetricFamily('frigate_camera_events', 'Count of camera events since exporter started', labels=['camera', 'label']) 337 | 338 | for camera, cam_dict in self.all_events.items(): 339 | for label, label_value in cam_dict.items(): 340 | camera_events.add_metric([camera, label], label_value) 341 | 342 | yield camera_events 343 | 344 | 345 | if __name__ == '__main__': 346 | logging.basicConfig(format='%(asctime)s - %(message)s', level=logging.INFO) 347 | try: 348 | url = os.environ['FRIGATE_STATS_URL'] 349 | except KeyError: 350 | logging.error( 351 | "Provide Frigate stats url as environment variable to container, " 352 | "e.g. FRIGATE_STATS_URL=http://:5000/api/stats") 353 | sys.exit() 354 | 355 | REGISTRY.register(CustomCollector(url)) 356 | port = int(os.environ.get('PORT', 9100)) 357 | start_http_server(port) 358 | 359 | logging.info('Started, Frigate API URL: %s', url) 360 | logging.info('Metrics at: http://localhost:%d/metrics', port) 361 | 362 | while True: 363 | time.sleep(1) 364 | --------------------------------------------------------------------------------