├── .dockerignore ├── .github ├── renovate.js └── workflows │ ├── codeql-analysis.yml │ ├── release.yml │ ├── renovate.yaml │ └── tests.yml ├── .gitignore ├── .goreleaser.yml ├── Dockerfile ├── LICENSE ├── Makefile ├── Readme.md ├── cmd ├── cmd └── mqtt2prometheus.go ├── config.yaml.dist ├── docs ├── overview └── overview.drawio.svg ├── examples ├── gosund_sp111.yaml ├── shelly_3em.yaml └── shelly_ht.yaml ├── freebsd └── mqtt2prometheus.rc ├── fuzzing ├── .gitignore ├── json_per_topic │ └── fuzz.go ├── metric_per_topic │ └── fuzz.go └── start.sh ├── go.mod ├── go.sum ├── hack ├── Readme.md ├── dht.env ├── dht22.yaml ├── docker-compose.yml ├── prometheus.yml ├── shelly.env ├── shelly.yaml └── shellyplusht.yaml ├── pkg ├── config │ ├── config.go │ ├── config_test.go │ └── runtime.go ├── metrics │ ├── collector.go │ ├── extractor.go │ ├── extractor_test.go │ ├── ingest.go │ ├── instrumentation.go │ ├── parser.go │ └── parser_test.go └── mqttclient │ └── mqttClient.go ├── release └── Dockerfile.scratch ├── renovate.json └── systemd ├── mqtt2prometheus └── mqtt2prometheus.service /.dockerignore: -------------------------------------------------------------------------------- 1 | bin/ 2 | .git 3 | systemd/ -------------------------------------------------------------------------------- /.github/renovate.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | "branchPrefix": "test-renovate/", 3 | "dryRun": null, 4 | "username": "renovate-release", 5 | "gitAuthor": "Renovate Bot ", 6 | "onboarding": false, 7 | "platform": "github", 8 | "includeForks": false, 9 | "repositories": ["hikhvar/mqtt2prometheus"], 10 | "packageRules": [ 11 | { 12 | "description": "lockFileMaintenance", 13 | "matchUpdateTypes": [ 14 | "pin", 15 | "digest", 16 | "patch", 17 | "minor", 18 | "major", 19 | "lockFileMaintenance" 20 | ], 21 | "dependencyDashboardApproval": false, 22 | "stabilityDays": 10 23 | } 24 | ] 25 | }; 26 | -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | # You may wish to alter this file to override the set of languages analyzed, 5 | # or to provide custom queries or build logic. 6 | # 7 | # ******** NOTE ******** 8 | # We have attempted to detect the languages in your repository. Please check 9 | # the `language` matrix defined below to confirm you have the correct set of 10 | # supported CodeQL languages. 11 | # 12 | name: "CodeQL" 13 | 14 | on: 15 | push: 16 | branches: [ master ] 17 | pull_request: 18 | # The branches below must be a subset of the branches above 19 | branches: [ master ] 20 | schedule: 21 | - cron: '41 18 * * 2' 22 | 23 | jobs: 24 | analyze: 25 | name: Analyze 26 | runs-on: ubuntu-latest 27 | 28 | strategy: 29 | fail-fast: false 30 | matrix: 31 | language: [ 'go' ] 32 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] 33 | # Learn more: 34 | # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed 35 | 36 | steps: 37 | - name: Checkout repository 38 | uses: actions/checkout@v4 39 | 40 | # Initializes the CodeQL tools for scanning. 41 | - name: Initialize CodeQL 42 | uses: github/codeql-action/init@v1 43 | with: 44 | languages: ${{ matrix.language }} 45 | # If you wish to specify custom queries, you can do so here or in a config file. 46 | # By default, queries listed here will override any specified in a config file. 47 | # Prefix the list here with "+" to use these queries and those in the config file. 48 | # queries: ./path/to/local/query, your-org/your-repo/queries@main 49 | 50 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 51 | # If this step fails, then you should remove it and run the build manually (see below) 52 | - name: Autobuild 53 | uses: github/codeql-action/autobuild@v1 54 | 55 | # ℹ️ Command-line programs to run using the OS shell. 56 | # 📚 https://git.io/JvXDl 57 | 58 | # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines 59 | # and modify them (or add more) to build your code if your project 60 | # uses a compiled language 61 | 62 | #- run: | 63 | # make bootstrap 64 | # make release 65 | 66 | - name: Perform CodeQL Analysis 67 | uses: github/codeql-action/analyze@v1 68 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: release 2 | 3 | on: 4 | push: 5 | tags: 6 | - '*' 7 | 8 | jobs: 9 | goreleaser: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - name: Checkout 13 | uses: actions/checkout@v4 14 | - name: Unshallow 15 | run: git fetch --prune --unshallow 16 | - name: Set up Go 17 | uses: actions/setup-go@v3 18 | with: 19 | go-version: 1.21.x 20 | - name: Test 21 | run: go test -cover ./... 22 | - name: Vet 23 | run: go vet ./... 24 | - name: Set up QEMU 25 | uses: docker/setup-qemu-action@v1 26 | - name: Set up Docker Buildx 27 | uses: docker/setup-buildx-action@v1 28 | # Login to the registries 29 | - name: Login to Github Packages 30 | run: echo ${{ secrets.GITHUB_TOKEN }} | docker login docker.pkg.github.com -u ${{ github.actor }} --password-stdin 31 | - name: Login to Dockerhub 32 | run: echo ${{ secrets.DOCKERHUB_ACCESS_KEY }} | docker login -u hikhvar --password-stdin 33 | # Github Registry does not support the github actions token 34 | - name: Login to Github Registry 35 | run: echo ${{secrets.PERSONAL_ACCESS_TOKEN }} | docker login ghcr.io -u hikhvar --password-stdin 36 | - name: Run GoReleaser 37 | uses: goreleaser/goreleaser-action@v3 38 | with: 39 | version: latest 40 | args: release --rm-dist 41 | env: 42 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 43 | DOCKER_CLI_EXPERIMENTAL: enabled 44 | -------------------------------------------------------------------------------- /.github/workflows/renovate.yaml: -------------------------------------------------------------------------------- 1 | name: Renovate 2 | on: 3 | schedule: 4 | # The "*" (#42, asterisk) character has special semantics in YAML, so this 5 | # string has to be quoted. 6 | - cron: '20 14 * * *' 7 | jobs: 8 | renovate: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - name: Get token 12 | id: get_token 13 | uses: machine-learning-apps/actions-app-token@master 14 | with: 15 | APP_PEM: ${{ secrets.APP_PEM }} 16 | APP_ID: ${{ secrets.APP_ID }} 17 | 18 | - name: Checkout 19 | uses: actions/checkout@v4 20 | 21 | - name: Self-hosted Renovate 22 | uses: renovatebot/github-action@v32.118.0 23 | with: 24 | configurationFile: .github/renovate.js 25 | token: 'x-access-token:${{ steps.get_token.outputs.app_token }}' 26 | -------------------------------------------------------------------------------- /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | on: 2 | push: 3 | branches: ["master"] 4 | pull_request: 5 | name: tests 6 | jobs: 7 | test: 8 | strategy: 9 | matrix: 10 | # Test oldest and newest supported version 11 | go-version: [1.18.x, 1.21.x, 1.22.x, 1.23.x] 12 | platform: [ubuntu-latest, macos-latest, windows-latest] 13 | runs-on: ${{ matrix.platform }} 14 | steps: 15 | - name: Install Go 16 | uses: actions/setup-go@v3 17 | with: 18 | go-version: ${{ matrix.go-version }} 19 | - name: Checkout code 20 | uses: actions/checkout@v4 21 | - name: Test 22 | run: go test ./... 23 | - name: Vet 24 | run: go vet ./... 25 | linting: 26 | runs-on: ubuntu-latest 27 | steps: 28 | - name: Checkout code 29 | uses: actions/checkout@v4 30 | - name: Set up Go 31 | uses: actions/setup-go@v3 32 | with: 33 | go-version: 1.22.x 34 | - name: Run golangci-lint 35 | uses: golangci/golangci-lint-action@v3 36 | with: 37 | only-new-issues: true 38 | 39 | goreleaser: 40 | needs: 41 | - test 42 | - linting 43 | runs-on: ubuntu-latest 44 | steps: 45 | - name: Checkout 46 | uses: actions/checkout@v4 47 | - name: Unshallow 48 | run: git fetch --prune --unshallow 49 | - name: Set up Go 50 | uses: actions/setup-go@v3 51 | with: 52 | go-version: 1.22.x 53 | - name: Set up QEMU 54 | uses: docker/setup-qemu-action@v1 55 | - name: Set up Docker Buildx 56 | uses: docker/setup-buildx-action@v1 57 | - name: Run GoReleaser 58 | uses: goreleaser/goreleaser-action@v3 59 | with: 60 | version: latest 61 | args: release --rm-dist --skip-publish 62 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | config.yaml 2 | bin/ 3 | vendor 4 | dist 5 | .vscode 6 | .idea 7 | -------------------------------------------------------------------------------- /.goreleaser.yml: -------------------------------------------------------------------------------- 1 | # This is an example goreleaser.yaml file with some sane defaults. 2 | # Make sure to check the documentation at http://goreleaser.com 3 | before: 4 | hooks: 5 | # You may remove this if you don't use go modules. 6 | - go mod download 7 | # you may remove this if you don't need go generate 8 | - go generate ./... 9 | release: 10 | prerelease: auto 11 | builds: 12 | - env: 13 | - CGO_ENABLED=0 14 | main: cmd/mqtt2prometheus.go 15 | # GOOS list to build for. 16 | # For more info refer to: https://golang.org/doc/install/source#environment 17 | # Defaults are darwin and linux. 18 | goos: 19 | - linux 20 | - darwin 21 | - freebsd 22 | - windows 23 | 24 | # GOARCH to build for. 25 | # For more info refer to: https://golang.org/doc/install/source#environment 26 | # Defaults are 386 and amd64. 27 | goarch: 28 | - amd64 29 | - arm 30 | - arm64 31 | - 386 32 | 33 | # GOARM to build for when GOARCH is arm. 34 | # For more info refer to: https://golang.org/doc/install/source#environment 35 | # Default is only 6. 36 | goarm: 37 | - 5 38 | - 6 39 | - 7 40 | 41 | # GOMIPS and GOMIPS64 to build when GOARCH is mips, mips64, mipsle or mips64le. 42 | # For more info refer to: https://golang.org/doc/install/source#environment 43 | # Default is empty. 44 | gomips: 45 | - hardfloat 46 | - softfloat 47 | archives: 48 | - name_template: >- 49 | {{- .ProjectName }}_ 50 | {{- .Version }}_ 51 | {{- title .Os }}_ 52 | {{- if eq .Arch "amd64" }}x86_64 53 | {{- else if eq .Arch "386" }}i386 54 | {{- else }}{{ .Arch }}{{ end }} 55 | {{- if .Arm }}v{{ .Arm }}{{ end -}} 56 | checksum: 57 | name_template: 'checksums.txt' 58 | snapshot: 59 | name_template: "{{ .Tag }}-next" 60 | changelog: 61 | sort: asc 62 | filters: 63 | exclude: 64 | - '^docs:' 65 | - '^test:' 66 | - '^ci:' 67 | 68 | nfpms: 69 | - id: default 70 | vendor: Christoph Petrausch 71 | homepage: https://github.com/hikhvar/mqtt2prometheus 72 | description: This exporter translates from MQTT topics to prometheus metrics. 73 | license: MIT License 74 | formats: 75 | - deb 76 | - rpm 77 | - apk 78 | conflicts: 79 | - prometheus-mqtt-exporter 80 | bindir: /usr/bin 81 | contents: 82 | # Simple config file 83 | - src: config.yaml.dist 84 | dst: /etc/mqtt2prometheus/config.yaml 85 | type: config 86 | - src: ./systemd/mqtt2prometheus.service 87 | dst: /etc/systemd/system/mqtt2prometheus.service 88 | type: config 89 | - src: ./systemd/mqtt2prometheus 90 | dst: /etc/default/mqtt2prometheus 91 | type: config 92 | 93 | 94 | dockers: 95 | 96 | - dockerfile: release/Dockerfile.scratch 97 | goos: linux 98 | goarch: amd64 99 | use: buildx 100 | build_flag_templates: 101 | - "--platform=linux/amd64" 102 | image_templates: 103 | - "hikhvar/mqtt2prometheus:{{ .Tag }}-amd64" 104 | - "hikhvar/mqtt2prometheus:latest-amd64" 105 | - "ghcr.io/hikhvar/mqtt2prometheus:{{ .Tag }}-amd64" 106 | - "ghcr.io/hikhvar/mqtt2prometheus:latest-amd64" 107 | 108 | - dockerfile: release/Dockerfile.scratch 109 | goos: linux 110 | goarch: arm64 111 | use: buildx 112 | build_flag_templates: 113 | - "--platform=linux/arm64" 114 | image_templates: 115 | - "hikhvar/mqtt2prometheus:{{ .Tag }}-arm64" 116 | - "hikhvar/mqtt2prometheus:latest-arm64" 117 | - "ghcr.io/hikhvar/mqtt2prometheus:{{ .Tag }}-arm64" 118 | - "ghcr.io/hikhvar/mqtt2prometheus:latest-arm64" 119 | 120 | 121 | - dockerfile: release/Dockerfile.scratch 122 | goos: linux 123 | goarch: arm 124 | goarm: 6 125 | use: buildx 126 | build_flag_templates: 127 | - "--platform=linux/arm/v6" 128 | image_templates: 129 | - "hikhvar/mqtt2prometheus:{{ .Tag }}-arm6" 130 | - "hikhvar/mqtt2prometheus:latest-arm6" 131 | - "ghcr.io/hikhvar/mqtt2prometheus:{{ .Tag }}-arm6" 132 | - "ghcr.io/hikhvar/mqtt2prometheus:latest-arm6" 133 | 134 | - dockerfile: release/Dockerfile.scratch 135 | goos: linux 136 | goarch: arm 137 | goarm: 7 138 | use: buildx 139 | build_flag_templates: 140 | - "--platform=linux/arm/v7" 141 | image_templates: 142 | - "hikhvar/mqtt2prometheus:{{ .Tag }}-arm7" 143 | - "hikhvar/mqtt2prometheus:latest-arm7" 144 | - "ghcr.io/hikhvar/mqtt2prometheus:{{ .Tag }}-arm7" 145 | - "ghcr.io/hikhvar/mqtt2prometheus:latest-arm7" 146 | 147 | - dockerfile: release/Dockerfile.scratch 148 | goos: linux 149 | goarch: 386 150 | use: buildx 151 | build_flag_templates: 152 | - "--platform=linux/386" 153 | image_templates: 154 | - "hikhvar/mqtt2prometheus:{{ .Tag }}-386" 155 | - "hikhvar/mqtt2prometheus:latest-386" 156 | - "ghcr.io/hikhvar/mqtt2prometheus:{{ .Tag }}-386" 157 | - "ghcr.io/hikhvar/mqtt2prometheus:latest-386" 158 | 159 | docker_manifests: 160 | # Docker Registry 161 | - name_template: hikhvar/mqtt2prometheus:{{ .Tag }} 162 | image_templates: 163 | - "hikhvar/mqtt2prometheus:{{ .Tag }}-amd64" 164 | - "hikhvar/mqtt2prometheus:{{ .Tag }}-arm64" 165 | - "hikhvar/mqtt2prometheus:{{ .Tag }}-arm6" 166 | - "hikhvar/mqtt2prometheus:{{ .Tag }}-arm7" 167 | - "hikhvar/mqtt2prometheus:{{ .Tag }}-386" 168 | 169 | - name_template: hikhvar/mqtt2prometheus:latest 170 | image_templates: 171 | - "hikhvar/mqtt2prometheus:latest-amd64" 172 | - "hikhvar/mqtt2prometheus:latest-arm64" 173 | - "hikhvar/mqtt2prometheus:latest-arm6" 174 | - "hikhvar/mqtt2prometheus:latest-arm7" 175 | - "hikhvar/mqtt2prometheus:latest-386" 176 | 177 | # Github Registry 178 | - name_template: ghcr.io/hikhvar/mqtt2prometheus:{{ .Tag }} 179 | image_templates: 180 | - "ghcr.io/hikhvar/mqtt2prometheus:{{ .Tag }}-amd64" 181 | - "ghcr.io/hikhvar/mqtt2prometheus:{{ .Tag }}-arm64" 182 | - "ghcr.io/hikhvar/mqtt2prometheus:{{ .Tag }}-arm6" 183 | - "ghcr.io/hikhvar/mqtt2prometheus:{{ .Tag }}-arm7" 184 | - "ghcr.io/hikhvar/mqtt2prometheus:{{ .Tag }}-386" 185 | 186 | - name_template: ghcr.io/hikhvar/mqtt2prometheus:latest 187 | image_templates: 188 | - "ghcr.io/hikhvar/mqtt2prometheus:latest-amd64" 189 | - "ghcr.io/hikhvar/mqtt2prometheus:latest-arm64" 190 | - "ghcr.io/hikhvar/mqtt2prometheus:latest-arm6" 191 | - "ghcr.io/hikhvar/mqtt2prometheus:latest-arm7" 192 | - "ghcr.io/hikhvar/mqtt2prometheus:latest-386" 193 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.23 as builder 2 | 3 | COPY . /build/mqtt2prometheus 4 | WORKDIR /build/mqtt2prometheus 5 | RUN make static_build TARGET_FILE=/bin/mqtt2prometheus 6 | 7 | FROM gcr.io/distroless/static-debian10:nonroot 8 | WORKDIR / 9 | COPY --from=builder /bin/mqtt2prometheus /mqtt2prometheus 10 | ENTRYPOINT ["/mqtt2prometheus"] 11 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Christoph Petrausch 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | ifndef GOBINARY 2 | GOBINARY:="go" 3 | endif 4 | 5 | ifndef GOPATH 6 | GOPATH:=$(shell $(GOBINARY) env GOPATH) 7 | endif 8 | 9 | ifndef GOBIN 10 | GOBIN:=$(GOPATH)/bin 11 | endif 12 | 13 | ifndef GOARCH 14 | GOARCH:=$(shell $(GOBINARY) env GOARCH) 15 | endif 16 | 17 | ifndef GOOS 18 | GOOS:=$(shell $(GOBINARY) env GOOS) 19 | endif 20 | 21 | ifndef GOARM 22 | GOARM:=$(shell $(GOBINARY) env GOARM) 23 | endif 24 | 25 | ifndef TARGET_FILE 26 | TARGET_FILE:=bin/mqtt2prometheus.$(GOOS)_$(GOARCH)$(GOARM) 27 | endif 28 | 29 | all: build 30 | 31 | GO111MODULE=on 32 | 33 | 34 | lint: 35 | golangci-lint run 36 | 37 | test: 38 | $(GOBINARY) test ./... 39 | $(GOBINARY) vet ./... 40 | 41 | build: 42 | GOOS=$(GOOS) GOARCH=$(GOARCH) $(GOBINARY) build -o $(TARGET_FILE) ./cmd 43 | 44 | static_build: 45 | CGO_ENABLED=0 GOOS=$(GOOS) GOARCH=$(GOARCH) $(GOBINARY) build -o $(TARGET_FILE) -a -tags netgo -ldflags '-w -extldflags "-static"' ./cmd 46 | 47 | container: 48 | docker build -t mqtt2prometheus:latest . 49 | 50 | test_release: 51 | goreleaser --rm-dist --skip-validate --skip-publish 52 | -------------------------------------------------------------------------------- /Readme.md: -------------------------------------------------------------------------------- 1 | # MQTT2Prometheus 2 | ![](https://github.com/hikhvar/mqtt2prometheus/workflows/tests/badge.svg) ![](https://github.com/hikhvar/mqtt2prometheus/workflows/release/badge.svg) 3 | 4 | 5 | This exporter translates from MQTT topics to prometheus metrics. The core design is that clients send arbitrary JSON messages 6 | on the topics. The translation between the MQTT representation and prometheus metrics is configured in the mqtt2prometheus exporter since we often can not change the IoT devices sending 7 | the messages. Clients can push metrics via MQTT to an MQTT Broker. This exporter subscribes to the broker and 8 | expose the received messages as prometheus metrics. Currently, the exporter supports only MQTT 3.1. 9 | 10 | ![Overview Diagram](docs/overview.drawio.svg) 11 | 12 | I wrote this exporter to expose metrics from small embedded sensors based on the NodeMCU to prometheus. 13 | The used arduino sketch can be found in the [dht22tomqtt](https://github.com/hikhvar/dht22tomqtt) repository. 14 | A local hacking environment with mqtt2prometheus, a MQTT broker and a prometheus server is in the [hack](https://github.com/hikhvar/mqtt2prometheus/tree/master/hack) directory. 15 | 16 | ## Assumptions about Messages and Topics 17 | This exporter makes some assumptions about the MQTT topics. This exporter assumes that each 18 | client publish the metrics into a dedicated topic. The regular expression in the configuration field `mqtt.device_id_regex` 19 | defines how to extract the device ID from the MQTT topic. This allows an arbitrary place of the device ID in the mqtt topic. 20 | For example the [tasmota](https://github.com/arendst/Tasmota) firmware pushes the telemetry data to the topics `tele//SENSOR`. 21 | 22 | Let us assume the default configuration from [configuration file](#config-file). A sensor publishes the following message 23 | ```json 24 | {"temperature":23.20,"humidity":51.60, "computed": {"heat_index":22.92} } 25 | ``` 26 | 27 | to the MQTT topic `devices/home/livingroom`. This message becomes the following prometheus metrics: 28 | 29 | ```text 30 | temperature{sensor="livingroom",topic="devices/home/livingroom"} 23.2 31 | heat_index{sensor="livingroom",topic="devices/home/livingroom"} 22.92 32 | humidity{sensor="livingroom",topic="devices/home/livingroom"} 51.6 33 | ``` 34 | 35 | The label `sensor` is extracted with the default `device_id_regex` `(.*/)?(?P.*)` from the MQTT topic `devices/home/livingroom`. 36 | The `device_id_regex` is able to extract exactly one label from the topic path. It extracts only the `deviceid` regex capture group into the `sensor` prometheus label. 37 | To extract more labels from the topic path, have a look at [this FAQ answer](#extract-more-labels-from-the-topic-path). 38 | 39 | The topic path can contain multiple wildcards. MQTT has two wildcards: 40 | * `+`: Single level of hierarchy in the topic path 41 | * `#`: Many levels of hierarchy in the topic path 42 | 43 | This [page](https://mosquitto.org/man/mqtt-7.html) explains the wildcard in depth. 44 | 45 | For example the `topic_path: devices/+/sensors/#` will match: 46 | * `devices/home/sensors/foo/bar` 47 | * `devices/workshop/sensors/temperature` 48 | 49 | ### JSON Separator 50 | The exporter interprets `mqtt_name` as [gojsonq](https://github.com/thedevsaddam/gojsonq) paths. Those paths will be used 51 | to find the value in the JSON message. 52 | For example `mqtt_name: computed.heat_index` 53 | addresses 54 | ```json 55 | { 56 | "computed": { 57 | "heat_index":22.92 58 | } 59 | } 60 | ``` 61 | Some sensors might use a `.` in the JSON keys. Therefore, there the configuration option `json_parsing.seperator` in 62 | the exporter config. This allows us to use any other string to separate hierarchies in the gojsonq path. 63 | E.g let's assume the following MQTT JSON message: 64 | ```json 65 | { 66 | "computed": { 67 | "heat.index":22.92 68 | } 69 | } 70 | ``` 71 | We can now set `json_parsing.seperator` to `/`. This allows us to specify `mqtt_name` as `computed/heat.index`. Keep in mind, 72 | `json_parsing.seperator` is a global setting. This affects all `mqtt_name` fields in your configuration. 73 | 74 | Some devices like Shelly Plus H&T publish one metric per-topic in a JSON format: 75 | ``` 76 | shellies/shellyplusht-xxx/status/humidity:0 {"id": 0,"rh":51.9} 77 | ``` 78 | You can use PayloadField to extract the desired value. 79 | 80 | ### Tasmota 81 | An example configuration for the tasmota based Gosund SP111 device is given in [examples/gosund_sp111.yaml](examples/gosund_sp111.yaml). 82 | 83 | ## Build 84 | 85 | To build the exporter run: 86 | 87 | ```bash 88 | make build 89 | ``` 90 | 91 | Only the latest two Go major versions are tested and supported. 92 | 93 | ### Docker 94 | 95 | #### Use Public Image 96 | 97 | To start the public available image run: 98 | ```bash 99 | docker run -it -v "$(pwd)/config.yaml:/config.yaml" -p 9641:9641 ghcr.io/hikhvar/mqtt2prometheus:latest 100 | ``` 101 | Please have a look at the [latest relase](https://github.com/hikhvar/mqtt2prometheus/releases/latest) to get a stable image tag. The latest tag may break at any moment in time since latest is pushed into the registries on every git commit in the master branch. 102 | 103 | #### Build The Image locally 104 | To build a docker container with the mqtt2prometheus exporter included run: 105 | 106 | ```bash 107 | make container 108 | ``` 109 | 110 | To run the container with a given config file: 111 | 112 | ```bash 113 | docker run -it -v "$(pwd)/config.yaml:/config.yaml" -p 9641:9641 mqtt2prometheus:latest 114 | ``` 115 | 116 | ## Configuration 117 | The exporter can be configured via command line and config file. 118 | 119 | ### Commandline 120 | Available command line flags: 121 | 122 | ```text 123 | Usage of ./mqtt2prometheus: 124 | -config string 125 | config file (default "config.yaml") 126 | -listen-address string 127 | listen address for HTTP server used to expose metrics (default "0.0.0.0") 128 | -listen-port string 129 | HTTP port used to expose metrics (default "9641") 130 | -log-format string 131 | set the desired log output format. Valid values are 'console' and 'json' (default "console") 132 | -log-level value 133 | sets the default loglevel (default: "info") 134 | -version 135 | show the builds version, date and commit 136 | -web-config-file string 137 | [EXPERIMENTAL] Path to configuration file that can enable TLS or authentication for metric scraping. 138 | -treat-mqtt-password-as-file-name bool (default: false) 139 | treat MQTT2PROM_MQTT_PASSWORD environment variable as a secret file path e.g. /var/run/secrets/mqtt-credential. Useful when docker secret or external credential management agents handle the secret file. 140 | ``` 141 | The logging is implemented via [zap](https://github.com/uber-go/zap). The logs are printed to `stderr` and valid log levels are 142 | those supported by zap. 143 | 144 | 145 | ### Config file 146 | The config file can look like this: 147 | 148 | ```yaml 149 | mqtt: 150 | # The MQTT broker to connect to 151 | server: tcp://127.0.0.1:1883 152 | # Optional: Username and Password for authenticating with the MQTT Server 153 | user: bob 154 | password: happylittleclouds 155 | # Optional: for TLS client certificates 156 | ca_cert: certs/AmazonRootCA1.pem 157 | client_cert: certs/xxxxx-certificate.pem.crt 158 | client_key: certs/xxxxx-private.pem.key 159 | # Optional: Used to specify ClientID. The default is - 160 | client_id: somedevice 161 | # The Topic path to subscribe to. Be aware that you have to specify the wildcard, if you want to follow topics for multiple sensors. 162 | topic_path: v1/devices/me/+ 163 | # Optional: Regular expression to extract the device ID from the topic path. The default regular expression, assumes 164 | # that the last "element" of the topic_path is the device id. 165 | # The regular expression must contain a named capture group with the name deviceid 166 | # For example the expression for tasamota based sensors is "tele/(?P.*)/.*" 167 | device_id_regex: "(.*/)?(?P.*)" 168 | # The MQTT QoS level 169 | qos: 0 170 | # NOTE: Only one of metric_per_topic_config or object_per_topic_config should be specified in the configuration 171 | # Optional: Configures mqtt2prometheus to expect a single metric to be published as the value on an mqtt topic. 172 | metric_per_topic_config: 173 | # A regex used for extracting the metric name from the topic. Must contain a named group for `metricname`. 174 | metric_name_regex: "(.*/)?(?P.*)" 175 | # Optional: Configures mqtt2prometheus to expect an object containing multiple metrics to be published as the value on an mqtt topic. 176 | # This is the default. 177 | object_per_topic_config: 178 | # The encoding of the object, currently only json is supported 179 | encoding: JSON 180 | cache: 181 | # Timeout. Each received metric will be presented for this time if no update is send via MQTT. 182 | # Set the timeout to -1 to disable the deletion of metrics from the cache. The exporter presents the ingest timestamp 183 | # to prometheus. 184 | timeout: 24h 185 | # Path to the directory to keep the state for monotonic metrics. 186 | state_directory: "/var/lib/mqtt2prometheus" 187 | json_parsing: 188 | # Separator. Used to split path to elements when accessing json fields. 189 | # You can access json fields with dots in it. F.E. {"key.name": {"nested": "value"}} 190 | # Just set separator to -> and use key.name->nested as mqtt_name 191 | separator: . 192 | # This is a list of valid metrics. Only metrics listed here will be exported 193 | metrics: 194 | # The name of the metric in prometheus 195 | - prom_name: temperature 196 | # The name of the metric in a MQTT JSON message 197 | mqtt_name: temperature 198 | # The prometheus help text for this metric 199 | help: DHT22 temperature reading 200 | # The prometheus type for this metric. Valid values are: "gauge" and "counter" 201 | type: gauge 202 | # A map of string to string for constant labels. This labels will be attached to every prometheus metric 203 | const_labels: 204 | sensor_type: dht22 205 | # A map of string to expression for dynamic labels. This labels will be attached to every prometheus metric 206 | # expression will be executed for each label every time a metric is processed 207 | # dynamic_labels: 208 | # raw_value: "raw_value" 209 | # The name of the metric in prometheus 210 | - prom_name: humidity 211 | # The name of the metric in a MQTT JSON message 212 | mqtt_name: humidity 213 | # The scale of the metric in a MQTT JSON message (prom_value = mqtt_value * scale) 214 | mqtt_value_scale: 100 215 | # The prometheus help text for this metric 216 | help: DHT22 humidity reading 217 | # The prometheus type for this metric. Valid values are: "gauge" and "counter" 218 | type: gauge 219 | # A map of string to string for constant labels. This labels will be attached to every prometheus metric 220 | const_labels: 221 | sensor_type: dht22 222 | # The name of the metric in prometheus 223 | - prom_name: heat_index 224 | # The path of the metric in a MQTT JSON message 225 | mqtt_name: computed.heat_index 226 | # The prometheus help text for this metric 227 | help: DHT22 heatIndex calculation 228 | # The prometheus type for this metric. Valid values are: "gauge" and "counter" 229 | type: gauge 230 | # A map of string to string for constant labels. This labels will be attached to every prometheus metric 231 | const_labels: 232 | sensor_type: dht22 233 | # The name of the metric in prometheus 234 | - prom_name: state 235 | # The name of the metric in a MQTT JSON message 236 | mqtt_name: state 237 | # Regular expression to only match sensors with the given name pattern 238 | sensor_name_filter: "^.*-light$" 239 | # The prometheus help text for this metric 240 | help: Light state 241 | # The prometheus type for this metric. Valid values are: "gauge" and "counter" 242 | type: gauge 243 | # according to prometheus exposition format timestamp is not mandatory, we can omit it if the reporting from the sensor is sporadic 244 | omit_timestamp: true 245 | # A map of string to string for constant labels. This labels will be attached to every prometheus metric 246 | const_labels: 247 | sensor_type: ikea 248 | # When specified, metric value to use if a value cannot be parsed (match cannot be found in the map above, invalid float parsing, expression fails, ...) 249 | # If not specified, parsing error will occur. 250 | error_value: 1 251 | # When specified, enables mapping between string values to metric values. 252 | string_value_mapping: 253 | # A map of string to metric value. 254 | map: 255 | off: 0 256 | low: 0 257 | # The name of the metric in prometheus 258 | - prom_name: total_light_usage_seconds 259 | # The name of the metric in a MQTT JSON message 260 | mqtt_name: state 261 | # Regular expression to only match sensors with the given name pattern 262 | sensor_name_filter: "^.*-light$" 263 | # The prometheus help text for this metric 264 | help: Total time the light was on, in seconds 265 | # The prometheus type for this metric. Valid values are: "gauge" and "counter" 266 | type: counter 267 | # according to prometheus exposition format timestamp is not mandatory, we can omit it if the reporting from the sensor is sporadic 268 | omit_timestamp: true 269 | # A map of string to string for constant labels. This labels will be attached to every prometheus metric 270 | const_labels: 271 | sensor_type: ikea 272 | # Metric value to use if a value cannot be parsed (match cannot be found in the map above, invalid float parsing, ...) 273 | # If not specified, parsing error will occur. 274 | error_value: 1 275 | # When specified, enables mapping between string values to metric values. 276 | string_value_mapping: 277 | # A map of string to metric value. 278 | map: 279 | off: 0 280 | low: 0 281 | # Sum up the time the light is on, see the section "Expressions" below. 282 | expression: "value > 0 ? last_result + elapsed.Seconds() : last_result" 283 | # The name of the metric in prometheus 284 | - prom_name: total_energy 285 | # The name of the metric in a MQTT JSON message 286 | mqtt_name: aenergy.total 287 | # Regular expression to only match sensors with the given name pattern 288 | sensor_name_filter: "^shellyplus1pm-.*$" 289 | # The prometheus help text for this metric 290 | help: Total energy used 291 | # The prometheus type for this metric. Valid values are: "gauge" and "counter" 292 | type: counter 293 | # This setting requires an almost monotonic counter as the source. When monotonicy is enforced, the metric value is regularly written to disk. Thus, resets in the source counter can be detected and corrected by adding an offset as if the reset did not happen. The result is a true monotonic increasing time series, like an ever growing counter. 294 | force_monotonicy: true 295 | - prom_name: linky_time 296 | # The name of the metric in a MQTT JSON message 297 | mqtt_name: linky_current_date 298 | # Regular expression to only match sensors with the given name pattern 299 | sensor_name_filter: "^linky.*$" 300 | # The prometheus help text for this metric 301 | help: current unix timestamp from linky 302 | # The prometheus type for this metric. Valid values are: "gauge" and "counter" 303 | type: gauge 304 | # convert dynamic datetime string to unix timestamp 305 | raw_expression: 'date(string(raw_value), "H060102150405", "Europe/Paris").Unix()' 306 | ``` 307 | 308 | ### Environment Variables 309 | 310 | Having the MQTT login details in the config file runs the risk of publishing them to a version control system. To avoid this, you can supply these parameters via environment variables. MQTT2Prometheus will look for `MQTT2PROM_MQTT_USER` and `MQTT2PROM_MQTT_PASSWORD` in the local environment and load them on startup. 311 | 312 | #### Example use with Docker 313 | 314 | Create a file to store your login details, for example at `~/secrets/mqtt2prom`: 315 | ```SHELL 316 | #!/bin/bash 317 | export MQTT2PROM_MQTT_USER="myUser" 318 | export MQTT2PROM_MQTT_PASSWORD="superpassword" 319 | ``` 320 | 321 | Then load that file into the environment before starting the container: 322 | ```SHELL 323 | source ~/secrets/mqtt2prom && \ 324 | docker run -it \ 325 | -e MQTT2PROM_MQTT_USER \ 326 | -e MQTT2PROM_MQTT_PASSWORD \ 327 | -v "$(pwd)/examples/config.yaml:/config.yaml" \ 328 | -p 9641:9641 \ 329 | ghcr.io/hikhvar/mqtt2prometheus:latest 330 | ``` 331 | 332 | #### Example use with Docker secret (in swarm) 333 | 334 | Create a docker secret to store the password(`mqtt-credential` in the example below), and pass the optional `treat-mqtt-password-as-file-name` command line argument. 335 | ```docker 336 | mqtt_exporter_tasmota: 337 | image: ghcr.io/hikhvar/mqtt2prometheus:latest 338 | secrets: 339 | - mqtt-credential 340 | environment: 341 | - MQTT2PROM_MQTT_USER=mqtt 342 | - MQTT2PROM_MQTT_PASSWORD=/var/run/secrets/mqtt-credential 343 | entrypoint: 344 | - /mqtt2prometheus 345 | - -log-level=debug 346 | - -treat-mqtt-password-as-file-name=true 347 | volumes: 348 | - config-tasmota.yml:/config.yaml:ro 349 | ``` 350 | 351 | ### Expressions 352 | 353 | Expression is a peace of code that is run dynamically for calculate metric value or generate dynamic labels. 354 | 355 | #### Metric value 356 | Metric values can be derived from sensor inputs using complex expressions. Set the metric config option `raw_expression` or `expression` to the desired formular to calculate the result from the input. `raw_expression` and `expression` are mutually exclusives: 357 | * `raw_expression` is run without raw value conversion. It's `raw_expression` duty to handle the conversion. Only `raw_value` is set while `value` is always set to 0.0. Here is an example which convert datetime (format `HYYMMDDhhmmss`) to unix timestamp: 358 | ```yaml 359 | raw_expression: 'date(string(raw_value), "H060102150405", "Europe/Paris").Unix()' 360 | ``` 361 | * `expression` is run after raw value conversion. If conversion fails, `expression` is not run. Here's an example which integrates all positive values over time: 362 | ```yaml 363 | expression: "value > 0 ? last_result + value * elapsed.Seconds() : last_result" 364 | ``` 365 | 366 | #### Dynamic labels 367 | Dynamic labels are derivated from sensor inputs using complex expressions. Define labels and the corresponding expression in the metric config otpion `dynamic_labels`. 368 | `raw_value` and `value` are both set in this context. The value returned from dynamic labels expression is not typed and will be converted to string before being exported. 369 | 370 | #### Expression 371 | During the evaluation, the following variables are available to the expression: 372 | * `raw_value` - the raw MQTT sensor value (without any conversion) 373 | * `value` - the current sensor value (after string-value mapping, if configured) 374 | * `last_value` - the `value` during the previous expression evaluation 375 | * `last_result` - the result from the previous expression evaluation (a float for `raw_expression`/`expression`, a string for `dynamic_labels`) 376 | * `elapsed` - the time that passed since the previous evaluation, as a [Duration](https://pkg.go.dev/time#Duration) value 377 | 378 | The [language definition](https://expr-lang.org/docs/v1.9/Language-Definition) describes the expression syntax. In addition, the following functions are available: 379 | * `now()` - the current time as a [Time](https://pkg.go.dev/time#Time) value 380 | * `int(x)` - convert `x` to an integer value 381 | * `float(x)` - convert `x` to a floating point value 382 | * `round(x)` - rounds value `x` to the nearest integer 383 | * `ceil(x)` - rounds value `x` up to the next higher integer 384 | * `floor(x)` - rounds value `x` down to the next lower integer 385 | * `abs(x)` - returns the `x` as a positive number 386 | * `min(x, y)` - returns the minimum of `x` and `y` 387 | * `max(x, y)` - returns the maximum of `x` and `y` 388 | 389 | [Time](https://pkg.go.dev/time#Time) and [Duration](https://pkg.go.dev/time#Duration) values come with their own methods which can be used in expressions. For example, `elapsed.Milliseconds()` yields the number of milliseconds that passed since the last evaluation, while `now().Sub(elapsed).Weekday()` returns the day of the week during the previous evaluation. 390 | 391 | The `last_value`, `last_result`, and the timestamp of the last evaluation are regularly stored on disk. When mqtt2prometheus is restarted, the data is read back for the next evaluation. This means that you can calculate stable, long-running time serious which depend on the previous result. 392 | 393 | #### Evaluation Order 394 | 395 | It is important to understand the sequence of transformations from a sensor input to the final output which is exported to Prometheus. The steps are as follows: 396 | 397 | If `raw_expression` is set, the generated value of the expression is exported to Prometheus. Otherwise: 398 | 1. The sensor input is converted to a number. If a `string_value_mapping` is configured, it is consulted for the conversion. 399 | 1. If an `expression` is configured, it is evaluated using the converted number. The result of the evaluation replaces the converted sensor value. 400 | 1. If `force_monotonicy` is set to `true`, any new value that is smaller than the previous one is considered to be a counter reset. When a reset is detected, the previous value becomes the value offset which is automatically added to each consecutive value. The offset is persistet between restarts of mqtt2prometheus. 401 | 1. If `mqtt_value_scale` is set to a non-zero value, it is applied to the the value to yield the final metric value. 402 | 403 | ## Frequently Asked Questions 404 | 405 | ### Listen to multiple Topic Pathes 406 | The exporter can only listen to one topic_path per instance. If you have to listen to two different topic_paths it is 407 | recommended to run two instances of the mqtt2prometheus exporter. You can run both on the same host or if you run in Kubernetes, 408 | even in the same pod. 409 | 410 | ### Extract more Labels from the Topic Path 411 | A regular use case is, that user want to extract more labels from the topic path. E.g. they have sensors not only in their `home` but also 412 | in their `workshop` and they encode the location in the topic path. E.g. a sensor pushes the message 413 | 414 | ```json 415 | {"temperature":3.0,"humidity":34.60, "computed": {"heat_index":15.92} } 416 | ``` 417 | 418 | to the topic `devices/workshop/storage`, this will produce the prometheus metrics with the default configuration. 419 | 420 | ```text 421 | temperature{sensor="storage",topic="devices/workshop/storage"} 3.0 422 | heat_index{sensor="storage",topic="devices/workshop/storage"} 15.92 423 | humidity{sensor="storage",topic="devices/workshop/storage"} 34.60 424 | ``` 425 | 426 | The following prometheus [relabel_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config) will extract the location from the topic path as well and attaches the `location` label. 427 | ```yaml 428 | relabel_config: 429 | - source_labels: [ "topic" ] 430 | target_label: location 431 | regex: '/devices/(.*)/.*' 432 | action: replace 433 | replacement: "$1" 434 | ``` 435 | 436 | With this config added to your prometheus scrape config you will get the following metrics in prometheus storage: 437 | 438 | ```text 439 | temperature{sensor="storage", location="workshop", topic="devices/workshop/storage"} 3.0 440 | heat_index{sensor="storage", location="workshop", topic="devices/workshop/storage"} 15.92 441 | humidity{sensor="storage", location="workshop", topic="devices/workshop/storage"} 34.60 442 | ``` 443 | -------------------------------------------------------------------------------- /cmd/cmd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hikhvar/mqtt2prometheus/8bd029b13cfc06fe9b8e9142b391dd5d3d92a6bb/cmd/cmd -------------------------------------------------------------------------------- /cmd/mqtt2prometheus.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "crypto/tls" 5 | "crypto/x509" 6 | "encoding/json" 7 | "flag" 8 | "fmt" 9 | "io/ioutil" 10 | "net/http" 11 | "os" 12 | "time" 13 | 14 | "go.uber.org/zap" 15 | "go.uber.org/zap/zapcore" 16 | 17 | mqtt "github.com/eclipse/paho.mqtt.golang" 18 | "github.com/go-kit/kit/log" 19 | kitzap "github.com/go-kit/kit/log/zap" 20 | "github.com/hikhvar/mqtt2prometheus/pkg/config" 21 | "github.com/hikhvar/mqtt2prometheus/pkg/metrics" 22 | "github.com/hikhvar/mqtt2prometheus/pkg/mqttclient" 23 | "github.com/prometheus/client_golang/prometheus" 24 | "github.com/prometheus/client_golang/prometheus/promhttp" 25 | "github.com/prometheus/exporter-toolkit/web" 26 | ) 27 | 28 | // These variables are set by goreleaser at linking time. 29 | var ( 30 | version string 31 | commit string 32 | date string 33 | ) 34 | 35 | var ( 36 | configFlag = flag.String( 37 | "config", 38 | "config.yaml", 39 | "config file", 40 | ) 41 | portFlag = flag.String( 42 | "listen-port", 43 | "9641", 44 | "HTTP port used to expose metrics", 45 | ) 46 | addressFlag = flag.String( 47 | "listen-address", 48 | "0.0.0.0", 49 | "listen address for HTTP server used to expose metrics", 50 | ) 51 | versionFlag = flag.Bool( 52 | "version", 53 | false, 54 | "show the builds version, date and commit", 55 | ) 56 | logLevelFlag = zap.LevelFlag("log-level", zap.InfoLevel, "sets the default loglevel (default: \"info\")") 57 | logEncodingFlag = flag.String( 58 | "log-format", 59 | "console", 60 | "set the desired log output format. Valid values are 'console' and 'json'", 61 | ) 62 | webConfigFlag = flag.String( 63 | "web-config-file", 64 | "", 65 | "[EXPERIMENTAL] Path to configuration file that can enable TLS or authentication for metric scraping.", 66 | ) 67 | usePasswordFromFile = flag.Bool( 68 | "treat-mqtt-password-as-file-name", 69 | false, 70 | "treat MQTT2PROM_MQTT_PASSWORD as a secret file path e.g. /var/run/secrets/mqtt-credential", 71 | ) 72 | ) 73 | 74 | func main() { 75 | flag.Parse() 76 | if *versionFlag { 77 | mustShowVersion() 78 | os.Exit(0) 79 | } 80 | logger := mustSetupLogger() 81 | defer logger.Sync() //nolint:errcheck 82 | c := make(chan os.Signal, 1) 83 | cfg, err := config.LoadConfig(*configFlag, logger) 84 | if err != nil { 85 | logger.Fatal("Could not load config", zap.Error(err)) 86 | } 87 | 88 | mqtt_user := os.Getenv("MQTT2PROM_MQTT_USER") 89 | if mqtt_user != "" { 90 | cfg.MQTT.User = mqtt_user 91 | } 92 | 93 | mqtt_password := os.Getenv("MQTT2PROM_MQTT_PASSWORD") 94 | if *usePasswordFromFile { 95 | if mqtt_password == "" { 96 | logger.Fatal("MQTT2PROM_MQTT_PASSWORD is required") 97 | } 98 | secret, err := ioutil.ReadFile(mqtt_password) 99 | if err != nil { 100 | logger.Fatal("unable to read mqtt password from secret file", zap.Error(err)) 101 | } 102 | cfg.MQTT.Password = string(secret) 103 | } else { 104 | if mqtt_password != "" { 105 | cfg.MQTT.Password = mqtt_password 106 | } 107 | } 108 | 109 | mqttClientOptions := mqtt.NewClientOptions() 110 | mqttClientOptions.AddBroker(cfg.MQTT.Server).SetCleanSession(true) 111 | mqttClientOptions.SetAutoReconnect(true) 112 | mqttClientOptions.SetUsername(cfg.MQTT.User) 113 | mqttClientOptions.SetPassword(cfg.MQTT.Password) 114 | 115 | if cfg.MQTT.ClientID != "" { 116 | mqttClientOptions.SetClientID(cfg.MQTT.ClientID) 117 | } else { 118 | mqttClientOptions.SetClientID(mustMQTTClientID()) 119 | } 120 | 121 | if cfg.MQTT.ClientCert != "" || cfg.MQTT.ClientKey != "" { 122 | tlsconfig, err := newTLSConfig(cfg) 123 | if err != nil { 124 | logger.Fatal("Invalid tls certificate settings", zap.Error(err)) 125 | } 126 | mqttClientOptions.SetTLSConfig(tlsconfig) 127 | } 128 | 129 | collector := metrics.NewCollector(cfg.Cache.Timeout, cfg.Metrics, logger) 130 | extractor, err := setupExtractor(cfg) 131 | if err != nil { 132 | logger.Fatal("could not setup a metric extractor", zap.Error(err)) 133 | } 134 | ingest := metrics.NewIngest(collector, extractor, cfg.MQTT.DeviceIDRegex) 135 | mqttClientOptions.SetOnConnectHandler(ingest.OnConnectHandler) 136 | mqttClientOptions.SetConnectionLostHandler(ingest.ConnectionLostHandler) 137 | errorChan := make(chan error, 1) 138 | 139 | for { 140 | err = mqttclient.Subscribe(mqttClientOptions, mqttclient.SubscribeOptions{ 141 | Topic: cfg.MQTT.TopicPath, 142 | QoS: cfg.MQTT.QoS, 143 | OnMessageReceived: ingest.SetupSubscriptionHandler(errorChan), 144 | Logger: logger, 145 | }) 146 | if err == nil { 147 | // connected, break loop 148 | break 149 | } 150 | logger.Warn("could not connect to mqtt broker, sleep 10 second", zap.Error(err)) 151 | time.Sleep(10 * time.Second) 152 | } 153 | 154 | var gatherer prometheus.Gatherer 155 | if cfg.EnableProfiling { 156 | gatherer = prometheus.DefaultGatherer 157 | } else { 158 | reg := prometheus.NewRegistry() 159 | reg.MustRegister(ingest.Collector()) 160 | reg.MustRegister(collector) 161 | gatherer = reg 162 | } 163 | http.Handle("/metrics", promhttp.HandlerFor(gatherer, promhttp.HandlerOpts{})) 164 | s := &http.Server{ 165 | Addr: getListenAddress(), 166 | Handler: http.DefaultServeMux, 167 | } 168 | go func() { 169 | err = web.ListenAndServe(s, *webConfigFlag, setupGoKitLogger(logger)) 170 | if err != nil { 171 | logger.Fatal("Error while serving http", zap.Error(err)) 172 | } 173 | }() 174 | 175 | for { 176 | select { 177 | case <-c: 178 | logger.Info("Terminated via Signal. Stop.") 179 | os.Exit(0) 180 | case err = <-errorChan: 181 | logger.Error("Error while processing message", zap.Error(err)) 182 | } 183 | } 184 | } 185 | 186 | func getListenAddress() string { 187 | return fmt.Sprintf("%s:%s", *addressFlag, *portFlag) 188 | } 189 | 190 | func mustShowVersion() { 191 | versionInfo := struct { 192 | Version string 193 | Commit string 194 | Date string 195 | }{ 196 | Version: version, 197 | Commit: commit, 198 | Date: date, 199 | } 200 | 201 | err := json.NewEncoder(os.Stdout).Encode(versionInfo) 202 | if err != nil { 203 | panic(err) 204 | } 205 | } 206 | 207 | func mustMQTTClientID() string { 208 | host, err := os.Hostname() 209 | if err != nil { 210 | panic(fmt.Sprintf("failed to get hostname: %v", err)) 211 | } 212 | pid := os.Getpid() 213 | return fmt.Sprintf("%s-%d", host, pid) 214 | } 215 | 216 | func mustSetupLogger() *zap.Logger { 217 | cfg := zap.NewProductionConfig() 218 | cfg.Level = zap.NewAtomicLevelAt(*logLevelFlag) 219 | cfg.Encoding = *logEncodingFlag 220 | if cfg.Encoding == "console" { 221 | cfg.EncoderConfig.EncodeTime = zapcore.RFC3339TimeEncoder 222 | } 223 | logger, err := cfg.Build() 224 | if err != nil { 225 | panic(fmt.Sprintf("failed to build logger: %v", err)) 226 | } 227 | 228 | config.SetProcessContext(logger) 229 | return logger 230 | } 231 | 232 | func setupGoKitLogger(l *zap.Logger) log.Logger { 233 | return kitzap.NewZapSugarLogger(l, zap.NewAtomicLevelAt(*logLevelFlag).Level()) 234 | } 235 | 236 | func setupExtractor(cfg config.Config) (metrics.Extractor, error) { 237 | parser := metrics.NewParser(cfg.Metrics, cfg.JsonParsing.Separator, cfg.Cache.StateDir) 238 | if cfg.MQTT.ObjectPerTopicConfig != nil { 239 | switch cfg.MQTT.ObjectPerTopicConfig.Encoding { 240 | case config.EncodingJSON: 241 | return metrics.NewJSONObjectExtractor(parser), nil 242 | default: 243 | return nil, fmt.Errorf("unsupported object format: %s", cfg.MQTT.ObjectPerTopicConfig.Encoding) 244 | } 245 | } 246 | if cfg.MQTT.MetricPerTopicConfig != nil { 247 | return metrics.NewMetricPerTopicExtractor(parser, cfg.MQTT.MetricPerTopicConfig.MetricNameRegex), nil 248 | } 249 | return nil, fmt.Errorf("no extractor configured") 250 | } 251 | 252 | func newTLSConfig(cfg config.Config) (*tls.Config, error) { 253 | certpool := x509.NewCertPool() 254 | if cfg.MQTT.CACert != "" { 255 | pemCerts, err := ioutil.ReadFile(cfg.MQTT.CACert) 256 | if err != nil { 257 | return nil, fmt.Errorf("failed to load ca_cert file: %w", err) 258 | } 259 | certpool.AppendCertsFromPEM(pemCerts) 260 | } 261 | 262 | cert, err := tls.LoadX509KeyPair(cfg.MQTT.ClientCert, cfg.MQTT.ClientKey) 263 | if err != nil { 264 | return nil, fmt.Errorf("failed to load client certificate: %w", err) 265 | } 266 | 267 | cert.Leaf, err = x509.ParseCertificate(cert.Certificate[0]) 268 | if err != nil { 269 | return nil, fmt.Errorf("failed to parse client certificate: %w", err) 270 | } 271 | 272 | return &tls.Config{ 273 | RootCAs: certpool, 274 | InsecureSkipVerify: false, 275 | Certificates: []tls.Certificate{cert}, 276 | }, nil 277 | } 278 | -------------------------------------------------------------------------------- /config.yaml.dist: -------------------------------------------------------------------------------- 1 | mqtt: 2 | # The MQTT broker to connect to 3 | server: tcp://127.0.0.1:1883 4 | # Optional: Username and Password for authenticating with the MQTT Server 5 | # user: bob 6 | # password: happylittleclouds 7 | # Optional: for TLS client certificates 8 | # ca_cert: certs/AmazonRootCA1.pem 9 | # client_cert: certs/xxxxx-certificate.pem.crt 10 | # client_key: certs/xxxxx-private.pem.key 11 | # Optional: Used to specify ClientID. The default is - 12 | # client_id: somedevice 13 | # The Topic path to subscribe to. Be aware that you have to specify the wildcard. 14 | topic_path: v1/devices/me/+ 15 | # Optional: Regular expression to extract the device ID from the topic path. The default regular expression, assumes 16 | # that the last "element" of the topic_path is the device id. 17 | # The regular expression must contain a named capture group with the name deviceid 18 | # For example the expression for tasamota based sensors is "tele/(?P.*)/.*" 19 | # device_id_regex: "(.*/)?(?P.*)" 20 | # The MQTT QoS level 21 | qos: 0 22 | # Export internal profiling metrics including CPU, Memory, uptime, open file 23 | # descriptors, as well as metrics exported by Go runtime such as information about 24 | # heap and garbage collection stats. 25 | enable_profiling_metrics: false 26 | cache: 27 | # Timeout. Each received metric will be presented for this time if no update is send via MQTT. 28 | # Set the timeout to -1 to disable the deletion of metrics from the cache. The exporter presents the ingest timestamp 29 | # to prometheus. 30 | timeout: 24h 31 | json_parsing: 32 | # Separator. Used to split path to elements when accessing json fields. 33 | # You can access json fields with dots in it. F.E. {"key.name": {"nested": "value"}} 34 | # Just set separator to -> and use key.name->nested as mqtt_name 35 | separator: . 36 | # This is a list of valid metrics. Only metrics listed here will be exported 37 | metrics: 38 | # The name of the metric in prometheus 39 | - prom_name: temperature 40 | # The name of the metric in a MQTT JSON message 41 | mqtt_name: temperature 42 | # The prometheus help text for this metric 43 | help: DHT22 temperature reading 44 | # The prometheus type for this metric. Valid values are: "gauge" and "counter" 45 | type: gauge 46 | # A map of string to string for constant labels. This labels will be attached to every prometheus metric 47 | const_labels: 48 | sensor_type: dht22 49 | # The name of the metric in prometheus 50 | - prom_name: humidity 51 | # The name of the metric in a MQTT JSON message 52 | mqtt_name: humidity 53 | # The prometheus help text for this metric 54 | help: DHT22 humidity reading 55 | # The prometheus type for this metric. Valid values are: "gauge" and "counter" 56 | type: gauge 57 | # A map of string to string for constant labels. This labels will be attached to every prometheus metric 58 | const_labels: 59 | sensor_type: dht22 60 | # The name of the metric in prometheus 61 | - prom_name: heat_index 62 | # The name of the metric in a MQTT JSON message 63 | mqtt_name: heat_index 64 | # The prometheus help text for this metric 65 | help: DHT22 heatIndex calculation 66 | # The prometheus type for this metric. Valid values are: "gauge" and "counter" 67 | type: gauge 68 | # A map of string to string for constant labels. This labels will be attached to every prometheus metric 69 | const_labels: 70 | sensor_type: dht22 71 | # The name of the metric in prometheus 72 | - prom_name: state 73 | # The name of the metric in a MQTT JSON message 74 | mqtt_name: state 75 | # Regular expression to only match sensors with the given name pattern 76 | sensor_name_filter: "^.*-light$" 77 | # The prometheus help text for this metric 78 | help: Light state 79 | # The prometheus type for this metric. Valid values are: "gauge" and "counter" 80 | type: gauge 81 | # A map of string to string for constant labels. This labels will be attached to every prometheus metric 82 | const_labels: 83 | sensor_type: ikea 84 | # Metric value to use if a value cannot be parsed (match cannot be found in the map above, invalid float parsing, ...) 85 | # If not specified, parsing error will occur. 86 | error_value: 1 87 | # When specified, enables mapping between string values to metric values. 88 | string_value_mapping: 89 | # A map of string to metric value. 90 | map: 91 | off: 0 92 | low: 0 93 | -------------------------------------------------------------------------------- /docs/overview: -------------------------------------------------------------------------------- 1 | 3ZjLcpswFIafhqUzXIxNlrXjpNNJpmmdSdtVR8AJKAFEhYjtPn2lIsCKwJc2duNuPOiXkMT3H+lINpxpuryiKI9vSAiJYZvh0nAuDNs+dz3+K4RVJYzdcSVEFIeVZLXCHP8EKZpSLXEIhdKQEZIwnKtiQLIMAqZoiFKyUJs9kEQdNUcRaMI8QImufsEhi6VqmWZb8R5wFMuhPVdW+Ch4iigpMzleRjKoalJUdyObFjEKyWJNcmaGM6WEsOopXU4hEVRrYtV7lz21zZQpZGyXF54/LL59vrK/e/fTQV7ef51fPl4M7KqXZ5SUEsXNp7s7rkwoeQIqZ85WNSj+Ebl4DFYJzkKgjuFMFjFmMM9RICoWPD64FrM04SWLP/qCDoTXfiM0zD6WjHcDUi+qsLBc/hxiyn3GJONCQUqBciKnCpTBspeB1ZDlsQokBUZXvIl8YVgbKuPU8mR5seZ6bVi8ZvhYakgGWtR03RLnDxL6HgaMNQM05oJV3vvxckkgv25u7gulgVBDGbs6FLMDyuhQUBwNyhyygujhSKvY4gVzeyDuFD/9Jun81vh4HXiGh8IzPA08dS8voktfcUeF5/bAK94mPVulZ/9jeucaJQh5JpVFQllMIpKhZNaqE5Vj2+aakFzSewTGVvJYgEpGutiKgTZv+3xepKQBbHefIRrBJlPsblMoJIjhZ3Uerw65XiJrMXpb+gku4k7618jnpzGFGEpwJFJnwAHxLO5MRGBift55JytSHIaVOcDTrkwewp6c4Iz9/iB3YrgXmyJ7l8zTH0S9ET8wz3gGspSwH8jizvxl77fia9aakIeHgvv+0qBmEn/hmZ63T2tlWPW5+20vjY6jQekXAcU+6Dv4Sa4Oqwew7N48sz1r/Kqro574UE3U6vsHXDqW5mn6gzE7p6LrGMpjpObNR+SR/fLe0HFE7ro3HOyIbLmnvt8Md9xvrB5rjrThjLTgvJqJC7LojY+Ag/9l23G3ZWX7fKRm5beflPWb0m3/pnKs/zZeYz8a/tG1wNt/N+LF9i+qypf2H0Bn9gs= -------------------------------------------------------------------------------- /docs/overview.drawio.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 |
Sensor
Sensor
Sensor
Sensor
Sensors
Sensors
Publish
Publish
Subscribes
Subscribes
mqtt2prometheus
mqtt2prometheus
GET /metrics
GET /metricsBrokerViewer does not support full SVG 1.1 -------------------------------------------------------------------------------- /examples/gosund_sp111.yaml: -------------------------------------------------------------------------------- 1 | # Settings for the MQTT Client. Currently only these three are supported 2 | mqtt: 3 | # The MQTT broker to connect to 4 | server: tcp://192.168.1.11:1883 5 | # Optional: Username and Password for authenticating with the MQTT Server 6 | # user: bob 7 | # password: happylittleclouds 8 | # The Topic path to subscribe to. Be aware that you have to specify the wildcard. 9 | topic_path: tele/+/SENSOR 10 | # Optional: Regular expression to extract the device ID from the topic path. The default regular expression, assumes 11 | # that the last "element" of the topic_path is the device id. 12 | # The regular expression must contain a named capture group with the name deviceid 13 | # For example the expression for tasamota based sensors is "tele/(?P.*)/.*" 14 | device_id_regex: "tele/(?P.*)/SENSOR" 15 | # The MQTT QoS level 16 | qos: 0 17 | cache: 18 | # Timeout. Each received metric will be presented for this time if no update is send via MQTT. 19 | # Set the timeout to -1 to disable the deletion of metrics from the cache. The exporter presents the ingest timestamp 20 | # to prometheus. 21 | timeout: 24h 22 | # This is a list of valid metrics. Only metrics listed here will be exported 23 | metrics: 24 | # The name of the metric in prometheus 25 | - prom_name: consumed_energy_kilowatthours_total 26 | mqtt_name: "ENERGY.Total" 27 | help: "total measured kilowatthours since flash" 28 | type: counter 29 | - prom_name: voltage_volts 30 | mqtt_name: "ENERGY.Voltage" 31 | help: "Currently measured voltage" 32 | type: gauge 33 | - prom_name: current_amperes 34 | mqtt_name: "ENERGY.Current" 35 | help: "Currently measured current" 36 | type: gauge 37 | - prom_name: power_watts 38 | mqtt_name: "ENERGY.Power" 39 | help: "Currently measured power" 40 | type: gauge 41 | - prom_name: apparent_power_watt 42 | mqtt_name: "ENERGY.ApparentPower" 43 | help: "Currently apparent power" 44 | type: gauge 45 | - prom_name: reactive_power_watt 46 | mqtt_name: "ENERGY.ReactivePower" 47 | help: "Currently reactive power" 48 | type: gauge -------------------------------------------------------------------------------- /examples/shelly_3em.yaml: -------------------------------------------------------------------------------- 1 | # Sample mqtt messages processed by this configuration file, 2 | # $ mosquitto_sub -t "shellies/shellyem3-123456789/emeter/+/+" -v 3 | # 4 | # shellies/shellyem3-123456789/emeter/0/power 41.25 5 | # shellies/shellyem3-123456789/emeter/0/pf 0.18 6 | # shellies/shellyem3-123456789/emeter/0/current 0.99 7 | # shellies/shellyem3-123456789/emeter/0/voltage 232.25 8 | # shellies/shellyem3-123456789/emeter/0/total 13372.4 9 | # shellies/shellyem3-123456789/emeter/0/total_returned 0.0 10 | # shellies/shellyem3-123456789/emeter/1/power 275.04 11 | # shellies/shellyem3-123456789/emeter/1/pf 0.72 12 | # shellies/shellyem3-123456789/emeter/1/current 1.65 13 | # shellies/shellyem3-123456789/emeter/1/voltage 232.83 14 | # shellies/shellyem3-123456789/emeter/1/total 27948.4 15 | # shellies/shellyem3-123456789/emeter/1/total_returned 0.0 16 | # shellies/shellyem3-123456789/emeter/2/power -2.23 17 | # shellies/shellyem3-123456789/emeter/2/pf -0.02 18 | # shellies/shellyem3-123456789/emeter/2/current 0.39 19 | # shellies/shellyem3-123456789/emeter/2/voltage 233.14 20 | # shellies/shellyem3-123456789/emeter/2/total 4107.8 21 | # shellies/shellyem3-123456789/emeter/2/total_returned 186.9 22 | 23 | # Settings for the MQTT Client. Currently only these three are supported 24 | mqtt: 25 | # The MQTT broker to connect to 26 | server: tcp://127.0.0.1:1883 27 | # Optional: Username and Password for authenticating with the MQTT Server 28 | # user: bob 29 | # password: happylittleclouds 30 | 31 | # The Topic path to subscribe to. Be aware that you have to specify the wildcard. 32 | topic_path: shellies/shellyem3-123456789/emeter/+/+ 33 | 34 | # Use the phase number as device_id in order to see all three phases in /metrics 35 | device_id_regex: "shellies/(.*)/emeter/(?P.*)/.*" 36 | 37 | # Metrics are being published on a per-topic basis. 38 | metric_per_topic_config: 39 | metric_name_regex: "shellies/(?P.*)/emeter/(.*)/(?P.*)" 40 | # The MQTT QoS level 41 | qos: 0 42 | cache: 43 | timeout: 60m 44 | 45 | metrics: 46 | - prom_name: power 47 | mqtt_name: power 48 | type: gauge 49 | const_labels: 50 | sensor_type: shelly 51 | 52 | - prom_name: voltage 53 | mqtt_name: voltage 54 | type: gauge 55 | const_labels: 56 | sensor_type: shelly 57 | -------------------------------------------------------------------------------- /examples/shelly_ht.yaml: -------------------------------------------------------------------------------- 1 | # Sample MQTT messages from Shelly H&T processed by this configuration file: 2 | # $ mosquitto_sub -h 127.0.0.1 -t 'shellies/+/sensor/+' -v -u bob -P happylittleclouds 3 | # 4 | # shellies/shellyht-CC2D76/sensor/temperature 24.75 5 | # shellies/shellyht-CC2D76/sensor/humidity 43.5 6 | # shellies/shellyht-CC2D76/sensor/battery 100 7 | # shellies/shellyht-CC2D76/sensor/ext_power false 8 | # shellies/shellyht-CC2D76/sensor/error 0 9 | # shellies/shellyht-CC2D76/sensor/act_reasons ["sensor"] 10 | 11 | mqtt: 12 | server: tcp://127.0.0.1:1883 13 | user: bob 14 | password: happylittleclouds 15 | topic_path: shellies/+/sensor/+ 16 | device_id_regex: "shellies/(?P.*)/sensor/.*" 17 | metric_per_topic_config: 18 | metric_name_regex: "shellies/(?P.*)/sensor/(?P.*)" 19 | qos: 0 20 | cache: 21 | timeout: 24h 22 | metrics: 23 | - prom_name: temperature 24 | mqtt_name: temperature 25 | type: gauge 26 | const_labels: 27 | sensor_type: shelly 28 | - prom_name: humidity 29 | mqtt_name: humidity 30 | type: gauge 31 | const_labels: 32 | sensor_type: shelly 33 | - prom_name: battery 34 | mqtt_name: battery 35 | type: gauge 36 | const_labels: 37 | sensor_type: shelly 38 | 39 | -------------------------------------------------------------------------------- /freebsd/mqtt2prometheus.rc: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # PROVIDE: mqtt2prometheus 4 | # REQUIRE: NETWORKING DAEMON 5 | 6 | . /etc/rc.subr 7 | 8 | name=mqtt2prometheus 9 | rcvar=mqtt2prometheus_enable 10 | mqtt2prometheus_config="/usr/local/etc/mqtt2prometheus/config.yaml" 11 | 12 | command="/usr/local/bin/mqtt2prometheus" 13 | 14 | start_cmd="/usr/sbin/daemon -T mqtt2prometheus -u nobody -c $command -config=${mqtt2prometheus_config}" 15 | 16 | load_rc_config $name 17 | run_rc_command "$1" 18 | -------------------------------------------------------------------------------- /fuzzing/.gitignore: -------------------------------------------------------------------------------- 1 | corpus 2 | crashers 3 | supressions 4 | fuzz-target 5 | -------------------------------------------------------------------------------- /fuzzing/json_per_topic/fuzz.go: -------------------------------------------------------------------------------- 1 | // +build gofuzz 2 | 3 | package json 4 | 5 | import ( 6 | "github.com/hikhvar/mqtt2prometheus/pkg/config" 7 | "github.com/hikhvar/mqtt2prometheus/pkg/metrics" 8 | ) 9 | 10 | func Fuzz(data []byte) int { 11 | p := metrics.NewParser([]config.MetricConfig{ 12 | { 13 | PrometheusName: "temperature", 14 | ValueType: "gauge", 15 | }, 16 | { 17 | PrometheusName: "enabled", 18 | ValueType: "gauge", 19 | ErrorValue: floatP(12333), 20 | StringValueMapping: &config.StringValueMappingConfig{ 21 | Map: map[string]float64{ 22 | "foo": 112, 23 | "bar": 2, 24 | }, 25 | }, 26 | }, 27 | { 28 | PrometheusName: "kartoffeln", 29 | ValueType: "counter", 30 | }, 31 | }, ".") 32 | json := metrics.NewJSONObjectExtractor(p) 33 | mc, err := json("foo", data, "bar") 34 | if err != nil && len(mc) > 0 { 35 | return 1 36 | } 37 | return 0 38 | } 39 | 40 | func floatP(f float64) *float64 { 41 | return &f 42 | } 43 | -------------------------------------------------------------------------------- /fuzzing/metric_per_topic/fuzz.go: -------------------------------------------------------------------------------- 1 | // +build gofuzz 2 | 3 | package metric_per_topic 4 | 5 | import ( 6 | "fmt" 7 | "github.com/hikhvar/mqtt2prometheus/pkg/config" 8 | "github.com/hikhvar/mqtt2prometheus/pkg/metrics" 9 | ) 10 | 11 | func Fuzz(data []byte) int { 12 | p := metrics.NewParser([]config.MetricConfig{ 13 | { 14 | PrometheusName: "temperature", 15 | ValueType: "gauge", 16 | }, 17 | { 18 | PrometheusName: "enabled", 19 | ValueType: "gauge", 20 | ErrorValue: floatP(12333), 21 | StringValueMapping: &config.StringValueMappingConfig{ 22 | Map: map[string]float64{ 23 | "foo": 112, 24 | "bar": 2, 25 | }, 26 | }, 27 | }, 28 | { 29 | PrometheusName: "kartoffeln", 30 | ValueType: "counter", 31 | }, 32 | }) 33 | json := metrics.NewMetricPerTopicExtractor(p, config.MustNewRegexp("shellies/(?P.*)/sensor/(?P.*)")) 34 | 35 | name := "enabled" 36 | consumed := 0 37 | if len(data) > 0 { 38 | 39 | name = []string{"temperature", "enabled", "kartoffel"}[data[0]%3] 40 | consumed += 1 41 | 42 | } 43 | mc, err := json(fmt.Sprintf("shellies/bar/sensor/%s", name), data[consumed:], "bar") 44 | if err != nil && len(mc) > 0 { 45 | return 1 46 | } 47 | return 0 48 | } 49 | 50 | func floatP(f float64) *float64 { 51 | return &f 52 | } 53 | -------------------------------------------------------------------------------- /fuzzing/start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | go-fuzz-build -o fuzz-target && go-fuzz -bin fuzz-target 4 | 5 | rm -rf fuzz-target 6 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/hikhvar/mqtt2prometheus 2 | 3 | go 1.18 4 | 5 | require ( 6 | github.com/eclipse/paho.mqtt.golang v1.3.5 7 | github.com/expr-lang/expr v1.16.9 8 | github.com/go-kit/kit v0.10.0 9 | github.com/patrickmn/go-cache v2.1.0+incompatible 10 | github.com/prometheus/client_golang v1.11.1 11 | github.com/prometheus/exporter-toolkit v0.7.3 12 | github.com/thedevsaddam/gojsonq/v2 v2.5.2 13 | go.uber.org/zap v1.16.0 14 | gopkg.in/yaml.v2 v2.4.0 15 | ) 16 | 17 | require ( 18 | github.com/beorn7/perks v1.0.1 // indirect 19 | github.com/cespare/xxhash/v2 v2.1.1 // indirect 20 | github.com/go-kit/log v0.1.0 // indirect 21 | github.com/go-logfmt/logfmt v0.5.0 // indirect 22 | github.com/golang/protobuf v1.5.0 // indirect 23 | github.com/gorilla/websocket v1.4.2 // indirect 24 | github.com/jpillora/backoff v1.0.0 // indirect 25 | github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect 26 | github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect 27 | github.com/pkg/errors v0.9.1 // indirect 28 | github.com/prometheus/client_model v0.2.0 // indirect 29 | github.com/prometheus/common v0.29.0 // indirect 30 | github.com/prometheus/procfs v0.6.0 // indirect 31 | go.uber.org/atomic v1.6.0 // indirect 32 | go.uber.org/multierr v1.5.0 // indirect 33 | golang.org/x/crypto v0.31.0 // indirect 34 | golang.org/x/net v0.21.0 // indirect 35 | golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c // indirect 36 | golang.org/x/sys v0.28.0 // indirect 37 | golang.org/x/text v0.21.0 // indirect 38 | google.golang.org/appengine v1.6.6 // indirect 39 | google.golang.org/protobuf v1.33.0 // indirect 40 | ) 41 | -------------------------------------------------------------------------------- /hack/Readme.md: -------------------------------------------------------------------------------- 1 | # Hack Scenarios 2 | 3 | Required is a MQTT client. I use this: https://github.com/shirou/mqttcli 4 | 5 | ## Shelly (Metric Per Topic) 6 | The scenario is the feature requested by issue https://github.com/hikhvar/mqtt2prometheus/issues/26. 7 | 8 | To start the scenario run: 9 | ```bash 10 | docker-compose --env-file shelly.env up 11 | ``` 12 | 13 | To publish a message run: 14 | ```bash 15 | mqttcli pub --host localhost -p 1883 -t shellies/living-room/sensor/temperature '15' 16 | ``` 17 | 18 | ## DHT22 (Object Per Topic) 19 | The default scenario 20 | 21 | To start the scenario run: 22 | ```bash 23 | docker-compose --env-file dht22.env up 24 | ``` 25 | 26 | To publish a message run: 27 | ```bash 28 | mqttcli pub --host localhost -p 1883 -t v1/devices/me/test -m '{"temperature":"12", "humidity":21}' 29 | ``` -------------------------------------------------------------------------------- /hack/dht.env: -------------------------------------------------------------------------------- 1 | CONFIG=dht22.yaml -------------------------------------------------------------------------------- /hack/dht22.yaml: -------------------------------------------------------------------------------- 1 | # Settings for the MQTT Client. Currently only these three are supported 2 | mqtt: 3 | # The MQTT broker to connect to 4 | server: tcp://mosquitto:1883 5 | # Optional: Username and Password for authenticating with the MQTT Server 6 | # user: bob 7 | # password: happylittleclouds 8 | # The Topic path to subscribe to. Be aware that you have to specify the wildcard. 9 | topic_path: v1/devices/me/+ 10 | # The MQTT QoS level 11 | qos: 0 12 | cache: 13 | # Timeout. Each received metric will be presented for this time if no update is send via MQTT 14 | timeout: 60m 15 | # This is a list of valid metrics. Only metrics listed here will be exported 16 | metrics: 17 | # The name of the metric in prometheus 18 | - prom_name: temperature 19 | # The name of the metric in a MQTT JSON message 20 | mqtt_name: temperature 21 | # The prometheus help text for this metric 22 | help: DHT22 temperature reading 23 | # The prometheus type for this metric. Valid values are: "gauge" and "counter" 24 | type: gauge 25 | # A map of string to string for constant labels. This labels will be attached to every prometheus metric 26 | const_labels: 27 | sensor_type: dht22 28 | # The name of the metric in prometheus 29 | - prom_name: humidity 30 | # The name of the metric in a MQTT JSON message 31 | mqtt_name: humidity 32 | # The prometheus help text for this metric 33 | help: DHT22 humidity reading 34 | # The prometheus type for this metric. Valid values are: "gauge" and "counter" 35 | type: gauge 36 | # A map of string to string for constant labels. This labels will be attached to every prometheus metric 37 | const_labels: 38 | sensor_type: dht22 39 | # The name of the metric in prometheus 40 | - prom_name: heat_index 41 | # The name of the metric in a MQTT JSON message 42 | mqtt_name: heat_index 43 | # The prometheus help text for this metric 44 | help: DHT22 heatIndex calculation 45 | # The prometheus type for this metric. Valid values are: "gauge" and "counter" 46 | type: gauge 47 | # A map of string to string for constant labels. This labels will be attached to every prometheus metric 48 | const_labels: 49 | sensor_type: dht22 50 | - prom_name: state 51 | # The name of the metric in a MQTT JSON message 52 | mqtt_name: state 53 | # The prometheus help text for this metric 54 | help: Light state 55 | # The prometheus type for this metric. Valid values are: "gauge" and "counter" 56 | type: gauge 57 | # A map of string to string for constant labels. This labels will be attached to every prometheus metric 58 | const_labels: 59 | sensor_type: ikea 60 | # Metric value to use if a value cannot be parsed (match cannot be found in the map above, invalid float parsing, ...) 61 | # If not specified, parsing error will occur. 62 | error_value: 1 63 | # When specified, enables mapping between string values to metric values. 64 | string_value_mapping: 65 | # A map of string to metric value. 66 | map: 67 | off: 0 68 | low: 0 69 | -------------------------------------------------------------------------------- /hack/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3.8" 2 | services: 3 | mqtt2prometheus: 4 | build: 5 | context: ../ 6 | dockerfile: Dockerfile 7 | command: 8 | - /mqtt2prometheus 9 | - -log-level 10 | - debug 11 | - -config 12 | - /config.yaml 13 | ports: 14 | - 9641:9641 15 | volumes: 16 | - type: bind 17 | source: ./${CONFIG:-dht22.yaml} 18 | target: /config.yaml 19 | mosquitto: 20 | image: eclipse-mosquitto:1.6.15 21 | ports: 22 | - 1883:1883 23 | - 9001:9001 24 | prometheus: 25 | image: prom/prometheus:v2.55.1 26 | ports: 27 | - 9090:9090 28 | volumes: 29 | - type: bind 30 | source: ./prometheus.yml 31 | target: /etc/prometheus/prometheus.yml 32 | -------------------------------------------------------------------------------- /hack/prometheus.yml: -------------------------------------------------------------------------------- 1 | global: 2 | scrape_interval: 15s 3 | scrape_timeout: 10s 4 | evaluation_interval: 15s 5 | alerting: 6 | alertmanagers: 7 | - static_configs: 8 | - targets: [] 9 | scheme: http 10 | timeout: 10s 11 | api_version: v1 12 | scrape_configs: 13 | - job_name: prometheus 14 | honor_timestamps: true 15 | scrape_interval: 15s 16 | scrape_timeout: 10s 17 | metrics_path: /metrics 18 | scheme: http 19 | static_configs: 20 | - targets: 21 | - localhost:9090 22 | - job_name: mqtt2prometheus 23 | honor_timestamps: true 24 | scrape_interval: 15s 25 | scrape_timeout: 10s 26 | metrics_path: /metrics 27 | scheme: http 28 | static_configs: 29 | - targets: 30 | - mqtt2prometheus:9641 31 | -------------------------------------------------------------------------------- /hack/shelly.env: -------------------------------------------------------------------------------- 1 | CONFIG=shelly.yaml -------------------------------------------------------------------------------- /hack/shelly.yaml: -------------------------------------------------------------------------------- 1 | mqtt: 2 | server: tcp://mosquitto:1883 3 | topic_path: shellies/+/sensor/+ 4 | device_id_regex: "shellies/(?P.*)/sensor" 5 | metric_per_topic_config: 6 | metric_name_regex: "shellies/(?P.*)/sensor/(?P.*)" 7 | qos: 0 8 | cache: 9 | timeout: 24h 10 | metrics: 11 | - prom_name: temperature 12 | # The name of the metric in a MQTT JSON message 13 | mqtt_name: temperature 14 | # The prometheus help text for this metric 15 | help: shelly temperature reading 16 | # The prometheus type for this metric. Valid values are: "gauge" and "counter" 17 | type: gauge 18 | # A map of string to string for constant labels. This labels will be attached to every prometheus metric 19 | const_labels: 20 | sensor_type: shelly 21 | # The name of the metric in prometheus 22 | - prom_name: humidity 23 | # The name of the metric in a MQTT JSON message 24 | mqtt_name: humidity 25 | # The prometheus help text for this metric 26 | help: shelly humidity reading 27 | # The prometheus type for this metric. Valid values are: "gauge" and "counter" 28 | type: gauge 29 | # A map of string to string for constant labels. This labels will be attached to every prometheus metric 30 | const_labels: 31 | sensor_type: shelly 32 | # The name of the metric in prometheus 33 | - prom_name: battery 34 | # The name of the metric in a MQTT JSON message 35 | mqtt_name: battery 36 | # The prometheus help text for this metric 37 | help: shelly battery reading 38 | # The prometheus type for this metric. Valid values are: "gauge" and "counter" 39 | type: gauge 40 | # A map of string to string for constant labels. This labels will be attached to every prometheus metric 41 | const_labels: 42 | sensor_type: shelly 43 | # The name of the metric in prometheus 44 | -------------------------------------------------------------------------------- /hack/shellyplusht.yaml: -------------------------------------------------------------------------------- 1 | mqtt: 2 | server: tcp://mosquitto:1883 3 | topic_path: shellies/+/sensor/+ 4 | device_id_regex: "shellies/(?P.*)/sensor" 5 | metric_per_topic_config: 6 | metric_name_regex: "shellies/(?P.*)/sensor/(?P.*)" 7 | qos: 0 8 | cache: 9 | timeout: 24h 10 | metrics: 11 | - prom_name: temperature 12 | # The name of the metric in a MQTT JSON message 13 | mqtt_name: status/temperature:0 14 | # The field to extract in JSON payload 15 | PayloadField: rh 16 | # The prometheus help text for this metric 17 | help: shelly temperature reading 18 | # The prometheus type for this metric. Valid values are: "gauge" and "counter" 19 | type: gauge 20 | # A map of string to string for constant labels. This labels will be attached to every prometheus metric 21 | const_labels: 22 | sensor_type: shellyplusht 23 | -------------------------------------------------------------------------------- /pkg/config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "os" 7 | "regexp" 8 | "sort" 9 | "time" 10 | 11 | "github.com/prometheus/client_golang/prometheus" 12 | "go.uber.org/zap" 13 | "gopkg.in/yaml.v2" 14 | ) 15 | 16 | const ( 17 | GaugeValueType = "gauge" 18 | CounterValueType = "counter" 19 | 20 | DeviceIDRegexGroup = "deviceid" 21 | MetricNameRegexGroup = "metricname" 22 | ) 23 | 24 | var MQTTConfigDefaults = MQTTConfig{ 25 | Server: "tcp://127.0.0.1:1883", 26 | TopicPath: "v1/devices/me", 27 | DeviceIDRegex: MustNewRegexp(fmt.Sprintf("(.*/)?(?P<%s>.*)", DeviceIDRegexGroup)), 28 | QoS: 0, 29 | } 30 | 31 | var CacheConfigDefaults = CacheConfig{ 32 | Timeout: 2 * time.Minute, 33 | StateDir: "/var/lib/mqtt2prometheus", 34 | } 35 | 36 | var JsonParsingConfigDefaults = JsonParsingConfig{ 37 | Separator: ".", 38 | } 39 | 40 | type Regexp struct { 41 | r *regexp.Regexp 42 | pattern string 43 | } 44 | 45 | func (rf *Regexp) UnmarshalYAML(unmarshal func(interface{}) error) error { 46 | var pattern string 47 | if err := unmarshal(&pattern); err != nil { 48 | return err 49 | } 50 | r, err := regexp.Compile(pattern) 51 | if err != nil { 52 | return err 53 | } 54 | rf.r = r 55 | rf.pattern = pattern 56 | return nil 57 | } 58 | 59 | func (rf *Regexp) MarshalYAML() (interface{}, error) { 60 | if rf == nil { 61 | return "", nil 62 | } 63 | return rf.pattern, nil 64 | } 65 | 66 | func (rf *Regexp) Match(s string) bool { 67 | return rf == nil || rf.r == nil || rf.r.MatchString(s) 68 | } 69 | 70 | // GroupValue returns the value of the given group. If the group is not part of the underlying regexp, returns the empty string. 71 | func (rf *Regexp) GroupValue(s string, groupName string) string { 72 | match := rf.r.FindStringSubmatch(s) 73 | groupValues := make(map[string]string) 74 | for i, name := range rf.r.SubexpNames() { 75 | if len(match) > i && name != "" { 76 | groupValues[name] = match[i] 77 | } 78 | } 79 | return groupValues[groupName] 80 | } 81 | 82 | func (rf *Regexp) RegEx() *regexp.Regexp { 83 | return rf.r 84 | } 85 | 86 | func MustNewRegexp(pattern string) *Regexp { 87 | return &Regexp{ 88 | pattern: pattern, 89 | r: regexp.MustCompile(pattern), 90 | } 91 | } 92 | 93 | type Config struct { 94 | JsonParsing *JsonParsingConfig `yaml:"json_parsing,omitempty"` 95 | Metrics []MetricConfig `yaml:"metrics"` 96 | MQTT *MQTTConfig `yaml:"mqtt,omitempty"` 97 | Cache *CacheConfig `yaml:"cache,omitempty"` 98 | EnableProfiling bool `yaml:"enable_profiling_metrics,omitempty"` 99 | } 100 | 101 | type CacheConfig struct { 102 | Timeout time.Duration `yaml:"timeout"` 103 | StateDir string `yaml:"state_directory"` 104 | } 105 | 106 | type JsonParsingConfig struct { 107 | Separator string `yaml:"separator"` 108 | } 109 | 110 | type MQTTConfig struct { 111 | Server string `yaml:"server"` 112 | TopicPath string `yaml:"topic_path"` 113 | DeviceIDRegex *Regexp `yaml:"device_id_regex"` 114 | User string `yaml:"user"` 115 | Password string `yaml:"password"` 116 | QoS byte `yaml:"qos"` 117 | ObjectPerTopicConfig *ObjectPerTopicConfig `yaml:"object_per_topic_config"` 118 | MetricPerTopicConfig *MetricPerTopicConfig `yaml:"metric_per_topic_config"` 119 | CACert string `yaml:"ca_cert"` 120 | ClientCert string `yaml:"client_cert"` 121 | ClientKey string `yaml:"client_key"` 122 | ClientID string `yaml:"client_id"` 123 | } 124 | 125 | const EncodingJSON = "JSON" 126 | 127 | type ObjectPerTopicConfig struct { 128 | Encoding string `yaml:"encoding"` // Currently only JSON is a valid value 129 | } 130 | 131 | type MetricPerTopicConfig struct { 132 | MetricNameRegex *Regexp `yaml:"metric_name_regex"` // Default 133 | } 134 | 135 | // Metrics Config is a mapping between a metric send on mqtt to a prometheus metric 136 | type MetricConfig struct { 137 | PrometheusName string `yaml:"prom_name"` 138 | MQTTName string `yaml:"mqtt_name"` 139 | PayloadField string `yaml:"payload_field"` 140 | SensorNameFilter Regexp `yaml:"sensor_name_filter"` 141 | Help string `yaml:"help"` 142 | ValueType string `yaml:"type"` 143 | OmitTimestamp bool `yaml:"omit_timestamp"` 144 | RawExpression string `yaml:"raw_expression"` 145 | Expression string `yaml:"expression"` 146 | ForceMonotonicy bool `yaml:"force_monotonicy"` 147 | ConstantLabels map[string]string `yaml:"const_labels"` 148 | DynamicLabels map[string]string `yaml:"dynamic_labels"` 149 | StringValueMapping *StringValueMappingConfig `yaml:"string_value_mapping"` 150 | MQTTValueScale float64 `yaml:"mqtt_value_scale"` 151 | // ErrorValue is used while error during value parsing 152 | ErrorValue *float64 `yaml:"error_value"` 153 | } 154 | 155 | // StringValueMappingConfig defines the mapping from string to float 156 | type StringValueMappingConfig struct { 157 | // ErrorValue was used when no mapping is found in Map 158 | // deprecated, a warning will be issued to migrate to metric level 159 | ErrorValue *float64 `yaml:"error_value"` 160 | Map map[string]float64 `yaml:"map"` 161 | } 162 | 163 | func (mc *MetricConfig) PrometheusDescription() *prometheus.Desc { 164 | labels := append([]string{"sensor", "topic"}, mc.DynamicLabelsKeys()...) 165 | return prometheus.NewDesc( 166 | mc.PrometheusName, mc.Help, labels, mc.ConstantLabels, 167 | ) 168 | } 169 | 170 | func (mc *MetricConfig) PrometheusValueType() prometheus.ValueType { 171 | switch mc.ValueType { 172 | case GaugeValueType: 173 | return prometheus.GaugeValue 174 | case CounterValueType: 175 | return prometheus.CounterValue 176 | default: 177 | return prometheus.UntypedValue 178 | } 179 | } 180 | 181 | func (mc *MetricConfig) DynamicLabelsKeys() []string { 182 | var labels []string 183 | for k := range mc.DynamicLabels { 184 | labels = append(labels, k) 185 | } 186 | sort.Strings(labels) 187 | return labels 188 | } 189 | 190 | func LoadConfig(configFile string, logger *zap.Logger) (Config, error) { 191 | configData, err := ioutil.ReadFile(configFile) 192 | if err != nil { 193 | return Config{}, err 194 | } 195 | var cfg Config 196 | if err = yaml.UnmarshalStrict(configData, &cfg); err != nil { 197 | return cfg, err 198 | } 199 | if cfg.MQTT == nil { 200 | cfg.MQTT = &MQTTConfigDefaults 201 | } 202 | if cfg.Cache == nil { 203 | cfg.Cache = &CacheConfigDefaults 204 | } 205 | if cfg.Cache.StateDir == "" { 206 | cfg.Cache.StateDir = CacheConfigDefaults.StateDir 207 | } 208 | if cfg.JsonParsing == nil { 209 | cfg.JsonParsing = &JsonParsingConfigDefaults 210 | } 211 | if cfg.MQTT.DeviceIDRegex == nil { 212 | cfg.MQTT.DeviceIDRegex = MQTTConfigDefaults.DeviceIDRegex 213 | } 214 | var validRegex bool 215 | for _, name := range cfg.MQTT.DeviceIDRegex.RegEx().SubexpNames() { 216 | if name == DeviceIDRegexGroup { 217 | validRegex = true 218 | } 219 | } 220 | if !validRegex { 221 | return Config{}, fmt.Errorf("device id regex %q does not contain required regex group %q", cfg.MQTT.DeviceIDRegex.pattern, DeviceIDRegexGroup) 222 | } 223 | 224 | if cfg.MQTT.ObjectPerTopicConfig != nil && cfg.MQTT.MetricPerTopicConfig != nil { 225 | return Config{}, fmt.Errorf("only one of object_per_topic_config and metric_per_topic_config can be specified") 226 | } 227 | 228 | if cfg.MQTT.ObjectPerTopicConfig == nil && cfg.MQTT.MetricPerTopicConfig == nil { 229 | cfg.MQTT.ObjectPerTopicConfig = &ObjectPerTopicConfig{ 230 | Encoding: EncodingJSON, 231 | } 232 | } 233 | 234 | if cfg.MQTT.MetricPerTopicConfig != nil { 235 | validRegex = false 236 | for _, name := range cfg.MQTT.MetricPerTopicConfig.MetricNameRegex.RegEx().SubexpNames() { 237 | if name == MetricNameRegexGroup { 238 | validRegex = true 239 | } 240 | } 241 | if !validRegex { 242 | return Config{}, fmt.Errorf("metric name regex %q does not contain required regex group %q", cfg.MQTT.DeviceIDRegex.pattern, MetricNameRegexGroup) 243 | } 244 | } 245 | 246 | // If any metric forces monotonicy, we need a state directory. 247 | forcesMonotonicy := false 248 | for _, m := range cfg.Metrics { 249 | if m.ForceMonotonicy { 250 | forcesMonotonicy = true 251 | } 252 | 253 | if m.StringValueMapping != nil && m.StringValueMapping.ErrorValue != nil { 254 | if m.ErrorValue != nil { 255 | return Config{}, fmt.Errorf("metric %s/%s: cannot set both string_value_mapping.error_value and error_value (string_value_mapping.error_value is deprecated).", m.MQTTName, m.PrometheusName) 256 | } 257 | logger.Warn("string_value_mapping.error_value is deprecated: please use error_value at the metric level.", zap.String("prometheusName", m.PrometheusName), zap.String("MQTTName", m.MQTTName)) 258 | } 259 | 260 | if m.Expression != "" && m.RawExpression != "" { 261 | return Config{}, fmt.Errorf("metric %s/%s: expression and raw_expression are mutually exclusive.", m.MQTTName, m.PrometheusName) 262 | } 263 | } 264 | if forcesMonotonicy { 265 | if err := os.MkdirAll(cfg.Cache.StateDir, 0755); err != nil { 266 | return Config{}, fmt.Errorf("failed to create directory %q: %w", cfg.Cache.StateDir, err) 267 | } 268 | } 269 | 270 | return cfg, nil 271 | } 272 | -------------------------------------------------------------------------------- /pkg/config/config_test.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | ) 7 | 8 | func TestRegexp_GroupValue(t *testing.T) { 9 | type args struct { 10 | s string 11 | groupName string 12 | } 13 | tests := []struct { 14 | name string 15 | pattern string 16 | args args 17 | want string 18 | }{ 19 | { 20 | name: "normal match", 21 | pattern: "(.*/)?(?P.*)", 22 | args: args{ 23 | s: "foo/bar", 24 | groupName: "deviceid", 25 | }, 26 | want: "bar", 27 | }, 28 | { 29 | name: "two groups", 30 | pattern: "(.*/)?(?P.*)/(?P.*)", 31 | args: args{ 32 | s: "foo/bar/batz", 33 | groupName: "deviceid", 34 | }, 35 | want: "bar", 36 | }, 37 | { 38 | name: "empty string match", 39 | pattern: "(.*/)?(?P.*)", 40 | args: args{ 41 | s: "", 42 | groupName: "deviceid", 43 | }, 44 | want: "", 45 | }, 46 | { 47 | name: "not match", 48 | pattern: "(.*)/(?P.*)", 49 | args: args{ 50 | s: "bar", 51 | groupName: "deviceid", 52 | }, 53 | want: "", 54 | }, 55 | { 56 | name: "empty pattern", 57 | pattern: "", 58 | args: args{ 59 | s: "bar", 60 | groupName: "deviceid", 61 | }, 62 | want: "", 63 | }, 64 | } 65 | for _, tt := range tests { 66 | t.Run(tt.name, func(t *testing.T) { 67 | rf := MustNewRegexp(tt.pattern) 68 | if got := rf.GroupValue(tt.args.s, tt.args.groupName); got != tt.want { 69 | t.Errorf("GroupValue() = %v, want %v", got, tt.want) 70 | } 71 | }) 72 | } 73 | } 74 | 75 | func TestRegexp_Match(t *testing.T) { 76 | tests := []struct { 77 | name string 78 | regex *Regexp 79 | args string 80 | want bool 81 | }{ 82 | { 83 | name: "nil regex matches everything", 84 | regex: nil, 85 | args: "foo", 86 | want: true, 87 | }, 88 | { 89 | name: "empty regex matches everything", 90 | regex: &Regexp{}, 91 | args: "foo", 92 | want: true, 93 | }, 94 | { 95 | name: "regex matches", 96 | regex: MustNewRegexp(".*"), 97 | args: "foo", 98 | want: true, 99 | }, 100 | 101 | { 102 | name: "regex matches", 103 | regex: MustNewRegexp("a.*"), 104 | args: "foo", 105 | want: false, 106 | }, 107 | } 108 | for _, tt := range tests { 109 | t.Run(tt.name, func(t *testing.T) { 110 | rf := tt.regex 111 | if got := rf.Match(tt.args); got != tt.want { 112 | t.Errorf("Match() = %v, want %v", got, tt.want) 113 | } 114 | }) 115 | } 116 | } 117 | 118 | func TestRegexp_MarshalYAML(t *testing.T) { 119 | tests := []struct { 120 | name string 121 | regex *Regexp 122 | want interface{} 123 | wantErr bool 124 | }{ 125 | { 126 | name: "empty", 127 | regex: nil, 128 | want: "", 129 | wantErr: false, 130 | }, 131 | { 132 | name: "with pattern", 133 | regex: MustNewRegexp("a.*"), 134 | want: "a.*", 135 | wantErr: false, 136 | }, 137 | } 138 | for _, tt := range tests { 139 | t.Run(tt.name, func(t *testing.T) { 140 | got, err := tt.regex.MarshalYAML() 141 | if (err != nil) != tt.wantErr { 142 | t.Errorf("MarshalYAML() error = %v, wantErr %v", err, tt.wantErr) 143 | return 144 | } 145 | if !reflect.DeepEqual(got, tt.want) { 146 | t.Errorf("MarshalYAML() got = %v, want %v", got, tt.want) 147 | } 148 | }) 149 | } 150 | } 151 | -------------------------------------------------------------------------------- /pkg/config/runtime.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import "go.uber.org/zap" 4 | 5 | // runtimeContext contains process global settings like the logger, 6 | type runtimeContext struct { 7 | logger *zap.Logger 8 | } 9 | 10 | func (r *runtimeContext) Logger() *zap.Logger { 11 | return r.logger 12 | } 13 | 14 | var ProcessContext runtimeContext 15 | 16 | func SetProcessContext(logger *zap.Logger) { 17 | ProcessContext = runtimeContext{ 18 | logger: logger, 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /pkg/metrics/collector.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "github.com/hikhvar/mqtt2prometheus/pkg/config" 8 | gocache "github.com/patrickmn/go-cache" 9 | "github.com/prometheus/client_golang/prometheus" 10 | "go.uber.org/zap" 11 | ) 12 | 13 | type Collector interface { 14 | prometheus.Collector 15 | Observe(deviceID string, collection MetricCollection) 16 | } 17 | 18 | type MemoryCachedCollector struct { 19 | cache *gocache.Cache 20 | descriptions []*prometheus.Desc 21 | logger *zap.Logger 22 | } 23 | 24 | type Metric struct { 25 | Description *prometheus.Desc 26 | Value float64 27 | ValueType prometheus.ValueType 28 | IngestTime time.Time 29 | Topic string 30 | Labels map[string]string 31 | LabelsKeys []string 32 | } 33 | 34 | type CacheItem struct { 35 | DeviceID string 36 | Metric Metric 37 | } 38 | 39 | type MetricCollection []Metric 40 | 41 | func NewCollector(defaultTimeout time.Duration, possibleMetrics []config.MetricConfig, logger *zap.Logger) Collector { 42 | var descs []*prometheus.Desc 43 | for _, m := range possibleMetrics { 44 | descs = append(descs, m.PrometheusDescription()) 45 | } 46 | return &MemoryCachedCollector{ 47 | cache: gocache.New(defaultTimeout, defaultTimeout*10), 48 | descriptions: descs, 49 | logger: logger, 50 | } 51 | } 52 | 53 | func (c *MemoryCachedCollector) Observe(deviceID string, collection MetricCollection) { 54 | for _, m := range collection { 55 | item := CacheItem{ 56 | DeviceID: deviceID, 57 | Metric: m, 58 | } 59 | c.cache.Set(fmt.Sprintf("%s-%s", deviceID, m.Description.String()), item, gocache.DefaultExpiration) 60 | } 61 | } 62 | 63 | func (c *MemoryCachedCollector) Describe(ch chan<- *prometheus.Desc) { 64 | for i := range c.descriptions { 65 | ch <- c.descriptions[i] 66 | } 67 | } 68 | 69 | func (c *MemoryCachedCollector) Collect(mc chan<- prometheus.Metric) { 70 | for _, metricsRaw := range c.cache.Items() { 71 | item := metricsRaw.Object.(CacheItem) 72 | device, metric := item.DeviceID, item.Metric 73 | if metric.Description == nil { 74 | c.logger.Warn("empty description", zap.String("topic", metric.Topic), zap.Float64("value", metric.Value)) 75 | } 76 | 77 | // set dynamic labels with the right order starting with "sensor" and "topic" 78 | labels := []string{device, metric.Topic} 79 | for _, k := range metric.LabelsKeys { 80 | labels = append(labels, metric.Labels[k]) 81 | } 82 | 83 | m := prometheus.MustNewConstMetric( 84 | metric.Description, 85 | metric.ValueType, 86 | metric.Value, 87 | labels..., 88 | ) 89 | 90 | if metric.IngestTime.IsZero() { 91 | mc <- m 92 | } else { 93 | mc <- prometheus.NewMetricWithTimestamp(metric.IngestTime, m) 94 | } 95 | 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /pkg/metrics/extractor.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "fmt" 5 | "regexp" 6 | 7 | "github.com/hikhvar/mqtt2prometheus/pkg/config" 8 | gojsonq "github.com/thedevsaddam/gojsonq/v2" 9 | ) 10 | 11 | type Extractor func(topic string, payload []byte, deviceID string) (MetricCollection, error) 12 | 13 | // metricID returns a deterministic identifier per metic config which is safe to use in a file path. 14 | func metricID(topic, metric, deviceID, promName string) string { 15 | re := regexp.MustCompile(`[^a-zA-Z0-9]`) 16 | deviceID = re.ReplaceAllString(deviceID, "_") 17 | topic = re.ReplaceAllString(topic, "_") 18 | metric = re.ReplaceAllString(metric, "_") 19 | promName = re.ReplaceAllString(promName, "_") 20 | return fmt.Sprintf("%s-%s-%s-%s", deviceID, topic, metric, promName) 21 | } 22 | 23 | func NewJSONObjectExtractor(p Parser) Extractor { 24 | return func(topic string, payload []byte, deviceID string) (MetricCollection, error) { 25 | var mc MetricCollection 26 | parsed := gojsonq.New(gojsonq.SetSeparator(p.separator)).FromString(string(payload)) 27 | 28 | for path := range p.config() { 29 | rawValue := parsed.Find(path) 30 | parsed.Reset() 31 | if rawValue == nil { 32 | continue 33 | } 34 | 35 | // Find all valid metric configs 36 | for _, config := range p.findMetricConfigs(path, deviceID) { 37 | id := metricID(topic, path, deviceID, config.PrometheusName) 38 | m, err := p.parseMetric(config, id, rawValue) 39 | if err != nil { 40 | return nil, fmt.Errorf("failed to parse valid value from '%v' for metric %q: %w", rawValue, config.PrometheusName, err) 41 | } 42 | m.Topic = topic 43 | mc = append(mc, m) 44 | } 45 | } 46 | return mc, nil 47 | } 48 | } 49 | 50 | func NewMetricPerTopicExtractor(p Parser, metricNameRegex *config.Regexp) Extractor { 51 | return func(topic string, payload []byte, deviceID string) (MetricCollection, error) { 52 | var mc MetricCollection 53 | metricName := metricNameRegex.GroupValue(topic, config.MetricNameRegexGroup) 54 | if metricName == "" { 55 | return nil, fmt.Errorf("failed to find valid metric in topic path") 56 | } 57 | 58 | // Find all valid metric configs 59 | for _, config := range p.findMetricConfigs(metricName, deviceID) { 60 | var rawValue interface{} 61 | if config.PayloadField != "" { 62 | parsed := gojsonq.New(gojsonq.SetSeparator(p.separator)).FromString(string(payload)) 63 | rawValue = parsed.Find(config.PayloadField) 64 | parsed.Reset() 65 | if rawValue == nil { 66 | return nil, fmt.Errorf("failed to extract field %q from payload %q for metric %q", config.PayloadField, payload, metricName) 67 | } 68 | } else { 69 | rawValue = string(payload) 70 | } 71 | 72 | id := metricID(topic, metricName, deviceID, config.PrometheusName) 73 | m, err := p.parseMetric(config, id, rawValue) 74 | if err != nil { 75 | return nil, fmt.Errorf("failed to parse valid value from '%v' for metric %q: %w", rawValue, config.PrometheusName, err) 76 | } 77 | m.Topic = topic 78 | mc = append(mc, m) 79 | } 80 | return mc, nil 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /pkg/metrics/extractor_test.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | 7 | "github.com/hikhvar/mqtt2prometheus/pkg/config" 8 | "github.com/prometheus/client_golang/prometheus" 9 | ) 10 | 11 | func TestNewJSONObjectExtractor_parseMetric(t *testing.T) { 12 | now = testNow 13 | type fields struct { 14 | metricConfigs map[string][]*config.MetricConfig 15 | } 16 | type args struct { 17 | metricPath string 18 | deviceID string 19 | value string 20 | } 21 | tests := []struct { 22 | name string 23 | separator string 24 | fields fields 25 | args args 26 | want Metric 27 | wantErr bool 28 | noValue bool 29 | }{ 30 | { 31 | name: "string value", 32 | separator: "->", 33 | fields: fields{ 34 | map[string][]*config.MetricConfig{ 35 | "SDS0X1->PM2->5": { 36 | { 37 | PrometheusName: "temperature", 38 | MQTTName: "SDS0X1.PM2.5", 39 | ValueType: "gauge", 40 | }, 41 | }, 42 | }, 43 | }, 44 | args: args{ 45 | metricPath: "topic", 46 | deviceID: "dht22", 47 | value: "{\"SDS0X1\":{\"PM2\":{\"5\":4.9}}}", 48 | }, 49 | want: Metric{ 50 | Description: prometheus.NewDesc("temperature", "", []string{"sensor", "topic"}, nil), 51 | ValueType: prometheus.GaugeValue, 52 | Value: 4.9, 53 | IngestTime: testNow(), 54 | Topic: "topic", 55 | }, 56 | }, { 57 | name: "string value with dots in path", 58 | separator: "->", 59 | fields: fields{ 60 | map[string][]*config.MetricConfig{ 61 | "SDS0X1->PM2.5": { 62 | { 63 | PrometheusName: "temperature", 64 | MQTTName: "SDS0X1->PM2.5", 65 | ValueType: "gauge", 66 | }, 67 | }, 68 | }, 69 | }, 70 | args: args{ 71 | metricPath: "topic", 72 | deviceID: "dht22", 73 | value: "{\"SDS0X1\":{\"PM2.5\":4.9,\"PM10\":8.5}}", 74 | }, 75 | want: Metric{ 76 | Description: prometheus.NewDesc("temperature", "", []string{"sensor", "topic"}, nil), 77 | ValueType: prometheus.GaugeValue, 78 | Value: 4.9, 79 | IngestTime: testNow(), 80 | Topic: "topic", 81 | }, 82 | }, { 83 | name: "metric matching SensorNameFilter", 84 | separator: ".", 85 | fields: fields{ 86 | map[string][]*config.MetricConfig{ 87 | "temperature": { 88 | { 89 | PrometheusName: "temperature", 90 | MQTTName: "temperature", 91 | ValueType: "gauge", 92 | SensorNameFilter: *config.MustNewRegexp(".*22$"), 93 | }, 94 | }, 95 | }, 96 | }, 97 | args: args{ 98 | metricPath: "topic", 99 | deviceID: "dht22", 100 | value: "{\"temperature\": 8.5}", 101 | }, 102 | want: Metric{ 103 | Description: prometheus.NewDesc("temperature", "", []string{"sensor", "topic"}, nil), 104 | ValueType: prometheus.GaugeValue, 105 | Value: 8.5, 106 | IngestTime: testNow(), 107 | Topic: "topic", 108 | }, 109 | }, { 110 | name: "metric not matching SensorNameFilter", 111 | separator: ".", 112 | fields: fields{ 113 | map[string][]*config.MetricConfig{ 114 | "temperature": { 115 | { 116 | PrometheusName: "temperature", 117 | MQTTName: "temperature", 118 | ValueType: "gauge", 119 | SensorNameFilter: *config.MustNewRegexp(".*fail$"), 120 | }, 121 | }, 122 | }, 123 | }, 124 | args: args{ 125 | metricPath: "topic", 126 | deviceID: "dht22", 127 | value: "{\"temperature\": 8.5}", 128 | }, 129 | want: Metric{}, 130 | noValue: true, 131 | }, 132 | } 133 | for _, tt := range tests { 134 | t.Run(tt.name, func(t *testing.T) { 135 | p := Parser{ 136 | separator: tt.separator, 137 | metricConfigs: tt.fields.metricConfigs, 138 | } 139 | extractor := NewJSONObjectExtractor(p) 140 | 141 | got, err := extractor(tt.args.metricPath, []byte(tt.args.value), tt.args.deviceID) 142 | if (err != nil) != tt.wantErr { 143 | t.Errorf("parseMetric() error = %v, wantErr %v", err, tt.wantErr) 144 | return 145 | } 146 | 147 | if len(got) == 0 { 148 | if !tt.noValue { 149 | t.Errorf("parseMetric() got = %v, want %v", nil, tt.want) 150 | } 151 | } else if !reflect.DeepEqual(got[0], tt.want) { 152 | t.Errorf("parseMetric() got = %v, want %v", got[0], tt.want) 153 | } else if len(got) > 1 { 154 | t.Errorf("unexpected result got = %v, want %v", got, tt.want) 155 | } 156 | }) 157 | } 158 | } 159 | -------------------------------------------------------------------------------- /pkg/metrics/ingest.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "fmt" 5 | "go.uber.org/zap" 6 | 7 | "github.com/eclipse/paho.mqtt.golang" 8 | "github.com/hikhvar/mqtt2prometheus/pkg/config" 9 | ) 10 | 11 | type Ingest struct { 12 | instrumentation 13 | extractor Extractor 14 | deviceIDRegex *config.Regexp 15 | collector Collector 16 | logger *zap.Logger 17 | } 18 | 19 | func NewIngest(collector Collector, extractor Extractor, deviceIDRegex *config.Regexp) *Ingest { 20 | 21 | return &Ingest{ 22 | instrumentation: defaultInstrumentation, 23 | extractor: extractor, 24 | deviceIDRegex: deviceIDRegex, 25 | collector: collector, 26 | logger: config.ProcessContext.Logger(), 27 | } 28 | } 29 | 30 | func (i *Ingest) store(topic string, payload []byte) error { 31 | deviceID := i.deviceID(topic) 32 | mc, err := i.extractor(topic, payload, deviceID) 33 | if err != nil { 34 | return fmt.Errorf("failed to extract metric values from topic: %w", err) 35 | } 36 | i.collector.Observe(deviceID, mc) 37 | return nil 38 | } 39 | 40 | func (i *Ingest) SetupSubscriptionHandler(errChan chan<- error) mqtt.MessageHandler { 41 | return func(c mqtt.Client, m mqtt.Message) { 42 | i.logger.Debug("Got message", zap.String("topic", m.Topic()), zap.String("payload", string(m.Payload()))) 43 | err := i.store(m.Topic(), m.Payload()) 44 | if err != nil { 45 | errChan <- fmt.Errorf("could not store metrics '%s' on topic %s: %s", string(m.Payload()), m.Topic(), err.Error()) 46 | i.CountStoreError(m.Topic()) 47 | return 48 | } 49 | i.CountSuccess(m.Topic()) 50 | } 51 | } 52 | 53 | // deviceID uses the configured DeviceIDRegex to extract the device ID from the given mqtt topic path. 54 | func (i *Ingest) deviceID(topic string) string { 55 | return i.deviceIDRegex.GroupValue(topic, config.DeviceIDRegexGroup) 56 | } 57 | -------------------------------------------------------------------------------- /pkg/metrics/instrumentation.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | mqtt "github.com/eclipse/paho.mqtt.golang" 5 | "github.com/prometheus/client_golang/prometheus" 6 | ) 7 | 8 | const ( 9 | storeError = "storeError" 10 | success = "success" 11 | ) 12 | 13 | var defaultInstrumentation = instrumentation{ 14 | messageMetric: prometheus.NewCounterVec( 15 | prometheus.CounterOpts{ 16 | Name: "mqtt2prometheus_received_messages_total", 17 | Help: "Total number of messages received per topic and status", 18 | }, []string{"status", "topic"}, 19 | ), 20 | connectedMetric: prometheus.NewGauge( 21 | prometheus.GaugeOpts{ 22 | Name: "mqtt2prometheus_connected", 23 | Help: "Whether the mqtt2prometheus exporter is connected to the broker", 24 | }, 25 | ), 26 | } 27 | 28 | type instrumentation struct { 29 | messageMetric *prometheus.CounterVec 30 | connectedMetric prometheus.Gauge 31 | } 32 | 33 | func (i *instrumentation) Collector() prometheus.Collector { 34 | return i 35 | } 36 | 37 | func (i *instrumentation) Describe(desc chan<- *prometheus.Desc) { 38 | prometheus.DescribeByCollect(i, desc) 39 | } 40 | 41 | func (i *instrumentation) Collect(metrics chan<- prometheus.Metric) { 42 | i.connectedMetric.Collect(metrics) 43 | i.messageMetric.Collect(metrics) 44 | } 45 | 46 | func (i *instrumentation) CountSuccess(topic string) { 47 | i.messageMetric.WithLabelValues(success, topic).Inc() 48 | } 49 | 50 | func (i *instrumentation) CountStoreError(topic string) { 51 | i.messageMetric.WithLabelValues(storeError, topic).Inc() 52 | } 53 | 54 | func (i *instrumentation) ConnectionLostHandler(client mqtt.Client, err error) { 55 | i.connectedMetric.Set(0) 56 | } 57 | 58 | func (i *instrumentation) OnConnectHandler(client mqtt.Client) { 59 | i.connectedMetric.Set(1) 60 | } 61 | -------------------------------------------------------------------------------- /pkg/metrics/parser.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "math" 7 | "os" 8 | "strconv" 9 | "strings" 10 | "time" 11 | 12 | "github.com/expr-lang/expr" 13 | "github.com/expr-lang/expr/vm" 14 | "github.com/hikhvar/mqtt2prometheus/pkg/config" 15 | "gopkg.in/yaml.v2" 16 | ) 17 | 18 | // dynamicState holds the runtime information for dynamic metric configs. 19 | type dynamicState struct { 20 | // Basline value to add to each parsed metric value to maintain monotonicy 21 | Offset float64 `yaml:"value_offset"` 22 | // Last value that was parsed before the offset was added 23 | LastRawValue float64 `yaml:"last_raw_value"` 24 | // Last value that was used for evaluating the given expression 25 | LastExprValue float64 `yaml:"last_expr_value"` 26 | // Last result returned from evaluating the given expression 27 | LastExprRawValue interface{} `yaml:"last_expr_raw_value"` 28 | // Last result returned from evaluating the given expression 29 | LastExprResult float64 `yaml:"last_expr_result"` 30 | // Last result (String) returned from evaluating the given expression 31 | LastExprResultString string `yaml:"last_expr_result_string"` 32 | // Last result returned from evaluating the given expression 33 | LastExprTimestamp time.Time `yaml:"last_expr_timestamp"` 34 | } 35 | 36 | // metricState holds runtime information per metric configuration. 37 | type metricState struct { 38 | dynamic dynamicState 39 | // The last time the state file was written 40 | lastWritten time.Time 41 | // Compiled evaluation expression 42 | program *vm.Program 43 | // Environment in which the expression is evaluated 44 | env map[string]interface{} 45 | } 46 | 47 | type Parser struct { 48 | separator string 49 | // Maps the mqtt metric name to a list of configs 50 | // The first that matches SensorNameFilter will be used 51 | metricConfigs map[string][]*config.MetricConfig 52 | // Directory holding state files 53 | stateDir string 54 | // Per-metric state 55 | states map[string]*metricState 56 | } 57 | 58 | // Identifiers within the expression evaluation environment. 59 | const ( 60 | env_raw_value = "raw_value" 61 | env_value = "value" 62 | env_last_value = "last_value" 63 | env_last_raw_value = "last_raw_value" 64 | env_last_result = "last_result" 65 | env_elapsed = "elapsed" 66 | env_now = "now" 67 | env_int = "int" 68 | env_float = "float" 69 | env_round = "round" 70 | env_ceil = "ceil" 71 | env_floor = "floor" 72 | env_abs = "abs" 73 | env_min = "min" 74 | env_max = "max" 75 | ) 76 | 77 | var now = time.Now 78 | 79 | func toInt64(i interface{}) int64 { 80 | switch v := i.(type) { 81 | case float32: 82 | return int64(v) 83 | case float64: 84 | return int64(v) 85 | case int: 86 | return int64(v) 87 | case int32: 88 | return int64(v) 89 | case int64: 90 | return v 91 | case time.Duration: 92 | return int64(v) 93 | case string: 94 | value, err := strconv.ParseInt(v, 10, 64) 95 | if err != nil { 96 | panic(err) 97 | } 98 | return value 99 | default: 100 | return v.(int64) // Hope for the best 101 | } 102 | } 103 | 104 | func toFloat64(i interface{}) float64 { 105 | switch v := i.(type) { 106 | case float32: 107 | return float64(v) 108 | case float64: 109 | return v 110 | case int: 111 | return float64(v) 112 | case int32: 113 | return float64(v) 114 | case int64: 115 | return float64(v) 116 | case time.Duration: 117 | return float64(v) 118 | case string: 119 | value, err := strconv.ParseFloat(v, 64) 120 | if err != nil { 121 | panic(err) 122 | } 123 | return value 124 | default: 125 | return v.(float64) // Hope for the best 126 | } 127 | } 128 | 129 | // defaultExprEnv returns the default environment for expression evaluation. 130 | func defaultExprEnv() map[string]interface{} { 131 | return map[string]interface{}{ 132 | // Variables 133 | env_raw_value: nil, 134 | env_value: 0.0, 135 | env_last_value: 0.0, 136 | env_last_result: 0.0, 137 | env_elapsed: time.Duration(0), 138 | // Functions 139 | env_now: now, 140 | env_int: toInt64, 141 | env_float: toFloat64, 142 | env_round: math.Round, 143 | env_ceil: math.Ceil, 144 | env_floor: math.Floor, 145 | env_abs: math.Abs, 146 | env_min: math.Min, 147 | env_max: math.Max, 148 | } 149 | } 150 | 151 | func NewParser(metrics []config.MetricConfig, separator, stateDir string) Parser { 152 | cfgs := make(map[string][]*config.MetricConfig) 153 | for i := range metrics { 154 | key := metrics[i].MQTTName 155 | cfgs[key] = append(cfgs[key], &metrics[i]) 156 | } 157 | return Parser{ 158 | separator: separator, 159 | metricConfigs: cfgs, 160 | stateDir: strings.TrimRight(stateDir, "/"), 161 | states: make(map[string]*metricState), 162 | } 163 | } 164 | 165 | // Config returns the underlying metrics config 166 | func (p *Parser) config() map[string][]*config.MetricConfig { 167 | return p.metricConfigs 168 | } 169 | 170 | // validMetric returns all configs matching the metric and deviceID. 171 | func (p *Parser) findMetricConfigs(metric string, deviceID string) []*config.MetricConfig { 172 | configs := []*config.MetricConfig{} 173 | for _, c := range p.metricConfigs[metric] { 174 | if c.SensorNameFilter.Match(deviceID) { 175 | configs = append(configs, c) 176 | } 177 | } 178 | return configs 179 | } 180 | 181 | // parseMetric parses the given value according to the given deviceID and metricPath. The config allows to 182 | // parse a metric value according to the device ID. 183 | func (p *Parser) parseMetric(cfg *config.MetricConfig, metricID string, value interface{}) (Metric, error) { 184 | var metricValue float64 185 | var err error 186 | 187 | if cfg.RawExpression != "" { 188 | if metricValue, err = p.evalExpressionValue(metricID, cfg.RawExpression, value, metricValue); err != nil { 189 | if cfg.ErrorValue != nil { 190 | metricValue = *cfg.ErrorValue 191 | } else { 192 | return Metric{}, err 193 | } 194 | } 195 | } else { 196 | 197 | if boolValue, ok := value.(bool); ok { 198 | if boolValue { 199 | metricValue = 1 200 | } else { 201 | metricValue = 0 202 | } 203 | } else if strValue, ok := value.(string); ok { 204 | 205 | // If string value mapping is defined, use that 206 | if cfg.StringValueMapping != nil { 207 | 208 | floatValue, ok := cfg.StringValueMapping.Map[strValue] 209 | if ok { 210 | metricValue = floatValue 211 | 212 | // deprecated, replaced by ErrorValue from the upper level 213 | } else if cfg.StringValueMapping.ErrorValue != nil { 214 | metricValue = *cfg.StringValueMapping.ErrorValue 215 | } else if cfg.ErrorValue != nil { 216 | metricValue = *cfg.ErrorValue 217 | } else { 218 | return Metric{}, fmt.Errorf("got unexpected string data '%s'", strValue) 219 | } 220 | 221 | } else { 222 | 223 | // otherwise try to parse float 224 | floatValue, err := strconv.ParseFloat(strValue, 64) 225 | if err != nil { 226 | if cfg.ErrorValue != nil { 227 | metricValue = *cfg.ErrorValue 228 | } else { 229 | return Metric{}, fmt.Errorf("got data with unexpectd type: %T ('%v') and failed to parse to float", value, value) 230 | } 231 | } else { 232 | metricValue = floatValue 233 | } 234 | 235 | } 236 | 237 | } else if floatValue, ok := value.(float64); ok { 238 | metricValue = floatValue 239 | } else if cfg.ErrorValue != nil { 240 | metricValue = *cfg.ErrorValue 241 | } else { 242 | return Metric{}, fmt.Errorf("got data with unexpectd type: %T ('%v')", value, value) 243 | } 244 | 245 | if cfg.Expression != "" { 246 | if metricValue, err = p.evalExpressionValue(metricID, cfg.Expression, value, metricValue); err != nil { 247 | if cfg.ErrorValue != nil { 248 | metricValue = *cfg.ErrorValue 249 | } else { 250 | return Metric{}, err 251 | } 252 | } 253 | } 254 | } 255 | 256 | if cfg.ForceMonotonicy { 257 | if metricValue, err = p.enforceMonotonicy(metricID, metricValue); err != nil { 258 | if cfg.ErrorValue != nil { 259 | metricValue = *cfg.ErrorValue 260 | } else { 261 | return Metric{}, err 262 | } 263 | } 264 | } 265 | 266 | if cfg.MQTTValueScale != 0 { 267 | metricValue = metricValue * cfg.MQTTValueScale 268 | } 269 | 270 | var ingestTime time.Time 271 | if !cfg.OmitTimestamp { 272 | ingestTime = now() 273 | } 274 | 275 | // generate dynamic labels 276 | var labels map[string]string 277 | if len(cfg.DynamicLabels) > 0 { 278 | labels = make(map[string]string, len(cfg.DynamicLabels)) 279 | for k, v := range cfg.DynamicLabels { 280 | value, err := p.evalExpressionLabel(metricID, k, v, value, metricValue) 281 | if err != nil { 282 | return Metric{}, err 283 | } 284 | labels[k] = value 285 | } 286 | } 287 | 288 | return Metric{ 289 | Description: cfg.PrometheusDescription(), 290 | Value: metricValue, 291 | ValueType: cfg.PrometheusValueType(), 292 | IngestTime: ingestTime, 293 | Labels: labels, 294 | LabelsKeys: cfg.DynamicLabelsKeys(), 295 | }, nil 296 | } 297 | 298 | func (p *Parser) stateFileName(metricID string) string { 299 | return fmt.Sprintf("%s/%s.yaml", p.stateDir, metricID) 300 | } 301 | 302 | // readMetricState parses the metric state from the configured path. 303 | // If the file does not exist, an empty state is returned. 304 | func (p *Parser) readMetricState(metricID string) (*metricState, error) { 305 | state := &metricState{} 306 | f, err := os.Open(p.stateFileName(metricID)) 307 | if err != nil { 308 | // The file does not exist for new metrics. 309 | if os.IsNotExist(err) { 310 | return state, nil 311 | } 312 | return state, fmt.Errorf("failed to read file %q: %v", f.Name(), err) 313 | } 314 | defer f.Close() 315 | 316 | var data []byte 317 | if info, err := f.Stat(); err == nil { 318 | data = make([]byte, int(info.Size())) 319 | } 320 | if _, err := f.Read(data); err != nil && err != io.EOF { 321 | return state, err 322 | } 323 | 324 | err = yaml.UnmarshalStrict(data, &state.dynamic) 325 | state.lastWritten = now() 326 | return state, err 327 | } 328 | 329 | // writeMetricState writes back the metric's current state to the configured path. 330 | func (p *Parser) writeMetricState(metricID string, state *metricState) error { 331 | out, err := yaml.Marshal(state.dynamic) 332 | if err != nil { 333 | return err 334 | } 335 | f, err := os.OpenFile(p.stateFileName(metricID), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) 336 | if err != nil { 337 | return err 338 | } 339 | defer f.Close() 340 | if _, err = f.Write(out); err != nil { 341 | return fmt.Errorf("failed to write file %q: %v", f.Name(), err) 342 | } 343 | return nil 344 | } 345 | 346 | // getMetricState returns the state of the given metric. 347 | // The state is read from and written back to disk as needed. 348 | func (p *Parser) getMetricState(metricID string) (*metricState, error) { 349 | var err error 350 | state, found := p.states[metricID] 351 | if !found { 352 | if state, err = p.readMetricState(metricID); err != nil { 353 | return nil, err 354 | } 355 | p.states[metricID] = state 356 | } 357 | // Write the state back to disc every minute. 358 | if now().Sub(state.lastWritten) >= time.Minute { 359 | if err = p.writeMetricState(metricID, state); err == nil { 360 | state.lastWritten = now() 361 | } 362 | } 363 | return state, err 364 | } 365 | 366 | // enforceMonotonicy makes sure the given values never decrease from one call to the next. 367 | // If the current value is smaller than the last one, a consistent offset is added. 368 | func (p *Parser) enforceMonotonicy(metricID string, value float64) (float64, error) { 369 | ms, err := p.getMetricState(metricID) 370 | if err != nil { 371 | return value, err 372 | } 373 | // When the source metric is reset, the last adjusted value becomes the new offset. 374 | if value < ms.dynamic.LastRawValue { 375 | ms.dynamic.Offset += ms.dynamic.LastRawValue 376 | // Trigger flushing the new state to disk. 377 | ms.lastWritten = time.Time{} 378 | } 379 | 380 | ms.dynamic.LastRawValue = value 381 | return value + ms.dynamic.Offset, nil 382 | } 383 | 384 | // evalExpressionValue runs the given code in the metric's environment and returns the result. 385 | // In case of an error, the original value is returned. 386 | func (p *Parser) evalExpressionValue(metricID, code string, raw_value interface{}, value float64) (float64, error) { 387 | ms, err := p.getMetricState(metricID) 388 | if err != nil { 389 | return value, err 390 | } 391 | if ms.program == nil { 392 | ms.env = defaultExprEnv() 393 | ms.program, err = expr.Compile(code, expr.Env(ms.env), expr.AsFloat64()) 394 | if err != nil { 395 | return value, fmt.Errorf("failed to compile expression %q: %w", code, err) 396 | } 397 | // Trigger flushing the new state to disk. 398 | ms.lastWritten = time.Time{} 399 | } 400 | 401 | // Update the environment 402 | ms.env[env_raw_value] = raw_value 403 | ms.env[env_value] = value 404 | ms.env[env_last_value] = ms.dynamic.LastExprValue 405 | ms.env[env_last_raw_value] = ms.dynamic.LastExprRawValue 406 | ms.env[env_last_result] = ms.dynamic.LastExprResult 407 | if ms.dynamic.LastExprTimestamp.IsZero() { 408 | ms.env[env_elapsed] = time.Duration(0) 409 | } else { 410 | ms.env[env_elapsed] = now().Sub(ms.dynamic.LastExprTimestamp) 411 | } 412 | 413 | result, err := expr.Run(ms.program, ms.env) 414 | if err != nil { 415 | return value, fmt.Errorf("failed to evaluate expression %q: %w", code, err) 416 | } 417 | // Type was statically checked above. 418 | ret := result.(float64) 419 | 420 | // Update the dynamic state 421 | ms.dynamic.LastExprResult = ret 422 | ms.dynamic.LastExprRawValue = raw_value 423 | ms.dynamic.LastExprValue = value 424 | ms.dynamic.LastExprTimestamp = now() 425 | 426 | return ret, nil 427 | } 428 | 429 | // evalExpressionLabel runs the given code in the metric's environment and returns the result. 430 | // In case of an error, the original value is returned. 431 | func (p *Parser) evalExpressionLabel(metricID, label, code string, rawValue interface{}, value float64) (string, error) { 432 | ms, err := p.getMetricState(label + "@" + metricID) 433 | if err != nil { 434 | return "", err 435 | } 436 | if ms.program == nil { 437 | ms.env = defaultExprEnv() 438 | ms.program, err = expr.Compile(code, expr.Env(ms.env)) 439 | if err != nil { 440 | return "", fmt.Errorf("failed to compile dynamic label expression %q: %w", code, err) 441 | } 442 | // Trigger flushing the new state to disk. 443 | ms.lastWritten = time.Time{} 444 | } 445 | 446 | // Update the environment 447 | ms.env[env_raw_value] = rawValue 448 | ms.env[env_value] = value 449 | ms.env[env_last_value] = ms.dynamic.LastExprValue 450 | ms.env[env_last_raw_value] = ms.dynamic.LastExprRawValue 451 | ms.env[env_last_result] = ms.dynamic.LastExprResultString 452 | if ms.dynamic.LastExprTimestamp.IsZero() { 453 | ms.env[env_elapsed] = time.Duration(0) 454 | } else { 455 | ms.env[env_elapsed] = now().Sub(ms.dynamic.LastExprTimestamp) 456 | } 457 | 458 | result, err := expr.Run(ms.program, ms.env) 459 | if err != nil { 460 | return "", fmt.Errorf("failed to evaluate dynamic label expression %q: %w", code, err) 461 | } 462 | 463 | // convert to string 464 | ret := fmt.Sprint(result) 465 | 466 | // Update the dynamic state 467 | ms.dynamic.LastExprResultString = ret 468 | ms.dynamic.LastExprValue = value 469 | ms.dynamic.LastExprTimestamp = now() 470 | 471 | return ret, nil 472 | } 473 | -------------------------------------------------------------------------------- /pkg/metrics/parser_test.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "os" 5 | "reflect" 6 | "testing" 7 | "time" 8 | 9 | "github.com/hikhvar/mqtt2prometheus/pkg/config" 10 | "github.com/prometheus/client_golang/prometheus" 11 | ) 12 | 13 | var testNowElapsed time.Duration 14 | 15 | func testNow() time.Time { 16 | now, err := time.Parse( 17 | time.RFC3339, 18 | "2020-11-01T22:08:41+00:00") 19 | if err != nil { 20 | panic(err) 21 | } 22 | now = now.Add(testNowElapsed) 23 | return now 24 | } 25 | 26 | func TestParser_parseMetric(t *testing.T) { 27 | stateDir, err := os.MkdirTemp("", "parser_test") 28 | if err != nil { 29 | t.Fatal(err) 30 | } 31 | defer os.RemoveAll(stateDir) 32 | 33 | now = testNow 34 | type fields struct { 35 | metricConfigs map[string][]*config.MetricConfig 36 | } 37 | type args struct { 38 | metricPath string 39 | deviceID string 40 | value interface{} 41 | } 42 | 43 | var errorValue float64 = 42.44 44 | 45 | tests := []struct { 46 | name string 47 | fields fields 48 | args args 49 | want Metric 50 | wantErr bool 51 | elapseNow time.Duration 52 | }{ 53 | { 54 | name: "value without timestamp", 55 | fields: fields{ 56 | map[string][]*config.MetricConfig{ 57 | "temperature": { 58 | { 59 | PrometheusName: "temperature", 60 | ValueType: "gauge", 61 | OmitTimestamp: true, 62 | }, 63 | }, 64 | }, 65 | }, 66 | args: args{ 67 | metricPath: "temperature", 68 | deviceID: "dht22", 69 | value: 12.6, 70 | }, 71 | want: Metric{ 72 | Description: prometheus.NewDesc("temperature", "", []string{"sensor", "topic"}, nil), 73 | ValueType: prometheus.GaugeValue, 74 | Value: 12.6, 75 | IngestTime: time.Time{}, 76 | Topic: "", 77 | }, 78 | }, 79 | { 80 | name: "string value", 81 | fields: fields{ 82 | map[string][]*config.MetricConfig{ 83 | "temperature": { 84 | { 85 | PrometheusName: "temperature", 86 | ValueType: "gauge", 87 | }, 88 | }, 89 | }, 90 | }, 91 | args: args{ 92 | metricPath: "temperature", 93 | deviceID: "dht22", 94 | value: "12.6", 95 | }, 96 | want: Metric{ 97 | Description: prometheus.NewDesc("temperature", "", []string{"sensor", "topic"}, nil), 98 | ValueType: prometheus.GaugeValue, 99 | Value: 12.6, 100 | IngestTime: testNow(), 101 | Topic: "", 102 | }, 103 | }, 104 | { 105 | name: "string value with dynamic label", 106 | fields: fields{ 107 | map[string][]*config.MetricConfig{ 108 | "temperature": { 109 | { 110 | PrometheusName: "temperature", 111 | ValueType: "gauge", 112 | DynamicLabels: map[string]string{"dynamic-label": `replace(raw_value, ".", "")`}, 113 | }, 114 | }, 115 | }, 116 | }, 117 | args: args{ 118 | metricPath: "temperature", 119 | deviceID: "dht22", 120 | value: "12.6", 121 | }, 122 | want: Metric{ 123 | Description: prometheus.NewDesc("temperature", "", []string{"sensor", "topic", "dynamic-label"}, nil), 124 | ValueType: prometheus.GaugeValue, 125 | Value: 12.6, 126 | IngestTime: testNow(), 127 | Topic: "", 128 | Labels: map[string]string{"dynamic-label": "126"}, 129 | LabelsKeys: []string{"dynamic-label"}, 130 | }, 131 | }, 132 | { 133 | name: "scaled string value", 134 | fields: fields{ 135 | map[string][]*config.MetricConfig{ 136 | "temperature": { 137 | { 138 | PrometheusName: "temperature", 139 | ValueType: "gauge", 140 | MQTTValueScale: 0.01, 141 | }, 142 | }, 143 | }, 144 | }, 145 | args: args{ 146 | metricPath: "temperature", 147 | deviceID: "dht22", 148 | value: "12.6", 149 | }, 150 | want: Metric{ 151 | Description: prometheus.NewDesc("temperature", "", []string{"sensor", "topic"}, nil), 152 | ValueType: prometheus.GaugeValue, 153 | Value: 0.126, 154 | IngestTime: testNow(), 155 | Topic: "", 156 | }, 157 | }, 158 | { 159 | name: "string value failure", 160 | fields: fields{ 161 | map[string][]*config.MetricConfig{ 162 | "temperature": { 163 | { 164 | PrometheusName: "temperature", 165 | ValueType: "gauge", 166 | }, 167 | }, 168 | }, 169 | }, 170 | args: args{ 171 | metricPath: "temperature", 172 | deviceID: "dht22", 173 | value: "12.6.5", 174 | }, 175 | wantErr: true, 176 | }, 177 | { 178 | name: "string value failure with errorValue", 179 | fields: fields{ 180 | map[string][]*config.MetricConfig{ 181 | "temperature": { 182 | { 183 | PrometheusName: "temperature", 184 | ValueType: "gauge", 185 | ErrorValue: &errorValue, 186 | }, 187 | }, 188 | }, 189 | }, 190 | args: args{ 191 | metricPath: "temperature", 192 | deviceID: "dht22", 193 | value: "12.6.5", 194 | }, 195 | want: Metric{ 196 | Description: prometheus.NewDesc("temperature", "", []string{"sensor", "topic"}, nil), 197 | ValueType: prometheus.GaugeValue, 198 | Value: errorValue, 199 | IngestTime: testNow(), 200 | Topic: "", 201 | }, 202 | }, 203 | { 204 | name: "float value", 205 | fields: fields{ 206 | map[string][]*config.MetricConfig{ 207 | "temperature": { 208 | { 209 | PrometheusName: "temperature", 210 | ValueType: "gauge", 211 | }, 212 | }, 213 | }, 214 | }, 215 | args: args{ 216 | metricPath: "temperature", 217 | deviceID: "dht22", 218 | value: 12.6, 219 | }, 220 | want: Metric{ 221 | Description: prometheus.NewDesc("temperature", "", []string{"sensor", "topic"}, nil), 222 | ValueType: prometheus.GaugeValue, 223 | Value: 12.6, 224 | IngestTime: testNow(), 225 | Topic: "", 226 | }, 227 | }, 228 | { 229 | name: "scaled float value", 230 | fields: fields{ 231 | map[string][]*config.MetricConfig{ 232 | "humidity": { 233 | { 234 | PrometheusName: "humidity", 235 | ValueType: "gauge", 236 | MQTTValueScale: 0.01, 237 | }, 238 | }, 239 | }, 240 | }, 241 | args: args{ 242 | metricPath: "humidity", 243 | deviceID: "dht22", 244 | value: 12.6, 245 | }, 246 | want: Metric{ 247 | Description: prometheus.NewDesc("humidity", "", []string{"sensor", "topic"}, nil), 248 | ValueType: prometheus.GaugeValue, 249 | Value: 0.126, 250 | IngestTime: testNow(), 251 | Topic: "", 252 | }, 253 | }, 254 | { 255 | name: "negative scaled float value", 256 | fields: fields{ 257 | map[string][]*config.MetricConfig{ 258 | "humidity": { 259 | { 260 | PrometheusName: "humidity", 261 | ValueType: "gauge", 262 | MQTTValueScale: -2, 263 | }, 264 | }, 265 | }, 266 | }, 267 | args: args{ 268 | metricPath: "humidity", 269 | deviceID: "dht22", 270 | value: 12.6, 271 | }, 272 | want: Metric{ 273 | Description: prometheus.NewDesc("humidity", "", []string{"sensor", "topic"}, nil), 274 | ValueType: prometheus.GaugeValue, 275 | Value: -25.2, 276 | IngestTime: testNow(), 277 | Topic: "", 278 | }, 279 | }, 280 | { 281 | name: "bool value true", 282 | fields: fields{ 283 | map[string][]*config.MetricConfig{ 284 | "enabled": { 285 | { 286 | PrometheusName: "enabled", 287 | ValueType: "gauge", 288 | }, 289 | }, 290 | }, 291 | }, 292 | args: args{ 293 | metricPath: "enabled", 294 | deviceID: "dht22", 295 | value: true, 296 | }, 297 | want: Metric{ 298 | Description: prometheus.NewDesc("enabled", "", []string{"sensor", "topic"}, nil), 299 | ValueType: prometheus.GaugeValue, 300 | Value: 1, 301 | IngestTime: testNow(), 302 | Topic: "", 303 | }, 304 | }, 305 | { 306 | name: "scaled bool value", 307 | fields: fields{ 308 | map[string][]*config.MetricConfig{ 309 | "enabled": { 310 | { 311 | PrometheusName: "enabled", 312 | ValueType: "gauge", 313 | MQTTValueScale: 0.5, 314 | }, 315 | }, 316 | }, 317 | }, 318 | args: args{ 319 | metricPath: "enabled", 320 | deviceID: "dht22", 321 | value: true, 322 | }, 323 | want: Metric{ 324 | Description: prometheus.NewDesc("enabled", "", []string{"sensor", "topic"}, nil), 325 | ValueType: prometheus.GaugeValue, 326 | Value: 0.5, 327 | IngestTime: testNow(), 328 | Topic: "", 329 | }, 330 | }, 331 | { 332 | name: "bool value false", 333 | fields: fields{ 334 | map[string][]*config.MetricConfig{ 335 | "enabled": { 336 | { 337 | PrometheusName: "enabled", 338 | ValueType: "gauge", 339 | }, 340 | }, 341 | }, 342 | }, 343 | args: args{ 344 | metricPath: "enabled", 345 | deviceID: "dht22", 346 | value: false, 347 | }, 348 | want: Metric{ 349 | Description: prometheus.NewDesc("enabled", "", []string{"sensor", "topic"}, nil), 350 | ValueType: prometheus.GaugeValue, 351 | Value: 0, 352 | IngestTime: testNow(), 353 | Topic: "", 354 | }, 355 | }, 356 | { 357 | name: "string mapping value success", 358 | fields: fields{ 359 | map[string][]*config.MetricConfig{ 360 | "enabled": { 361 | { 362 | PrometheusName: "enabled", 363 | ValueType: "gauge", 364 | StringValueMapping: &config.StringValueMappingConfig{ 365 | Map: map[string]float64{ 366 | "foo": 112, 367 | "bar": 2, 368 | }, 369 | }, 370 | }, 371 | }, 372 | }, 373 | }, 374 | args: args{ 375 | metricPath: "enabled", 376 | deviceID: "dht22", 377 | value: "foo", 378 | }, 379 | want: Metric{ 380 | Description: prometheus.NewDesc("enabled", "", []string{"sensor", "topic"}, nil), 381 | ValueType: prometheus.GaugeValue, 382 | Value: 112, 383 | IngestTime: testNow(), 384 | Topic: "", 385 | }, 386 | }, 387 | { 388 | name: "string mapping value failure default to error value", 389 | fields: fields{ 390 | map[string][]*config.MetricConfig{ 391 | "enabled": { 392 | { 393 | PrometheusName: "enabled", 394 | ValueType: "gauge", 395 | ErrorValue: floatP(12333), 396 | StringValueMapping: &config.StringValueMappingConfig{ 397 | Map: map[string]float64{ 398 | "foo": 112, 399 | "bar": 2, 400 | }, 401 | }, 402 | }, 403 | }, 404 | }, 405 | }, 406 | args: args{ 407 | metricPath: "enabled", 408 | deviceID: "dht22", 409 | value: "asd", 410 | }, 411 | want: Metric{ 412 | Description: prometheus.NewDesc("enabled", "", []string{"sensor", "topic"}, nil), 413 | ValueType: prometheus.GaugeValue, 414 | Value: 12333, 415 | IngestTime: testNow(), 416 | Topic: "", 417 | }, 418 | }, 419 | { 420 | name: "string mapping value failure no error value", 421 | fields: fields{ 422 | map[string][]*config.MetricConfig{ 423 | "enabled": { 424 | { 425 | PrometheusName: "enabled", 426 | ValueType: "gauge", 427 | StringValueMapping: &config.StringValueMappingConfig{ 428 | Map: map[string]float64{ 429 | "foo": 112, 430 | "bar": 2, 431 | }, 432 | }, 433 | }, 434 | }, 435 | }, 436 | }, 437 | args: args{ 438 | metricPath: "enabled", 439 | deviceID: "dht22", 440 | value: "asd", 441 | }, 442 | wantErr: true, 443 | }, 444 | { 445 | name: "metric not configured", 446 | fields: fields{ 447 | map[string][]*config.MetricConfig{ 448 | "enabled": { 449 | { 450 | PrometheusName: "enabled", 451 | ValueType: "gauge", 452 | ErrorValue: floatP(12333), 453 | StringValueMapping: &config.StringValueMappingConfig{ 454 | Map: map[string]float64{ 455 | "foo": 112, 456 | "bar": 2, 457 | }, 458 | }, 459 | }, 460 | }, 461 | }, 462 | }, 463 | args: args{ 464 | metricPath: "enabled1", 465 | deviceID: "dht22", 466 | value: "asd", 467 | }, 468 | wantErr: true, 469 | }, 470 | { 471 | name: "unexpected type", 472 | fields: fields{ 473 | map[string][]*config.MetricConfig{ 474 | "enabled": { 475 | { 476 | PrometheusName: "enabled", 477 | ValueType: "gauge", 478 | StringValueMapping: &config.StringValueMappingConfig{ 479 | Map: map[string]float64{ 480 | "foo": 112, 481 | "bar": 2, 482 | }, 483 | }, 484 | }, 485 | }, 486 | }, 487 | }, 488 | args: args{ 489 | metricPath: "enabled", 490 | deviceID: "dht22", 491 | value: []int{3}, 492 | }, 493 | wantErr: true, 494 | }, 495 | { 496 | name: "monotonic gauge, step 1: initial value", 497 | fields: fields{ 498 | map[string][]*config.MetricConfig{ 499 | "aenergy.total": { 500 | { 501 | PrometheusName: "total_energy", 502 | ValueType: "gauge", 503 | OmitTimestamp: true, 504 | ForceMonotonicy: true, 505 | }, 506 | }, 507 | }, 508 | }, 509 | args: args{ 510 | metricPath: "aenergy.total", 511 | deviceID: "shellyplus1pm-foo", 512 | value: 1.0, 513 | }, 514 | want: Metric{ 515 | Description: prometheus.NewDesc("total_energy", "", []string{"sensor", "topic"}, nil), 516 | ValueType: prometheus.GaugeValue, 517 | Value: 1.0, 518 | }, 519 | }, 520 | { 521 | name: "monotonic gauge, step 2: monotonic increase does not add offset", 522 | fields: fields{ 523 | map[string][]*config.MetricConfig{ 524 | "aenergy.total": { 525 | { 526 | PrometheusName: "total_energy", 527 | ValueType: "gauge", 528 | OmitTimestamp: true, 529 | ForceMonotonicy: true, 530 | }, 531 | }, 532 | }, 533 | }, 534 | args: args{ 535 | metricPath: "aenergy.total", 536 | deviceID: "shellyplus1pm-foo", 537 | value: 2.0, 538 | }, 539 | want: Metric{ 540 | Description: prometheus.NewDesc("total_energy", "", []string{"sensor", "topic"}, nil), 541 | ValueType: prometheus.GaugeValue, 542 | Value: 2.0, 543 | }, 544 | }, 545 | { 546 | name: "monotonic gauge, step 3: raw metric is reset, last value becomes the new offset", 547 | fields: fields{ 548 | map[string][]*config.MetricConfig{ 549 | "aenergy.total": { 550 | { 551 | PrometheusName: "total_energy", 552 | ValueType: "gauge", 553 | OmitTimestamp: true, 554 | ForceMonotonicy: true, 555 | }, 556 | }, 557 | }, 558 | }, 559 | args: args{ 560 | metricPath: "aenergy.total", 561 | deviceID: "shellyplus1pm-foo", 562 | value: 0.0, 563 | }, 564 | want: Metric{ 565 | Description: prometheus.NewDesc("total_energy", "", []string{"sensor", "topic"}, nil), 566 | ValueType: prometheus.GaugeValue, 567 | Value: 2.0, 568 | }, 569 | }, 570 | { 571 | name: "monotonic gauge, step 4: monotonic increase with offset", 572 | fields: fields{ 573 | map[string][]*config.MetricConfig{ 574 | "aenergy.total": { 575 | { 576 | PrometheusName: "total_energy", 577 | ValueType: "gauge", 578 | OmitTimestamp: true, 579 | ForceMonotonicy: true, 580 | }, 581 | }, 582 | }, 583 | }, 584 | args: args{ 585 | metricPath: "aenergy.total", 586 | deviceID: "shellyplus1pm-foo", 587 | value: 1.0, 588 | }, 589 | want: Metric{ 590 | Description: prometheus.NewDesc("total_energy", "", []string{"sensor", "topic"}, nil), 591 | ValueType: prometheus.GaugeValue, 592 | Value: 3.0, 593 | }, 594 | }, 595 | { 596 | name: "integrate positive values using expressions, step 1", 597 | fields: fields{ 598 | map[string][]*config.MetricConfig{ 599 | "apower": { 600 | { 601 | PrometheusName: "total_energy", 602 | ValueType: "gauge", 603 | OmitTimestamp: true, 604 | Expression: "value > 0 ? last_result + value * elapsed.Hours() : last_result", 605 | }, 606 | }, 607 | }, 608 | }, 609 | args: args{ 610 | metricPath: "apower", 611 | deviceID: "shellyplus1pm-foo", 612 | value: 60.0, 613 | }, 614 | want: Metric{ 615 | Description: prometheus.NewDesc("total_energy", "", []string{"sensor", "topic"}, nil), 616 | ValueType: prometheus.GaugeValue, 617 | Value: 0.0, // No elapsed time yet, hence no integration 618 | }, 619 | }, 620 | { 621 | name: "integrate positive values using expressions, step 2", 622 | fields: fields{ 623 | map[string][]*config.MetricConfig{ 624 | "apower": { 625 | { 626 | PrometheusName: "total_energy", 627 | ValueType: "gauge", 628 | OmitTimestamp: true, 629 | Expression: "value > 0 ? last_result + value * elapsed.Hours() : last_result", 630 | }, 631 | }, 632 | }, 633 | }, 634 | elapseNow: time.Minute, 635 | args: args{ 636 | metricPath: "apower", 637 | deviceID: "shellyplus1pm-foo", 638 | value: 60.0, 639 | }, 640 | want: Metric{ 641 | Description: prometheus.NewDesc("total_energy", "", []string{"sensor", "topic"}, nil), 642 | ValueType: prometheus.GaugeValue, 643 | Value: 1.0, // 60 watts for 1 minute = 1 Wh 644 | }, 645 | }, 646 | { 647 | name: "integrate positive values using expressions, step 3", 648 | fields: fields{ 649 | map[string][]*config.MetricConfig{ 650 | "apower": { 651 | { 652 | PrometheusName: "total_energy", 653 | ValueType: "gauge", 654 | OmitTimestamp: true, 655 | Expression: "value > 0 ? last_result + value * elapsed.Hours() : last_result", 656 | }, 657 | }, 658 | }, 659 | }, 660 | elapseNow: 2 * time.Minute, 661 | args: args{ 662 | metricPath: "apower", 663 | deviceID: "shellyplus1pm-foo", 664 | value: -60.0, 665 | }, 666 | want: Metric{ 667 | Description: prometheus.NewDesc("total_energy", "", []string{"sensor", "topic"}, nil), 668 | ValueType: prometheus.GaugeValue, 669 | Value: 1.0, // negative input is ignored 670 | }, 671 | }, 672 | { 673 | name: "integrate positive values using expressions, step 4", 674 | fields: fields{ 675 | map[string][]*config.MetricConfig{ 676 | "apower": { 677 | { 678 | PrometheusName: "total_energy", 679 | ValueType: "gauge", 680 | OmitTimestamp: true, 681 | Expression: "value > 0 ? last_result + value * elapsed.Hours() : last_result", 682 | }, 683 | }, 684 | }, 685 | }, 686 | elapseNow: 3 * time.Minute, 687 | args: args{ 688 | metricPath: "apower", 689 | deviceID: "shellyplus1pm-foo", 690 | value: 600.0, 691 | }, 692 | want: Metric{ 693 | Description: prometheus.NewDesc("total_energy", "", []string{"sensor", "topic"}, nil), 694 | ValueType: prometheus.GaugeValue, 695 | Value: 11.0, // 600 watts for 1 minute = 10 Wh 696 | }, 697 | }, 698 | { 699 | name: "raw expression, step 1", 700 | fields: fields{ 701 | map[string][]*config.MetricConfig{ 702 | "apower": { 703 | { 704 | PrometheusName: "total_energy", 705 | ValueType: "gauge", 706 | OmitTimestamp: true, 707 | RawExpression: `float(join(filter(split(string(raw_value), ""), { # matches "^[0-9\\.]$" }), ""))`, 708 | }, 709 | }, 710 | }, 711 | }, 712 | elapseNow: 3 * time.Minute, 713 | args: args{ 714 | metricPath: "apower", 715 | deviceID: "shellyplus1pm-foo", 716 | value: "H42Jj.j44", 717 | }, 718 | want: Metric{ 719 | Description: prometheus.NewDesc("total_energy", "", []string{"sensor", "topic"}, nil), 720 | ValueType: prometheus.GaugeValue, 721 | Value: 42.44, 722 | }, 723 | }, 724 | } 725 | for _, tt := range tests { 726 | t.Run(tt.name, func(t *testing.T) { 727 | testNowElapsed = tt.elapseNow 728 | defer func() { testNowElapsed = time.Duration(0) }() 729 | 730 | p := NewParser(nil, config.JsonParsingConfigDefaults.Separator, stateDir) 731 | p.metricConfigs = tt.fields.metricConfigs 732 | 733 | // Find a valid metrics config 734 | configs := p.findMetricConfigs(tt.args.metricPath, tt.args.deviceID) 735 | if len(configs) != 1 { 736 | if !tt.wantErr { 737 | t.Errorf("MetricConfig not found") 738 | } 739 | return 740 | } 741 | config := configs[0] 742 | 743 | id := metricID("", tt.args.metricPath, tt.args.deviceID, config.PrometheusName) 744 | got, err := p.parseMetric(config, id, tt.args.value) 745 | if (err != nil) != tt.wantErr { 746 | t.Errorf("parseMetric() error = %v, wantErr %v", err, tt.wantErr) 747 | return 748 | } 749 | if !reflect.DeepEqual(got, tt.want) { 750 | t.Errorf("parseMetric() got = %v, want %v", got, tt.want) 751 | } 752 | 753 | if config.ForceMonotonicy || config.Expression != "" { 754 | if err = p.writeMetricState(id, p.states[id]); err != nil { 755 | t.Errorf("failed to write metric state: %v", err) 756 | } 757 | } 758 | }) 759 | } 760 | } 761 | 762 | func floatP(f float64) *float64 { 763 | return &f 764 | } 765 | 766 | func TestParser_evalExpression(t *testing.T) { 767 | now = testNow 768 | testNowElapsed = time.Duration(0) 769 | id := "metric" 770 | 771 | tests := []struct { 772 | expression string 773 | values []float64 774 | results []float64 775 | }{ 776 | { 777 | expression: "value + value", 778 | values: []float64{1, 0, -4}, 779 | results: []float64{2, 0, -8}, 780 | }, 781 | { 782 | expression: "value - last_value", 783 | values: []float64{1, 2, 5, 7}, 784 | results: []float64{1, 1, 3, 2}, 785 | }, 786 | { 787 | expression: "last_result + value", 788 | values: []float64{1, 2, 3, 4}, 789 | results: []float64{1, 3, 6, 10}, 790 | }, 791 | { 792 | expression: "last_result + elapsed.Milliseconds()", 793 | values: []float64{0, 0, 0, 0}, 794 | results: []float64{0, 1000, 2000, 3000}, 795 | }, 796 | { 797 | expression: "now().Unix()", 798 | values: []float64{0, 0}, 799 | results: []float64{float64(testNow().Unix()), float64(testNow().Unix() + 1)}, 800 | }, 801 | { 802 | expression: "int(1.1) + int(1.9)", 803 | values: []float64{0}, 804 | results: []float64{2}, 805 | }, 806 | { 807 | expression: "float(elapsed)", 808 | values: []float64{0, 0}, 809 | results: []float64{0, float64(time.Second)}, 810 | }, 811 | { 812 | expression: "round(value)", 813 | values: []float64{1.1, 2.5, 3.9}, 814 | results: []float64{1, 3, 4}, 815 | }, 816 | { 817 | expression: "ceil(value)", 818 | values: []float64{1.1, 2.9, 4.0}, 819 | results: []float64{2, 3, 4}, 820 | }, 821 | { 822 | expression: "floor(value)", 823 | values: []float64{1.1, 2.9, 4.0}, 824 | results: []float64{1, 2, 4}, 825 | }, 826 | { 827 | expression: "abs(value)", 828 | values: []float64{0, 1, -2}, 829 | results: []float64{0, 1, 2}, 830 | }, 831 | { 832 | expression: "min(value, 0)", 833 | values: []float64{1, -2, 3, -4}, 834 | results: []float64{0, -2, 0, -4}, 835 | }, 836 | { 837 | expression: "max(value, 0)", 838 | values: []float64{1, -2, 3, -4}, 839 | results: []float64{1, 0, 3, 0}, 840 | }, 841 | } 842 | 843 | for _, tt := range tests { 844 | t.Run(tt.expression, func(t *testing.T) { 845 | stateDir, err := os.MkdirTemp("", "parser_test") 846 | if err != nil { 847 | t.Fatal(err) 848 | } 849 | defer os.RemoveAll(stateDir) 850 | defer func() { testNowElapsed = time.Duration(0) }() 851 | 852 | p := NewParser(nil, ".", stateDir) 853 | for i, value := range tt.values { 854 | got, err := p.evalExpressionValue(id, tt.expression, value, value) 855 | want := tt.results[i] 856 | if err != nil { 857 | t.Errorf("evaluating the %dth value '%v' failed: %v", i, value, err) 858 | } 859 | if got != want { 860 | t.Errorf("unexpected result for %dth value, got %v, want %v", i, got, want) 861 | } 862 | // Advance the clock by one second for every sample 863 | testNowElapsed = testNowElapsed + time.Second 864 | } 865 | }) 866 | } 867 | } 868 | -------------------------------------------------------------------------------- /pkg/mqttclient/mqttClient.go: -------------------------------------------------------------------------------- 1 | package mqttclient 2 | 3 | import ( 4 | mqtt "github.com/eclipse/paho.mqtt.golang" 5 | "go.uber.org/zap" 6 | ) 7 | 8 | type SubscribeOptions struct { 9 | Topic string 10 | QoS byte 11 | OnMessageReceived mqtt.MessageHandler 12 | Logger *zap.Logger 13 | } 14 | 15 | func Subscribe(connectionOptions *mqtt.ClientOptions, subscribeOptions SubscribeOptions) error { 16 | oldConnect := connectionOptions.OnConnect 17 | connectionOptions.OnConnect = func(client mqtt.Client) { 18 | logger := subscribeOptions.Logger 19 | oldConnect(client) 20 | logger.Info("Connected to MQTT Broker") 21 | logger.Info("Will subscribe to topic", zap.String("topic", subscribeOptions.Topic)) 22 | if token := client.Subscribe(subscribeOptions.Topic, subscribeOptions.QoS, subscribeOptions.OnMessageReceived); token.Wait() && token.Error() != nil { 23 | logger.Error("Could not subscribe", zap.Error(token.Error())) 24 | } 25 | } 26 | client := mqtt.NewClient(connectionOptions) 27 | if token := client.Connect(); token.Wait() && token.Error() != nil { 28 | return token.Error() 29 | } 30 | 31 | return nil 32 | } 33 | -------------------------------------------------------------------------------- /release/Dockerfile.scratch: -------------------------------------------------------------------------------- 1 | FROM alpine as donor 2 | RUN apk add tzdata 3 | FROM scratch 4 | COPY mqtt2prometheus /mqtt2prometheus 5 | # Copy CA Certificates 6 | COPY --from=donor /etc/ssl/certs /etc/ssl/certs 7 | # Copy Time Zone Data 8 | COPY --from=donor /usr/share/zoneinfo /usr/share/zoneinfo 9 | ENTRYPOINT ["/mqtt2prometheus"] 10 | -------------------------------------------------------------------------------- /renovate.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://docs.renovatebot.com/renovate-schema.json", 3 | "extends": [ 4 | "config:base", 5 | ":automergeRequireAllStatusChecks", 6 | ":separateMultipleMajorReleases", 7 | ":separatePatchReleases", 8 | ":renovatePrefix", 9 | ":semanticPrefixChore", 10 | ":prHourlyLimitNone", 11 | ":prConcurrentLimit10" 12 | ], 13 | "automergeType": "pr", 14 | "postUpdateOptions": [ 15 | "gomodTidy" 16 | ], 17 | "labels": [ 18 | "dependencies", 19 | "versions", 20 | "automated" 21 | ], 22 | "rebaseWhen": "behind-base-branch", 23 | "stabilityDays": 10, 24 | "internalChecksFilter": "strict" 25 | } 26 | -------------------------------------------------------------------------------- /systemd/mqtt2prometheus: -------------------------------------------------------------------------------- 1 | # Command line options for mqtt2prometheus.service 2 | # See also /etc/mqtt2prometheus/config.yaml 3 | 4 | ARGS="-config /etc/mqtt2prometheus/config.yaml" 5 | -------------------------------------------------------------------------------- /systemd/mqtt2prometheus.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Simple translator from mqtt messages to prometheus. Analog to pushgateway 3 | Documentation=https://github.com/hikhvar/mqtt2prometheus 4 | After=network.target 5 | Before=prometheus.service 6 | 7 | [Service] 8 | Restart=always 9 | EnvironmentFile=/etc/default/mqtt2prometheus 10 | ExecStart=/usr/bin/mqtt2prometheus $ARGS 11 | TimeoutStopSec=20s 12 | 13 | # Extra security hardening options 14 | # See systemd.exec(5) for more information regarding these options. 15 | 16 | # Empty because mqtt2prometheus does not require any special capability. See capabilities(7) for more information. 17 | CapabilityBoundingSet= 18 | DynamicUser=true 19 | LockPersonality=true 20 | MemoryDenyWriteExecute=true 21 | NoNewPrivileges=true 22 | PrivateDevices=true 23 | PrivateTmp=true 24 | PrivateUsers=true 25 | ProtectClock=true 26 | ProtectControlGroups=true 27 | ProtectHome=true 28 | ProtectHostname=true 29 | ProtectKernelLogs=true 30 | ProtectKernelModules=true 31 | ProtectKernelTunables=true 32 | ProtectSystem=strict 33 | RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 34 | RestrictNamespaces=true 35 | RestrictRealtime=true 36 | RestrictSUIDSGID=true 37 | SystemCallArchitectures=native 38 | SystemCallErrorNumber=EPERM 39 | SystemCallFilter=@system-service 40 | UMask=077 41 | 42 | # See systemd.resource-control(5) for more information 43 | #IPAddressAllow=127.0.0.0/8 44 | #IPAddressDeny=any # the allow-list is evaluated before the deny list. Since the default is to allow, we need to deny everything. 45 | 46 | [Install] 47 | WantedBy=multi-user.target 48 | --------------------------------------------------------------------------------