├── .gitignore ├── contrib ├── icinga2-service-example.conf ├── test_notify-alertmanager-example.py ├── notify-alertmanager-example.py └── icinga2-commands.conf ├── .github ├── dependabot.yml ├── ISSUE_TEMPLATE │ ├── question.yaml │ ├── documentation.yaml │ ├── feature_request.yaml │ └── bug_report.yaml └── workflows │ ├── golangci-lint.yml │ └── build.yml ├── Makefile ├── testdata ├── unittest │ ├── queryDataset2.json │ ├── queryDataset3.json │ ├── queryDataset4.json │ ├── queryDataset1.json │ ├── alertDataset2.json │ ├── alertDataset4.json │ ├── alertDataset3.json │ └── alertDataset1.json ├── prometheus.yml ├── docker-compose.yml ├── README.md └── alertmanager │ ├── alert.rules │ └── alertmanager.yml ├── cmd ├── config_test.go ├── root.go ├── health.go ├── config.go ├── health_test.go ├── query.go ├── alert.go ├── alert_test.go └── query_test.go ├── main.go ├── .golangci.yml ├── .goreleaser.yml ├── go.mod ├── internal ├── alert │ ├── alert.go │ └── alert_test.go └── client │ └── client.go ├── go.sum ├── README.md └── LICENSE /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | check_prometheus 3 | vendor/ 4 | coverage.html 5 | coverage.out 6 | dist/ 7 | __pycache__/ -------------------------------------------------------------------------------- /contrib/icinga2-service-example.conf: -------------------------------------------------------------------------------- 1 | apply Service "prometheus_go_routines" { 2 | check_command = "prometheus-query" 3 | 4 | vars.prometheus_query = "go_goroutines" 5 | vars.prometheus_warning = "10" 6 | vars.prometheus_critical = "20" 7 | } -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: gomod 4 | directory: "/" 5 | schedule: 6 | interval: weekly 7 | time: '10:00' 8 | open-pull-requests-limit: 10 9 | 10 | - package-ecosystem: "github-actions" 11 | directory: "/" 12 | schedule: 13 | interval: monthly 14 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/question.yaml: -------------------------------------------------------------------------------- 1 | name: Question 2 | description: Ask a question 3 | title: "[Question]: " 4 | labels: ["question"] 5 | body: 6 | - type: textarea 7 | id: description 8 | attributes: 9 | label: Ask a question 10 | description: Please provide as much context as possible. You can use markdown. 11 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: test coverage lint vet 2 | 3 | build: 4 | CGO_ENABLED=0 go build 5 | lint: 6 | go fmt $(go list ./... | grep -v /vendor/) 7 | vet: 8 | go vet $(go list ./... | grep -v /vendor/) 9 | test: 10 | go test -v -cover ./... 11 | coverage: 12 | go test -v -cover -coverprofile=coverage.out ./... &&\ 13 | go tool cover -html=coverage.out -o coverage.html 14 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/documentation.yaml: -------------------------------------------------------------------------------- 1 | name: Documentation 2 | description: Suggest documentation improvements 3 | title: "[Documentation]: " 4 | labels: ["documentation"] 5 | body: 6 | - type: textarea 7 | id: description 8 | attributes: 9 | label: Describe the improvements you'd like. 10 | description: Please provide as much context as possible. You can use markdown. 11 | -------------------------------------------------------------------------------- /testdata/unittest/queryDataset2.json: -------------------------------------------------------------------------------- 1 | { 2 | "status": "success", 3 | "data": { 4 | "resultType": "vector", 5 | "result": [ 6 | { 7 | "metric": { 8 | "__name__": "up", 9 | "instance": "localhost", 10 | "job": "prometheus" 11 | }, 12 | "value": [ 13 | 1668782473.835, 14 | "1" 15 | ] 16 | } 17 | ] 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /testdata/unittest/queryDataset3.json: -------------------------------------------------------------------------------- 1 | { 2 | "status": "success", 3 | "data": { 4 | "resultType": "vector", 5 | "result": [ 6 | { 7 | "metric": { 8 | "__name__": "up", 9 | "instance": "localhost", 10 | "job": "prometheus" 11 | }, 12 | "value": [ 13 | 1668782473.835, 14 | "100" 15 | ] 16 | } 17 | ] 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /testdata/unittest/queryDataset4.json: -------------------------------------------------------------------------------- 1 | { 2 | "status": "success", 3 | "data": { 4 | "resultType": "vector", 5 | "result": [ 6 | { 7 | "metric": { 8 | "__name__": "up", 9 | "instance": "localhost", 10 | "job": "prometheus" 11 | }, 12 | "value": [ 13 | 1668782473.835, 14 | "-100" 15 | ] 16 | } 17 | ] 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /cmd/config_test.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | ) 7 | 8 | func loadTestdata(filepath string) []byte { 9 | data, _ := os.ReadFile(filepath) 10 | return data 11 | } 12 | 13 | func TestConfig(t *testing.T) { 14 | c := cliConfig.NewClient() 15 | expected := "http://localhost:9090/" 16 | if c.URL != "http://localhost:9090/" { 17 | t.Error("\nActual: ", c.URL, "\nExpected: ", expected) 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /.github/workflows/golangci-lint.yml: -------------------------------------------------------------------------------- 1 | name: golangci-lint 2 | on: 3 | push: 4 | tags: 5 | - v* 6 | branches: 7 | - main 8 | pull_request: 9 | 10 | jobs: 11 | golangci: 12 | name: lint 13 | runs-on: ubuntu-latest 14 | steps: 15 | - uses: actions/checkout@v6 16 | 17 | - name: golangci-lint 18 | uses: golangci/golangci-lint-action@v9 19 | with: 20 | version: v2.1.6 21 | -------------------------------------------------------------------------------- /testdata/prometheus.yml: -------------------------------------------------------------------------------- 1 | --- 2 | global: 3 | scrape_interval: 5s 4 | external_labels: 5 | monitor: 'my-monitor' 6 | 7 | rule_files: 8 | - /alertmanager/alert.rules 9 | 10 | scrape_configs: 11 | - job_name: 'prometheus' 12 | static_configs: 13 | - targets: ['localhost:9090'] 14 | 15 | - job_name: 'node-exporter' 16 | static_configs: 17 | - targets: ['node-exporter:9100'] 18 | 19 | alerting: 20 | alertmanagers: 21 | - static_configs: 22 | - targets: ['alertmanager:9093'] 23 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.yaml: -------------------------------------------------------------------------------- 1 | name: Feature Request 2 | description: Request a feature or enhancement 3 | title: "[Feature]: " 4 | labels: ["feature", "needs-triage"] 5 | body: 6 | - type: markdown 7 | attributes: 8 | value: | 9 | Please try to fill out as much of the information below as you can. Thank you! 10 | **Note:** If you want to sponsor new features, contact us at info@netways.de 11 | - type: textarea 12 | id: description 13 | attributes: 14 | label: Describe the feature request 15 | description: Please provide a concise description of the feature. You can use markdown. 16 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/NETWAYS/check_prometheus/cmd" 7 | ) 8 | 9 | // nolint: gochecknoglobals 10 | var ( 11 | // These get filled at build time with the proper vaules. 12 | version = "development" 13 | commit = "HEAD" 14 | date = "latest" 15 | ) 16 | 17 | func main() { 18 | cmd.Execute(buildVersion()) 19 | } 20 | 21 | func buildVersion() string { 22 | result := version 23 | 24 | if commit != "" { 25 | result = fmt.Sprintf("%s\ncommit: %s", result, commit) 26 | } 27 | 28 | if date != "" { 29 | result = fmt.Sprintf("%s\ndate: %s", result, date) 30 | } 31 | 32 | result += "\n" + cmd.License 33 | 34 | return result 35 | } 36 | -------------------------------------------------------------------------------- /contrib/test_notify-alertmanager-example.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import importlib 3 | 4 | loader = importlib.machinery.SourceFileLoader('notify', 'notify-alertmanager-example.py') 5 | mod = loader.load_module() 6 | 7 | class TestNotify(unittest.TestCase): 8 | 9 | def test_generate_alert(self): 10 | args = mod.cli(['--debug', '--alert-api-url', 'https://localhost', '--hostname', 'barfoo', '--state', '1', '--service', 'foobar']) 11 | actual = mod.generate_alert(args)[0] 12 | self.assertEqual(actual['labels']['service'], 'foobar') 13 | self.assertEqual(actual['labels']['instance'], 'barfoo') 14 | self.assertEqual(actual['labels']['alertname'], 'foobar_barfoo') 15 | 16 | if __name__ == '__main__': 17 | unittest.main() 18 | -------------------------------------------------------------------------------- /testdata/unittest/queryDataset1.json: -------------------------------------------------------------------------------- 1 | { 2 | "status": "success", 3 | "data": { 4 | "resultType": "matrix", 5 | "result": [ 6 | { 7 | "metric": { 8 | "__name__": "up", 9 | "instance": "localhost", 10 | "job": "node" 11 | }, 12 | "values": [ 13 | [ 14 | 1670340712.988, 15 | "1" 16 | ], 17 | [ 18 | 1670340772.988, 19 | "1" 20 | ], 21 | [ 22 | 1670340832.99, 23 | "1" 24 | ], 25 | [ 26 | 1670340892.99, 27 | "1" 28 | ], 29 | [ 30 | 1670340952.99, 31 | "1" 32 | ] 33 | ] 34 | } 35 | ] 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: Go 2 | on: 3 | push: 4 | branches: [ main ] 5 | tags: 6 | - v* 7 | pull_request: 8 | 9 | jobs: 10 | build: 11 | runs-on: ubuntu-latest 12 | 13 | steps: 14 | - name: Check out code into the Go module directory 15 | uses: actions/checkout@v6 16 | with: 17 | fetch-depth: 0 18 | 19 | - name: Set up Go 20 | uses: actions/setup-go@v6 21 | 22 | - name: Test 23 | run: go test -v ./... 24 | 25 | - name: Build 26 | run: go build -v . 27 | 28 | - name: Run goreleaser in release mode 29 | if: success() && startsWith(github.ref, 'refs/tags/v') 30 | uses: goreleaser/goreleaser-action@v6 31 | with: 32 | version: latest 33 | args: release --clean 34 | env: 35 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 36 | -------------------------------------------------------------------------------- /testdata/docker-compose.yml: -------------------------------------------------------------------------------- 1 | --- 2 | version: '2' 3 | services: 4 | prometheus: 5 | image: docker.io/prom/prometheus 6 | privileged: true 7 | volumes: 8 | - ./prometheus.yml:/etc/prometheus/prometheus.yml 9 | - ./alertmanager/alert.rules:/alertmanager/alert.rules 10 | command: 11 | - '--config.file=/etc/prometheus/prometheus.yml' 12 | - '--web.enable-lifecycle' 13 | ports: 14 | - '9090:9090' 15 | 16 | node-exporter: 17 | image: docker.io/prom/node-exporter 18 | ports: 19 | - '9100:9100' 20 | 21 | alertmanager: 22 | image: docker.io/prom/alertmanager 23 | privileged: true 24 | volumes: 25 | - ./alertmanager/alertmanager.yml:/alertmanager.yml 26 | command: 27 | - '--config.file=/alertmanager.yml' 28 | ports: 29 | - '9093:9093' 30 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | version: "2" 2 | run: 3 | tests: false 4 | linters: 5 | default: all 6 | disable: 7 | - cyclop 8 | - depguard 9 | - err113 10 | - exhaustruct 11 | - forbidigo 12 | - forcetypeassert 13 | - gochecknoglobals 14 | - gochecknoinits 15 | - godot 16 | - godox 17 | - lll 18 | - mnd 19 | - musttag 20 | - nakedret 21 | - nlreturn 22 | - nolintlint 23 | - nonamedreturns 24 | - tagliatelle 25 | - varnamelen 26 | - wrapcheck 27 | - funlen 28 | exclusions: 29 | generated: lax 30 | presets: 31 | - comments 32 | - common-false-positives 33 | - legacy 34 | - std-error-handling 35 | paths: 36 | - third_party$ 37 | - builtin$ 38 | - examples$ 39 | formatters: 40 | enable: 41 | - gofmt 42 | - goimports 43 | exclusions: 44 | generated: lax 45 | paths: 46 | - third_party$ 47 | - builtin$ 48 | - examples$ 49 | -------------------------------------------------------------------------------- /testdata/README.md: -------------------------------------------------------------------------------- 1 | # Container Setup for Testing 2 | 3 | ```bash 4 | $ docker-compose up -d 5 | ``` 6 | 7 | - altertmanager: http://localhost:9093/#/alerts 8 | - prometheus: http://localhost:9090/graph 9 | - node-exporter: http://localhost:9100/metrics 10 | 11 | # Examples 12 | 13 | ```bash 14 | $ curl -X PUT localhost:9090/-/reload 15 | ``` 16 | 17 | ```bash 18 | $ curl localhost:9090/-/healthy 19 | Prometheus Server is Healthy. 20 | ``` 21 | 22 | ```bash 23 | $ curl localhost:9090/-/ready 24 | Prometheus Server is Ready. 25 | ``` 26 | 27 | ```bash 28 | curl -g 'http://localhost:9090/api/v1/series?' --data-urlencode 'match[]=up' --data-urlencode 'match[]=process_start_time_seconds{job="prometheus"}' 29 | 30 | {"status":"success","data":[{"__name__":"process_start_time_seconds","instance":"localhost:9090","job":"prometheus"},{"__name__":"up","instance":"localhost:9090","job":"prometheus"},{"__name__":"up","instance":"node-exporter:9100","job":"node-exporter"}]} 31 | (base) 32 | ``` 33 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.yaml: -------------------------------------------------------------------------------- 1 | name: Bug Report 2 | description: File a bug report 3 | title: "[Bug]: " 4 | labels: ["bug", "needs-triage"] 5 | body: 6 | - type: checkboxes 7 | id: terms 8 | attributes: 9 | label: Please try to fill out as much of the information below as you can. Thank you! 10 | options: 11 | - label: Yes, I've searched similar issues on GitHub and didn't find any. 12 | required: true 13 | - type: input 14 | id: app_version 15 | attributes: 16 | label: Which version contains the bug? 17 | placeholder: 1.0.0 18 | - type: textarea 19 | id: description 20 | attributes: 21 | label: Describe the bug 22 | description: Please provide a concise description of the bug, add any relevant output or error messages. You can use markdown. 23 | - type: textarea 24 | id: recreate 25 | attributes: 26 | label: How to recreate the bug? 27 | description: Please provide the steps to recreate the issue. 28 | -------------------------------------------------------------------------------- /testdata/unittest/alertDataset2.json: -------------------------------------------------------------------------------- 1 | { 2 | "status": "success", 3 | "data": { 4 | "groups": [ 5 | { 6 | "name": "Foo", 7 | "file": "alerts.yaml", 8 | "rules": [ 9 | { 10 | "state": "inactive", 11 | "name": "InactiveAlert", 12 | "query": "foo", 13 | "duration": 120, 14 | "labels": { 15 | "severity": "critical" 16 | }, 17 | "annotations": { 18 | "description": "Inactive", 19 | "summary": "Inactive" 20 | }, 21 | "alerts": [], 22 | "health": "ok", 23 | "evaluationTime": 0.000462382, 24 | "lastEvaluation": "2022-11-18T14:01:07.597034323Z", 25 | "type": "alerting" 26 | } 27 | ], 28 | "interval": 10, 29 | "limit": 0, 30 | "evaluationTime": 0.000478395, 31 | "lastEvaluation": "2022-11-18T14:01:07.597021953Z" 32 | } 33 | ] 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /.goreleaser.yml: -------------------------------------------------------------------------------- 1 | # See documentation at https://goreleaser.com 2 | version: 2 3 | before: 4 | hooks: 5 | - go mod download 6 | builds: 7 | - env: 8 | - CGO_ENABLED=0 9 | goarch: 10 | - amd64 11 | - arm64 12 | goos: 13 | - linux 14 | - windows 15 | - darwin 16 | ldflags: 17 | - >- 18 | -s -w -X main.version={{.Version}} 19 | -X main.commit={{.Commit}} 20 | -X main.date={{.CommitDate}} 21 | release: 22 | draft: true 23 | github: 24 | owner: NETWAYS 25 | name: check_prometheus 26 | archives: 27 | - format: binary 28 | name_template: >- 29 | {{ .ProjectName }}_{{ .Tag }}_ 30 | {{- if eq .Os "linux" }}Linux{{ end }} 31 | {{- if eq .Os "windows" }}Windows{{ end }} 32 | {{- if eq .Os "darwin" }}Darwin{{ end }} 33 | {{- if eq .Arch "amd64" }}_x86_64{{ end }} 34 | {{- if eq .Arch "arm64" }}_arm64{{ end }} 35 | checksum: 36 | name_template: 'checksums.txt' 37 | snapshot: 38 | name_template: '{{ .Tag }}-SNAPSHOT-{{.ShortCommit}}' 39 | changelog: 40 | sort: asc 41 | filters: 42 | exclude: 43 | - '^docs:' 44 | - '^test:' 45 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/NETWAYS/check_prometheus 2 | 3 | go 1.24.0 4 | 5 | require ( 6 | github.com/NETWAYS/go-check v0.6.3 7 | github.com/prometheus/client_golang v1.23.2 8 | github.com/prometheus/common v0.67.4 9 | github.com/spf13/cobra v1.10.2 10 | ) 11 | 12 | require ( 13 | github.com/beorn7/perks v1.0.1 // indirect 14 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 15 | github.com/golang-jwt/jwt/v5 v5.3.0 // indirect 16 | github.com/google/uuid v1.6.0 // indirect 17 | github.com/inconshreveable/mousetrap v1.1.0 // indirect 18 | github.com/jpillora/backoff v1.0.0 // indirect 19 | github.com/json-iterator/go v1.1.12 // indirect 20 | github.com/kr/text v0.2.0 // indirect 21 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 22 | github.com/modern-go/reflect2 v1.0.2 // indirect 23 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 24 | github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect 25 | github.com/prometheus/client_model v0.6.2 // indirect 26 | github.com/prometheus/procfs v0.16.1 // indirect 27 | github.com/spf13/pflag v1.0.10 // indirect 28 | go.yaml.in/yaml/v2 v2.4.3 // indirect 29 | golang.org/x/net v0.46.0 // indirect 30 | golang.org/x/oauth2 v0.32.0 // indirect 31 | golang.org/x/sys v0.37.0 // indirect 32 | golang.org/x/text v0.30.0 // indirect 33 | google.golang.org/protobuf v1.36.10 // indirect 34 | ) 35 | -------------------------------------------------------------------------------- /testdata/unittest/alertDataset4.json: -------------------------------------------------------------------------------- 1 | { 2 | "status": "success", 3 | "data": { 4 | "groups": [ 5 | { 6 | "name": "example", 7 | "file": "recoding.yaml", 8 | "rules": [ 9 | { 10 | "name": "job:foo", 11 | "query": "sum by(job) (requests_total)", 12 | "health": "ok", 13 | "evaluationTime": 0.000391321, 14 | "lastEvaluation": "2023-01-13T14:26:08.687065894Z", 15 | "type": "recording" 16 | } 17 | ], 18 | "interval": 10, 19 | "evaluationTime": 0.000403777, 20 | "lastEvaluation": "2023-01-13T14:26:08.687058029Z" 21 | }, 22 | { 23 | "name": "Foo", 24 | "file": "alerts.yaml", 25 | "rules": [ 26 | { 27 | "state": "inactive", 28 | "name": "InactiveAlert", 29 | "query": "foo", 30 | "duration": 120, 31 | "labels": { 32 | "severity": "critical" 33 | }, 34 | "annotations": { 35 | "description": "Inactive", 36 | "summary": "Inactive" 37 | }, 38 | "alerts": [], 39 | "health": "ok", 40 | "evaluationTime": 0.000462382, 41 | "lastEvaluation": "2022-11-18T14:01:07.597034323Z", 42 | "type": "alerting" 43 | } 44 | ], 45 | "interval": 10, 46 | "limit": 0, 47 | "evaluationTime": 0.000478395, 48 | "lastEvaluation": "2022-11-18T14:01:07.597021953Z" 49 | } 50 | ] 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /testdata/alertmanager/alert.rules: -------------------------------------------------------------------------------- 1 | groups: 2 | - name: Test Alerts for check_plugin 3 | rules: 4 | 5 | - alert: PrometheusTargetMissing 6 | expr: up == 0 7 | for: 0m 8 | labels: 9 | severity: critical 10 | annotations: 11 | summary: Prometheus target missing (instance {{ $labels.instance }}) 12 | description: "A Prometheus target has disappeared. An exporter might be crashed.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" 13 | 14 | - alert: PrometheusAlertmanagerJobMissing 15 | expr: absent(up{job="alertmanager"}) 16 | for: 0m 17 | labels: 18 | severity: warning 19 | annotations: 20 | summary: Prometheus AlertManager job missing (instance {{ $labels.instance }}) 21 | description: "A Prometheus AlertManager job has disappeared\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" 22 | 23 | - alert: HostOutOfMemory 24 | expr: node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes * 100 < 10 25 | for: 2m 26 | labels: 27 | severity: warning 28 | annotations: 29 | summary: Host out of memory (instance {{ $labels.instance }}) 30 | description: "Node memory is filling up (< 10% left)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" 31 | 32 | - alert: HostHighCpuLoad 33 | expr: 100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[2m])) * 100) > 80 34 | for: 0m 35 | labels: 36 | severity: warning 37 | annotations: 38 | summary: Host high CPU load (instance {{ $labels.instance }}) 39 | description: "CPU load is > 80%\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" 40 | -------------------------------------------------------------------------------- /testdata/unittest/alertDataset3.json: -------------------------------------------------------------------------------- 1 | { 2 | "status": "success", 3 | "data": { 4 | "groups": [ 5 | { 6 | "name": "k8s", 7 | "file": "/etc/prometheus/rules/al.yaml", 8 | "rules": [ 9 | { 10 | "state": "inactive", 11 | "name": "NodeHasMemoryPressure", 12 | "query": "kube_node{condition=\"MemoryPressure\",status=\"true\"} == 1", 13 | "duration": 300, 14 | "keepFiringFor": 0, 15 | "labels": {}, 16 | "annotations": { 17 | "summary": "Memory pressure on instance {{ $labels.instance }}" 18 | }, 19 | "alerts": [], 20 | "health": "ok", 21 | "evaluationTime": 0.00023339, 22 | "lastEvaluation": "2024-12-18T17:50:01.483161228Z", 23 | "type": "alerting" 24 | } 25 | ], 26 | "interval": 15, 27 | "limit": 0, 28 | "evaluationTime": 0.000262616, 29 | "lastEvaluation": "2024-12-18T17:50:01.483135426Z" 30 | }, 31 | { 32 | "name": "example", 33 | "file": "/etc/prometheus/rules/rec.yaml", 34 | "rules": [ 35 | { 36 | "name": "rule:prometheus_http_requests_total:sum", 37 | "query": "sum by (code) (rate(prometheus_http_requests_total[5m]))", 38 | "health": "ok", 39 | "evaluationTime": 0.000472562, 40 | "lastEvaluation": "2024-12-18T17:50:12.420737469Z", 41 | "type": "recording" 42 | } 43 | ], 44 | "interval": 15, 45 | "limit": 0, 46 | "evaluationTime": 0.000497618, 47 | "lastEvaluation": "2024-12-18T17:50:12.42071533Z" 48 | } 49 | ], 50 | "groupNextToken:omitempty": "" 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /cmd/root.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "os" 5 | 6 | "github.com/NETWAYS/go-check" 7 | "github.com/spf13/cobra" 8 | ) 9 | 10 | var Timeout = 30 11 | 12 | var rootCmd = &cobra.Command{ 13 | Use: "check_prometheus", 14 | Short: "An Icinga check plugin to check Prometheus", 15 | PersistentPreRun: func(_ *cobra.Command, _ []string) { 16 | go check.HandleTimeout(Timeout) 17 | }, 18 | Run: Usage, 19 | } 20 | 21 | func Execute(version string) { 22 | defer check.CatchPanic() 23 | 24 | rootCmd.Version = version 25 | rootCmd.VersionTemplate() 26 | 27 | if err := rootCmd.Execute(); err != nil { 28 | check.ExitError(err) 29 | } 30 | } 31 | 32 | func init() { 33 | rootCmd.CompletionOptions.DisableDefaultCmd = true 34 | rootCmd.DisableAutoGenTag = true 35 | 36 | rootCmd.SetHelpCommand(&cobra.Command{ 37 | Use: "no-help", 38 | Hidden: true, 39 | }) 40 | 41 | pfs := rootCmd.PersistentFlags() 42 | pfs.StringVarP(&cliConfig.Hostname, "hostname", "H", "localhost", 43 | "Hostname of the Prometheus server (CHECK_PROMETHEUS_HOSTNAME)") 44 | pfs.IntVarP(&cliConfig.Port, "port", "p", 9090, 45 | "Port of the Prometheus server") 46 | pfs.StringVarP(&cliConfig.URL, "url", "U", "/", 47 | "URL/Path to append to the Promethes Hostname (CHECK_PROMETHEUS_URL)") 48 | pfs.BoolVarP(&cliConfig.Secure, "secure", "s", false, 49 | "Use a HTTPS connection") 50 | pfs.BoolVarP(&cliConfig.Insecure, "insecure", "i", false, 51 | "Skip the verification of the server's TLS certificate") 52 | pfs.StringVarP(&cliConfig.Bearer, "bearer", "b", "", 53 | "Specify the Bearer Token for server authentication (CHECK_PROMETHEUS_BEARER)") 54 | pfs.StringVarP(&cliConfig.BasicAuth, "user", "u", "", 55 | "Specify the user name and password for server authentication (CHECK_PROMETHEUS_BASICAUTH)") 56 | pfs.StringVarP(&cliConfig.CAFile, "ca-file", "", "", 57 | "Specify the CA File for TLS authentication (CHECK_PROMETHEUS_CA_FILE)") 58 | pfs.StringVarP(&cliConfig.CertFile, "cert-file", "", "", 59 | "Specify the Certificate File for TLS authentication (CHECK_PROMETHEUS_CERT_FILE)") 60 | pfs.StringVarP(&cliConfig.KeyFile, "key-file", "", "", 61 | "Specify the Key File for TLS authentication (CHECK_PROMETHEUS_KEY_FILE)") 62 | pfs.IntVarP(&Timeout, "timeout", "t", Timeout, 63 | "Timeout in seconds for the CheckPlugin") 64 | pfs.StringSliceVarP(&cliConfig.Headers, "header", "", nil, 65 | `Additional HTTP header to include in the request. Can be used multiple times. 66 | Keys and values are separated by a colon (--header "X-Custom: example").`) 67 | 68 | rootCmd.Flags().SortFlags = false 69 | pfs.SortFlags = false 70 | 71 | help := rootCmd.HelpTemplate() 72 | rootCmd.SetHelpTemplate(help + Copyright) 73 | 74 | check.LoadFromEnv(&cliConfig) 75 | } 76 | 77 | func Usage(cmd *cobra.Command, _ []string) { 78 | _ = cmd.Usage() 79 | 80 | os.Exit(3) 81 | } 82 | -------------------------------------------------------------------------------- /internal/alert/alert.go: -------------------------------------------------------------------------------- 1 | package alert 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "slices" 7 | "strconv" 8 | "strings" 9 | 10 | "github.com/NETWAYS/go-check" 11 | v1 "github.com/prometheus/client_golang/api/prometheus/v1" 12 | "github.com/prometheus/common/model" 13 | ) 14 | 15 | // Internal representation of Prometheus Rules. 16 | // Alert attribute will be used when iterating over multiple AlertingRules. 17 | type Rule struct { 18 | AlertingRule v1.AlertingRule 19 | Alert *v1.Alert 20 | } 21 | 22 | func FlattenRules(groups []v1.RuleGroup, wantedGroups []string) []Rule { 23 | // Flattens a list of RuleGroup containing a list of Rules into 24 | // a list of internal Alertingrules. 25 | var l int 26 | // Set initial capacity to reduce memory allocations. 27 | for _, grp := range groups { 28 | if wantedGroups != nil { 29 | if !slices.Contains(wantedGroups, grp.Name) { 30 | continue 31 | } 32 | } 33 | 34 | l += len(grp.Rules) 35 | } 36 | 37 | rules := make([]Rule, 0, l) 38 | 39 | var r Rule 40 | 41 | for _, grp := range groups { 42 | if wantedGroups != nil { 43 | if !slices.Contains(wantedGroups, grp.Name) { 44 | continue 45 | } 46 | } 47 | 48 | for _, rl := range grp.Rules { 49 | // For now we only care about AlertingRules, 50 | // since RecodingRules can simply be queried. 51 | if _, ok := rl.(v1.AlertingRule); ok { 52 | r.AlertingRule = rl.(v1.AlertingRule) 53 | rules = append(rules, r) 54 | } 55 | } 56 | } 57 | 58 | return rules 59 | } 60 | 61 | func (a *Rule) GetStatus() (status int) { 62 | switch a.AlertingRule.State { 63 | case string(v1.AlertStateFiring): 64 | status = check.Critical 65 | case string(v1.AlertStatePending): 66 | status = check.Warning 67 | case string(v1.AlertStateInactive): 68 | status = check.OK 69 | default: 70 | status = check.Unknown 71 | } 72 | 73 | return status 74 | } 75 | 76 | func (a *Rule) GetOutput() (output string) { 77 | if a.Alert == nil { 78 | return fmt.Sprintf("[%s] is %s", 79 | a.AlertingRule.Name, 80 | a.AlertingRule.State) 81 | } 82 | 83 | var ( 84 | value float64 85 | v model.LabelValue 86 | ok bool 87 | out strings.Builder 88 | ) 89 | 90 | // Base Output 91 | out.WriteString(fmt.Sprintf("[%s]", a.AlertingRule.Name)) 92 | 93 | // Add job if available 94 | v, ok = a.Alert.Labels["job"] 95 | if ok { 96 | out.WriteString(fmt.Sprintf(" - Job: [%s]", string(v))) 97 | } 98 | 99 | // Add instance if available 100 | v, ok = a.Alert.Labels["instance"] 101 | if ok { 102 | out.WriteString(fmt.Sprintf(" on Instance: [%s]", string(v))) 103 | } 104 | 105 | // Add current value to output 106 | value, _ = strconv.ParseFloat(a.Alert.Value, 32) 107 | out.WriteString(fmt.Sprintf(" is %s - value: %.2f", a.AlertingRule.State, value)) 108 | 109 | // Add labels to the output 110 | l, err := json.Marshal(a.Alert.Labels) 111 | 112 | if err == nil { 113 | out.WriteString(" - ") 114 | out.Write(l) 115 | } 116 | 117 | return out.String() 118 | } 119 | -------------------------------------------------------------------------------- /contrib/notify-alertmanager-example.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | 4 | from argparse import ArgumentParser 5 | from urllib import request 6 | import json 7 | import socket 8 | import sys 9 | 10 | 11 | DEBUG = False 12 | 13 | 14 | def cli(args): 15 | """ 16 | Commmand Line Arguments 17 | """ 18 | 19 | parser = ArgumentParser(description='Send notifications to the Prometheus Alertmanager') 20 | 21 | parser.add_argument('--hostname', type=str, required=True) 22 | parser.add_argument('--service', type=str, default='hostalive', required=True) 23 | parser.add_argument('--output', type=str, default='') 24 | parser.add_argument('--state', type=int, required=True, choices=[0,1,2,3]) 25 | parser.add_argument('--alert-api-url', type=str, default='http://localhost:9093/api/v1/alerts') 26 | 27 | parser.add_argument('--debug', action='store_true') 28 | parser.set_defaults(debug=False) 29 | 30 | return parser.parse_args(args) 31 | 32 | 33 | def post_alert(url, data): 34 | """ 35 | HTTP Post the data to URL 36 | """ 37 | if DEBUG: 38 | print('DEBUG: Posting Alert') 39 | 40 | content = None 41 | try: 42 | req = request.Request(url, method="POST") 43 | req.add_header('Content-Type', 'application/json') 44 | r = request.urlopen(req, data=data) 45 | content = r.read() 46 | except Exception as e: 47 | print('ERROR:', e) 48 | sys.exit(1) 49 | 50 | if DEBUG: 51 | print('DEBUG: Server Return:') 52 | print(content) 53 | 54 | 55 | def generate_alert(args): 56 | if DEBUG: 57 | print('DEBUG: Generating Alert') 58 | 59 | # Default Value 60 | state_name_mapping = { 61 | "0": "Up", 62 | "1": "Up", 63 | "2": "Down", 64 | "3": "Down" 65 | } 66 | 67 | # Change mapping if hostalive 68 | if args.service != "hostalive": 69 | state_name_mapping = { 70 | "0": "OK", 71 | "1": "Warning", 72 | "2": "Critical", 73 | "3": "Unknown" 74 | } 75 | 76 | alert_state_name_mapping = { 77 | "0": "resolved", 78 | "1": "firing", 79 | "2": "firing", 80 | "3": "firing" 81 | } 82 | 83 | status = state_name_mapping[str(args.state)] 84 | alert_status = alert_state_name_mapping[str(args.state)] 85 | 86 | return [{ 87 | "status": alert_status, 88 | "generatorURL": "foo", 89 | "labels": { 90 | "alertname": f"{args.service}_{args.hostname}", 91 | "instance": args.hostname, 92 | "service": args.service, 93 | }, 94 | "annotations": { 95 | "summary": f"Service {args.service} on {args.hostname} is {status}", 96 | } 97 | }] 98 | 99 | def main(args): 100 | 101 | DEBUG = args.debug 102 | 103 | if DEBUG: 104 | print('DEBUG: CLI Arguments') 105 | print('DEBUG:', args) 106 | 107 | alert = generate_alert(args) 108 | alert_json = json.dumps(alert).encode() 109 | 110 | if DEBUG: 111 | print('DEBUG: Alert to be posted') 112 | print('DEBUG:', alert) 113 | 114 | post_alert(args.alert_api_url, alert_json) 115 | 116 | if __name__ == "__main__": 117 | ARGS = cli(sys.argv[1:]) 118 | main(ARGS) 119 | -------------------------------------------------------------------------------- /cmd/health.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "errors" 5 | 6 | "github.com/NETWAYS/go-check" 7 | "github.com/NETWAYS/go-check/result" 8 | "github.com/spf13/cobra" 9 | ) 10 | 11 | var healthCmd = &cobra.Command{ 12 | Use: "health", 13 | Short: "Checks the health or readiness status of the Prometheus server", 14 | Long: `Checks the health or readiness status of the Prometheus server 15 | Health: Checks the health of an endpoint, which returns OK if the Prometheus server is healthy. 16 | Ready: Checks the readiness of an endpoint, which returns OK if the Prometheus server is ready to serve traffic (i.e. respond to queries).`, 17 | Example: ` 18 | $ check_prometheus health --hostname 'localhost' --port 9090 --insecure 19 | OK - Prometheus Server is Healthy. | statuscode=200 20 | 21 | $ check_prometheus --bearer secrettoken health --ready 22 | OK - Prometheus Server is Ready. | statuscode=200`, 23 | Run: func(_ *cobra.Command, _ []string) { 24 | var ( 25 | rc int 26 | ) 27 | 28 | overall := result.Overall{} 29 | 30 | // Creating an client and connecting to the API 31 | c := cliConfig.NewClient() 32 | err := c.Connect() 33 | if err != nil { 34 | check.ExitError(err) 35 | } 36 | 37 | // Getting the preconfigured context 38 | ctx, cancel := cliConfig.timeoutContext() 39 | defer cancel() 40 | 41 | if cliConfig.PReady { 42 | // Getting the ready status 43 | rc, _, output, err := c.GetStatus(ctx, "ready") 44 | 45 | if err != nil { 46 | check.ExitError(errors.New(output)) 47 | } 48 | 49 | partialResult := result.NewPartialResult() 50 | 51 | _ = partialResult.SetState(rc) 52 | partialResult.Output = output 53 | overall.AddSubcheck(partialResult) 54 | 55 | check.ExitRaw(overall.GetStatus(), overall.GetOutput()) 56 | } 57 | 58 | if cliConfig.Info { 59 | // Displays various build information properties about the Prometheus server 60 | info, err := c.API.Buildinfo(ctx) 61 | if err != nil { 62 | check.ExitError(err) 63 | } 64 | partialResult := result.NewPartialResult() 65 | 66 | _ = partialResult.SetState(rc) 67 | 68 | partialResult.Output = "Prometheus Server information\n\n" + 69 | "Version: " + info.Version + "\n" + 70 | "Branch: " + info.Branch + "\n" + 71 | "BuildDate: " + info.BuildDate + "\n" + 72 | "BuildUser: " + info.BuildUser + "\n" + 73 | "Revision: " + info.Revision 74 | 75 | overall.AddSubcheck(partialResult) 76 | 77 | check.ExitRaw(overall.GetStatus(), overall.GetOutput()) 78 | } 79 | 80 | // Getting the health status is the default 81 | rc, _, output, err := c.GetStatus(ctx, "healthy") 82 | 83 | if err != nil { 84 | check.ExitError(errors.New(output)) 85 | } 86 | 87 | partialResult := result.NewPartialResult() 88 | _ = partialResult.SetState(rc) 89 | partialResult.Output = output 90 | overall.AddSubcheck(partialResult) 91 | 92 | check.ExitRaw(overall.GetStatus(), overall.GetOutput()) 93 | }, 94 | } 95 | 96 | func init() { 97 | rootCmd.AddCommand(healthCmd) 98 | 99 | fs := healthCmd.Flags() 100 | fs.BoolVarP(&cliConfig.PReady, "ready", "r", false, 101 | "Checks the readiness of an endpoint") 102 | fs.BoolVarP(&cliConfig.Info, "info", "I", false, 103 | "Displays various build information properties about the Prometheus server") 104 | 105 | fs.SortFlags = false 106 | } 107 | -------------------------------------------------------------------------------- /internal/client/client.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/http" 7 | "net/url" 8 | "strings" 9 | 10 | "github.com/NETWAYS/go-check" 11 | "github.com/prometheus/client_golang/api" 12 | v1 "github.com/prometheus/client_golang/api/prometheus/v1" 13 | ) 14 | 15 | type Client struct { 16 | URL string 17 | Client api.Client 18 | API v1.API 19 | RoundTripper http.RoundTripper 20 | } 21 | 22 | func NewClient(url string, rt http.RoundTripper) *Client { 23 | return &Client{ 24 | URL: url, 25 | RoundTripper: rt, 26 | } 27 | } 28 | 29 | func (c *Client) Connect() error { 30 | cfg, err := api.NewClient(api.Config{ 31 | Address: c.URL, 32 | RoundTripper: c.RoundTripper, 33 | }) 34 | 35 | if err != nil { 36 | return fmt.Errorf("error creating client: %w", err) 37 | } 38 | 39 | c.Client = cfg 40 | c.API = v1.NewAPI(c.Client) 41 | 42 | return nil 43 | } 44 | 45 | func (c *Client) GetStatus(ctx context.Context, endpoint string) (returncode int, statuscode int, body string, err error) { 46 | // Parses the response from the Prometheus /healthy and /ready endpoint 47 | // Return: Exit Status Code, HTTP Status Code, HTTP Body, Error 48 | // Building the final URL with the endpoint parameter 49 | u, _ := url.JoinPath(c.URL, "/-/", endpoint) 50 | req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil) 51 | 52 | if err != nil { 53 | e := fmt.Sprintf("could not create request: %s", err) 54 | return check.Unknown, 0, e, err 55 | } 56 | 57 | // Making the request with the preconfigured Client 58 | // So that we can reuse the preconfigured Roundtripper 59 | resp, b, err := c.Client.Do(ctx, req) 60 | 61 | if err != nil { 62 | e := fmt.Sprintf("could not get status: %s", err) 63 | return check.Unknown, 0, e, err 64 | } 65 | 66 | defer resp.Body.Close() 67 | 68 | // Getting the response body 69 | respBody := strings.TrimSpace(string(b)) 70 | 71 | // What we expect from the Prometheus Server 72 | statusOk := "is Healthy." 73 | if endpoint == "ready" { 74 | statusOk = "is Ready." 75 | } 76 | 77 | if resp.StatusCode == http.StatusOK && strings.Contains(respBody, statusOk) { 78 | return check.OK, resp.StatusCode, respBody, err 79 | } 80 | 81 | if resp.StatusCode != http.StatusOK { 82 | return check.Critical, resp.StatusCode, respBody, err 83 | } 84 | 85 | return check.Unknown, resp.StatusCode, respBody, err 86 | } 87 | 88 | type headersRoundTripper struct { 89 | headers map[string]string 90 | rt http.RoundTripper 91 | } 92 | 93 | // NewHeadersRoundTripper adds the given headers to a request 94 | func NewHeadersRoundTripper(headers map[string]string, rt http.RoundTripper) http.RoundTripper { 95 | return &headersRoundTripper{headers, rt} 96 | } 97 | 98 | func (rt *headersRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { 99 | // RoundTrip should not modify the request, except for 100 | // consuming and closing the Request's Body. 101 | req = cloneRequest(req) 102 | 103 | for key, value := range rt.headers { 104 | req.Header.Add(key, value) 105 | } 106 | 107 | return rt.rt.RoundTrip(req) 108 | } 109 | 110 | // cloneRequest returns a clone of the provided *http.Request 111 | func cloneRequest(r *http.Request) *http.Request { 112 | // Shallow copy of the struct. 113 | r2 := new(http.Request) 114 | *r2 = *r 115 | // Deep copy of the Header. 116 | r2.Header = make(http.Header) 117 | for k, s := range r.Header { 118 | r2.Header[k] = s 119 | } 120 | 121 | return r2 122 | } 123 | -------------------------------------------------------------------------------- /testdata/alertmanager/alertmanager.yml: -------------------------------------------------------------------------------- 1 | # Sample configuration. 2 | # See https://prometheus.io/docs/alerting/configuration/ for documentation. 3 | 4 | global: 5 | # The smarthost and SMTP sender used for mail notifications. 6 | smtp_smarthost: 'localhost:25' 7 | smtp_from: 'alertmanager@example.org' 8 | smtp_auth_username: 'alertmanager' 9 | smtp_auth_password: 'password' 10 | 11 | # The directory from which notification templates are read. 12 | templates: 13 | - '/etc/prometheus/alertmanager_templates/*.tmpl' 14 | 15 | # The root route on which each incoming alert enters. 16 | route: 17 | # The labels by which incoming alerts are grouped together. For example, 18 | # multiple alerts coming in for cluster=A and alertname=LatencyHigh would 19 | # be batched into a single group. 20 | group_by: ['alertname', 'cluster', 'service'] 21 | 22 | # When a new group of alerts is created by an incoming alert, wait at 23 | # least 'group_wait' to send the initial notification. 24 | # This way ensures that you get multiple alerts for the same group that start 25 | # firing shortly after another are batched together on the first 26 | # notification. 27 | group_wait: 30s 28 | 29 | # When the first notification was sent, wait 'group_interval' to send a batch 30 | # of new alerts that started firing for that group. 31 | group_interval: 5m 32 | 33 | # If an alert has successfully been sent, wait 'repeat_interval' to 34 | # resend them. 35 | repeat_interval: 3h 36 | 37 | # A default receiver 38 | receiver: team-X-mails 39 | 40 | # All the above attributes are inherited by all child routes and can 41 | # overwritten on each. 42 | 43 | # The child route trees. 44 | routes: 45 | # This routes performs a regular expression match on alert labels to 46 | # catch alerts that are related to a list of services. 47 | - match_re: 48 | service: ^(foo1|foo2|baz)$ 49 | receiver: team-X-mails 50 | # The service has a sub-route for critical alerts, any alerts 51 | # that do not match, i.e. severity != critical, fall-back to the 52 | # parent node and are sent to 'team-X-mails' 53 | routes: 54 | - match: 55 | severity: critical 56 | receiver: team-X-pager 57 | - match: 58 | service: files 59 | receiver: team-Y-mails 60 | 61 | routes: 62 | - match: 63 | severity: critical 64 | receiver: team-Y-pager 65 | 66 | # This route handles all alerts coming from a database service. If there's 67 | # no team to handle it, it defaults to the DB team. 68 | - match: 69 | service: database 70 | receiver: team-DB-pager 71 | # Also group alerts by affected database. 72 | group_by: [alertname, cluster, database] 73 | routes: 74 | - match: 75 | owner: team-X 76 | receiver: team-X-pager 77 | - match: 78 | owner: team-Y 79 | receiver: team-Y-pager 80 | 81 | 82 | # Inhibition rules allow to mute a set of alerts given that another alert is 83 | # firing. 84 | # We use this to mute any warning-level notifications if the same alert is 85 | # already critical. 86 | inhibit_rules: 87 | - source_match: 88 | severity: 'critical' 89 | target_match: 90 | severity: 'warning' 91 | # Apply inhibition if the alertname is the same. 92 | equal: ['alertname', 'cluster', 'service'] 93 | 94 | 95 | receivers: 96 | - name: 'team-X-mails' 97 | email_configs: 98 | - to: 'team-X+alerts@example.org' 99 | 100 | - name: 'team-X-pager' 101 | email_configs: 102 | - to: 'team-X+alerts-critical@example.org' 103 | pagerduty_configs: 104 | - service_key: 105 | 106 | - name: 'team-Y-mails' 107 | email_configs: 108 | - to: 'team-Y+alerts@example.org' 109 | 110 | - name: 'team-Y-pager' 111 | pagerduty_configs: 112 | - service_key: 113 | 114 | - name: 'team-DB-pager' 115 | pagerduty_configs: 116 | - service_key: 117 | -------------------------------------------------------------------------------- /cmd/config.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "net" 7 | "net/http" 8 | "net/url" 9 | "strconv" 10 | "strings" 11 | "time" 12 | 13 | "github.com/NETWAYS/check_prometheus/internal/client" 14 | "github.com/NETWAYS/go-check" 15 | "github.com/prometheus/common/config" 16 | ) 17 | 18 | type Config struct { 19 | BasicAuth string `env:"CHECK_PROMETHEUS_BASICAUTH"` 20 | Bearer string `env:"CHECK_PROMETHEUS_BEARER"` 21 | CAFile string `env:"CHECK_PROMETHEUS_CA_FILE"` 22 | CertFile string `env:"CHECK_PROMETHEUS_CERT_FILE"` 23 | KeyFile string `env:"CHECK_PROMETHEUS_KEY_FILE"` 24 | Hostname string `env:"CHECK_PROMETHEUS_HOSTNAME"` 25 | URL string `env:"CHECK_PROMETHEUS_URL"` 26 | Headers []string 27 | Port int 28 | Info bool 29 | Insecure bool 30 | PReady bool 31 | Secure bool 32 | } 33 | 34 | const Copyright = ` 35 | Copyright (C) 2022 NETWAYS GmbH 36 | ` 37 | 38 | const License = ` 39 | Copyright (C) 2022 NETWAYS GmbH 40 | 41 | This program is free software: you can redistribute it and/or modify 42 | it under the terms of the GNU General Public License as published by 43 | the Free Software Foundation, either version 3 of the License, or 44 | (at your option) any later version. 45 | 46 | This program is distributed in the hope that it will be useful, 47 | but WITHOUT ANY WARRANTY; without even the implied warranty of 48 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 49 | GNU General Public License for more details. 50 | 51 | You should have received a copy of the GNU General Public License 52 | along with this program. If not, see https://www.gnu.org/licenses/. 53 | ` 54 | 55 | var cliConfig Config 56 | 57 | func (c *Config) NewClient() *client.Client { 58 | u := url.URL{ 59 | Scheme: "http", 60 | Host: c.Hostname + ":" + strconv.Itoa(c.Port), 61 | Path: c.URL, 62 | } 63 | 64 | if c.Secure { 65 | u.Scheme = "https" 66 | } 67 | 68 | // Create TLS configuration for default RoundTripper 69 | tlsConfig, err := config.NewTLSConfig(&config.TLSConfig{ 70 | InsecureSkipVerify: c.Insecure, 71 | CAFile: c.CAFile, 72 | KeyFile: c.KeyFile, 73 | CertFile: c.CertFile, 74 | }) 75 | 76 | if err != nil { 77 | check.ExitError(err) 78 | } 79 | 80 | var rt http.RoundTripper = &http.Transport{ 81 | Proxy: http.ProxyFromEnvironment, 82 | DialContext: (&net.Dialer{ 83 | Timeout: 30 * time.Second, 84 | KeepAlive: 30 * time.Second, 85 | }).DialContext, 86 | TLSHandshakeTimeout: 10 * time.Second, 87 | TLSClientConfig: tlsConfig, 88 | } 89 | 90 | // Using a Bearer Token for authentication 91 | if c.Bearer != "" { 92 | var t = config.NewInlineSecret(c.Bearer) 93 | rt = config.NewAuthorizationCredentialsRoundTripper("Bearer", t, rt) 94 | } 95 | 96 | // Using a BasicAuth for authentication 97 | if c.BasicAuth != "" { 98 | s := strings.Split(c.BasicAuth, ":") 99 | if len(s) != 2 { 100 | check.ExitError(errors.New("specify the user name and password for server authentication ")) 101 | } 102 | 103 | var u = config.NewInlineSecret(s[0]) 104 | 105 | var p = config.NewInlineSecret(s[1]) 106 | 107 | rt = config.NewBasicAuthRoundTripper(u, p, rt) 108 | } 109 | 110 | // If extra headers are set, parse them and add them to the request 111 | if len(c.Headers) > 0 { 112 | headers := make(map[string]string) 113 | 114 | for _, h := range c.Headers { 115 | head := strings.Split(h, ":") 116 | if len(head) == 2 { 117 | headers[strings.TrimSpace(head[0])] = strings.TrimSpace(head[1]) 118 | } 119 | } 120 | 121 | rt = client.NewHeadersRoundTripper(headers, rt) 122 | } 123 | 124 | return client.NewClient(u.String(), rt) 125 | } 126 | 127 | func (c *Config) timeoutContext() (context.Context, func()) { 128 | return context.WithTimeout(context.Background(), 10*time.Second) 129 | } 130 | -------------------------------------------------------------------------------- /testdata/unittest/alertDataset1.json: -------------------------------------------------------------------------------- 1 | { 2 | "status": "success", 3 | "data": { 4 | "groups": [ 5 | { 6 | "name": "Foo", 7 | "file": "alerts.yaml", 8 | "rules": [ 9 | { 10 | "state": "inactive", 11 | "name": "HostOutOfMemory", 12 | "query": "up", 13 | "duration": 120, 14 | "labels": { 15 | "severity": "critical" 16 | }, 17 | "annotations": { 18 | "description": "Foo", 19 | "summary": "Foo" 20 | }, 21 | "alerts": [], 22 | "health": "ok", 23 | "evaluationTime": 0.000553928, 24 | "lastEvaluation": "2022-11-24T14:08:17.597083058Z", 25 | "type": "alerting" 26 | } 27 | ], 28 | "interval": 10, 29 | "limit": 0, 30 | "evaluationTime": 0.000581212, 31 | "lastEvaluation": "2022-11-24T14:08:17.59706083Z" 32 | }, 33 | { 34 | "name": "SQL", 35 | "file": "alerts.yaml", 36 | "rules": [ 37 | { 38 | "state": "pending", 39 | "name": "SqlAccessDeniedRate", 40 | "query": "mysql", 41 | "duration": 17280000, 42 | "labels": { 43 | "severity": "warning" 44 | }, 45 | "annotations": { 46 | "description": "MySQL", 47 | "summary": "MySQL" 48 | }, 49 | "alerts": [ 50 | { 51 | "labels": { 52 | "alertname": "SqlAccessDeniedRate", 53 | "instance": "localhost", 54 | "job": "mysql", 55 | "severity": "warning" 56 | }, 57 | "annotations": { 58 | "description": "MySQL", 59 | "summary": "MySQL" 60 | }, 61 | "state": "pending", 62 | "activeAt": "2022-11-21T10:38:35.373483748Z", 63 | "value": "4.03448275862069e-01" 64 | } 65 | ], 66 | "health": "ok", 67 | "evaluationTime": 0.002909617, 68 | "lastEvaluation": "2022-11-24T14:08:25.375220595Z", 69 | "type": "alerting" 70 | } 71 | ], 72 | "interval": 10, 73 | "limit": 0, 74 | "evaluationTime": 0.003046259, 75 | "lastEvaluation": "2022-11-24T14:08:25.375096825Z" 76 | }, 77 | { 78 | "name": "TLS", 79 | "file": "alerts.yaml", 80 | "rules": [ 81 | { 82 | "state": "firing", 83 | "name": "BlackboxTLS", 84 | "query": "SSL", 85 | "duration": 0, 86 | "labels": { 87 | "severity": "critical" 88 | }, 89 | "annotations": { 90 | "description": "TLS", 91 | "summary": "TLS" 92 | }, 93 | "alerts": [ 94 | { 95 | "labels": { 96 | "alertname": "TLS", 97 | "instance": "https://localhost:443", 98 | "job": "blackbox", 99 | "severity": "critical" 100 | }, 101 | "annotations": { 102 | "description": "TLS", 103 | "summary": "TLS" 104 | }, 105 | "state": "firing", 106 | "activeAt": "2022-11-24T05:11:27.211699259Z", 107 | "value": "-6.065338210999966e+06" 108 | } 109 | ], 110 | "health": "ok", 111 | "evaluationTime": 0.000713955, 112 | "lastEvaluation": "2022-11-24T14:08:17.212720815Z", 113 | "type": "alerting" 114 | } 115 | ], 116 | "interval": 10, 117 | "limit": 0, 118 | "evaluationTime": 0.000738927, 119 | "lastEvaluation": "2022-11-24T14:08:17.212700182Z" 120 | } 121 | ] 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /contrib/icinga2-commands.conf: -------------------------------------------------------------------------------- 1 | template CheckCommand "prometheus" { 2 | command = [ PluginContribDir + "/check_prometheus" ] 3 | 4 | arguments = { 5 | "--hostname" = { 6 | value = "$prometheus_hostname$" 7 | description = "Hostname of the Prometheus server (default 'localhost')" 8 | } 9 | "--port" = { 10 | value = "$prometheus_port$" 11 | description = "Port of the Prometheus server (default 9090)" 12 | } 13 | "--secure" = { 14 | set_if = "$prometheus_secure$" 15 | description = "Use a HTTPS connection" 16 | } 17 | "--insecure" = { 18 | set_if = "$prometheus_insecure$" 19 | description = "Skip the verification of the server's TLS certificate" 20 | } 21 | "--bearer" = { 22 | value = "$prometheus_bearer$" 23 | description = "Specify the Bearer Token for server authentication" 24 | } 25 | "--user" = { 26 | value = "$prometheus_user$" 27 | description = "Specify the user name and password for server authentication " 28 | } 29 | "--ca-file" = { 30 | value = "$prometheus_ca_file$" 31 | description = "Specify the CA File for TLS authentication" 32 | } 33 | "--cert-file" = { 34 | value = "$prometheus_cert_file$" 35 | description = "Specify the Certificate File for TLS authentication" 36 | } 37 | "--key-file" = { 38 | value = "$prometheus_key_file$" 39 | description = "Specify the Key File for TLS authentication" 40 | } 41 | "--timeout" = { 42 | value = "$prometheus_timeout$" 43 | description = "Timeout in seconds for the CheckPlugin (default 30)" 44 | } 45 | } 46 | 47 | vars.prometheus_hostname = "localhost" 48 | vars.prometheus_port = 9090 49 | vars.prometheus_timeout = 30 50 | } 51 | 52 | object CheckCommand "prometheus-alert" { 53 | import "prometheus" 54 | 55 | command += [ "alert" ] 56 | 57 | arguments += { 58 | "--name" = { 59 | value = "$prometheus_alert$" 60 | repeat_key = true 61 | description = "The name of one or more specific alerts to check. This parameter can be repeated e.G.: '--name alert1 --name alert2' If no name is given, all alerts will be evaluated" 62 | } 63 | "--problems" = { 64 | value = "$prometheus_alert_problems$" 65 | description = "Display only alerts which status is not inactive/OK. Note that in combination with the --name flag this might result in no alerts being displayed" 66 | } 67 | } 68 | } 69 | 70 | object CheckCommand "prometheus-health" { 71 | import "prometheus" 72 | 73 | command += [ "health" ] 74 | 75 | arguments += { 76 | "--ready" = { 77 | value = "$prometheus_health_ready$" 78 | description = "Checks the readiness of an endpoint" 79 | } 80 | "--info" = { 81 | value = "$prometheus_health_info$" 82 | description = "Displays various build information properties about the Prometheus server" 83 | } 84 | } 85 | } 86 | 87 | object CheckCommand "prometheus-query" { 88 | import "prometheus" 89 | 90 | command += [ "query" ] 91 | 92 | arguments += { 93 | "--query" = { 94 | value = "$prometheus_query$" 95 | description = "An Prometheus query which will be performed and the value result will be evaluated" 96 | } 97 | "--warning" = { 98 | value = "$prometheus_query_warning$" 99 | description = "The warning threshold for a value (default '10')" 100 | } 101 | "--critical" = { 102 | value = "$prometheus_query_critical$" 103 | description = "The critical threshold for a value (default '20')" 104 | } 105 | } 106 | 107 | vars.prometheus_query_warning = "10" 108 | vars.prometheus_query_critical = "20" 109 | } 110 | -------------------------------------------------------------------------------- /internal/alert/alert_test.go: -------------------------------------------------------------------------------- 1 | package alert 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/NETWAYS/go-check" 8 | v1 "github.com/prometheus/client_golang/api/prometheus/v1" 9 | "github.com/prometheus/common/model" 10 | ) 11 | 12 | func TestGetStatus(t *testing.T) { 13 | 14 | testTime := time.Now() 15 | 16 | ar := v1.AlertingRule{ 17 | Alerts: []*v1.Alert{ 18 | { 19 | ActiveAt: testTime.UTC(), 20 | Annotations: model.LabelSet{ 21 | "summary": "High request latency", 22 | }, 23 | Labels: model.LabelSet{ 24 | "alertname": "HighRequestLatency", 25 | "severity": "page", 26 | }, 27 | State: v1.AlertStateFiring, 28 | Value: "1e+00", 29 | }, 30 | }, 31 | Annotations: model.LabelSet{ 32 | "summary": "High request latency", 33 | }, 34 | Labels: model.LabelSet{ 35 | "severity": "page", 36 | }, 37 | Duration: 600, 38 | Health: v1.RuleHealthGood, 39 | Name: "HighRequestLatency", 40 | Query: "job:request_latency_seconds:mean5m{job=\"myjob\"} > 0.5", 41 | LastError: "", 42 | EvaluationTime: 0.5, 43 | LastEvaluation: time.Date(2020, 5, 18, 15, 52, 53, 450311300, time.UTC), 44 | State: "firing", 45 | } 46 | 47 | r := Rule{ 48 | AlertingRule: ar, 49 | Alert: ar.Alerts[0], 50 | } 51 | 52 | actual := r.GetStatus() 53 | if actual != check.Critical { 54 | t.Error("\nActual: ", actual, "\nExpected: ", check.Critical) 55 | } 56 | 57 | r.AlertingRule.State = "pending" 58 | actual = r.GetStatus() 59 | if actual != check.Warning { 60 | t.Error("\nActual: ", actual, "\nExpected: ", check.Warning) 61 | } 62 | 63 | } 64 | 65 | func TestGetOutput(t *testing.T) { 66 | 67 | testTime := time.Now() 68 | 69 | ar := v1.AlertingRule{ 70 | Alerts: []*v1.Alert{ 71 | { 72 | ActiveAt: testTime.UTC(), 73 | Annotations: model.LabelSet{ 74 | "summary": "High request latency", 75 | }, 76 | Labels: model.LabelSet{ 77 | "alertname": "HighRequestLatency", 78 | "instance": "foo", 79 | "job": "bar", 80 | }, 81 | State: v1.AlertStateFiring, 82 | Value: "1e+00", 83 | }, 84 | }, 85 | Annotations: model.LabelSet{ 86 | "summary": "High request latency", 87 | }, 88 | Labels: model.LabelSet{ 89 | "severity": "page", 90 | }, 91 | Duration: 600, 92 | Health: v1.RuleHealthGood, 93 | Name: "HighRequestLatency", 94 | Query: "job:request_latency_seconds:mean5m{job=\"myjob\"} > 0.5", 95 | LastError: "", 96 | EvaluationTime: 0.5, 97 | LastEvaluation: time.Date(2020, 5, 18, 15, 52, 53, 450311300, time.UTC), 98 | State: "firing", 99 | } 100 | 101 | r := Rule{ 102 | AlertingRule: ar, 103 | Alert: ar.Alerts[0], 104 | } 105 | 106 | var expected string 107 | 108 | expected = `[HighRequestLatency] - Job: [bar] on Instance: [foo] is firing - value: 1.00 - {"alertname":"HighRequestLatency","instance":"foo","job":"bar"}` 109 | if r.GetOutput() != expected { 110 | t.Error("\nActual: ", r.GetOutput(), "\nExpected: ", expected) 111 | } 112 | 113 | r.AlertingRule.Alerts[0].Labels = model.LabelSet{ 114 | "alertname": "HighRequestLatency", 115 | } 116 | 117 | expected = `[HighRequestLatency] is firing - value: 1.00 - {"alertname":"HighRequestLatency"}` 118 | if r.GetOutput() != expected { 119 | t.Error("\nActual: ", r.GetOutput(), "\nExpected: ", expected) 120 | } 121 | 122 | r.AlertingRule.State = "inactive" 123 | 124 | expected = `[HighRequestLatency] is inactive - value: 1.00 - {"alertname":"HighRequestLatency"}` 125 | if r.GetOutput() != expected { 126 | t.Error("\nActual: ", r.GetOutput(), "\nExpected: ", expected) 127 | } 128 | 129 | r.Alert = nil 130 | expected = "[HighRequestLatency] is inactive" 131 | if r.GetOutput() != expected { 132 | t.Error("\nActual: ", r.GetOutput(), "\nExpected: ", expected) 133 | } 134 | } 135 | 136 | func TestFlattenRules(t *testing.T) { 137 | testTime := time.Now() 138 | 139 | rg := []v1.RuleGroup{ 140 | { 141 | Name: "example", 142 | File: "/rules.yaml", 143 | Interval: 60, 144 | Rules: []interface{}{ 145 | v1.AlertingRule{ 146 | Alerts: []*v1.Alert{ 147 | { 148 | ActiveAt: testTime.UTC(), 149 | Annotations: model.LabelSet{ 150 | "summary": "High request latency", 151 | }, 152 | Labels: model.LabelSet{ 153 | "alertname": "HighRequestLatency", 154 | "severity": "page", 155 | }, 156 | State: v1.AlertStateFiring, 157 | Value: "1e+00", 158 | }, 159 | }, 160 | Annotations: model.LabelSet{ 161 | "summary": "High request latency", 162 | }, 163 | Labels: model.LabelSet{ 164 | "severity": "page", 165 | }, 166 | Duration: 600, 167 | Health: v1.RuleHealthGood, 168 | Name: "HighRequestLatency", 169 | Query: "job:request_latency_seconds:mean5m{job=\"myjob\"} > 0.5", 170 | LastError: "", 171 | }, 172 | v1.RecordingRule{ 173 | Health: v1.RuleHealthGood, 174 | Name: "job:http_inprogress_requests:sum", 175 | Query: "sum(http_inprogress_requests) by (job)", 176 | LastError: "", 177 | }, 178 | }, 179 | }, 180 | } 181 | 182 | fr := FlattenRules(rg, nil) 183 | if len(fr) != 1 { 184 | t.Error("\nActual: ", fr) 185 | } 186 | 187 | } 188 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/NETWAYS/go-check v0.6.3 h1:Rcunu4JvlulbKjIef1H4pUiFO88atANpJ1YAqIwcJxI= 2 | github.com/NETWAYS/go-check v0.6.3/go.mod h1:8/GWnq8SirreAixgRmcp82JG16NnEl38rHq9phICy9s= 3 | github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= 4 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= 5 | github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= 6 | github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 7 | github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= 8 | github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= 9 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 10 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 11 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 12 | github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= 13 | github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= 14 | github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= 15 | github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= 16 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 17 | github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= 18 | github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 19 | github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= 20 | github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= 21 | github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= 22 | github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= 23 | github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= 24 | github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= 25 | github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= 26 | github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= 27 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 28 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 29 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 30 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= 31 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 32 | github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= 33 | github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= 34 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= 35 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= 36 | github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= 37 | github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= 38 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 39 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 40 | github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= 41 | github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= 42 | github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= 43 | github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= 44 | github.com/prometheus/common v0.67.4 h1:yR3NqWO1/UyO1w2PhUvXlGQs/PtFmoveVO0KZ4+Lvsc= 45 | github.com/prometheus/common v0.67.4/go.mod h1:gP0fq6YjjNCLssJCQp0yk4M8W6ikLURwkdd/YKtTbyI= 46 | github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= 47 | github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= 48 | github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= 49 | github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= 50 | github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= 51 | github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= 52 | github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= 53 | github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= 54 | github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= 55 | github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= 56 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 57 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 58 | github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= 59 | github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= 60 | go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= 61 | go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= 62 | go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= 63 | go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= 64 | go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= 65 | golang.org/x/net v0.46.0 h1:giFlY12I07fugqwPuWJi68oOnpfqFnJIJzaIIm2JVV4= 66 | golang.org/x/net v0.46.0/go.mod h1:Q9BGdFy1y4nkUwiLvT5qtyhAnEHgnQ/zd8PfU6nc210= 67 | golang.org/x/oauth2 v0.32.0 h1:jsCblLleRMDrxMN29H3z/k1KliIvpLgCkE6R8FXXNgY= 68 | golang.org/x/oauth2 v0.32.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= 69 | golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ= 70 | golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= 71 | golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k= 72 | golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM= 73 | google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= 74 | google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= 75 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 76 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= 77 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= 78 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 79 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 80 | -------------------------------------------------------------------------------- /cmd/health_test.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "net/http" 5 | "net/http/httptest" 6 | "net/url" 7 | "os/exec" 8 | "strings" 9 | "testing" 10 | ) 11 | 12 | func TestHealth_ConnectionRefused(t *testing.T) { 13 | 14 | cmd := exec.Command("go", "run", "../main.go", "health", "--port", "9999") 15 | out, _ := cmd.CombinedOutput() 16 | 17 | actual := string(out) 18 | expected := "[UNKNOWN] - could not get status: Get \"http://localhost:9999/-/healthy\": dial" 19 | 20 | if !strings.Contains(actual, expected) { 21 | t.Error("\nActual: ", actual, "\nExpected: ", expected) 22 | } 23 | } 24 | 25 | type HealthTest struct { 26 | name string 27 | server *httptest.Server 28 | args []string 29 | expected string 30 | } 31 | 32 | func TestHealthCmd(t *testing.T) { 33 | tests := []HealthTest{ 34 | { 35 | name: "health-ok", 36 | server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 37 | w.WriteHeader(http.StatusOK) 38 | w.Write([]byte(`Prometheus Server is Healthy.`)) 39 | })), 40 | args: []string{"run", "../main.go", "health"}, 41 | expected: "[OK] - states: ok=1\n\\_ [OK] Prometheus Server is Healthy.\n\n", 42 | }, 43 | { 44 | name: "health-ok-older-versions", 45 | server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 46 | w.WriteHeader(http.StatusOK) 47 | w.Write([]byte(`Prometheus is Healthy.`)) 48 | })), 49 | args: []string{"run", "../main.go", "health"}, 50 | expected: "[OK] - states: ok=1\n\\_ [OK] Prometheus is Healthy.\n\n", 51 | }, 52 | { 53 | name: "ready-ok", 54 | server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 55 | w.WriteHeader(http.StatusOK) 56 | w.Write([]byte(`Prometheus Server is Ready.`)) 57 | })), 58 | args: []string{"run", "../main.go", "health", "--ready"}, 59 | expected: "[OK] - states: ok=1\n\\_ [OK] Prometheus Server is Ready.\n\n", 60 | }, 61 | { 62 | name: "info-ok", 63 | server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 64 | w.WriteHeader(http.StatusOK) 65 | w.Write([]byte(`{"status":"success","data":{"version":"2.30.3","revision":"foo","branch":"HEAD","buildUser":"root@foo","buildDate":"20211005-16:10:52","goVersion":"go1.17.1"}}`)) 66 | })), 67 | args: []string{"run", "../main.go", "health", "--info"}, 68 | expected: "[OK] - states: ok=1\n\\_ [OK] Prometheus Server information\n\nVersion: 2.30.3\nBranch: HEAD\nBuildDate: 20211005-16:10:52\nBuildUser: root@foo\nRevision: foo\n\n", 69 | }, 70 | { 71 | name: "health-bearer-ok", 72 | server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 73 | token := r.Header.Get("Authorization") 74 | if token == "Bearer secret" { 75 | // Just for testing, this is now how to handle tokens properly 76 | w.WriteHeader(http.StatusOK) 77 | w.Write([]byte(`Prometheus Server is Healthy.`)) 78 | return 79 | } 80 | w.WriteHeader(http.StatusUnauthorized) 81 | w.Write([]byte(`The Authorization header wasn't set`)) 82 | })), 83 | args: []string{"run", "../main.go", "--bearer", "secret", "health"}, 84 | expected: "[OK] - states: ok=1\n\\_ [OK] Prometheus Server is Healthy.\n\n", 85 | }, 86 | { 87 | name: "health-bearer-unauthorized", 88 | server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 89 | token := r.Header.Get("Authorization") 90 | if token == "Bearer right-token" { 91 | // Just for testing, this is now how to handle BasicAuth properly 92 | w.WriteHeader(http.StatusOK) 93 | w.Write([]byte(`Prometheus Server is Healthy.`)) 94 | return 95 | } 96 | w.WriteHeader(http.StatusUnauthorized) 97 | w.Write([]byte(`Access Denied!`)) 98 | })), 99 | args: []string{"run", "../main.go", "--bearer", "wrong-token", "health"}, 100 | expected: "[CRITICAL] - states: critical=1\n\\_ [CRITICAL] Access Denied!\n\nexit status 2\n", 101 | }, 102 | { 103 | name: "health-basic-auth-ok", 104 | server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 105 | user, pass, ok := r.BasicAuth() 106 | if ok { 107 | // Just for testing, this is now how to handle BasicAuth properly 108 | if user == "username" && pass == "password" { 109 | w.WriteHeader(http.StatusOK) 110 | w.Write([]byte(`Prometheus Server is Healthy.`)) 111 | return 112 | } 113 | } 114 | w.WriteHeader(http.StatusUnauthorized) 115 | w.Write([]byte(`The Authorization header wasn't set`)) 116 | })), 117 | args: []string{"run", "../main.go", "--user", "username:password", "health"}, 118 | expected: "[OK] - states: ok=1\n\\_ [OK] Prometheus Server is Healthy.\n\n", 119 | }, 120 | { 121 | name: "health-basic-auth-unauthorized", 122 | server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 123 | user, pass, ok := r.BasicAuth() 124 | if ok { 125 | // Just for testing, this is now how to handle BasicAuth properly 126 | if user == "wrong" && pass == "kong" { 127 | w.WriteHeader(http.StatusOK) 128 | w.Write([]byte(`Prometheus Server is Healthy.`)) 129 | return 130 | } 131 | } 132 | w.WriteHeader(http.StatusUnauthorized) 133 | w.Write([]byte(`Access Denied!`)) 134 | })), 135 | args: []string{"run", "../main.go", "health"}, 136 | expected: "[CRITICAL] - states: critical=1\n\\_ [CRITICAL] Access Denied!\n\nexit status 2\n", 137 | }, 138 | { 139 | name: "health-basic-auth-wrong-use", 140 | server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 141 | w.WriteHeader(http.StatusUnauthorized) 142 | w.Write([]byte(`Access Denied!`)) 143 | })), 144 | args: []string{"run", "../main.go", "--user", "passwordmissing", "health"}, 145 | expected: "[UNKNOWN] - specify the user name and password for server authentication (*errors.errorString)\nexit status 3\n", 146 | }, 147 | { 148 | name: "health-extra-header", 149 | server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 150 | foobar := r.Header.Get("X-Foobar") 151 | if foobar == "Barfoo" { 152 | w.WriteHeader(http.StatusOK) 153 | w.Write([]byte(`Prometheus Server is Healthy.`)) 154 | return 155 | } 156 | w.WriteHeader(http.StatusUnauthorized) 157 | w.Write([]byte(`Wrong Header!`)) 158 | })), 159 | args: []string{"run", "../main.go", "--header", "X-Foobar: Barfoo", "health"}, 160 | expected: "[OK] - states: ok=1\n\\_ [OK] Prometheus Server is Healthy.\n\n", 161 | }, 162 | } 163 | 164 | for _, test := range tests { 165 | t.Run(test.name, func(t *testing.T) { 166 | defer test.server.Close() 167 | 168 | // We need the random Port extracted 169 | u, _ := url.Parse(test.server.URL) 170 | cmd := exec.Command("go", append(test.args, "--port", u.Port())...) 171 | out, _ := cmd.CombinedOutput() 172 | 173 | actual := string(out) 174 | 175 | if actual != test.expected { 176 | t.Errorf("\nActual:\n%#v\nExpected:\n%#v\n", actual, test.expected) 177 | } 178 | 179 | }) 180 | } 181 | } 182 | -------------------------------------------------------------------------------- /cmd/query.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "math" 7 | "strconv" 8 | "strings" 9 | "time" 10 | 11 | "github.com/NETWAYS/go-check" 12 | "github.com/NETWAYS/go-check/perfdata" 13 | goresult "github.com/NETWAYS/go-check/result" 14 | "github.com/prometheus/common/model" 15 | "github.com/spf13/cobra" 16 | ) 17 | 18 | type QueryConfig struct { 19 | RawQuery string 20 | Warning string 21 | Critical string 22 | ShowAll bool 23 | UnixTime bool 24 | } 25 | 26 | type User struct { 27 | Name string 28 | Occupation string 29 | } 30 | 31 | var cliQueryConfig QueryConfig 32 | 33 | var replacer = strings.NewReplacer("{", "_", "}", "", "\"", "", ",", "_", " ", "") 34 | 35 | func generateMetricOutput(metric string, value string) string { 36 | // Format the metric and RC output for console output 37 | return fmt.Sprintf(" %s - value: %s", metric, value) 38 | } 39 | 40 | type Number interface { 41 | ~int | ~int8 | ~int16 | ~int32 | ~int64 | ~uint | ~uint8 | ~uint16 | ~uint32 | ~uint64 | ~uintptr | ~float32 | ~float64 42 | } 43 | 44 | func generatePerfdata[T Number](metric string, value T, warning, critical *check.Threshold) perfdata.Perfdata { 45 | // We trim the trailing "} from the string, so that the Perfdata won't have a trailing _ 46 | return perfdata.Perfdata{ 47 | Label: replacer.Replace(metric), 48 | Value: value, 49 | Warn: warning, 50 | Crit: critical, 51 | } 52 | } 53 | 54 | var queryCmd = &cobra.Command{ 55 | Use: "query", 56 | Short: "Checks the status of a Prometheus query", 57 | Long: `Checks the status of a Prometheus query and evaluates the result of the alert. 58 | Note: Time range values e.G. 'go_memstats_alloc_bytes_total[0s]' only the latest value will be evaluated, other values will be ignored!`, 59 | Example: ` 60 | $ check_prometheus query -q 'go_gc_duration_seconds_count' -c 5000 -w 2000 61 | CRITICAL - 2 Metrics: 1 Critical - 0 Warning - 1 Ok 62 | \_[OK] go_gc_duration_seconds_count{instance="localhost:9090", job="prometheus"} - value: 1599 63 | \_[CRITICAL] go_gc_duration_seconds_count{instance="node-exporter:9100", job="node-exporter"} - value: 79610 64 | | value_go_gc_duration_seconds_count_localhost:9090_prometheus=1599 value_go_gc_duration_seconds_count_node-exporter:9100_node-exporter=79610`, 65 | PreRun: func(_ *cobra.Command, _ []string) { 66 | if cliQueryConfig.Warning == "" || cliQueryConfig.Critical == "" { 67 | check.ExitError(errors.New("please specify warning and critical thresholds")) 68 | } 69 | }, 70 | Run: func(_ *cobra.Command, _ []string) { 71 | crit, err := check.ParseThreshold(cliQueryConfig.Critical) 72 | if err != nil { 73 | check.ExitError(err) 74 | } 75 | 76 | warn, err := check.ParseThreshold(cliQueryConfig.Warning) 77 | if err != nil { 78 | check.ExitError(err) 79 | } 80 | 81 | c := cliConfig.NewClient() 82 | err = c.Connect() 83 | if err != nil { 84 | check.ExitError(err) 85 | } 86 | 87 | ctx, cancel := cliConfig.timeoutContext() 88 | defer cancel() 89 | 90 | result, warnings, err := c.API.Query(ctx, cliQueryConfig.RawQuery, time.Now()) 91 | 92 | if err != nil { 93 | if strings.Contains(err.Error(), "unmarshalerDecoder: unexpected value type \"string\"") { 94 | err = errors.New("string value results are not supported") 95 | } 96 | check.ExitError(err) 97 | } 98 | 99 | overall := goresult.Overall{} 100 | 101 | switch result.Type() { 102 | default: 103 | check.ExitError(errors.New("none value results are not supported")) 104 | // Scalar - a simple numeric floating point value 105 | case model.ValScalar: 106 | check.ExitError(errors.New("scalar value results are not supported")) 107 | case model.ValNone: 108 | check.ExitError(errors.New("none value results are not supported")) 109 | case model.ValString: 110 | // String - a simple string value; currently unused 111 | check.ExitError(errors.New("string value results are not supported")) 112 | case model.ValVector: 113 | // Instant vector - a set of time series containing a single sample for each time series, all sharing the same timestamp 114 | vectorVal := result.(model.Vector) 115 | 116 | // Set initial capacity to reduce memory allocations 117 | for _, sample := range vectorVal { 118 | 119 | numberValue := float64(sample.Value) 120 | partial := goresult.NewPartialResult() 121 | 122 | if crit.DoesViolate(numberValue) { 123 | _ = partial.SetState(check.Critical) 124 | } else if warn.DoesViolate(numberValue) { 125 | _ = partial.SetState(check.Warning) 126 | } else { 127 | _ = partial.SetState(check.OK) 128 | } 129 | 130 | // Format the metric and RC output for console output 131 | partial.Output = generateMetricOutput(sample.Metric.String(), sample.Value.String()) 132 | 133 | // Generate Perfdata from API return 134 | if math.IsInf(numberValue, 0) || math.IsNaN(numberValue) { 135 | continue 136 | } 137 | 138 | perf := generatePerfdata(sample.Metric.String(), numberValue, warn, crit) 139 | partial.Perfdata.Add(&perf) 140 | overall.AddSubcheck(partial) 141 | } 142 | 143 | case model.ValMatrix: 144 | // Range vector - a set of time series containing a range of data points over time for each time series -> Matrix 145 | // An example query for a matrix 'go_goroutines{job="prometheus"}[5m]' 146 | 147 | // Note: Only the latest value will be evaluated, other values will be ignored! 148 | matrixVal := result.(model.Matrix) 149 | 150 | for _, samplestream := range matrixVal { 151 | samplepair := samplestream.Values[len(samplestream.Values)-1] 152 | 153 | numberValue := float64(samplepair.Value) 154 | 155 | partial := goresult.NewPartialResult() 156 | 157 | if crit.DoesViolate(numberValue) { 158 | _ = partial.SetState(check.Critical) 159 | } else if warn.DoesViolate(numberValue) { 160 | _ = partial.SetState(check.Warning) 161 | } else { 162 | _ = partial.SetState(check.OK) 163 | } 164 | 165 | // Format the metric and RC output for console output 166 | partial.Output = generateMetricOutput(samplepair.String(), samplepair.Value.String()) 167 | 168 | valueString := samplepair.Value.String() 169 | 170 | valueNumber, err := strconv.ParseFloat(valueString, 64) 171 | if err == nil { 172 | pd := generatePerfdata(samplestream.Metric.String(), valueNumber, warn, crit) 173 | 174 | // Generate Perfdata from API return 175 | if !math.IsInf(numberValue, 0) && !math.IsNaN(numberValue) { 176 | partial.Perfdata.Add(&pd) 177 | } 178 | } 179 | 180 | overall.AddSubcheck(partial) 181 | } 182 | } 183 | 184 | if len(warnings) != 0 { 185 | appendum := fmt.Sprintf("HTTP Warnings: %v", strings.Join(warnings, ", ")) 186 | overall.Summary = overall.GetOutput() + appendum 187 | } 188 | check.ExitRaw(overall.GetStatus(), overall.GetOutput()) 189 | }, 190 | } 191 | 192 | func init() { 193 | rootCmd.AddCommand(queryCmd) 194 | fs := queryCmd.Flags() 195 | fs.StringVarP(&cliQueryConfig.RawQuery, "query", "q", "", 196 | "An Prometheus query which will be performed and the value result will be evaluated") 197 | fs.BoolVar(&cliQueryConfig.ShowAll, "show-all", false, 198 | "Displays all metrics regardless of the status") 199 | 200 | _ = fs.MarkHidden("show-all") 201 | 202 | fs.StringVarP(&cliQueryConfig.Warning, "warning", "w", "10", 203 | "The warning threshold for a value") 204 | fs.StringVarP(&cliQueryConfig.Critical, "critical", "c", "20", 205 | "The critical threshold for a value") 206 | 207 | fs.SortFlags = false 208 | _ = queryCmd.MarkFlagRequired("query") 209 | } 210 | -------------------------------------------------------------------------------- /cmd/alert.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "regexp" 7 | "strings" 8 | 9 | "github.com/NETWAYS/check_prometheus/internal/alert" 10 | "github.com/NETWAYS/go-check" 11 | "github.com/NETWAYS/go-check/perfdata" 12 | "github.com/NETWAYS/go-check/result" 13 | "github.com/spf13/cobra" 14 | ) 15 | 16 | type AlertConfig struct { 17 | AlertName []string 18 | Group []string 19 | ExcludeAlerts []string 20 | ProblemsOnly bool 21 | NoAlertsState string 22 | } 23 | 24 | var cliAlertConfig AlertConfig 25 | 26 | func contains(s string, list []string) bool { 27 | // Tiny helper to see if a string is in a list of strings 28 | for _, elem := range list { 29 | if s == elem { 30 | return true 31 | } 32 | } 33 | 34 | return false 35 | } 36 | 37 | var alertCmd = &cobra.Command{ 38 | Use: "alert", 39 | Short: "Checks the status of a Prometheus alert", 40 | Long: `Checks the status of a Prometheus alert and evaluates the status of the alert: 41 | firing = 2 42 | pending = 1 43 | inactive = 0`, 44 | Example: ` 45 | $ check_prometheus alert --name "PrometheusAlertmanagerJobMissing" 46 | CRITICAL - 1 Alerts: 1 Firing - 0 Pending - 0 Inactive 47 | \_[CRITICAL] [PrometheusAlertmanagerJobMissing] - Job: [alertmanager] is firing - value: 1.00 48 | | firing=1 pending=0 inactive=0 49 | 50 | $ check_prometheus alert --name "PrometheusAlertmanagerJobMissing" --name "PrometheusTargetMissing" 51 | CRITICAL - 2 Alerts: 1 Firing - 0 Pending - 1 Inactive 52 | \_[OK] [PrometheusTargetMissing] is inactive 53 | \_[CRITICAL] [PrometheusAlertmanagerJobMissing] - Job: [alertmanager] is firing - value: 1.00 54 | | total=2 firing=1 pending=0 inactive=1`, 55 | Run: func(_ *cobra.Command, _ []string) { 56 | // Convert --no-alerts-state to integer and validate input 57 | noAlertsState, err := convertStateToInt(cliAlertConfig.NoAlertsState) 58 | if err != nil { 59 | check.ExitError(fmt.Errorf("invalid value for --no-alerts-state: %s", cliAlertConfig.NoAlertsState)) 60 | } 61 | 62 | var ( 63 | counterFiring int 64 | counterPending int 65 | counterInactive int 66 | ) 67 | 68 | c := cliConfig.NewClient() 69 | err = c.Connect() 70 | 71 | if err != nil { 72 | check.ExitError(err) 73 | } 74 | 75 | ctx, cancel := cliConfig.timeoutContext() 76 | defer cancel() 77 | // We use the Rules endpoint since it contains 78 | // the state of inactive Alert Rules, unlike the Alert endpoint 79 | // Search requested Alert in all Groups and all Rules 80 | alerts, err := c.API.Rules(ctx) 81 | if err != nil { 82 | check.ExitError(err) 83 | } 84 | 85 | // Get all rules from all groups into a single list 86 | rules := alert.FlattenRules(alerts.Groups, cliAlertConfig.Group) 87 | 88 | // If there are no rules we can exit early 89 | if len(rules) == 0 { 90 | // Just an empty PerfdataList to have consistent perfdata output 91 | pdlist := perfdata.PerfdataList{ 92 | {Label: "total", Value: 0}, 93 | {Label: "firing", Value: 0}, 94 | {Label: "pending", Value: 0}, 95 | {Label: "inactive", Value: 0}, 96 | } 97 | 98 | // Since the user is expecting the state of a certain alert and 99 | // it that is not present it might be noteworthy. 100 | if cliAlertConfig.AlertName != nil { 101 | check.ExitRaw(check.Unknown, "No such alert defined", "|", pdlist.String()) 102 | } 103 | check.ExitRaw(noAlertsState, "No alerts defined", "|", pdlist.String()) 104 | } 105 | 106 | // Set initial capacity to reduce memory allocations 107 | var l int 108 | for _, rl := range rules { 109 | l *= len(rl.AlertingRule.Alerts) 110 | } 111 | 112 | var overall result.Overall 113 | 114 | for _, rl := range rules { 115 | 116 | // If it's not the Alert we're looking for, Skip! 117 | if cliAlertConfig.AlertName != nil { 118 | if !contains(rl.AlertingRule.Name, cliAlertConfig.AlertName) { 119 | continue 120 | } 121 | } 122 | 123 | // Skip inactive alerts if flag is set 124 | if len(rl.AlertingRule.Alerts) == 0 && cliAlertConfig.ProblemsOnly { 125 | continue 126 | } 127 | 128 | alertMatched, regexErr := matches(rl.AlertingRule.Name, cliAlertConfig.ExcludeAlerts) 129 | 130 | if regexErr != nil { 131 | check.ExitRaw(check.Unknown, "Invalid regular expression provided:", regexErr.Error()) 132 | } 133 | 134 | if alertMatched { 135 | // If the alert matches a regex from the list we can skip it. 136 | continue 137 | } 138 | 139 | // Handle Inactive Alerts 140 | if len(rl.AlertingRule.Alerts) == 0 { 141 | // Counting states for perfdata 142 | switch rl.GetStatus() { 143 | case 0: 144 | counterInactive++ 145 | case 1: 146 | counterPending++ 147 | case 2: 148 | counterFiring++ 149 | } 150 | 151 | sc := result.NewPartialResult() 152 | 153 | _ = sc.SetState(rl.GetStatus()) 154 | sc.Output = rl.GetOutput() 155 | overall.AddSubcheck(sc) 156 | } 157 | 158 | // Handle active alerts 159 | if len(rl.AlertingRule.Alerts) > 0 { 160 | // Handle Pending or Firing Alerts 161 | for _, alert := range rl.AlertingRule.Alerts { 162 | // Counting states for perfdata 163 | switch rl.GetStatus() { 164 | case 0: 165 | counterInactive++ 166 | case 1: 167 | counterPending++ 168 | case 2: 169 | counterFiring++ 170 | } 171 | 172 | sc := result.NewPartialResult() 173 | 174 | _ = sc.SetState(rl.GetStatus()) 175 | // Set the alert in the internal Type to generate the output 176 | rl.Alert = alert 177 | sc.Output = rl.GetOutput() 178 | overall.AddSubcheck(sc) 179 | } 180 | } 181 | } 182 | 183 | counterAlert := counterFiring + counterPending + counterInactive 184 | 185 | perfList := perfdata.PerfdataList{ 186 | {Label: "total", Value: counterAlert}, 187 | {Label: "firing", Value: counterFiring}, 188 | {Label: "pending", Value: counterPending}, 189 | {Label: "inactive", Value: counterInactive}, 190 | } 191 | 192 | // When there are no alerts we add an empty PartialResult just to have consistent output 193 | if len(overall.PartialResults) == 0 { 194 | sc := result.NewPartialResult() 195 | // We already make sure it's valid 196 | //nolint: errcheck 197 | sc.SetDefaultState(noAlertsState) 198 | sc.Output = "No alerts retrieved" 199 | overall.AddSubcheck(sc) 200 | } 201 | 202 | overall.PartialResults[0].Perfdata = append(overall.PartialResults[0].Perfdata, perfList...) 203 | 204 | overall.Summary = fmt.Sprintf("%d Alerts: %d Firing - %d Pending - %d Inactive", 205 | counterAlert, 206 | counterFiring, 207 | counterPending, 208 | counterInactive) 209 | 210 | check.ExitRaw(overall.GetStatus(), overall.GetOutput()) 211 | }, 212 | } 213 | 214 | func init() { 215 | rootCmd.AddCommand(alertCmd) 216 | 217 | fs := alertCmd.Flags() 218 | 219 | fs.StringVarP(&cliAlertConfig.NoAlertsState, "no-alerts-state", "T", "OK", "State to assign when no alerts are found (0, 1, 2, 3, OK, WARNING, CRITICAL, UNKNOWN). If not set this defaults to OK") 220 | 221 | fs.StringArrayVar(&cliAlertConfig.ExcludeAlerts, "exclude-alert", []string{}, "Alerts to ignore. Can be used multiple times and supports regex.") 222 | 223 | fs.StringSliceVarP(&cliAlertConfig.AlertName, "name", "n", nil, 224 | "The name of one or more specific alerts to check."+ 225 | "\nThis parameter can be repeated e.G.: '--name alert1 --name alert2'"+ 226 | "\nIf no name is given, all alerts will be evaluated") 227 | 228 | fs.StringSliceVarP(&cliAlertConfig.Group, "group", "g", nil, 229 | "The name of one or more specific groups to check for alerts."+ 230 | "\nThis parameter can be repeated e.G.: '--group group1 --group group2'"+ 231 | "\nIf no group is given, all groups will be scanned for alerts") 232 | 233 | fs.BoolVarP(&cliAlertConfig.ProblemsOnly, "problems", "P", false, 234 | "Display only alerts which status is not inactive/OK. Note that in combination with the --name flag this might result in no alerts being displayed") 235 | } 236 | 237 | // Function to convert state to integer. 238 | func convertStateToInt(state string) (int, error) { 239 | state = strings.ToUpper(state) 240 | switch state { 241 | case "OK", "0": 242 | return check.OK, nil 243 | case "WARNING", "1": 244 | return check.Warning, nil 245 | case "CRITICAL", "2": 246 | return check.Critical, nil 247 | case "UNKNOWN", "3": 248 | return check.Unknown, nil 249 | default: 250 | return check.Unknown, errors.New("invalid state") 251 | } 252 | } 253 | 254 | // Matches a list of regular expressions against a string. 255 | func matches(input string, regexToExclude []string) (bool, error) { 256 | for _, regex := range regexToExclude { 257 | re, err := regexp.Compile(regex) 258 | 259 | if err != nil { 260 | return false, err 261 | } 262 | 263 | if re.MatchString(input) { 264 | return true, nil 265 | } 266 | } 267 | 268 | return false, nil 269 | } 270 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # check_prometheus 2 | 3 | An Icinga check plugin to check Prometheus. 4 | 5 | ## Usage 6 | 7 | ```bash 8 | Usage: 9 | check_prometheus [flags] 10 | check_prometheus [command] 11 | 12 | Available Commands: 13 | alert Checks the status of a Prometheus alert 14 | health Checks the health or readiness status of the Prometheus server 15 | query Checks the status of a Prometheus query 16 | 17 | Flags: 18 | -H, --hostname string Hostname of the Prometheus server (CHECK_PROMETHEUS_HOSTNAME) (default "localhost") 19 | -p, --port int Port of the Prometheus server (default 9090) 20 | -U, --url string URL/Path to append to the Promethes Hostname (CHECK_PROMETHEUS_URL) (default "/") 21 | -s, --secure Use a HTTPS connection 22 | -i, --insecure Skip the verification of the server's TLS certificate 23 | -b, --bearer string Specify the Bearer Token for server authentication (CHECK_PROMETHEUS_BEARER) 24 | -u, --user string Specify the user name and password for server authentication (CHECK_PROMETHEUS_BASICAUTH) 25 | --ca-file string Specify the CA File for TLS authentication (CHECK_PROMETHEUS_CA_FILE) 26 | --cert-file string Specify the Certificate File for TLS authentication (CHECK_PROMETHEUS_CERT_FILE) 27 | --key-file string Specify the Key File for TLS authentication (CHECK_PROMETHEUS_KEY_FILE) 28 | -t, --timeout int Timeout in seconds for the CheckPlugin (default 30) 29 | --header strings Additional HTTP header to include in the request. Can be used multiple times. 30 | Keys and values are separated by a colon (--header "X-Custom: example"). 31 | -h, --help help for check_prometheus 32 | -v, --version version for check_prometheus 33 | ``` 34 | 35 | The check plugin respects the environment variables `HTTP_PROXY`, `HTTPS_PROXY` and `NO_PROXY`. 36 | 37 | Various flags can be set with environment variables, refer to the help to see which flags. 38 | 39 | In the case Prometheus runs behind a reverse proxy, the `--url` parameter can be used: 40 | 41 | ```bash 42 | # https://monitoring.example.com:443/subpath 43 | 44 | $ check_prometheus health -H 'monitoring.example.com' --port 443 --secure --url /subpath 45 | OK - Prometheus Server is Healthy. | statuscode=200 46 | ``` 47 | 48 | ### Health 49 | 50 | Checks the health or readiness status of the Prometheus server. 51 | 52 | * `Health`: Checks the health of an endpoint, which returns OK if the Prometheus server is healthy. 53 | * `Ready`: Checks the readiness of an endpoint, which returns OK if the Prometheus server is ready to serve traffic (i.e. respond to queries). 54 | 55 | ```bash 56 | Usage: 57 | check_prometheus health [flags] 58 | 59 | Examples: 60 | $ check_prometheus health --hostname 'localhost' --port 9090 --insecure 61 | OK - Prometheus Server is Healthy. | statuscode=200 62 | 63 | Flags: 64 | -r, --ready Checks the readiness of an endpoint 65 | -I, --info Displays various build information properties about the Prometheus server 66 | -h, --help help for health 67 | ``` 68 | 69 | ```bash 70 | $ check_prometheus health --hostname 'localhost' --port 9090 --insecure 71 | OK - Prometheus Server is Healthy. | statuscode=200 72 | 73 | $ check_prometheus health --ready 74 | OK - Prometheus Server is Ready. | statuscode=200 75 | ``` 76 | 77 | ### Query 78 | 79 | Checks the status of a Prometheus query and evaluates the result of the alert. 80 | The warning and critical support thresholds in the common Nagios format (e.g. `~:10`). 81 | 82 | >Note: Time range values e.G. 'go_memstats_alloc_bytes_total[10s]', only the latest value will be evaluated, other values will be ignored! 83 | 84 | ```bash 85 | Usage: 86 | check_prometheus query [flags] 87 | 88 | Examples: 89 | $ check_prometheus query -q 'go_gc_duration_seconds_count' -c 5000 -w 2000 90 | CRITICAL - 2 Metrics: 1 Critical - 0 Warning - 1 Ok 91 | \_[OK] go_gc_duration_seconds_count{instance="localhost:9090", job="prometheus"} - value: 1599 92 | \_[CRITICAL] go_gc_duration_seconds_count{instance="node-exporter:9100", job="node-exporter"} - value: 79610 93 | | value_go_gc_duration_seconds_count_localhost:9090_prometheus=1599 value_go_gc_duration_seconds_count_node-exporter:9100_node-exporter=79610 94 | 95 | Flags: 96 | -q, --query string An Prometheus query which will be performed and the value result will be evaluated 97 | -w, --warning string The warning threshold for a value (default "10") 98 | -c, --critical string The critical threshold for a value (default "20") 99 | -h, --help help for query 100 | ``` 101 | 102 | #### Checking a single metric with ONE direct vector result 103 | 104 | ```bash 105 | $ check_prometheus query -q 'go_goroutines{job="prometheus"}' -c 40 -w 27 106 | WARNING - 1 Metrics: 0 Critical - 1 Warning - 0 Ok 107 | \_[WARNING] go_goroutines{instance="localhost:9090", job="prometheus"} - value: 37 108 | | value_go_goroutines_localhost:9090_prometheus=37 109 | ``` 110 | 111 | #### Checking a single metric with multiple vector results 112 | 113 | ```bash 114 | $ check_prometheus query -q 'go_goroutines' -c 40 -w 27 115 | WARNING - 2 Metrics: 0 Critical - 1 Warning - 1 Ok 116 | \_[WARNING] go_goroutines{instance="localhost:9090", job="prometheus"} - value: 37 117 | \_[OK] go_goroutines{instance="node-exporter:9100", job="node-exporter"} - value: 7 118 | | value_go_goroutines_localhost:9090_prometheus=37 value_go_goroutines_node-exporter:9100_node-exporter=7 119 | ``` 120 | 121 | #### Checking a time series matrix result 122 | 123 | Hint: Currently only the latest value will be evaluated, other values will be ignored. 124 | 125 | ```bash 126 | $ check_prometheus query -q 'go_goroutines{job="prometheus"}[10s]' -c5 -w 10 127 | CRITICAL - 1 Metrics: 1 Critical - 0 Warning - 0 Ok 128 | \_[CRITICAL] go_goroutines{instance="localhost:9090", job="prometheus"} - value: 37 129 | | value_go_goroutines_localhost:9090_prometheus=37 130 | 131 | $ check_prometheus query -q 'go_goroutines[10s]' -c 50 -w 40 132 | OK - 2 Metrics OK | value_go_goroutines_localhost:9090_prometheus=37 value_go_goroutines_node-exporter:9100_node-exporter=7 133 | ``` 134 | 135 | ### Alert 136 | 137 | Checks the status of a Prometheus alert and evaluates the status of the alert. 138 | 139 | ```bash 140 | Usage: 141 | check_prometheus alert [flags] 142 | 143 | Examples: 144 | $ check_prometheus alert --name "PrometheusAlertmanagerJobMissing" 145 | CRITICAL - 1 Alerts: 1 Firing - 0 Pending - 0 Inactive 146 | \_[CRITICAL] [PrometheusAlertmanagerJobMissing] - Job: [alertmanager] is firing - value: 1.00 147 | | firing=1 pending=0 inactive=0 148 | 149 | $ check_prometheus a alert --name "PrometheusAlertmanagerJobMissing" --name "PrometheusTargetMissing" 150 | CRITICAL - 2 Alerts: 1 Firing - 0 Pending - 1 Inactive 151 | \_[OK] [PrometheusTargetMissing] is inactive 152 | \_[CRITICAL] [PrometheusAlertmanagerJobMissing] - Job: [alertmanager] is firing - value: 1.00 153 | | total=2 firing=1 pending=0 inactive=1 154 | 155 | Flags: 156 | --exclude-alert stringArray Alerts to ignore. Can be used multiple times and supports regex. 157 | -h, --help help for alert 158 | -n, --name strings The name of one or more specific alerts to check. 159 | This parameter can be repeated e.G.: '--name alert1 --name alert2' 160 | If no name is given, all alerts will be evaluated 161 | -g, --group strings The name of one or more specific groups to check. 162 | This parameter can be repeated e.G.: '--group group1 --group group2' 163 | If no group is given, all groups will be scanned for alerts 164 | -T, --no-alerts-state string State to assign when no alerts are found (0, 1, 2, 3, OK, WARNING, CRITICAL, UNKNOWN). If not set this defaults to OK (default "OK") 165 | -P, --problems Display only alerts which status is not inactive/OK. Note that in combination with the --name flag this might result in no alerts being displayed 166 | ``` 167 | 168 | #### Checking all defined alerts 169 | 170 | ```bash 171 | $ check_prometheus alert 172 | CRITICAL - 6 Alerts: 3 Firing - 0 Pending - 3 Inactive 173 | \_[OK] [PrometheusTargetMissing] is inactive 174 | \_[CRITICAL] [PrometheusAlertmanagerJobMissing] - Job: [alertmanager] is firing - value: 1.00 175 | \_[OK] [HostOutOfMemory] - Job: [alertmanager] 176 | \_[OK] [HostHighCpuLoad] - Job: [alertmanager] 177 | \_[CRITICAL] [HighResultLatency] - Job: [prometheus] on Instance: [localhost:9090] is firing - value: 11.00 178 | \_[CRITICAL] [HighResultLatency] - Job: [node-exporter] on Instance: [node-exporter:9100] is firing - value: 10.00 179 | | total=6 firing=3 pending=0 inactive=3 180 | 181 | ``` 182 | 183 | #### Checking multiple alerts 184 | 185 | ```bash 186 | $ check_prometheus alert --name "HostHighCpuLoad" --name "HighResultLatency" 187 | CRITICAL - 3 Alerts: 2 Firing - 0 Pending - 1 Inactive 188 | \_[OK] [HostHighCpuLoad] is inactive 189 | \_[CRITICAL] [HighResultLatency] - Job: [prometheus] on Instance: [localhost:9090] is firing - value: 11.00 190 | \_[CRITICAL] [HighResultLatency] - Job: [node-exporter] on Instance: [node-exporter:9100] is firing - value: 10.00 191 | | total=3 firing=2 pending=0 inactive=1 192 | ``` 193 | 194 | ```bash 195 | $ check_prometheus alert --name "HostHighCpuLoad" --name "PrometheusTargetMissing" 196 | OK - Alerts inactive | total=2 firing=0 pending=0 inactive=2 197 | ``` 198 | 199 | ## License 200 | 201 | Copyright (c) 2022 [NETWAYS GmbH](mailto:info@netways.de) 202 | 203 | This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public 204 | License as published by the Free Software Foundation, either version 2 of the License, or 205 | (at your option) any later version. 206 | 207 | This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied 208 | warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. 209 | 210 | You should have received a copy of the GNU General Public License along with this program. If not, 211 | see [gnu.org/licenses](https://www.gnu.org/licenses/). 212 | -------------------------------------------------------------------------------- /cmd/alert_test.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "net/http" 5 | "net/http/httptest" 6 | "net/url" 7 | "os/exec" 8 | "strings" 9 | "testing" 10 | ) 11 | 12 | func TestAlert_ConnectionRefused(t *testing.T) { 13 | 14 | cmd := exec.Command("go", "run", "../main.go", "alert", "--port", "9999") 15 | out, _ := cmd.CombinedOutput() 16 | 17 | actual := string(out) 18 | expected := "[UNKNOWN] - Get \"http://localhost:9999/api/v1/rules\"" 19 | 20 | if !strings.Contains(actual, expected) { 21 | t.Error("\nActual: ", actual, "\nExpected: ", expected) 22 | } 23 | } 24 | 25 | type AlertTest struct { 26 | name string 27 | server *httptest.Server 28 | args []string 29 | expected string 30 | } 31 | 32 | func TestAlertCmd(t *testing.T) { 33 | 34 | alertTestDataSet1 := "../testdata/unittest/alertDataset1.json" 35 | 36 | alertTestDataSet2 := "../testdata/unittest/alertDataset2.json" 37 | 38 | alertTestDataSet3 := "../testdata/unittest/alertDataset3.json" 39 | 40 | alertTestDataSet4 := "../testdata/unittest/alertDataset4.json" 41 | 42 | tests := []AlertTest{ 43 | { 44 | name: "alert-none", 45 | server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 46 | w.WriteHeader(http.StatusOK) 47 | w.Write([]byte(`{"status":"success","data":{"groups":[]}}`)) 48 | })), 49 | args: []string{"run", "../main.go", "alert"}, 50 | expected: "[OK] - No alerts defined | total=0 firing=0 pending=0 inactive=0\n", 51 | }, 52 | { 53 | name: "alert-none-with-problems", 54 | server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 55 | w.WriteHeader(http.StatusOK) 56 | w.Write([]byte(`{"status":"success","data":{"groups":[]}}`)) 57 | })), 58 | args: []string{"run", "../main.go", "alert", "--problems"}, 59 | expected: "[OK] - No alerts defined | total=0 firing=0 pending=0 inactive=0\n", 60 | }, 61 | { 62 | name: "alert-none-with-no-state", 63 | server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 64 | w.WriteHeader(http.StatusOK) 65 | w.Write([]byte(`{"status":"success","data":{"groups":[]}}`)) 66 | })), 67 | args: []string{"run", "../main.go", "alert", "--no-alerts-state", "3"}, 68 | expected: "[UNKNOWN] - No alerts defined | total=0 firing=0 pending=0 inactive=0\nexit status 3\n", 69 | }, 70 | { 71 | name: "alert-none-with-name", 72 | server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 73 | w.WriteHeader(http.StatusOK) 74 | w.Write([]byte(`{"status":"success","data":{"groups":[]}}`)) 75 | })), 76 | args: []string{"run", "../main.go", "alert", "--name", "MyPreciousAlert"}, 77 | expected: "[UNKNOWN] - No such alert defined | total=0 firing=0 pending=0 inactive=0\nexit status 3\n", 78 | }, 79 | { 80 | name: "alert-default", 81 | server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 82 | w.WriteHeader(http.StatusOK) 83 | w.Write(loadTestdata(alertTestDataSet1)) 84 | })), 85 | args: []string{"run", "../main.go", "alert"}, 86 | expected: `[CRITICAL] - 3 Alerts: 1 Firing - 1 Pending - 1 Inactive 87 | \_ [OK] [HostOutOfMemory] is inactive 88 | \_ [WARNING] [SqlAccessDeniedRate] - Job: [mysql] on Instance: [localhost] is pending - value: 0.40 - {"alertname":"SqlAccessDeniedRate","instance":"localhost","job":"mysql","severity":"warning"} 89 | \_ [CRITICAL] [BlackboxTLS] - Job: [blackbox] on Instance: [https://localhost:443] is firing - value: -6065338.00 - {"alertname":"TLS","instance":"https://localhost:443","job":"blackbox","severity":"critical"} 90 | |total=3 firing=1 pending=1 inactive=1 91 | 92 | exit status 2 93 | `, 94 | }, 95 | { 96 | name: "alert-problems-only", 97 | server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 98 | w.WriteHeader(http.StatusOK) 99 | w.Write(loadTestdata(alertTestDataSet1)) 100 | })), 101 | args: []string{"run", "../main.go", "alert", "--problems"}, 102 | expected: `[CRITICAL] - 2 Alerts: 1 Firing - 1 Pending - 0 Inactive 103 | \_ [WARNING] [SqlAccessDeniedRate] - Job: [mysql] on Instance: [localhost] is pending - value: 0.40 - {"alertname":"SqlAccessDeniedRate","instance":"localhost","job":"mysql","severity":"warning"} 104 | \_ [CRITICAL] [BlackboxTLS] - Job: [blackbox] on Instance: [https://localhost:443] is firing - value: -6065338.00 - {"alertname":"TLS","instance":"https://localhost:443","job":"blackbox","severity":"critical"} 105 | |total=2 firing=1 pending=1 inactive=0 106 | 107 | exit status 2 108 | `, 109 | }, 110 | { 111 | name: "alert-problems-only-with-exlude-on-one-group", 112 | server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 113 | w.WriteHeader(http.StatusOK) 114 | w.Write(loadTestdata(alertTestDataSet1)) 115 | })), 116 | args: []string{"run", "../main.go", "alert", "--problems", "-g", "TLS"}, 117 | expected: `[CRITICAL] - 1 Alerts: 1 Firing - 0 Pending - 0 Inactive 118 | \_ [CRITICAL] [BlackboxTLS] - Job: [blackbox] on Instance: [https://localhost:443] is firing - value: -6065338.00 - {"alertname":"TLS","instance":"https://localhost:443","job":"blackbox","severity":"critical"} 119 | |total=1 firing=1 pending=0 inactive=0 120 | 121 | exit status 2 122 | `, 123 | }, 124 | { 125 | name: "alert-problems-only-with-exlude-on-two-groups", 126 | server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 127 | w.WriteHeader(http.StatusOK) 128 | w.Write(loadTestdata(alertTestDataSet1)) 129 | })), 130 | args: []string{"run", "../main.go", "alert", "--problems", "-g", "SQL", "-g", "TLS"}, 131 | expected: `[CRITICAL] - 2 Alerts: 1 Firing - 1 Pending - 0 Inactive 132 | \_ [WARNING] [SqlAccessDeniedRate] - Job: [mysql] on Instance: [localhost] is pending - value: 0.40 - {"alertname":"SqlAccessDeniedRate","instance":"localhost","job":"mysql","severity":"warning"} 133 | \_ [CRITICAL] [BlackboxTLS] - Job: [blackbox] on Instance: [https://localhost:443] is firing - value: -6065338.00 - {"alertname":"TLS","instance":"https://localhost:443","job":"blackbox","severity":"critical"} 134 | |total=2 firing=1 pending=1 inactive=0 135 | 136 | exit status 2 137 | `, 138 | }, 139 | { 140 | name: "alert-problems-only-with-exlude", 141 | server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 142 | w.WriteHeader(http.StatusOK) 143 | w.Write(loadTestdata(alertTestDataSet1)) 144 | })), 145 | args: []string{"run", "../main.go", "alert", "--problems", "--exclude-alert", "Sql.*DeniedRate"}, 146 | expected: `[CRITICAL] - 1 Alerts: 1 Firing - 0 Pending - 0 Inactive 147 | \_ [CRITICAL] [BlackboxTLS] - Job: [blackbox] on Instance: [https://localhost:443] is firing - value: -6065338.00 - {"alertname":"TLS","instance":"https://localhost:443","job":"blackbox","severity":"critical"} 148 | |total=1 firing=1 pending=0 inactive=0 149 | 150 | exit status 2 151 | `, 152 | }, 153 | { 154 | name: "alert-with-exclude-error", 155 | server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 156 | w.WriteHeader(http.StatusOK) 157 | w.Write(loadTestdata(alertTestDataSet3)) 158 | })), 159 | args: []string{"run", "../main.go", "alert", "--exclude-alert", "[a-z"}, 160 | expected: "[UNKNOWN] - Invalid regular expression provided: error parsing regexp: missing closing ]: `[a-z`\nexit status 3\n", 161 | }, 162 | { 163 | name: "alert-no-such-alert", 164 | server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 165 | w.WriteHeader(http.StatusOK) 166 | w.Write(loadTestdata(alertTestDataSet2)) 167 | })), 168 | args: []string{"run", "../main.go", "alert", "--name", "NoSuchAlert", "-T", "3"}, 169 | expected: `[UNKNOWN] - 0 Alerts: 0 Firing - 0 Pending - 0 Inactive 170 | \_ [UNKNOWN] No alerts retrieved 171 | |total=0 firing=0 pending=0 inactive=0 172 | 173 | exit status 3 174 | `, 175 | }, 176 | { 177 | name: "alert-inactive-with-problems", 178 | server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 179 | w.WriteHeader(http.StatusOK) 180 | w.Write(loadTestdata(alertTestDataSet2)) 181 | })), 182 | args: []string{"run", "../main.go", "alert", "--name", "InactiveAlert", "--problems", "-T", "3"}, 183 | expected: `[UNKNOWN] - 0 Alerts: 0 Firing - 0 Pending - 0 Inactive 184 | \_ [UNKNOWN] No alerts retrieved 185 | |total=0 firing=0 pending=0 inactive=0 186 | 187 | exit status 3 188 | `, 189 | }, 190 | { 191 | name: "alert-multiple-alerts", 192 | server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 193 | w.WriteHeader(http.StatusOK) 194 | w.Write(loadTestdata(alertTestDataSet1)) 195 | })), 196 | args: []string{"run", "../main.go", "alert", "--name", "HostOutOfMemory", "--name", "BlackboxTLS"}, 197 | expected: `[CRITICAL] - 2 Alerts: 1 Firing - 0 Pending - 1 Inactive 198 | \_ [OK] [HostOutOfMemory] is inactive 199 | \_ [CRITICAL] [BlackboxTLS] - Job: [blackbox] on Instance: [https://localhost:443] is firing - value: -6065338.00 - {"alertname":"TLS","instance":"https://localhost:443","job":"blackbox","severity":"critical"} 200 | |total=2 firing=1 pending=0 inactive=1 201 | 202 | exit status 2 203 | `, 204 | }, 205 | { 206 | name: "alert-multiple-alerts-problems-only", 207 | server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 208 | w.WriteHeader(http.StatusOK) 209 | w.Write(loadTestdata(alertTestDataSet1)) 210 | })), 211 | args: []string{"run", "../main.go", "alert", "--name", "HostOutOfMemory", "--name", "BlackboxTLS", "--problems"}, 212 | expected: `[CRITICAL] - 1 Alerts: 1 Firing - 0 Pending - 0 Inactive 213 | \_ [CRITICAL] [BlackboxTLS] - Job: [blackbox] on Instance: [https://localhost:443] is firing - value: -6065338.00 - {"alertname":"TLS","instance":"https://localhost:443","job":"blackbox","severity":"critical"} 214 | |total=1 firing=1 pending=0 inactive=0 215 | 216 | exit status 2 217 | `, 218 | }, 219 | { 220 | name: "alert-inactive", 221 | server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 222 | w.WriteHeader(http.StatusOK) 223 | w.Write(loadTestdata(alertTestDataSet2)) 224 | })), 225 | args: []string{"run", "../main.go", "alert", "--name", "InactiveAlert"}, 226 | expected: "[OK] - 1 Alerts: 0 Firing - 0 Pending - 1 Inactive\n\\_ [OK] [InactiveAlert] is inactive\n|total=1 firing=0 pending=0 inactive=1\n\n", 227 | }, 228 | { 229 | name: "alert-recording-rule", 230 | server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 231 | w.WriteHeader(http.StatusOK) 232 | w.Write(loadTestdata(alertTestDataSet4)) 233 | })), 234 | args: []string{"run", "../main.go", "alert", "--name", "InactiveAlert"}, 235 | expected: "[OK] - 1 Alerts: 0 Firing - 0 Pending - 1 Inactive\n\\_ [OK] [InactiveAlert] is inactive\n|total=1 firing=0 pending=0 inactive=1\n\n", 236 | }, 237 | } 238 | 239 | for _, test := range tests { 240 | t.Run(test.name, func(t *testing.T) { 241 | defer test.server.Close() 242 | 243 | // We need the random Port extracted 244 | u, _ := url.Parse(test.server.URL) 245 | cmd := exec.Command("go", append(test.args, "--port", u.Port())...) 246 | out, _ := cmd.CombinedOutput() 247 | 248 | actual := string(out) 249 | 250 | if actual != test.expected { 251 | t.Error("\nActual: ", actual, "\nExpected: ", test.expected) 252 | } 253 | 254 | }) 255 | } 256 | } 257 | -------------------------------------------------------------------------------- /cmd/query_test.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "net/http" 5 | "net/http/httptest" 6 | "net/url" 7 | "os/exec" 8 | "strings" 9 | "testing" 10 | ) 11 | 12 | func TestQuery_ConnectionRefused(t *testing.T) { 13 | 14 | cmd := exec.Command("go", "run", "../main.go", "query", "--query", "foo", "--critical", "1", "--warning", "2", "--port", "9999") 15 | out, _ := cmd.CombinedOutput() 16 | 17 | actual := string(out) 18 | expected := "[UNKNOWN] - Post \"http://localhost:9999/api/v1/query\"" 19 | 20 | if !strings.Contains(actual, expected) { 21 | t.Error("\nActual: ", actual, "\nExpected: ", expected) 22 | } 23 | } 24 | 25 | func TestQuery_MissingParameter(t *testing.T) { 26 | 27 | cmd := exec.Command("go", "run", "../main.go", "query") 28 | out, _ := cmd.CombinedOutput() 29 | 30 | expected := "[UNKNOWN] - required flag(s) \"query\" not set (*errors.errorString)" 31 | 32 | actual := string(out) 33 | 34 | if !strings.Contains(actual, expected) { 35 | t.Error("\nActual: ", actual, "\nExpected: ", expected) 36 | } 37 | } 38 | 39 | type QueryTest struct { 40 | name string 41 | server *httptest.Server 42 | args []string 43 | expected string 44 | } 45 | 46 | func TestQueryCmd(t *testing.T) { 47 | 48 | queryTestDataSet1 := "../testdata/unittest/queryDataset1.json" 49 | 50 | queryTestDataSet2 := "../testdata/unittest/queryDataset2.json" 51 | 52 | queryTestDataSet3 := "../testdata/unittest/queryDataset3.json" 53 | 54 | queryTestDataSet4 := "../testdata/unittest/queryDataset4.json" 55 | 56 | tests := []QueryTest{ 57 | { 58 | name: "query-warning", 59 | server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 60 | w.WriteHeader(http.StatusOK) 61 | w.Write([]byte(`{"status":"success","data":{"resultType":"vector","result":[]},"warnings": ["hic sunt dracones", "foo"]}`)) 62 | })), 63 | args: []string{"run", "../main.go", "query", "--query", "foo"}, 64 | expected: "[UNKNOWN] - No status information\nHTTP Warnings: hic sunt dracones, foo\n\nexit status 3\n", 65 | }, 66 | { 67 | name: "query-no-such-metric", 68 | server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 69 | w.WriteHeader(http.StatusOK) 70 | w.Write([]byte(`{"status":"success","data":{"resultType":"vector","result":[]}}`)) 71 | })), 72 | args: []string{"run", "../main.go", "query", "--query", "foo"}, 73 | expected: "[UNKNOWN] - No status information\n\nexit status 3\n", 74 | }, 75 | { 76 | name: "query-no-such-matrix", 77 | server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 78 | w.WriteHeader(http.StatusOK) 79 | w.Write([]byte(`{"status":"success","data":{"resultType":"matrix","result":[]}}`)) 80 | })), 81 | args: []string{"run", "../main.go", "query", "--query", "foo"}, 82 | expected: "[UNKNOWN] - No status information\n\nexit status 3\n", 83 | }, 84 | { 85 | name: "query-scalar", 86 | server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 87 | w.WriteHeader(http.StatusOK) 88 | w.Write([]byte(`{"status":"success","data":{"resultType":"scalar","result":[1670339013.992,"1"]}}`)) 89 | })), 90 | args: []string{"run", "../main.go", "query", "--query", "1"}, 91 | expected: "[UNKNOWN] - scalar value results are not supported (*errors.errorString)\nexit status 3\n", 92 | }, 93 | { 94 | name: "query-string", 95 | server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 96 | w.WriteHeader(http.StatusOK) 97 | w.Write([]byte(`{"status":"success","data":{"resultType":"string","result":[1670339013.992,"up"]}}`)) 98 | })), 99 | args: []string{"run", "../main.go", "query", "--query", "up"}, 100 | expected: "[UNKNOWN] - string value results are not supported (*errors.errorString)\nexit status 3\n", 101 | }, 102 | { 103 | name: "query-matrix-exists", 104 | server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 105 | w.WriteHeader(http.StatusOK) 106 | w.Write(loadTestdata(queryTestDataSet1)) 107 | })), 108 | args: []string{"run", "../main.go", "query", "--query", "up{job=\"prometheus\"}[5m]"}, 109 | expected: "[OK] - states: ok=1\n\\_ [OK] 1 @[1670340952.99] - value: 1\n|up_instance_localhost_job_node=1;10;20\n\n", 110 | }, 111 | { 112 | name: "query-metric-exists2", 113 | server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 114 | w.WriteHeader(http.StatusOK) 115 | w.Write(loadTestdata(queryTestDataSet2)) 116 | })), 117 | args: []string{"run", "../main.go", "query", "--query", "up{job=\"prometheus\"}"}, 118 | expected: "[OK] - states: ok=1\n\\_ [OK] up{instance=\"localhost\", job=\"prometheus\"} - value: 1\n|up_instance_localhost_job_prometheus=1;10;20\n\n", 119 | }, 120 | { 121 | name: "query-threshold-ok", 122 | server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 123 | w.WriteHeader(http.StatusOK) 124 | w.Write(loadTestdata(queryTestDataSet3)) 125 | })), 126 | args: []string{"run", "../main.go", "query", "--query", "up{job=\"prometheus\"}", "-w", "0:", "-c", "0:"}, 127 | expected: "[OK] - states: ok=1\n\\_ [OK] up{instance=\"localhost\", job=\"prometheus\"} - value: 100\n|up_instance_localhost_job_prometheus=100\n\n", 128 | }, 129 | { 130 | name: "query-threshold-critical", 131 | server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 132 | w.WriteHeader(http.StatusOK) 133 | w.Write(loadTestdata(queryTestDataSet4)) 134 | })), 135 | args: []string{"run", "../main.go", "query", "--query", "up{job=\"prometheus\"}", "-w", "0:", "-c", "0:"}, 136 | expected: "[CRITICAL] - states: critical=1\n\\_ [CRITICAL] up{instance=\"localhost\", job=\"prometheus\"} - value: -100\n|up_instance_localhost_job_prometheus=-100\n\nexit status 2\n", 137 | }, 138 | } 139 | 140 | for _, test := range tests { 141 | t.Run(test.name, func(t *testing.T) { 142 | defer test.server.Close() 143 | 144 | // We need the random Port extracted 145 | u, _ := url.Parse(test.server.URL) 146 | cmd := exec.Command("go", append(test.args, "--port", u.Port())...) 147 | out, _ := cmd.CombinedOutput() 148 | 149 | actual := string(out) 150 | 151 | if actual != test.expected { 152 | // t.Error("\nActual: ", actual, "\nExpected: ", test.expected) 153 | t.Error("\nActual: ", actual, "\nExpected: ", test.expected) 154 | } 155 | 156 | }) 157 | } 158 | } 159 | 160 | func TestExtendedQueryCmd(t *testing.T) { 161 | tests := []QueryTest{ 162 | { 163 | name: "vector-multiple-ok", 164 | server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 165 | w.WriteHeader(http.StatusOK) 166 | w.Write([]byte(`{"status":"success","data":{"resultType":"vector","result":[{"metric":{"__name__":"up","instance":"localhost:9100","job":"node"},"value":[1696589905.608,"1"]},{"metric":{"__name__":"up","instance":"localhost:9104","job":"mysqld"},"value":[1696589905.608,"99"]},{"metric":{"__name__":"up","instance":"localhost:9117","job":"apache"},"value":[1696589905.608,"1"]}]}}`)) 167 | })), 168 | args: []string{"run", "../main.go", "query", "--query", "up", "-w", "100", "-c", "200"}, 169 | expected: "OK] - states: ok=3\n\\_ [OK] up{instance=\"localhost:9100\", job=\"node\"} - value: 1\n\\_ [OK] up{instance=\"localhost:9104\", job=\"mysqld\"} - value: 99\n\\_ [OK] up{instance=\"localhost:9117\", job=\"apache\"} - value: 1\n|up_instance_localhost:9100_job_node=1;100;200 up_instance_localhost:9104_job_mysqld=99;100;200 up_instance_localhost:9117_job_apache=1;100;200\n\n", 170 | }, 171 | { 172 | name: "vector-multiple-critical", 173 | server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 174 | w.WriteHeader(http.StatusOK) 175 | w.Write([]byte(`{"status":"success","data":{"resultType":"vector","result":[{"metric":{"__name__":"up","instance":"localhost:9100","job":"node"},"value":[1696589905.608,"1"]},{"metric":{"__name__":"up","instance":"localhost:9104","job":"mysqld"},"value":[1696589905.608,"11"]},{"metric":{"__name__":"up","instance":"localhost:9117","job":"apache"},"value":[1696589905.608,"6"]}]}}`)) 176 | })), 177 | args: []string{"run", "../main.go", "query", "--query", "up", "-w", "5", "-c", "10"}, 178 | expected: "[CRITICAL] - states: critical=1 warning=1 ok=1\n\\_ [OK] up{instance=\"localhost:9100\", job=\"node\"} - value: 1\n\\_ [CRITICAL] up{instance=\"localhost:9104\", job=\"mysqld\"} - value: 11\n\\_ [WARNING] up{instance=\"localhost:9117\", job=\"apache\"} - value: 6\n|up_instance_localhost:9100_job_node=1;5;10 up_instance_localhost:9104_job_mysqld=11;5;10 up_instance_localhost:9117_job_apache=6;5;10\n\nexit status 2\n", 179 | }, 180 | { 181 | name: "matrix-multiple-critical", 182 | server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 183 | w.WriteHeader(http.StatusOK) 184 | w.Write([]byte(`{"status":"success","data":{"resultType":"matrix","result":[{"metric":{"__name__":"up","instance":"localhost:9100","job":"node"},"values":[[1696589212.987,"1"],[1696589272.987,"1"],[1696589332.987,"1"],[1696589392.987,"1"],[1696589452.987,"1"]]},{"metric":{"__name__":"up","instance":"localhost:9104","job":"mysqld"},"values":[[1696589209.089,"25"],[1696589269.089,"25"],[1696589329.089,"25"],[1696589389.089,"25"],[1696589449.089,"25"]]},{"metric":{"__name__":"up","instance":"localhost:9117","job":"apache"},"values":[[1696589209.369,"1"],[1696589269.369,"1"],[1696589329.369,"1"],[1696589389.369,"1"],[1696589449.369,"1"]]}]}}`)) 185 | })), 186 | args: []string{"run", "../main.go", "query", "--query", "up", "-w", "10", "-c", "20"}, 187 | expected: "[CRITICAL] - states: critical=1 ok=2\n\\_ [OK] 1 @[1696589452.987] - value: 1\n\\_ [CRITICAL] 25 @[1696589449.089] - value: 25\n\\_ [OK] 1 @[1696589449.369] - value: 1\n|up_instance_localhost:9100_job_node=1;10;20 up_instance_localhost:9104_job_mysqld=25;10;20 up_instance_localhost:9117_job_apache=1;10;20\n\nexit status 2\n", 188 | }, 189 | { 190 | name: "matrix-multiple-warning", 191 | server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 192 | w.WriteHeader(http.StatusOK) 193 | w.Write([]byte(`{"status":"success","data":{"resultType":"matrix","result":[{"metric":{"__name__":"up","instance":"localhost:9100","job":"node"},"values":[[1696589212.987,"1"],[1696589272.987,"1"],[1696589332.987,"1"],[1696589392.987,"1"],[1696589452.987,"1"]]},{"metric":{"__name__":"up","instance":"localhost:9104","job":"mysqld"},"values":[[1696589209.089,"15"],[1696589269.089,"15"],[1696589329.089,"15"],[1696589389.089,"15"],[1696589449.089,"15"]]},{"metric":{"__name__":"up","instance":"localhost:9117","job":"apache"},"values":[[1696589209.369,"1"],[1696589269.369,"1"],[1696589329.369,"1"],[1696589389.369,"1"],[1696589449.369,"1"]]}]}}`)) 194 | })), 195 | args: []string{"run", "../main.go", "query", "--query", "up", "-w", "10", "-c", "20"}, 196 | expected: "WARNING] - states: warning=1 ok=2\n\\_ [OK] 1 @[1696589452.987] - value: 1\n\\_ [WARNING] 15 @[1696589449.089] - value: 15\n\\_ [OK] 1 @[1696589449.369] - value: 1\n|up_instance_localhost:9100_job_node=1;10;20 up_instance_localhost:9104_job_mysqld=15;10;20 up_instance_localhost:9117_job_apache=1;10;20\n\nexit status 1\n", 197 | }, 198 | } 199 | 200 | for _, test := range tests { 201 | t.Run(test.name, func(t *testing.T) { 202 | defer test.server.Close() 203 | 204 | // We need the random Port extracted 205 | u, _ := url.Parse(test.server.URL) 206 | cmd := exec.Command("go", append(test.args, "--port", u.Port())...) 207 | out, _ := cmd.CombinedOutput() 208 | 209 | actual := string(out) 210 | 211 | if !strings.Contains(actual, test.expected) { 212 | t.Error("\nActual: ", actual, "\nExpected: ", test.expected) 213 | } 214 | 215 | }) 216 | } 217 | } 218 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 2, June 1991 3 | 4 | Copyright (C) 1989, 1991 Free Software Foundation, Inc., 5 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 6 | Everyone is permitted to copy and distribute verbatim copies 7 | of this license document, but changing it is not allowed. 8 | 9 | Preamble 10 | 11 | The licenses for most software are designed to take away your 12 | freedom to share and change it. By contrast, the GNU General Public 13 | License is intended to guarantee your freedom to share and change free 14 | software--to make sure the software is free for all its users. This 15 | General Public License applies to most of the Free Software 16 | Foundation's software and to any other program whose authors commit to 17 | using it. (Some other Free Software Foundation software is covered by 18 | the GNU Lesser General Public License instead.) You can apply it to 19 | your programs, too. 20 | 21 | When we speak of free software, we are referring to freedom, not 22 | price. Our General Public Licenses are designed to make sure that you 23 | have the freedom to distribute copies of free software (and charge for 24 | this service if you wish), that you receive source code or can get it 25 | if you want it, that you can change the software or use pieces of it 26 | in new free programs; and that you know you can do these things. 27 | 28 | To protect your rights, we need to make restrictions that forbid 29 | anyone to deny you these rights or to ask you to surrender the rights. 30 | These restrictions translate to certain responsibilities for you if you 31 | distribute copies of the software, or if you modify it. 32 | 33 | For example, if you distribute copies of such a program, whether 34 | gratis or for a fee, you must give the recipients all the rights that 35 | you have. You must make sure that they, too, receive or can get the 36 | source code. And you must show them these terms so they know their 37 | rights. 38 | 39 | We protect your rights with two steps: (1) copyright the software, and 40 | (2) offer you this license which gives you legal permission to copy, 41 | distribute and/or modify the software. 42 | 43 | Also, for each author's protection and ours, we want to make certain 44 | that everyone understands that there is no warranty for this free 45 | software. If the software is modified by someone else and passed on, we 46 | want its recipients to know that what they have is not the original, so 47 | that any problems introduced by others will not reflect on the original 48 | authors' reputations. 49 | 50 | Finally, any free program is threatened constantly by software 51 | patents. We wish to avoid the danger that redistributors of a free 52 | program will individually obtain patent licenses, in effect making the 53 | program proprietary. To prevent this, we have made it clear that any 54 | patent must be licensed for everyone's free use or not licensed at all. 55 | 56 | The precise terms and conditions for copying, distribution and 57 | modification follow. 58 | 59 | GNU GENERAL PUBLIC LICENSE 60 | TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 61 | 62 | 0. This License applies to any program or other work which contains 63 | a notice placed by the copyright holder saying it may be distributed 64 | under the terms of this General Public License. The "Program", below, 65 | refers to any such program or work, and a "work based on the Program" 66 | means either the Program or any derivative work under copyright law: 67 | that is to say, a work containing the Program or a portion of it, 68 | either verbatim or with modifications and/or translated into another 69 | language. (Hereinafter, translation is included without limitation in 70 | the term "modification".) Each licensee is addressed as "you". 71 | 72 | Activities other than copying, distribution and modification are not 73 | covered by this License; they are outside its scope. The act of 74 | running the Program is not restricted, and the output from the Program 75 | is covered only if its contents constitute a work based on the 76 | Program (independent of having been made by running the Program). 77 | Whether that is true depends on what the Program does. 78 | 79 | 1. You may copy and distribute verbatim copies of the Program's 80 | source code as you receive it, in any medium, provided that you 81 | conspicuously and appropriately publish on each copy an appropriate 82 | copyright notice and disclaimer of warranty; keep intact all the 83 | notices that refer to this License and to the absence of any warranty; 84 | and give any other recipients of the Program a copy of this License 85 | along with the Program. 86 | 87 | You may charge a fee for the physical act of transferring a copy, and 88 | you may at your option offer warranty protection in exchange for a fee. 89 | 90 | 2. You may modify your copy or copies of the Program or any portion 91 | of it, thus forming a work based on the Program, and copy and 92 | distribute such modifications or work under the terms of Section 1 93 | above, provided that you also meet all of these conditions: 94 | 95 | a) You must cause the modified files to carry prominent notices 96 | stating that you changed the files and the date of any change. 97 | 98 | b) You must cause any work that you distribute or publish, that in 99 | whole or in part contains or is derived from the Program or any 100 | part thereof, to be licensed as a whole at no charge to all third 101 | parties under the terms of this License. 102 | 103 | c) If the modified program normally reads commands interactively 104 | when run, you must cause it, when started running for such 105 | interactive use in the most ordinary way, to print or display an 106 | announcement including an appropriate copyright notice and a 107 | notice that there is no warranty (or else, saying that you provide 108 | a warranty) and that users may redistribute the program under 109 | these conditions, and telling the user how to view a copy of this 110 | License. (Exception: if the Program itself is interactive but 111 | does not normally print such an announcement, your work based on 112 | the Program is not required to print an announcement.) 113 | 114 | These requirements apply to the modified work as a whole. If 115 | identifiable sections of that work are not derived from the Program, 116 | and can be reasonably considered independent and separate works in 117 | themselves, then this License, and its terms, do not apply to those 118 | sections when you distribute them as separate works. But when you 119 | distribute the same sections as part of a whole which is a work based 120 | on the Program, the distribution of the whole must be on the terms of 121 | this License, whose permissions for other licensees extend to the 122 | entire whole, and thus to each and every part regardless of who wrote it. 123 | 124 | Thus, it is not the intent of this section to claim rights or contest 125 | your rights to work written entirely by you; rather, the intent is to 126 | exercise the right to control the distribution of derivative or 127 | collective works based on the Program. 128 | 129 | In addition, mere aggregation of another work not based on the Program 130 | with the Program (or with a work based on the Program) on a volume of 131 | a storage or distribution medium does not bring the other work under 132 | the scope of this License. 133 | 134 | 3. You may copy and distribute the Program (or a work based on it, 135 | under Section 2) in object code or executable form under the terms of 136 | Sections 1 and 2 above provided that you also do one of the following: 137 | 138 | a) Accompany it with the complete corresponding machine-readable 139 | source code, which must be distributed under the terms of Sections 140 | 1 and 2 above on a medium customarily used for software interchange; or, 141 | 142 | b) Accompany it with a written offer, valid for at least three 143 | years, to give any third party, for a charge no more than your 144 | cost of physically performing source distribution, a complete 145 | machine-readable copy of the corresponding source code, to be 146 | distributed under the terms of Sections 1 and 2 above on a medium 147 | customarily used for software interchange; or, 148 | 149 | c) Accompany it with the information you received as to the offer 150 | to distribute corresponding source code. (This alternative is 151 | allowed only for noncommercial distribution and only if you 152 | received the program in object code or executable form with such 153 | an offer, in accord with Subsection b above.) 154 | 155 | The source code for a work means the preferred form of the work for 156 | making modifications to it. For an executable work, complete source 157 | code means all the source code for all modules it contains, plus any 158 | associated interface definition files, plus the scripts used to 159 | control compilation and installation of the executable. However, as a 160 | special exception, the source code distributed need not include 161 | anything that is normally distributed (in either source or binary 162 | form) with the major components (compiler, kernel, and so on) of the 163 | operating system on which the executable runs, unless that component 164 | itself accompanies the executable. 165 | 166 | If distribution of executable or object code is made by offering 167 | access to copy from a designated place, then offering equivalent 168 | access to copy the source code from the same place counts as 169 | distribution of the source code, even though third parties are not 170 | compelled to copy the source along with the object code. 171 | 172 | 4. You may not copy, modify, sublicense, or distribute the Program 173 | except as expressly provided under this License. Any attempt 174 | otherwise to copy, modify, sublicense or distribute the Program is 175 | void, and will automatically terminate your rights under this License. 176 | However, parties who have received copies, or rights, from you under 177 | this License will not have their licenses terminated so long as such 178 | parties remain in full compliance. 179 | 180 | 5. You are not required to accept this License, since you have not 181 | signed it. However, nothing else grants you permission to modify or 182 | distribute the Program or its derivative works. These actions are 183 | prohibited by law if you do not accept this License. Therefore, by 184 | modifying or distributing the Program (or any work based on the 185 | Program), you indicate your acceptance of this License to do so, and 186 | all its terms and conditions for copying, distributing or modifying 187 | the Program or works based on it. 188 | 189 | 6. Each time you redistribute the Program (or any work based on the 190 | Program), the recipient automatically receives a license from the 191 | original licensor to copy, distribute or modify the Program subject to 192 | these terms and conditions. You may not impose any further 193 | restrictions on the recipients' exercise of the rights granted herein. 194 | You are not responsible for enforcing compliance by third parties to 195 | this License. 196 | 197 | 7. If, as a consequence of a court judgment or allegation of patent 198 | infringement or for any other reason (not limited to patent issues), 199 | conditions are imposed on you (whether by court order, agreement or 200 | otherwise) that contradict the conditions of this License, they do not 201 | excuse you from the conditions of this License. If you cannot 202 | distribute so as to satisfy simultaneously your obligations under this 203 | License and any other pertinent obligations, then as a consequence you 204 | may not distribute the Program at all. For example, if a patent 205 | license would not permit royalty-free redistribution of the Program by 206 | all those who receive copies directly or indirectly through you, then 207 | the only way you could satisfy both it and this License would be to 208 | refrain entirely from distribution of the Program. 209 | 210 | If any portion of this section is held invalid or unenforceable under 211 | any particular circumstance, the balance of the section is intended to 212 | apply and the section as a whole is intended to apply in other 213 | circumstances. 214 | 215 | It is not the purpose of this section to induce you to infringe any 216 | patents or other property right claims or to contest validity of any 217 | such claims; this section has the sole purpose of protecting the 218 | integrity of the free software distribution system, which is 219 | implemented by public license practices. Many people have made 220 | generous contributions to the wide range of software distributed 221 | through that system in reliance on consistent application of that 222 | system; it is up to the author/donor to decide if he or she is willing 223 | to distribute software through any other system and a licensee cannot 224 | impose that choice. 225 | 226 | This section is intended to make thoroughly clear what is believed to 227 | be a consequence of the rest of this License. 228 | 229 | 8. If the distribution and/or use of the Program is restricted in 230 | certain countries either by patents or by copyrighted interfaces, the 231 | original copyright holder who places the Program under this License 232 | may add an explicit geographical distribution limitation excluding 233 | those countries, so that distribution is permitted only in or among 234 | countries not thus excluded. In such case, this License incorporates 235 | the limitation as if written in the body of this License. 236 | 237 | 9. The Free Software Foundation may publish revised and/or new versions 238 | of the General Public License from time to time. Such new versions will 239 | be similar in spirit to the present version, but may differ in detail to 240 | address new problems or concerns. 241 | 242 | Each version is given a distinguishing version number. If the Program 243 | specifies a version number of this License which applies to it and "any 244 | later version", you have the option of following the terms and conditions 245 | either of that version or of any later version published by the Free 246 | Software Foundation. If the Program does not specify a version number of 247 | this License, you may choose any version ever published by the Free Software 248 | Foundation. 249 | 250 | 10. If you wish to incorporate parts of the Program into other free 251 | programs whose distribution conditions are different, write to the author 252 | to ask for permission. For software which is copyrighted by the Free 253 | Software Foundation, write to the Free Software Foundation; we sometimes 254 | make exceptions for this. Our decision will be guided by the two goals 255 | of preserving the free status of all derivatives of our free software and 256 | of promoting the sharing and reuse of software generally. 257 | 258 | NO WARRANTY 259 | 260 | 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY 261 | FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN 262 | OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES 263 | PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED 264 | OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 265 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS 266 | TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE 267 | PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, 268 | REPAIR OR CORRECTION. 269 | 270 | 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 271 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR 272 | REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, 273 | INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING 274 | OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED 275 | TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY 276 | YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER 277 | PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE 278 | POSSIBILITY OF SUCH DAMAGES. 279 | 280 | END OF TERMS AND CONDITIONS 281 | 282 | How to Apply These Terms to Your New Programs 283 | 284 | If you develop a new program, and you want it to be of the greatest 285 | possible use to the public, the best way to achieve this is to make it 286 | free software which everyone can redistribute and change under these terms. 287 | 288 | To do so, attach the following notices to the program. It is safest 289 | to attach them to the start of each source file to most effectively 290 | convey the exclusion of warranty; and each file should have at least 291 | the "copyright" line and a pointer to where the full notice is found. 292 | 293 | 294 | Copyright (C) 295 | 296 | This program is free software; you can redistribute it and/or modify 297 | it under the terms of the GNU General Public License as published by 298 | the Free Software Foundation; either version 2 of the License, or 299 | (at your option) any later version. 300 | 301 | This program is distributed in the hope that it will be useful, 302 | but WITHOUT ANY WARRANTY; without even the implied warranty of 303 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 304 | GNU General Public License for more details. 305 | 306 | You should have received a copy of the GNU General Public License along 307 | with this program; if not, write to the Free Software Foundation, Inc., 308 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 309 | 310 | Also add information on how to contact you by electronic and paper mail. 311 | 312 | If the program is interactive, make it output a short notice like this 313 | when it starts in an interactive mode: 314 | 315 | Gnomovision version 69, Copyright (C) year name of author 316 | Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 317 | This is free software, and you are welcome to redistribute it 318 | under certain conditions; type `show c' for details. 319 | 320 | The hypothetical commands `show w' and `show c' should show the appropriate 321 | parts of the General Public License. Of course, the commands you use may 322 | be called something other than `show w' and `show c'; they could even be 323 | mouse-clicks or menu items--whatever suits your program. 324 | 325 | You should also get your employer (if you work as a programmer) or your 326 | school, if any, to sign a "copyright disclaimer" for the program, if 327 | necessary. Here is a sample; alter the names: 328 | 329 | Yoyodyne, Inc., hereby disclaims all copyright interest in the program 330 | `Gnomovision' (which makes passes at compilers) written by James Hacker. 331 | 332 | , 1 April 1989 333 | Ty Coon, President of Vice 334 | 335 | This General Public License does not permit incorporating your program into 336 | proprietary programs. If your program is a subroutine library, you may 337 | consider it more useful to permit linking proprietary applications with the 338 | library. If this is what you want to do, use the GNU Lesser General 339 | Public License instead of this License. 340 | --------------------------------------------------------------------------------