├── .gitlab-ci.yml ├── Dockerfile.amd64 ├── Dockerfile.arm64 ├── Dockerfile.armhf ├── Gopkg.lock ├── Gopkg.toml ├── LICENSE ├── Makefile ├── README.md ├── db.d ├── mysql │ ├── 0.0.1-bootstrap.sql │ └── 0.1.0-fingerprint.sql └── postgres │ ├── 0.0.1-bootstrap.sql │ └── 0.1.0-fingerprint.sql ├── docker-compose.yml ├── go.mod ├── go.sum ├── internal ├── db │ ├── db.go │ ├── db_test.go │ ├── mysql.go │ ├── nulldb.go │ ├── nulldb_test.go │ └── postgres.go ├── internal.go ├── metrics │ ├── metrics.go │ └── metrics_test.go ├── server │ └── server.go └── webhook │ ├── sample-payload-invalid-ends-at.json │ ├── sample-payload.json │ ├── webhook.go │ └── webhook_test.go ├── main.go ├── script.d ├── bootstrap_mysql.sh ├── bootstrap_postgres.sh └── test.sh └── version ├── version.go └── version_test.go /.gitlab-ci.yml: -------------------------------------------------------------------------------- 1 | image: registry.gitlab.com/yakshaving.art/dockerfiles/go-builder:latest 2 | stages: 3 | - build 4 | - image 5 | - release 6 | 7 | variables: 8 | CGO_ENABLED: 0 9 | GOCACHE: ${CI_PROJECT_DIR}/.cache/go-build 10 | 11 | 12 | test_mysql: 13 | stage: build 14 | cache: 15 | key: build-cache 16 | paths: 17 | - ${CI_PROJECT_DIR}/.cache/go-build 18 | services: 19 | - mysql:5.7 20 | variables: 21 | MYSQL_DATABASE: alertsnitch 22 | MYSQL_ROOT_PASSWORD: mysql 23 | ALERTSNITCH_DSN: "root:${MYSQL_ROOT_PASSWORD}@tcp(mysql)/${MYSQL_DATABASE}" 24 | ALERTSNITCH_BACKEND: mysql 25 | coverage: '/^total:\s+\(statements\)\s+(\d+.\d+)%$/' 26 | script: 27 | - apk --no-cache add mysql-client bash 28 | - bash script.d/bootstrap_mysql.sh 29 | - bash script.d/test.sh 30 | 31 | test_postgres: 32 | stage: build 33 | services: 34 | - name: postgres:11 35 | alias: postgres 36 | variables: 37 | POSTGRES_DB: alertsnitch 38 | POSTGRES_USER: runner 39 | POSTGRES_PASSWORD: "" 40 | POSTGRES_HOST_AUTH_METHOD: trust 41 | ALERTSNITCH_DSN: "sslmode=disable user=${POSTGRES_USER} password='' host=postgres database=${POSTGRES_DB}" 42 | ALERTSNITCH_BACKEND: postgres 43 | coverage: '/^total:\s+\(statements\)\s+(\d+.\d+)%$/' 44 | script: 45 | - apk --no-cache add postgresql-client bash 46 | - bash script.d/bootstrap_postgres.sh 47 | - bash script.d/test.sh 48 | 49 | build: 50 | stage: build 51 | artifacts: 52 | paths: 53 | - alertsnitch-* 54 | script: 55 | - make build 56 | - GOARCH=arm64 make build 57 | - GOARCH=arm GOARM=6 make build 58 | 59 | .docker: &docker 60 | image: docker:stable 61 | services: 62 | - docker:dind 63 | variables: 64 | DOCKER_CLI_EXPERIMENTAL: enabled 65 | DOCKER_HOST: tcp://docker:2375 66 | DOCKER_DRIVER: overlay2 67 | DOCKER_TLS_CERTDIR: "" 68 | before_script: 69 | - echo ${CI_JOB_TOKEN} | docker login -u gitlab-ci-token --password-stdin ${CI_REGISTRY} 70 | after_script: 71 | - docker logout 72 | 73 | .build_image: &build_image 74 | <<: *docker 75 | stage: image 76 | script: 77 | - echo Building ${ARCH} image 78 | - cp Dockerfile.${ARCH} Dockerfile 79 | - docker build --pull -t ${CI_REGISTRY_IMAGE}:${ARCH}-latest . 80 | - docker push ${CI_REGISTRY_IMAGE}:${ARCH}-latest 81 | - rm Dockerfile 82 | 83 | build_arm64: 84 | <<: *build_image 85 | variables: 86 | ARCH: arm64 87 | 88 | build_amd64: 89 | <<: *build_image 90 | variables: 91 | ARCH: amd64 92 | 93 | build_armhf: 94 | <<: *build_image 95 | variables: 96 | ARCH: armhf 97 | 98 | release_latest: 99 | <<: *docker 100 | stage: release 101 | script: 102 | - docker manifest create ${CI_REGISTRY_IMAGE}:latest 103 | ${CI_REGISTRY_IMAGE}:amd64-latest 104 | ${CI_REGISTRY_IMAGE}:arm64-latest 105 | ${CI_REGISTRY_IMAGE}:armhf-latest 106 | - docker manifest annotate ${CI_REGISTRY_IMAGE} 107 | ${CI_REGISTRY_IMAGE}:arm64-latest --os linux --arch arm64 108 | - docker manifest annotate ${CI_REGISTRY_IMAGE} 109 | ${CI_REGISTRY_IMAGE}:armhf-latest --os linux --arch arm --variant 6 110 | - docker manifest push ${CI_REGISTRY_IMAGE}:latest 111 | only: 112 | - master 113 | 114 | release_tag: 115 | <<: *docker 116 | stage: release 117 | script: 118 | - docker manifest create ${CI_REGISTRY_IMAGE}:${CI_COMMIT_TAG} 119 | ${CI_REGISTRY_IMAGE}:amd64-latest 120 | ${CI_REGISTRY_IMAGE}:arm64-latest 121 | ${CI_REGISTRY_IMAGE}:armhf-latest 122 | - docker manifest annotate ${CI_REGISTRY_IMAGE}:${CI_COMMIT_TAG} 123 | ${CI_REGISTRY_IMAGE}:arm64-latest --os linux --arch arm64 124 | - docker manifest annotate ${CI_REGISTRY_IMAGE}:${CI_COMMIT_TAG} 125 | ${CI_REGISTRY_IMAGE}:armhf-latest --os linux --arch arm --variant 6 126 | - docker manifest push ${CI_REGISTRY_IMAGE}:${CI_COMMIT_TAG} 127 | only: 128 | - tags 129 | -------------------------------------------------------------------------------- /Dockerfile.amd64: -------------------------------------------------------------------------------- 1 | FROM registry.yakshaving.art:443/tools/multiarch-alpine-base:latest 2 | 3 | COPY alertsnitch-amd64 /alertsnitch 4 | 5 | EXPOSE 9567 6 | 7 | ENTRYPOINT [ "/alertsnitch" ] 8 | -------------------------------------------------------------------------------- /Dockerfile.arm64: -------------------------------------------------------------------------------- 1 | FROM registry.yakshaving.art:443/tools/multiarch-alpine-base:latest 2 | 3 | COPY alertsnitch-arm64 /alertsnitch 4 | 5 | EXPOSE 9567 6 | 7 | ENTRYPOINT [ "/alertsnitch" ] 8 | -------------------------------------------------------------------------------- /Dockerfile.armhf: -------------------------------------------------------------------------------- 1 | FROM registry.yakshaving.art:443/tools/multiarch-alpine-base:latest 2 | 3 | COPY alertsnitch-arm /alertsnitch 4 | 5 | EXPOSE 9567 6 | 7 | ENTRYPOINT [ "/alertsnitch" ] 8 | -------------------------------------------------------------------------------- /Gopkg.lock: -------------------------------------------------------------------------------- 1 | # This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. 2 | 3 | 4 | [[projects]] 5 | branch = "master" 6 | digest = "1:d6afaeed1502aa28e80a4ed0981d570ad91b2579193404256ce672ed0a609e0d" 7 | name = "github.com/beorn7/perks" 8 | packages = ["quantile"] 9 | pruneopts = "UT" 10 | revision = "3a771d992973f24aa725d07868b467d1ddfceafb" 11 | 12 | [[projects]] 13 | digest = "1:ffe9824d294da03b391f44e1ae8281281b4afc1bdaa9588c9097785e3af10cec" 14 | name = "github.com/davecgh/go-spew" 15 | packages = ["spew"] 16 | pruneopts = "UT" 17 | revision = "8991bc29aa16c548c550c7ff78260e27b9ab7c73" 18 | version = "v1.1.1" 19 | 20 | [[projects]] 21 | digest = "1:ec6f9bf5e274c833c911923c9193867f3f18788c461f76f05f62bb1510e0ae65" 22 | name = "github.com/go-sql-driver/mysql" 23 | packages = ["."] 24 | pruneopts = "UT" 25 | revision = "72cd26f257d44c1114970e19afddcd812016007e" 26 | version = "v1.4.1" 27 | 28 | [[projects]] 29 | digest = "1:97df918963298c287643883209a2c3f642e6593379f97ab400c2a2e219ab647d" 30 | name = "github.com/golang/protobuf" 31 | packages = ["proto"] 32 | pruneopts = "UT" 33 | revision = "aa810b61a9c79d51363740d207bb46cf8e620ed5" 34 | version = "v1.2.0" 35 | 36 | [[projects]] 37 | digest = "1:c79fb010be38a59d657c48c6ba1d003a8aa651fa56b579d959d74573b7dff8e1" 38 | name = "github.com/gorilla/context" 39 | packages = ["."] 40 | pruneopts = "UT" 41 | revision = "08b5f424b9271eedf6f9f0ce86cb9396ed337a42" 42 | version = "v1.1.1" 43 | 44 | [[projects]] 45 | digest = "1:e73f5b0152105f18bc131fba127d9949305c8693f8a762588a82a48f61756f5f" 46 | name = "github.com/gorilla/mux" 47 | packages = ["."] 48 | pruneopts = "UT" 49 | revision = "e3702bed27f0d39777b0b37b664b6280e8ef8fbf" 50 | version = "v1.6.2" 51 | 52 | [[projects]] 53 | digest = "1:ff5ebae34cfbf047d505ee150de27e60570e8c394b3b8fdbb720ff6ac71985fc" 54 | name = "github.com/matttproud/golang_protobuf_extensions" 55 | packages = ["pbutil"] 56 | pruneopts = "UT" 57 | revision = "c12348ce28de40eed0136aa2b644d0ee0650e56c" 58 | version = "v1.0.1" 59 | 60 | [[projects]] 61 | digest = "1:0028cb19b2e4c3112225cd871870f2d9cf49b9b4276531f03438a88e94be86fe" 62 | name = "github.com/pmezard/go-difflib" 63 | packages = ["difflib"] 64 | pruneopts = "UT" 65 | revision = "792786c7400a136282c1664665ae0a8db921c6c2" 66 | version = "v1.0.0" 67 | 68 | [[projects]] 69 | digest = "1:93a746f1060a8acbcf69344862b2ceced80f854170e1caae089b2834c5fbf7f4" 70 | name = "github.com/prometheus/client_golang" 71 | packages = [ 72 | "prometheus", 73 | "prometheus/internal", 74 | "prometheus/promhttp", 75 | ] 76 | pruneopts = "UT" 77 | revision = "505eaef017263e299324067d40ca2c48f6a2cf50" 78 | version = "v0.9.2" 79 | 80 | [[projects]] 81 | branch = "master" 82 | digest = "1:2d5cd61daa5565187e1d96bae64dbbc6080dacf741448e9629c64fd93203b0d4" 83 | name = "github.com/prometheus/client_model" 84 | packages = ["go"] 85 | pruneopts = "UT" 86 | revision = "5c3871d89910bfb32f5fcab2aa4b9ec68e65a99f" 87 | 88 | [[projects]] 89 | branch = "master" 90 | digest = "1:ce62b400185bf6b16ef6088011b719e449f5c15c4adb6821589679f752c2788e" 91 | name = "github.com/prometheus/common" 92 | packages = [ 93 | "expfmt", 94 | "internal/bitbucket.org/ww/goautoneg", 95 | "model", 96 | ] 97 | pruneopts = "UT" 98 | revision = "b1c43a6df3aedba268353d940b5974f05037ed5c" 99 | 100 | [[projects]] 101 | branch = "master" 102 | digest = "1:08eb8b60450efe841e37512d66ce366a87d187505d7c67b99307a6c1803483a2" 103 | name = "github.com/prometheus/procfs" 104 | packages = [ 105 | ".", 106 | "internal/util", 107 | "nfs", 108 | "xfs", 109 | ] 110 | pruneopts = "UT" 111 | revision = "b1a0a9a36d7453ba0f62578b99712f3a6c5f82d1" 112 | 113 | [[projects]] 114 | digest = "1:972c2427413d41a1e06ca4897e8528e5a1622894050e2f527b38ddf0f343f759" 115 | name = "github.com/stretchr/testify" 116 | packages = ["assert"] 117 | pruneopts = "UT" 118 | revision = "ffdc059bfe9ce6a4e144ba849dbedead332c6053" 119 | version = "v1.3.0" 120 | 121 | [[projects]] 122 | digest = "1:c25289f43ac4a68d88b02245742347c94f1e108c534dda442188015ff80669b3" 123 | name = "google.golang.org/appengine" 124 | packages = ["cloudsql"] 125 | pruneopts = "UT" 126 | revision = "e9657d882bb81064595ca3b56cbe2546bbabf7b1" 127 | version = "v1.4.0" 128 | 129 | [solve-meta] 130 | analyzer-name = "dep" 131 | analyzer-version = 1 132 | input-imports = [ 133 | "github.com/go-sql-driver/mysql", 134 | "github.com/gorilla/mux", 135 | "github.com/prometheus/client_golang/prometheus", 136 | "github.com/prometheus/client_golang/prometheus/promhttp", 137 | "github.com/stretchr/testify/assert", 138 | ] 139 | solver-name = "gps-cdcl" 140 | solver-version = 1 141 | -------------------------------------------------------------------------------- /Gopkg.toml: -------------------------------------------------------------------------------- 1 | # Gopkg.toml example 2 | # 3 | # Refer to https://golang.github.io/dep/docs/Gopkg.toml.html 4 | # for detailed Gopkg.toml documentation. 5 | # 6 | # required = ["github.com/user/thing/cmd/thing"] 7 | # ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"] 8 | # 9 | # [[constraint]] 10 | # name = "github.com/user/project" 11 | # version = "1.0.0" 12 | # 13 | # [[constraint]] 14 | # name = "github.com/user/project2" 15 | # branch = "dev" 16 | # source = "github.com/myfork/project2" 17 | # 18 | # [[override]] 19 | # name = "github.com/x/y" 20 | # version = "2.4.0" 21 | # 22 | # [prune] 23 | # non-go = false 24 | # go-tests = true 25 | # unused-packages = true 26 | 27 | 28 | [prune] 29 | go-tests = true 30 | unused-packages = true 31 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Yakshaving Art 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for gitlab group manager 2 | # vim: set ft=make ts=8 noet 3 | # Copyright Yakshaving.art 4 | # Licence MIT 5 | 6 | # Variables 7 | # UNAME := $(shell uname -s) 8 | 9 | COMMIT_ID := `git log -1 --format=%H` 10 | COMMIT_DATE := `git log -1 --format=%aI` 11 | VERSION := $${CI_COMMIT_TAG:-SNAPSHOT-$(COMMIT_ID)} 12 | SHELL := /bin/bash 13 | 14 | GOOS ?= linux 15 | GOARCH ?= amd64 16 | 17 | # this is godly 18 | # https://news.ycombinator.com/item?id=11939200 19 | .PHONY: help 20 | help: ### this screen. Keep it first target to be default 21 | ifeq ($(UNAME), Linux) 22 | @grep -P '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | \ 23 | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' 24 | else 25 | @# this is not tested, but prepared in advance for you, Mac drivers 26 | @awk -F ':.*###' '$$0 ~ FS {printf "%15s%s\n", $$1 ":", $$2}' \ 27 | $(MAKEFILE_LIST) | grep -v '@awk' | sort 28 | endif 29 | 30 | # Targets 31 | # 32 | .PHONY: debug 33 | debug: ### debug Makefile itself 34 | @echo $(UNAME) 35 | 36 | .PHONY: check 37 | check: ### sanity checks 38 | @find . -type f \( -name \*.yml -o -name \*yaml \) \! -path './vendor/*' \ 39 | | xargs -r yq '.' # >/dev/null 40 | 41 | .PHONY: lint 42 | lint: check 43 | lint: ### run all the lints 44 | gometalinter 45 | 46 | .PHONY: test 47 | test: ### run all the unit tests 48 | # test: lint 49 | @go test -v -coverprofile=coverage.out $$(go list ./... | grep -v '/vendor/') \ 50 | && go tool cover -func=coverage.out 51 | 52 | .PHONY: integration 53 | integration: ### run integration tests (requires a bootstrapped local environment) 54 | @go test -v ./... -tags "integration" -coverprofile=coverage.out $$(go list ./... | grep -v '/vendor/') \ 55 | && go tool cover -func=coverage.out 56 | 57 | .PHONY: build 58 | build: ### build the binary applying the correct version from git 59 | @GOOS=$(GOOS) GOARCH=$(GOARCH) go build -ldflags "-X \ 60 | gitlab.com/yakshaving.art/alertsnitch/version.Version=$(VERSION) -X \ 61 | gitlab.com/yakshaving.art/alertsnitch/version.Commit=$(COMMIT_ID) -X \ 62 | gitlab.com/yakshaving.art/alertsnitch/version.Date=$(COMMIT_DATE)" \ 63 | -o alertsnitch-$(GOARCH) 64 | 65 | CURRENT_DIR:=$(shell pwd) 66 | 67 | .PHONY: bootstrap_local_testing 68 | bootstrap_local_testing: ### builds and bootstraps a local integration testing environment using docker-compose 69 | @if [[ -z "$(MYSQL_ROOT_PASSWORD)" ]]; then echo "MYSQL_ROOT_PASSWORD is not set" ; exit 1; fi 70 | @if [[ -z "$(MYSQL_DATABASE)" ]]; then echo "MYSQL_DATABASE is not set" ; exit 1; fi 71 | @echo "Launching alertsnitch-mysql integration container" 72 | @docker run --rm --name alertsnitch-mysql \ 73 | -e MYSQL_ROOT_PASSWORD=$(MYSQL_ROOT_PASSWORD) \ 74 | -e MYSQL_DATABASE=$(MYSQL_DATABASE) \ 75 | -p 3306:3306 \ 76 | -v $(CURRENT_DIR)/db.d/mysql:/db.scripts \ 77 | -d \ 78 | mysql:5.7 79 | @while ! docker exec alertsnitch-mysql mysql --database=$(MYSQL_DATABASE) --password=$(MYSQL_ROOT_PASSWORD) -e "SELECT 1" >/dev/null 2>&1 ; do \ 80 | echo "Waiting for database connection..." ; \ 81 | sleep 1 ; \ 82 | done 83 | @echo "Bootstrapping model" 84 | @docker exec alertsnitch-mysql sh -c "exec mysql -uroot -p$(MYSQL_ROOT_PASSWORD) $(MYSQL_DATABASE) < /db.scripts/0.0.1-bootstrap.sql" 85 | @docker exec alertsnitch-mysql sh -c "exec mysql -uroot -p$(MYSQL_ROOT_PASSWORD) $(MYSQL_DATABASE) < /db.scripts/0.1.0-fingerprint.sql" 86 | @echo "Everything is ready to run 'make integration'; remember to teardown_local_testing when you are done" 87 | 88 | .PHONY: teardown_local_testing 89 | teardown_local_testing: ### Tears down the integration testing environment 90 | docker stop alertsnitch-mysql 91 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # AlertSnitch 2 | 3 | Captures Prometheus AlertManager alerts and writes them in a MySQL or 4 | Postgres database for future examination. 5 | 6 | Because given a noisy enough alerting environment, offline querying 7 | capabilities of triggered alerts is extremely valuable. 8 | 9 | ## How does it work 10 | 11 | 1. You stand up one of these however you like (multi-arch Docker images provided) 12 | 1. You setup AlertManager to point at it and propagate your alerts in. 13 | 1. Every alert that gets triggered reaches your database. 14 | 1. Profit. 15 | 16 | ```mermaid 17 | graph TD 18 | A[alertmanager] -->|POST|B(AlertSnitch) 19 | B --> |Save|C(MySQL/PG Database) 20 | C -.-|Graph|G[Grafana] 21 | C -.-|Query|D[MySQL/PG Client] 22 | style B fill:#f9f,stroke:#333,stroke-width:1px 23 | style C fill:#00A0A0,stroke:#333,stroke-width:1px 24 | style D fill:#00C000 25 | style G fill:#00C000 26 | ``` 27 | 28 | ## Local install 29 | 30 | Simply install to your $GOPATH using your GO tools 31 | 32 | ```sh 33 | $ go get gitlab.com/yakshaving.art/alertsnitch` 34 | ``` 35 | 36 | ## Requirements 37 | 38 | To run AlertSnitch requires a MySQL or Postgres database to write to. 39 | 40 | The database must be initialized with AlertSnitch model. 41 | 42 | AlertSnitch will not become online until the model is up to date with the 43 | expected one. Bootstrapping scripts are provided in the [scripts][./script.d] 44 | folder. 45 | 46 | ## Configuration 47 | 48 | ### MySQL 49 | 50 | For specifics about how to set up the MySQL DSN refer to [Go MySQL client driver][1] 51 | 52 | This is a sample of a DSN that would connect to the local host over a Unix socket 53 | 54 | ```bash 55 | export ALERTSNITCH_BACKEND="mysql" 56 | export ALERTSNITCH_DSN="${MYSQL_USER}:${MYSQL_PASSWORD}@/${MYSQL_DATABASE}" 57 | ``` 58 | 59 | ### Postgres 60 | 61 | ```bash 62 | export ALERTSNITCH_BACKEND="postgres" 63 | export ALERTSNITCH_DSN="sslmode=disable user=${PGUSER} password=${PGPASSWORD} host=${PGHOST} database=${PGDATABASE}" 64 | ``` 65 | 66 | ## How to run 67 | 68 | ### Running with Docker 69 | 70 | **Run using docker in this very registry, for ex.** 71 | 72 | ```sh 73 | $ docker run --rm \ 74 | -p 9567:9567 \ 75 | -e ALERTSNITCH_DSN \ 76 | -e ALERTSNITCH_BACKEND \ 77 | registry.gitlab.com/yakshaving.art/alertsnitch 78 | ``` 79 | 80 | ### Running Manually 81 | 82 | 1. Open a terminal and run the following 83 | 1. Copy the AlertSnitch binary from your $GOPATH to `/usr/local/bin` with `sudo cp ~/go/bin/alertsnitch /usr/local/bin` 84 | 1. Now run AlertSnitch as with just `alertsnitch` 85 | - To just see the alerts that are being received, use the *null* backend with `ALERTSNITCH_BACKEND=null` 86 | 87 | ### Setting up in AlertManager 88 | 89 | Once AlertSnitch is up and running, configure the Prometheus Alert Manager to 90 | forward every alert to it on the `/webhooks` path. 91 | 92 | ```yaml 93 | --- 94 | receivers: 95 | - name: alertsnitch 96 | webhook_configs: 97 | - url: http://:9567/webhook 98 | ``` 99 | 100 | Then add the route 101 | 102 | ```yaml 103 | # We want to send all alerts to alertsnitch and then continue to the 104 | # appropiate handler. 105 | route: 106 | routes: 107 | - receiver: alertsnitch 108 | continue: true 109 | ``` 110 | 111 | ### Command line arguments 112 | 113 | * **-database-backend** sets the database backend to connect to, supported are `mysql`, `postgres` and `null` 114 | * **-debug** dumps the received WebHook payloads to the log so you can understand what is going on 115 | * **-listen.address** _string_ address in which to listen for HTTP requests (default ":9567") 116 | * **-version** prints the version and exit 117 | 118 | ### Environment variables 119 | 120 | - **ALERTSNITCH_DSN** *required* database connection query string 121 | - **ALERTSNITCH_ADDR** same as **-listen.address** 122 | - **ALERTSNITCH_BACKEND** same as **-database-backend** 123 | 124 | ### Readiness probe 125 | 126 | AlertSnitch offers a `/-/ready` endpoint which will return 200 if the 127 | application is ready to accept WebHook posts. 128 | 129 | During startup AlertSnitch will probe the MySQL database and the database 130 | model version. If everything is as expected it will set itself as ready. 131 | 132 | In case of failure it will return a 500 and will write the error in the 133 | response payload. 134 | 135 | ### Liveliness probe 136 | 137 | AlertSnitch offers a `/-/health` endpoint which will return 200 as long as 138 | the MySQL/Postgres database is reachable. 139 | 140 | In case of error it will return a 500 and will write the error in the 141 | response payload. 142 | 143 | ### Metrics 144 | 145 | AlertSnitch provides Prometheus metrics on `/metrics` as per Prometheus 146 | convention. 147 | 148 | ### Security 149 | 150 | There is no offering of security of any kind. AlertSnitch is not ment to be 151 | exposed to the internet but to be executed in an internal network reachable 152 | by the alert manager. 153 | 154 | ### Grafana Compatibility 155 | 156 | AlertSnitch writes alerts in such a way that they can be explored using 157 | Grafana's MySQL/Postgres Data Source plugin. Refer to Grafana documentation 158 | for further instructions. 159 | 160 | ## Testing locally 161 | 162 | We provide a couple of Makefile tasks to make it easy to run integration tests 163 | locally, to get a full coverage sample run: 164 | 165 | ```sh 166 | make bootstrap_local_testing 167 | make integration 168 | go tool cover -html=coverage.out 169 | make teardown_local_testing 170 | ``` 171 | 172 | [1]: https://github.com/go-sql-driver/mysql 173 | -------------------------------------------------------------------------------- /db.d/mysql/0.0.1-bootstrap.sql: -------------------------------------------------------------------------------- 1 | DROP PROCEDURE IF EXISTS bootstrap; 2 | 3 | DELIMITER // 4 | CREATE PROCEDURE bootstrap() 5 | BEGIN 6 | SET @exists := (SELECT 1 FROM information_schema.tables I WHERE I.table_name = "Model" AND I.table_schema = database()); 7 | IF @exists IS NULL THEN 8 | 9 | CREATE TABLE `Model` ( 10 | `ID` enum('1') NOT NULL, 11 | `version` VARCHAR(20) NOT NULL, 12 | PRIMARY KEY (`ID`) 13 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8; 14 | 15 | INSERT INTO `Model` (`version`) VALUES ("0.0.1"); 16 | 17 | ELSE 18 | SIGNAL SQLSTATE '42000' SET MESSAGE_TEXT='Model Table Exists, quitting...'; 19 | END IF; 20 | END; 21 | // 22 | DELIMITER ; 23 | 24 | -- Execute the procedure 25 | CALL bootstrap(); 26 | 27 | -- Drop the procedure 28 | DROP PROCEDURE bootstrap; 29 | 30 | -- Create the rest of the tables 31 | CREATE TABLE `AlertGroup` ( 32 | `ID` INT NOT NULL AUTO_INCREMENT, 33 | `time` TIMESTAMP NOT NULL, 34 | `receiver` VARCHAR(100) NOT NULL, 35 | `status` VARCHAR(50) NOT NULL, 36 | `externalURL` TEXT NOT NULL, 37 | `groupKey` VARCHAR(255) NOT NULL, 38 | KEY `idx_time` (`time`) USING BTREE, 39 | KEY `idx_status_ts` (`status`, `time`) USING BTREE, 40 | PRIMARY KEY (`ID`) 41 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8; 42 | 43 | CREATE TABLE `GroupLabel` ( 44 | `ID` INT NOT NULL AUTO_INCREMENT, 45 | `AlertGroupID` INT NOT NULL, 46 | `GroupLabel` VARCHAR(100) NOT NULL, 47 | `Value` VARCHAR(1000) NOT NULL, 48 | FOREIGN KEY (AlertGroupID) REFERENCES AlertGroup (ID) ON DELETE CASCADE, 49 | PRIMARY KEY (`ID`) 50 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8; 51 | 52 | CREATE TABLE `CommonLabel` ( 53 | `ID` INT NOT NULL AUTO_INCREMENT, 54 | `AlertGroupID` INT NOT NULL, 55 | `Label` VARCHAR(100) NOT NULL, 56 | `Value` VARCHAR(1000) NOT NULL, 57 | FOREIGN KEY (AlertGroupID) REFERENCES AlertGroup (ID) ON DELETE CASCADE, 58 | PRIMARY KEY (`ID`) 59 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8; 60 | 61 | CREATE TABLE `CommonAnnotation` ( 62 | `ID` INT NOT NULL AUTO_INCREMENT, 63 | `AlertGroupID` INT NOT NULL, 64 | `Annotation` VARCHAR(100) NOT NULL, 65 | `Value` VARCHAR(1000) NOT NULL, 66 | FOREIGN KEY (AlertGroupID) REFERENCES AlertGroup (ID) ON DELETE CASCADE, 67 | PRIMARY KEY (`ID`) 68 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8; 69 | 70 | CREATE TABLE `Alert` ( 71 | `ID` INT NOT NULL AUTO_INCREMENT, 72 | `alertGroupID` INT NOT NULL, 73 | `status` VARCHAR(50) NOT NULL, 74 | `startsAt` DATETIME NOT NULL, 75 | `endsAt` DATETIME DEFAULT NULL, 76 | `generatorURL` TEXT NOT NULL, 77 | FOREIGN KEY (alertGroupID) REFERENCES AlertGroup (ID) ON DELETE CASCADE, 78 | PRIMARY KEY (`ID`) 79 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8; 80 | 81 | CREATE TABLE `AlertLabel` ( 82 | `ID` INT NOT NULL AUTO_INCREMENT, 83 | `AlertID` INT NOT NULL, 84 | `Label` VARCHAR(100) NOT NULL, 85 | `Value` VARCHAR(1000) NOT NULL, 86 | FOREIGN KEY (AlertID) REFERENCES Alert (ID) ON DELETE CASCADE, 87 | PRIMARY KEY (`ID`) 88 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8; 89 | 90 | CREATE TABLE `AlertAnnotation` ( 91 | `ID` INT NOT NULL AUTO_INCREMENT, 92 | `AlertID` INT NOT NULL, 93 | `Annotation` VARCHAR(100) NOT NULL, 94 | `Value` VARCHAR(1000) NOT NULL, 95 | FOREIGN KEY (AlertID) REFERENCES Alert (ID) ON DELETE CASCADE, 96 | PRIMARY KEY (`ID`) 97 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8; 98 | -------------------------------------------------------------------------------- /db.d/mysql/0.1.0-fingerprint.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE Alert 2 | ADD `fingerprint` TEXT NOT NULL 3 | ; 4 | 5 | UPDATE `Model` SET `version`="0.1.0"; -------------------------------------------------------------------------------- /db.d/postgres/0.0.1-bootstrap.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE Model ( 2 | ID INTEGER UNIQUE DEFAULT(1), 3 | version VARCHAR(20) NOT NULL, 4 | CONSTRAINT single_row CHECK (ID = 1) 5 | ); 6 | 7 | INSERT INTO Model (version) VALUES ('0.0.1'); 8 | 9 | -- Create the rest of the tables 10 | CREATE TABLE AlertGroup ( 11 | ID SERIAL NOT NULL PRIMARY KEY, 12 | time TIMESTAMP NOT NULL, 13 | receiver VARCHAR(100) NOT NULL, 14 | status VARCHAR(50) NOT NULL, 15 | externalURL TEXT NOT NULL, 16 | groupKey VARCHAR(255) NOT NULL 17 | ); 18 | 19 | CREATE INDEX ON AlertGroup (time); 20 | 21 | CREATE INDEX ON AlertGroup (status, time); 22 | 23 | CREATE TABLE GroupLabel ( 24 | ID SERIAL NOT NULL PRIMARY KEY, 25 | AlertGroupID INT NOT NULL references AlertGroup(ID), 26 | GroupLabel VARCHAR(100) NOT NULL, 27 | Value VARCHAR(1000) NOT NULL 28 | ); 29 | 30 | CREATE TABLE CommonLabel ( 31 | ID SERIAL NOT NULL PRIMARY KEY, 32 | AlertGroupID INT NOT NULL references AlertGroup (ID), 33 | Label VARCHAR(100) NOT NULL, 34 | Value VARCHAR(1000) NOT NULL 35 | ); 36 | 37 | CREATE TABLE CommonAnnotation ( 38 | ID SERIAL NOT NULL PRIMARY KEY, 39 | AlertGroupID INT NOT NULL REFERENCES AlertGroup (ID), 40 | Annotation VARCHAR(100) NOT NULL, 41 | Value VARCHAR(1000) NOT NULL 42 | ); 43 | 44 | CREATE TABLE Alert ( 45 | ID SERIAL NOT NULL PRIMARY KEY, 46 | alertGroupID INT NOT NULL REFERENCES AlertGroup (ID), 47 | status VARCHAR(50) NOT NULL, 48 | startsAt TIMESTAMP NOT NULL, 49 | endsAt TIMESTAMP NULL, 50 | generatorURL TEXT NOT NULL 51 | ); 52 | 53 | CREATE TABLE AlertLabel ( 54 | ID SERIAL NOT NULL PRIMARY KEY, 55 | AlertID INT NOT NULL REFERENCES Alert (ID), 56 | Label VARCHAR(100) NOT NULL, 57 | Value VARCHAR(1000) NOT NULL 58 | ); 59 | 60 | CREATE TABLE AlertAnnotation ( 61 | ID SERIAL NOT NULL PRIMARY KEY, 62 | AlertID INT NOT NULL REFERENCES Alert (ID), 63 | Annotation VARCHAR(100) NOT NULL, 64 | Value VARCHAR(1000) NOT NULL 65 | ); 66 | -------------------------------------------------------------------------------- /db.d/postgres/0.1.0-fingerprint.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE Alert 2 | ADD fingerprint TEXT NOT NULL default '' 3 | ; 4 | 5 | UPDATE Model SET version='0.1.0'; -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.1' 2 | 3 | services: 4 | alertsnitch: 5 | image: registry.gitlab.com/yakshaving.art/alertsnitch:0.2 6 | ports: 7 | - "9567:9567" 8 | environment: 9 | ALERTSNITCH_DSN: "alertsnitch:alertsnitch@tcp(mysqldb)/alertsnitch" 10 | ALERSTNITCH_BACKEND: "mysql" 11 | depends_on: 12 | mysqldb: 13 | condition: service_healthy 14 | 15 | mysqldb: 16 | restart: always 17 | image: mysql:5.7 18 | command: --default-authentication-plugin=mysql_native_password 19 | volumes: 20 | - ./db.d/mysql:/docker-entrypoint-initdb.d 21 | ports: 22 | - "3306:3306" 23 | environment: 24 | MYSQL_DATABASE: alertsnitch 25 | MYSQL_USER: "alertsnitch" 26 | MYSQL_PASSWORD: "alertsnitch" 27 | MYSQL_ROOT_PASSWORD: "root" 28 | healthcheck: 29 | test: ["CMD", "mysqladmin" ,"ping", "-h", "localhost", "-proot"] 30 | timeout: 20s 31 | retries: 10 -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module gitlab.com/yakshaving.art/alertsnitch 2 | 3 | go 1.14 4 | 5 | require ( 6 | github.com/go-sql-driver/mysql v1.5.0 7 | github.com/gorilla/context v1.1.1 // indirect 8 | github.com/gorilla/mux v1.6.2 9 | github.com/lib/pq v1.9.0 10 | github.com/prometheus/client_golang v0.9.2 11 | github.com/prometheus/common v0.0.0-20190104105734-b1c43a6df3ae // indirect 12 | github.com/prometheus/procfs v0.0.0-20190104112138-b1a0a9a36d74 // indirect 13 | github.com/sirupsen/logrus v1.2.0 14 | github.com/stretchr/testify v1.3.0 15 | ) 16 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= 2 | github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= 3 | github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= 4 | github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= 5 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 6 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 7 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 8 | github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= 9 | github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= 10 | github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= 11 | github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= 12 | github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= 13 | github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= 14 | github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= 15 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 16 | github.com/gorilla/context v1.1.1 h1:AWwleXJkX/nhcU9bZSnZoi3h/qGYqQAGhq6zZe/aQW8= 17 | github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= 18 | github.com/gorilla/mux v1.6.2 h1:Pgr17XVTNXAk3q/r4CpKzC5xBM/qW1uVLV+IhRZpIIk= 19 | github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= 20 | github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= 21 | github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= 22 | github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 23 | github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= 24 | github.com/lib/pq v1.9.0 h1:L8nSXQQzAYByakOFMTwpjRoHsMJklur4Gi59b6VivR8= 25 | github.com/lib/pq v1.9.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= 26 | github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= 27 | github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= 28 | github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= 29 | github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 30 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 31 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 32 | github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= 33 | github.com/prometheus/client_golang v0.9.2 h1:awm861/B8OKDd2I/6o1dy3ra4BamzKhYOiGItCeZ740= 34 | github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= 35 | github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8= 36 | github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= 37 | github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= 38 | github.com/prometheus/common v0.0.0-20190104105734-b1c43a6df3ae h1:iq3e1tH4dCzdqscIkWimcnzYt6Pkz0zOzHSgV9cb5DE= 39 | github.com/prometheus/common v0.0.0-20190104105734-b1c43a6df3ae/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= 40 | github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= 41 | github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= 42 | github.com/prometheus/procfs v0.0.0-20190104112138-b1a0a9a36d74 h1:d1Xoc24yp/pXmWl2leBiBA+Tptce6cQsA+MMx/nOOcY= 43 | github.com/prometheus/procfs v0.0.0-20190104112138-b1a0a9a36d74/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= 44 | github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo= 45 | github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= 46 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 47 | github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 48 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 49 | github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= 50 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 51 | golang.org/x/crypto v0.0.0-20180904163835-0709b304e793 h1:u+LnwYTOOW7Ukr/fppxEb1Nwz0AtPflrblfvUudpo+I= 52 | golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 53 | golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 54 | golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 55 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f h1:Bl/8QSvNqXvPGPGXa2z5xUTmV7VDcZyvRZ+QQXkXTZQ= 56 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 57 | golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 58 | golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5 h1:mzjBh+S5frKOsOBobWIMAbXavqjmgO17k/2puhcFR94= 59 | golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 60 | gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= 61 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 62 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 63 | -------------------------------------------------------------------------------- /internal/db/db.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "fmt" 5 | 6 | "gitlab.com/yakshaving.art/alertsnitch/internal" 7 | ) 8 | 9 | // SupportedModel stores the model that is supported by this application 10 | const SupportedModel = "0.1.0" 11 | 12 | // ConnectionArgs required to create a MySQL connection 13 | type ConnectionArgs struct { 14 | DSN string 15 | MaxIdleConns int 16 | MaxOpenConns int 17 | MaxConnLifetimeSeconds int 18 | } 19 | 20 | // Connect connects to a backend database 21 | func Connect(backend string, args ConnectionArgs) (internal.Storer, error) { 22 | switch backend { 23 | case "mysql": 24 | return connectMySQL(args) 25 | 26 | case "postgres": 27 | return connectPG(args) 28 | 29 | case "null": 30 | return NullDB{}, nil 31 | 32 | default: 33 | return nil, fmt.Errorf("Invalid backend %q", backend) 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /internal/db/db_test.go: -------------------------------------------------------------------------------- 1 | // +build integration 2 | 3 | package db_test 4 | 5 | import ( 6 | "io/ioutil" 7 | "os" 8 | "testing" 9 | 10 | _ "github.com/go-sql-driver/mysql" 11 | _ "github.com/lib/pq" 12 | 13 | "github.com/stretchr/testify/assert" 14 | 15 | "gitlab.com/yakshaving.art/alertsnitch/internal" 16 | "gitlab.com/yakshaving.art/alertsnitch/internal/db" 17 | "gitlab.com/yakshaving.art/alertsnitch/internal/webhook" 18 | ) 19 | 20 | func TestPingingDatabaseWorks(t *testing.T) { 21 | backend := os.Getenv("ALERTSNITCH_BACKEND") 22 | 23 | a := assert.New(t) 24 | driver, err := db.Connect(backend, connectionArgs()) 25 | a.NoError(err) 26 | a.NotNilf(driver, "database driver is nil?") 27 | a.NoErrorf(driver.Ping(), "failed to ping database") 28 | a.NoErrorf(driver.CheckModel(), "failed to check the model") 29 | } 30 | 31 | func TestSavingAnAlertWorks(t *testing.T) { 32 | a := assert.New(t) 33 | 34 | b, err := ioutil.ReadFile("../webhook/sample-payload.json") 35 | a.NoError(err) 36 | 37 | data, err := webhook.Parse(b) 38 | a.NoError(err) 39 | 40 | backend := os.Getenv("ALERTSNITCH_BACKEND") 41 | 42 | driver, err := db.Connect(backend, connectionArgs()) 43 | a.NoError(err) 44 | 45 | a.NoError(driver.Save(data)) 46 | } 47 | 48 | func TestSavingAFiringAlertWorks(t *testing.T) { 49 | a := assert.New(t) 50 | 51 | b, err := ioutil.ReadFile("../webhook/sample-payload-invalid-ends-at.json") 52 | a.NoError(err) 53 | 54 | data, err := webhook.Parse(b) 55 | a.NoError(err) 56 | 57 | backend := os.Getenv("ALERTSNITCH_BACKEND") 58 | driver, err := db.Connect(backend, connectionArgs()) 59 | a.NoError(err) 60 | 61 | a.NoError(driver.Save(data)) 62 | } 63 | 64 | func connectionArgs() db.ConnectionArgs { 65 | return db.ConnectionArgs{ 66 | DSN: os.Getenv(internal.DSNVar), 67 | MaxIdleConns: 1, 68 | MaxOpenConns: 2, 69 | MaxConnLifetimeSeconds: 600, 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /internal/db/mysql.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "time" 7 | 8 | "database/sql" 9 | 10 | "github.com/sirupsen/logrus" 11 | "gitlab.com/yakshaving.art/alertsnitch/internal" 12 | "gitlab.com/yakshaving.art/alertsnitch/internal/metrics" 13 | ) 14 | 15 | // MySQLDB A database that does nothing 16 | type MySQLDB struct { 17 | db *sql.DB 18 | } 19 | 20 | // ConnectMySQL connect to a MySQL database using the provided data source name 21 | func connectMySQL(args ConnectionArgs) (*MySQLDB, error) { 22 | if args.DSN == "" { 23 | return nil, fmt.Errorf("Empty DSN provided, can't connect to MySQL database") 24 | } 25 | 26 | logrus.Debugf("Connecting to MySQL database with DSN: %s", args.DSN) 27 | 28 | connection, err := sql.Open("mysql", args.DSN) 29 | if err != nil { 30 | return nil, fmt.Errorf("failed to open MySQL connection: %s", err) 31 | } 32 | 33 | connection.SetMaxIdleConns(args.MaxIdleConns) 34 | connection.SetMaxOpenConns(args.MaxOpenConns) 35 | connection.SetConnMaxLifetime(time.Duration(args.MaxConnLifetimeSeconds) * time.Second) 36 | 37 | database := &MySQLDB{ 38 | db: connection, 39 | } 40 | 41 | err = database.Ping() 42 | if err != nil { 43 | return nil, err 44 | } 45 | logrus.Debug("Connected to MySQL database") 46 | 47 | return database, database.CheckModel() 48 | } 49 | 50 | // Save implements Storer interface 51 | func (d MySQLDB) Save(data *internal.AlertGroup) error { 52 | return d.unitOfWork(func(tx *sql.Tx) error { 53 | r, err := tx.Exec(` 54 | INSERT INTO AlertGroup (time, receiver, status, externalURL, groupKey) 55 | VALUES (now(), ?, ?, ?, ?)`, data.Receiver, data.Status, data.ExternalURL, data.GroupKey) 56 | if err != nil { 57 | return fmt.Errorf("failed to insert into AlertGroups: %s", err) 58 | } 59 | 60 | alertGroupID, err := r.LastInsertId() // alertGroupID 61 | if err != nil { 62 | return fmt.Errorf("failed to get AlertGroups inserted id: %s", err) 63 | } 64 | 65 | for k, v := range data.GroupLabels { 66 | _, err := tx.Exec(` 67 | INSERT INTO GroupLabel (alertGroupID, GroupLabel, Value) 68 | VALUES (?, ?, ?)`, alertGroupID, k, v) 69 | if err != nil { 70 | return fmt.Errorf("failed to insert into GroupLabel: %s", err) 71 | } 72 | } 73 | for k, v := range data.CommonLabels { 74 | _, err := tx.Exec(` 75 | INSERT INTO CommonLabel (alertGroupID, Label, Value) 76 | VALUES (?, ?, ?)`, alertGroupID, k, v) 77 | if err != nil { 78 | return fmt.Errorf("failed to insert into CommonLabel: %s", err) 79 | } 80 | } 81 | for k, v := range data.CommonAnnotations { 82 | _, err := tx.Exec(` 83 | INSERT INTO CommonAnnotation (alertGroupID, Annotation, Value) 84 | VALUES (?, ?, ?)`, alertGroupID, k, v) 85 | if err != nil { 86 | return fmt.Errorf("failed to insert into CommonAnnotation: %s", err) 87 | } 88 | } 89 | 90 | for _, alert := range data.Alerts { 91 | var result sql.Result 92 | if alert.EndsAt.Before(alert.StartsAt) { 93 | result, err = tx.Exec(` 94 | INSERT INTO Alert (alertGroupID, status, startsAt, generatorURL, fingerprint) 95 | VALUES (?, ?, ?, ?, ?)`, 96 | alertGroupID, alert.Status, alert.StartsAt, alert.GeneratorURL, alert.Fingerprint) 97 | } else { 98 | result, err = tx.Exec(` 99 | INSERT INTO Alert (alertGroupID, status, startsAt, endsAt, generatorURL, fingerprint) 100 | VALUES (?, ?, ?, ?, ?, ?)`, 101 | alertGroupID, alert.Status, alert.StartsAt, alert.EndsAt, alert.GeneratorURL, alert.Fingerprint) 102 | } 103 | if err != nil { 104 | return fmt.Errorf("failed to insert into Alert: %s", err) 105 | } 106 | 107 | alertID, err := result.LastInsertId() 108 | if err != nil { 109 | return fmt.Errorf("failed to get Alert inserted id: %s", err) 110 | } 111 | 112 | for k, v := range alert.Labels { 113 | _, err := tx.Exec(` 114 | INSERT INTO AlertLabel (AlertID, Label, Value) 115 | VALUES (?, ?, ?)`, alertID, k, v) 116 | if err != nil { 117 | return fmt.Errorf("failed to insert into AlertLabel: %s", err) 118 | } 119 | } 120 | for k, v := range alert.Annotations { 121 | _, err := tx.Exec(` 122 | INSERT INTO AlertAnnotation (AlertID, Annotation, Value) 123 | VALUES (?, ?, ?)`, alertID, k, v) 124 | if err != nil { 125 | return fmt.Errorf("failed to insert into AlertAnnotation: %s", err) 126 | } 127 | } 128 | } 129 | 130 | return nil 131 | }) 132 | } 133 | 134 | func (d MySQLDB) unitOfWork(f func(*sql.Tx) error) error { 135 | tx, err := d.db.Begin() 136 | if err != nil { 137 | return fmt.Errorf("failed to begin transaction: %s", err) 138 | } 139 | 140 | err = f(tx) 141 | 142 | if err != nil { 143 | e := tx.Rollback() 144 | if e != nil { 145 | return fmt.Errorf("failed to rollback transaction (%s) after failing execution: %s", e, err) 146 | } 147 | return fmt.Errorf("failed execution: %s", err) 148 | } 149 | err = tx.Commit() 150 | if err != nil { 151 | return fmt.Errorf("failed to commit transaction: %s", err) 152 | } 153 | return nil 154 | } 155 | 156 | // Ping implements Storer interface 157 | func (d MySQLDB) Ping() error { 158 | ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) 159 | defer cancel() 160 | 161 | if err := d.db.PingContext(ctx); err != nil { 162 | metrics.DatabaseUp.Set(0) 163 | logrus.Debugf("Failed to ping database: %s", err) 164 | return err 165 | } 166 | metrics.DatabaseUp.Set(1) 167 | 168 | logrus.Debugf("Pinged database...") 169 | return nil 170 | } 171 | 172 | // CheckModel implements Storer interface 173 | func (d MySQLDB) CheckModel() error { 174 | rows, err := d.db.Query("SELECT version FROM Model") 175 | if err != nil { 176 | return fmt.Errorf("failed to fetch model version from the database: %s", err) 177 | } 178 | defer rows.Close() 179 | 180 | if !rows.Next() { 181 | return fmt.Errorf("failed to read model version from the database: empty resultset") 182 | } 183 | 184 | var model string 185 | if err := rows.Scan(&model); err != nil { 186 | return fmt.Errorf("failed to read model version from the database: %s", err) 187 | } 188 | 189 | if model != SupportedModel { 190 | return fmt.Errorf("database model '%s' is not supported by this application (%s)", model, SupportedModel) 191 | } 192 | 193 | return nil 194 | } 195 | 196 | func (MySQLDB) String() string { 197 | return "mysql database driver" 198 | } 199 | -------------------------------------------------------------------------------- /internal/db/nulldb.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "log" 5 | 6 | "gitlab.com/yakshaving.art/alertsnitch/internal" 7 | ) 8 | 9 | // NullDB A database that does nothing 10 | type NullDB struct{} 11 | 12 | // Save implements Storer interface 13 | func (NullDB) Save(data *internal.AlertGroup) error { 14 | log.Printf("save alert %#v\n", data) 15 | return nil 16 | } 17 | 18 | // Ping implements Storer interface 19 | func (NullDB) Ping() error { 20 | log.Println("pong") 21 | return nil 22 | } 23 | 24 | // CheckModel implements Storer interface 25 | func (NullDB) CheckModel() error { 26 | log.Println("check model") 27 | return nil 28 | } 29 | 30 | func (NullDB) String() string { 31 | return "null database driver" 32 | } 33 | -------------------------------------------------------------------------------- /internal/db/nulldb_test.go: -------------------------------------------------------------------------------- 1 | package db_test 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "gitlab.com/yakshaving.art/alertsnitch/internal/db" 6 | "testing" 7 | ) 8 | 9 | func TestNullDBObject(t *testing.T) { 10 | a := assert.New(t) 11 | 12 | n := db.NullDB{} 13 | a.Equal(n.String(), "null database driver") 14 | 15 | a.Nil(n.Save(nil)) 16 | a.NoError(n.Ping()) 17 | a.NoError(n.CheckModel()) 18 | } 19 | -------------------------------------------------------------------------------- /internal/db/postgres.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "time" 7 | 8 | "database/sql" 9 | 10 | "github.com/sirupsen/logrus" 11 | "gitlab.com/yakshaving.art/alertsnitch/internal" 12 | "gitlab.com/yakshaving.art/alertsnitch/internal/metrics" 13 | ) 14 | 15 | // PostgresDB A database that does nothing 16 | type PostgresDB struct { 17 | db *sql.DB 18 | } 19 | 20 | // ConnectPG connect to a Postgres database using the provided data source name 21 | func connectPG(args ConnectionArgs) (*PostgresDB, error) { 22 | if args.DSN == "" { 23 | return nil, fmt.Errorf("Empty DSN provided, can't connect to Postgres database") 24 | } 25 | 26 | logrus.Debugf("Connecting to Postgres database with DSN: %s", args.DSN) 27 | 28 | connection, err := sql.Open("postgres", args.DSN) 29 | if err != nil { 30 | return nil, fmt.Errorf("failed to open Postgres connection: %s", err) 31 | } 32 | 33 | connection.SetMaxIdleConns(args.MaxIdleConns) 34 | connection.SetMaxOpenConns(args.MaxOpenConns) 35 | connection.SetConnMaxLifetime(time.Duration(args.MaxConnLifetimeSeconds) * time.Second) 36 | 37 | database := &PostgresDB{ 38 | db: connection, 39 | } 40 | 41 | err = database.Ping() 42 | if err != nil { 43 | return nil, err 44 | } 45 | logrus.Debugf("Connected to Postgres database") 46 | 47 | return database, database.CheckModel() 48 | } 49 | 50 | // Save implements Storer interface 51 | func (d PostgresDB) Save(data *internal.AlertGroup) error { 52 | return d.unitOfWork(func(tx *sql.Tx) error { 53 | r := tx.QueryRow(` 54 | INSERT INTO AlertGroup (time, receiver, status, externalURL, groupKey) 55 | VALUES (current_timestamp, $1, $2, $3, $4) RETURNING ID`, data.Receiver, data.Status, data.ExternalURL, data.GroupKey) 56 | 57 | var alertGroupID int64 58 | err := r.Scan(&alertGroupID) 59 | if err != nil { 60 | return fmt.Errorf("failed to insert into AlertGroups: %s", err) 61 | } 62 | 63 | for k, v := range data.GroupLabels { 64 | _, err := tx.Exec(` 65 | INSERT INTO GroupLabel (alertGroupID, GroupLabel, Value) 66 | VALUES ($1, $2, $3)`, alertGroupID, k, v) 67 | if err != nil { 68 | return fmt.Errorf("failed to insert into GroupLabel: %s", err) 69 | } 70 | } 71 | for k, v := range data.CommonLabels { 72 | _, err := tx.Exec(` 73 | INSERT INTO CommonLabel (alertGroupID, Label, Value) 74 | VALUES ($1, $2, $3)`, alertGroupID, k, v) 75 | if err != nil { 76 | return fmt.Errorf("failed to insert into CommonLabel: %s", err) 77 | } 78 | } 79 | for k, v := range data.CommonAnnotations { 80 | _, err := tx.Exec(` 81 | INSERT INTO CommonAnnotation (alertGroupID, Annotation, Value) 82 | VALUES ($1, $2, $3)`, alertGroupID, k, v) 83 | if err != nil { 84 | return fmt.Errorf("failed to insert into CommonAnnotation: %s", err) 85 | } 86 | } 87 | 88 | for _, alert := range data.Alerts { 89 | if alert.EndsAt.Before(alert.StartsAt) { 90 | r = tx.QueryRow(` 91 | INSERT INTO Alert (alertGroupID, status, startsAt, generatorURL, fingerprint) 92 | VALUES ($1, $2, $3, $4, $5) RETURNING ID`, 93 | alertGroupID, alert.Status, alert.StartsAt, alert.GeneratorURL, alert.Fingerprint) 94 | } else { 95 | r = tx.QueryRow(` 96 | INSERT INTO Alert (alertGroupID, status, startsAt, endsAt, generatorURL, fingerprint) 97 | VALUES ($1, $2, $3, $4, $5, $6) RETURNING ID`, 98 | alertGroupID, alert.Status, alert.StartsAt, alert.EndsAt, alert.GeneratorURL, alert.Fingerprint) 99 | } 100 | var alertID int64 101 | if err := r.Scan(&alertID); err != nil { 102 | return fmt.Errorf("failed to insert into Alert: %s", err) 103 | } 104 | 105 | for k, v := range alert.Labels { 106 | _, err := tx.Exec(` 107 | INSERT INTO AlertLabel (AlertID, Label, Value) 108 | VALUES ($1, $2, $3)`, alertID, k, v) 109 | if err != nil { 110 | return fmt.Errorf("failed to insert into AlertLabel: %s", err) 111 | } 112 | } 113 | for k, v := range alert.Annotations { 114 | _, err := tx.Exec(` 115 | INSERT INTO AlertAnnotation (AlertID, Annotation, Value) 116 | VALUES ($1, $2, $3)`, alertID, k, v) 117 | if err != nil { 118 | return fmt.Errorf("failed to insert into AlertAnnotation: %s", err) 119 | } 120 | } 121 | } 122 | 123 | return nil 124 | }) 125 | } 126 | 127 | func (d PostgresDB) unitOfWork(f func(*sql.Tx) error) error { 128 | tx, err := d.db.Begin() 129 | if err != nil { 130 | return fmt.Errorf("failed to begin transaction: %s", err) 131 | } 132 | 133 | err = f(tx) 134 | 135 | if err != nil { 136 | e := tx.Rollback() 137 | if e != nil { 138 | return fmt.Errorf("failed to rollback transaction (%s) after failing execution: %s", e, err) 139 | } 140 | return fmt.Errorf("failed execution: %s", err) 141 | } 142 | err = tx.Commit() 143 | if err != nil { 144 | return fmt.Errorf("failed to commit transaction: %s", err) 145 | } 146 | return nil 147 | } 148 | 149 | // Ping implements Storer interface 150 | func (d PostgresDB) Ping() error { 151 | ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) 152 | defer cancel() 153 | 154 | if err := d.db.PingContext(ctx); err != nil { 155 | metrics.DatabaseUp.Set(0) 156 | logrus.Debugf("Failed to ping database: %s", err) 157 | 158 | return err 159 | } 160 | metrics.DatabaseUp.Set(1) 161 | logrus.Debugf("Pinged database...") 162 | return nil 163 | } 164 | 165 | // CheckModel implements Storer interface 166 | func (d PostgresDB) CheckModel() error { 167 | rows, err := d.db.Query("SELECT version FROM Model") 168 | if err != nil { 169 | return fmt.Errorf("failed to fetch model version from the database: %s", err) 170 | } 171 | defer rows.Close() 172 | 173 | if !rows.Next() { 174 | return fmt.Errorf("failed to read model version from the database: empty resultset") 175 | } 176 | 177 | var model string 178 | if err := rows.Scan(&model); err != nil { 179 | return fmt.Errorf("failed to read model version from the database: %s", err) 180 | } 181 | 182 | if model != SupportedModel { 183 | return fmt.Errorf("model '%s' is not supported by this application (%s)", model, SupportedModel) 184 | } 185 | 186 | return nil 187 | } 188 | 189 | func (PostgresDB) String() string { 190 | return "postgres database driver" 191 | } 192 | -------------------------------------------------------------------------------- /internal/internal.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | // DSNVar Environment variable in which the DSN is stored 8 | const DSNVar = "ALERTSNITCH_DSN" 9 | 10 | // Storer saves an Alert Data into a persistence engine 11 | type Storer interface { 12 | Save(*AlertGroup) error 13 | Ping() error 14 | CheckModel() error 15 | } 16 | 17 | // AlertGroup is the data read from a webhook call 18 | type AlertGroup struct { 19 | Version string `json:"version"` 20 | GroupKey string `json:"groupKey"` 21 | 22 | Receiver string `json:"receiver"` 23 | Status string `json:"status"` 24 | Alerts Alerts `json:"alerts"` 25 | 26 | GroupLabels map[string]string `json:"groupLabels"` 27 | CommonLabels map[string]string `json:"commonLabels"` 28 | CommonAnnotations map[string]string `json:"commonAnnotations"` 29 | 30 | ExternalURL string `json:"externalURL"` 31 | } 32 | 33 | // Alerts is a slice of Alert 34 | type Alerts []Alert 35 | 36 | // Alert holds one alert for notification templates. 37 | type Alert struct { 38 | Status string `json:"status"` 39 | Labels map[string]string `json:"labels"` 40 | Annotations map[string]string `json:"annotations"` 41 | StartsAt time.Time `json:"startsAt"` 42 | EndsAt time.Time `json:"endsAt"` 43 | GeneratorURL string `json:"generatorURL"` 44 | Fingerprint string `json:"fingerprint"` 45 | } 46 | -------------------------------------------------------------------------------- /internal/metrics/metrics.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/prometheus/client_golang/prometheus" 7 | 8 | "gitlab.com/yakshaving.art/alertsnitch/version" 9 | ) 10 | 11 | var ( 12 | namespace = "alertsnitch" 13 | 14 | bootTime = prometheus.NewGauge(prometheus.GaugeOpts{ 15 | Namespace: namespace, 16 | Name: "boot_time_seconds", 17 | Help: "unix timestamp of when the service was started", 18 | }) 19 | 20 | buildInfo = prometheus.NewGaugeVec(prometheus.GaugeOpts{ 21 | Namespace: namespace, 22 | Name: "build_info", 23 | Help: "Build information", 24 | }, []string{"version", "commit", "date"}) 25 | ) 26 | 27 | // Exported metrics 28 | var ( 29 | AlertsReceivedTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ 30 | Namespace: namespace, 31 | Subsystem: "alerts", 32 | Name: "received_total", 33 | Help: "total number of valid alerts received", 34 | }, []string{"receiver", "status"}) 35 | WebhooksReceivedTotal = prometheus.NewCounter(prometheus.CounterOpts{ 36 | Namespace: namespace, 37 | Subsystem: "webhooks", 38 | Name: "received_total", 39 | Help: "total number of webhooks posts received", 40 | }) 41 | InvalidWebhooksTotal = prometheus.NewCounter(prometheus.CounterOpts{ 42 | Namespace: namespace, 43 | Subsystem: "webhooks", 44 | Name: "invalid_total", 45 | Help: "total number of invalid webhooks received", 46 | }) 47 | 48 | DatabaseUp = prometheus.NewGauge(prometheus.GaugeOpts{ 49 | Namespace: namespace, 50 | Subsystem: "database", 51 | Name: "up", 52 | Help: "wether the database is accessible or not", 53 | }) 54 | 55 | AlertsSavedTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ 56 | Namespace: namespace, 57 | Subsystem: "alerts", 58 | Name: "saved_total", 59 | Help: "total number of alerts saved in the database", 60 | }, []string{"receiver", "status"}) 61 | AlertsSavingFailuresTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ 62 | Namespace: namespace, 63 | Subsystem: "alerts", 64 | Name: "saving_failures_total", 65 | Help: "total number of alerts that failed to be saved in the database", 66 | }, []string{"receiver", "status"}) 67 | ) 68 | 69 | func init() { 70 | bootTime.Set(float64(time.Now().Unix())) 71 | 72 | buildInfo.WithLabelValues(version.Version, version.Commit, version.Date).Set(1) 73 | 74 | prometheus.MustRegister(bootTime) 75 | prometheus.MustRegister(buildInfo) 76 | prometheus.MustRegister(DatabaseUp) 77 | 78 | prometheus.MustRegister(AlertsReceivedTotal) 79 | prometheus.MustRegister(AlertsSavedTotal) 80 | prometheus.MustRegister(AlertsSavingFailuresTotal) 81 | 82 | prometheus.MustRegister(WebhooksReceivedTotal) 83 | prometheus.MustRegister(InvalidWebhooksTotal) 84 | 85 | } 86 | -------------------------------------------------------------------------------- /internal/metrics/metrics_test.go: -------------------------------------------------------------------------------- 1 | package metrics_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/prometheus/client_golang/prometheus" 7 | "github.com/stretchr/testify/assert" 8 | 9 | "gitlab.com/yakshaving.art/alertsnitch/internal/metrics" 10 | ) 11 | 12 | func TestMetricsAreRegistered(t *testing.T) { 13 | a := assert.New(t) 14 | a.True(prometheus.DefaultRegisterer.Unregister(metrics.AlertsReceivedTotal), 15 | "alerts received total not registered") 16 | a.True(prometheus.DefaultRegisterer.Unregister(metrics.AlertsSavedTotal), 17 | "alerts saved total not registered") 18 | a.True(prometheus.DefaultRegisterer.Unregister(metrics.DatabaseUp), 19 | "database up") 20 | a.True(prometheus.DefaultRegisterer.Unregister(metrics.InvalidWebhooksTotal), 21 | "invalid webhooks total") 22 | a.True(prometheus.DefaultRegisterer.Unregister(metrics.WebhooksReceivedTotal), 23 | "webhooks received total") 24 | a.True(prometheus.DefaultRegisterer.Unregister(metrics.AlertsSavingFailuresTotal), 25 | "alerts failed to be saved") 26 | 27 | } 28 | -------------------------------------------------------------------------------- /internal/server/server.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "log" 7 | "net/http" 8 | 9 | "github.com/gorilla/mux" 10 | "github.com/prometheus/client_golang/prometheus/promhttp" 11 | 12 | "gitlab.com/yakshaving.art/alertsnitch/internal" 13 | "gitlab.com/yakshaving.art/alertsnitch/internal/metrics" 14 | "gitlab.com/yakshaving.art/alertsnitch/internal/webhook" 15 | ) 16 | 17 | // SupportedWebhookVersion is the alert webhook data version that is supported 18 | // by this app 19 | const SupportedWebhookVersion = "4" 20 | 21 | // Server represents a web server that processes webhooks 22 | type Server struct { 23 | db internal.Storer 24 | r *mux.Router 25 | 26 | debug bool 27 | } 28 | 29 | // New returns a new web server 30 | func New(db internal.Storer, debug bool) Server { 31 | r := mux.NewRouter() 32 | 33 | s := Server{ 34 | db: db, 35 | r: r, 36 | 37 | debug: debug, 38 | } 39 | 40 | r.HandleFunc("/webhook", s.webhookPost).Methods("POST") 41 | r.HandleFunc("/-/ready", s.readyProbe).Methods("GET") 42 | r.HandleFunc("/-/health", s.healthyProbe).Methods("GET") 43 | r.Handle("/metrics", promhttp.Handler()) 44 | 45 | return s 46 | } 47 | 48 | // Start starts a new server on the given address 49 | func (s Server) Start(address string) { 50 | log.Println("Starting listener on", address, "using", s.db) 51 | log.Fatal(http.ListenAndServe(address, s.r)) 52 | } 53 | 54 | func (s Server) webhookPost(w http.ResponseWriter, r *http.Request) { 55 | defer r.Body.Close() 56 | 57 | metrics.WebhooksReceivedTotal.Inc() 58 | 59 | body, err := ioutil.ReadAll(r.Body) 60 | if err != nil { 61 | metrics.InvalidWebhooksTotal.Inc() 62 | log.Printf("Failed to read payload: %s\n", err) 63 | http.Error(w, fmt.Sprintf("Failed to read payload: %s", err), http.StatusBadRequest) 64 | return 65 | } 66 | 67 | if s.debug { 68 | log.Println("Received webhook payload", string(body)) 69 | } 70 | 71 | data, err := webhook.Parse(body) 72 | if err != nil { 73 | metrics.InvalidWebhooksTotal.Inc() 74 | 75 | log.Printf("Invalid payload: %s\n", err) 76 | http.Error(w, fmt.Sprintf("Invalid payload: %s", err), http.StatusBadRequest) 77 | return 78 | } 79 | 80 | if data.Version != SupportedWebhookVersion { 81 | metrics.InvalidWebhooksTotal.Inc() 82 | 83 | log.Printf("Invalid payload: webhook version %s is not supported\n", data.Version) 84 | http.Error(w, fmt.Sprintf("Invalid payload: webhook version %s is not supported", data.Version), http.StatusBadRequest) 85 | return 86 | } 87 | 88 | metrics.AlertsReceivedTotal.WithLabelValues(data.Receiver, data.Status).Add(float64(len(data.Alerts))) 89 | 90 | if err = s.db.Save(data); err != nil { 91 | metrics.AlertsSavingFailuresTotal.WithLabelValues(data.Receiver, data.Status).Add(float64(len(data.Alerts))) 92 | 93 | log.Printf("failed to save alerts: %s\n", err) 94 | http.Error(w, fmt.Sprintf("failed to save alerts: %s", err), http.StatusInternalServerError) 95 | return 96 | } 97 | metrics.AlertsSavedTotal.WithLabelValues(data.Receiver, data.Status).Add(float64(len(data.Alerts))) 98 | } 99 | 100 | func (s Server) healthyProbe(w http.ResponseWriter, r *http.Request) { 101 | if err := s.db.Ping(); err != nil { 102 | log.Printf("failed to ping database server: %s", err) 103 | http.Error(w, fmt.Sprintf("failed to ping database server: %s", err), http.StatusServiceUnavailable) 104 | return 105 | } 106 | } 107 | 108 | func (s Server) readyProbe(w http.ResponseWriter, r *http.Request) { 109 | if err := s.db.Ping(); err != nil { 110 | log.Printf("database is not reachable: %s", err) 111 | http.Error(w, fmt.Sprintf("database is not reachable: %s", err), http.StatusServiceUnavailable) 112 | return 113 | } 114 | if err := s.db.CheckModel(); err != nil { 115 | log.Printf("invalid model: %s", err) 116 | http.Error(w, fmt.Sprintf("invalid model: %s", err), http.StatusServiceUnavailable) 117 | return 118 | } 119 | } 120 | -------------------------------------------------------------------------------- /internal/webhook/sample-payload-invalid-ends-at.json: -------------------------------------------------------------------------------- 1 | { 2 | "receiver": "alertsnitch", 3 | "status": "resolved", 4 | "alerts": [ 5 | { 6 | "status": "resolved", 7 | "labels": { 8 | "alertname": "MyAlertName", 9 | "hostname": "myhostname", 10 | "instance": "192.168.1.2:9100", 11 | "job": "node-exporter", 12 | "severity": "critical", 13 | "tier": "svc" 14 | }, 15 | "annotations": { 16 | "summary": "Non ephemeral host is DOWN" 17 | }, 18 | "startsAt": "2019-01-02T10:31:46.05445419Z", 19 | "endsAt": "0001-01-01T00:00:00Z", 20 | "generatorURL": "http://prometheus.int/graph?g0.expr=up%7Bjob%3D%22node-exporter%22%2Ctier%21%3D%22ephemeral%22%7D+%3D%3D+0&g0.tab=1" 21 | } 22 | ], 23 | "groupLabels": { 24 | "alertname": "NonEphemeralHostIsDown" 25 | }, 26 | "commonLabels": { 27 | "alertname": "NonEphemeralHostIsDown", 28 | "hostname": "myhostname", 29 | "instance": "192.168.1.2:9100", 30 | "job": "node-exporter", 31 | "severity": "critical", 32 | "tier": "svc" 33 | }, 34 | "commonAnnotations": { 35 | "host_tier": "myhostname", 36 | "summary": "Non ephemeral host is DOWN" 37 | }, 38 | "externalURL": "http://alertmanager:9093", 39 | "version": "4", 40 | "groupKey": "{}/{}:{alertname=\"NonEphemeralHostIsDown\"}" 41 | } -------------------------------------------------------------------------------- /internal/webhook/sample-payload.json: -------------------------------------------------------------------------------- 1 | { 2 | "receiver": "alertsnitch", 3 | "status": "resolved", 4 | "alerts": [ 5 | { 6 | "status": "resolved", 7 | "labels": { 8 | "alertname": "MyAlertName", 9 | "hostname": "myhostname", 10 | "instance": "192.168.1.2:9100", 11 | "job": "node-exporter", 12 | "severity": "critical", 13 | "tier": "svc" 14 | }, 15 | "annotations": { 16 | "summary": "Non ephemeral host is DOWN" 17 | }, 18 | "startsAt": "2019-01-02T10:31:46.05445419Z", 19 | "endsAt": "2019-01-02T10:36:46.05445419Z", 20 | "generatorURL": "http://prometheus.int/graph?g0.expr=up%7Bjob%3D%22node-exporter%22%2Ctier%21%3D%22ephemeral%22%7D+%3D%3D+0&g0.tab=1", 21 | "fingerprint": "dd19ae3d4e06ac55" 22 | } 23 | ], 24 | "groupLabels": { 25 | "alertname": "NonEphemeralHostIsDown" 26 | }, 27 | "commonLabels": { 28 | "alertname": "NonEphemeralHostIsDown", 29 | "hostname": "myhostname", 30 | "instance": "192.168.1.2:9100", 31 | "job": "node-exporter", 32 | "severity": "critical", 33 | "tier": "svc" 34 | }, 35 | "commonAnnotations": { 36 | "host_tier": "myhostname", 37 | "summary": "Non ephemeral host is DOWN" 38 | }, 39 | "externalURL": "http://alertmanager:9093", 40 | "version": "4", 41 | "groupKey": "{}/{}:{alertname=\"NonEphemeralHostIsDown\"}" 42 | } -------------------------------------------------------------------------------- /internal/webhook/webhook.go: -------------------------------------------------------------------------------- 1 | package webhook 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | 7 | "gitlab.com/yakshaving.art/alertsnitch/internal" 8 | ) 9 | 10 | // Parse gets a webhook payload and parses it returning a prometheus 11 | // template.Data object if successful 12 | func Parse(payload []byte) (*internal.AlertGroup, error) { 13 | d := internal.AlertGroup{} 14 | err := json.Unmarshal(payload, &d) 15 | if err != nil { 16 | return nil, fmt.Errorf("failed to decode json webhook payload: %s", err) 17 | } 18 | return &d, nil 19 | } 20 | -------------------------------------------------------------------------------- /internal/webhook/webhook_test.go: -------------------------------------------------------------------------------- 1 | package webhook_test 2 | 3 | import ( 4 | "io/ioutil" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | "gitlab.com/yakshaving.art/alertsnitch/internal/webhook" 9 | ) 10 | 11 | func TestParsingPayloadWithEmptyPayloadFails(t *testing.T) { 12 | _, err := webhook.Parse([]byte("")) 13 | assert.EqualError(t, err, "failed to decode json webhook payload: unexpected end of JSON input") 14 | } 15 | 16 | func TestParsingPayloadWithInvalidPayloadFails(t *testing.T) { 17 | _, err := webhook.Parse([]byte("error")) 18 | assert.EqualError(t, err, "failed to decode json webhook payload: invalid character 'e' looking for beginning of value") 19 | } 20 | 21 | func TestParsingValidPayloadWorks(t *testing.T) { 22 | a := assert.New(t) 23 | b, err := ioutil.ReadFile("sample-payload.json") 24 | 25 | a.NoError(err) 26 | 27 | d, err := webhook.Parse(b) 28 | 29 | a.NoError(err) 30 | a.NotNil(d) 31 | 32 | a.Equal(d.Status, "resolved") 33 | a.Equal(d.ExternalURL, "http://alertmanager:9093") 34 | } 35 | 36 | func TestParsingValidPayloadWithoutEndsAtWorks(t *testing.T) { 37 | a := assert.New(t) 38 | b, err := ioutil.ReadFile("sample-payload-invalid-ends-at.json") 39 | 40 | a.NoError(err) 41 | 42 | d, err := webhook.Parse(b) 43 | 44 | a.NoError(err) 45 | a.NotNil(d) 46 | 47 | a.Equal(d.Status, "resolved") 48 | a.Equal(d.ExternalURL, "http://alertmanager:9093") 49 | a.True(d.Alerts[0].EndsAt.Before(d.Alerts[0].StartsAt)) 50 | } 51 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "os" 7 | 8 | _ "github.com/go-sql-driver/mysql" 9 | _ "github.com/lib/pq" 10 | 11 | "github.com/sirupsen/logrus" 12 | 13 | "gitlab.com/yakshaving.art/alertsnitch/internal" 14 | "gitlab.com/yakshaving.art/alertsnitch/internal/db" 15 | "gitlab.com/yakshaving.art/alertsnitch/internal/server" 16 | "gitlab.com/yakshaving.art/alertsnitch/version" 17 | ) 18 | 19 | // Args are the arguments that can be passed to alertsnitch 20 | type Args struct { 21 | Address string 22 | DBBackend string 23 | DSN string 24 | MaxIdleConns int 25 | MaxOpenConns int 26 | MaxConnLifetimeSeconds int 27 | 28 | Debug bool 29 | DryRun bool 30 | Version bool 31 | } 32 | 33 | func main() { 34 | args := Args{} 35 | 36 | flag.BoolVar(&args.Version, "version", false, "print the version and exit") 37 | flag.StringVar(&args.Address, "listen.address", envWithDefault("ALERTSNITCH_ADDR", ":9567"), "address in which to listen for http requests, (also ALERTSNITCH_ADDR)") 38 | flag.BoolVar(&args.Debug, "debug", false, "enable debug mode, which dumps alerts payloads to the log as they arrive") 39 | 40 | flag.StringVar(&args.DBBackend, "database-backend", envWithDefault("ALERTSNITCH_BACKEND", "mysql"), "database backend, allowed are mysql, postgres, and null (also ALERTSNITCH_BACKEND") 41 | flag.StringVar(&args.DSN, "dsn", os.Getenv(internal.DSNVar), "Database DSN (also ALERTSNITCH_DSN)") 42 | 43 | flag.IntVar(&args.MaxOpenConns, "max-open-connections", 2, "maximum number of connections in the pool") 44 | flag.IntVar(&args.MaxIdleConns, "max-idle-connections", 1, "maximum number of idle connections in the pool") 45 | flag.IntVar(&args.MaxConnLifetimeSeconds, "max-connection-lifetyme-seconds", 600, "maximum number of seconds a connection is kept alive in the pool") 46 | 47 | flag.Parse() 48 | 49 | if args.Version { 50 | fmt.Println(version.GetVersion()) 51 | os.Exit(0) 52 | } 53 | 54 | if args.Debug { 55 | logrus.SetLevel(logrus.DebugLevel) 56 | } 57 | 58 | driver, err := db.Connect(args.DBBackend, db.ConnectionArgs{ 59 | DSN: args.DSN, 60 | MaxIdleConns: args.MaxIdleConns, 61 | MaxOpenConns: args.MaxOpenConns, 62 | MaxConnLifetimeSeconds: args.MaxConnLifetimeSeconds, 63 | }) 64 | if err != nil { 65 | fmt.Println("failed to connect to database:", err) 66 | os.Exit(1) 67 | } 68 | 69 | s := server.New(driver, args.Debug) 70 | s.Start(args.Address) 71 | } 72 | 73 | func envWithDefault(key, defaultValue string) string { 74 | v := os.Getenv(key) 75 | if v == "" { 76 | return defaultValue 77 | } 78 | return v 79 | } 80 | -------------------------------------------------------------------------------- /script.d/bootstrap_mysql.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -EeufCo pipefail 4 | IFS=$'\t\n' 5 | 6 | echo "Creating DB" 7 | mysql --user=root --password="${MYSQL_ROOT_PASSWORD}" --host=mysql -e "CREATE DATABASE IF NOT EXISTS ${MYSQL_DATABASE};" 8 | 9 | echo "Creating bootstrapped model" 10 | mysql --user=root --password="${MYSQL_ROOT_PASSWORD}" --host=mysql "${MYSQL_DATABASE}" < db.d/mysql/0.0.1-bootstrap.sql 11 | 12 | echo "Applying fingerprint model update" 13 | mysql --user=root --password="${MYSQL_ROOT_PASSWORD}" --host=mysql "${MYSQL_DATABASE}" < db.d/mysql/0.1.0-fingerprint.sql 14 | 15 | echo "Done creating model" -------------------------------------------------------------------------------- /script.d/bootstrap_postgres.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -EeufCo pipefail 4 | IFS=$'\t\n' 5 | 6 | echo "Creating bootstrapped model" 7 | psql -h "postgres" -U "${POSTGRES_USER}" -d "${POSTGRES_DB}" -f db.d/postgres/0.0.1-bootstrap.sql 8 | 9 | echo "Applying fingerprint model update" 10 | psql -h "postgres" -U "${POSTGRES_USER}" -d "${POSTGRES_DB}" -f db.d/postgres/0.1.0-fingerprint.sql 11 | 12 | echo "Done creating model" -------------------------------------------------------------------------------- /script.d/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -EeufCo pipefail 4 | IFS=$'\t\n' 5 | 6 | go mod download 7 | 8 | go test -v ./... -tags "integration" -coverprofile=coverage.out $(go list ./... | grep -v '/vendor/') 9 | 10 | go tool cover -func=coverage.out 11 | -------------------------------------------------------------------------------- /version/version.go: -------------------------------------------------------------------------------- 1 | package version 2 | 3 | import "fmt" 4 | 5 | // Name is the application name 6 | const Name = "alertsnitch" 7 | 8 | // Version is the application Version 9 | var Version string 10 | 11 | // Date is the built date and time 12 | var Date string 13 | 14 | // Commit is the commit in which the package is based 15 | var Commit string 16 | 17 | // GetVersion returns the version as a string 18 | func GetVersion() string { 19 | return fmt.Sprintf("%s Version: %s Commit: %s Date: %s", Name, Version, Commit, Date) 20 | } 21 | -------------------------------------------------------------------------------- /version/version_test.go: -------------------------------------------------------------------------------- 1 | package version_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "gitlab.com/yakshaving.art/alertsnitch/version" 7 | ) 8 | 9 | func TestVersionString(t *testing.T) { 10 | version.Commit = "mycommit" 11 | version.Date = "today" 12 | version.Version = "0.0.1" 13 | 14 | expected := "alertsnitch Version: 0.0.1 Commit: mycommit Date: today" 15 | if version.GetVersion() != expected { 16 | t.Fatalf("invalid version %s expected %s", version.GetVersion(), expected) 17 | } 18 | } 19 | --------------------------------------------------------------------------------