├── .github ├── CODEOWNERS └── workflows │ └── ci.yaml ├── .gitignore ├── CHANGELOG.md ├── LICENSE ├── Makefile ├── README.md ├── cmd └── alertgram │ ├── config.go │ └── main.go ├── docker ├── dev │ └── Dockerfile └── prod │ └── Dockerfile ├── docs ├── alertmanager │ └── README.md └── kubernetes │ ├── README.md │ ├── alertmanager-cfg.yaml │ └── deploy.yaml ├── go.mod ├── go.sum ├── internal ├── deadmansswitch │ ├── deadmansswitch.go │ ├── deadmansswitch_test.go │ └── metrics.go ├── forward │ ├── forward.go │ ├── forward_test.go │ ├── metrics.go │ └── notify.go ├── http │ ├── alertmanager │ │ ├── alertmanager.go │ │ ├── alertmanager_test.go │ │ ├── handler.go │ │ └── mapper.go │ └── server.go ├── internalerrors │ └── errors.go ├── log │ ├── log.go │ └── logrus │ │ └── logrus.go ├── metrics │ └── prometheus │ │ └── prometheus.go ├── mocks │ ├── deadmansswitch │ │ └── service.go │ ├── doc.go │ ├── forward │ │ ├── notifier.go │ │ └── service.go │ └── notify │ │ ├── telegram │ │ └── client.go │ │ └── template_renderer.go ├── model │ └── model.go └── notify │ ├── metrics.go │ ├── notify.go │ ├── telegram │ ├── telegram.go │ └── telegram_test.go │ ├── template.go │ └── template_test.go ├── scripts ├── build │ ├── build-image.sh │ ├── build.sh │ └── publish-image.sh ├── check │ ├── check.sh │ ├── integration-test.sh │ └── unit-test.sh ├── deps.sh └── mockgen.sh └── testdata ├── alerts └── base.json └── templates └── simple.tmpl /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @slok 2 | 3 | -------------------------------------------------------------------------------- /.github/workflows/ci.yaml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | check: 7 | name: Check 8 | runs-on: ubuntu-latest 9 | # Execute the checks inside the container instead the VM. 10 | container: golangci/golangci-lint:v1.34.1-alpine 11 | steps: 12 | - uses: actions/checkout@v1 13 | - run: golangci-lint run -E goimports 14 | 15 | test: 16 | name: Test 17 | runs-on: ubuntu-latest 18 | steps: 19 | - uses: actions/checkout@v2 20 | - uses: actions/setup-go@v2 21 | with: 22 | go-version: 1.15 23 | - run: make ci-test 24 | 25 | release-image: 26 | if: startsWith(github.ref, 'refs/tags/') 27 | env: 28 | TAG_IMAGE_LATEST: "true" 29 | PROD_IMAGE_NAME: ${GITHUB_REPOSITORY} 30 | #PROD_IMAGE_NAME: docker.pkg.github.com/${GITHUB_REPOSITORY}/app 31 | #PROD_IMAGE_NAME: registry.gitlab.com/${GITHUB_REPOSITORY} 32 | needs: [check, test] 33 | name: Release image 34 | runs-on: ubuntu-latest 35 | # Only run in master and when has a tag. 36 | steps: 37 | - run: echo "VERSION=${GITHUB_REF#refs/*/}" >> ${GITHUB_ENV} # Sets VERSION env var. 38 | - uses: actions/checkout@v2 39 | - name: Build image 40 | run: make build-image 41 | - name: Docker login 42 | run: docker login ${DOCKER_HOST} -u ${DOCKER_USER} -p ${DOCKER_TOKEN} 43 | env: 44 | # DockerHub. 45 | DOCKER_HOST: "" 46 | DOCKER_USER: slok 47 | DOCKER_TOKEN: ${{secrets.DOCKER_HUB_TOKEN}} 48 | ## Github. 49 | # DOCKER_HOST: docker.pkg.github.com 50 | # DOCKER_TOKEN: ${{secrets.GITHUB_TOKEN}} 51 | # DOCKER_USER: ${GITHUB_ACTOR} 52 | ## Gitlab. 53 | # DOCKER_HOST: registry.gitlab.com 54 | # DOCKER_USER: slok 55 | # DOCKER_TOKEN: ${{secrets.GITLAB_TOKEN}} 56 | 57 | - name: Publish image 58 | run: make publish-image 59 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.dll 4 | *.so 5 | *.dylib 6 | 7 | # Test binary, build with `go test -c` 8 | *.test 9 | 10 | # Output of the go coverage tool, specifically when used with LiteIDE 11 | *.out 12 | 13 | # Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736 14 | .glide/ 15 | 16 | # binary 17 | bin/ 18 | 19 | # vendor 20 | vendor/ -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## [unreleased] - YYYY-MM-DD 4 | 5 | ## [0.3.2] - 2021-01-03 6 | 7 | ### Added 8 | 9 | - Default template supports resolved alerts 10 | - Helpers on alert group model to group alerts by status. 11 | 12 | ### Changed 13 | 14 | - Update libraries. 15 | - Use Go 1.15. 16 | 17 | ## [0.3.1] - 2019-06-02 18 | 19 | ### Changed 20 | 21 | - Update libraries. 22 | - Use Go 1.14. 23 | 24 | ## [0.3.0] - 2019-12-20 25 | 26 | ### Added 27 | 28 | - Forward alerts by custom chat ID based on alert labels. 29 | 30 | ### Changed 31 | 32 | - Telegram required flags/envs are not required when using notify dry run mode. 33 | 34 | ## [0.2.1] - 2019-12-17 35 | 36 | ### Added 37 | 38 | - Metrics to dead man's switch service. 39 | 40 | ### Changed 41 | 42 | - Dead man's switch default interval is 15m. 43 | 44 | ## [0.2.0] - 2019-12-16 45 | 46 | ### Added 47 | 48 | - Dead man's switch option with Alertmanager. 49 | - Alertmanager API accepts a query string param with a custom chat ID. 50 | - Telegram notifier can send to customized chats. 51 | 52 | ## [0.1.0] - 2019-12-13 53 | 54 | ### Added 55 | 56 | - Custom templates. 57 | - Docs for Kubernetes deployment and Alertmanager configuration. 58 | - Simple health check 59 | - Prometheus metrics. 60 | - Telegram notifier. 61 | - Dry-run notifier. 62 | - Default message template. 63 | - Alertmanager compatible webhook API. 64 | - Models and forwarding domain service. 65 | 66 | [unreleased]: https://github.com/slok/alertgram/compare/v0.3.2...HEAD 67 | [0.3.2]: https://github.com/slok/alertgram/compare/v0.3.1...v0.3.2 68 | [0.3.1]: https://github.com/slok/alertgram/compare/v0.3.0...v0.3.1 69 | [0.3.0]: https://github.com/slok/alertgram/compare/v0.2.1...v0.3.0 70 | [0.2.1]: https://github.com/slok/alertgram/compare/v0.2.0...v0.2.1 71 | [0.2.0]: https://github.com/slok/alertgram/compare/v0.1.0...v0.2.0 72 | [0.1.0]: https://github.com/slok/alertgram/releases/tag/v0.1.0 73 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [2018] [Spotahome Ltd.] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | 2 | SHELL := $(shell which bash) 3 | OSTYPE := $(shell uname) 4 | DOCKER := $(shell command -v docker) 5 | GID := $(shell id -g) 6 | UID := $(shell id -u) 7 | VERSION ?= $(shell git describe --tags --always) 8 | 9 | UNIT_TEST_CMD := ./scripts/check/unit-test.sh 10 | INTEGRATION_TEST_CMD := ./scripts/check/integration-test.sh 11 | CHECK_CMD := ./scripts/check/check.sh 12 | 13 | DEV_IMAGE_NAME := slok/alertgram-dev 14 | PROD_IMAGE_NAME ?= slok/alertgram 15 | 16 | DOCKER_RUN_CMD := docker run --env ostype=$(OSTYPE) -v ${PWD}:/src --rm -it ${DEV_IMAGE_NAME} 17 | BUILD_BINARY_CMD := VERSION=${VERSION} ./scripts/build/build.sh 18 | BUILD_DEV_IMAGE_CMD := IMAGE=${DEV_IMAGE_NAME} DOCKER_FILE_PATH=./docker/dev/Dockerfile VERSION=latest ./scripts/build/build-image.sh 19 | BUILD_PROD_IMAGE_CMD := IMAGE=${PROD_IMAGE_NAME} DOCKER_FILE_PATH=./docker/prod/Dockerfile VERSION=${VERSION} ./scripts/build/build-image.sh 20 | PUBLISH_PROD_IMAGE_CMD := IMAGE=${PROD_IMAGE_NAME} VERSION=${VERSION} ./scripts/build/publish-image.sh 21 | 22 | 23 | help: ## Show this help 24 | @echo "Help" 25 | @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf " \033[36m%-20s\033[93m %s\n", $$1, $$2}' 26 | 27 | .PHONY: default 28 | default: build 29 | 30 | .PHONY: build-image 31 | build-image: ## Builds the production docker image. 32 | @$(BUILD_PROD_IMAGE_CMD) 33 | 34 | .PHONY: publish-image 35 | publish-image: ## Publishes the production docker image. 36 | @$(PUBLISH_PROD_IMAGE_CMD) 37 | 38 | .PHONY: build-dev-image 39 | build-dev-image: ## Builds the development docker image. 40 | @$(BUILD_DEV_IMAGE_CMD) 41 | 42 | build: build-dev-image ## Builds the production binary. 43 | @$(DOCKER_RUN_CMD) /bin/sh -c '$(BUILD_BINARY_CMD)' 44 | 45 | .PHONY: test 46 | test: build-dev-image ## Runs unit test. 47 | @$(DOCKER_RUN_CMD) /bin/sh -c '$(UNIT_TEST_CMD)' 48 | 49 | .PHONY: check 50 | check: build-dev-image ## Runs checks. 51 | @$(DOCKER_RUN_CMD) /bin/sh -c '$(CHECK_CMD)' 52 | 53 | .PHONY: integration 54 | integration: build-dev-image ## Runs integration test. 55 | @$(DOCKER_RUN_CMD) /bin/sh -c '$(INTEGRATION_TEST_CMD)' 56 | 57 | .PHONY: mocks 58 | mocks: build-dev-image ## Generates the mocks. 59 | @$(DOCKER_RUN_CMD) /bin/sh -c './scripts/mockgen.sh' 60 | 61 | .PHONY: deps 62 | deps: ## Fixes the dependencies 63 | @$(DOCKER_RUN_CMD) /bin/sh -c './scripts/deps.sh' 64 | 65 | .PHONY: ci-unit-test 66 | ci-test: ## Runs unit test in CI environment (without docker). 67 | @$(UNIT_TEST_CMD) 68 | 69 | .PHONY: ci-check 70 | ci-check: ## Runs checks in CI environment (without docker). 71 | @$(CHECK_CMD) 72 | 73 | .PHONY: ci-integration-test 74 | ci-integration: ## Runs integraton test in CI environment (without docker). 75 | @$(INTEGRATION_TEST_CMD) 76 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Alertgram [![Build Status][github-actions-image]][github-actions-url] [![Go Report Card][goreport-image]][goreport-url] 2 | 3 | Alertgram is the easiest way to forward alerts to [Telegram] (Supports [Prometheus alertmanager] alerts). 4 | 5 |

6 | alertgram 7 |

8 | 9 | ## Table of contents 10 | 11 | - [Introduction](#introduction) 12 | - [Features](#features) 13 | - [Input alerts](#input-alerts) 14 | - [Options](#options) 15 | - [Run](#run) 16 | - [Simple example](#simple-example) 17 | - [Production](#production) 18 | - [Metrics](#metrics) 19 | - [Development and debugging](#development-and-debugging) 20 | - [FAQ](#faq) 21 | - [Only alertmanager alerts are supported?](#only-alertmanager-alerts-are-supported) 22 | - [Where does alertgram listen to alertmanager alerts?](#where-does-alertgram-listen-to-alertmanager-alerts) 23 | - [Can I notify to different chats?](#can-i-notify-to-different-chats) 24 | - [Can I use custom templates?](#can-i-use-custom-templates) 25 | - [Dead man's switch?](#dead-mans-switch) 26 | 27 | ## Introduction 28 | 29 | Everything started as a way of forwarding [Prometheus alertmanager] alerts to [Telegram] because the solutions that I found were too complex, I just wanted to forward alerts to channels without trouble. And Alertgram is just that, a simple app that forwards alerts to Telegram groups and channels and some small features that help like metrics and dead man's switch. 30 | 31 | ## Features 32 | 33 | - Alertmanager alerts webhook receiver compatibility. 34 | - Telegram notifications. 35 | - Metrics in Prometheus format. 36 | - Optional dead man switch endpoint. 37 | - Optional customizable templates. 38 | - Configurable notification chat ID targets (with fallback to default chat ID). 39 | - Easy setup and flexible. 40 | - Lightweight. 41 | - Perfect for any environment, from a company cluster to home cheap clusters (e.g [K3S]). 42 | 43 | ## Input alerts 44 | 45 | Alertgram is developed in a decoupled way so in a future may be extended to more inputs apart from Alertmanager's webhook API (ask for a new input if you want). 46 | 47 | ## Options 48 | 49 | Use `--help` flag to show the options. 50 | 51 | The configuration of the app is based on flags that also can be set as env vars prepending `ALERTGRAM` to the var. e.g: the flag `--telegram.api-token` would be `ALERTGRAM_TELEGRAM_API_TOKEN`. You can combine both, flags have preference. 52 | 53 | ## Run 54 | 55 | To forward alerts to Telegram the minimum options that need to be set are `--telegram.api-token` and `--telegram.chat-id` 56 | 57 | ### Simple example 58 | 59 | ```bash 60 | docker run -p8080:8080 -p8081:8081 slok/alertgram:latest --telegram.api-token=XXXXX --telegram.chat-id=YYYYY 61 | ``` 62 | 63 | ### Production 64 | 65 | - [Get telegram API token][telegram-token] 66 | - [Get telegram chat IDs][telegram-chat-id] 67 | - [Configure Alertmanager][alertmanager-configuration] 68 | - [Deploy on Kubernetes][kubernetes-deployment] 69 | 70 | ## Metrics 71 | 72 | The app comes with [Prometheus] metrics, it measures the forwarded alerts, HTTP requests, errors... with rate and latency. 73 | 74 | By default are served on `/metrics` on `0.0.0.0:8081` 75 | 76 | ## Development and debugging 77 | 78 | You can use the `--notify.dry-run` to show the alerts on the terminal instead of forwarding them to telegram. 79 | 80 | Also remember that you can use `--debug` flag. 81 | 82 | ## FAQ 83 | 84 | ### Only alertmanager alerts are supported? 85 | 86 | At this moment yes, but we can add more input alert systems if you want, create an issue 87 | so we can discuss and implement. 88 | 89 | ### Where does alertgram listen to alertmanager alerts? 90 | 91 | By default in `0.0.0.0:8080/alerts`, but you can use `--alertmanager.listen-address` and 92 | `--alertmanager.webhook-path` to customize. 93 | 94 | ### Can I notify to different chats? 95 | 96 | There are 3 levels where you could customize the notification chat: 97 | 98 | - By default: Using the required `--telegram.chat-id` flag. 99 | - At URL level: using [query string] parameter, e.g. `0.0.0.0:8080/alerts?chat-id=-1009876543210`. 100 | This query param can be customized with `--alertmanager.chat-id-query-string` flag. 101 | - At alert level: If alerts have a label with the chat ID the alert notification will be forwarded to 102 | that label content. Use the flag `--alert.label-chat-id` to customize the label name, by default 103 | is `chat_id`. 104 | 105 | The preference is in order from highest to lowest: Alert, URL, Default. 106 | 107 | ### Can I use custom templates? 108 | 109 | Yes!, use the flag `--notify.template-path`. You can check [testdata/templates](testdata/templates) for examples. 110 | 111 | The templates are [HTML Go templates] with [Sprig] functions, so you can use these also. 112 | 113 | You can use also the notification dry run mode to check your templates without the need 114 | to notify on telegram: 115 | 116 | ```bash 117 | export ALERTGRAM_TELEGRAM_API_TOKEN=fake 118 | export ALERTGRAM_TELEGRAM_CHAT_ID=1234567890 119 | 120 | go run ./cmd/alertgram/ --notify.template-path=./testdata/templates/simple.tmpl --debug --notify.dry-run 121 | ``` 122 | 123 | To send an alert easily and check the template rendering without an alertmanager, prometheus, alerts... you can use the test alerts that are on [testdata/alerts](testdata/alerts): 124 | 125 | ```bash 126 | curl -i http://127.0.0.1:8080/alerts -d @./testdata/alerts/base.json 127 | ``` 128 | 129 | ### Dead man's switch? 130 | 131 | A [dead man's switch][dms] (from now on, DMS) is a technique or process where at regular intervals a signal must be received 132 | so the DMS is disabled, if that signal is not received it will be activated. 133 | 134 | In monitoring this would be: If an alert is not received at regular intervals, the switch will be activated and notify that we 135 | are not receiving alerts, this is mostly used to know that our alerting system is working. 136 | 137 | For example we would set Prometheus triggering an alert continously, Alertmanager sending this specific alert 138 | every `7m` to the DMS endpoint in Alertgram, and Alertgram would be configured with a `10m` interval DMS. 139 | 140 | With this setup if Prometheus fails creating the alert, Alertmanager sending the alert to Alertgram, or Alertgram not receiving 141 | this alert (e.g. network problems), Alertmanager will send an alert to Telegram to notify us that our monitoring system is broken. 142 | 143 | You could use the same alertgram or another instance, usually in other machine, cluster... so if the cluster/machine fails, your 144 | is isolated and could notify you. 145 | 146 | To Enable Alertgram's DMS use `--dead-mans-switch.enable` to enable. By default it will be listening in `/alert/dms`, with a 147 | `15m` interval and use the telegrams default notifier and chat ID. To customize this settings use: 148 | 149 | - `--dead-mans-switch.interval`: To configure the interval. 150 | - `--dead-mans-switch.chat-id`: To configure the notifier chat, is independent of the notifier 151 | although at this moment is Telegram, if not set it will use the notifier default chat target. 152 | - `--alertmanager.dead-mans-switch-path` To configure the path the alertmanager can send the DMS alerts. 153 | 154 | [github-actions-image]: https://github.com/slok/alertgram/workflows/CI/badge.svg 155 | [github-actions-url]: https://github.com/slok/alertgram/actions 156 | [goreport-image]: https://goreportcard.com/badge/github.com/slok/alertgram 157 | [goreport-url]: https://goreportcard.com/report/github.com/slok/alertgram 158 | [prometheus alertmanager]: https://github.com/prometheus/alertmanager 159 | [prometheus]: https://prometheus.io/ 160 | [telegram]: https://telegram.org/ 161 | [telegram-token]: https://core.telegram.org/bots#6-botfather 162 | [telegram-chat-id]: https://github.com/GabrielRF/telegram-id 163 | [alertmanager-configuration]: docs/alertmanager 164 | [kubernetes-deployment]: docs/kubernetes 165 | [html go templates]: https://golang.org/pkg/html/template/ 166 | [sprig]: http://masterminds.github.io/sprig 167 | [query string]: https://en.wikipedia.org/wiki/Query_string 168 | [k3s]: https://k3s.io/ 169 | [dms]: https://en.wikipedia.org/wiki/Dead_man%27s_switch 170 | -------------------------------------------------------------------------------- /cmd/alertgram/config.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "errors" 5 | "os" 6 | "time" 7 | 8 | "gopkg.in/alecthomas/kingpin.v2" 9 | ) 10 | 11 | var ( 12 | // Version will be populated in compilation time. 13 | Version = "dev" 14 | ) 15 | 16 | // flag descriptions. 17 | const ( 18 | descAMListenAddr = "The listen address where the server will be listening to alertmanager's webhook request." 19 | descAMWebhookPath = "The path where the server will be handling the alertmanager webhook alert requests." 20 | descAMChatIDQS = "The optional query string key used to customize the chat id of the notification. Does not depend on the notifier type." 21 | descAMDMSPath = "The path for the dead man switch alerts from the Alertmanger." 22 | descTelegramAPIToken = "The token that will be used to use the telegram API to send the alerts." 23 | descTelegramDefChatID = "The default ID of the chat (group/channel) in telegram where the alerts will be sent." 24 | descMetricsListenAddr = "The listen address where the metrics will be being served." 25 | descMetricsPath = "The path where the metrics will be being served." 26 | descMetricsHCPath = "The path where the healthcheck will be being served, it uses the same port as the metrics." 27 | descDMSEnable = "Enables the dead man switch, that will send an alert if no alert is received at regular intervals." 28 | descDMSInterval = "The interval the dead mans switch needs to receive an alert to not activate and send a notification alert (in Go time duration)." 29 | descDMSChatID = "The chat ID (group/channel/room) the dead man's witch will sent the alerts. Does not depend on the notifier type and if not set it will be used notifier default chat ID." 30 | descDebug = "Run the application in debug mode." 31 | descNotifyDryRun = "Dry run the notification and show in the terminal instead of sending." 32 | descNotifyTemplatePath = "The path to set a custom template for the notification messages." 33 | descAlertLabelChatID = "The label of the alert that will carry the chat id to forward the alert." 34 | ) 35 | 36 | const ( 37 | defAMListenAddr = ":8080" 38 | defAMWebhookPath = "/alerts" 39 | defAMChatIDQS = "chat-id" 40 | defAMDMSPath = "/alerts/dms" 41 | defMetricsListenAddr = ":8081" 42 | defMetricsPath = "/metrics" 43 | defMetricsHCPath = "/status" 44 | defDMSInterval = "15m" 45 | defAlertLabelChatID = "chat_id" 46 | ) 47 | 48 | // Config has the configuration of the application. 49 | type Config struct { 50 | AlertmanagerListenAddr string 51 | AlertmanagerWebhookPath string 52 | AlertmanagerChatIDQQueryString string 53 | AlertmanagerDMSPath string 54 | TeletramAPIToken string 55 | TelegramChatID int64 56 | MetricsListenAddr string 57 | MetricsPath string 58 | MetricsHCPath string 59 | DMSInterval time.Duration 60 | DMSEnable bool 61 | DMSChatID string 62 | NotifyTemplate *os.File 63 | DebugMode bool 64 | NotifyDryRun bool 65 | AlertLabelChatID string 66 | 67 | app *kingpin.Application 68 | } 69 | 70 | // NewConfig returns a new configuration for the apps. 71 | func NewConfig() (*Config, error) { 72 | c := &Config{ 73 | app: kingpin.New("alertgram", "Forward your alerts to telegram.").DefaultEnvars(), 74 | } 75 | c.app.Version(Version) 76 | 77 | c.registerFlags() 78 | 79 | if _, err := c.app.Parse(os.Args[1:]); err != nil { 80 | return nil, err 81 | } 82 | if err := c.validate(); err != nil { 83 | return nil, err 84 | } 85 | 86 | return c, nil 87 | } 88 | 89 | func (c *Config) registerFlags() { 90 | c.app.Flag("alertmanager.listen-address", descAMListenAddr).Default(defAMListenAddr).StringVar(&c.AlertmanagerListenAddr) 91 | c.app.Flag("alertmanager.webhook-path", descAMWebhookPath).Default(defAMWebhookPath).StringVar(&c.AlertmanagerWebhookPath) 92 | c.app.Flag("alertmanager.chat-id-query-string", descAMChatIDQS).Default(defAMChatIDQS).StringVar(&c.AlertmanagerChatIDQQueryString) 93 | c.app.Flag("alertmanager.dead-mans-switch-path", descAMDMSPath).Default(defAMDMSPath).StringVar(&c.AlertmanagerDMSPath) 94 | c.app.Flag("telegram.api-token", descTelegramAPIToken).StringVar(&c.TeletramAPIToken) 95 | c.app.Flag("telegram.chat-id", descTelegramDefChatID).Int64Var(&c.TelegramChatID) 96 | c.app.Flag("metrics.listen-address", descMetricsListenAddr).Default(defMetricsListenAddr).StringVar(&c.MetricsListenAddr) 97 | c.app.Flag("metrics.path", descMetricsPath).Default(defMetricsPath).StringVar(&c.MetricsPath) 98 | c.app.Flag("metrics.health-path", descMetricsHCPath).Default(defMetricsHCPath).StringVar(&c.MetricsHCPath) 99 | c.app.Flag("dead-mans-switch.enable", descDMSEnable).BoolVar(&c.DMSEnable) 100 | c.app.Flag("dead-mans-switch.interval", descDMSInterval).Default(defDMSInterval).DurationVar(&c.DMSInterval) 101 | c.app.Flag("dead-mans-switch.chat-id", descDMSChatID).StringVar(&c.DMSChatID) 102 | c.app.Flag("notify.dry-run", descNotifyDryRun).BoolVar(&c.NotifyDryRun) 103 | c.app.Flag("notify.template-path", descNotifyTemplatePath).FileVar(&c.NotifyTemplate) 104 | c.app.Flag("alert.label-chat-id", descAlertLabelChatID).Default(defAlertLabelChatID).StringVar(&c.AlertLabelChatID) 105 | c.app.Flag("debug", descDebug).BoolVar(&c.DebugMode) 106 | } 107 | 108 | func (c *Config) validate() error { 109 | if !c.NotifyDryRun { 110 | if c.TeletramAPIToken == "" { 111 | return errors.New("telegram api token is required") 112 | } 113 | 114 | if c.TelegramChatID == 0 { 115 | return errors.New("telegram default chat ID is required") 116 | } 117 | } 118 | return nil 119 | } 120 | -------------------------------------------------------------------------------- /cmd/alertgram/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io/ioutil" 7 | "net/http" 8 | "os" 9 | "os/signal" 10 | "syscall" 11 | 12 | tgbotapi "github.com/go-telegram-bot-api/telegram-bot-api" 13 | "github.com/oklog/run" 14 | "github.com/prometheus/client_golang/prometheus" 15 | "github.com/prometheus/client_golang/prometheus/promhttp" 16 | metricsmiddleware "github.com/slok/go-http-metrics/middleware" 17 | metricsmiddlewarestd "github.com/slok/go-http-metrics/middleware/std" 18 | 19 | "github.com/slok/alertgram/internal/deadmansswitch" 20 | "github.com/slok/alertgram/internal/forward" 21 | internalhttp "github.com/slok/alertgram/internal/http" 22 | "github.com/slok/alertgram/internal/http/alertmanager" 23 | "github.com/slok/alertgram/internal/log" 24 | "github.com/slok/alertgram/internal/log/logrus" 25 | metricsprometheus "github.com/slok/alertgram/internal/metrics/prometheus" 26 | "github.com/slok/alertgram/internal/notify" 27 | "github.com/slok/alertgram/internal/notify/telegram" 28 | ) 29 | 30 | // Main is the main application. 31 | type Main struct { 32 | cfg *Config 33 | logger log.Logger 34 | } 35 | 36 | // Run runs the main application. 37 | func (m *Main) Run() error { 38 | // Initialization and setup. 39 | var err error 40 | m.cfg, err = NewConfig() 41 | if err != nil { 42 | return err 43 | } 44 | m.logger = logrus.New(m.cfg.DebugMode).WithValues(log.KV{"version": Version}) 45 | 46 | // Dependencies. 47 | metricsRecorder := metricsprometheus.New(prometheus.DefaultRegisterer) 48 | 49 | // Select the kind of template renderer: default or custom template. 50 | var tmplRenderer notify.TemplateRenderer 51 | if m.cfg.NotifyTemplate != nil { 52 | tmpl, err := ioutil.ReadAll(m.cfg.NotifyTemplate) 53 | if err != nil { 54 | return err 55 | } 56 | _ = m.cfg.NotifyTemplate.Close() 57 | tmplRenderer, err = notify.NewHTMLTemplateRenderer(string(tmpl)) 58 | if err != nil { 59 | return err 60 | } 61 | tmplRenderer = notify.NewMeasureTemplateRenderer("custom", metricsRecorder, tmplRenderer) 62 | m.logger.Infof("using custom template at %s", m.cfg.NotifyTemplate.Name()) 63 | } else { 64 | tmplRenderer = notify.NewMeasureTemplateRenderer("default", metricsRecorder, notify.DefaultTemplateRenderer) 65 | } 66 | 67 | var notifier forward.Notifier 68 | if m.cfg.NotifyDryRun { 69 | notifier = notify.NewLogger(tmplRenderer, m.logger) 70 | } else { 71 | tgCli, err := tgbotapi.NewBotAPI(m.cfg.TeletramAPIToken) 72 | if err != nil { 73 | return err 74 | } 75 | 76 | notifier, err = telegram.NewNotifier(telegram.Config{ 77 | TemplateRenderer: tmplRenderer, 78 | Client: tgCli, 79 | DefaultTelegramChatID: m.cfg.TelegramChatID, 80 | Logger: m.logger, 81 | }) 82 | if err != nil { 83 | return err 84 | } 85 | } 86 | notifier = forward.NewMeasureNotifier(metricsRecorder, notifier) 87 | 88 | var g run.Group 89 | 90 | // Alertmanager webhook server. 91 | { 92 | 93 | // Alert forward. 94 | forwardSvc, err := forward.NewService(forward.ServiceConfig{ 95 | AlertLabelChatID: m.cfg.AlertLabelChatID, 96 | Notifiers: []forward.Notifier{notifier}, 97 | Logger: m.logger, 98 | }) 99 | if err != nil { 100 | return err 101 | } 102 | forwardSvc = forward.NewMeasureService(metricsRecorder, forwardSvc) 103 | 104 | // Dead man's switch. 105 | ctx, ctxCancel := context.WithCancel(context.Background()) 106 | var deadMansSwitchSvc deadmansswitch.Service = deadmansswitch.DisabledService // By default disabled. 107 | if m.cfg.DMSEnable { 108 | deadMansSwitchSvc, err = deadmansswitch.NewService(ctx, deadmansswitch.Config{ 109 | CustomChatID: m.cfg.DMSChatID, 110 | Notifiers: []forward.Notifier{notifier}, 111 | Interval: m.cfg.DMSInterval, 112 | Logger: m.logger, 113 | }) 114 | if err != nil { 115 | ctxCancel() 116 | return err 117 | } 118 | 119 | deadMansSwitchSvc = deadmansswitch.NewMeasureService(metricsRecorder, deadMansSwitchSvc) 120 | } 121 | 122 | // API server. 123 | logger := m.logger.WithValues(log.KV{"server": "alertmanager-handler"}) 124 | h, err := alertmanager.NewHandler(alertmanager.Config{ 125 | Debug: m.cfg.DebugMode, 126 | MetricsRecorder: metricsRecorder, 127 | WebhookPath: m.cfg.AlertmanagerWebhookPath, 128 | DeadMansSwitchService: deadMansSwitchSvc, 129 | DeadMansSwitchPath: m.cfg.AlertmanagerDMSPath, 130 | ForwardService: forwardSvc, 131 | Logger: logger, 132 | }) 133 | if err != nil { 134 | ctxCancel() 135 | return err 136 | } 137 | server, err := internalhttp.NewServer(internalhttp.Config{ 138 | Handler: h, 139 | ListenAddress: m.cfg.AlertmanagerListenAddr, 140 | Logger: logger, 141 | }) 142 | if err != nil { 143 | ctxCancel() 144 | return err 145 | } 146 | 147 | g.Add( 148 | func() error { 149 | return server.ListenAndServe() 150 | }, 151 | func(_ error) { 152 | ctxCancel() 153 | if err := server.DrainAndShutdown(); err != nil { 154 | logger.Errorf("error while draining connections") 155 | } 156 | }) 157 | } 158 | 159 | // Metrics. 160 | { 161 | logger := m.logger.WithValues(log.KV{"server": "metrics"}) 162 | mux := http.NewServeMux() 163 | mux.Handle(m.cfg.MetricsPath, promhttp.Handler()) 164 | mux.Handle(m.cfg.MetricsHCPath, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte(`{"status":"ok"}`)) })) 165 | mdlw := metricsmiddleware.New(metricsmiddleware.Config{Service: "metrics", Recorder: metricsRecorder}) 166 | h := metricsmiddlewarestd.Handler("", mdlw, mux) 167 | server, err := internalhttp.NewServer(internalhttp.Config{ 168 | Handler: h, 169 | ListenAddress: m.cfg.MetricsListenAddr, 170 | Logger: logger, 171 | }) 172 | if err != nil { 173 | return err 174 | } 175 | 176 | g.Add( 177 | func() error { 178 | return server.ListenAndServe() 179 | }, 180 | func(_ error) { 181 | if err := server.DrainAndShutdown(); err != nil { 182 | logger.Errorf("error while draining connections") 183 | } 184 | }) 185 | } 186 | 187 | // Capture signals. 188 | { 189 | logger := m.logger.WithValues(log.KV{"service": "main"}) 190 | sigC := make(chan os.Signal, 1) 191 | exitC := make(chan struct{}) 192 | signal.Notify(sigC, syscall.SIGTERM, syscall.SIGINT) 193 | g.Add( 194 | func() error { 195 | select { 196 | case <-sigC: 197 | logger.Infof("signal captured") 198 | case <-exitC: 199 | } 200 | return nil 201 | }, 202 | func(_ error) { 203 | close(exitC) 204 | }) 205 | } 206 | 207 | return g.Run() 208 | } 209 | 210 | func main() { 211 | m := Main{} 212 | if err := m.Run(); err != nil { 213 | fmt.Fprintf(os.Stderr, "error running the app: %s\n", err) 214 | os.Exit(1) 215 | } 216 | 217 | fmt.Println("goodbye!") 218 | } 219 | -------------------------------------------------------------------------------- /docker/dev/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.15 2 | 3 | ARG GOLANGCI_LINT_VERSION="1.34.1" 4 | ARG MOCKERY_VERSION="2.5.1" 5 | ARG ostype=Linux 6 | 7 | RUN apt-get update && apt-get install -y \ 8 | git \ 9 | bash \ 10 | zip 11 | 12 | 13 | RUN wget https://github.com/golangci/golangci-lint/releases/download/v${GOLANGCI_LINT_VERSION}/golangci-lint-${GOLANGCI_LINT_VERSION}-linux-amd64.tar.gz && \ 14 | tar zxvf golangci-lint-${GOLANGCI_LINT_VERSION}-linux-amd64.tar.gz --strip 1 -C /usr/local/bin/ && \ 15 | rm golangci-lint-${GOLANGCI_LINT_VERSION}-linux-amd64.tar.gz && \ 16 | \ 17 | wget https://github.com/vektra/mockery/releases/download/v${MOCKERY_VERSION}/mockery_${MOCKERY_VERSION}_Linux_x86_64.tar.gz && \ 18 | tar zxvf mockery_${MOCKERY_VERSION}_Linux_x86_64.tar.gz -C /tmp && \ 19 | mv /tmp/mockery /usr/local/bin/ && \ 20 | rm mockery_${MOCKERY_VERSION}_Linux_x86_64.tar.gz 21 | 22 | 23 | # Create user. 24 | ARG uid=1000 25 | ARG gid=1000 26 | 27 | RUN bash -c 'if [ ${ostype} == Linux ]; then addgroup -gid $gid app; else addgroup app; fi && \ 28 | adduser --disabled-password -uid $uid --ingroup app --gecos "" app && \ 29 | chown app:app -R /go' 30 | 31 | # Fill go mod cache. 32 | RUN mkdir /tmp/cache 33 | COPY go.mod /tmp/cache 34 | COPY go.sum /tmp/cache 35 | RUN chown app:app -R /tmp/cache 36 | USER app 37 | RUN cd /tmp/cache && \ 38 | go mod download 39 | 40 | WORKDIR /src 41 | -------------------------------------------------------------------------------- /docker/prod/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.14-alpine as build-stage 2 | 3 | RUN apk --no-cache add \ 4 | g++ \ 5 | git \ 6 | make \ 7 | bash 8 | 9 | ARG VERSION 10 | ENV VERSION=${VERSION} 11 | ARG ostype=Linux 12 | 13 | WORKDIR /src 14 | COPY . . 15 | RUN ./scripts/build/build.sh 16 | 17 | # Final image. 18 | FROM alpine:latest 19 | RUN apk --no-cache add \ 20 | ca-certificates 21 | COPY --from=build-stage /src/bin/alertgram-linux-amd64 /usr/local/bin/alertgram 22 | ENTRYPOINT ["/usr/local/bin/alertgram"] -------------------------------------------------------------------------------- /docs/alertmanager/README.md: -------------------------------------------------------------------------------- 1 | # Configure alertmanager 2 | 3 | To configure the alertmanager with alertgram you need to use the alertmanager's 4 | webhook receiver. 5 | 6 | For example if we have our alertgram service listening to requests in 7 | `http://alertgram:8080/alerts` our alertmanager configuration `receiver` would be: 8 | 9 | ```yaml 10 | receivers: 11 | # ... 12 | - name: "telegram" 13 | webhook_config: 14 | - url: "http://alertgram:8080/alerts" 15 | ``` 16 | 17 | Now we could use `telegram` in the `receiver` setting to forward the alerts to 18 | Telegram. 19 | 20 | A full example: 21 | 22 | ```yaml 23 | global: 24 | resolve_timeout: 5m 25 | route: 26 | group_wait: 30s 27 | group_interval: 5m 28 | repeat_interval: 3h 29 | receiver: telegram 30 | routes: 31 | # Only important alerts. 32 | - match_re: 33 | severity: ^(oncall|critical)$ 34 | receiver: telegram-oncall 35 | 36 | receivers: 37 | - name: telegram 38 | webhook_configs: 39 | - url: 'http://alertgram:8080/alerts' 40 | send_resolved: false 41 | 42 | - name: telegram-oncall 43 | webhook_configs: 44 | - url: 'http://alertgram:8080/alerts?chat-id=-1001111111111' 45 | ``` 46 | -------------------------------------------------------------------------------- /docs/kubernetes/README.md: -------------------------------------------------------------------------------- 1 | # Kubernetes deployment 2 | 3 | ## Alertgram 4 | 5 | First of all check you meet the requirements: 6 | 7 | - [Telegram bot token.][telegram-token] 8 | - [Telegram channel or group ID.][telegram-chat-id] 9 | - Add your Telegram bot to the Telegram channel/group. 10 | - Write a message to the bot at least once (not sure if required currently). 11 | 12 | The manifest file is in [docs/kubernetes/deploy.yaml](deploy.yaml). 13 | 14 | before aplying the manifests check these: 15 | 16 | - They will be deployed on `monitoring` 17 | - It will set a ServiceMonitor for the alertgram metrics, the prometheus that 18 | will scrape the metrics is `prometheus`, change the labels if required. 19 | - Change the string `CHANGE_ME_TELEGRAM_API_TOKEN`for your Telegram API token. 20 | - Change the string `CHANGE_ME_TELEGRAM_CHAT_ID`for your Telegram chat ID. 21 | - Check the alertgram container image version and update to the one you want `image: slok/alertgram:XXXXXXXX`. 22 | - Alertgram will be listening on `http://alertgram:8080/alerts` or `http://alertgram.monitoring.svc.cluster.local:8080/alerts`. 23 | 24 | Now you are ready to `kubectly apply -f ./deploy.yaml`. 25 | 26 | ## Alertmanager 27 | 28 | As an example in [docs/kubernetes/alertmanager-cfg.yaml](alertmanager-cfg.yaml), 29 | is an alertmanager configuration example that connects with alertgram. 30 | 31 | Before deploying this, we assume: 32 | 33 | - You are using prometheus-operator. 34 | - Your alertmanager is `main`, if not change accordingly (labels, names...) 35 | - Is deployed on `monitoring` namespace. 36 | 37 | [telegram-token]: https://core.telegram.org/bots#6-botfather 38 | [telegram-chat-id]: https://github.com/GabrielRF/telegram-id 39 | -------------------------------------------------------------------------------- /docs/kubernetes/alertmanager-cfg.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Secret 3 | type: Opaque 4 | metadata: 5 | name: alertmanager-main 6 | namespace: monitoring 7 | labels: 8 | app: alertmanager 9 | alertmanager: main 10 | stringData: 11 | alertmanager.yaml: | 12 | global: 13 | resolve_timeout: 5m 14 | route: 15 | group_wait: 30s 16 | group_interval: 5m 17 | repeat_interval: 12h 18 | receiver: 'telegram-default' 19 | routes: 20 | # Warning alerts not important. 21 | - match: 22 | severity: warning 23 | receiver: telegram-warning 24 | 25 | # Only important alerts. 26 | - match_re: 27 | severity: ^(oncall|critical)$ 28 | receiver: telegram-oncall 29 | 30 | receivers: 31 | # Use default alert channel for the default alerts. 32 | - name: telegram-default 33 | webhook_configs: 34 | - url: 'http://alertgram:8080/alerts' 35 | send_resolved: false 36 | 37 | # Critical and oncall alerts to special channel. 38 | - name: telegram-oncall 39 | webhook_configs: 40 | - url: 'http://alertgram:8080/alerts?chat-id=-1001111111111' 41 | send_resolved: false 42 | 43 | # Warning alerts to a more public informative channel. 44 | - name: telegram-warning 45 | webhook_configs: 46 | - url: 'http://alertgram:8080/alerts?chat-id=-1002222222222' 47 | send_resolved: false 48 | -------------------------------------------------------------------------------- /docs/kubernetes/deploy.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: alertgram 5 | namespace: monitoring 6 | spec: 7 | replicas: 1 8 | selector: 9 | matchLabels: 10 | app: alertgram 11 | template: 12 | metadata: 13 | labels: 14 | app: alertgram 15 | spec: 16 | containers: 17 | - name: alertgram 18 | image: slok/alertgram:v0.3.2 19 | envFrom: 20 | - secretRef: 21 | name: alertgram 22 | ports: 23 | - name: http 24 | containerPort: 8080 25 | - name: metrics 26 | containerPort: 8081 27 | readinessProbe: 28 | httpGet: 29 | path: /status 30 | port: 8081 31 | initialDelaySeconds: 5 32 | timeoutSeconds: 5 33 | resources: 34 | requests: 35 | cpu: 20m 36 | memory: 40Mi 37 | 38 | --- 39 | apiVersion: v1 40 | kind: Secret 41 | type: Opaque 42 | metadata: 43 | name: alertgram 44 | namespace: monitoring 45 | labels: 46 | app: alertgram 47 | stringData: 48 | ALERTGRAM_TELEGRAM_API_TOKEN: "CHANGE_ME_TELEGRAM_API_TOKEN" 49 | ALERTGRAM_TELEGRAM_CHAT_ID: "CHANGE_ME_TELEGRAM_CHAT_ID" 50 | 51 | --- 52 | kind: Service 53 | apiVersion: v1 54 | metadata: 55 | name: alertgram 56 | namespace: monitoring 57 | labels: 58 | app: alertgram 59 | spec: 60 | selector: 61 | app: alertgram 62 | type: ClusterIP 63 | ports: 64 | - name: http 65 | port: 8080 66 | - name: metrics 67 | port: 8081 68 | 69 | --- 70 | apiVersion: monitoring.coreos.com/v1 71 | kind: ServiceMonitor 72 | metadata: 73 | name: alertgram 74 | namespace: monitoring 75 | labels: 76 | app: alertgram 77 | prometheus: prometheus 78 | spec: 79 | selector: 80 | matchLabels: 81 | app: alertgram 82 | endpoints: 83 | - port: metrics 84 | honorLabels: false 85 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/slok/alertgram 2 | 3 | require ( 4 | github.com/Masterminds/sprig/v3 v3.1.0 5 | github.com/gin-gonic/gin v1.6.3 6 | github.com/go-telegram-bot-api/telegram-bot-api v4.6.4+incompatible 7 | github.com/oklog/run v1.1.0 8 | github.com/prometheus/alertmanager v0.21.0 9 | github.com/prometheus/client_golang v1.6.0 10 | github.com/prometheus/common v0.10.0 11 | github.com/sirupsen/logrus v1.7.0 12 | github.com/slok/go-http-metrics v0.8.0 13 | github.com/stretchr/testify v1.6.1 14 | github.com/technoweenie/multipartstreamer v1.0.1 // indirect 15 | gopkg.in/alecthomas/kingpin.v2 v2.2.6 16 | ) 17 | 18 | go 1.15 19 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 2 | cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 3 | contrib.go.opencensus.io/exporter/prometheus v0.1.0/go.mod h1:cGFniUXGZlKRjzOyuZJ6mgB+PgBcCIa79kEKR8YCW+A= 4 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 5 | github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= 6 | github.com/Masterminds/goutils v1.1.0 h1:zukEsf/1JZwCMgHiK3GZftabmxiCw4apj3a28RPBiVg= 7 | github.com/Masterminds/goutils v1.1.0/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= 8 | github.com/Masterminds/semver/v3 v3.1.0 h1:Y2lUDsFKVRSYGojLJ1yLxSXdMmMYTYls0rCvoqmMUQk= 9 | github.com/Masterminds/semver/v3 v3.1.0/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= 10 | github.com/Masterminds/sprig/v3 v3.1.0 h1:j7GpgZ7PdFqNsmncycTHsLmVPf5/3wJtlgW9TNDYD9Y= 11 | github.com/Masterminds/sprig/v3 v3.1.0/go.mod h1:ONGMf7UfYGAbMXCZmQLy8x3lCDIPrEZE/rU8pmrbihA= 12 | github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= 13 | github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= 14 | github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= 15 | github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= 16 | github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= 17 | github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= 18 | github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= 19 | github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= 20 | github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= 21 | github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= 22 | github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= 23 | github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= 24 | github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= 25 | github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= 26 | github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 h1:Hs82Z41s6SdL1CELW+XaDYmOH4hkBN4/N9og/AsOv7E= 27 | github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= 28 | github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= 29 | github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= 30 | github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= 31 | github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= 32 | github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I= 33 | github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= 34 | github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= 35 | github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= 36 | github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= 37 | github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= 38 | github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= 39 | github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= 40 | github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= 41 | github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= 42 | github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= 43 | github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= 44 | github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= 45 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= 46 | github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= 47 | github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= 48 | github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= 49 | github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= 50 | github.com/cenkalti/backoff/v4 v4.0.2 h1:JIufpQLbh4DkbQoii76ItQIUFzevQSqOLZca4eamEDs= 51 | github.com/cenkalti/backoff/v4 v4.0.2/go.mod h1:eEew/i+1Q6OrCDZh3WiXYv3+nJwBASZ8Bog/87DQnVg= 52 | github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= 53 | github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= 54 | github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= 55 | github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= 56 | github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 57 | github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= 58 | github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= 59 | github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= 60 | github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= 61 | github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= 62 | github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= 63 | github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= 64 | github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= 65 | github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= 66 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 67 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 68 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 69 | github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= 70 | github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= 71 | github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= 72 | github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= 73 | github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= 74 | github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= 75 | github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= 76 | github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= 77 | github.com/emicklei/go-restful v2.12.0+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= 78 | github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= 79 | github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= 80 | github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= 81 | github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= 82 | github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= 83 | github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= 84 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= 85 | github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= 86 | github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= 87 | github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= 88 | github.com/gin-gonic/gin v1.6.3 h1:ahKqKTFpO5KTPHxWZjEdPScmYaGtLo8Y4DMHoEsnp14= 89 | github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= 90 | github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= 91 | github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= 92 | github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= 93 | github.com/go-kit/kit v0.9.0 h1:wDJmvq38kDhkVxi50ni9ykkdUr1PKgqKOoi01fa0Mdk= 94 | github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= 95 | github.com/go-kit/kit v0.10.0 h1:dXFJfIHVvUcpSgDOV+Ne6t7jXri8Tfv2uOLHUZ2XNuo= 96 | github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= 97 | github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= 98 | github.com/go-logfmt/logfmt v0.4.0 h1:MP4Eh7ZCb31lleYCFuwm0oe4/YGak+5l1vA2NOE80nA= 99 | github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= 100 | github.com/go-logfmt/logfmt v0.5.0 h1:TrB8swr/68K7m9CcGut2g3UOihhbcbiMAYiuTXdEih4= 101 | github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= 102 | github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI= 103 | github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= 104 | github.com/go-openapi/analysis v0.18.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= 105 | github.com/go-openapi/analysis v0.19.2/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9snX6gxi44djMjk= 106 | github.com/go-openapi/analysis v0.19.4/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9snX6gxi44djMjk= 107 | github.com/go-openapi/analysis v0.19.5/go.mod h1:hkEAkxagaIvIP7VTn8ygJNkd4kAYON2rCu0v0ObL0AU= 108 | github.com/go-openapi/analysis v0.19.10/go.mod h1:qmhS3VNFxBlquFJ0RGoDtylO9y4pgTAUNE9AEEMdlJQ= 109 | github.com/go-openapi/errors v0.17.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= 110 | github.com/go-openapi/errors v0.18.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= 111 | github.com/go-openapi/errors v0.19.2/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94= 112 | github.com/go-openapi/errors v0.19.3/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94= 113 | github.com/go-openapi/errors v0.19.4/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94= 114 | github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= 115 | github.com/go-openapi/jsonpointer v0.18.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= 116 | github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= 117 | github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= 118 | github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= 119 | github.com/go-openapi/jsonreference v0.18.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= 120 | github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= 121 | github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= 122 | github.com/go-openapi/loads v0.17.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= 123 | github.com/go-openapi/loads v0.18.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= 124 | github.com/go-openapi/loads v0.19.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= 125 | github.com/go-openapi/loads v0.19.2/go.mod h1:QAskZPMX5V0C2gvfkGZzJlINuP7Hx/4+ix5jWFxsNPs= 126 | github.com/go-openapi/loads v0.19.3/go.mod h1:YVfqhUCdahYwR3f3iiwQLhicVRvLlU/WO5WPaZvcvSI= 127 | github.com/go-openapi/loads v0.19.4/go.mod h1:zZVHonKd8DXyxyw4yfnVjPzBjIQcLt0CCsn0N0ZrQsk= 128 | github.com/go-openapi/loads v0.19.5/go.mod h1:dswLCAdonkRufe/gSUC3gN8nTSaB9uaS2es0x5/IbjY= 129 | github.com/go-openapi/runtime v0.0.0-20180920151709-4f900dc2ade9/go.mod h1:6v9a6LTXWQCdL8k1AO3cvqx5OtZY/Y9wKTgaoP6YRfA= 130 | github.com/go-openapi/runtime v0.19.0/go.mod h1:OwNfisksmmaZse4+gpV3Ne9AyMOlP1lt4sK4FXt0O64= 131 | github.com/go-openapi/runtime v0.19.4/go.mod h1:X277bwSUBxVlCYR3r7xgZZGKVvBd/29gLDlFGtJ8NL4= 132 | github.com/go-openapi/runtime v0.19.15/go.mod h1:dhGWCTKRXlAfGnQG0ONViOZpjfg0m2gUt9nTQPQZuoo= 133 | github.com/go-openapi/spec v0.17.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= 134 | github.com/go-openapi/spec v0.18.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= 135 | github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY= 136 | github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= 137 | github.com/go-openapi/spec v0.19.6/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk= 138 | github.com/go-openapi/spec v0.19.8/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk= 139 | github.com/go-openapi/strfmt v0.17.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= 140 | github.com/go-openapi/strfmt v0.18.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= 141 | github.com/go-openapi/strfmt v0.19.0/go.mod h1:+uW+93UVvGGq2qGaZxdDeJqSAqBqBdl+ZPMF/cC8nDY= 142 | github.com/go-openapi/strfmt v0.19.2/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6+bY1AVL9LU= 143 | github.com/go-openapi/strfmt v0.19.3/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6+bY1AVL9LU= 144 | github.com/go-openapi/strfmt v0.19.4/go.mod h1:eftuHTlB/dI8Uq8JJOyRlieZf+WkkxUuk0dgdHXr2Qk= 145 | github.com/go-openapi/strfmt v0.19.5/go.mod h1:eftuHTlB/dI8Uq8JJOyRlieZf+WkkxUuk0dgdHXr2Qk= 146 | github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= 147 | github.com/go-openapi/swag v0.18.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= 148 | github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= 149 | github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= 150 | github.com/go-openapi/swag v0.19.7/go.mod h1:ao+8BpOPyKdpQz3AOJfbeEVpLmWAvlT1IfTe5McPyhY= 151 | github.com/go-openapi/swag v0.19.9/go.mod h1:ao+8BpOPyKdpQz3AOJfbeEVpLmWAvlT1IfTe5McPyhY= 152 | github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= 153 | github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA= 154 | github.com/go-openapi/validate v0.19.3/go.mod h1:90Vh6jjkTn+OT1Eefm0ZixWNFjhtOH7vS9k0lo6zwJo= 155 | github.com/go-openapi/validate v0.19.8/go.mod h1:8DJv2CVJQ6kGNpFW6eV9N3JviE1C85nY1c2z52x1Gk4= 156 | github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A= 157 | github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= 158 | github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= 159 | github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= 160 | github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no= 161 | github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= 162 | github.com/go-playground/validator/v10 v10.2.0 h1:KgJ0snyC2R9VXYN2rneOtQcw5aHQB1Vv0sFl1UcHBOY= 163 | github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI= 164 | github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= 165 | github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= 166 | github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= 167 | github.com/go-telegram-bot-api/telegram-bot-api v4.6.4+incompatible h1:2cauKuaELYAEARXRkq2LrJ0yDDv1rW7+wrTEdVL3uaU= 168 | github.com/go-telegram-bot-api/telegram-bot-api v4.6.4+incompatible/go.mod h1:qf9acutJ8cwBUhm1bqgz6Bei9/C/c93FPDljKWwsOgM= 169 | github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= 170 | github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY= 171 | github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg= 172 | github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= 173 | github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= 174 | github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs= 175 | github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= 176 | github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= 177 | github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk= 178 | github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28= 179 | github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo= 180 | github.com/gobuffalo/genny v0.1.1/go.mod h1:5TExbEyY48pfunL4QSXxlDOmdsD44RRq4mVZ0Ex28Xk= 181 | github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211/go.mod h1:vEHJk/E9DmhejeLeNt7UVvlSGv3ziL+djtTr3yyzcOw= 182 | github.com/gobuffalo/gogen v0.0.0-20190315121717-8f38393713f5/go.mod h1:V9QVDIxsgKNZs6L2IYiGR8datgMhB577vzTDqypH360= 183 | github.com/gobuffalo/gogen v0.1.0/go.mod h1:8NTelM5qd8RZ15VjQTFkAW6qOMx5wBbW4dSCS3BY8gg= 184 | github.com/gobuffalo/gogen v0.1.1/go.mod h1:y8iBtmHmGc4qa3urIyo1shvOD8JftTtfcKi+71xfDNE= 185 | github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8= 186 | github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= 187 | github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= 188 | github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= 189 | github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= 190 | github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ= 191 | github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= 192 | github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= 193 | github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= 194 | github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= 195 | github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= 196 | github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= 197 | github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= 198 | github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= 199 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= 200 | github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= 201 | github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= 202 | github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 203 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 204 | github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 205 | github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= 206 | github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 207 | github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= 208 | github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= 209 | github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= 210 | github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= 211 | github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= 212 | github.com/golang/protobuf v1.4.0 h1:oOuy+ugB+P/kBdUnG5QaMXSIyJ1q38wWSojYCb3z5VQ= 213 | github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= 214 | github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 215 | github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 216 | github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c h1:964Od4U6p2jUkFxvCydnIczKteheJEzHRToSGK3Bnlw= 217 | github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= 218 | github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= 219 | github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= 220 | github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= 221 | github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 222 | github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= 223 | github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 224 | github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= 225 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 226 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 227 | github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= 228 | github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 229 | github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= 230 | github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 231 | github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= 232 | github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= 233 | github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= 234 | github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= 235 | github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= 236 | github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= 237 | github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= 238 | github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= 239 | github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= 240 | github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= 241 | github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= 242 | github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= 243 | github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= 244 | github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0= 245 | github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= 246 | github.com/hashicorp/go-msgpack v0.5.3 h1:zKjpN5BK/P5lMYrLmBHdBULWbJ0XpYR+7NGzqkZzoD4= 247 | github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= 248 | github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o= 249 | github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= 250 | github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= 251 | github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= 252 | github.com/hashicorp/go-sockaddr v1.0.2 h1:ztczhD1jLxIRjVejw8gFomI1BQZOe2WoVOu0SyteCQc= 253 | github.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjGlgmH/UkBUC97A= 254 | github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= 255 | github.com/hashicorp/go-uuid v1.0.0 h1:RS8zrF7PhGwyNPOtxSClXXj9HA8feRnJzgnI1RJCSnM= 256 | github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= 257 | github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE= 258 | github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= 259 | github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= 260 | github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= 261 | github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo= 262 | github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= 263 | github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= 264 | github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= 265 | github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= 266 | github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= 267 | github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= 268 | github.com/hashicorp/memberlist v0.2.2 h1:5+RffWKwqJ71YPu9mWsF7ZOscZmwfasdA8kbdC7AO2g= 269 | github.com/hashicorp/memberlist v0.2.2/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= 270 | github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= 271 | github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= 272 | github.com/huandu/xstrings v1.3.1 h1:4jgBlKK6tLKFvO8u5pmYjG91cqytmDCDvGh7ECVFfFs= 273 | github.com/huandu/xstrings v1.3.1/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= 274 | github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= 275 | github.com/imdario/mergo v0.3.8 h1:CGgOkSJeqMRmt0D9XLWExdT4m4F1vd3FV3VPt+0VxkQ= 276 | github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= 277 | github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= 278 | github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= 279 | github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= 280 | github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= 281 | github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= 282 | github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= 283 | github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= 284 | github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 285 | github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 286 | github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns= 287 | github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 288 | github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= 289 | github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= 290 | github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= 291 | github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= 292 | github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= 293 | github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= 294 | github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= 295 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 296 | github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= 297 | github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= 298 | github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 299 | github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 300 | github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY= 301 | github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= 302 | github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= 303 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 304 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 305 | github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= 306 | github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= 307 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 308 | github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= 309 | github.com/labstack/echo/v4 v4.1.16/go.mod h1:awO+5TzAjvL8XpibdsfXxPgHr+orhtXZJZIQCVjogKI= 310 | github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= 311 | github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= 312 | github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= 313 | github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= 314 | github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= 315 | github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= 316 | github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= 317 | github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= 318 | github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= 319 | github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= 320 | github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= 321 | github.com/mailru/easyjson v0.7.1/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= 322 | github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= 323 | github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= 324 | github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= 325 | github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= 326 | github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= 327 | github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= 328 | github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= 329 | github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= 330 | github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg= 331 | github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= 332 | github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= 333 | github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= 334 | github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= 335 | github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= 336 | github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= 337 | github.com/miekg/dns v1.0.14 h1:9jZdLNd/P4+SfEJ0TNyxYpsK8N4GtfylBLqtbYN1sbA= 338 | github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= 339 | github.com/miekg/dns v1.1.26 h1:gPxPSwALAeHJSjarOs00QjVdV9QoBvc1D2ujQUr5BzU= 340 | github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= 341 | github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= 342 | github.com/mitchellh/copystructure v1.0.0 h1:Laisrj+bAB6b/yJwB5Bt3ITZhGJdqmxquMKeZ+mmkFQ= 343 | github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= 344 | github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= 345 | github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= 346 | github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= 347 | github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= 348 | github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= 349 | github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= 350 | github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= 351 | github.com/mitchellh/reflectwalk v1.0.0 h1:9D+8oIskB4VJBN5SFlmc27fSlIBZaov1Wpk/IfikLNY= 352 | github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= 353 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 354 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= 355 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 356 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 357 | github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= 358 | github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 359 | github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= 360 | github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223 h1:F9x/1yl3T2AeKLr2AMdilSD8+f9bvMnNN8VS5iDtovc= 361 | github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= 362 | github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= 363 | github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= 364 | github.com/nats-io/nats-server/v2 v2.1.2/go.mod h1:Afk+wRZqkMQs/p45uXdrVLuab3gwv3Z8C4HTBu8GD/k= 365 | github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= 366 | github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= 367 | github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= 368 | github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= 369 | github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= 370 | github.com/oklog/run v1.0.0 h1:Ru7dDtJNOyC66gQ5dQmaCa0qIsAUFY3sFpK1Xk8igrw= 371 | github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= 372 | github.com/oklog/run v1.1.0 h1:GEenZ1cK0+q0+wsJew9qUg/DyD8k3JzYsZAi5gYi2mA= 373 | github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU= 374 | github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= 375 | github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= 376 | github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= 377 | github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 378 | github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 379 | github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= 380 | github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= 381 | github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= 382 | github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= 383 | github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= 384 | github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= 385 | github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5/go.mod h1:/wsWhb9smxSfWAKL3wpBW7V8scJMt8N8gnaMCS9E/cA= 386 | github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= 387 | github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= 388 | github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= 389 | github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= 390 | github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c h1:Lgl0gzECD8GnQ5QCWA8o6BtfL6mDH5rQgM4/fX3avOs= 391 | github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= 392 | github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= 393 | github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= 394 | github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= 395 | github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= 396 | github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= 397 | github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 398 | github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= 399 | github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 400 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 401 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 402 | github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= 403 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 404 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 405 | github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= 406 | github.com/prometheus/alertmanager v0.21.0 h1:qK51JcUR9l/unhawGA9F9B64OCYfcGewhPNprem/Acc= 407 | github.com/prometheus/alertmanager v0.21.0/go.mod h1:h7tJ81NA0VLWvWEayi1QltevFkLF3KxmC/malTcT8Go= 408 | github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= 409 | github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= 410 | github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= 411 | github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= 412 | github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= 413 | github.com/prometheus/client_golang v1.6.0 h1:YVPodQOcK15POxhgARIvnDRVpLcuK8mglnMrWfyrw6A= 414 | github.com/prometheus/client_golang v1.6.0/go.mod h1:ZLOG9ck3JLRdB5MgO8f+lLTe83AXG6ro35rLTxvnIl4= 415 | github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= 416 | github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= 417 | github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 418 | github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM= 419 | github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 420 | github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 421 | github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= 422 | github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 423 | github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= 424 | github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= 425 | github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= 426 | github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= 427 | github.com/prometheus/common v0.9.1 h1:KOMtN28tlbam3/7ZKEYKHhKoJZYYj3gMH4uc62x7X7U= 428 | github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= 429 | github.com/prometheus/common v0.10.0 h1:RyRA7RzGXQZiW+tGMr7sxa85G1z0yOpM1qq5c8lNawc= 430 | github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= 431 | github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= 432 | github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= 433 | github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= 434 | github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= 435 | github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= 436 | github.com/prometheus/procfs v0.0.11 h1:DhHlBtkHWPYi8O2y31JkK0TF+DGM+51OopZjH/Ia5qI= 437 | github.com/prometheus/procfs v0.0.11/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= 438 | github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= 439 | github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= 440 | github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= 441 | github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= 442 | github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= 443 | github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= 444 | github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= 445 | github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= 446 | github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= 447 | github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= 448 | github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= 449 | github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= 450 | github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= 451 | github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= 452 | github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= 453 | github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749 h1:bUGsEnyNbVPw06Bs80sCeARAlK8lhwqGyi6UT8ymuGk= 454 | github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= 455 | github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= 456 | github.com/shurcooL/vfsgen v0.0.0-20181202132449-6a9ea43bcacd h1:ug7PpSOB5RBPK1Kg6qskGBoP3Vnj/aNYFTznWvlkGo0= 457 | github.com/shurcooL/vfsgen v0.0.0-20181202132449-6a9ea43bcacd/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw= 458 | github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= 459 | github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= 460 | github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= 461 | github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= 462 | github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= 463 | github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM= 464 | github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= 465 | github.com/slok/go-http-metrics v0.8.0 h1:rsIKW30MzLjbWRBkCQoe/Oxh/F283MKT6afdH3mXTaA= 466 | github.com/slok/go-http-metrics v0.8.0/go.mod h1:f22ekj0Ht4taz2clntVmLRSK4D+feX33zkdDW0Eytvk= 467 | github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= 468 | github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= 469 | github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= 470 | github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= 471 | github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= 472 | github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= 473 | github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= 474 | github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= 475 | github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= 476 | github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= 477 | github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= 478 | github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= 479 | github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= 480 | github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= 481 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 482 | github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 483 | github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= 484 | github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= 485 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 486 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 487 | github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= 488 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 489 | github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= 490 | github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= 491 | github.com/stretchr/testify v1.6.0 h1:jlIyCplCJFULU/01vCkhKuTyc3OorI3bJFuw6obfgho= 492 | github.com/stretchr/testify v1.6.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 493 | github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= 494 | github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 495 | github.com/technoweenie/multipartstreamer v1.0.1 h1:XRztA5MXiR1TIRHxH2uNxXxaIkKQDeX7m2XsSOlQEnM= 496 | github.com/technoweenie/multipartstreamer v1.0.1/go.mod h1:jNVxdtShOxzAsukZwTSw6MDx5eUJoiEBsSvzDU9uzog= 497 | github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= 498 | github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= 499 | github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo= 500 | github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= 501 | github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs= 502 | github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= 503 | github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= 504 | github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= 505 | github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= 506 | github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= 507 | github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= 508 | github.com/valyala/fasttemplate v1.1.0/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= 509 | github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= 510 | github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= 511 | github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= 512 | github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= 513 | github.com/xlab/treeprint v1.0.0/go.mod h1:IoImgRak9i3zJyuxOKUP1v4UZd1tMoKkq/Cimt1uhCg= 514 | github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 515 | go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= 516 | go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= 517 | go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= 518 | go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= 519 | go.mongodb.org/mongo-driver v1.1.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= 520 | go.mongodb.org/mongo-driver v1.3.0/go.mod h1:MSWZXKOynuguX+JSvwP8i+58jYCXxbia8HS3gZBapIE= 521 | go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= 522 | go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= 523 | go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= 524 | go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= 525 | go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= 526 | go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= 527 | go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= 528 | go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= 529 | go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= 530 | go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= 531 | go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= 532 | go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= 533 | goji.io v2.0.2+incompatible/go.mod h1:sbqFwrtqZACxLBTQcdgVjFh54yGVCvwq8+w49MVMMIk= 534 | golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 535 | golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 536 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 537 | golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 538 | golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= 539 | golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 540 | golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 541 | golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 542 | golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 543 | golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 544 | golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= 545 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 546 | golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d h1:1ZiEyfaQIg3Qh0EoqpwAakHVhecoE5wlSg5GjnafJGw= 547 | golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 548 | golang.org/x/crypto v0.0.0-20200414173820-0848c9571904 h1:bXoxMPcSLOq08zI3/c5dEBT6lE4eh+jOh886GHrn6V8= 549 | golang.org/x/crypto v0.0.0-20200414173820-0848c9571904/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 550 | golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 551 | golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 552 | golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= 553 | golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 554 | golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 555 | golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 556 | golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= 557 | golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= 558 | golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 559 | golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 560 | golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 561 | golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 562 | golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 563 | golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 564 | golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 565 | golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 566 | golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 567 | golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 568 | golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 569 | golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 570 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 571 | golang.org/x/net v0.0.0-20190320064053-1272bf9dcd53/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 572 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 573 | golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= 574 | golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 575 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 576 | golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 577 | golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 578 | golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 579 | golang.org/x/net v0.0.0-20200226121028-0de0cce0169b h1:0mm1VjtFUOIlE1SbDlwjYaDxZVDP2S5ou6y0gSgXHu8= 580 | golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 581 | golang.org/x/net v0.0.0-20200513185701-a91f0712d120 h1:EZ3cVSzKOlJxAd8e8YAJ7no8nNypTxexh/YE/xW3ZEY= 582 | golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= 583 | golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= 584 | golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= 585 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 586 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 587 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 588 | golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 589 | golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 590 | golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= 591 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 592 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= 593 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 594 | golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 595 | golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 596 | golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 597 | golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 598 | golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 599 | golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 600 | golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 601 | golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 602 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 603 | golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 604 | golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 605 | golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 606 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 607 | golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 608 | golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 609 | golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 610 | golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 611 | golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 612 | golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 613 | golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 614 | golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 615 | golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 616 | golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 617 | golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 618 | golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 619 | golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 620 | golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 621 | golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 622 | golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 623 | golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f h1:gWF768j/LaZugp8dyS4UwsslYCYz9XgFxvlgsn0n9H8= 624 | golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 625 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 626 | golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= 627 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 628 | golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 629 | golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 630 | golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 631 | golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 632 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 633 | golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 634 | golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 635 | golang.org/x/tools v0.0.0-20190125232054-d66bd3c5d5a6/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 636 | golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= 637 | golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 638 | golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 639 | golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 640 | golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 641 | golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 642 | golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 643 | golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= 644 | golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= 645 | golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= 646 | golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= 647 | golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= 648 | golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 649 | golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 650 | golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 651 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 652 | golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= 653 | golang.org/x/tools v0.0.0-20200513201620-d5fe73897c97 h1:DAuln/hGp+aJiHpID1Y1hYzMEPP5WLwtZHPb50mN0OE= 654 | golang.org/x/tools v0.0.0-20200513201620-d5fe73897c97/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= 655 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 656 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 657 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= 658 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 659 | google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= 660 | google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= 661 | google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 662 | google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 663 | google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= 664 | google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 665 | google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 666 | google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= 667 | google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= 668 | google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= 669 | google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= 670 | google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= 671 | google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= 672 | google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= 673 | google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= 674 | google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= 675 | google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= 676 | google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= 677 | google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= 678 | google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= 679 | google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= 680 | google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= 681 | google.golang.org/protobuf v1.21.0 h1:qdOKuR/EIArgaWNjetjgTzgVTAZ+S/WXVrq9HW9zimw= 682 | google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= 683 | gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= 684 | gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= 685 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 686 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= 687 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 688 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= 689 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 690 | gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= 691 | gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= 692 | gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= 693 | gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= 694 | gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= 695 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= 696 | gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= 697 | gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= 698 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 699 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 700 | gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= 701 | gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 702 | gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 703 | gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= 704 | gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 705 | gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= 706 | gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 707 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= 708 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 709 | honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 710 | honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 711 | honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 712 | honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= 713 | sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= 714 | sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= 715 | -------------------------------------------------------------------------------- /internal/deadmansswitch/deadmansswitch.go: -------------------------------------------------------------------------------- 1 | package deadmansswitch 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "time" 7 | 8 | "github.com/slok/alertgram/internal/forward" 9 | "github.com/slok/alertgram/internal/log" 10 | "github.com/slok/alertgram/internal/model" 11 | ) 12 | 13 | // Service is a Dead man's switch 14 | // 15 | // A dead man's switch is a process where at regular intervals if some kind of signal is 16 | // not received it will be activated. This usually is used to check that some kind 17 | // of system is working, in this case if we don't receive an alert we assume that something 18 | // is not working and we should notify. 19 | type Service interface { 20 | // PushSwitch will disable the dead man's switch when it's pushed and reset 21 | // the interval for activation. 22 | PushSwitch(ctx context.Context, alertGroup *model.AlertGroup) error 23 | } 24 | 25 | // Config is the Service configuration. 26 | type Config struct { 27 | CustomChatID string 28 | Interval time.Duration 29 | Notifiers []forward.Notifier 30 | Logger log.Logger 31 | } 32 | 33 | func (c *Config) defaults() error { 34 | if c.Logger == nil { 35 | c.Logger = log.Dummy 36 | } 37 | return nil 38 | } 39 | 40 | type service struct { 41 | cfg Config 42 | dmsSwitch chan *model.AlertGroup 43 | notifiers []forward.Notifier 44 | logger log.Logger 45 | } 46 | 47 | // NewService returns a Dead mans's switch service. 48 | // When creating a new instance it will start the dead man's switch interval 49 | // it can only stop once and it's done when the received context is done. 50 | func NewService(ctx context.Context, cfg Config) (Service, error) { 51 | err := cfg.defaults() 52 | if err != nil { 53 | return nil, fmt.Errorf("invalid dead man's switch service configuration: %w", err) 54 | } 55 | s := &service{ 56 | cfg: cfg, 57 | dmsSwitch: make(chan *model.AlertGroup, 1), 58 | notifiers: cfg.Notifiers, 59 | logger: cfg.Logger.WithValues(log.KV{"service": "deadMansSwitch"}), 60 | } 61 | go s.startDMS(ctx) 62 | 63 | return s, nil 64 | } 65 | 66 | func (s *service) PushSwitch(_ context.Context, alertGroup *model.AlertGroup) error { 67 | if alertGroup != nil { 68 | s.dmsSwitch <- alertGroup 69 | } 70 | return nil 71 | } 72 | 73 | func (s *service) activate(ctx context.Context) error { 74 | dmsNotification := forward.Notification{ 75 | ChatID: s.cfg.CustomChatID, 76 | AlertGroup: model.AlertGroup{ 77 | ID: "DeadMansSwitchActive", 78 | Alerts: []model.Alert{ 79 | model.Alert{ 80 | ID: "DeadMansSwitchActive", 81 | Name: "DeadMansSwitchActive", 82 | StartsAt: time.Now(), 83 | Status: model.AlertStatusFiring, 84 | Labels: map[string]string{ 85 | "alertname": "DeadMansSwitchActive", 86 | "severity": "critical", 87 | "origin": "alertgram", 88 | }, 89 | Annotations: map[string]string{ 90 | "message": "The Dead man's switch has been activated! This usually means that your monitoring/alerting system is not working", 91 | }, 92 | }, 93 | }, 94 | }, 95 | } 96 | 97 | // TODO(slok): Add concurrency using workers. 98 | for _, not := range s.notifiers { 99 | err := not.Notify(ctx, dmsNotification) 100 | if err != nil { 101 | s.logger.WithValues(log.KV{"notifier": not.Type(), "alertGroupID": dmsNotification.AlertGroup.ID}). 102 | Errorf("could not notify alert group: %s", err) 103 | } 104 | } 105 | return nil 106 | } 107 | 108 | // startDMS will start the DeadMansSwitch process. 109 | // It will be listening to the signals to know 110 | // that we are alive, if not received in the interval the 111 | // Dead mans switch should assume we are dead and will activate 112 | // this means executing the received function. 113 | func (s *service) startDMS(ctx context.Context) { 114 | logger := s.logger.WithValues(log.KV{"interval": s.cfg.Interval}) 115 | logger.Infof("dead man's switch started with an interval of %s", s.cfg.Interval) 116 | 117 | for { 118 | select { 119 | case <-ctx.Done(): 120 | logger.Infof("context done, stopping dead man's switch") 121 | return 122 | case <-time.After(s.cfg.Interval): 123 | logger.Infof("no switch pushed during interval wait, dead mans switch activated!") 124 | err := s.activate(ctx) 125 | if err != nil { 126 | logger.Errorf("something happened when activating the dead man's switch") 127 | } 128 | case <-s.dmsSwitch: 129 | logger.Debugf("dead mans switch pushed, deactivated") 130 | } 131 | } 132 | } 133 | 134 | // DisabledService is a Dead man switch service that doesn't do anything. 135 | const DisabledService = dummyService(0) 136 | 137 | type dummyService int 138 | 139 | func (dummyService) PushSwitch(ctx context.Context, alertGroup *model.AlertGroup) error { return nil } 140 | -------------------------------------------------------------------------------- /internal/deadmansswitch/deadmansswitch_test.go: -------------------------------------------------------------------------------- 1 | package deadmansswitch_test 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "testing" 7 | "time" 8 | 9 | "github.com/stretchr/testify/assert" 10 | "github.com/stretchr/testify/mock" 11 | "github.com/stretchr/testify/require" 12 | 13 | "github.com/slok/alertgram/internal/deadmansswitch" 14 | "github.com/slok/alertgram/internal/forward" 15 | forwardmock "github.com/slok/alertgram/internal/mocks/forward" 16 | "github.com/slok/alertgram/internal/model" 17 | ) 18 | 19 | func TestServiceDeadMansSwitch(t *testing.T) { 20 | tests := map[string]struct { 21 | cfg deadmansswitch.Config 22 | exec func(svc deadmansswitch.Service) error 23 | mock func(ns []*forwardmock.Notifier) 24 | expErr error 25 | }{ 26 | "If the alert is not received in the interval it should notify.": { 27 | cfg: deadmansswitch.Config{ 28 | Interval: 10 * time.Millisecond, 29 | }, 30 | exec: func(svc deadmansswitch.Service) error { 31 | // Give time to interval to act. 32 | time.Sleep(15 * time.Millisecond) 33 | return nil 34 | }, 35 | mock: func(ns []*forwardmock.Notifier) { 36 | for _, n := range ns { 37 | n.On("Notify", mock.Anything, mock.Anything).Once().Return(nil) 38 | n.On("Type").Maybe().Return("") 39 | } 40 | }, 41 | }, 42 | 43 | "If the alert is received in the interval it should not notify.": { 44 | cfg: deadmansswitch.Config{ 45 | Interval: 10 * time.Millisecond, 46 | }, 47 | exec: func(svc deadmansswitch.Service) error { 48 | // Give time to interval to act. 49 | time.Sleep(6 * time.Millisecond) 50 | err := svc.PushSwitch(context.TODO(), &model.AlertGroup{}) 51 | time.Sleep(6 * time.Millisecond) 52 | return err 53 | }, 54 | mock: func(ns []*forwardmock.Notifier) {}, 55 | }, 56 | 57 | "If the alert is received and then stops being received in the interval it should not notify.": { 58 | cfg: deadmansswitch.Config{ 59 | Interval: 10 * time.Millisecond, 60 | }, 61 | exec: func(svc deadmansswitch.Service) error { 62 | time.Sleep(6 * time.Millisecond) 63 | err := svc.PushSwitch(context.TODO(), &model.AlertGroup{}) 64 | if err != nil { 65 | return err 66 | } 67 | time.Sleep(6 * time.Millisecond) 68 | err = svc.PushSwitch(context.TODO(), &model.AlertGroup{}) 69 | time.Sleep(15 * time.Millisecond) 70 | return err 71 | }, 72 | mock: func(ns []*forwardmock.Notifier) { 73 | for _, n := range ns { 74 | n.On("Notify", mock.Anything, mock.Anything).Once().Return(nil) 75 | n.On("Type").Maybe().Return("") 76 | } 77 | }, 78 | }, 79 | } 80 | 81 | for name, test := range tests { 82 | t.Run(name, func(t *testing.T) { 83 | assert := assert.New(t) 84 | require := require.New(t) 85 | 86 | mn1 := &forwardmock.Notifier{} 87 | mn2 := &forwardmock.Notifier{} 88 | test.mock([]*forwardmock.Notifier{mn1, mn2}) 89 | 90 | test.cfg.Notifiers = []forward.Notifier{mn1, mn2} 91 | ctx, cancel := context.WithCancel(context.Background()) 92 | defer cancel() 93 | svc, err := deadmansswitch.NewService(ctx, test.cfg) 94 | require.NoError(err) 95 | err = test.exec(svc) 96 | 97 | if test.expErr != nil && assert.Error(err) { 98 | assert.True(errors.Is(err, test.expErr)) 99 | } else if assert.NoError(err) { 100 | mn1.AssertExpectations(t) 101 | mn2.AssertExpectations(t) 102 | } 103 | }) 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /internal/deadmansswitch/metrics.go: -------------------------------------------------------------------------------- 1 | package deadmansswitch 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | "github.com/slok/alertgram/internal/model" 8 | ) 9 | 10 | // ServiceMetricsRecorder knows how to record metrics on deadmansswitch.Service. 11 | type ServiceMetricsRecorder interface { 12 | ObserveDMSServiceOpDuration(ctx context.Context, op string, success bool, t time.Duration) 13 | } 14 | 15 | type measureService struct { 16 | rec ServiceMetricsRecorder 17 | next Service 18 | } 19 | 20 | // NewMeasureService wraps a service and measures using metrics. 21 | func NewMeasureService(rec ServiceMetricsRecorder, next Service) Service { 22 | return &measureService{ 23 | rec: rec, 24 | next: next, 25 | } 26 | } 27 | 28 | func (m measureService) PushSwitch(ctx context.Context, ag *model.AlertGroup) (err error) { 29 | defer func(t0 time.Time) { 30 | m.rec.ObserveDMSServiceOpDuration(ctx, "PushSwitch", err == nil, time.Since(t0)) 31 | }(time.Now()) 32 | return m.next.PushSwitch(ctx, ag) 33 | } 34 | -------------------------------------------------------------------------------- /internal/forward/forward.go: -------------------------------------------------------------------------------- 1 | package forward 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | 8 | "github.com/slok/alertgram/internal/internalerrors" 9 | "github.com/slok/alertgram/internal/log" 10 | "github.com/slok/alertgram/internal/model" 11 | ) 12 | 13 | // Properties are the properties an AlertGroup can have 14 | // when the forwarding process is done. 15 | type Properties struct { 16 | // CustomChatID can be used when the forward should be done 17 | // to a different target (chat, group, channel, user...) 18 | // instead of using the default one. 19 | CustomChatID string 20 | } 21 | 22 | // Service is the domain service that forwards alerts 23 | type Service interface { 24 | // Forward knows how to forward alerts from an input to an output. 25 | Forward(ctx context.Context, props Properties, alertGroup *model.AlertGroup) error 26 | } 27 | 28 | // ServiceConfig is the service configuration. 29 | type ServiceConfig struct { 30 | AlertLabelChatID string 31 | Notifiers []Notifier 32 | Logger log.Logger 33 | } 34 | 35 | func (c *ServiceConfig) defaults() error { 36 | if len(c.Notifiers) == 0 { 37 | return errors.New("notifiers can't be empty") 38 | } 39 | 40 | if c.Logger == nil { 41 | c.Logger = log.Dummy 42 | } 43 | 44 | return nil 45 | } 46 | 47 | type service struct { 48 | cfg ServiceConfig 49 | notifiers []Notifier 50 | logger log.Logger 51 | } 52 | 53 | // NewService returns a new forward.Service. 54 | func NewService(cfg ServiceConfig) (Service, error) { 55 | err := cfg.defaults() 56 | if err != nil { 57 | err := fmt.Errorf("%w: %s", internalerrors.ErrInvalidConfiguration, err) 58 | return nil, fmt.Errorf("could not create forward service instance because invalid configuration: %w", err) 59 | } 60 | 61 | return &service{ 62 | cfg: cfg, 63 | notifiers: cfg.Notifiers, 64 | logger: cfg.Logger.WithValues(log.KV{"service": "forward.Service"}), 65 | }, nil 66 | } 67 | 68 | var ( 69 | // ErrInvalidAlertGroup will be used when the alertgroup is not valid. 70 | ErrInvalidAlertGroup = errors.New("invalid alert group") 71 | ) 72 | 73 | func (s service) Forward(ctx context.Context, props Properties, alertGroup *model.AlertGroup) error { 74 | // TODO(slok): Add better validation. 75 | if alertGroup == nil { 76 | return fmt.Errorf("alertgroup can't be empty: %w", ErrInvalidAlertGroup) 77 | } 78 | 79 | notifications, err := s.createNotifications(props, alertGroup) 80 | if err != nil { 81 | return fmt.Errorf("could not prepare notifications from the alerts: %w", err) 82 | } 83 | 84 | // TODO(slok): Add concurrency using workers. 85 | for _, notifier := range s.notifiers { 86 | for _, notification := range notifications { 87 | err := notifier.Notify(ctx, *notification) 88 | if err != nil { 89 | s.logger.WithValues(log.KV{"notifier": notifier.Type(), "alertGroupID": alertGroup.ID, "chatID": notification.ChatID}). 90 | Errorf("could not notify alert group: %s", err) 91 | } 92 | } 93 | } 94 | 95 | return nil 96 | } 97 | 98 | func (s service) createNotifications(props Properties, alertGroup *model.AlertGroup) (ns []*Notification, err error) { 99 | // Decompose the alerts in groups by chat IDs based on the 100 | // alert chat ID labels. If the alerts don't have the chat ID 101 | // label they will remain on the default group. 102 | agByChatID := map[string]*model.AlertGroup{} 103 | for _, a := range alertGroup.Alerts { 104 | chatID := a.Labels[s.cfg.AlertLabelChatID] 105 | ag, ok := agByChatID[chatID] 106 | if !ok { 107 | id := alertGroup.ID 108 | if chatID != "" { 109 | id = fmt.Sprintf("%s-%s", alertGroup.ID, chatID) 110 | } 111 | ag = &model.AlertGroup{ 112 | ID: id, 113 | Labels: alertGroup.Labels, 114 | } 115 | agByChatID[chatID] = ag 116 | } 117 | 118 | ag.Alerts = append(ag.Alerts, a) 119 | } 120 | 121 | // Create notifications based on the alertgroups. 122 | notifications := []*Notification{} 123 | for chatID, ag := range agByChatID { 124 | // If no custom alert based chat then fallback to 125 | // properties custom chat (normally received by upper 126 | // layers by URL). 127 | if chatID == "" { 128 | chatID = props.CustomChatID 129 | } 130 | notifications = append(notifications, &Notification{ 131 | AlertGroup: *ag, 132 | ChatID: chatID, 133 | }) 134 | } 135 | 136 | return notifications, nil 137 | } 138 | -------------------------------------------------------------------------------- /internal/forward/forward_test.go: -------------------------------------------------------------------------------- 1 | package forward_test 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | "github.com/stretchr/testify/mock" 10 | "github.com/stretchr/testify/require" 11 | 12 | "github.com/slok/alertgram/internal/forward" 13 | forwardmock "github.com/slok/alertgram/internal/mocks/forward" 14 | "github.com/slok/alertgram/internal/model" 15 | ) 16 | 17 | var errTest = errors.New("whatever") 18 | 19 | func TestServiceForward(t *testing.T) { 20 | tests := map[string]struct { 21 | cfg forward.ServiceConfig 22 | props forward.Properties 23 | alertGroup *model.AlertGroup 24 | mock func(ns []*forwardmock.Notifier) 25 | expErr error 26 | }{ 27 | "A nil alert group should fail.": { 28 | mock: func(ns []*forwardmock.Notifier) {}, 29 | expErr: forward.ErrInvalidAlertGroup, 30 | }, 31 | 32 | "A forwarded alerts should be send to all notifiers.": { 33 | props: forward.Properties{ 34 | CustomChatID: "-1001234567890", 35 | }, 36 | alertGroup: &model.AlertGroup{ 37 | ID: "test-group", 38 | Alerts: []model.Alert{model.Alert{Name: "test"}}, 39 | }, 40 | mock: func(ns []*forwardmock.Notifier) { 41 | expNotification := forward.Notification{ 42 | ChatID: "-1001234567890", 43 | AlertGroup: model.AlertGroup{ 44 | ID: "test-group", 45 | Alerts: []model.Alert{model.Alert{Name: "test"}}, 46 | }, 47 | } 48 | for _, n := range ns { 49 | n.On("Notify", mock.Anything, expNotification).Once().Return(nil) 50 | } 51 | }, 52 | }, 53 | 54 | "Errors from notifiers should be ignored to the callers and all notifiers should be called.": { 55 | alertGroup: &model.AlertGroup{ 56 | ID: "test-group", 57 | Alerts: []model.Alert{model.Alert{Name: "test"}}, 58 | }, 59 | mock: func(ns []*forwardmock.Notifier) { 60 | expNotification := forward.Notification{ 61 | AlertGroup: model.AlertGroup{ 62 | ID: "test-group", 63 | Alerts: []model.Alert{model.Alert{Name: "test"}}, 64 | }, 65 | } 66 | for i, n := range ns { 67 | err := errTest 68 | // Set error in the first one. 69 | if i != 0 { 70 | err = nil 71 | } 72 | n.On("Notify", mock.Anything, expNotification).Once().Return(err) 73 | n.On("Type").Maybe().Return("") 74 | } 75 | }, 76 | }, 77 | 78 | "Alerts that have the label for custom chat ids should be grouped together.": { 79 | cfg: forward.ServiceConfig{ 80 | AlertLabelChatID: "test_chat_id", 81 | }, 82 | props: forward.Properties{ 83 | CustomChatID: "-1001234567890", 84 | }, 85 | alertGroup: &model.AlertGroup{ 86 | ID: "test-group", 87 | Alerts: []model.Alert{ 88 | {Name: "test-1", Labels: map[string]string{"test_chat_id": ""}}, 89 | {Name: "test-2", Labels: map[string]string{"test_chat_id": "chat2"}}, 90 | {Name: "test-3", Labels: map[string]string{"test_chat_id": "chat1"}}, 91 | {Name: "test-4", Labels: map[string]string{"test_chat_id": "chat2"}}, 92 | {Name: "test-3", Labels: map[string]string{"test_chat_id": "chat1"}}, 93 | {Name: "test-6", Labels: map[string]string{"test_chat_id": "chat3"}}, 94 | {Name: "test-7", Labels: map[string]string{"test_chat_id": ""}}, 95 | {Name: "test-8", Labels: map[string]string{"test_chat_id": ""}}, 96 | {Name: "test-9", Labels: map[string]string{"test_chat_id": "chat1"}}, 97 | }, 98 | }, 99 | mock: func(ns []*forwardmock.Notifier) { 100 | expNotChatDef := forward.Notification{ 101 | ChatID: "-1001234567890", 102 | AlertGroup: model.AlertGroup{ID: "test-group", 103 | Alerts: []model.Alert{ 104 | {Name: "test-1", Labels: map[string]string{"test_chat_id": ""}}, 105 | {Name: "test-7", Labels: map[string]string{"test_chat_id": ""}}, 106 | {Name: "test-8", Labels: map[string]string{"test_chat_id": ""}}, 107 | }, 108 | }, 109 | } 110 | expNotChat1 := forward.Notification{ 111 | ChatID: "chat1", 112 | AlertGroup: model.AlertGroup{ID: "test-group-chat1", 113 | Alerts: []model.Alert{ 114 | {Name: "test-3", Labels: map[string]string{"test_chat_id": "chat1"}}, 115 | {Name: "test-3", Labels: map[string]string{"test_chat_id": "chat1"}}, 116 | {Name: "test-9", Labels: map[string]string{"test_chat_id": "chat1"}}, 117 | }, 118 | }, 119 | } 120 | expNotChat2 := forward.Notification{ 121 | ChatID: "chat2", 122 | AlertGroup: model.AlertGroup{ID: "test-group-chat2", 123 | Alerts: []model.Alert{ 124 | {Name: "test-2", Labels: map[string]string{"test_chat_id": "chat2"}}, 125 | {Name: "test-4", Labels: map[string]string{"test_chat_id": "chat2"}}, 126 | }, 127 | }, 128 | } 129 | expNotChat3 := forward.Notification{ 130 | ChatID: "chat3", 131 | AlertGroup: model.AlertGroup{ID: "test-group-chat3", 132 | Alerts: []model.Alert{ 133 | {Name: "test-6", Labels: map[string]string{"test_chat_id": "chat3"}}, 134 | }, 135 | }, 136 | } 137 | for _, n := range ns { 138 | n.On("Notify", mock.Anything, expNotChatDef).Once().Return(nil) 139 | n.On("Notify", mock.Anything, expNotChat1).Once().Return(nil) 140 | n.On("Notify", mock.Anything, expNotChat2).Once().Return(nil) 141 | n.On("Notify", mock.Anything, expNotChat3).Once().Return(nil) 142 | } 143 | }, 144 | }, 145 | } 146 | 147 | for name, test := range tests { 148 | t.Run(name, func(t *testing.T) { 149 | assert := assert.New(t) 150 | require := require.New(t) 151 | 152 | mn1 := &forwardmock.Notifier{} 153 | mn2 := &forwardmock.Notifier{} 154 | test.mock([]*forwardmock.Notifier{mn1, mn2}) 155 | 156 | test.cfg.Notifiers = []forward.Notifier{mn1, mn2} 157 | svc, err := forward.NewService(test.cfg) 158 | require.NoError(err) 159 | 160 | err = svc.Forward(context.TODO(), test.props, test.alertGroup) 161 | 162 | if test.expErr != nil && assert.Error(err) { 163 | assert.True(errors.Is(err, test.expErr)) 164 | } else if assert.NoError(err) { 165 | mn1.AssertExpectations(t) 166 | mn2.AssertExpectations(t) 167 | } 168 | }) 169 | } 170 | } 171 | -------------------------------------------------------------------------------- /internal/forward/metrics.go: -------------------------------------------------------------------------------- 1 | package forward 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | "github.com/slok/alertgram/internal/model" 8 | ) 9 | 10 | // ServiceMetricsRecorder knows how to record metrics on forward.Service. 11 | type ServiceMetricsRecorder interface { 12 | ObserveForwardServiceOpDuration(ctx context.Context, op string, success bool, t time.Duration) 13 | } 14 | 15 | type measureService struct { 16 | rec ServiceMetricsRecorder 17 | next Service 18 | } 19 | 20 | // NewMeasureService wraps a service and measures using metrics. 21 | func NewMeasureService(rec ServiceMetricsRecorder, next Service) Service { 22 | return &measureService{ 23 | rec: rec, 24 | next: next, 25 | } 26 | } 27 | 28 | func (m measureService) Forward(ctx context.Context, props Properties, ag *model.AlertGroup) (err error) { 29 | defer func(t0 time.Time) { 30 | m.rec.ObserveForwardServiceOpDuration(ctx, "Forward", err == nil, time.Since(t0)) 31 | }(time.Now()) 32 | return m.next.Forward(ctx, props, ag) 33 | } 34 | 35 | // NotifierMetricsRecorder knows how to record metrics on forward.Notifier. 36 | type NotifierMetricsRecorder interface { 37 | ObserveForwardNotifierOpDuration(ctx context.Context, notifierType string, op string, success bool, t time.Duration) 38 | } 39 | 40 | type measureNotifier struct { 41 | notifierType string 42 | rec NotifierMetricsRecorder 43 | next Notifier 44 | } 45 | 46 | // NewMeasureNotifier wraps a notifier and measures using metrics. 47 | func NewMeasureNotifier(rec NotifierMetricsRecorder, next Notifier) Notifier { 48 | return &measureNotifier{ 49 | notifierType: next.Type(), 50 | rec: rec, 51 | next: next, 52 | } 53 | } 54 | 55 | func (m measureNotifier) Notify(ctx context.Context, n Notification) (err error) { 56 | defer func(t0 time.Time) { 57 | m.rec.ObserveForwardNotifierOpDuration(ctx, m.notifierType, "Notify", err == nil, time.Since(t0)) 58 | }(time.Now()) 59 | return m.next.Notify(ctx, n) 60 | } 61 | 62 | func (m measureNotifier) Type() string { 63 | defer func(t0 time.Time) { 64 | m.rec.ObserveForwardNotifierOpDuration(context.TODO(), m.notifierType, "Type", true, time.Since(t0)) 65 | }(time.Now()) 66 | return m.next.Type() 67 | } 68 | -------------------------------------------------------------------------------- /internal/forward/notify.go: -------------------------------------------------------------------------------- 1 | package forward 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/slok/alertgram/internal/model" 7 | ) 8 | 9 | // Notification is the notification that wants to be send 10 | // via a notifier. 11 | type Notification struct { 12 | // ChatID is an ID to send the notification. In 13 | // Telegram could be a channel/group ID, in Slack 14 | // a room or a user. 15 | ChatID string 16 | AlertGroup model.AlertGroup 17 | } 18 | 19 | // Notifier knows how to notify alerts to different backends. 20 | type Notifier interface { 21 | Notify(ctx context.Context, notification Notification) error 22 | Type() string 23 | } 24 | -------------------------------------------------------------------------------- /internal/http/alertmanager/alertmanager.go: -------------------------------------------------------------------------------- 1 | package alertmanager 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | 7 | "github.com/gin-gonic/gin" 8 | "github.com/slok/go-http-metrics/metrics" 9 | metricsmiddleware "github.com/slok/go-http-metrics/middleware" 10 | metricsmiddlewaregin "github.com/slok/go-http-metrics/middleware/gin" 11 | 12 | "github.com/slok/alertgram/internal/deadmansswitch" 13 | "github.com/slok/alertgram/internal/forward" 14 | "github.com/slok/alertgram/internal/log" 15 | ) 16 | 17 | // Config is the configuration of the WebhookHandler. 18 | type Config struct { 19 | MetricsRecorder metrics.Recorder 20 | WebhookPath string 21 | ChatIDQueryString string 22 | ForwardService forward.Service 23 | DeadMansSwitchPath string 24 | DeadMansSwitchService deadmansswitch.Service 25 | Debug bool 26 | Logger log.Logger 27 | } 28 | 29 | func (c *Config) defaults() error { 30 | if c.WebhookPath == "" { 31 | c.WebhookPath = "/alerts" 32 | } 33 | 34 | if c.ForwardService == nil { 35 | return fmt.Errorf("forward can't be nil") 36 | } 37 | 38 | if c.ChatIDQueryString == "" { 39 | c.ChatIDQueryString = "chat-id" 40 | } 41 | 42 | if c.DeadMansSwitchService == nil { 43 | c.DeadMansSwitchService = deadmansswitch.DisabledService 44 | } 45 | 46 | if c.DeadMansSwitchPath == "" { 47 | c.DeadMansSwitchPath = "/alerts/dms" 48 | } 49 | 50 | if c.Logger == nil { 51 | c.Logger = log.Dummy 52 | } 53 | 54 | return nil 55 | } 56 | 57 | // More info here: https://prometheus.io/docs/alerting/configuration/#webhook_config. 58 | type webhookHandler struct { 59 | cfg Config 60 | engine *gin.Engine 61 | forwarder forward.Service 62 | deadmansswitcher deadmansswitch.Service 63 | logger log.Logger 64 | } 65 | 66 | // NewHandler is an HTTP handler that knows how to handle 67 | // alertmanager webhook alerts. 68 | func NewHandler(cfg Config) (http.Handler, error) { 69 | err := cfg.defaults() 70 | if err != nil { 71 | return nil, err 72 | } 73 | 74 | if !cfg.Debug { 75 | gin.SetMode(gin.ReleaseMode) 76 | } 77 | 78 | w := webhookHandler{ 79 | cfg: cfg, 80 | engine: gin.New(), 81 | forwarder: cfg.ForwardService, 82 | deadmansswitcher: cfg.DeadMansSwitchService, 83 | logger: cfg.Logger, 84 | } 85 | 86 | // Metrics middleware. 87 | mdlw := metricsmiddleware.New(metricsmiddleware.Config{ 88 | Service: "alertmanager-api", 89 | Recorder: cfg.MetricsRecorder, 90 | }) 91 | w.engine.Use(metricsmiddlewaregin.Handler("", mdlw)) 92 | 93 | // Register routes. 94 | w.routes() 95 | 96 | return w.engine, nil 97 | } 98 | 99 | func (w webhookHandler) routes() { 100 | w.engine.POST(w.cfg.WebhookPath, w.HandleAlerts()) 101 | 102 | // Only enable dead man's switch if required. 103 | if w.deadmansswitcher != nil && w.deadmansswitcher != deadmansswitch.DisabledService { 104 | w.engine.POST(w.cfg.DeadMansSwitchPath, w.HandleDeadMansSwitch()) 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /internal/http/alertmanager/alertmanager_test.go: -------------------------------------------------------------------------------- 1 | package alertmanager_test 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "fmt" 7 | "net/http" 8 | "net/http/httptest" 9 | "strings" 10 | "testing" 11 | "time" 12 | 13 | "github.com/prometheus/alertmanager/notify/webhook" 14 | "github.com/prometheus/alertmanager/template" 15 | prommodel "github.com/prometheus/common/model" 16 | "github.com/stretchr/testify/assert" 17 | "github.com/stretchr/testify/mock" 18 | "github.com/stretchr/testify/require" 19 | 20 | "github.com/slok/alertgram/internal/forward" 21 | "github.com/slok/alertgram/internal/http/alertmanager" 22 | "github.com/slok/alertgram/internal/internalerrors" 23 | deadmansswitchmock "github.com/slok/alertgram/internal/mocks/deadmansswitch" 24 | forwardmock "github.com/slok/alertgram/internal/mocks/forward" 25 | "github.com/slok/alertgram/internal/model" 26 | ) 27 | 28 | var t0 = time.Now().UTC() 29 | 30 | func getBaseAlert() model.Alert { 31 | return model.Alert{ 32 | ID: "test-alert", 33 | Name: "test-alert-name", 34 | StartsAt: t0.Add(-10 * time.Minute), 35 | EndsAt: t0.Add(-3 * time.Minute), 36 | Status: model.AlertStatusFiring, 37 | Labels: map[string]string{prommodel.AlertNameLabel: "test-alert-name", "lK1": "lV1", "lK2": "lV2"}, 38 | Annotations: map[string]string{"aK1": "aV1", "aK2": "aV2"}, 39 | GeneratorURL: "http://test.com", 40 | } 41 | } 42 | 43 | func getBaseAlerts() *model.AlertGroup { 44 | al1 := getBaseAlert() 45 | al1.ID += "-1" 46 | al2 := getBaseAlert() 47 | al2.ID += "-2" 48 | al3 := getBaseAlert() 49 | al3.ID += "-3" 50 | al3.Status = model.AlertStatusResolved 51 | 52 | return &model.AlertGroup{ 53 | ID: "test-group", 54 | Labels: map[string]string{"glK1": "glV1", "glK2": "glV2"}, 55 | Alerts: []model.Alert{al1, al2, al3}, 56 | } 57 | } 58 | 59 | func getBaseAlertmanagerAlert() template.Alert { 60 | return template.Alert{ 61 | Fingerprint: "test-alert", 62 | Status: string(prommodel.AlertFiring), 63 | Labels: map[string]string{prommodel.AlertNameLabel: "test-alert-name", "lK1": "lV1", "lK2": "lV2"}, 64 | Annotations: map[string]string{"aK1": "aV1", "aK2": "aV2"}, 65 | StartsAt: t0.Add(-10 * time.Minute), 66 | EndsAt: t0.Add(-3 * time.Minute), 67 | GeneratorURL: "http://test.com", 68 | } 69 | } 70 | 71 | func getBaseAlertmanagerAlerts() webhook.Message { 72 | al1 := getBaseAlertmanagerAlert() 73 | al1.Fingerprint += "-1" 74 | al2 := getBaseAlertmanagerAlert() 75 | al2.Fingerprint += "-2" 76 | al3 := getBaseAlertmanagerAlert() 77 | al3.Fingerprint += "-3" 78 | al3.Status = string(prommodel.AlertResolved) 79 | 80 | return webhook.Message{ 81 | Data: &template.Data{ 82 | Receiver: "test-recv", 83 | Status: string(prommodel.AlertFiring), 84 | Alerts: template.Alerts{al1, al2, al3}, 85 | GroupLabels: map[string]string{"glK1": "glV1", "glK2": "glV2"}, 86 | CommonLabels: map[string]string{"gclK1": "gclV1", "gclK2": "gclV2"}, 87 | CommonAnnotations: map[string]string{"gcaK1": "gcaV1", "gcaK2": "gcaV2"}, 88 | ExternalURL: "http://test.com", 89 | }, 90 | Version: "4", 91 | GroupKey: "test-group", 92 | } 93 | } 94 | 95 | func TestHandleAlerts(t *testing.T) { 96 | tests := map[string]struct { 97 | config alertmanager.Config 98 | urlPath string 99 | webhookAlertJSON func(t *testing.T) string 100 | mock func(t *testing.T, msvc *forwardmock.Service) 101 | expCode int 102 | }{ 103 | "Alertmanager webhook alerts request should be handled correctly (with defaults).": { 104 | urlPath: "/alerts", 105 | webhookAlertJSON: func(t *testing.T) string { 106 | wa := getBaseAlertmanagerAlerts() 107 | body, err := json.Marshal(wa) 108 | require.NoError(t, err) 109 | return string(body) 110 | }, 111 | mock: func(t *testing.T, msvc *forwardmock.Service) { 112 | expAlerts := getBaseAlerts() 113 | expProps := forward.Properties{} 114 | msvc.On("Forward", mock.Anything, expProps, expAlerts).Once().Return(nil) 115 | }, 116 | expCode: http.StatusOK, 117 | }, 118 | 119 | "Alertmanager webhook alerts request should be handled correctly (with custom params).": { 120 | config: alertmanager.Config{ 121 | WebhookPath: "/test-alerts", 122 | ChatIDQueryString: "custom-telegram-chat-id", 123 | }, 124 | urlPath: "/test-alerts?custom-telegram-chat-id=-1009876543210", 125 | webhookAlertJSON: func(t *testing.T) string { 126 | wa := getBaseAlertmanagerAlerts() 127 | body, err := json.Marshal(wa) 128 | require.NoError(t, err) 129 | return string(body) 130 | }, 131 | mock: func(t *testing.T, msvc *forwardmock.Service) { 132 | expAlerts := getBaseAlerts() 133 | expProps := forward.Properties{ 134 | CustomChatID: "-1009876543210", 135 | } 136 | msvc.On("Forward", mock.Anything, expProps, expAlerts).Once().Return(nil) 137 | }, 138 | expCode: http.StatusOK, 139 | }, 140 | 141 | "Alertmanager webhook internal errors should be propagated to clients (forwarding).": { 142 | urlPath: "/alerts", 143 | webhookAlertJSON: func(t *testing.T) string { 144 | wa := getBaseAlertmanagerAlerts() 145 | body, err := json.Marshal(wa) 146 | require.NoError(t, err) 147 | return string(body) 148 | }, 149 | mock: func(t *testing.T, msvc *forwardmock.Service) { 150 | expAlerts := getBaseAlerts() 151 | expProps := forward.Properties{} 152 | msvc.On("Forward", mock.Anything, expProps, expAlerts).Once().Return(errors.New("whatever")) 153 | }, 154 | expCode: http.StatusInternalServerError, 155 | }, 156 | 157 | "Alertmanager webhook configuration errors should be propagated to clients (forwarding).": { 158 | urlPath: "/alerts", 159 | webhookAlertJSON: func(t *testing.T) string { 160 | wa := getBaseAlertmanagerAlerts() 161 | body, err := json.Marshal(wa) 162 | require.NoError(t, err) 163 | return string(body) 164 | }, 165 | mock: func(t *testing.T, msvc *forwardmock.Service) { 166 | expAlerts := getBaseAlerts() 167 | expProps := forward.Properties{} 168 | err := fmt.Errorf("custom error: %w", internalerrors.ErrInvalidConfiguration) 169 | msvc.On("Forward", mock.Anything, expProps, expAlerts).Once().Return(err) 170 | }, 171 | expCode: http.StatusBadRequest, 172 | }, 173 | 174 | "Alertmanager webhook configuration errors on notification should be propagated to clients (alert mapping).": { 175 | urlPath: "/alerts", 176 | webhookAlertJSON: func(t *testing.T) string { 177 | wa := getBaseAlertmanagerAlerts() 178 | wa.Version = "v3" 179 | body, err := json.Marshal(wa) 180 | require.NoError(t, err) 181 | return string(body) 182 | }, 183 | mock: func(t *testing.T, msvc *forwardmock.Service) {}, 184 | expCode: http.StatusBadRequest, 185 | }, 186 | 187 | "Alertmanager webhook configuration errors on notification should be propagated to clients (JSON formatting).": { 188 | urlPath: "/alerts", 189 | webhookAlertJSON: func(t *testing.T) string { 190 | return "{" 191 | }, 192 | mock: func(t *testing.T, msvc *forwardmock.Service) {}, 193 | expCode: http.StatusBadRequest, 194 | }, 195 | } 196 | 197 | for name, test := range tests { 198 | t.Run(name, func(t *testing.T) { 199 | assert := assert.New(t) 200 | require := require.New(t) 201 | 202 | // Mocks. 203 | msvc := &forwardmock.Service{} 204 | test.mock(t, msvc) 205 | 206 | // Execute. 207 | test.config.ForwardService = msvc 208 | h, _ := alertmanager.NewHandler(test.config) 209 | srv := httptest.NewServer(h) 210 | defer srv.Close() 211 | req, err := http.NewRequest(http.MethodPost, srv.URL+test.urlPath, strings.NewReader(test.webhookAlertJSON(t))) 212 | require.NoError(err) 213 | resp, err := http.DefaultClient.Do(req) 214 | require.NoError(err) 215 | 216 | // Check. 217 | assert.Equal(test.expCode, resp.StatusCode) 218 | msvc.AssertExpectations(t) 219 | }) 220 | } 221 | } 222 | 223 | func TestHandleDeadMansSwitch(t *testing.T) { 224 | tests := map[string]struct { 225 | config alertmanager.Config 226 | urlPath string 227 | webhookAlertJSON func(t *testing.T) string 228 | mock func(t *testing.T, msvc *deadmansswitchmock.Service) 229 | expCode int 230 | }{ 231 | 232 | "Dead man's switch request should be handled correctly (with defaults).": { 233 | urlPath: "/alerts/dms", 234 | webhookAlertJSON: func(t *testing.T) string { 235 | wa := getBaseAlertmanagerAlerts() 236 | body, err := json.Marshal(wa) 237 | require.NoError(t, err) 238 | return string(body) 239 | }, 240 | mock: func(t *testing.T, msvc *deadmansswitchmock.Service) { 241 | expAlerts := getBaseAlerts() 242 | msvc.On("PushSwitch", mock.Anything, expAlerts).Once().Return(nil) 243 | }, 244 | expCode: http.StatusOK, 245 | }, 246 | 247 | "Dead man's switch request should be handled correctly (with custom params).": { 248 | config: alertmanager.Config{ 249 | DeadMansSwitchPath: "/dead-mans-switch", 250 | }, 251 | urlPath: "/dead-mans-switch", 252 | webhookAlertJSON: func(t *testing.T) string { 253 | wa := getBaseAlertmanagerAlerts() 254 | body, err := json.Marshal(wa) 255 | require.NoError(t, err) 256 | return string(body) 257 | }, 258 | mock: func(t *testing.T, msvc *deadmansswitchmock.Service) { 259 | expAlerts := getBaseAlerts() 260 | msvc.On("PushSwitch", mock.Anything, expAlerts).Once().Return(nil) 261 | }, 262 | expCode: http.StatusOK, 263 | }, 264 | 265 | "Dead man's switch webhook internal errors should be propagated to clients (PushSwitch).": { 266 | urlPath: "/alerts/dms", 267 | webhookAlertJSON: func(t *testing.T) string { 268 | wa := getBaseAlertmanagerAlerts() 269 | body, err := json.Marshal(wa) 270 | require.NoError(t, err) 271 | return string(body) 272 | }, 273 | mock: func(t *testing.T, msvc *deadmansswitchmock.Service) { 274 | expAlerts := getBaseAlerts() 275 | msvc.On("PushSwitch", mock.Anything, expAlerts).Once().Return(errors.New("whatever")) 276 | }, 277 | expCode: http.StatusInternalServerError, 278 | }, 279 | 280 | "Dead man's switch webhook configuration errors should be propagated to clients (PushSwitch).": { 281 | urlPath: "/alerts/dms", 282 | webhookAlertJSON: func(t *testing.T) string { 283 | wa := getBaseAlertmanagerAlerts() 284 | body, err := json.Marshal(wa) 285 | require.NoError(t, err) 286 | return string(body) 287 | }, 288 | mock: func(t *testing.T, msvc *deadmansswitchmock.Service) { 289 | expAlerts := getBaseAlerts() 290 | err := fmt.Errorf("custom error: %w", internalerrors.ErrInvalidConfiguration) 291 | msvc.On("PushSwitch", mock.Anything, expAlerts).Once().Return(err) 292 | }, 293 | expCode: http.StatusBadRequest, 294 | }, 295 | 296 | "Dead man's switch webhook configuration errors on notification should be propagated to clients (alert mapping).": { 297 | urlPath: "/alerts/dms", 298 | webhookAlertJSON: func(t *testing.T) string { 299 | wa := getBaseAlertmanagerAlerts() 300 | wa.Version = "v3" 301 | body, err := json.Marshal(wa) 302 | require.NoError(t, err) 303 | return string(body) 304 | }, 305 | mock: func(t *testing.T, msvc *deadmansswitchmock.Service) {}, 306 | expCode: http.StatusBadRequest, 307 | }, 308 | 309 | "Dead man's switch configuration errors on notification should be propagated to clients (JSON formatting).": { 310 | urlPath: "/alerts/dms", 311 | webhookAlertJSON: func(t *testing.T) string { 312 | return "{" 313 | }, 314 | mock: func(t *testing.T, msvc *deadmansswitchmock.Service) {}, 315 | expCode: http.StatusBadRequest, 316 | }, 317 | } 318 | 319 | for name, test := range tests { 320 | t.Run(name, func(t *testing.T) { 321 | assert := assert.New(t) 322 | require := require.New(t) 323 | 324 | // Mocks. 325 | mfw := &forwardmock.Service{} 326 | mdms := &deadmansswitchmock.Service{} 327 | test.mock(t, mdms) 328 | 329 | // Execute. 330 | test.config.ForwardService = mfw 331 | test.config.DeadMansSwitchService = mdms 332 | h, err := alertmanager.NewHandler(test.config) 333 | require.NoError(err) 334 | srv := httptest.NewServer(h) 335 | defer srv.Close() 336 | req, err := http.NewRequest(http.MethodPost, srv.URL+test.urlPath, strings.NewReader(test.webhookAlertJSON(t))) 337 | require.NoError(err) 338 | resp, err := http.DefaultClient.Do(req) 339 | require.NoError(err) 340 | 341 | // Check. 342 | assert.Equal(test.expCode, resp.StatusCode) 343 | mfw.AssertExpectations(t) 344 | mdms.AssertExpectations(t) 345 | }) 346 | } 347 | } 348 | -------------------------------------------------------------------------------- /internal/http/alertmanager/handler.go: -------------------------------------------------------------------------------- 1 | package alertmanager 2 | 3 | import ( 4 | "errors" 5 | "net/http" 6 | 7 | "github.com/gin-gonic/gin" 8 | "github.com/slok/alertgram/internal/forward" 9 | "github.com/slok/alertgram/internal/internalerrors" 10 | ) 11 | 12 | func (w webhookHandler) HandleAlerts() gin.HandlerFunc { 13 | return func(ctx *gin.Context) { 14 | reqAlerts := alertGroupV4{} 15 | err := ctx.BindJSON(&reqAlerts) 16 | if err != nil { 17 | w.logger.Errorf("error unmarshalling JSON: %s", err) 18 | _ = ctx.AbortWithError(http.StatusBadRequest, err).SetType(gin.ErrorTypePublic) 19 | return 20 | } 21 | 22 | model, err := reqAlerts.toDomain() 23 | if err != nil { 24 | w.logger.Errorf("error mapping to domain models: %s", err) 25 | _ = ctx.AbortWithError(http.StatusBadRequest, err).SetType(gin.ErrorTypePublic) 26 | return 27 | } 28 | 29 | props := forward.Properties{ 30 | CustomChatID: ctx.Query(w.cfg.ChatIDQueryString), 31 | } 32 | err = w.forwarder.Forward(ctx.Request.Context(), props, model) 33 | if err != nil { 34 | w.logger.Errorf("error forwarding alert: %s", err) 35 | 36 | if errors.Is(err, internalerrors.ErrInvalidConfiguration) { 37 | _ = ctx.AbortWithError(http.StatusBadRequest, err).SetType(gin.ErrorTypePublic) 38 | return 39 | } 40 | 41 | _ = ctx.AbortWithError(http.StatusInternalServerError, err).SetType(gin.ErrorTypePublic) 42 | return 43 | } 44 | } 45 | } 46 | 47 | func (w webhookHandler) HandleDeadMansSwitch() gin.HandlerFunc { 48 | return func(ctx *gin.Context) { 49 | reqAlerts := alertGroupV4{} 50 | err := ctx.BindJSON(&reqAlerts) 51 | if err != nil { 52 | w.logger.Errorf("error unmarshalling JSON: %s", err) 53 | _ = ctx.AbortWithError(http.StatusBadRequest, err).SetType(gin.ErrorTypePublic) 54 | return 55 | } 56 | 57 | model, err := reqAlerts.toDomain() 58 | if err != nil { 59 | w.logger.Errorf("error mapping to domain models: %s", err) 60 | _ = ctx.AbortWithError(http.StatusBadRequest, err).SetType(gin.ErrorTypePublic) 61 | return 62 | } 63 | 64 | err = w.deadmansswitcher.PushSwitch(ctx.Request.Context(), model) 65 | if err != nil { 66 | w.logger.Errorf("error pushing dead mans switch push: %s", err) 67 | 68 | if errors.Is(err, internalerrors.ErrInvalidConfiguration) { 69 | _ = ctx.AbortWithError(http.StatusBadRequest, err).SetType(gin.ErrorTypePublic) 70 | return 71 | } 72 | 73 | _ = ctx.AbortWithError(http.StatusInternalServerError, err).SetType(gin.ErrorTypePublic) 74 | return 75 | } 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /internal/http/alertmanager/mapper.go: -------------------------------------------------------------------------------- 1 | package alertmanager 2 | 3 | import ( 4 | "errors" 5 | 6 | "github.com/prometheus/alertmanager/notify/webhook" 7 | prommodel "github.com/prometheus/common/model" 8 | 9 | "github.com/slok/alertgram/internal/model" 10 | ) 11 | 12 | var ( 13 | // ErrCantDeserialize will be used when for some reason the 14 | // received data can't be deserialized. 15 | ErrCantDeserialize = errors.New("can't deserialize the received alerts") 16 | ) 17 | 18 | // alertGroupV4 are the alertGroup received by the webhook 19 | // It uses the V4 version of the webhook format. 20 | // 21 | // https://github.com/prometheus/alertmanager/blob/5cb556e4b2247f2c5d8cebdef88e2a634a46863a/notify/webhook/webhook.go#L85 22 | type alertGroupV4 webhook.Message 23 | 24 | func (a alertGroupV4) toDomain() (*model.AlertGroup, error) { 25 | if a.Version != "4" { 26 | return nil, errors.New("not supported alert group version") 27 | } 28 | 29 | // Map alerts. 30 | alerts := make([]model.Alert, 0, len(a.Alerts)) 31 | for _, alert := range a.Alerts { 32 | modelAlert := model.Alert{ 33 | ID: alert.Fingerprint, 34 | Name: alert.Labels[prommodel.AlertNameLabel], 35 | StartsAt: alert.StartsAt, 36 | EndsAt: alert.EndsAt, 37 | Status: alertStatusToDomain(alert.Status), 38 | Labels: alert.Labels, 39 | Annotations: alert.Annotations, 40 | GeneratorURL: alert.GeneratorURL, 41 | } 42 | alerts = append(alerts, modelAlert) 43 | } 44 | 45 | ag := &model.AlertGroup{ 46 | ID: a.GroupKey, 47 | Labels: a.GroupLabels, 48 | Alerts: alerts, 49 | } 50 | 51 | return ag, nil 52 | } 53 | 54 | func alertStatusToDomain(st string) model.AlertStatus { 55 | switch prommodel.AlertStatus(st) { 56 | case prommodel.AlertFiring: 57 | return model.AlertStatusFiring 58 | case prommodel.AlertResolved: 59 | return model.AlertStatusResolved 60 | default: 61 | return model.AlertStatusUnknown 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /internal/http/server.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/http" 7 | "time" 8 | 9 | "github.com/slok/alertgram/internal/log" 10 | ) 11 | 12 | const ( 13 | drainTimeoutDef = 2 * time.Second 14 | listenAddressDef = ":8080" 15 | ) 16 | 17 | // Config is the server configuration. 18 | type Config struct { 19 | // ListenAddress is where the server will be listening. 20 | // By default will listen on :8080. 21 | ListenAddress string 22 | // DrainTimeout is the draining timeout, by default is 2 seconds. 23 | DrainTimeout time.Duration 24 | // Handler is the handler that will serve the server. 25 | Handler http.Handler 26 | // Logger is the logger used by the server. 27 | Logger log.Logger 28 | } 29 | 30 | func (c *Config) defaults() error { 31 | if c.Handler == nil { 32 | return fmt.Errorf("handler is required") 33 | } 34 | 35 | if c.Logger == nil { 36 | c.Logger = log.Dummy 37 | } 38 | 39 | if c.ListenAddress == "" { 40 | c.ListenAddress = listenAddressDef 41 | } 42 | 43 | if c.DrainTimeout == 0 { 44 | c.DrainTimeout = drainTimeoutDef 45 | } 46 | 47 | return nil 48 | } 49 | 50 | // Server is a Server that serves a handler. 51 | type Server struct { 52 | server *http.Server 53 | listenAddress string 54 | drainTimeout time.Duration 55 | logger log.Logger 56 | } 57 | 58 | // NewServer returns a new HTTP server. 59 | func NewServer(cfg Config) (*Server, error) { 60 | // fulfill with default configuration if needed. 61 | err := cfg.defaults() 62 | if err != nil { 63 | return nil, err 64 | } 65 | 66 | // Create the handler mux and the internal http server. 67 | httpServer := &http.Server{ 68 | Handler: cfg.Handler, 69 | Addr: cfg.ListenAddress, 70 | } 71 | 72 | // Create our HTTP Server. 73 | return &Server{ 74 | server: httpServer, 75 | listenAddress: cfg.ListenAddress, 76 | drainTimeout: cfg.DrainTimeout, 77 | logger: cfg.Logger.WithValues(log.KV{ 78 | "service": "http-server", 79 | "addr": cfg.ListenAddress, 80 | }), 81 | }, nil 82 | } 83 | 84 | // ListenAndServe runs the server. 85 | func (s *Server) ListenAndServe() error { 86 | s.logger.Infof("server listening on %s...", s.listenAddress) 87 | return s.server.ListenAndServe() 88 | } 89 | 90 | // DrainAndShutdown will drain the connections and shutdown the server. 91 | func (s *Server) DrainAndShutdown() error { 92 | s.logger.Infof("start draining connections...") 93 | 94 | ctx, cancel := context.WithTimeout(context.Background(), s.drainTimeout) 95 | defer cancel() 96 | 97 | err := s.server.Shutdown(ctx) 98 | if err != nil { 99 | return fmt.Errorf("something happened while draining connections: %w", err) 100 | } 101 | 102 | s.logger.Infof("connections drained") 103 | return nil 104 | } 105 | -------------------------------------------------------------------------------- /internal/internalerrors/errors.go: -------------------------------------------------------------------------------- 1 | package internalerrors 2 | 3 | import "errors" 4 | 5 | var ( 6 | // ErrInvalidConfiguration will be used when a configuration is invalid. 7 | ErrInvalidConfiguration = errors.New("configuration is invalid") 8 | ) 9 | -------------------------------------------------------------------------------- /internal/log/log.go: -------------------------------------------------------------------------------- 1 | package log 2 | 3 | // KV is a helper type to 4 | type KV map[string]interface{} 5 | 6 | // Logger knows how to log. 7 | type Logger interface { 8 | WithValues(d map[string]interface{}) Logger 9 | Debugf(format string, args ...interface{}) 10 | Infof(format string, args ...interface{}) 11 | Warningf(format string, args ...interface{}) 12 | Errorf(format string, args ...interface{}) 13 | } 14 | 15 | type dummy int 16 | 17 | // Dummy is a dummy logger. 18 | const Dummy = dummy(0) 19 | 20 | func (d dummy) WithValues(map[string]interface{}) Logger { return d } 21 | func (dummy) Infof(string, ...interface{}) {} 22 | func (dummy) Warningf(string, ...interface{}) {} 23 | func (dummy) Errorf(string, ...interface{}) {} 24 | func (dummy) Debugf(format string, args ...interface{}) {} 25 | -------------------------------------------------------------------------------- /internal/log/logrus/logrus.go: -------------------------------------------------------------------------------- 1 | package logrus 2 | 3 | import ( 4 | "github.com/sirupsen/logrus" 5 | 6 | "github.com/slok/alertgram/internal/log" 7 | ) 8 | 9 | type logger struct { 10 | *logrus.Entry 11 | } 12 | 13 | // New returns a new implementation of a logrus logger. 14 | // If not debug mode it will use JSON logging. 15 | func New(debug bool) log.Logger { 16 | l := logrus.New() 17 | if debug { 18 | l.SetLevel(logrus.DebugLevel) 19 | } else { 20 | l.SetFormatter(&logrus.JSONFormatter{}) 21 | } 22 | 23 | return &logger{Entry: logrus.NewEntry(l)} 24 | } 25 | 26 | func (l logger) WithValues(vals map[string]interface{}) log.Logger { 27 | return logger{l.Entry.WithFields(vals)} 28 | } 29 | -------------------------------------------------------------------------------- /internal/metrics/prometheus/prometheus.go: -------------------------------------------------------------------------------- 1 | package prometheus 2 | 3 | import ( 4 | "context" 5 | "strconv" 6 | "time" 7 | 8 | "github.com/prometheus/client_golang/prometheus" 9 | httpmetrics "github.com/slok/go-http-metrics/metrics" 10 | httpmetricsprometheus "github.com/slok/go-http-metrics/metrics/prometheus" 11 | 12 | "github.com/slok/alertgram/internal/deadmansswitch" 13 | "github.com/slok/alertgram/internal/forward" 14 | "github.com/slok/alertgram/internal/notify" 15 | ) 16 | 17 | const prefix = "alertgram" 18 | 19 | // Recorder knows how to measure the different metrics 20 | // interfaces of the application. 21 | type Recorder struct { 22 | httpmetrics.Recorder 23 | 24 | forwardServiceOpDurHistogram *prometheus.HistogramVec 25 | forwardNotifierOpDurHistogram *prometheus.HistogramVec 26 | templateRendererOpDurHistogram *prometheus.HistogramVec 27 | deadmansswitchServiceOpDurHistogram *prometheus.HistogramVec 28 | } 29 | 30 | // New returns a new Prometheus recorder for the app. 31 | func New(reg prometheus.Registerer) *Recorder { 32 | r := &Recorder{ 33 | Recorder: httpmetricsprometheus.NewRecorder(httpmetricsprometheus.Config{ 34 | Registry: reg, 35 | }), 36 | 37 | forwardServiceOpDurHistogram: prometheus.NewHistogramVec(prometheus.HistogramOpts{ 38 | Namespace: prefix, 39 | Subsystem: "forward", 40 | Name: "operation_duration_seconds", 41 | Help: "The duration of the operation in forward service.", 42 | Buckets: prometheus.DefBuckets, 43 | }, []string{"operation", "success"}), 44 | 45 | forwardNotifierOpDurHistogram: prometheus.NewHistogramVec(prometheus.HistogramOpts{ 46 | Namespace: prefix, 47 | Subsystem: "notifier", 48 | Name: "operation_duration_seconds", 49 | Help: "The duration of the operation in notifier.", 50 | Buckets: prometheus.DefBuckets, 51 | }, []string{"type", "operation", "success"}), 52 | 53 | templateRendererOpDurHistogram: prometheus.NewHistogramVec(prometheus.HistogramOpts{ 54 | Namespace: prefix, 55 | Subsystem: "template_renderer", 56 | Name: "operation_duration_seconds", 57 | Help: "The duration of the operation in template renderer.", 58 | Buckets: prometheus.DefBuckets, 59 | }, []string{"type", "operation", "success"}), 60 | 61 | deadmansswitchServiceOpDurHistogram: prometheus.NewHistogramVec(prometheus.HistogramOpts{ 62 | Namespace: prefix, 63 | Subsystem: "dead_mans_switch", 64 | Name: "operation_duration_seconds", 65 | Help: "The duration of the operation in dead man's switch service.", 66 | Buckets: []float64{.0001, .0005, .001, .005, .01, .025, .05, .1, .25, .5, 1}, 67 | }, []string{"operation", "success"}), 68 | } 69 | 70 | // Register all the metrics. 71 | reg.MustRegister( 72 | r.forwardServiceOpDurHistogram, 73 | r.forwardNotifierOpDurHistogram, 74 | r.templateRendererOpDurHistogram, 75 | r.deadmansswitchServiceOpDurHistogram, 76 | ) 77 | 78 | return r 79 | } 80 | 81 | // ObserveForwardNotifierOpDuration satifies forward.NotifierMetricsRecorder interface. 82 | func (r Recorder) ObserveForwardNotifierOpDuration(ctx context.Context, notType string, op string, success bool, t time.Duration) { 83 | r.forwardNotifierOpDurHistogram.WithLabelValues(notType, op, strconv.FormatBool(success)).Observe(t.Seconds()) 84 | } 85 | 86 | // ObserveForwardServiceOpDuration satisfies forward.ServiceMetricsRecorder interface. 87 | func (r Recorder) ObserveForwardServiceOpDuration(ctx context.Context, op string, success bool, t time.Duration) { 88 | r.forwardServiceOpDurHistogram.WithLabelValues(op, strconv.FormatBool(success)).Observe(t.Seconds()) 89 | } 90 | 91 | // ObserveTemplateRendererOpDuration satisfies notify.TemplateRendererMetricsRecorder interface. 92 | func (r Recorder) ObserveTemplateRendererOpDuration(ctx context.Context, rendererType string, op string, success bool, t time.Duration) { 93 | r.templateRendererOpDurHistogram.WithLabelValues(rendererType, op, strconv.FormatBool(success)).Observe(t.Seconds()) 94 | } 95 | 96 | // ObserveDMSServiceOpDuration satisfies deadmansswitch.ServiceMetricsRecorder interface. 97 | func (r Recorder) ObserveDMSServiceOpDuration(ctx context.Context, op string, success bool, t time.Duration) { 98 | r.deadmansswitchServiceOpDurHistogram.WithLabelValues(op, strconv.FormatBool(success)).Observe(t.Seconds()) 99 | } 100 | 101 | // Ensure that the recorder implements the different interfaces of the app. 102 | var _ forward.NotifierMetricsRecorder = &Recorder{} 103 | var _ forward.ServiceMetricsRecorder = &Recorder{} 104 | var _ deadmansswitch.ServiceMetricsRecorder = &Recorder{} 105 | var _ notify.TemplateRendererMetricsRecorder = &Recorder{} 106 | var _ httpmetrics.Recorder = &Recorder{} 107 | -------------------------------------------------------------------------------- /internal/mocks/deadmansswitch/service.go: -------------------------------------------------------------------------------- 1 | // Code generated by mockery v1.0.0. DO NOT EDIT. 2 | 3 | package mocks 4 | 5 | import ( 6 | context "context" 7 | 8 | mock "github.com/stretchr/testify/mock" 9 | 10 | model "github.com/slok/alertgram/internal/model" 11 | ) 12 | 13 | // Service is an autogenerated mock type for the Service type 14 | type Service struct { 15 | mock.Mock 16 | } 17 | 18 | // PushSwitch provides a mock function with given fields: ctx, alertGroup 19 | func (_m *Service) PushSwitch(ctx context.Context, alertGroup *model.AlertGroup) error { 20 | ret := _m.Called(ctx, alertGroup) 21 | 22 | var r0 error 23 | if rf, ok := ret.Get(0).(func(context.Context, *model.AlertGroup) error); ok { 24 | r0 = rf(ctx, alertGroup) 25 | } else { 26 | r0 = ret.Error(0) 27 | } 28 | 29 | return r0 30 | } 31 | -------------------------------------------------------------------------------- /internal/mocks/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Package mocks will have all the mocks of the app. 3 | */ 4 | package mocks 5 | 6 | //go:generate mockery -case underscore -output ./forward -dir ../forward -name Notifier 7 | //go:generate mockery -case underscore -output ./forward -dir ../forward -name Service 8 | //go:generate mockery -case underscore -output ./deadmansswitch -dir ../deadmansswitch -name Service 9 | 10 | //go:generate mockery -case underscore -output ./notify/telegram -dir ../notify/telegram -name Client 11 | //go:generate mockery -case underscore -output ./notify -dir ../notify -name TemplateRenderer 12 | -------------------------------------------------------------------------------- /internal/mocks/forward/notifier.go: -------------------------------------------------------------------------------- 1 | // Code generated by mockery v1.0.0. DO NOT EDIT. 2 | 3 | package mocks 4 | 5 | import ( 6 | context "context" 7 | 8 | forward "github.com/slok/alertgram/internal/forward" 9 | mock "github.com/stretchr/testify/mock" 10 | ) 11 | 12 | // Notifier is an autogenerated mock type for the Notifier type 13 | type Notifier struct { 14 | mock.Mock 15 | } 16 | 17 | // Notify provides a mock function with given fields: ctx, notification 18 | func (_m *Notifier) Notify(ctx context.Context, notification forward.Notification) error { 19 | ret := _m.Called(ctx, notification) 20 | 21 | var r0 error 22 | if rf, ok := ret.Get(0).(func(context.Context, forward.Notification) error); ok { 23 | r0 = rf(ctx, notification) 24 | } else { 25 | r0 = ret.Error(0) 26 | } 27 | 28 | return r0 29 | } 30 | 31 | // Type provides a mock function with given fields: 32 | func (_m *Notifier) Type() string { 33 | ret := _m.Called() 34 | 35 | var r0 string 36 | if rf, ok := ret.Get(0).(func() string); ok { 37 | r0 = rf() 38 | } else { 39 | r0 = ret.Get(0).(string) 40 | } 41 | 42 | return r0 43 | } 44 | -------------------------------------------------------------------------------- /internal/mocks/forward/service.go: -------------------------------------------------------------------------------- 1 | // Code generated by mockery v1.0.0. DO NOT EDIT. 2 | 3 | package mocks 4 | 5 | import ( 6 | context "context" 7 | 8 | forward "github.com/slok/alertgram/internal/forward" 9 | mock "github.com/stretchr/testify/mock" 10 | 11 | model "github.com/slok/alertgram/internal/model" 12 | ) 13 | 14 | // Service is an autogenerated mock type for the Service type 15 | type Service struct { 16 | mock.Mock 17 | } 18 | 19 | // Forward provides a mock function with given fields: ctx, props, alertGroup 20 | func (_m *Service) Forward(ctx context.Context, props forward.Properties, alertGroup *model.AlertGroup) error { 21 | ret := _m.Called(ctx, props, alertGroup) 22 | 23 | var r0 error 24 | if rf, ok := ret.Get(0).(func(context.Context, forward.Properties, *model.AlertGroup) error); ok { 25 | r0 = rf(ctx, props, alertGroup) 26 | } else { 27 | r0 = ret.Error(0) 28 | } 29 | 30 | return r0 31 | } 32 | -------------------------------------------------------------------------------- /internal/mocks/notify/telegram/client.go: -------------------------------------------------------------------------------- 1 | // Code generated by mockery v1.0.0. DO NOT EDIT. 2 | 3 | package mocks 4 | 5 | import ( 6 | mock "github.com/stretchr/testify/mock" 7 | 8 | tgbotapi "github.com/go-telegram-bot-api/telegram-bot-api" 9 | ) 10 | 11 | // Client is an autogenerated mock type for the Client type 12 | type Client struct { 13 | mock.Mock 14 | } 15 | 16 | // Send provides a mock function with given fields: c 17 | func (_m *Client) Send(c tgbotapi.Chattable) (tgbotapi.Message, error) { 18 | ret := _m.Called(c) 19 | 20 | var r0 tgbotapi.Message 21 | if rf, ok := ret.Get(0).(func(tgbotapi.Chattable) tgbotapi.Message); ok { 22 | r0 = rf(c) 23 | } else { 24 | r0 = ret.Get(0).(tgbotapi.Message) 25 | } 26 | 27 | var r1 error 28 | if rf, ok := ret.Get(1).(func(tgbotapi.Chattable) error); ok { 29 | r1 = rf(c) 30 | } else { 31 | r1 = ret.Error(1) 32 | } 33 | 34 | return r0, r1 35 | } 36 | -------------------------------------------------------------------------------- /internal/mocks/notify/template_renderer.go: -------------------------------------------------------------------------------- 1 | // Code generated by mockery v1.0.0. DO NOT EDIT. 2 | 3 | package mocks 4 | 5 | import ( 6 | context "context" 7 | 8 | model "github.com/slok/alertgram/internal/model" 9 | mock "github.com/stretchr/testify/mock" 10 | ) 11 | 12 | // TemplateRenderer is an autogenerated mock type for the TemplateRenderer type 13 | type TemplateRenderer struct { 14 | mock.Mock 15 | } 16 | 17 | // Render provides a mock function with given fields: ctx, ag 18 | func (_m *TemplateRenderer) Render(ctx context.Context, ag *model.AlertGroup) (string, error) { 19 | ret := _m.Called(ctx, ag) 20 | 21 | var r0 string 22 | if rf, ok := ret.Get(0).(func(context.Context, *model.AlertGroup) string); ok { 23 | r0 = rf(ctx, ag) 24 | } else { 25 | r0 = ret.Get(0).(string) 26 | } 27 | 28 | var r1 error 29 | if rf, ok := ret.Get(1).(func(context.Context, *model.AlertGroup) error); ok { 30 | r1 = rf(ctx, ag) 31 | } else { 32 | r1 = ret.Error(1) 33 | } 34 | 35 | return r0, r1 36 | } 37 | -------------------------------------------------------------------------------- /internal/model/model.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | // AlertStatus is the status of an alert. 8 | type AlertStatus int 9 | 10 | const ( 11 | // AlertStatusUnknown is unknown alert status. 12 | AlertStatusUnknown AlertStatus = iota 13 | // AlertStatusFiring is when the alert is active and firing. 14 | AlertStatusFiring 15 | // AlertStatusResolved is when the alert was being triggered and now 16 | // is not being triggered anymore. 17 | AlertStatusResolved 18 | ) 19 | 20 | // Alert represents an alert. 21 | type Alert struct { 22 | // ID is the ID of the alert. 23 | ID string 24 | // Name is the name of the alert. 25 | Name string 26 | // StartsAt is when the alert has been started. 27 | StartsAt time.Time 28 | // End is when the alert has been ended. 29 | EndsAt time.Time 30 | // Status is the status of the alert. 31 | Status AlertStatus 32 | // Labels is data that defines the alert. 33 | Labels map[string]string 34 | // Annotations is a simple map of values that can be used to 35 | // add more info to the alert but don't define the alert nature 36 | // commonly this is used to add description, titles... 37 | Annotations map[string]string 38 | // GeneratorURL is the url that generated the alert (eg. Prometheus metrics). 39 | GeneratorURL string 40 | } 41 | 42 | // IsFiring returns if the alerts is firing. 43 | func (a Alert) IsFiring() bool { return a.Status == AlertStatusFiring } 44 | 45 | // AlertGroup is a group of alerts that share some of 46 | // the information like the state, common metadata... 47 | // and can be grouped in order to notify at the same 48 | // time. 49 | type AlertGroup struct { 50 | // ID is the group of alerts ID. 51 | ID string 52 | // Labels are the labels of the group. 53 | Labels map[string]string 54 | // Alerts are all the alerts in the group (firing, resolved, unknown...). 55 | Alerts []Alert 56 | } 57 | 58 | // FiringAlerts returns the firing alerts. 59 | func (a AlertGroup) FiringAlerts() []Alert { return a.groupByStatusAlerts(AlertStatusFiring) } 60 | 61 | // ResolvedAlerts returns the resolved alerts. 62 | func (a AlertGroup) ResolvedAlerts() []Alert { return a.groupByStatusAlerts(AlertStatusResolved) } 63 | 64 | // HasFiring returns true if it has firing alerts. 65 | func (a AlertGroup) HasFiring() bool { return a.hasAlertByStatus(AlertStatusFiring) } 66 | 67 | // HasResolved returns true if it has resolved alerts. 68 | func (a AlertGroup) HasResolved() bool { return a.hasAlertByStatus(AlertStatusResolved) } 69 | 70 | func (a AlertGroup) hasAlertByStatus(status AlertStatus) bool { 71 | for _, al := range a.Alerts { 72 | if al.Status == status { 73 | return true 74 | } 75 | } 76 | 77 | return false 78 | } 79 | 80 | func (a AlertGroup) groupByStatusAlerts(status AlertStatus) []Alert { 81 | var alerts []Alert 82 | for _, al := range a.Alerts { 83 | if al.Status == status { 84 | alerts = append(alerts, al) 85 | } 86 | } 87 | 88 | return alerts 89 | } 90 | -------------------------------------------------------------------------------- /internal/notify/metrics.go: -------------------------------------------------------------------------------- 1 | package notify 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | "github.com/slok/alertgram/internal/model" 8 | ) 9 | 10 | // TemplateRendererMetricsRecorder knows how to record the metrics in the TemplateRenderer. 11 | type TemplateRendererMetricsRecorder interface { 12 | ObserveTemplateRendererOpDuration(ctx context.Context, rendererType string, op string, success bool, t time.Duration) 13 | } 14 | 15 | type measureTemplateRenderer struct { 16 | rendererType string 17 | rec TemplateRendererMetricsRecorder 18 | next TemplateRenderer 19 | } 20 | 21 | // NewMeasureTemplateRenderer wraps a template renderer and measures using metrics. 22 | func NewMeasureTemplateRenderer(rendererType string, rec TemplateRendererMetricsRecorder, next TemplateRenderer) TemplateRenderer { 23 | return &measureTemplateRenderer{ 24 | rendererType: rendererType, 25 | rec: rec, 26 | next: next, 27 | } 28 | } 29 | 30 | func (m measureTemplateRenderer) Render(ctx context.Context, ag *model.AlertGroup) (_ string, err error) { 31 | defer func(t0 time.Time) { 32 | m.rec.ObserveTemplateRendererOpDuration(ctx, m.rendererType, "Render", err == nil, time.Since(t0)) 33 | }(time.Now()) 34 | return m.next.Render(ctx, ag) 35 | } 36 | -------------------------------------------------------------------------------- /internal/notify/notify.go: -------------------------------------------------------------------------------- 1 | package notify 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/slok/alertgram/internal/forward" 7 | "github.com/slok/alertgram/internal/log" 8 | ) 9 | 10 | type dummy int 11 | 12 | // Dummy is a dummy notifier. 13 | const Dummy = dummy(0) 14 | 15 | func (dummy) Notify(ctx context.Context, notification forward.Notification) error { return nil } 16 | func (dummy) Type() string { return "dummy" } 17 | 18 | type logger struct { 19 | renderer TemplateRenderer 20 | logger log.Logger 21 | } 22 | 23 | // NewLogger returns a notifier that only logs the renderer alerts, 24 | // normally used to develop or dry/run. 25 | func NewLogger(r TemplateRenderer, l log.Logger) forward.Notifier { 26 | return &logger{ 27 | renderer: r, 28 | logger: l.WithValues(log.KV{"notifier": "logger"}), 29 | } 30 | } 31 | 32 | func (l logger) Notify(ctx context.Context, n forward.Notification) error { 33 | logger := l.logger.WithValues(log.KV{"chatID": n.ChatID, "alertGroup": n.AlertGroup.ID, "alertsNumber": len(n.AlertGroup.Alerts)}) 34 | 35 | alertText, err := l.renderer.Render(ctx, &n.AlertGroup) 36 | if err != nil { 37 | return err 38 | } 39 | logger.Infof("alert: %s", alertText) 40 | 41 | return nil 42 | } 43 | func (logger) Type() string { return "logger" } 44 | -------------------------------------------------------------------------------- /internal/notify/telegram/telegram.go: -------------------------------------------------------------------------------- 1 | package telegram 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "strconv" 8 | 9 | tgbotapi "github.com/go-telegram-bot-api/telegram-bot-api" 10 | 11 | "github.com/slok/alertgram/internal/forward" 12 | "github.com/slok/alertgram/internal/internalerrors" 13 | "github.com/slok/alertgram/internal/log" 14 | "github.com/slok/alertgram/internal/notify" 15 | ) 16 | 17 | var ( 18 | // ErrComm will be used when the communication to telegram fails. 19 | ErrComm = errors.New("error communicating with telegram") 20 | ) 21 | 22 | // Config is the configuration of the Notifier. 23 | type Config struct { 24 | // This is the ID of the channel or group where the alerts 25 | // will be sent by default. 26 | // Got from here https://github.com/GabrielRF/telegram-id#web-channel-id 27 | // You ca get the the ID like this: 28 | // - Enter the telegram web app and there to the channel/group. 29 | // - Check the URL, it has this schema: https://web.telegram.org/#/im?p=c1234567891_12345678912345678912 30 | // - Get the `c1234567891_`, get this part: `1234567891`. 31 | // - Add `-100` (until you have 13 characters), this should be the chat ID: `-1001234567891` 32 | DefaultTelegramChatID int64 33 | // TemplateRenderer is the renderer that will be used to render the 34 | // notifications before sending to Telegram. 35 | TemplateRenderer notify.TemplateRenderer 36 | // Client is the telegram client is compatible with "github.com/go-telegram-bot-api/telegram-bot-api" 37 | // library client API. 38 | Client Client 39 | // Logger is the logger. 40 | Logger log.Logger 41 | } 42 | 43 | func (c *Config) defaults() error { 44 | if c.Client == nil { 45 | return fmt.Errorf("telegram client is required") 46 | } 47 | 48 | if c.TemplateRenderer == nil { 49 | c.TemplateRenderer = notify.DefaultTemplateRenderer 50 | } 51 | 52 | if c.Logger == nil { 53 | c.Logger = log.Dummy 54 | } 55 | 56 | return nil 57 | } 58 | 59 | type notifier struct { 60 | tplRenderer notify.TemplateRenderer 61 | cfg Config 62 | client Client 63 | logger log.Logger 64 | } 65 | 66 | // NewNotifier returns a notifier is a Telegram notifier 67 | // that knows how to send alerts to telegram. 68 | func NewNotifier(cfg Config) (forward.Notifier, error) { 69 | err := cfg.defaults() 70 | if err != nil { 71 | return nil, fmt.Errorf("%s: %w", err, internalerrors.ErrInvalidConfiguration) 72 | } 73 | 74 | return ¬ifier{ 75 | cfg: cfg, 76 | tplRenderer: cfg.TemplateRenderer, 77 | client: cfg.Client, 78 | logger: cfg.Logger.WithValues(log.KV{"notifier": "telegram"}), 79 | }, nil 80 | } 81 | 82 | func (n notifier) Notify(ctx context.Context, notification forward.Notification) error { 83 | ag := notification.AlertGroup 84 | 85 | logger := n.logger.WithValues(log.KV{"alertGroup": ag.ID, "alertsNumber": len(ag.Alerts)}) 86 | select { 87 | case <-ctx.Done(): 88 | logger.Infof("context cancelled, not notifying alerts") 89 | return nil 90 | default: 91 | } 92 | 93 | msg, err := n.createMessage(ctx, notification) 94 | if err != nil { 95 | return fmt.Errorf("could not format the alerts to message: %w", err) 96 | } 97 | logger = n.logger.WithValues(log.KV{"telegramChatID": msg.ChatID}) 98 | 99 | res, err := n.client.Send(msg) 100 | if err != nil { 101 | err = fmt.Errorf("%w: %s", ErrComm, err) 102 | return fmt.Errorf("error sending telegram message: %w", err) 103 | } 104 | logger.Infof("telegram message sent") 105 | logger.Debugf("telegram response: %+v", res) 106 | 107 | return nil 108 | } 109 | 110 | func (n notifier) getChatID(notification forward.Notification) (int64, error) { 111 | if notification.ChatID == "" { 112 | return n.cfg.DefaultTelegramChatID, nil 113 | } 114 | 115 | chatID, err := strconv.ParseInt(notification.ChatID, 10, 64) 116 | if err != nil { 117 | return 0, fmt.Errorf("%w: %s", internalerrors.ErrInvalidConfiguration, err) 118 | } 119 | 120 | return chatID, nil 121 | } 122 | 123 | func (n notifier) createMessage(ctx context.Context, notification forward.Notification) (tgbotapi.MessageConfig, error) { 124 | chatID, err := n.getChatID(notification) 125 | if err != nil { 126 | return tgbotapi.MessageConfig{}, fmt.Errorf("could not get a valid telegran chat ID: %w", err) 127 | } 128 | 129 | data, err := n.tplRenderer.Render(ctx, ¬ification.AlertGroup) 130 | if err != nil { 131 | return tgbotapi.MessageConfig{}, fmt.Errorf("error rendering alerts to template: %w", err) 132 | } 133 | 134 | msg := tgbotapi.NewMessage(chatID, data) 135 | msg.ParseMode = "HTML" 136 | msg.DisableWebPagePreview = true // TODO(slok): Make it configurable? 137 | 138 | return msg, nil 139 | } 140 | 141 | func (n notifier) Type() string { return "telegram" } 142 | 143 | // Client is an small abstraction for the telegram-bot-api client. 144 | // the client of this lib should satisfy directly. 145 | // More info here: https://godoc.org/github.com/go-telegram-bot-api/telegram-bot-api. 146 | type Client interface { 147 | Send(c tgbotapi.Chattable) (tgbotapi.Message, error) 148 | } 149 | -------------------------------------------------------------------------------- /internal/notify/telegram/telegram_test.go: -------------------------------------------------------------------------------- 1 | package telegram_test 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "testing" 7 | 8 | tgbotapi "github.com/go-telegram-bot-api/telegram-bot-api" 9 | "github.com/stretchr/testify/assert" 10 | "github.com/stretchr/testify/mock" 11 | "github.com/stretchr/testify/require" 12 | 13 | "github.com/slok/alertgram/internal/forward" 14 | "github.com/slok/alertgram/internal/internalerrors" 15 | notifymock "github.com/slok/alertgram/internal/mocks/notify" 16 | telegrammock "github.com/slok/alertgram/internal/mocks/notify/telegram" 17 | "github.com/slok/alertgram/internal/model" 18 | "github.com/slok/alertgram/internal/notify/telegram" 19 | ) 20 | 21 | func GetBaseAlertGroup() model.AlertGroup { 22 | return model.AlertGroup{ 23 | ID: "test-alert", 24 | Alerts: []model.Alert{ 25 | { 26 | Labels: map[string]string{ 27 | "alertname": "ServicePodIsRestarting", 28 | }, 29 | Annotations: map[string]string{ 30 | "message": "There has been restarting more than 5 times over 20 minutes", 31 | }, 32 | }, 33 | { 34 | Labels: map[string]string{ 35 | "alertname": "ServicePodIsRestarting", 36 | "chatid": "-1001234567890", 37 | }, 38 | Annotations: map[string]string{ 39 | "message": "There has been restarting more than 5 times over 20 minutes", 40 | "graph": "https://prometheus.test/my-graph", 41 | }, 42 | }, 43 | }, 44 | } 45 | } 46 | 47 | var errTest = errors.New("whatever") 48 | 49 | func TestNotify(t *testing.T) { 50 | tests := map[string]struct { 51 | cfg telegram.Config 52 | mocks func(t *testing.T, mcli *telegrammock.Client, mr *notifymock.TemplateRenderer) 53 | notification forward.Notification 54 | expErr error 55 | }{ 56 | "A alertGroup should be rendered and send the message to telegram.": { 57 | cfg: telegram.Config{ 58 | DefaultTelegramChatID: 1234, 59 | }, 60 | mocks: func(t *testing.T, mcli *telegrammock.Client, mr *notifymock.TemplateRenderer) { 61 | expMsgData := "rendered template" 62 | expAlertGroup := GetBaseAlertGroup() 63 | mr.On("Render", mock.Anything, &expAlertGroup).Once().Return(expMsgData, nil) 64 | 65 | expMsg := tgbotapi.MessageConfig{ 66 | BaseChat: tgbotapi.BaseChat{ChatID: 1234}, 67 | ParseMode: "HTML", 68 | DisableWebPagePreview: true, 69 | Text: expMsgData, 70 | } 71 | mcli.On("Send", expMsg).Once().Return(tgbotapi.Message{}, nil) 72 | }, 73 | notification: forward.Notification{ 74 | AlertGroup: GetBaseAlertGroup(), 75 | }, 76 | }, 77 | 78 | "If using a custom chat ID based on notificaiton it should send to that chat.": { 79 | cfg: telegram.Config{ 80 | DefaultTelegramChatID: 1234, 81 | }, 82 | mocks: func(t *testing.T, mcli *telegrammock.Client, mr *notifymock.TemplateRenderer) { 83 | expMsgData := "rendered template" 84 | expAlertGroup := GetBaseAlertGroup() 85 | mr.On("Render", mock.Anything, &expAlertGroup).Once().Return(expMsgData, nil) 86 | 87 | expMsg := tgbotapi.MessageConfig{ 88 | BaseChat: tgbotapi.BaseChat{ChatID: -1009876543210}, 89 | ParseMode: "HTML", 90 | DisableWebPagePreview: true, 91 | Text: expMsgData, 92 | } 93 | mcli.On("Send", expMsg).Once().Return(tgbotapi.Message{}, nil) 94 | }, 95 | notification: forward.Notification{ 96 | ChatID: "-1009876543210", 97 | AlertGroup: GetBaseAlertGroup(), 98 | }, 99 | }, 100 | 101 | "A error in the template rendering process should be processed.": { 102 | cfg: telegram.Config{ 103 | DefaultTelegramChatID: 1234, 104 | }, 105 | mocks: func(t *testing.T, mcli *telegrammock.Client, mr *notifymock.TemplateRenderer) { 106 | expAlertGroup := GetBaseAlertGroup() 107 | mr.On("Render", mock.Anything, &expAlertGroup).Once().Return("", errTest) 108 | }, 109 | notification: forward.Notification{ 110 | AlertGroup: GetBaseAlertGroup(), 111 | }, 112 | expErr: errTest, 113 | }, 114 | 115 | "A error with an invalid custom Chat ID should be propagated.": { 116 | cfg: telegram.Config{ 117 | DefaultTelegramChatID: 1234, 118 | }, 119 | mocks: func(t *testing.T, mcli *telegrammock.Client, mr *notifymock.TemplateRenderer) {}, 120 | notification: forward.Notification{ 121 | ChatID: "notAnInt64", 122 | AlertGroup: GetBaseAlertGroup(), 123 | }, 124 | expErr: internalerrors.ErrInvalidConfiguration, 125 | }, 126 | 127 | "A error in the notification send process should be processed with communication error.": { 128 | cfg: telegram.Config{ 129 | DefaultTelegramChatID: 1234, 130 | }, 131 | mocks: func(t *testing.T, mcli *telegrammock.Client, mr *notifymock.TemplateRenderer) { 132 | expMsgData := "rendered template" 133 | expAlertGroup := GetBaseAlertGroup() 134 | mr.On("Render", mock.Anything, &expAlertGroup).Once().Return(expMsgData, nil) 135 | 136 | expMsg := tgbotapi.MessageConfig{ 137 | BaseChat: tgbotapi.BaseChat{ChatID: 1234}, 138 | ParseMode: "HTML", 139 | DisableWebPagePreview: true, 140 | Text: expMsgData, 141 | } 142 | mcli.On("Send", expMsg).Once().Return(tgbotapi.Message{}, errTest) 143 | }, 144 | notification: forward.Notification{ 145 | AlertGroup: GetBaseAlertGroup(), 146 | }, 147 | expErr: telegram.ErrComm, 148 | }, 149 | } 150 | 151 | for name, test := range tests { 152 | t.Run(name, func(t *testing.T) { 153 | assert := assert.New(t) 154 | require := require.New(t) 155 | 156 | // Mocks. 157 | mcli := &telegrammock.Client{} 158 | mr := ¬ifymock.TemplateRenderer{} 159 | test.mocks(t, mcli, mr) 160 | test.cfg.Client = mcli 161 | test.cfg.TemplateRenderer = mr 162 | 163 | // Execute. 164 | n, err := telegram.NewNotifier(test.cfg) 165 | require.NoError(err) 166 | err = n.Notify(context.TODO(), test.notification) 167 | 168 | // Check. 169 | if test.expErr != nil && assert.Error(err) { 170 | assert.True(errors.Is(err, test.expErr)) 171 | } else if assert.NoError(err) { 172 | mcli.AssertExpectations(t) 173 | mr.AssertExpectations(t) 174 | } 175 | }) 176 | } 177 | } 178 | -------------------------------------------------------------------------------- /internal/notify/template.go: -------------------------------------------------------------------------------- 1 | package notify 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "errors" 7 | "fmt" 8 | "html/template" 9 | 10 | "github.com/Masterminds/sprig/v3" 11 | "github.com/slok/alertgram/internal/model" 12 | ) 13 | 14 | // ErrRenderTemplate will be used when there is an error rendering the alerts 15 | // to a template. 16 | var ErrRenderTemplate = errors.New("error rendering template") 17 | 18 | // TemplateRenderer knows how to render an alertgroup to get the 19 | // final notification message. 20 | type TemplateRenderer interface { 21 | Render(ctx context.Context, ag *model.AlertGroup) (string, error) 22 | } 23 | 24 | // TemplateRendererFunc is a helper function to use funcs as TemplateRenderer types. 25 | type TemplateRendererFunc func(ctx context.Context, ag *model.AlertGroup) (string, error) 26 | 27 | // Render satisfies TemplateRenderer interface. 28 | func (t TemplateRendererFunc) Render(ctx context.Context, ag *model.AlertGroup) (string, error) { 29 | return t(ctx, ag) 30 | } 31 | 32 | // NewHTMLTemplateRenderer returns a new template renderer using the go HTML 33 | // template renderer. 34 | // The templates use https://github.com/Masterminds/sprig to render. 35 | func NewHTMLTemplateRenderer(tpl string) (TemplateRenderer, error) { 36 | t, err := template.New("tpl").Funcs(sprig.FuncMap()).Parse(tpl) 37 | if err != nil { 38 | return nil, fmt.Errorf("error rendering template: %w", err) 39 | } 40 | 41 | return TemplateRendererFunc(func(_ context.Context, ag *model.AlertGroup) (string, error) { 42 | return renderAlertGroup(ag, t) 43 | }), nil 44 | } 45 | 46 | // renderAlertGroup takes an alertGroup and renders on the given template. 47 | func renderAlertGroup(ag *model.AlertGroup, t *template.Template) (string, error) { 48 | var b bytes.Buffer 49 | err := t.Execute(&b, ag) 50 | if err != nil { 51 | return "", fmt.Errorf("%w: %s", ErrRenderTemplate, err) 52 | } 53 | 54 | return b.String(), nil 55 | } 56 | 57 | type defRenderer int 58 | 59 | // DefaultTemplateRenderer is the default renderer that will render the 60 | // alerts using a premade HTML template. 61 | const DefaultTemplateRenderer = defRenderer(0) 62 | 63 | func (defRenderer) Render(_ context.Context, ag *model.AlertGroup) (string, error) { 64 | return renderAlertGroup(ag, defTemplate) 65 | } 66 | 67 | var defTemplate = template.Must(template.New("def").Funcs(sprig.FuncMap()).Parse(` 68 | {{- if .HasFiring }} 69 | 🚨🚨 FIRING ALERTS 🚨🚨 70 | {{- range .FiringAlerts }} 71 | 72 | 💥💥💥 {{ .Labels.alertname }} 💥💥💥 73 | {{ .Annotations.message }} 74 | {{- range $key, $value := .Labels }} 75 | {{- if ne $key "alertname" }} 76 | {{- if hasPrefix "http" $value }} 77 | 🔹 {{ $key }} 78 | {{- else }} 79 | 🔹 {{ $key }}: {{ $value }} 80 | {{- end}} 81 | {{- end }} 82 | {{- end}} 83 | {{- range $key, $value := .Annotations }} 84 | {{- if ne $key "message" }} 85 | {{- if hasPrefix "http" $value }} 86 | 🔸 {{ $key }} 87 | {{- else }} 88 | 🔸 {{ $key }}: {{ $value }} 89 | {{- end}} 90 | {{- end}} 91 | {{- end}} 92 | {{- end }} 93 | {{- end }} 94 | {{- if .HasResolved }} 95 | 96 | ✅✅ RESOLVED ALERTS ✅✅ 97 | {{- range .ResolvedAlerts }} 98 | 99 | 🟢🟢🟢 {{ .Labels.alertname }} 🟢🟢🟢 100 | {{ .Annotations.message }} 101 | {{- range $key, $value := .Labels }} 102 | {{- if ne $key "alertname" }} 103 | {{- if hasPrefix "http" $value }} 104 | 🔹 {{ $key }} 105 | {{- else }} 106 | 🔹 {{ $key }}: {{ $value }} 107 | {{- end}} 108 | {{- end }} 109 | {{- end}} 110 | {{- range $key, $value := .Annotations }} 111 | {{- if ne $key "message" }} 112 | {{- if hasPrefix "http" $value }} 113 | 🔸 {{ $key }} 114 | {{- else }} 115 | 🔸 {{ $key }}: {{ $value }} 116 | {{- end}} 117 | {{- end}} 118 | {{- end}} 119 | {{- end }} 120 | {{- end }} 121 | `)) 122 | -------------------------------------------------------------------------------- /internal/notify/template_test.go: -------------------------------------------------------------------------------- 1 | package notify_test 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | 10 | "github.com/slok/alertgram/internal/model" 11 | "github.com/slok/alertgram/internal/notify" 12 | ) 13 | 14 | func TestTemplateRenderer(t *testing.T) { 15 | tests := map[string]struct { 16 | alertGroup func() *model.AlertGroup 17 | renderer func() notify.TemplateRenderer 18 | expData string 19 | expErr error 20 | }{ 21 | "Invalid template should return an error.": { 22 | alertGroup: func() *model.AlertGroup { return nil }, 23 | renderer: func() notify.TemplateRenderer { 24 | r, _ := notify.NewHTMLTemplateRenderer("{{ .ID }}") 25 | return r 26 | }, 27 | expErr: notify.ErrRenderTemplate, 28 | }, 29 | 30 | "Custom template should render the alerts correctly.": { 31 | alertGroup: func() *model.AlertGroup { 32 | return &model.AlertGroup{ 33 | ID: "test-alert", 34 | Alerts: []model.Alert{{}, {}, {}}, 35 | } 36 | }, 37 | renderer: func() notify.TemplateRenderer { 38 | r, _ := notify.NewHTMLTemplateRenderer("{{ .ID }} has {{ .Alerts | len }} alerts.") 39 | return r 40 | }, 41 | expData: "test-alert has 3 alerts.", 42 | }, 43 | 44 | "Default template should render the alerts correctly.": { 45 | alertGroup: func() *model.AlertGroup { 46 | al1 := model.Alert{ 47 | Status: model.AlertStatusFiring, 48 | Labels: map[string]string{ 49 | "alertname": "ServicePodIsRestarting", 50 | "chatid": "-1001234567890", 51 | "job": "kubernetes-metrics", 52 | "owner": "team1", 53 | "pod": "ns1/pod-service1-f76c976c4-9hlgv", 54 | "severity": "telegram", 55 | }, 56 | Annotations: map[string]string{ 57 | "message": "There has been restarting more than 5 times over 20 minutes", 58 | "graph": "https://prometheus.test/my-graph", 59 | "runbook": "https://github.test/runbooks/pod-restarting.md", 60 | }, 61 | } 62 | al2 := model.Alert{ 63 | Status: model.AlertStatusFiring, 64 | Labels: map[string]string{ 65 | "alertname": "ServicePodIsRestarting", 66 | "chatid": "-1001234567890", 67 | "job": "kubernetes-metrics", 68 | "owner": "team1", 69 | "pod": "ns1/pod-service64-f5c7dd9cfc5-8scht", 70 | "severity": "telegram", 71 | }, 72 | Annotations: map[string]string{ 73 | "message": "There has been restarting more than 5 times over 20 minutes", 74 | "graph": "https://prometheus.test/my-graph", 75 | "runbook": "https://github.test/runbooks/pod-restarting.md", 76 | }, 77 | } 78 | al3 := model.Alert{ 79 | Status: model.AlertStatusResolved, 80 | Labels: map[string]string{ 81 | "alertname": "ServicePodIsRestarting", 82 | }, 83 | Annotations: map[string]string{ 84 | "message": "There has been restarting more than 5 times over 20 minutes", 85 | }, 86 | } 87 | return &model.AlertGroup{ 88 | ID: "test-alert", 89 | Alerts: []model.Alert{al1, al2, al3}, 90 | } 91 | }, 92 | expData: ` 93 | 🚨🚨 FIRING ALERTS 🚨🚨 94 | 95 | 💥💥💥 ServicePodIsRestarting 💥💥💥 96 | There has been restarting more than 5 times over 20 minutes 97 | 🔹 chatid: -1001234567890 98 | 🔹 job: kubernetes-metrics 99 | 🔹 owner: team1 100 | 🔹 pod: ns1/pod-service1-f76c976c4-9hlgv 101 | 🔹 severity: telegram 102 | 🔸 graph 103 | 🔸 runbook 104 | 105 | 💥💥💥 ServicePodIsRestarting 💥💥💥 106 | There has been restarting more than 5 times over 20 minutes 107 | 🔹 chatid: -1001234567890 108 | 🔹 job: kubernetes-metrics 109 | 🔹 owner: team1 110 | 🔹 pod: ns1/pod-service64-f5c7dd9cfc5-8scht 111 | 🔹 severity: telegram 112 | 🔸 graph 113 | 🔸 runbook 114 | 115 | ✅✅ RESOLVED ALERTS ✅✅ 116 | 117 | 🟢🟢🟢 ServicePodIsRestarting 🟢🟢🟢 118 | There has been restarting more than 5 times over 20 minutes 119 | `, 120 | renderer: func() notify.TemplateRenderer { return notify.DefaultTemplateRenderer }, 121 | }, 122 | } 123 | 124 | for name, test := range tests { 125 | t.Run(name, func(t *testing.T) { 126 | assert := assert.New(t) 127 | 128 | r := test.renderer() 129 | gotData, err := r.Render(context.TODO(), test.alertGroup()) 130 | 131 | if test.expErr != nil && assert.Error(err) { 132 | assert.True(errors.Is(err, test.expErr)) 133 | } else if assert.NoError(err) { 134 | assert.Equal(test.expData, gotData) 135 | } 136 | }) 137 | } 138 | } 139 | -------------------------------------------------------------------------------- /scripts/build/build-image.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | set -e 4 | 5 | if [ -z ${VERSION} ]; then 6 | echo "IMAGE_VERSION env var needs to be set" 7 | exit 1 8 | fi 9 | 10 | if [ -z ${IMAGE} ]; then 11 | echo "IMAGE env var needs to be set" 12 | exit 1 13 | fi 14 | 15 | if [ -z ${DOCKER_FILE_PATH} ]; then 16 | echo "DOCKER_FILE_PATH env var needs to be set" 17 | exit 1 18 | fi 19 | 20 | echo "Building image ${IMAGE}:${VERSION}..." 21 | docker build \ 22 | --build-arg VERSION=${VERSION} \ 23 | -t ${IMAGE}:${VERSION} \ 24 | -f ${DOCKER_FILE_PATH} . 25 | 26 | if [ ! -z ${TAG_IMAGE_LATEST} ]; then 27 | echo "Tagged image ${IMAGE}:${VERSION} with ${IMAGE}:latest" 28 | docker tag ${IMAGE}:${VERSION} ${IMAGE}:latest 29 | fi -------------------------------------------------------------------------------- /scripts/build/build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -o errexit 4 | set -o nounset 5 | 6 | src=./cmd/alertgram 7 | out=./bin/alertgram 8 | 9 | goarch=amd64 10 | goos=linux 11 | goarm=7 12 | 13 | if [ $ostype == 'Linux' ]; then 14 | echo "Building linux release..." 15 | goos=linux 16 | binary_ext=-linux-amd64 17 | elif [ $ostype == 'Darwin' ]; then 18 | echo "Building darwin release..." 19 | goos=darwin 20 | binary_ext=-darwin-amd64 21 | elif [ $ostype == 'Windows' ]; then 22 | echo "Building windows release..." 23 | goos=windows 24 | binary_ext=-windows-amd64.exe 25 | elif [ $ostype == 'ARM64' ]; then 26 | echo "Building ARM64 release..." 27 | goos=linux 28 | goarch=arm64 29 | binary_ext=-linux-arm64 30 | elif [ $ostype == 'ARM' ]; then 31 | echo "Building ARM release..." 32 | goos=linux 33 | goarch=arm 34 | goarm=7 35 | binary_ext=-linux-arm-v7 36 | else 37 | echo "ostype env var required" 38 | exit 1 39 | fi 40 | 41 | final_out=${out}${binary_ext} 42 | ldf_cmp="-w -extldflags '-static'" 43 | f_ver="-X main.Version=${VERSION:-dev}" 44 | 45 | echo "Building binary at ${final_out}" 46 | GOOS=${goos} GOARCH=${goarch} GOARM=${goarm} CGO_ENABLED=0 go build -o ${final_out} --ldflags "${ldf_cmp} ${f_ver}" ${src} -------------------------------------------------------------------------------- /scripts/build/publish-image.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | set -e 4 | 5 | if [ -z ${VERSION} ]; then 6 | echo "IMAGE_VERSION env var needs to be set" 7 | exit 1 8 | fi 9 | 10 | if [ -z ${IMAGE} ]; then 11 | echo "IMAGE env var needs to be set" 12 | exit 1 13 | fi 14 | 15 | echo "Pushing image ${IMAGE}:${VERSION}..." 16 | docker push ${IMAGE}:${VERSION} 17 | 18 | if [ ! -z ${TAG_IMAGE_LATEST} ]; then 19 | echo "Pushing image ${IMAGE}:latest..." 20 | docker push ${IMAGE}:latest 21 | fi -------------------------------------------------------------------------------- /scripts/check/check.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | set -o errexit 4 | set -o nounset 5 | 6 | golangci-lint run -E goimports -------------------------------------------------------------------------------- /scripts/check/integration-test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | set -o errexit 4 | set -o nounset 5 | 6 | go test -race -v -tags='integration' ./... -------------------------------------------------------------------------------- /scripts/check/unit-test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | set -o errexit 4 | set -o nounset 5 | 6 | go test -race -v ./... -------------------------------------------------------------------------------- /scripts/deps.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | set -o errexit 4 | set -o nounset 5 | 6 | go mod tidy -------------------------------------------------------------------------------- /scripts/mockgen.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | set -o errexit 4 | set -o nounset 5 | 6 | go generate ./internal/mocks -------------------------------------------------------------------------------- /testdata/alerts/base.json: -------------------------------------------------------------------------------- 1 | { 2 | "receiver": "team1", 3 | "status": "firing", 4 | "alerts": [ 5 | { 6 | "status": "firing", 7 | "labels": { 8 | "alertname": "ServicePodIsRestarting", 9 | "chat_id": "myCustomChat1", 10 | "instance": "10.0.0.221:20464", 11 | "job": "kubernetes-metrics", 12 | "owner": "team1", 13 | "severity": "telegram", 14 | "env": "production", 15 | "pod": "ns1/pod-service1-f76c976c4-9hlgv" 16 | }, 17 | "annotations": { 18 | "message": "The has been restarting more than 5 times over 20 minutes", 19 | "Graph": "https://prometheus.test/d/taQlRuxik/k8s-cluster-summary?orgId=1&refresh=30s&fullscreen&panelId=11&from=now-3h&to=now&var-cluster=prometheus&var-ds=&var-datasource=prometheus&var-node=k8s-master&var-namespace=monitoring", 20 | "Runbook": "https://github.test/runbooks/pod-restarting.md" 21 | }, 22 | "startsAt": "2019-12-06T20:46:37.903Z", 23 | "generatorURL": "https://example.com/graph#..." 24 | }, 25 | { 26 | "status": "firing", 27 | "labels": { 28 | "alertname": "ServicePodIsRestarting", 29 | "instance": "10.0.0.221:20464", 30 | "job": "kubernetes-metrics", 31 | "owner": "team1", 32 | "severity": "telegram", 33 | "env": "production", 34 | "pod": "ns1/pod-service64-f5c7dd9cfc5-8scht" 35 | }, 36 | "annotations": { 37 | "message": "The has been restarting more than 5 times over 20 minutes", 38 | "Graph": "https://prometheus.test/d/taQlRuxik/k8s-cluster-summary?orgId=1&refresh=30s&fullscreen&panelId=11&from=now-3h&to=now&var-cluster=prometheus&var-ds=&var-datasource=prometheus&var-node=k8s-master&var-namespace=monitoring", 39 | "Runbook": "https://github.test/runbooks/pod-restarting.md" 40 | }, 41 | "startsAt": "2019-12-06T20:46:37.903Z", 42 | "generatorURL": "https://example.com/graph#..." 43 | }, 44 | { 45 | "status": "firing", 46 | "labels": { 47 | "alertname": "ServicePodIsPending", 48 | "instance": "10.0.0.221:20464", 49 | "job": "kubernetes-metrics", 50 | "owner": "team1", 51 | "severity": "telegram", 52 | "env": "production", 53 | "pod": "ns2/pod-service2-954bb8c86-p6svs" 54 | }, 55 | "annotations": { 56 | "message": "The has been pending more than 10 minutes", 57 | "Graph": "https://prometheus.test/d/taQlRuxik/k8s-cluster-summary?orgId=1&refresh=30s&fullscreen&panelId=11&from=now-3h&to=now&var-cluster=prometheus&var-ds=&var-datasource=prometheus&var-node=k8s-master&var-namespace=monitoring", 58 | "Runbook": "https://github.test/runbooks/pod-restarting.md" 59 | }, 60 | "startsAt": "2019-12-06T20:46:37.903Z", 61 | "generatorURL": "https://example.com/graph" 62 | }, 63 | { 64 | "status": "resolved", 65 | "labels": { 66 | "alertname": "ServicePodIsPending", 67 | "instance": "10.0.0.221:20464", 68 | "job": "kubernetes-metrics2", 69 | "owner": "team1", 70 | "severity": "telegram", 71 | "env": "production", 72 | "pod": "ns3/pod-service3-954bb8c86-p6svs" 73 | }, 74 | "annotations": { 75 | "message": "The has been pending more than 10 minutes", 76 | "Graph": "https://prometheus.test/d/taQlRuxik/k8s-cluster-summary?orgId=1&refresh=30s&fullscreen&panelId=11&from=now-3h&to=now&var-cluster=prometheus&var-ds=&var-datasource=prometheus&var-node=k8s-master&var-namespace=monitoring", 77 | "Runbook": "https://github.test/runbooks/pod-restarting.md" 78 | }, 79 | "startsAt": "2019-12-06T20:46:37.903Z", 80 | "generatorURL": "https://example.com/graph" 81 | } 82 | ], 83 | "groupLabels": { 84 | "alertname": "something_happend", 85 | "instance": "server01.int:9100" 86 | }, 87 | "externalURL": "https://prometheus.example.com", 88 | "version": "4", 89 | "groupKey": "1234567890" 90 | } 91 | -------------------------------------------------------------------------------- /testdata/templates/simple.tmpl: -------------------------------------------------------------------------------- 1 | {{- range .Alerts }} 2 | {{- if .IsFiring }} 3 | 🚨{{ .Labels.alertname }} 4 | ➡️ {{ .Annotations.message }} 5 | {{- end }} 6 | {{- end }} 7 | 8 | --------------------------------------------------------------------------------