├── .circleci └── config.yml ├── .dockerignore ├── .eslintignore ├── .eslintrc.js ├── .eslintrc.json ├── .github ├── dependabot.yml └── workflows │ ├── dependency-review.yml │ └── yarn_upgrade.yml ├── .gitignore ├── Dockerfile ├── LICENSE ├── README.md ├── charts └── polkadot-watcher │ ├── Chart.yaml │ ├── templates │ ├── _helpers.tpl │ ├── alertrules.yaml │ ├── configmap.yaml │ ├── deployment.yaml │ ├── service.yaml │ └── servicemonitor.yaml │ └── values.yaml ├── config └── main.sample.yaml ├── package.json ├── scripts ├── integration-tests.sh ├── patch.sh └── test_prometheus_rules.sh ├── src ├── actions │ └── start.ts ├── client.ts ├── constants.ts ├── gitConfigLoader │ ├── disabled.ts │ ├── gitConfigLoaderFactory.ts │ ├── gitConfigLoaderInterface.ts │ └── gitLabPrivate.ts ├── index.ts ├── logger.ts ├── prometheus.ts ├── subscriber.ts ├── types.ts └── utils.ts ├── test ├── mocks.ts ├── prometheus │ └── alertrules.yaml └── subscriber.ts ├── tsconfig.json └── yarn.lock /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | jobs: 4 | unitTest: 5 | docker: 6 | - image: web3f/node-dind:v3 7 | - image: parity/polkadot:latest 8 | name: polkadot 9 | command: --chain=kusama-dev --ws-port 11000 --alice --ws-external --rpc-methods=Unsafe --rpc-cors=all 10 | steps: 11 | - checkout 12 | - run: yarn 13 | - run: yarn test 14 | 15 | 16 | yarnLint: 17 | docker: 18 | - image: web3f/node-dind:v3 19 | steps: 20 | - checkout 21 | - run: yarn 22 | - run: yarn lint 23 | 24 | helmLint: 25 | docker: 26 | - image: web3f/ci-commons:v3 27 | steps: 28 | - checkout 29 | - run: 30 | command: | 31 | helm lint ./charts/polkadot-watcher 32 | 33 | buildImage: 34 | docker: 35 | - image: web3f/ci-commons:v3 36 | steps: 37 | - checkout 38 | - setup_remote_docker: 39 | docker_layer_caching: true 40 | - run: 41 | command: | 42 | /scripts/build-image.sh web3f/polkadot-watcher . 43 | 44 | integrationTests: 45 | docker: 46 | - image: web3f/ci-commons:v3 47 | steps: 48 | - checkout 49 | - setup_remote_docker 50 | - run: 51 | description: run integration tests 52 | command: | 53 | /scripts/integration-tests.sh kindest/node:v1.19.11 54 | 55 | testPrometheusRules: 56 | docker: 57 | - image: web3f/ci-commons:v3 58 | steps: 59 | - checkout 60 | - run: 61 | name: Install missing dependencies 62 | command: | 63 | YQ_VER=4.16.1 64 | wget -O /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/v${YQ_VER}/yq_linux_amd64 65 | 66 | chmod +x /usr/local/bin/yq 67 | 68 | PROM_VER=2.31.1 69 | wget -O /tmp/prometheus.tgz https://github.com/prometheus/prometheus/releases/download/v${PROM_VER}/prometheus-${PROM_VER}.linux-amd64.tar.gz 70 | tar -xvf /tmp/prometheus.tgz prometheus-${PROM_VER}.linux-amd64/promtool -C /tmp 71 | mv /tmp/prometheus-$PROM_VER.linux-amd64/promtool /usr/local/bin/ 72 | - run: 73 | command: | 74 | scripts/test_prometheus_rules.sh 75 | 76 | 77 | publishImage: 78 | docker: 79 | - image: web3f/ci-commons:v3 80 | steps: 81 | - checkout 82 | - setup_remote_docker 83 | - run: 84 | command: | 85 | /scripts/publish-image.sh web3f/polkadot-watcher 86 | 87 | publishChart: 88 | docker: 89 | - image: web3f/ci-commons:v3 90 | steps: 91 | - checkout 92 | - run: 93 | command: | 94 | /scripts/publish-chart.sh 95 | 96 | workflows: 97 | version: 2 98 | test_and_deploy: 99 | jobs: 100 | - helmLint: 101 | filters: 102 | tags: 103 | only: /.*/ 104 | - yarnLint: 105 | filters: 106 | tags: 107 | only: /.*/ 108 | # - unitTest: 109 | # filters: 110 | # tags: 111 | # only: /.*/ 112 | # requires: 113 | # - helmLint 114 | # - yarnLint 115 | - testPrometheusRules: 116 | filters: 117 | tags: 118 | only: /.*/ 119 | requires: 120 | - helmLint 121 | - yarnLint 122 | - buildImage: 123 | context: dockerhub-bot 124 | filters: 125 | tags: 126 | only: /.*/ 127 | requires: 128 | - helmLint 129 | - yarnLint 130 | - testPrometheusRules 131 | - integrationTests: 132 | filters: 133 | tags: 134 | only: /.*/ 135 | requires: 136 | - buildImage 137 | - publishImage: 138 | context: dockerhub-bot 139 | filters: 140 | branches: 141 | ignore: /.*/ 142 | tags: 143 | only: /^v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$/ 144 | requires: 145 | - integrationTests 146 | - publishChart: 147 | context: github-bot 148 | filters: 149 | branches: 150 | ignore: /.*/ 151 | tags: 152 | only: /^v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$/ 153 | requires: 154 | - integrationTests 155 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | * 2 | !src 3 | !package.json 4 | !tsconfig.json 5 | !yarn.lock -------------------------------------------------------------------------------- /.eslintignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | dist 3 | coverage 4 | -------------------------------------------------------------------------------- /.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | root: true, 3 | parser: '@typescript-eslint/parser', 4 | plugins: [ 5 | '@typescript-eslint', 6 | ], 7 | extends: [ 8 | 'eslint:recommended', 9 | 'plugin:@typescript-eslint/eslint-recommended', 10 | 'plugin:@typescript-eslint/recommended', 11 | ], 12 | "rules": { 13 | "@typescript-eslint/camelcase": ["error", { "properties": "never" } ] 14 | }, 15 | env: { 16 | node: true, 17 | }, 18 | }; 19 | -------------------------------------------------------------------------------- /.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "env": { 3 | "node": true, 4 | "commonjs": true, 5 | "es6": true, 6 | "mocha": true 7 | }, 8 | "extends": "eslint:recommended", 9 | "globals": { 10 | "Atomics": "readonly", 11 | "SharedArrayBuffer": "readonly" 12 | }, 13 | "parserOptions": { 14 | "ecmaVersion": 2018 15 | }, 16 | "rules": { 17 | "no-console": 0, 18 | "no-useless-escape": 0 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: npm 4 | directory: "/" 5 | schedule: 6 | interval: weekly 7 | open-pull-requests-limit: 10 8 | reviewers: 9 | - "w3f/infrastructure" 10 | ignore: 11 | - dependency-name: "*" 12 | update-types: ["version-update:semver-minor","version-update:semver-patch"] 13 | 14 | - package-ecosystem: docker 15 | directory: "/" 16 | schedule: 17 | interval: weekly 18 | open-pull-requests-limit: 10 19 | ignore: 20 | - dependency-name: "*" 21 | update-types: ["version-update:semver-minor","version-update:semver-patch"] -------------------------------------------------------------------------------- /.github/workflows/dependency-review.yml: -------------------------------------------------------------------------------- 1 | # Dependency Review Action 2 | # 3 | # This Action will scan dependency manifest files that change as part of a Pull Request, surfacing known-vulnerable versions of the packages declared or updated in the PR. Once installed, if the workflow run is marked as required, PRs introducing known-vulnerable packages will be blocked from merging. 4 | # 5 | # Source repository: https://github.com/actions/dependency-review-action 6 | # Public documentation: https://docs.github.com/en/code-security/supply-chain-security/understanding-your-software-supply-chain/about-dependency-review#dependency-review-enforcement 7 | name: 'Dependency Review' 8 | on: [pull_request] 9 | 10 | permissions: 11 | contents: read 12 | 13 | jobs: 14 | dependency-review: 15 | runs-on: ubuntu-latest 16 | steps: 17 | - name: 'Checkout Repository' 18 | uses: actions/checkout@v3 19 | - name: 'Dependency Review' 20 | uses: actions/dependency-review-action@v2 -------------------------------------------------------------------------------- /.github/workflows/yarn_upgrade.yml: -------------------------------------------------------------------------------- 1 | name: 'Yarn Upgrade' 2 | on: 3 | schedule: 4 | - cron: '0 10 * * 1' 5 | workflow_dispatch: 6 | 7 | jobs: 8 | yarn-upgrade: 9 | uses: w3f/base-services-charts/.github/workflows/yarn_upgrade.yml@master 10 | secrets: 11 | PR_PAT: ${{ secrets.BOT_PAT }} -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | config/* 3 | !config/*sample* 4 | config/secrets.yaml 5 | dist/ 6 | helmfile.d/.cache 7 | helmfile.d/.config 8 | helmfile.d/.kube 9 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:18-alpine 2 | 3 | WORKDIR /app 4 | 5 | COPY package.json yarn.lock ./ 6 | RUN yarn --ignore-scripts 7 | 8 | COPY . . 9 | RUN yarn && \ 10 | yarn build 11 | 12 | ENTRYPOINT ["yarn", "start"] 13 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![CircleCI](https://circleci.com/gh/w3f/polkadot-watcher-validator.svg?style=svg)](https://circleci.com/gh/w3f/polkadot-watcher-validator) 2 | 3 | # polkadot-watcher-validator 4 | 5 | ## How to Run 6 | 7 | ### Requirements 8 | - yarn: https://classic.yarnpkg.com/en/docs/install/ 9 | 10 | ```bash 11 | git clone https://github.com/w3f/polkadot-watcher.git 12 | cd polkadot-watcher 13 | cp config/main.sample.yaml config/main.yaml 14 | #just the first time 15 | 16 | yarn 17 | yarn build 18 | yarn start 19 | ``` 20 | 21 | ## About 22 | 23 | The watcher is a nodeJs application meant to be connected with a substrate based node through a web socket. 24 | It can then monitor the status of the node, leveraging on mechanisms such as the builtin heartbeat. 25 | 26 | ### Monitoring Features 27 | 28 | - A validator has been reported for Slash 29 | - A validator is not seleceted by Phragmen alorithm to be part of the active set 30 | - A validator has changed his payout address destination 31 | - A validator has an unexpected payout address destination 32 | - A validator has changed his commission rate 33 | - A validator has an unexpected commission rate 34 | 35 | #### Unexpected payout address destination 36 | 37 | Possible Payout Destination Types are `'Staked' | 'Stash' | 'Controller' | 'Account' | 'None'`. 38 | If an expected destination address is specified in the config file, it is implicit that you are expecting a type `'Account'` to be matched. 39 | 40 | ### Resources 41 | 42 | - session: https://wiki.polkadot.network/docs/en/glossary#session 43 | - era: https://wiki.polkadot.network/docs/en/glossary#era 44 | - polkadotJs library (raccomended, Nodejs + typescript): https://polkadot.js.org/docs/ 45 | - event, validator SlashReported: https://polkadot.js.org/docs/substrate/events/#slashreportedaccountid32-perbill-u32 46 | 47 | ## Configuration 48 | 49 | A sample config file is provided [here](config/main.sample.yaml) 50 | 51 | ### Validators from Git 52 | 53 | The list of your addresses can be dynamically retrieved (app startup/restart) from a Git file. Check the [GitConfigLoader](src/gitConfigLoader) implementation. 54 | 55 | - [GitLab API](https://docs.gitlab.com/ee/api/repository_files.html) 56 | 57 | ### Prometheus 58 | 59 | A Prometheus instance can be attached to this application thanks to the endpoint exposed at [/metrics](https://github.com/w3f/polkadot-watcher-validator/blob/master/src/prometheus.ts#L114). 60 | An Alerting system can be then built on top of that, see [here](charts/polkadot-watcher/templates/alertrules.yaml) 61 | 62 | ## Deployment 63 | 64 | The [Dockerfile](Dockerfile) can be deployed into a Kubernetes cluster thanks to the polkadot-watcher [chart](charts/polkadot-watcher). 65 | 66 | 67 | -------------------------------------------------------------------------------- /charts/polkadot-watcher/Chart.yaml: -------------------------------------------------------------------------------- 1 | description: Polkadot Watcher 2 | name: polkadot-watcher 3 | version: v4.4.7 4 | appVersion: v4.4.7 5 | apiVersion: v2 6 | -------------------------------------------------------------------------------- /charts/polkadot-watcher/templates/_helpers.tpl: -------------------------------------------------------------------------------- 1 | {{/* Returns the app name */}} 2 | {{- define "app.name" -}} 3 | {{- default .Release.Name .Values.nameOverride -}} 4 | {{- end }} -------------------------------------------------------------------------------- /charts/polkadot-watcher/templates/alertrules.yaml: -------------------------------------------------------------------------------- 1 | # max by(job,namespace,network,address,name) is the primary key which all the metrics are aggregated by to avoid alerts flapping: i.e. when a K8s pod restarts 2 | # The same result could be achieved by applying max without(instance,pod) 3 | # -- 4 | # last_over_time is used to avoid alerts flapping when the alert is firing and the alert disappear, to override and extend the default endsAt behaviour: i.e. when the watcher takes a while to restart 5 | # -- 6 | # All the queries are also filtered by the "environment" label: i.e. to not mix metrics coming from staging and production environments 7 | 8 | {{ if and .Values.prometheusRules.enabled ( ne .Values.config.environment "ci" ) }} 9 | apiVersion: monitoring.coreos.com/v1 10 | kind: PrometheusRule 11 | metadata: 12 | labels: 13 | {{ toYaml .Values.prometheusRules.labels | indent 4 }} 14 | name: {{ include "app.name" . }} 15 | spec: 16 | groups: 17 | - name: {{ include "app.name" . }}-{{ .Values.config.environment }}.rules 18 | rules: 19 | - alert: ValidatorOutOfActiveSet 20 | annotations: 21 | message: 'Target {{`{{ $labels.name }}`}} is not present in the current validation active set because it has not been selected by Phragmen' 22 | expr: max without(instance,pod) (last_over_time(polkadot_validator_out_of_active_set_state{environment="{{ .Values.config.environment }}"}[10m])) > 0 23 | for: 2m 24 | labels: 25 | severity: info 26 | origin: {{ .Values.prometheusRules.origin }} 27 | {{ if ne .Values.prometheusRules.producerStall false }} 28 | - alert: ProducerStallShort 29 | annotations: 30 | message: 'Blocks were not produced for 1 hour by {{`{{ $labels.name }}`}}' 31 | runbook_url: "https://github.com/w3f/infrastructure/wiki/Producer-Stall" 32 | expr: max without(instance,pod) (increase(polkadot_validator_blocks_produced{environment="{{ .Values.config.environment }}"}[10m])) == 0 and max without(instance,pod) (last_over_time(polkadot_validator_out_of_active_set_state{environment="{{ .Values.config.environment }}"}[10m])) == 0 33 | for: 60m 34 | labels: 35 | severity: warning 36 | origin: {{ .Values.prometheusRules.origin }} 37 | - alert: ProducerStallLong 38 | annotations: 39 | message: 'Blocks were not produced for 3 hours by {{`{{ $labels.name }}`}}' 40 | runbook_url: "https://github.com/w3f/infrastructure/wiki/Producer-Stall" 41 | expr: max without(instance,pod) (increase(polkadot_validator_blocks_produced{environment="{{ .Values.config.environment }}"}[10m])) == 0 and max without(instance,pod) (last_over_time(polkadot_validator_out_of_active_set_state{environment="{{ .Values.config.environment }}"}[10m])) == 0 42 | for: 180m 43 | labels: 44 | severity: critical 45 | origin: {{ .Values.prometheusRules.origin }} 46 | {{ end }} 47 | - alert: ValidatorSlashed 48 | annotations: 49 | message: 'Target {{`{{ $labels.name }}`}} was reported for Slash, an advanced double check can be carried here. This message is going to RESOLVE by itself soon.' 50 | runbook_url: "https://github.com/w3f/infrastructure/wiki/Validator-Slashed" 51 | expr: max without(instance,pod) (increase(polkadot_validator_slashed_reports{environment="{{ .Values.config.environment }}"}[5m])) > 0 52 | for: 30s 53 | labels: 54 | severity: critical 55 | origin: {{ .Values.prometheusRules.origin }} 56 | {{ if ne .Values.prometheusRules.changed false }} 57 | - alert: ValidatorRewardDestinationChanged 58 | annotations: 59 | message: 'Target {{`{{ $labels.name }}`}} may have changed his reward destination recently, please double check. This message is going to RESOLVE by itself soon.' 60 | expr: max without(instance,pod) (increase(polkadot_validator_payee_changed_reports{environment="{{ .Values.config.environment }}"}[5m])) > 0 61 | for: 30s 62 | labels: 63 | severity: warning 64 | origin: {{ .Values.prometheusRules.origin }} 65 | - alert: ValidatorCommissionRateChanged 66 | annotations: 67 | message: 'Target {{`{{ $labels.name }}`}} may have changed his commission rate recently, please double check. This message is going to RESOLVE by itself soon.' 68 | expr: max without(instance,pod) (increase(polkadot_validator_commission_changed_reports{environment="{{ .Values.config.environment }}"}[5m])) > 0 69 | for: 30s 70 | labels: 71 | severity: warning 72 | origin: {{ .Values.prometheusRules.origin }} 73 | {{ end }} 74 | - alert: ValidatorCommissionRateUnexpected 75 | annotations: 76 | message: 'Target {{`{{ $labels.name }}`}} has an unexpected commission rate, please double check.' 77 | expr: max without(instance,pod) (last_over_time(polkadot_validator_unexpected_commission_state{environment="{{ .Values.config.environment }}"}[10m])) > 0 78 | for: 1m 79 | labels: 80 | severity: warning 81 | origin: {{ .Values.prometheusRules.origin }} 82 | - alert: ValidatorRewardDestinationUnexpected 83 | annotations: 84 | message: 'Target {{`{{ $labels.name }}`}} has an unexpected reward destination, please double check.' 85 | expr: max without(instance,pod) (last_over_time(polkadot_validator_unexpected_payee_state{environment="{{ .Values.config.environment }}"}[10m])) > 0 86 | for: 1m 87 | labels: 88 | severity: warning 89 | origin: {{ .Values.prometheusRules.origin }} 90 | {{ end }} 91 | -------------------------------------------------------------------------------- /charts/polkadot-watcher/templates/configmap.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: {{ include "app.name" . }} 5 | data: 6 | main.yaml: |- 7 | {{ toYaml .Values.config | indent 4 }} 8 | -------------------------------------------------------------------------------- /charts/polkadot-watcher/templates/deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: {{ include "app.name" . }} 5 | labels: 6 | app: {{ include "app.name" . }} 7 | spec: 8 | replicas: 1 9 | revisionHistoryLimit: 3 10 | strategy: 11 | type: RollingUpdate 12 | selector: 13 | matchLabels: 14 | app: {{ include "app.name" . }} 15 | template: 16 | metadata: 17 | labels: 18 | app: {{ include "app.name" . }} 19 | annotations: 20 | checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} 21 | spec: 22 | containers: 23 | - name: {{ include "app.name" . }} 24 | image: {{ .Values.image.repo }}:{{ .Values.image.tag | default .Chart.AppVersion }} 25 | imagePullPolicy: {{ .Values.image.pullPolicy | default "Always" }} 26 | args: 27 | - -c 28 | - /app/config/main.yaml 29 | ports: 30 | - name: metrics 31 | containerPort: {{ .Values.config.port }} 32 | livenessProbe: 33 | httpGet: 34 | path: /healthcheck 35 | port: {{ .Values.config.port }} 36 | initialDelaySeconds: 10 37 | timeoutSeconds: 200 38 | {{ if ne .Values.config.environment "ci" }} 39 | resources: 40 | {{- toYaml .Values.resources | nindent 10 }} 41 | {{ end }} 42 | volumeMounts: 43 | - name: config 44 | mountPath: /app/config 45 | volumes: 46 | - name: config 47 | configMap: 48 | name: {{ include "app.name" . }} 49 | -------------------------------------------------------------------------------- /charts/polkadot-watcher/templates/service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: {{ include "app.name" . }} 5 | labels: 6 | app: {{ include "app.name" . }} 7 | spec: 8 | ports: 9 | - name: metrics 10 | port: {{ .Values.config.port }} 11 | selector: 12 | app: {{ include "app.name" . }} 13 | -------------------------------------------------------------------------------- /charts/polkadot-watcher/templates/servicemonitor.yaml: -------------------------------------------------------------------------------- 1 | {{ if and .Values.serviceMonitor.enabled ( ne .Values.config.environment "ci" ) }} 2 | apiVersion: monitoring.coreos.com/v1 3 | kind: ServiceMonitor 4 | metadata: 5 | name: {{ include "app.name" . }} 6 | labels: 7 | {{ toYaml .Values.serviceMonitor.labels | indent 4 }} 8 | spec: 9 | selector: 10 | matchLabels: 11 | app: {{ include "app.name" . }} 12 | endpoints: 13 | - port: metrics 14 | {{ end }} 15 | -------------------------------------------------------------------------------- /charts/polkadot-watcher/values.yaml: -------------------------------------------------------------------------------- 1 | image: 2 | repo: web3f/polkadot-watcher 3 | # tag: latest 4 | 5 | config: 6 | endpoint: "wss://kusama-rpc.polkadot.io" 7 | port: 3000 8 | logLevel: info 9 | environment: production #it is used also to filter the prometheusrules, i.e. to avoid conflicts between production and staging 10 | validators: [] #optional 11 | validatorsFromGit: #optional 12 | enabled: false 13 | platform: gitLab 14 | private: 15 | enabled: true 16 | apiToken: xxx 17 | network: kusama 18 | url: http://your.gitlab.domain/api/v4/projects/number/repository/files/accounts.yaml/raw?ref=main 19 | 20 | serviceMonitor: 21 | enabled: true #to be enabled for each instance 22 | labels: 23 | group: w3f 24 | release: prometheus-operator 25 | 26 | prometheusRules: 27 | enabled: false #if you have multiple instances, enabled it just once to avoid duplicated alerts 28 | labels: 29 | app: w3f 30 | origin: cluster 31 | producerStall: true 32 | changed: true 33 | 34 | resources: 35 | requests: 36 | cpu: "100m" 37 | memory: "500Mi" 38 | limits: 39 | cpu: "100m" 40 | memory: "500Mi" 41 | -------------------------------------------------------------------------------- /config/main.sample.yaml: -------------------------------------------------------------------------------- 1 | endpoint: "wss://kusama-rpc.polkadot.io/" 2 | port: 3000 3 | logLevel: info 4 | environment: production 5 | commissionLogic: eq #optional | lt | eq 6 | validatorsFromGit: #optional 7 | enabled: false 8 | platform: gitLab 9 | private: 10 | enabled: true 11 | apiToken: xxx 12 | network: kusama 13 | url: http://your.gitlab.domain/api/v4/projects/number/repository/files/accounts.yaml/raw?ref=main 14 | validators: #optional 15 | - name: test 16 | address: Gt6HqWBhdu4Sy1u8ASTbS1qf2Ac5gwdegwr8tWN8saMxPt5 17 | expected: #optional 18 | commission: 0.17 #optional 19 | payee: Gt6HqWBhdu4Sy1u8ASTbS1qf2Ac5gwdegwr8tWN8saMxPt5 #optional | Staked | Stash | Controller | None -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "polkadot-watcher", 3 | "version": "4.4.7", 4 | "description": "Monitor events on Polkadot networks", 5 | "repository": "git@github.com:w3f/polkadot-watcher.git", 6 | "author": "W3F Infrastructure Team ", 7 | "license": "Apache-2.0", 8 | "main": "./dist/index.js", 9 | "types": "./dist/index.d.ts", 10 | "files": [ 11 | "dist/**/*" 12 | ], 13 | "scripts": { 14 | "lint": "yarn eslint . --ext .js,.jsx,.ts,.tsx", 15 | "build": "tsc --build tsconfig.json", 16 | "prepare": "yarn build", 17 | "pretest": "yarn lint", 18 | "test": "mocha --timeout 60000 --require ts-node/register --exit test/*.ts test/**/*.ts", 19 | "e2e-test": "mocha --timeout 300000 --require ts-node/register --exit e2e-test/**/*.ts", 20 | "start": "node ./dist/index.js start" 21 | }, 22 | "dependencies": { 23 | "@polkadot/api": "^14.3.1", 24 | "@w3f/config": "^0.1.1", 25 | "@w3f/logger": "^0.4.2", 26 | "commander": "^4.0.0", 27 | "express": "^4.18.1", 28 | "got": "^11.8.5", 29 | "node-fetch": "^2.6.6", 30 | "prom-client": "^14.0.1", 31 | "ws": "^6.1.2" 32 | }, 33 | "devDependencies": { 34 | "@types/chai": "^4.3.1", 35 | "@types/express": "^4.17.13", 36 | "@types/fs-extra": "^8.1.2", 37 | "@types/lodash": "^4.14.182", 38 | "@types/mocha": "^9.1.1", 39 | "@types/node": "^14.18.24", 40 | "@types/tmp": "^0.2.3", 41 | "@typescript-eslint/eslint-plugin": "^2.34.0", 42 | "@typescript-eslint/parser": "^2.34.0", 43 | "@w3f/test-utils": "^1.2.30", 44 | "chai": "^4.3.6", 45 | "eslint": "^7.32.0", 46 | "fs-extra": "^9.1.0", 47 | "lodash": "^4.17.21", 48 | "mocha": "^9.2.2", 49 | "nock": "^12.0.3", 50 | "sinon": "^8.1.1", 51 | "tmp": "^0.2.1", 52 | "ts-node": "^10.9.1", 53 | "typescript": "4.7.4" 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /scripts/integration-tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | source /scripts/common.sh 4 | source /scripts/bootstrap-helm.sh 5 | 6 | 7 | run_tests() { 8 | echo Running tests... 9 | 10 | wait_pod_ready polkadot-watcher 11 | } 12 | 13 | teardown() { 14 | helm delete polkadot-watcher 15 | } 16 | 17 | main(){ 18 | if [ -z "$KEEP_W3F_POLKADOT_WATCHER" ]; then 19 | trap teardown EXIT 20 | fi 21 | 22 | /scripts/build-helm.sh \ 23 | --set config.environment=ci \ 24 | --set image.tag="${CIRCLE_SHA1}" \ 25 | polkadot-watcher \ 26 | ./charts/polkadot-watcher 27 | 28 | run_tests 29 | } 30 | 31 | main 32 | -------------------------------------------------------------------------------- /scripts/patch.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | echo patching... 5 | sed -i "/parity\/polkadot/c\ - image: parity\/polkadot:$latest_upstream " .circleci/config.yml 6 | -------------------------------------------------------------------------------- /scripts/test_prometheus_rules.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | set -x 5 | 6 | TEST_FOLDER="test/prometheus" 7 | CHART_NAME="polkadot-watcher" 8 | for file in $(find "${TEST_FOLDER}" -name *yaml | sed 's|^'"${TEST_FOLDER}"/'||'); do 9 | template_name=${file##*/} 10 | 11 | helm template --set prometheusRules.enabled=true -s "templates/${template_name}" "charts/${CHART_NAME}" | yq e '.spec' - | promtool check rules /dev/stdin 12 | helm template --set prometheusRules.enabled=true -s "templates/${template_name}" "charts/${CHART_NAME}" | yq e '.spec' - | promtool test rules "${TEST_FOLDER}/${template_name}" 13 | done -------------------------------------------------------------------------------- /src/actions/start.ts: -------------------------------------------------------------------------------- 1 | import express from 'express'; 2 | import { LoggerSingleton } from '../logger' 3 | import { Config } from '@w3f/config'; 4 | import { register } from 'prom-client'; 5 | import { Subscriber } from '../subscriber'; 6 | import { Prometheus } from '../prometheus'; 7 | import { InputConfig } from '../types'; 8 | import { Client } from '../client'; 9 | import { environment } from '../constants'; 10 | import { GitConfigLoaderFactory } from '../gitConfigLoader/gitConfigLoaderFactory'; 11 | 12 | const _addTestEndpoint = (server: express.Application, subscriber: Subscriber): void =>{ 13 | 14 | server.get('/test', 15 | async (_req: express.Request, res: express.Response): Promise => { 16 | subscriber.triggerConnectivityTest() 17 | res.status(200).send('A test alert should fire and then resolve...') 18 | }) 19 | } 20 | 21 | const _loadConfig = async (config: any): Promise =>{ 22 | const cfg = new Config().parse(config); 23 | const gitList = await new GitConfigLoaderFactory(cfg).makeGitConfigLoader().downloadAndLoad(); 24 | 25 | const seen = new Set(); 26 | if(!cfg.validators) cfg.validators = [] 27 | const filteredArr = [...cfg.validators,...gitList].filter(el=>{ //priority given to locals over downloaded ones 28 | const isDuplicate = seen.has(el.name); 29 | seen.add(el.name) 30 | return !isDuplicate 31 | }) 32 | cfg.validators = filteredArr 33 | return cfg 34 | } 35 | 36 | export async function startAction(cmd): Promise { 37 | const cfg = await _loadConfig(cmd.config) 38 | 39 | const logger = LoggerSingleton.getInstance(cfg.logLevel) 40 | logger.info(`loaded ${cfg.validators.length} accounts`) 41 | 42 | const server = express(); 43 | server.get('/healthcheck', 44 | async (req: express.Request, res: express.Response): Promise => { 45 | res.status(200).send('OK!') 46 | }) 47 | server.get('/metrics', async (req: express.Request, res: express.Response) => { 48 | res.set('Content-Type', register.contentType) 49 | res.end(await register.metrics()) 50 | }) 51 | server.listen(cfg.port); 52 | 53 | const api = await new Client(cfg).connect() 54 | const chain = await api.rpc.system.chain() 55 | const networkId = chain.toString().toLowerCase() 56 | const env = cfg.environment ? cfg.environment : environment 57 | 58 | const promClient = new Prometheus(networkId,env); 59 | promClient.startCollection(); 60 | 61 | const subscriber = new Subscriber(cfg, api, promClient); 62 | await subscriber.start(); 63 | 64 | _addTestEndpoint(server,subscriber) 65 | } 66 | -------------------------------------------------------------------------------- /src/client.ts: -------------------------------------------------------------------------------- 1 | import { ApiPromise, WsProvider } from '@polkadot/api'; 2 | import { LoggerSingleton, Logger } from './logger'; 3 | 4 | import { 5 | InputConfig, 6 | } from './types'; 7 | 8 | export class Client { 9 | private api: ApiPromise; 10 | private endpoint: string; 11 | private readonly logger: Logger = LoggerSingleton.getInstance() 12 | 13 | constructor(cfg: InputConfig) { 14 | this.endpoint = cfg.endpoint; 15 | } 16 | 17 | public async connect(): Promise { 18 | try { 19 | await this._initAPI(); 20 | } catch (error) { 21 | this.logger.error("initAPI error... exiting: "+JSON.stringify(error)) 22 | process.exit(1) 23 | } 24 | return this.api 25 | } 26 | 27 | private async _initAPI(): Promise { 28 | const provider = new WsProvider(this.endpoint); 29 | this.api = new ApiPromise({provider}) 30 | if(this.api){ 31 | this.api.on("error", error => { 32 | if( error.toString().includes("FATAL") || JSON.stringify(error).includes("FATAL") ){ 33 | this.logger.error("The API had a FATAL error... exiting!") 34 | process.exit(1) 35 | } 36 | }) 37 | } 38 | await this.api.isReadyOrError; 39 | 40 | const [chain, nodeName, nodeVersion] = await Promise.all([ 41 | this.api.rpc.system.chain(), 42 | this.api.rpc.system.name(), 43 | this.api.rpc.system.version() 44 | ]); 45 | this.logger.info( 46 | `You are connected to chain ${chain} using ${nodeName} v${nodeVersion}` 47 | ); 48 | } 49 | 50 | } 51 | -------------------------------------------------------------------------------- /src/constants.ts: -------------------------------------------------------------------------------- 1 | import { Balance } from '@polkadot/types/interfaces'; 2 | import BN from 'bn.js'; 3 | 4 | export const ZeroBN = new BN(0); 5 | export const ZeroBalance = ZeroBN as Balance; 6 | export const payeeMetricAutoresolveMillis = 1200000; //20 minutes 7 | export const commissionMetricAutoresolveMillis = 1200000; //20 minutes 8 | export const environment = "production" 9 | -------------------------------------------------------------------------------- /src/gitConfigLoader/disabled.ts: -------------------------------------------------------------------------------- 1 | import { GitConfigLoader } from "./gitConfigLoaderInterface"; 2 | import { Subscribable } from "../types"; 3 | 4 | export class Disabled implements GitConfigLoader { 5 | async downloadAndLoad(): Promise> { 6 | return [] 7 | } 8 | } -------------------------------------------------------------------------------- /src/gitConfigLoader/gitConfigLoaderFactory.ts: -------------------------------------------------------------------------------- 1 | import { InputConfig } from "../types" 2 | import { Disabled } from "./disabled" 3 | import { GitConfigLoader } from "./gitConfigLoaderInterface" 4 | import { GitLabPrivate } from "./gitLabPrivate" 5 | 6 | export class GitConfigLoaderFactory { 7 | constructor(private readonly cfg: InputConfig){} 8 | makeGitConfigLoader = (): GitConfigLoader => { 9 | 10 | const gitConfig = this.cfg.validatorsFromGit 11 | 12 | if(!gitConfig?.enabled) 13 | return new Disabled() 14 | 15 | switch (gitConfig.platform.toLowerCase()) { 16 | case "gitlab": 17 | if(gitConfig.private.enabled) return new GitLabPrivate(gitConfig.url,gitConfig.private.apiToken,gitConfig.network) //implemented just GitLab private for now 18 | else new Disabled() 19 | break; 20 | default: 21 | return new Disabled() 22 | } 23 | } 24 | } -------------------------------------------------------------------------------- /src/gitConfigLoader/gitConfigLoaderInterface.ts: -------------------------------------------------------------------------------- 1 | import { Subscribable } from "../types"; 2 | 3 | export interface GitConfigLoader { 4 | downloadAndLoad(): Promise>; 5 | } -------------------------------------------------------------------------------- /src/gitConfigLoader/gitLabPrivate.ts: -------------------------------------------------------------------------------- 1 | import { GitConfigLoader } from "./gitConfigLoaderInterface"; 2 | import fetch from 'node-fetch'; 3 | import fs from 'fs'; 4 | import { Config } from '@w3f/config'; 5 | import { InputConfigFromGit, Subscribable } from "../types"; 6 | 7 | export class GitLabPrivate implements GitConfigLoader { 8 | 9 | constructor( 10 | protected readonly url: string, 11 | protected readonly apiToken: string, 12 | protected readonly network: string 13 | ) { } 14 | 15 | async downloadAndLoad(): Promise> { 16 | const response = await fetch(this.url, { 17 | headers: { 18 | 'PRIVATE-TOKEN': this.apiToken 19 | } 20 | }); 21 | const data = await response.text(); 22 | if(!response.ok) throw new Error("git config download probelm: " + data) 23 | 24 | fs.writeFileSync("./tmp.yaml",data) 25 | const cfg = new Config().parse("./tmp.yaml"); 26 | fs.rmSync("./tmp.yaml") 27 | 28 | switch (this.network.toLowerCase()) { 29 | case "kusama": 30 | return cfg.Kusama 31 | 32 | case "polkadot": 33 | return cfg.Polkadot 34 | 35 | default: 36 | throw new Error("unexpected configuration") 37 | } 38 | } 39 | } -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | import process from 'process'; 2 | import program from 'commander'; 3 | import { startAction } from './actions/start'; 4 | import '@polkadot/api-augment'; //https://github.com/polkadot-js/api/issues/4450 5 | 6 | 7 | program 8 | .command('start') 9 | .description('Starts the watcher.') 10 | .option('-c, --config [path]', 'Path to config file.', './config/main.yaml') 11 | .action(startAction); 12 | 13 | program.allowUnknownOption(false); 14 | 15 | program.parse(process.argv); 16 | -------------------------------------------------------------------------------- /src/logger.ts: -------------------------------------------------------------------------------- 1 | import { createLogger, Logger as LoggerW3f } from '@w3f/logger'; 2 | 3 | export class LoggerSingleton { 4 | private static instance: LoggerW3f 5 | private constructor() { 6 | //do nothing 7 | } 8 | public static setInstance(level: string): void { 9 | LoggerSingleton.instance = createLogger(level) 10 | } 11 | public static getInstance(level?: string): Logger { 12 | if (!LoggerSingleton.instance) { 13 | LoggerSingleton.instance = createLogger(level) 14 | } 15 | return LoggerSingleton.instance 16 | } 17 | } 18 | 19 | export type Logger = LoggerW3f 20 | -------------------------------------------------------------------------------- /src/prometheus.ts: -------------------------------------------------------------------------------- 1 | import * as promClient from 'prom-client'; 2 | import { Logger, LoggerSingleton } from './logger'; 3 | import { PromClient } from './types'; 4 | 5 | export class Prometheus implements PromClient { 6 | 7 | private blocksProducedReports: promClient.Counter<"network" | "name" | "address" | "environment">; 8 | private slashedReports: promClient.Counter<"network" | "name" | "address" | "environment">; 9 | private stateOutOfActiveSet: promClient.Gauge<"network" | "name" | "address" | "environment">; 10 | 11 | private payeeChangedReports: promClient.Counter<"network" | "name" | "address" | "environment">; 12 | private stateUnexpectedPayee: promClient.Gauge<"network" | "name" | "address" | "environment">; 13 | 14 | private commissionChangedReports: promClient.Counter<"network" | "name" | "address" | "environment">; 15 | private stateUnexpectedCommission: promClient.Gauge<"network" | "name" | "address" | "environment">; 16 | 17 | private readonly logger: Logger = LoggerSingleton.getInstance() 18 | 19 | constructor(private readonly network: string, private readonly environment: string) { 20 | this._initMetrics() 21 | } 22 | 23 | startCollection(): void { 24 | this.logger.info( 25 | 'Starting the collection of metrics, the metrics are available on /metrics' 26 | ); 27 | promClient.collectDefaultMetrics(); 28 | } 29 | 30 | increaseBlocksProducedReports(name: string, address: string): void { 31 | this.blocksProducedReports.inc({network:this.network, name, address, environment: this.environment }) 32 | } 33 | 34 | increaseSlashedReports(name: string, address: string): void { 35 | this.slashedReports.inc({network:this.network, name, address, environment: this.environment }); 36 | } 37 | 38 | setStatusOutOfActiveSet(name: string, address: string): void{ 39 | this.stateOutOfActiveSet.set({network:this.network, name, address, environment: this.environment }, 1); 40 | } 41 | 42 | resetStatusOutOfActiveSet(name: string, address: string): void{ 43 | this.stateOutOfActiveSet.set({network:this.network, name, address, environment: this.environment }, 0); 44 | } 45 | 46 | increasePayeeChangedReports(name: string, address: string): void{ 47 | this.payeeChangedReports.inc({network:this.network, name, address, environment: this.environment}); 48 | } 49 | 50 | setStatusValidatorPayeeUnexpected(name: string, address: string): void{ 51 | this.stateUnexpectedPayee.set({network:this.network, name,address, environment: this.environment }, 1); 52 | } 53 | 54 | resetStatusValidatorPayeeUnexpected(name: string, address: string): void{ 55 | this.stateUnexpectedPayee.set({network:this.network, name,address, environment: this.environment }, 0); 56 | } 57 | 58 | increaseCommissionChangedReports(name: string, address: string): void{ 59 | this.commissionChangedReports.inc({network:this.network, name, address, environment: this.environment}); 60 | } 61 | 62 | setStatusValidatorCommissionUnexpected(name: string, address: string): void{ 63 | this.stateUnexpectedCommission.set({network:this.network, name, address, environment: this.environment }, 1); 64 | } 65 | 66 | resetStatusValidatorCommissionUnexpected(name: string, address: string): void{ 67 | this.stateUnexpectedCommission.set({network:this.network, name, address, environment: this.environment }, 0); 68 | } 69 | 70 | _initMetrics(): void { 71 | this.blocksProducedReports = new promClient.Counter({ 72 | name: 'polkadot_validator_blocks_produced', 73 | help: 'Number of blocks produced by a validator', 74 | labelNames: ['network', 'name', 'address', 'environment'] 75 | }); 76 | this.slashedReports = new promClient.Gauge({ 77 | name: 'polkadot_validator_slashed_reports', 78 | help: 'Times a validator has been reported for slashing', 79 | labelNames: ['network', 'name', 'address', 'environment'] 80 | }); 81 | this.stateOutOfActiveSet = new promClient.Gauge({ 82 | name: 'polkadot_validator_out_of_active_set_state', 83 | help: 'Whether a validator is reported as outside of the current Era validators active set', 84 | labelNames: ['network', 'name', 'address', 'environment'] 85 | }); 86 | this.payeeChangedReports = new promClient.Counter({ 87 | name: 'polkadot_validator_payee_changed_reports', 88 | help: 'Times a validator has changed the payee destination', 89 | labelNames: ['network', 'name', 'address', 'environment'] 90 | }); 91 | this.stateUnexpectedPayee = new promClient.Gauge({ 92 | name: 'polkadot_validator_unexpected_payee_state', 93 | help: 'Whether a validator has an unexpected payee destination', 94 | labelNames: ['network', 'name', 'address', 'environment'] 95 | }); 96 | this.commissionChangedReports = new promClient.Counter({ 97 | name: 'polkadot_validator_commission_changed_reports', 98 | help: 'Times a validator has changed the commission rate', 99 | labelNames: ['network', 'name', 'address', 'environment'] 100 | }); 101 | this.stateUnexpectedCommission = new promClient.Gauge({ 102 | name: 'polkadot_validator_unexpected_commission_state', 103 | help: 'Whether a validator has an unexpected commission rate', 104 | labelNames: ['network', 'name', 'address', 'environment'] 105 | }); 106 | } 107 | } 108 | -------------------------------------------------------------------------------- /src/subscriber.ts: -------------------------------------------------------------------------------- 1 | import { ApiPromise } from '@polkadot/api'; 2 | import { Event } from '@polkadot/types/interfaces/system'; 3 | import { Header, SessionIndex, ValidatorId, Address } from '@polkadot/types/interfaces'; 4 | import { DeriveStakingQuery } from '@polkadot/api-derive/types'; 5 | import { Vec } from '@polkadot/types/codec'; 6 | import { LoggerSingleton } from './logger'; 7 | 8 | import { 9 | InputConfig, 10 | Subscribable, 11 | PromClient, 12 | } from './types'; 13 | import { getActiveEraIndex } from './utils'; 14 | 15 | export class Subscriber { 16 | private validators: Array; 17 | private currentEraIndex: number; 18 | private validatorActiveSet: Vec; 19 | private sessionIndex: SessionIndex; 20 | 21 | private readonly logger = LoggerSingleton.getInstance() 22 | 23 | constructor( 24 | private readonly cfg: InputConfig, 25 | private readonly api: ApiPromise, 26 | private readonly promClient: PromClient) { 27 | 28 | this.validators = cfg.validators 29 | } 30 | 31 | public async start(): Promise { 32 | 33 | await this._initInstanceVariables(); 34 | this._initCounterMetrics() 35 | 36 | await this._handleNewHeadSubscriptions(); 37 | await this._subscribeEvents(); 38 | } 39 | 40 | public triggerConnectivityTest(): void { 41 | const testAccountName = "CONNECTIVITY_TEST_NO_ACTION_REQUIRED" 42 | this.promClient.increaseSlashedReports(testAccountName,testAccountName); 43 | } 44 | 45 | private async _initInstanceVariables(): Promise{ 46 | this.sessionIndex = await this.api.query.session.currentIndex(); 47 | this.currentEraIndex = await getActiveEraIndex(this.api); 48 | this.validatorActiveSet = await this.api.query.session.validators(); 49 | await this._initValidatorsControllers() 50 | } 51 | 52 | private async _initValidatorsControllers(): Promise{ 53 | for (const validator of this.validators) { 54 | const controller = await this.api.query.staking.bonded(validator.address) 55 | validator.controllerAddress = controller.unwrapOr("").toString() 56 | } 57 | } 58 | 59 | private async _handleNewHeadSubscriptions(): Promise { 60 | this.api.rpc.chain.subscribeNewHeads(async (header) => { 61 | this._producerHandler(header); 62 | this._validatorStatusHandler(); 63 | this._payeeChangeHandler(header); 64 | this._commissionChangeHandler(header); 65 | this._checkUnexpected(); 66 | }) 67 | } 68 | 69 | private async _checkUnexpected(): Promise { 70 | const tmp = await this.api.derive.staking.queryMulti(this.validators.map(v=>v.address),{withDestination:true,withPrefs:true}) 71 | const stakingMap = new Map() 72 | tmp.forEach(t=>stakingMap.set(t.accountId.toString(),t)) 73 | this.validators.forEach(v => { 74 | let isOkCommission = true 75 | const actualCommission = stakingMap.get(v.address).validatorPrefs.commission.toNumber() //expressed in ppb 76 | if(v.expected?.commission){ 77 | isOkCommission = false 78 | switch (this.cfg.commissionLogic) { 79 | case "lt": 80 | isOkCommission = actualCommission <= v.expected.commission*10000000 81 | break; 82 | default: 83 | isOkCommission = actualCommission == v.expected.commission*10000000 84 | } 85 | } 86 | if(isOkCommission) 87 | this.promClient.resetStatusValidatorCommissionUnexpected(v.name,v.address) 88 | else { 89 | this.logger.info(`Detected Unexpected commission for validator ${v.name}: expected percentage ${v.expected.commission}, actual in ppb ${actualCommission}`) 90 | this.promClient.setStatusValidatorCommissionUnexpected(v.name,v.address) 91 | } 92 | 93 | let isOkPayee = true 94 | const actualRewardDestination = stakingMap.get(v.address).rewardDestination 95 | if(v.expected?.payee){ 96 | isOkPayee = false 97 | switch (v.expected.payee) { 98 | case "Staked": 99 | isOkPayee = actualRewardDestination?.isStaked 100 | break; 101 | case "Stash": 102 | isOkPayee = actualRewardDestination?.isStash 103 | break; 104 | case "Controller": 105 | isOkPayee = actualRewardDestination?.isController 106 | break; 107 | case "None": 108 | isOkPayee = actualRewardDestination?.isNone 109 | break; 110 | default: 111 | isOkPayee = actualRewardDestination?.isAccount && v.expected.payee == actualRewardDestination?.asAccount.toString() 112 | break; 113 | } 114 | } 115 | if(isOkPayee){ 116 | this.promClient.resetStatusValidatorPayeeUnexpected(v.name,v.address) 117 | } else { 118 | this.logger.info(`Detected Unexpected payee for validator ${v.name}: expected ${v.expected.payee}, actual ${JSON.stringify(actualRewardDestination)}`) 119 | this.promClient.setStatusValidatorPayeeUnexpected(v.name,v.address) 120 | } 121 | }) 122 | } 123 | 124 | private async _subscribeEvents(): Promise { 125 | 126 | this.api.query.system.events((events) => { 127 | 128 | events.forEach(async (record) => { 129 | const { event } = record; 130 | 131 | if(this.api.events.staking.SlashReported.is(event)){ 132 | this._slashedEventHandler(event) 133 | } 134 | 135 | if(this.api.events.session.NewSession.is(event)){ 136 | await this._newSessionEventHandler() 137 | } 138 | }); 139 | }); 140 | } 141 | 142 | private async _producerHandler(header: Header): Promise { 143 | // get block author 144 | const hash = await this.api.rpc.chain.getBlockHash(header.number.toNumber()); 145 | const deriveHeader = await this.api.derive.chain.getHeader(hash); 146 | const author = deriveHeader.author; 147 | if (author) { 148 | const account = this.validators.find((producer) => producer.address == author.toString()); 149 | if (account) { 150 | this.logger.info(`New block produced by ${account.name}`); 151 | this.promClient.increaseBlocksProducedReports(account.name, account.address); 152 | } 153 | } 154 | } 155 | 156 | private async _validatorStatusHandler(): Promise { 157 | 158 | this.validators.forEach(async account => { 159 | 160 | const validatorActiveSetIndex = this.validatorActiveSet.indexOf(account.address) 161 | if ( validatorActiveSetIndex < 0 ) { 162 | this.logger.debug(`Target ${account.name} is not present in the validation active set of era ${this.currentEraIndex}`); 163 | this.promClient.setStatusOutOfActiveSet(account.name,account.address); 164 | } else { 165 | this.promClient.resetStatusOutOfActiveSet(account.name,account.address); 166 | } 167 | }) 168 | 169 | } 170 | 171 | private async _payeeChangeHandler(header: Header): Promise { 172 | 173 | const currentBlock = header.number.unwrap() 174 | const blockHash = await this.api.rpc.chain.getBlockHash(currentBlock) 175 | const block = await this.api.rpc.chain.getBlock(blockHash) 176 | 177 | block.block.extrinsics.forEach( async (extrinsic) => { 178 | 179 | const { signer } = extrinsic; 180 | if(this.api.tx.staking.setPayee.is(extrinsic) || this.api.tx.staking.bond.is(extrinsic)){ 181 | this._handlePayeeChangeDetection(signer) 182 | } 183 | else if(this.api.tx.utility.batch.is(extrinsic) || this.api.tx.utility.batchAll.is(extrinsic)){ 184 | //this.logger.debug(`detected new utility > batch extrinsic`) 185 | const { signer, method: { args } } = extrinsic; 186 | for (const callAny of args[0] as any) { 187 | const call = this.api.registry.createType('Call',callAny) 188 | if(this.api.tx.staking.setPayee.is(call) || this.api.tx.staking.bond.is(call)){ 189 | this._handlePayeeChangeDetection(signer) 190 | } 191 | } 192 | } 193 | }) 194 | } 195 | 196 | private _handlePayeeChangeDetection(signer: Address): void{ 197 | for (const validator of this.validators) { 198 | if(signer.toString() == validator.address || signer.toString() == validator.controllerAddress){ 199 | this.logger.info(`Found setPayee or bond extrinsic for validator ${validator.name}`) 200 | this.promClient.increasePayeeChangedReports(validator.name, validator.address) 201 | } 202 | } 203 | } 204 | 205 | private async _commissionChangeHandler(header: Header): Promise { 206 | 207 | const currentBlock = header.number.unwrap() 208 | const blockHash = await this.api.rpc.chain.getBlockHash(currentBlock) 209 | const block = await this.api.rpc.chain.getBlock(blockHash) 210 | 211 | block.block.extrinsics.forEach( async (extrinsic) => { 212 | 213 | const { signer } = extrinsic; 214 | if(this.api.tx.staking.validate.is(extrinsic)){ 215 | this._handleCommissionChangeDetection(signer) 216 | } 217 | else if(this.api.tx.utility.batch.is(extrinsic) || this.api.tx.utility.batchAll.is(extrinsic)){ 218 | //this.logger.debug(`detected new utility > batch extrinsic`) 219 | const { signer, method: { args } } = extrinsic; 220 | for (const callAny of args[0] as any) { 221 | const call = this.api.registry.createType('Call',callAny) 222 | if(this.api.tx.staking.validate.is(call)){ 223 | this._handleCommissionChangeDetection(signer) 224 | } 225 | } 226 | } 227 | }) 228 | } 229 | 230 | private _handleCommissionChangeDetection(signer: Address): void{ 231 | for (const validator of this.validators) { 232 | if(signer.toString() == validator.address || signer.toString() == validator.controllerAddress){ 233 | this.logger.info(`Found validate extrinsic for validator ${validator.name}`) 234 | this.promClient.increaseCommissionChangedReports(validator.name, validator.address) 235 | } 236 | } 237 | } 238 | 239 | private _slashedEventHandler(event: Event): void { 240 | const validator = event.data[0].toString(); 241 | 242 | this.logger.debug(`${validator} has been reported for Slash`); 243 | const account = this.validators.find((subject) => subject.address == validator); 244 | 245 | if (account) { 246 | this.logger.info(`Really bad... Target ${account.name} has been reported for Slash`); 247 | this.promClient.increaseSlashedReports(account.name, account.address); 248 | } 249 | } 250 | 251 | private async _newSessionEventHandler(): Promise { 252 | this.sessionIndex = await this.api.query.session.currentIndex(); // TODO improve, check if it is present in event 253 | 254 | const newEraIndex = await getActiveEraIndex(this.api); 255 | if( newEraIndex > this.currentEraIndex ){ 256 | await this._newEraHandler(newEraIndex) 257 | } 258 | } 259 | 260 | private async _newEraHandler(newEraIndex: number): Promise{ 261 | this.currentEraIndex = newEraIndex; 262 | this.validatorActiveSet = await this.api.query.session.validators(); 263 | await this._initValidatorsControllers(); 264 | } 265 | 266 | private _initCounterMetrics(): void { 267 | this._initBlocksProducedMetrics(); 268 | this._initSlashedReportsMetrics() 269 | this._initPayeeChangedMetrics(); 270 | this._initCommissionChangedMetrics(); 271 | } 272 | 273 | private _initBlocksProducedMetrics(): void { 274 | this.validators.forEach((account) => { 275 | // always increase counters even the first time, so that we initialize the time series 276 | // https://github.com/prometheus/prometheus/issues/1673 277 | this.promClient.increaseBlocksProducedReports(account.name, account.address) 278 | }); 279 | } 280 | 281 | private _initSlashedReportsMetrics(): void { 282 | this.validators.forEach((account) => { 283 | // always increase counters even the first time, so that we initialize the time series 284 | // https://github.com/prometheus/prometheus/issues/1673 285 | this.promClient.increaseSlashedReports(account.name, account.address); 286 | }); 287 | } 288 | 289 | private _initPayeeChangedMetrics(): void { 290 | this.validators.forEach((account) => { 291 | // always increase counters even the first time, so that we initialize the time series 292 | // https://github.com/prometheus/prometheus/issues/1673 293 | this.promClient.increasePayeeChangedReports(account.name, account.address) 294 | }); 295 | } 296 | 297 | private _initCommissionChangedMetrics(): void { 298 | this.validators.forEach((account) => { 299 | // always increase counters even the first time, so that we initialize the time series 300 | // https://github.com/prometheus/prometheus/issues/1673 301 | this.promClient.increaseCommissionChangedReports(account.name, account.address) 302 | }); 303 | } 304 | 305 | } 306 | -------------------------------------------------------------------------------- /src/types.ts: -------------------------------------------------------------------------------- 1 | import { SessionIndex, ValidatorId } from '@polkadot/types/interfaces'; 2 | import { Vec } from '@polkadot/types/codec'; 3 | 4 | export interface Subscribable { 5 | name: string; 6 | address: string; 7 | controllerAddress?: string; 8 | expected?: { 9 | commission?: number; 10 | payee?: string; 11 | }; 12 | } 13 | 14 | export interface InputConfig { 15 | logLevel: string; 16 | environment: string; 17 | port: number; 18 | endpoint: string; 19 | validators?: Array; 20 | commissionLogic?: string; 21 | validatorsFromGit?: { 22 | enabled: boolean; 23 | platform: string; 24 | private: { 25 | enabled: boolean; 26 | apiToken: string; 27 | }; 28 | network: string; 29 | url: string; 30 | }; 31 | } 32 | 33 | export interface InputConfigFromGit { 34 | Kusama: Array; 35 | Polkadot: Array; 36 | } 37 | 38 | export interface PromClient { 39 | increaseBlocksProducedReports(name: string, address: string): void; 40 | increaseSlashedReports(name: string, address: string): void; 41 | setStatusOutOfActiveSet(name: string, address: string): void; 42 | resetStatusOutOfActiveSet(name: string, address: string): void; 43 | increasePayeeChangedReports(name: string, address: string): void; 44 | increaseCommissionChangedReports(name: string, address: string): void; 45 | setStatusValidatorPayeeUnexpected(name: string, address: string): void; 46 | resetStatusValidatorPayeeUnexpected(name: string, address: string): void; 47 | setStatusValidatorCommissionUnexpected(name: string, address: string): void; 48 | resetStatusValidatorCommissionUnexpected(name: string, address: string): void; 49 | } 50 | 51 | interface LabelMap { 52 | alertname: string; 53 | severity: string; 54 | } 55 | 56 | interface Annotation { 57 | description: string; 58 | } 59 | 60 | interface Alert { 61 | status: string; 62 | labels: LabelMap; 63 | annotations: Annotation; 64 | } 65 | 66 | export interface MatrixbotMsg { 67 | receiver: string; 68 | status: string; 69 | alerts: Array; 70 | version: string; 71 | } 72 | 73 | export interface ValidatorImOnlineParameters { 74 | sessionIndex: SessionIndex; 75 | eraIndex: number; 76 | validatorActiveSet: Vec; 77 | } 78 | -------------------------------------------------------------------------------- /src/utils.ts: -------------------------------------------------------------------------------- 1 | /*eslint @typescript-eslint/no-use-before-define: ["error", { "variables": false }]*/ 2 | 3 | import { ApiPromise } from '@polkadot/api'; 4 | import { LoggerSingleton } from './logger'; 5 | 6 | const logger = LoggerSingleton.getInstance() 7 | 8 | export const getActiveEraIndex = async (api: ApiPromise): Promise => { 9 | return (await api.query.staking.activeEra()).toJSON()['index']; 10 | } 11 | 12 | export async function asyncForEach(array: Array, callback: (arg0: T, arg1: number, arg2: Array) => void): Promise { 13 | for (let index = 0; index < array.length; index++) { 14 | await callback(array[index], index, array); 15 | } 16 | } -------------------------------------------------------------------------------- /test/mocks.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable @typescript-eslint/no-empty-function */ 2 | /* eslint-disable @typescript-eslint/no-unused-vars */ 3 | 4 | import { PromClient } from "../src/types"; 5 | 6 | 7 | export class PrometheusMock implements PromClient { 8 | private _blocksProducedReports = 0; 9 | private _slashedReports = 0; 10 | private _stateOutOfActiveSet = 0; 11 | private _payeeChangedReports = 0; 12 | private _commissionChangedReports = 0; 13 | private _statePayeeUnexpected = 0; 14 | private _stateCommissionUnexpected = 0; 15 | 16 | increaseBlocksProducedReports(name: string, address: string): void { 17 | this._blocksProducedReports++; 18 | } 19 | 20 | increaseSlashedReports(name: string, address: string): void { 21 | this._slashedReports++; 22 | } 23 | 24 | setStatusOutOfActiveSet(name: string): void { 25 | this._stateOutOfActiveSet = 1 26 | } 27 | resetStatusOutOfActiveSet(name: string): void { 28 | this._stateOutOfActiveSet = 0 29 | } 30 | increasePayeeChangedReports(name: string, address: string): void { 31 | this._payeeChangedReports++; 32 | } 33 | increaseCommissionChangedReports(name: string, address: string): void { 34 | this._commissionChangedReports++; 35 | } 36 | 37 | setStatusValidatorPayeeUnexpected(name: string): void { 38 | this._statePayeeUnexpected = 1 39 | } 40 | resetStatusValidatorPayeeUnexpected(name: string): void { 41 | this._statePayeeUnexpected = 0 42 | } 43 | setStatusValidatorCommissionUnexpected(name: string): void { 44 | this._stateCommissionUnexpected = 1 45 | } 46 | resetStatusValidatorCommissionUnexpected(name: string): void { 47 | this._stateCommissionUnexpected = 0 48 | } 49 | 50 | 51 | get blocksProducedReports(): number { 52 | return this._blocksProducedReports; 53 | } 54 | get slashedReports(): number { 55 | return this._slashedReports; 56 | } 57 | get statusOutOfActiveSet(): number { 58 | return this._stateOutOfActiveSet; 59 | } 60 | get payeeChangedReports(): number { 61 | return this._payeeChangedReports; 62 | } 63 | get commissionChangedReports(): number { 64 | return this._commissionChangedReports; 65 | } 66 | get statusPayeeUnexpected(): number { 67 | return this._statePayeeUnexpected; 68 | } 69 | get statusCommissionUnexpected(): number { 70 | return this._stateCommissionUnexpected; 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /test/prometheus/alertrules.yaml: -------------------------------------------------------------------------------- 1 | rule_files: 2 | - /dev/stdin 3 | 4 | evaluation_interval: 1m 5 | 6 | tests: 7 | - interval: 1m 8 | input_series: 9 | - series: 'polkadot_validator_out_of_active_set_state{network="kusama",name="node0",address="Gt6HqWBhdu4Sy1u8ASTbS1qf2Ac5gwdegwr8tWN8saMxPt5",environment="production"}' 10 | values: '0 0 1 1 1 0 0+0x190 1+0x20' 11 | - series: 'polkadot_validator_payee_changed_reports{network="kusama",name="node0",address="Gt6HqWBhdu4Sy1u8ASTbS1qf2Ac5gwdegwr8tWN8saMxPt5",environment="production"}' 12 | values: '0 0 1 1 1 1 1' 13 | - series: 'polkadot_validator_commission_changed_reports{network="kusama",name="node0",address="Gt6HqWBhdu4Sy1u8ASTbS1qf2Ac5gwdegwr8tWN8saMxPt5",environment="production"}' 14 | values: '0 0 1 1 1 1 1' 15 | - series: 'polkadot_validator_slashed_reports{network="kusama",name="node0",address="Gt6HqWBhdu4Sy1u8ASTbS1qf2Ac5gwdegwr8tWN8saMxPt5",environment="production"}' 16 | values: '0 1 0 0 0 0 0' 17 | - series: 'polkadot_validator_blocks_produced{network="kusama",name="node0",address="Gt6HqWBhdu4Sy1u8ASTbS1qf2Ac5gwdegwr8tWN8saMxPt5",environment="production"}' 18 | values: '1 1+0x190' # more than 3 hour 19 | 20 | alert_rule_test: 21 | # Test ValidatorOutOfActiveSet alert 22 | - eval_time: 3m # Values: 0 0 1 1 23 | alertname: ValidatorOutOfActiveSet 24 | exp_alerts: 25 | - eval_time: 4m # Values: 0 0 1 1 1 26 | alertname: ValidatorOutOfActiveSet 27 | exp_alerts: 28 | - exp_labels: 29 | severity: info 30 | origin: cluster 31 | name: node0 32 | address: Gt6HqWBhdu4Sy1u8ASTbS1qf2Ac5gwdegwr8tWN8saMxPt5 33 | network: kusama 34 | environment: production 35 | exp_annotations: 36 | message: 'Target node0 is not present in the current validation active set because it has not been selected by Phragmen' 37 | 38 | # Test ValidatorRewardDestinationChanged alert 39 | - eval_time: 2m # Values: 0 0 1 40 | alertname: ValidatorRewardDestinationChanged 41 | exp_alerts: 42 | - eval_time: 3m # Values: 0 0 1 1 43 | alertname: ValidatorRewardDestinationChanged 44 | exp_alerts: 45 | - exp_labels: 46 | severity: warning 47 | origin: cluster 48 | name: node0 49 | address: Gt6HqWBhdu4Sy1u8ASTbS1qf2Ac5gwdegwr8tWN8saMxPt5 50 | network: kusama 51 | environment: production 52 | exp_annotations: 53 | message: 'Target node0 may have changed his reward destination recently, please double check. This message is going to RESOLVE by itself soon.' 54 | - eval_time: 7m # Values: 0 0 1 1 1 1 1 55 | alertname: ValidatorRewardDestinationChanged 56 | exp_alerts: 57 | 58 | # Test ValidatorCommissionRateChanged alert 59 | - eval_time: 2m # Values: 0 0 1 60 | alertname: ValidatorCommissionRateChanged 61 | exp_alerts: 62 | - eval_time: 3m # Values: 0 0 1 1 63 | alertname: ValidatorCommissionRateChanged 64 | exp_alerts: 65 | - exp_labels: 66 | severity: warning 67 | origin: cluster 68 | name: node0 69 | address: Gt6HqWBhdu4Sy1u8ASTbS1qf2Ac5gwdegwr8tWN8saMxPt5 70 | network: kusama 71 | environment: production 72 | exp_annotations: 73 | message: 'Target node0 may have changed his commission rate recently, please double check. This message is going to RESOLVE by itself soon.' 74 | - eval_time: 7m # Values: 0 0 1 1 1 1 1 75 | alertname: ValidatorCommissionRateChanged 76 | exp_alerts: 77 | 78 | # Test ValidatorSlashed alert 79 | - eval_time: 1m # Values: 0 0 80 | alertname: ValidatorSlashed 81 | exp_alerts: 82 | - eval_time: 2m # Values: 0 1 83 | alertname: ValidatorSlashed 84 | exp_alerts: 85 | - exp_labels: 86 | severity: critical 87 | origin: cluster 88 | name: node0 89 | address: Gt6HqWBhdu4Sy1u8ASTbS1qf2Ac5gwdegwr8tWN8saMxPt5 90 | network: kusama 91 | environment: production 92 | exp_annotations: 93 | message: 'Target node0 was reported for Slash, an advanced double check can be carried here. This message is going to RESOLVE by itself soon.' 94 | runbook_url: "https://github.com/w3f/infrastructure/wiki/Validator-Slashed" 95 | - eval_time: 6m # Values: 0 1 0 0 0 0 96 | alertname: ValidatorSlashed 97 | exp_alerts: 98 | 99 | # Test ProducerStallX alert 100 | - eval_time: 5m 101 | alertname: ProducerStallShort 102 | exp_alerts: 103 | - eval_time: 5m 104 | alertname: ProducerStallLong 105 | exp_alerts: 106 | - eval_time: 65m 107 | alertname: ProducerStallShort 108 | exp_alerts: 109 | - exp_labels: 110 | severity: warning 111 | origin: cluster 112 | name: node0 113 | address: Gt6HqWBhdu4Sy1u8ASTbS1qf2Ac5gwdegwr8tWN8saMxPt5 114 | network: kusama 115 | environment: production 116 | exp_annotations: 117 | message: 'Blocks were not produced for 1 hour by node0' 118 | runbook_url: "https://github.com/w3f/infrastructure/wiki/Producer-Stall" 119 | - eval_time: 65m 120 | alertname: ProducerStallLong 121 | - eval_time: 185m 122 | alertname: ProducerStallShort 123 | exp_alerts: 124 | - exp_labels: 125 | severity: warning 126 | origin: cluster 127 | name: node0 128 | address: Gt6HqWBhdu4Sy1u8ASTbS1qf2Ac5gwdegwr8tWN8saMxPt5 129 | network: kusama 130 | environment: production 131 | exp_annotations: 132 | message: 'Blocks were not produced for 1 hour by node0' 133 | runbook_url: "https://github.com/w3f/infrastructure/wiki/Producer-Stall" 134 | - eval_time: 185m 135 | alertname: ProducerStallLong 136 | exp_alerts: 137 | - exp_labels: 138 | severity: critical 139 | origin: cluster 140 | name: node0 141 | address: Gt6HqWBhdu4Sy1u8ASTbS1qf2Ac5gwdegwr8tWN8saMxPt5 142 | network: kusama 143 | environment: production 144 | exp_annotations: 145 | message: 'Blocks were not produced for 3 hours by node0' 146 | runbook_url: "https://github.com/w3f/infrastructure/wiki/Producer-Stall" 147 | - eval_time: 200m 148 | alertname: ProducerStallShort 149 | exp_alerts: 150 | - eval_time: 200m 151 | alertname: ProducerStallLong -------------------------------------------------------------------------------- /test/subscriber.ts: -------------------------------------------------------------------------------- 1 | import '@polkadot/api-augment'; //https://github.com/polkadot-js/api/issues/4450 2 | import { TestPolkadotRPC } from '@w3f/test-utils'; 3 | import { LoggerSingleton } from '../src/logger' 4 | import { should } from 'chai'; 5 | 6 | import { Subscriber } from '../src/subscriber'; 7 | import { Client } from '../src/client'; 8 | import { 9 | PrometheusMock 10 | } from './mocks'; 11 | 12 | import { cryptoWaitReady} from '@polkadot/util-crypto' 13 | import { Keyring } from '@polkadot/api'; 14 | import { KeyringPair } from '@polkadot/keyring/types'; 15 | 16 | should(); 17 | 18 | let alice: KeyringPair 19 | let keyring: Keyring 20 | 21 | const cfg = { 22 | logLevel: 'info', 23 | port: 3000, 24 | endpoint: 'some_endpoint', 25 | environment: 'test', 26 | validators: [{ 27 | name: 'Alice', 28 | address: 'GsvVmjr1CBHwQHw84pPHMDxgNY3iBLz6Qn7qS3CH8qPhrHz' 29 | }] 30 | }; 31 | 32 | const cfg2 = { 33 | logLevel: 'info', 34 | port: 3000, 35 | endpoint: 'some_endpoint', 36 | environment: 'test', 37 | validators: [{ 38 | name: 'NotActiveAccount', 39 | address: 'FoQJpPyadYccjavVdTWxpxU7rUEaYhfLCPwXgkfD6Zat9QP' 40 | }] 41 | }; 42 | 43 | const cfg3 = { 44 | logLevel: 'info', 45 | port: 3000, 46 | endpoint: 'some_endpoint', 47 | environment: 'test', 48 | validators: [{ 49 | name: 'Alice', 50 | address: 'GsvVmjr1CBHwQHw84pPHMDxgNY3iBLz6Qn7qS3CH8qPhrHz', 51 | expected: { 52 | commission: 17, 53 | payee: 'FoQJpPyadYccjavVdTWxpxU7rUEaYhfLCPwXgkfD6Zat9QP' 54 | } 55 | }] 56 | }; 57 | 58 | LoggerSingleton.setInstance("info") 59 | 60 | function delay(ms: number): Promise { 61 | return new Promise(resolve => setTimeout(resolve, ms)); 62 | } 63 | 64 | describe('with a started WASM interface', async () => { 65 | await cryptoWaitReady() 66 | keyring = new Keyring({ type: 'sr25519' }); 67 | alice = keyring.addFromUri('//Alice', { name: 'Alice default' }); 68 | }) 69 | 70 | describe('Subscriber cfg1, with a started network', async () => { 71 | const testRPC = new TestPolkadotRPC(); 72 | const prometheus = new PrometheusMock(); 73 | let subject: Subscriber; 74 | before(async () => { 75 | await testRPC.start(); 76 | cfg.endpoint = testRPC.endpoint() 77 | const api = await new Client(cfg).connect() 78 | subject = new Subscriber(cfg, api, prometheus); 79 | }); 80 | 81 | after(async () => { 82 | await testRPC.stop(); 83 | }); 84 | 85 | describe('with an started instance', () => { 86 | before(async () => { 87 | await subject.start(); 88 | }); 89 | 90 | describe('validator status', async () => { 91 | it('should record produced blocks...', async () => { 92 | await delay(6000); 93 | 94 | prometheus.blocksProducedReports.should.be.gt(1); //counters are init to 1 95 | prometheus.slashedReports.should.be.eq(1) //counters are init to 1 96 | prometheus.statusOutOfActiveSet.should.be.eq(0) 97 | prometheus.payeeChangedReports.should.be.eq(1) //counters are init to 1 98 | prometheus.commissionChangedReports.should.be.eq(1) //counters are init to 1 99 | prometheus.statusCommissionUnexpected.should.be.eq(0) 100 | prometheus.statusPayeeUnexpected.should.be.eq(0) 101 | }); 102 | 103 | it('should detect a payee change attempt...', async () => { 104 | await delay(6000); 105 | 106 | const current = prometheus.payeeChangedReports 107 | 108 | const call = testRPC.api().tx.staking.setPayee({Staked:{}}) //About Staked: not so relevant what the value is, it will be detected anyway 109 | await call.signAndSend(alice) 110 | 111 | await delay(6000); 112 | 113 | prometheus.payeeChangedReports.should.be.eq(current+1) 114 | }); 115 | 116 | it('should detect a payee change attempt 2...', async () => { 117 | await delay(6000); 118 | 119 | const current = prometheus.payeeChangedReports 120 | 121 | const call = testRPC.api().tx.staking.bond(cfg.validators[0].address,1,{Stash:{}}) //About Stash: not so relevant what the value is, it will be detected anyway 122 | await call.signAndSend(alice) 123 | 124 | await delay(6000); 125 | 126 | prometheus.payeeChangedReports.should.be.eq(current+1) 127 | }); 128 | it('should detect a commission rate change attempt...', async () => { 129 | await delay(6000); 130 | 131 | const current = prometheus.commissionChangedReports 132 | 133 | const call = testRPC.api().tx.staking.validate({commission: 100000000}) //10 percent in ppb 134 | await call.signAndSend(alice) 135 | 136 | await delay(6000); 137 | 138 | prometheus.commissionChangedReports.should.be.eq(current+1) 139 | }); 140 | }); 141 | 142 | }); 143 | }); 144 | 145 | describe('Subscriber cfg2, with a started network', () => { 146 | const testRPC = new TestPolkadotRPC(); 147 | const prometheus = new PrometheusMock(); 148 | let subject: Subscriber; 149 | before(async () => { 150 | await testRPC.start(); 151 | cfg2.endpoint = testRPC.endpoint() 152 | const api = await new Client(cfg2).connect() 153 | subject = new Subscriber(cfg2, api, prometheus); 154 | }); 155 | 156 | after(async () => { 157 | await testRPC.stop(); 158 | }); 159 | 160 | describe('with an started instance', () => { 161 | before(async () => { 162 | await subject.start(); 163 | }); 164 | 165 | describe('validator status', async () => { 166 | it('should NOT record blocks produced...', async () => { 167 | await delay(6000); 168 | 169 | prometheus.blocksProducedReports.should.be.eq(1); //counters are init to 1 170 | prometheus.slashedReports.should.be.eq(1) //counters are init to 1 171 | prometheus.statusOutOfActiveSet.should.be.eq(1) 172 | prometheus.payeeChangedReports.should.be.eq(1) //counters are init to 1 173 | prometheus.commissionChangedReports.should.be.eq(1) //counters are init to 1 174 | }); 175 | it('should NOT detect a payee change attempt, Alice is not under monitoring...', async () => { 176 | await delay(6000); 177 | 178 | const call = testRPC.api().tx.staking.bond(alice.address,1,{Stash:{}}) //About Stash: not so relevant what the value is 179 | await call.signAndSend(alice) 180 | 181 | await delay(6000); 182 | 183 | prometheus.payeeChangedReports.should.be.eq(1) //counters are init to 1 184 | }); 185 | it('should NOT detect a commission rate change attempt, Alice is not under monitoring...', async () => { 186 | await delay(6000); 187 | 188 | const call = testRPC.api().tx.staking.validate({commission: 10}) 189 | await call.signAndSend(alice) 190 | 191 | await delay(6000); 192 | 193 | prometheus.commissionChangedReports.should.be.eq(1) //counters are init to 1 194 | }); 195 | }); 196 | }); 197 | }); 198 | 199 | describe('Subscriber cfg3, with a started network', async () => { 200 | const testRPC = new TestPolkadotRPC(); 201 | const prometheus = new PrometheusMock(); 202 | let subject: Subscriber; 203 | before(async () => { 204 | await testRPC.start(); 205 | cfg3.endpoint = testRPC.endpoint() 206 | const api = await new Client(cfg3).connect() 207 | subject = new Subscriber(cfg3, api, prometheus); 208 | }); 209 | 210 | after(async () => { 211 | await testRPC.stop(); 212 | }); 213 | 214 | describe('with an started instance', () => { 215 | before(async () => { 216 | await subject.start(); 217 | }); 218 | 219 | describe('validator status', async () => { 220 | it('should detected an unexpected behaviour...', async () => { 221 | await delay(6000); 222 | 223 | prometheus.statusCommissionUnexpected.should.be.eq(1) 224 | prometheus.statusPayeeUnexpected.should.be.eq(1) 225 | }); 226 | 227 | it('should detect an expected payee resolution...', async () => { 228 | await delay(6000); 229 | 230 | const current = prometheus.payeeChangedReports 231 | 232 | const call = testRPC.api().tx.staking.setPayee({Account: cfg3.validators[0].expected.payee}) 233 | await call.signAndSend(alice) 234 | 235 | await delay(6000); 236 | 237 | prometheus.statusPayeeUnexpected.should.be.eq(0) 238 | prometheus.payeeChangedReports.should.be.eq(current+1) 239 | }); 240 | 241 | it('should detect an expected commission resolution...', async () => { 242 | await delay(6000); 243 | 244 | const current = prometheus.commissionChangedReports 245 | 246 | const call = testRPC.api().tx.staking.validate({commission: cfg3.validators[0].expected.commission*10000000}) //percentage to ppb conversion 247 | await call.signAndSend(alice) 248 | 249 | await delay(6000); 250 | 251 | 252 | prometheus.statusCommissionUnexpected.should.be.eq(0) 253 | prometheus.commissionChangedReports.should.be.eq(current+1) 254 | }); 255 | }); 256 | 257 | }); 258 | }); 259 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "esModuleInterop": true, 4 | "declaration": true, 5 | "noImplicitAny": false, 6 | "module": "commonjs", 7 | "target": "es6", 8 | "skipLibCheck": true, 9 | "outDir": "./dist", 10 | "experimentalDecorators": true, 11 | "emitDecoratorMetadata": true 12 | }, 13 | "exclude": [ 14 | "node_modules", 15 | "**/*.spec.ts", 16 | "test/**/*", 17 | "e2e-test/**/*", 18 | "dist/**/*" 19 | ], 20 | "_comment": "The polkadot-watcher app contains typescript errors" 21 | } 22 | --------------------------------------------------------------------------------