├── .dockerignore ├── .github ├── release-drafter.yml └── workflows │ ├── docker.yaml │ ├── pr-title.yaml │ ├── release-drafter.yaml │ ├── release.yaml │ └── test.yaml ├── .gitignore ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── assets └── cosmos-validator-watcher-screenshot.jpg ├── go.mod ├── go.sum ├── main.go ├── pkg ├── app │ ├── cmd.go │ ├── debug.go │ ├── flags.go │ ├── http.go │ └── run.go ├── crypto │ ├── bls12_318.go │ ├── bn254.go │ └── utils.go ├── metrics │ ├── metrics.go │ ├── metrics_test.go │ ├── utils.go │ └── utils_test.go ├── rpc │ ├── node.go │ └── pool.go ├── watcher │ ├── babylon.go │ ├── block.go │ ├── block_test.go │ ├── block_types.go │ ├── commissions.go │ ├── commissions_test.go │ ├── slashing.go │ ├── slashing_test.go │ ├── status.go │ ├── types.go │ ├── types_test.go │ ├── upgrade.go │ ├── upgrade_test.go │ ├── validators.go │ ├── validators_test.go │ ├── votes.go │ └── votes_test.go └── webhook │ └── webhook.go └── tests └── e2e.go /.dockerignore: -------------------------------------------------------------------------------- 1 | /.git 2 | /.github 3 | /Dockerfile 4 | -------------------------------------------------------------------------------- /.github/release-drafter.yml: -------------------------------------------------------------------------------- 1 | name-template: 'v$RESOLVED_VERSION' 2 | tag-template: 'v$RESOLVED_VERSION' 3 | change-template: '* $TITLE (#$NUMBER) by @$AUTHOR' 4 | template: | 5 | $CHANGES 6 | 7 | **Full Changelog**: https://github.com/$OWNER/$REPOSITORY/compare/$PREVIOUS_TAG...v$RESOLVED_VERSION 8 | 9 | sort-direction: ascending 10 | 11 | categories: 12 | - title: '⚠️ BREAKING CHANGES' 13 | label: 'breaking' 14 | - title: '💫 Features' 15 | label: 'feature' 16 | - title: '🛠️ Bug fixes' 17 | label: 'fix' 18 | - title: '🕹️ Others' 19 | label: 'chore' 20 | 21 | version-resolver: 22 | # Major is not meant to be used at the moment. 23 | # Should be used with label breaking in the future. 24 | major: 25 | labels: 26 | - 'major' 27 | minor: 28 | labels: 29 | - 'breaking' 30 | - 'feature' 31 | - 'chore' 32 | patch: 33 | labels: 34 | - 'fix' 35 | 36 | exclude-labels: 37 | - 'skip-changelog' 38 | 39 | autolabeler: 40 | - label: 'breaking' 41 | title: 42 | - '/!:/i' 43 | - label: 'chore' 44 | title: 45 | - '/^chore/i' 46 | - label: 'fix' 47 | title: 48 | - '/^fix/i' 49 | - label: 'feature' 50 | title: 51 | - '/^feat/i' 52 | -------------------------------------------------------------------------------- /.github/workflows/docker.yaml: -------------------------------------------------------------------------------- 1 | name: Docker 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | paths: 8 | - 'pkg/**' 9 | - '*.go' 10 | - 'go.*' 11 | - Dockerfile 12 | - .github/workflows/docker.yaml 13 | pull_request: 14 | branches: 15 | - main 16 | paths: 17 | - 'pkg/**' 18 | - '*.go' 19 | - 'go.*' 20 | - Dockerfile 21 | - .github/workflows/docker.yaml 22 | release: 23 | types: 24 | - published 25 | 26 | env: 27 | REGISTRY: ghcr.io 28 | IMAGE_NAME: ${{ github.repository }} 29 | PLATFORMS: ${{ github.event_name == 'pull_request' && 'linux/amd64' || 'linux/amd64,linux/arm64' }} 30 | 31 | jobs: 32 | build: 33 | runs-on: ubuntu-latest 34 | permissions: 35 | contents: read 36 | packages: write 37 | 38 | steps: 39 | - name: Checkout repository 40 | uses: actions/checkout@v4 41 | with: 42 | fetch-depth: 0 43 | fetch-tags: true 44 | 45 | - name: Set up QEMU 46 | uses: docker/setup-qemu-action@v2 47 | 48 | - name: "Generate Build ID (main)" 49 | if: github.ref == 'refs/heads/main' && github.event_name == 'push' 50 | run: | 51 | branch=${GITHUB_REF##*/} 52 | sha=${GITHUB_SHA::8} 53 | ts=$(date +%s) 54 | echo "BUILD_ID=${branch}-${ts}-${sha}" >> $GITHUB_ENV 55 | 56 | - name: "Generate Build ID (PR)" 57 | if: github.event_name == 'pull_request' 58 | run: | 59 | echo "BUILD_ID=pr-${{ github.event.number }}-$GITHUB_RUN_ID" >> $GITHUB_ENV 60 | 61 | - name: "Generate Build ID (Release)" 62 | if: github.event_name == 'release' 63 | run: | 64 | echo "BUILD_ID=${GITHUB_REF##*/}" >> $GITHUB_ENV 65 | 66 | - name: 'Generate App Version' 67 | run: echo "VERSION=$(make version)" >> $GITHUB_ENV 68 | 69 | - name: Log in to the Container registry 70 | uses: docker/login-action@v2 71 | with: 72 | registry: ${{ env.REGISTRY }} 73 | username: ${{ github.actor }} 74 | password: ${{ secrets.GITHUB_TOKEN }} 75 | 76 | - name: Set up Docker Buildx 77 | uses: docker/setup-buildx-action@v2 78 | 79 | - name: Extract metadata (tags, labels) for Docker 80 | id: meta 81 | uses: docker/metadata-action@v4 82 | with: 83 | images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} 84 | tags: | 85 | type=ref,event=pr 86 | type=ref,event=branch 87 | type=raw,value=${{ env.BUILD_ID }} 88 | type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', github.event.repository.default_branch) }} 89 | 90 | - name: Build and push Docker image 91 | uses: docker/build-push-action@v4 92 | with: 93 | context: . 94 | push: true 95 | tags: ${{ steps.meta.outputs.tags }} 96 | labels: ${{ steps.meta.outputs.labels }} 97 | platforms: ${{ env.PLATFORMS }} 98 | cache-from: type=gha 99 | cache-to: type=gha,mode=max 100 | build-args: | 101 | VERSION=${{ env.VERSION }} 102 | -------------------------------------------------------------------------------- /.github/workflows/pr-title.yaml: -------------------------------------------------------------------------------- 1 | name: 'Validate PR title' 2 | 3 | on: 4 | pull_request: 5 | branches: [ main ] 6 | types: [ opened, reopened, synchronize ] 7 | 8 | permissions: 9 | pull-requests: read 10 | statuses: write 11 | 12 | jobs: 13 | main: 14 | name: Validate PR title 15 | runs-on: ubuntu-latest 16 | steps: 17 | # Please look up the latest version from 18 | # https://github.com/amannn/action-semantic-pull-request/releases 19 | - uses: amannn/action-semantic-pull-request@v5 20 | env: 21 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 22 | with: 23 | types: | 24 | fix 25 | feat 26 | chore 27 | requireScope: false 28 | wip: true 29 | validateSingleCommit: false 30 | -------------------------------------------------------------------------------- /.github/workflows/release-drafter.yaml: -------------------------------------------------------------------------------- 1 | name: Release Drafter 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | pull_request: 7 | branches: [ main ] 8 | types: [ opened, reopened, synchronize ] 9 | 10 | permissions: 11 | contents: read 12 | 13 | jobs: 14 | update_release_draft: 15 | permissions: 16 | contents: write 17 | pull-requests: write 18 | runs-on: ubuntu-latest 19 | steps: 20 | - uses: release-drafter/release-drafter@v5 21 | env: 22 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 23 | -------------------------------------------------------------------------------- /.github/workflows/release.yaml: -------------------------------------------------------------------------------- 1 | name: Release 2 | 3 | on: 4 | release: 5 | types: [published] 6 | 7 | permissions: 8 | contents: write 9 | packages: write 10 | 11 | jobs: 12 | releases-matrix: 13 | name: Release Go Binary 14 | runs-on: ubuntu-latest 15 | strategy: 16 | fail-fast: false 17 | matrix: 18 | goos: [linux, windows, darwin] 19 | # goarch: ["386", amd64, arm64] 20 | goarch: [amd64, arm64] 21 | exclude: 22 | - goarch: "386" 23 | goos: darwin 24 | - goarch: arm64 25 | goos: windows 26 | steps: 27 | - uses: actions/checkout@v4 28 | with: 29 | fetch-depth: 0 30 | fetch-tags: true 31 | 32 | - uses: wangyoucao577/go-release-action@v1 33 | env: 34 | BUILD_FOLDER: . 35 | with: 36 | build_command: make build 37 | extra_files: LICENSE README.md 38 | github_token: ${{ secrets.GITHUB_TOKEN }} 39 | goos: ${{ matrix.goos }} 40 | goarch: ${{ matrix.goarch }} 41 | goversion: "1.22" 42 | md5sum: false 43 | -------------------------------------------------------------------------------- /.github/workflows/test.yaml: -------------------------------------------------------------------------------- 1 | name: Test 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | paths: 8 | - 'pkg/**' 9 | - '*.go' 10 | - 'go.*' 11 | - '.github/workflows/test.yaml' 12 | pull_request: 13 | paths: 14 | - 'pkg/**' 15 | - '*.go' 16 | - 'go.*' 17 | - '.github/workflows/test.yaml' 18 | 19 | env: 20 | GO_VERSION: "1.23" 21 | 22 | jobs: 23 | unit-tests: 24 | name: 'Unit tests' 25 | runs-on: ubuntu-latest 26 | steps: 27 | - uses: actions/checkout@v4 28 | 29 | - uses: actions/setup-go@v4 30 | with: 31 | go-version: ${{ env.GO_VERSION }} 32 | 33 | - id: go-cache-paths 34 | run: | 35 | echo "go-mod=$(go env GOMODCACHE)" >> $GITHUB_OUTPUT 36 | 37 | # Cache go mod cache to speedup deps downloads 38 | - uses: actions/cache@v3 39 | with: 40 | path: ${{ steps.go-cache-paths.outputs.go-mod }} 41 | key: ${{ runner.os }}-go-mod-${{ hashFiles('**/go.sum') }} 42 | restore-keys: ${{ runner.os }}-go-mod- 43 | 44 | - name: 'Run unit tests' 45 | run: | 46 | make test 47 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /build 2 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Builder 2 | FROM golang:1.23-alpine3.21 as builder 3 | 4 | ARG VERSION="0.0.0-build" 5 | ENV VERSION=$VERSION 6 | 7 | WORKDIR /go/src/app 8 | 9 | RUN apk add --no-cache \ 10 | bash build-base git make 11 | 12 | # Copy module files and download dependencies 13 | COPY go.mod ./ 14 | COPY go.sum ./ 15 | RUN go mod download 16 | 17 | # Then copy the rest of the source code and build 18 | COPY . ./ 19 | RUN make build 20 | 21 | # Final image 22 | FROM alpine:3.21 23 | RUN apk upgrade && apk add --no-cache bash curl 24 | 25 | RUN addgroup -g 1001 app 26 | RUN adduser -D -G app -u 1001 app 27 | 28 | COPY --from=builder /go/src/app/build/cosmos-validator-watcher / 29 | 30 | WORKDIR / 31 | ENTRYPOINT ["/cosmos-validator-watcher"] 32 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Kiln 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | BUILD_FOLDER ?= build 2 | BINARY_NAME ?= cosmos-validator-watcher 3 | PACKAGES ?= $(shell go list ./... | egrep -v "testutils" ) 4 | VERSION ?= $(shell git describe --tags) 5 | 6 | .PHONY: build 7 | build: 8 | @go build -o $(BUILD_FOLDER)/$(BINARY_NAME) -v -ldflags="-X 'main.Version=$(VERSION)'" 9 | 10 | .PHONY: test 11 | test: 12 | @go test -v $(PACKAGES) 13 | 14 | .PHONY: version 15 | version: 16 | @echo $(VERSION) 17 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Cosmos Validator Watcher 2 | 3 | [![License](https://img.shields.io/badge/license-MIT-blue)](https://opensource.org/licenses/MIT) 4 | 5 | **Cosmos Validator Watcher** is a Prometheus exporter to help you monitor missed blocks on 6 | any cosmos-based blockchains in real-time. 7 | 8 | Features: 9 | 10 | - Track when your validator **missed a block** (with solo option) 11 | - Check how many validators missed the signatures for each block 12 | - Track the current active set and check if your validator is **bonded** or **jailed** 13 | - Track the **staked amount** as well as the min seat price 14 | - Track **pending proposals** and check if your validator has voted (including proposal end time) 15 | - Expose **upgrade plan** to know when the next upgrade will happen (including pending proposals) 16 | - Trigger webhook when an upgrade happens 17 | 18 | ![Cosmos Validator Watcher Screenshot](assets/cosmos-validator-watcher-screenshot.jpg) 19 | 20 | ## ✨ Usage 21 | 22 | Example for cosmoshub using 2 public RPC nodes and tracking 4 validators (with custom aliases). 23 | 24 | ### Via compiled binary 25 | 26 | Compiled binary can be found on the [Releases page](https://github.com/kilnfi/cosmos-validator-watcher/releases). 27 | 28 | ```bash 29 | cosmos-validator-watcher \ 30 | --node https://cosmos-rpc.publicnode.com:443 \ 31 | --node https://cosmos-rpc.polkachu.com:443 \ 32 | --validator 3DC4DD610817606AD4A8F9D762A068A81E8741E2:kiln \ 33 | --validator 25445D0EB353E9050AB11EC6197D5DCB611986DB:allnodes \ 34 | --validator 9DF8E338C85E879BC84B0AAA28A08B431BD5B548:9df8e338 \ 35 | --validator ABC1239871ABDEBCDE761D718978169BCD019739:random-name 36 | ``` 37 | 38 | ### Via Docker 39 | 40 | Latest Docker image can be found on the [Packages page](https://github.com/kilnfi/cosmos-validator-watcher/pkgs/container/cosmos-validator-watcher). 41 | 42 | ```bash 43 | docker run --rm ghcr.io/kilnfi/cosmos-validator-watcher:latest \ 44 | --node https://cosmos-rpc.publicnode.com:443 \ 45 | --node https://cosmos-rpc.polkachu.com:443 \ 46 | --validator 3DC4DD610817606AD4A8F9D762A068A81E8741E2:kiln \ 47 | --validator 25445D0EB353E9050AB11EC6197D5DCB611986DB:allnodes \ 48 | --validator 9DF8E338C85E879BC84B0AAA28A08B431BD5B548:9df8e338 \ 49 | --validator ABC1239871ABDEBCDE761D718978169BCD019739:random-name 50 | ``` 51 | 52 | ### Available options 53 | 54 | ``` 55 | cosmos-validator-watcher --help 56 | 57 | NAME: 58 | cosmos-validator-watcher - Real-time Cosmos-based chains monitoring tool 59 | 60 | USAGE: 61 | cosmos-validator-watcher [global options] command [command options] [arguments...] 62 | 63 | COMMANDS: 64 | help, h Shows a list of commands or help for one command 65 | 66 | GLOBAL OPTIONS: 67 | --babylon enable babylon watcher (checkpoint votes & finality providers) (default: false) 68 | --chain-id value to ensure all nodes matches the specific network (dismiss to auto-detected) 69 | --debug shortcut for --log-level=debug (default: false) 70 | --denom value denom used in metrics label (eg. atom or uatom) 71 | --denom-exponent value denom exponent (eg. 6 for atom, 1 for uatom) (default: 0) 72 | --finality-provider value [ --finality-provider value ] list of finality providers to watch (requires --babylon) 73 | --http-addr value http server address (default: ":8080") 74 | --log-level value log level (debug, info, warn, error) (default: "info") 75 | --namespace value namespace for Prometheus metrics (default: "cosmos_validator_watcher") 76 | --no-color disable colored output (default: false) 77 | --no-commission disable calls to get validator commission (useful for chains without distribution module) (default: false) 78 | --no-gov disable calls to gov module (useful for consumer chains) (default: false) 79 | --no-slashing disable calls to slashing module (default: false) 80 | --no-staking disable calls to staking module (useful for consumer chains) (default: false) 81 | --no-upgrade disable calls to upgrade module (for chains created without the upgrade module) (default: false) 82 | --node value [ --node value ] rpc node endpoint to connect to (specify multiple for high availability) (default: "http://localhost:26657") 83 | --start-timeout value timeout to wait on startup for one node to be ready (default: 10s) 84 | --stop-timeout value timeout to wait on stop (default: 10s) 85 | --validator value [ --validator value ] validator address(es) to track (use :my-label to add a custom label in metrics & output) 86 | --webhook-custom-block value [ --webhook-custom-block value ] trigger a custom webhook at a given block number (experimental) 87 | --webhook-url value endpoint where to send upgrade webhooks (experimental) 88 | --x-gov value version of the gov module to use (v1|v1beta1) (default: "v1") 89 | --help, -h show help 90 | --version, -v print the version 91 | ``` 92 | 93 | 94 | ## ❇️ Endpoints 95 | 96 | - `/metrics` exposed Prometheus metrics (see next section) 97 | - `/ready` responds OK when at least one of the nodes is synced (ie. `.SyncInfo.catching_up` is `false`) 98 | - `/live` responds OK as soon as server is up & running correctly 99 | 100 | 101 | ## 📊 Prometheus metrics 102 | 103 | All metrics are by default prefixed by `cosmos_validator_watcher` but this can be changed through options. 104 | 105 | Metrics (without prefix) | Description 106 | --------------------------------|------------------------------------------------------------------------- 107 | `active_set` | Number of validators in the active set 108 | `block_height` | Latest known block height (all nodes mixed up) 109 | `commission` | Earned validator commission 110 | `consecutive_missed_blocks` | Number of consecutive missed blocks per validator (for a bonded validator) 111 | `downtime_jail_duration` | Duration of the jail period for a validator in seconds 112 | `empty_blocks` | Number of empty blocks (blocks with zero transactions) proposed by validator 113 | `is_bonded` | Set to 1 if the validator is bonded 114 | `is_jailed` | Set to 1 if the validator is jailed 115 | `min_signed_blocks_per_window` | Minimum number of blocks required to be signed per signing window 116 | `missed_blocks_window` | Number of missed blocks per validator for the current signing window (for a bonded validator) 117 | `missed_blocks` | Number of missed blocks per validator (for a bonded validator) 118 | `node_block_height` | Latest fetched block height for each node 119 | `node_synced` | Set to 1 is the node is synced (ie. not catching-up) 120 | `proposal_end_time` | Timestamp of the voting end time of a proposal 121 | `proposed_blocks` | Number of proposed blocks per validator (for a bonded validator) 122 | `rank` | Rank of the validator 123 | `seat_price` | Min seat price to be in the active set (ie. bonded tokens of the latest validator) 124 | `signed_blocks_window` | Number of blocks per signing window 125 | `skipped_blocks` | Number of blocks skipped (ie. not tracked) since start 126 | `slash_fraction_double_sign` | Slash penaltiy for double-signing 127 | `slash_fraction_downtime` | Slash penaltiy for downtime 128 | `solo_missed_blocks` | Number of missed blocks per validator, unless the block is missed by many other validators 129 | `tokens` | Number of staked tokens per validator 130 | `tracked_blocks` | Number of blocks tracked since start 131 | `transactions` | Number of transactions since start 132 | `upgrade_plan` | Block height of the upcoming upgrade (hard fork) 133 | `validated_blocks` | Number of validated blocks per validator (for a bonded validator) 134 | `vote` | Set to 1 if the validator has voted on a proposal 135 | 136 | 137 | ### Chain specific metrics 138 | 139 | **Babylon** (requires the `--babylon` flag). 140 | 141 | Metrics (without prefix) | Description 142 | ----------------------------------------------|------------------------------------------------------------------------- 143 | `babylon_epoch` | Babylon epoch 144 | `babylon_checkpoint_vote` | Count of checkpoint votes since start (equal to number of epochs) 145 | `babylon_committed_checkpoint_vote` | Number of committed checkpoint votes for a validator 146 | `babylon_missed_checkpoint_vote` | Number of missed checkpoint votes for a validator 147 | `babylon_consecutive_missed_checkpoint_vote` | Number of consecutive missed checkpoint votes for a validator 148 | `babylon_finality_votes` | Count of total finality provider slots since start 149 | `babylon_committed_finality_votes` | Number of votes for a finality provider 150 | `babylon_missed_finality_votes` | Number of missed votes for a finality provider 151 | `babylon_consecutive_missed_finality_votes` | Number of consecutive missed votes for a finality provider 152 | 153 | 154 | ### Grafana dashboard 155 | 156 | For an example of a Prometheus and Grafana dashboard setup using Docker Compose, you can refer to [21state/cosmos-watcher-stack](https://github.com/21state/cosmos-watcher-stack/). 157 | 158 | 159 | ## ❓FAQ 160 | 161 | ### Which blockchains are compatible? 162 | 163 | Any blockchains based on the cosmos-sdk should work: 164 | 165 | - cosmoshub 166 | - celestia 167 | - cronos 168 | - dydx 169 | - evmos 170 | - injective 171 | - kava 172 | - osmosis 173 | - persistence 174 | - dymension 175 | - zetachain 176 | - ... 177 | 178 | This app is using the [CometBFT library](https://github.com/cometbft/cometbft/) (successor of Tendermint) as well as the `x/staking` module from the [Cosmos-SDK](https://github.com/cosmos/cosmos-sdk). 179 | 180 | ### How to get your validator pubkey address? 181 | 182 | **Option 1**: use `tendermint show-validator` to get the pubkey and `debug pubkey` to convert to hex format. 183 | 184 | ```bash 185 | CLI_NAME=gaiad 186 | ADDRESS="$($CLI_NAME debug pubkey "$($CLI_NAME tendermint show-validator)" 2>&1 | grep "Address")" 187 | ADDRESS="${ADDRESS##* 0x}" 188 | ADDRESS="${ADDRESS##* }" 189 | echo "${ADDRESS^^}" 190 | ``` 191 | 192 | (replace `gaiad` by the binary name or the desired chain, eg. `evmosd`, `strided`, `injectived`, …). 193 | 194 | **Option 2**: use the `cosmos-validator-watcher debug consensus-key` sub command: 195 | 196 | ```bash 197 | cosmos-validator-watcher debug validator \ 198 | --node https://cosmos-rpc.publicnode.com:443 \ 199 | cosmosvaloper1uxlf7mvr8nep3gm7udf2u9remms2jyjqvwdul2 200 | ``` 201 | 202 | Notes: 203 | - the `--node` flag must be placed before the validator address) 204 | - this doesns't work for consumer chains (neutron, stride) since they don't rely on the `staking` module 205 | 206 | 207 | ## 📃 License 208 | 209 | [MIT License](LICENSE). 210 | -------------------------------------------------------------------------------- /assets/cosmos-validator-watcher-screenshot.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kilnfi/cosmos-validator-watcher/85bf55d162394cd55c2d3732f111da160d239181/assets/cosmos-validator-watcher-screenshot.jpg -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/kilnfi/cosmos-validator-watcher 2 | 3 | go 1.23.1 4 | 5 | toolchain go1.23.3 6 | 7 | require ( 8 | cosmossdk.io/math v1.4.0 9 | cosmossdk.io/x/upgrade v0.1.4 10 | github.com/avast/retry-go/v4 v4.6.0 11 | github.com/babylonlabs-io/babylon v0.18.2 12 | github.com/cometbft/cometbft v0.38.15 13 | github.com/cosmos/cosmos-sdk v0.50.9 14 | github.com/fatih/color v1.17.0 15 | github.com/gogo/protobuf v1.3.2 16 | github.com/prometheus/client_golang v1.20.5 17 | github.com/rs/zerolog v1.33.0 18 | github.com/samber/lo v1.39.0 19 | github.com/shopspring/decimal v1.4.0 20 | github.com/stretchr/testify v1.9.0 21 | github.com/urfave/cli/v2 v2.27.2 22 | golang.org/x/sync v0.8.0 23 | google.golang.org/grpc v1.67.1 24 | gotest.tools v2.2.0+incompatible 25 | ) 26 | 27 | require ( 28 | cosmossdk.io/api v0.7.5 // indirect 29 | cosmossdk.io/collections v0.4.0 // indirect 30 | cosmossdk.io/core v0.11.1 // indirect 31 | cosmossdk.io/depinject v1.0.0 // indirect 32 | cosmossdk.io/errors v1.0.1 // indirect 33 | cosmossdk.io/log v1.4.1 // indirect 34 | cosmossdk.io/store v1.1.0 // indirect 35 | cosmossdk.io/x/tx v0.13.4 // indirect 36 | filippo.io/edwards25519 v1.0.0 // indirect 37 | github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect 38 | github.com/99designs/keyring v1.2.1 // indirect 39 | github.com/DataDog/datadog-go v3.2.0+incompatible // indirect 40 | github.com/DataDog/zstd v1.5.5 // indirect 41 | github.com/aead/siphash v1.0.1 // indirect 42 | github.com/beorn7/perks v1.0.1 // indirect 43 | github.com/bgentry/speakeasy v0.1.1-0.20220910012023-760eaf8b6816 // indirect 44 | github.com/boljen/go-bitmap v0.0.0-20151001105940-23cd2fb0ce7d // indirect 45 | github.com/btcsuite/btcd v0.24.2 // indirect 46 | github.com/btcsuite/btcd/btcec/v2 v2.3.2 // indirect 47 | github.com/btcsuite/btcd/btcutil v1.1.6 // indirect 48 | github.com/btcsuite/btcd/chaincfg/chainhash v1.1.0 // indirect 49 | github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f // indirect 50 | github.com/cenkalti/backoff/v4 v4.2.0 // indirect 51 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 52 | github.com/cockroachdb/errors v1.11.3 // indirect 53 | github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce // indirect 54 | github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect 55 | github.com/cockroachdb/pebble v1.1.2 // indirect 56 | github.com/cockroachdb/redact v1.1.5 // indirect 57 | github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect 58 | github.com/cometbft/cometbft-db v0.15.0 // indirect 59 | github.com/cosmos/btcutil v1.0.5 // indirect 60 | github.com/cosmos/cosmos-db v1.0.2 // indirect 61 | github.com/cosmos/cosmos-proto v1.0.0-beta.5 // indirect 62 | github.com/cosmos/go-bip39 v1.0.0 // indirect 63 | github.com/cosmos/gogogateway v1.2.0 // indirect 64 | github.com/cosmos/gogoproto v1.7.0 // indirect 65 | github.com/cosmos/iavl v1.2.0 // indirect 66 | github.com/cosmos/ics23/go v0.10.0 // indirect 67 | github.com/cosmos/ledger-cosmos-go v0.13.3 // indirect 68 | github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect 69 | github.com/danieljoos/wincred v1.1.2 // indirect 70 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect 71 | github.com/decred/dcrd/crypto/blake256 v1.0.1 // indirect 72 | github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 // indirect 73 | github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f // indirect 74 | github.com/dgraph-io/badger/v4 v4.3.0 // indirect 75 | github.com/dgraph-io/ristretto v0.1.2-0.20240116140435-c67e07994f91 // indirect 76 | github.com/dustin/go-humanize v1.0.1 // indirect 77 | github.com/dvsekhvalnov/jose2go v1.6.0 // indirect 78 | github.com/emicklei/dot v1.6.1 // indirect 79 | github.com/felixge/httpsnoop v1.0.4 // indirect 80 | github.com/fsnotify/fsnotify v1.7.0 // indirect 81 | github.com/getsentry/sentry-go v0.27.0 // indirect 82 | github.com/go-kit/kit v0.13.0 // indirect 83 | github.com/go-kit/log v0.2.1 // indirect 84 | github.com/go-logfmt/logfmt v0.6.0 // indirect 85 | github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect 86 | github.com/gogo/googleapis v1.4.1 // indirect 87 | github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect 88 | github.com/golang/mock v1.6.0 // indirect 89 | github.com/golang/protobuf v1.5.4 // indirect 90 | github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect 91 | github.com/google/btree v1.1.3 // indirect 92 | github.com/google/flatbuffers v1.12.1 // indirect 93 | github.com/google/go-cmp v0.6.0 // indirect 94 | github.com/gorilla/handlers v1.5.2 // indirect 95 | github.com/gorilla/mux v1.8.1 // indirect 96 | github.com/gorilla/websocket v1.5.3 // indirect 97 | github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect 98 | github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect 99 | github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect 100 | github.com/hashicorp/go-hclog v1.5.0 // indirect 101 | github.com/hashicorp/go-immutable-radix v1.3.1 // indirect 102 | github.com/hashicorp/go-metrics v0.5.3 // indirect 103 | github.com/hashicorp/go-plugin v1.5.2 // indirect 104 | github.com/hashicorp/golang-lru v1.0.2 // indirect 105 | github.com/hashicorp/hcl v1.0.0 // indirect 106 | github.com/hashicorp/yamux v0.1.1 // indirect 107 | github.com/hdevalence/ed25519consensus v0.1.0 // indirect 108 | github.com/huandu/skiplist v1.2.0 // indirect 109 | github.com/iancoleman/strcase v0.3.0 // indirect 110 | github.com/improbable-eng/grpc-web v0.15.0 // indirect 111 | github.com/inconshreveable/mousetrap v1.1.0 // indirect 112 | github.com/jmhodges/levigo v1.0.0 // indirect 113 | github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23 // indirect 114 | github.com/klauspost/compress v1.17.9 // indirect 115 | github.com/kr/pretty v0.3.1 // indirect 116 | github.com/kr/text v0.2.0 // indirect 117 | github.com/kylelemons/godebug v1.1.0 // indirect 118 | github.com/linxGnu/grocksdb v1.9.3 // indirect 119 | github.com/magiconair/properties v1.8.7 // indirect 120 | github.com/mattn/go-colorable v0.1.13 // indirect 121 | github.com/mattn/go-isatty v0.0.20 // indirect 122 | github.com/mitchellh/go-testing-interface v1.14.1 // indirect 123 | github.com/mitchellh/mapstructure v1.5.0 // indirect 124 | github.com/mtibben/percent v0.2.1 // indirect 125 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 126 | github.com/oasisprotocol/curve25519-voi v0.0.0-20230904125328-1f23a7beb09a // indirect 127 | github.com/oklog/run v1.1.0 // indirect 128 | github.com/pelletier/go-toml/v2 v2.2.2 // indirect 129 | github.com/petermattis/goid v0.0.0-20240813172612-4fcff4a6cae7 // indirect 130 | github.com/pkg/errors v0.9.1 // indirect 131 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect 132 | github.com/prometheus/client_model v0.6.1 // indirect 133 | github.com/prometheus/common v0.60.1 // indirect 134 | github.com/prometheus/procfs v0.15.1 // indirect 135 | github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect 136 | github.com/rogpeppe/go-internal v1.12.0 // indirect 137 | github.com/rs/cors v1.11.1 // indirect 138 | github.com/russross/blackfriday/v2 v2.1.0 // indirect 139 | github.com/sagikazarmark/locafero v0.4.0 // indirect 140 | github.com/sagikazarmark/slog-shim v0.1.0 // indirect 141 | github.com/sasha-s/go-deadlock v0.3.5 // indirect 142 | github.com/sourcegraph/conc v0.3.0 // indirect 143 | github.com/spf13/afero v1.11.0 // indirect 144 | github.com/spf13/cast v1.7.0 // indirect 145 | github.com/spf13/cobra v1.8.1 // indirect 146 | github.com/spf13/pflag v1.0.5 // indirect 147 | github.com/spf13/viper v1.19.0 // indirect 148 | github.com/stretchr/objx v0.5.2 // indirect 149 | github.com/subosito/gotenv v1.6.0 // indirect 150 | github.com/supranational/blst v0.3.11 // indirect 151 | github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d // indirect 152 | github.com/tendermint/go-amino v0.16.0 // indirect 153 | github.com/tidwall/btree v1.7.0 // indirect 154 | github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect 155 | github.com/zondax/hid v0.9.2 // indirect 156 | github.com/zondax/ledger-go v0.14.3 // indirect 157 | go.etcd.io/bbolt v1.4.0-alpha.0.0.20240404170359-43604f3112c5 // indirect 158 | go.opencensus.io v0.24.0 // indirect 159 | go.uber.org/multierr v1.11.0 // indirect 160 | golang.org/x/crypto v0.28.0 // indirect 161 | golang.org/x/exp v0.0.0-20240604190554-fc45aab8b7f8 // indirect 162 | golang.org/x/net v0.30.0 // indirect 163 | golang.org/x/sys v0.26.0 // indirect 164 | golang.org/x/term v0.25.0 // indirect 165 | golang.org/x/text v0.19.0 // indirect 166 | google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de // indirect 167 | google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142 // indirect 168 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142 // indirect 169 | google.golang.org/protobuf v1.35.1 // indirect 170 | gopkg.in/ini.v1 v1.67.0 // indirect 171 | gopkg.in/yaml.v2 v2.4.0 // indirect 172 | gopkg.in/yaml.v3 v3.0.1 // indirect 173 | gotest.tools/v3 v3.5.1 // indirect 174 | nhooyr.io/websocket v1.8.6 // indirect 175 | pgregory.net/rapid v1.1.0 // indirect 176 | sigs.k8s.io/yaml v1.4.0 // indirect 177 | ) 178 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "os" 7 | 8 | "github.com/kilnfi/cosmos-validator-watcher/pkg/app" 9 | "github.com/rs/zerolog/log" 10 | "github.com/urfave/cli/v2" 11 | ) 12 | 13 | var Version = "v0.0.0-dev" // generated at build time 14 | 15 | func main() { 16 | app := &cli.App{ 17 | Name: "cosmos-validator-watcher", 18 | Usage: "Real-time Cosmos-based chains monitoring tool", 19 | Flags: app.Flags, 20 | Action: app.RunFunc, 21 | Commands: app.Commands, 22 | Version: Version, 23 | } 24 | 25 | if err := app.Run(os.Args); err != nil && !errors.Is(err, context.Canceled) { 26 | log.Error().Err(err).Msg("") 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /pkg/app/cmd.go: -------------------------------------------------------------------------------- 1 | package app 2 | 3 | import "github.com/urfave/cli/v2" 4 | 5 | var Commands = []*cli.Command{ 6 | { 7 | Name: "debug", 8 | Usage: "Debug utilities", 9 | Subcommands: []*cli.Command{ 10 | { 11 | Name: "consensus-key", 12 | Action: DebugConsensusKeyRun, 13 | Flags: Flags, 14 | }, 15 | { 16 | Name: "denom", 17 | Action: DebugDenomRun, 18 | Flags: Flags, 19 | }, 20 | { 21 | Name: "validator", 22 | Action: DebugValidatorRun, 23 | Flags: Flags, 24 | }, 25 | }, 26 | }, 27 | } 28 | -------------------------------------------------------------------------------- /pkg/app/debug.go: -------------------------------------------------------------------------------- 1 | package app 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/cometbft/cometbft/rpc/client/http" 7 | "github.com/cosmos/cosmos-sdk/client" 8 | "github.com/cosmos/cosmos-sdk/codec" 9 | bank "github.com/cosmos/cosmos-sdk/x/bank/types" 10 | staking "github.com/cosmos/cosmos-sdk/x/staking/types" 11 | "github.com/kilnfi/cosmos-validator-watcher/pkg/crypto" 12 | "github.com/urfave/cli/v2" 13 | ) 14 | 15 | func DebugDenomRun(cCtx *cli.Context) error { 16 | var ( 17 | ctx = cCtx.Context 18 | nodes = cCtx.StringSlice("node") 19 | ) 20 | 21 | if len(nodes) < 1 { 22 | return cli.Exit("at least one node must be specified", 1) 23 | } 24 | 25 | endpoint := nodes[0] 26 | 27 | rpcClient, err := http.New(endpoint, "/websocket") 28 | if err != nil { 29 | return fmt.Errorf("failed to create client: %w", err) 30 | } 31 | 32 | clientCtx := (client.Context{}).WithClient(rpcClient) 33 | queryClient := bank.NewQueryClient(clientCtx) 34 | 35 | resp, err := queryClient.DenomsMetadata(ctx, &bank.QueryDenomsMetadataRequest{}) 36 | if err != nil { 37 | return err 38 | } 39 | 40 | cdc := codec.NewLegacyAmino() 41 | j, err := cdc.MarshalJSONIndent(resp.Metadatas, "", " ") 42 | fmt.Println(string(j)) 43 | 44 | return nil 45 | } 46 | 47 | func DebugValidatorRun(cCtx *cli.Context) error { 48 | var ( 49 | ctx = cCtx.Context 50 | nodes = cCtx.StringSlice("node") 51 | ) 52 | 53 | if cCtx.Args().Len() < 1 { 54 | return cli.Exit("validator address must be specified", 1) 55 | } 56 | if len(nodes) < 1 { 57 | return cli.Exit("at least one node must be specified", 1) 58 | } 59 | 60 | endpoint := nodes[0] 61 | validatorAddr := cCtx.Args().First() 62 | 63 | rpcClient, err := http.New(endpoint, "/websocket") 64 | if err != nil { 65 | return fmt.Errorf("failed to create client: %w", err) 66 | } 67 | 68 | clientCtx := (client.Context{}).WithClient(rpcClient) 69 | queryClient := staking.NewQueryClient(clientCtx) 70 | 71 | resp, err := queryClient.Validator(ctx, &staking.QueryValidatorRequest{ 72 | ValidatorAddr: validatorAddr, 73 | }) 74 | if err != nil { 75 | return err 76 | } 77 | 78 | val := resp.Validator 79 | 80 | cdc := codec.NewLegacyAmino() 81 | j, err := cdc.MarshalJSONIndent(val, "", " ") 82 | if err != nil { 83 | return err 84 | } 85 | fmt.Println(string(j)) 86 | 87 | return nil 88 | } 89 | 90 | func DebugConsensusKeyRun(cCtx *cli.Context) error { 91 | var ( 92 | ctx = cCtx.Context 93 | nodes = cCtx.StringSlice("node") 94 | ) 95 | 96 | if cCtx.Args().Len() < 1 { 97 | return cli.Exit("validator address must be specified", 1) 98 | } 99 | if len(nodes) < 1 { 100 | return cli.Exit("at least one node must be specified", 1) 101 | } 102 | 103 | endpoint := nodes[0] 104 | validatorAddr := cCtx.Args().First() 105 | 106 | rpcClient, err := http.New(endpoint, "/websocket") 107 | if err != nil { 108 | return fmt.Errorf("failed to create client: %w", err) 109 | } 110 | 111 | clientCtx := (client.Context{}).WithClient(rpcClient) 112 | queryClient := staking.NewQueryClient(clientCtx) 113 | 114 | resp, err := queryClient.Validator(ctx, &staking.QueryValidatorRequest{ 115 | ValidatorAddr: validatorAddr, 116 | }) 117 | if err != nil { 118 | return err 119 | } 120 | 121 | val := resp.Validator 122 | address := crypto.PubKeyAddress(val.ConsensusPubkey) 123 | 124 | fmt.Println(address) 125 | 126 | return nil 127 | } 128 | -------------------------------------------------------------------------------- /pkg/app/flags.go: -------------------------------------------------------------------------------- 1 | package app 2 | 3 | import ( 4 | "sort" 5 | "time" 6 | 7 | "github.com/urfave/cli/v2" 8 | ) 9 | 10 | var Flags = []cli.Flag{ 11 | &cli.StringFlag{ 12 | Name: "chain-id", 13 | Usage: "to ensure all nodes matches the specific network (dismiss to auto-detected)", 14 | }, 15 | &cli.BoolFlag{ 16 | Name: "debug", 17 | Usage: "shortcut for --log-level=debug", 18 | }, 19 | &cli.StringFlag{ 20 | Name: "http-addr", 21 | Usage: "http server address", 22 | Value: ":8080", 23 | }, 24 | &cli.StringFlag{ 25 | Name: "log-level", 26 | Usage: "log level (debug, info, warn, error)", 27 | Value: "info", 28 | }, 29 | &cli.StringFlag{ 30 | Name: "namespace", 31 | Usage: "namespace for Prometheus metrics", 32 | Value: "cosmos_validator_watcher", 33 | }, 34 | &cli.BoolFlag{ 35 | Name: "no-color", 36 | Usage: "disable colored output", 37 | }, 38 | &cli.StringSliceFlag{ 39 | Name: "node", 40 | Usage: "rpc node endpoint to connect to (specify multiple for high availability)", 41 | Value: cli.NewStringSlice("http://localhost:26657"), 42 | }, 43 | &cli.BoolFlag{ 44 | Name: "no-gov", 45 | Usage: "disable calls to gov module (useful for consumer chains)", 46 | }, 47 | &cli.BoolFlag{ 48 | Name: "no-staking", 49 | Usage: "disable calls to staking module (useful for consumer chains)", 50 | }, 51 | &cli.BoolFlag{ 52 | Name: "no-slashing", 53 | Usage: "disable calls to slashing module", 54 | }, 55 | &cli.BoolFlag{ 56 | Name: "no-commission", 57 | Usage: "disable calls to get validator commission (useful for chains without distribution module)", 58 | }, 59 | &cli.BoolFlag{ 60 | Name: "no-upgrade", 61 | Usage: "disable calls to upgrade module (for chains created without the upgrade module)", 62 | }, 63 | &cli.StringFlag{ 64 | Name: "denom", 65 | Usage: "denom used in metrics label (eg. atom or uatom)", 66 | }, 67 | &cli.UintFlag{ 68 | Name: "denom-exponent", 69 | Usage: "denom exponent (eg. 6 for atom, 1 for uatom)", 70 | }, 71 | &cli.DurationFlag{ 72 | Name: "start-timeout", 73 | Usage: "timeout to wait on startup for one node to be ready", 74 | Value: 10 * time.Second, 75 | }, 76 | &cli.DurationFlag{ 77 | Name: "stop-timeout", 78 | Usage: "timeout to wait on stop", 79 | Value: 10 * time.Second, 80 | }, 81 | &cli.StringSliceFlag{ 82 | Name: "validator", 83 | Usage: "validator address(es) to track (use :my-label to add a custom label in metrics & output)", 84 | }, 85 | &cli.StringFlag{ 86 | Name: "webhook-url", 87 | Usage: "endpoint where to send upgrade webhooks (experimental)", 88 | }, 89 | &cli.StringSliceFlag{ 90 | Name: "webhook-custom-block", 91 | Usage: "trigger a custom webhook at a given block number (experimental)", 92 | }, 93 | &cli.StringFlag{ 94 | Name: "x-gov", 95 | Usage: "version of the gov module to use (v1|v1beta1)", 96 | Value: "v1", 97 | }, 98 | &cli.BoolFlag{ 99 | Name: "babylon", 100 | Usage: "enable babylon watcher (checkpoint votes & finality providers)", 101 | }, 102 | &cli.StringSliceFlag{ 103 | Name: "finality-provider", 104 | Usage: "list of finality providers to watch (requires --babylon)", 105 | }, 106 | } 107 | 108 | func init() { 109 | sort.SliceStable(Flags, func(i, j int) bool { 110 | return Flags[i].Names()[0] < Flags[j].Names()[0] 111 | }) 112 | } 113 | -------------------------------------------------------------------------------- /pkg/app/http.go: -------------------------------------------------------------------------------- 1 | package app 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/http" 7 | 8 | "github.com/prometheus/client_golang/prometheus" 9 | "github.com/prometheus/client_golang/prometheus/promhttp" 10 | ) 11 | 12 | type Probe func() bool 13 | 14 | var upProbe Probe = func() bool { return true } 15 | 16 | type HTTPServer struct { 17 | *http.Server 18 | } 19 | 20 | type HTTPMuxOption func(*http.ServeMux) 21 | 22 | func withProbe(path string, probe Probe) HTTPMuxOption { 23 | return func(mux *http.ServeMux) { 24 | mux.HandleFunc(path, probeHandler(probe)) 25 | } 26 | } 27 | 28 | func WithReadyProbe(probe Probe) HTTPMuxOption { 29 | return withProbe("/ready", probe) 30 | } 31 | 32 | func WithLiveProbe(probe Probe) HTTPMuxOption { 33 | return withProbe("/live", probe) 34 | } 35 | 36 | func WithMetrics(registry *prometheus.Registry) HTTPMuxOption { 37 | return func(mux *http.ServeMux) { 38 | mux.Handle("/metrics", promhttp.HandlerFor(registry, promhttp.HandlerOpts{})) 39 | } 40 | } 41 | 42 | func NewHTTPServer(addr string, options ...HTTPMuxOption) *HTTPServer { 43 | mux := http.NewServeMux() 44 | server := &HTTPServer{ 45 | Server: &http.Server{ 46 | Addr: addr, 47 | Handler: mux, 48 | }, 49 | } 50 | 51 | for _, option := range options { 52 | option(mux) 53 | } 54 | 55 | return server 56 | } 57 | 58 | func (s *HTTPServer) Run() error { 59 | return s.listenAndServe() 60 | } 61 | 62 | func (s *HTTPServer) Shutdown(ctx context.Context) error { 63 | return s.Server.Shutdown(ctx) 64 | } 65 | 66 | func (s *HTTPServer) listenAndServe() error { 67 | err := s.Server.ListenAndServe() 68 | 69 | if err != nil && err != http.ErrServerClosed { 70 | return fmt.Errorf("failed starting HTTP server: %w", err) 71 | } 72 | 73 | return nil 74 | } 75 | 76 | func probeHandler(probe Probe) http.HandlerFunc { 77 | return func(w http.ResponseWriter, r *http.Request) { 78 | if probe() { 79 | w.WriteHeader(http.StatusNoContent) 80 | } else { 81 | w.WriteHeader(http.StatusServiceUnavailable) 82 | } 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /pkg/app/run.go: -------------------------------------------------------------------------------- 1 | package app 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/url" 7 | "os" 8 | "os/signal" 9 | "strconv" 10 | "syscall" 11 | 12 | upgrade "cosmossdk.io/x/upgrade/types" 13 | "github.com/cometbft/cometbft/rpc/client/http" 14 | "github.com/cosmos/cosmos-sdk/client" 15 | "github.com/cosmos/cosmos-sdk/types/query" 16 | staking "github.com/cosmos/cosmos-sdk/x/staking/types" 17 | "github.com/fatih/color" 18 | "github.com/kilnfi/cosmos-validator-watcher/pkg/crypto" 19 | "github.com/kilnfi/cosmos-validator-watcher/pkg/metrics" 20 | "github.com/kilnfi/cosmos-validator-watcher/pkg/rpc" 21 | "github.com/kilnfi/cosmos-validator-watcher/pkg/watcher" 22 | "github.com/kilnfi/cosmos-validator-watcher/pkg/webhook" 23 | "github.com/rs/zerolog" 24 | "github.com/rs/zerolog/log" 25 | "github.com/samber/lo" 26 | "github.com/urfave/cli/v2" 27 | "golang.org/x/sync/errgroup" 28 | ) 29 | 30 | func RunFunc(cCtx *cli.Context) error { 31 | var ( 32 | ctx = cCtx.Context 33 | 34 | // Config flags 35 | chainID = cCtx.String("chain-id") 36 | debug = cCtx.Bool("debug") 37 | httpAddr = cCtx.String("http-addr") 38 | logLevel = cCtx.String("log-level") 39 | namespace = cCtx.String("namespace") 40 | noColor = cCtx.Bool("no-color") 41 | nodes = cCtx.StringSlice("node") 42 | noGov = cCtx.Bool("no-gov") 43 | noStaking = cCtx.Bool("no-staking") 44 | noUpgrade = cCtx.Bool("no-upgrade") 45 | noCommission = cCtx.Bool("no-commission") 46 | noSlashing = cCtx.Bool("no-slashing") 47 | denom = cCtx.String("denom") 48 | denomExpon = cCtx.Uint("denom-exponent") 49 | startTimeout = cCtx.Duration("start-timeout") 50 | stopTimeout = cCtx.Duration("stop-timeout") 51 | validators = cCtx.StringSlice("validator") 52 | webhookURL = cCtx.String("webhook-url") 53 | webhookCustomBlocks = cCtx.StringSlice("webhook-custom-block") 54 | xGov = cCtx.String("x-gov") 55 | 56 | // Babylon specific flags 57 | babylonEnabled = cCtx.Bool("babylon") 58 | finalityProviders = cCtx.StringSlice("finality-provider") 59 | ) 60 | 61 | // 62 | // Setup 63 | // 64 | // Logger setup 65 | zerolog.TimeFieldFormat = zerolog.TimeFormatUnix 66 | log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr}) 67 | zerolog.SetGlobalLevel(logLevelFromString(logLevel)) 68 | if debug { 69 | zerolog.SetGlobalLevel(zerolog.DebugLevel) 70 | } 71 | 72 | // Disable colored output if requested 73 | color.NoColor = noColor 74 | 75 | // Handle signals via context 76 | ctx, stop := signal.NotifyContext(ctx, os.Interrupt, syscall.SIGINT, syscall.SIGTERM) 77 | defer stop() 78 | 79 | // Create errgroup to manage all goroutines 80 | errg, ctx := errgroup.WithContext(ctx) 81 | 82 | // Timeout to wait for at least one node to be ready 83 | startCtx, cancel := context.WithTimeout(ctx, startTimeout) 84 | defer cancel() 85 | 86 | // Test connection to nodes 87 | pool, err := createNodePool(startCtx, nodes) 88 | if err != nil { 89 | return err 90 | } 91 | 92 | // Detect cosmos modules to automatically disable features 93 | modules, err := detectCosmosModules(startCtx, pool.GetSyncedNode()) 94 | if err != nil { 95 | log.Warn().Err(err).Msg("failed to detect cosmos modules") 96 | } else { 97 | noGov = !ensureCosmosModule("gov", modules) || noGov 98 | noStaking = !ensureCosmosModule("staking", modules) || noStaking 99 | noSlashing = !ensureCosmosModule("slashing", modules) || noSlashing 100 | noCommission = !ensureCosmosModule("distribution", modules) || noCommission 101 | noUpgrade = !ensureCosmosModule("upgrade", modules) || noUpgrade 102 | } 103 | log.Info(). 104 | Bool("commission", !noCommission). 105 | Bool("gov", !noGov). 106 | Bool("slashing", !noSlashing). 107 | Bool("staking", !noStaking). 108 | Bool("upgrade", !noUpgrade). 109 | Msg("cosmos modules features status") 110 | 111 | // Parse validators into name & address 112 | trackedValidators, err := createTrackedValidators(ctx, pool, validators, noStaking) 113 | if err != nil { 114 | return err 115 | } 116 | 117 | var wh *webhook.Webhook 118 | if webhookURL != "" { 119 | whURL, err := url.Parse(webhookURL) 120 | if err != nil { 121 | return fmt.Errorf("failed to parse webhook endpoint: %w", err) 122 | } 123 | wh = webhook.New(*whURL) 124 | } 125 | 126 | // Custom block webhooks 127 | blockWebhooks := []watcher.BlockWebhook{} 128 | for _, block := range webhookCustomBlocks { 129 | blockHeight, err := strconv.ParseInt(block, 10, 64) 130 | if err != nil { 131 | return fmt.Errorf("failed to parse block height for custom webhook (%s): %w", block, err) 132 | } 133 | blockWebhooks = append(blockWebhooks, watcher.BlockWebhook{ 134 | Height: blockHeight, 135 | Metadata: map[string]string{}, 136 | }) 137 | } 138 | 139 | // 140 | // Node Watchers 141 | // 142 | metrics := metrics.New(namespace) 143 | metrics.Register() 144 | blockWatcher := watcher.NewBlockWatcher(trackedValidators, metrics, os.Stdout, wh, blockWebhooks) 145 | errg.Go(func() error { 146 | return blockWatcher.Start(ctx) 147 | }) 148 | statusWatcher := watcher.NewStatusWatcher(chainID, metrics) 149 | errg.Go(func() error { 150 | return statusWatcher.Start(ctx) 151 | }) 152 | if !noCommission { 153 | commissionWatcher := watcher.NewCommissionsWatcher(trackedValidators, metrics, pool) 154 | errg.Go(func() error { 155 | return commissionWatcher.Start(ctx) 156 | }) 157 | } 158 | if babylonEnabled { 159 | finalityProviders := lo.Map(finalityProviders, func(val string, _ int) watcher.BabylonFinalityProvider { 160 | return watcher.ParseBabylonFinalityProvider(val) 161 | }) 162 | babylonWatcher := watcher.NewBabylonWatcher(trackedValidators, finalityProviders, pool, metrics, os.Stdout) 163 | errg.Go(func() error { 164 | return babylonWatcher.Start(ctx) 165 | }) 166 | pool.OnNodeEvent(rpc.EventNewBlock, babylonWatcher.OnNewBlock) 167 | } 168 | 169 | // 170 | // Slashing watchers 171 | // 172 | if !noSlashing { 173 | slashingWatcher := watcher.NewSlashingWatcher(metrics, pool) 174 | errg.Go(func() error { 175 | return slashingWatcher.Start(ctx) 176 | }) 177 | } 178 | 179 | // 180 | // Pool watchers 181 | // 182 | if !noStaking { 183 | validatorsWatcher := watcher.NewValidatorsWatcher(trackedValidators, metrics, pool, watcher.ValidatorsWatcherOptions{ 184 | Denom: denom, 185 | DenomExponent: denomExpon, 186 | NoSlashing: noSlashing, 187 | }) 188 | errg.Go(func() error { 189 | return validatorsWatcher.Start(ctx) 190 | }) 191 | } 192 | if xGov != "v1beta1" && xGov != "v1" { 193 | log.Warn().Msgf("unknown gov module version: %s (fallback to v1)", xGov) 194 | xGov = "v1" 195 | } 196 | if !noGov { 197 | votesWatcher := watcher.NewVotesWatcher(trackedValidators, metrics, pool, watcher.VotesWatcherOptions{ 198 | GovModuleVersion: xGov, 199 | }) 200 | errg.Go(func() error { 201 | return votesWatcher.Start(ctx) 202 | }) 203 | } 204 | 205 | var upgradeWatcher *watcher.UpgradeWatcher 206 | if !noUpgrade { 207 | upgradeWatcher = watcher.NewUpgradeWatcher(metrics, pool, wh, watcher.UpgradeWatcherOptions{ 208 | CheckPendingProposals: !noGov, 209 | GovModuleVersion: xGov, 210 | }) 211 | errg.Go(func() error { 212 | return upgradeWatcher.Start(ctx) 213 | }) 214 | } 215 | 216 | // 217 | // Register watchers on nodes events 218 | // 219 | pool.OnNodeStart(blockWatcher.OnNodeStart) 220 | pool.OnNodeStatus(statusWatcher.OnNodeStatus) 221 | pool.OnNodeEvent(rpc.EventNewBlock, blockWatcher.OnNewBlock) 222 | if upgradeWatcher != nil { 223 | pool.OnNodeEvent(rpc.EventNewBlock, upgradeWatcher.OnNewBlock) 224 | } 225 | 226 | // 227 | // Start Pool 228 | // 229 | errg.Go(func() error { 230 | return pool.Start(ctx) 231 | }) 232 | 233 | // 234 | // HTTP server 235 | // 236 | log.Info().Msgf("starting HTTP server on %s", httpAddr) 237 | readyProbe := func() bool { 238 | // ready when at least one watcher is synced 239 | return pool.GetSyncedNode() != nil 240 | } 241 | httpServer := NewHTTPServer( 242 | httpAddr, 243 | WithReadyProbe(readyProbe), 244 | WithLiveProbe(upProbe), 245 | WithMetrics(metrics.Registry), 246 | ) 247 | errg.Go(func() error { 248 | return httpServer.Run() 249 | }) 250 | 251 | // 252 | // Wait for context to be cancelled (via signals or error from errgroup) 253 | // 254 | <-ctx.Done() 255 | log.Info().Msg("shutting down") 256 | 257 | // 258 | // Stop all watchers and exporter 259 | // 260 | ctx, cancel = context.WithTimeout(context.Background(), stopTimeout) 261 | defer cancel() 262 | 263 | if err := pool.Stop(ctx); err != nil { 264 | log.Error().Err(fmt.Errorf("failed to stop node pool: %w", err)).Msg("") 265 | } 266 | if err := httpServer.Shutdown(ctx); err != nil { 267 | log.Error().Err(fmt.Errorf("failed to stop http server: %w", err)).Msg("") 268 | } 269 | 270 | // Wait for all goroutines to finish 271 | return errg.Wait() 272 | } 273 | 274 | func logLevelFromString(level string) zerolog.Level { 275 | switch level { 276 | case "debug": 277 | return zerolog.DebugLevel 278 | case "info": 279 | return zerolog.InfoLevel 280 | case "warn": 281 | return zerolog.WarnLevel 282 | case "error": 283 | return zerolog.ErrorLevel 284 | default: 285 | return zerolog.InfoLevel 286 | } 287 | } 288 | 289 | func createNodePool(ctx context.Context, nodes []string) (*rpc.Pool, error) { 290 | rpcNodes := make([]*rpc.Node, len(nodes)) 291 | for i, endpoint := range nodes { 292 | client, err := http.New(endpoint, "/websocket") 293 | if err != nil { 294 | return nil, fmt.Errorf("failed to create client: %w", err) 295 | } 296 | 297 | opts := []rpc.NodeOption{} 298 | 299 | // Check is query string websocket is present in the endpoint 300 | if u, err := url.Parse(endpoint); err == nil { 301 | if u.Query().Get("__websocket") == "0" { 302 | opts = append(opts, rpc.DisableWebsocket()) 303 | } 304 | } 305 | 306 | rpcNodes[i] = rpc.NewNode(client, opts...) 307 | 308 | status, err := rpcNodes[i].Status(ctx) 309 | if err != nil { 310 | log.Error().Err(err).Msgf("failed to connect to %s", rpcNodes[i].Redacted()) 311 | continue 312 | } 313 | 314 | chainID := status.NodeInfo.Network 315 | blockHeight := status.SyncInfo.LatestBlockHeight 316 | 317 | logger := log.With().Int64("height", blockHeight).Str("chainID", chainID).Logger() 318 | 319 | if rpcNodes[i].IsSynced() { 320 | logger.Info().Msgf("connected to %s", rpcNodes[i].Redacted()) 321 | } else { 322 | logger.Warn().Msgf("connected to %s (but node is catching up)", rpcNodes[i].Redacted()) 323 | } 324 | } 325 | 326 | var rpcNode *rpc.Node 327 | var chainID string 328 | for _, node := range rpcNodes { 329 | if chainID == "" { 330 | chainID = node.ChainID() 331 | } else if chainID != node.ChainID() && node.ChainID() != "" { 332 | return nil, fmt.Errorf("nodes are on different chains: %s != %s", chainID, node.ChainID()) 333 | } 334 | if node.IsSynced() { 335 | rpcNode = node 336 | } 337 | } 338 | if rpcNode == nil { 339 | return nil, fmt.Errorf("no nodes synced") 340 | } 341 | 342 | return rpc.NewPool(chainID, rpcNodes), nil 343 | } 344 | 345 | func detectCosmosModules(ctx context.Context, node *rpc.Node) ([]*upgrade.ModuleVersion, error) { 346 | if node == nil { 347 | return nil, fmt.Errorf("no node available") 348 | } 349 | 350 | clientCtx := (client.Context{}).WithClient(node.Client) 351 | queryClient := upgrade.NewQueryClient(clientCtx) 352 | resp, err := queryClient.ModuleVersions(ctx, &upgrade.QueryModuleVersionsRequest{}) 353 | if err != nil { 354 | return nil, err 355 | } 356 | 357 | log.Debug().Msgf("detected %d cosmos modules", len(resp.ModuleVersions)) 358 | 359 | for _, module := range resp.ModuleVersions { 360 | log.Debug().Str("module", module.Name).Uint64("version", module.Version).Msg("detected cosmos module") 361 | } 362 | 363 | return resp.ModuleVersions, nil 364 | } 365 | 366 | func ensureCosmosModule(name string, modules []*upgrade.ModuleVersion) bool { 367 | for _, module := range modules { 368 | if module.Name == name { 369 | return true 370 | } 371 | } 372 | return false 373 | } 374 | 375 | func createTrackedValidators(ctx context.Context, pool *rpc.Pool, validators []string, noStaking bool) ([]watcher.TrackedValidator, error) { 376 | var stakingValidators []staking.Validator 377 | if !noStaking { 378 | node := pool.GetSyncedNode() 379 | clientCtx := (client.Context{}).WithClient(node.Client) 380 | queryClient := staking.NewQueryClient(clientCtx) 381 | 382 | resp, err := queryClient.Validators(ctx, &staking.QueryValidatorsRequest{ 383 | Pagination: &query.PageRequest{ 384 | Limit: 3000, 385 | }, 386 | }) 387 | if err != nil { 388 | return nil, err 389 | } 390 | stakingValidators = resp.Validators 391 | } 392 | 393 | trackedValidators := lo.Map(validators, func(v string, _ int) watcher.TrackedValidator { 394 | val := watcher.ParseValidator(v) 395 | 396 | for _, stakingVal := range stakingValidators { 397 | address := crypto.PubKeyAddress(stakingVal.ConsensusPubkey) 398 | if address == val.Address { 399 | hrp := crypto.GetHrpPrefix(stakingVal.OperatorAddress) + "valcons" 400 | val.Moniker = stakingVal.Description.Moniker 401 | val.OperatorAddress = stakingVal.OperatorAddress 402 | val.ConsensusAddress = crypto.PubKeyBech32Address(stakingVal.ConsensusPubkey, hrp) 403 | } 404 | } 405 | 406 | log.Info(). 407 | Str("alias", val.Name). 408 | Str("moniker", val.Moniker). 409 | Msgf("tracking validator %s", val.Address) 410 | 411 | log.Debug(). 412 | Str("account", val.AccountAddress()). 413 | Str("address", val.Address). 414 | Str("alias", val.Name). 415 | Str("moniker", val.Moniker). 416 | Str("operator", val.OperatorAddress). 417 | Str("consensus", val.ConsensusAddress). 418 | Msgf("validator info") 419 | 420 | return val 421 | }) 422 | 423 | return trackedValidators, nil 424 | } 425 | -------------------------------------------------------------------------------- /pkg/crypto/bls12_318.go: -------------------------------------------------------------------------------- 1 | package crypto 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | 7 | "github.com/cometbft/cometbft/crypto" 8 | "github.com/cometbft/cometbft/crypto/tmhash" 9 | cmtjson "github.com/cometbft/cometbft/libs/json" 10 | ) 11 | 12 | const ( 13 | Bls12PubKeyName = "cometbft/PubKeyBls12_381" 14 | ) 15 | 16 | func init() { 17 | cmtjson.RegisterType(Bls12PubKey{}, Bls12PubKeyName) 18 | } 19 | 20 | var _ crypto.PubKey = Bls12PubKey{} 21 | 22 | type Bls12PubKey []byte 23 | 24 | // Address is the SHA256-20 of the raw pubkey bytes. 25 | func (pubKey Bls12PubKey) Address() crypto.Address { 26 | // if len(pubKey) != PubKeySize { 27 | // panic("pubkey is incorrect size") 28 | // } 29 | return crypto.Address(tmhash.SumTruncated(pubKey)) 30 | } 31 | 32 | // Bytes returns the PubKey byte format. 33 | func (pubKey Bls12PubKey) Bytes() []byte { 34 | return []byte(pubKey) 35 | } 36 | 37 | func (pubKey Bls12PubKey) VerifySignature(msg []byte, sig []byte) bool { 38 | return false 39 | } 40 | 41 | func (pubKey Bls12PubKey) String() string { 42 | return fmt.Sprintf("PubKeyBls12_381{%X}", []byte(pubKey)) 43 | } 44 | 45 | func (Bls12PubKey) Type() string { 46 | return "bls12_381" 47 | } 48 | 49 | func (pubKey Bls12PubKey) Equals(other crypto.PubKey) bool { 50 | if otherEd, ok := other.(Bls12PubKey); ok { 51 | return bytes.Equal(pubKey[:], otherEd[:]) 52 | } 53 | 54 | return false 55 | } 56 | -------------------------------------------------------------------------------- /pkg/crypto/bn254.go: -------------------------------------------------------------------------------- 1 | package crypto 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | 7 | "github.com/cometbft/cometbft/crypto" 8 | "github.com/cometbft/cometbft/crypto/tmhash" 9 | cmtjson "github.com/cometbft/cometbft/libs/json" 10 | ) 11 | 12 | const ( 13 | Bn254PubKeyName = "cometbft/PubKeyBn254" 14 | ) 15 | 16 | func init() { 17 | cmtjson.RegisterType(Bn254PubKey{}, Bn254PubKeyName) 18 | } 19 | 20 | var _ crypto.PubKey = Bn254PubKey{} 21 | 22 | type Bn254PubKey []byte 23 | 24 | // Address is the SHA256-20 of the raw pubkey bytes. 25 | func (pubKey Bn254PubKey) Address() crypto.Address { 26 | // if len(pubKey) != PubKeySize { 27 | // panic("pubkey is incorrect size") 28 | // } 29 | return crypto.Address(tmhash.SumTruncated(pubKey)) 30 | } 31 | 32 | // Bytes returns the PubKey byte format. 33 | func (pubKey Bn254PubKey) Bytes() []byte { 34 | return []byte(pubKey) 35 | } 36 | 37 | func (pubKey Bn254PubKey) VerifySignature(msg []byte, sig []byte) bool { 38 | return false 39 | } 40 | 41 | func (pubKey Bn254PubKey) String() string { 42 | return fmt.Sprintf("PubKeyBn254{%X}", []byte(pubKey)) 43 | } 44 | 45 | func (Bn254PubKey) Type() string { 46 | return "Bn254" 47 | } 48 | 49 | func (pubKey Bn254PubKey) Equals(other crypto.PubKey) bool { 50 | if otherEd, ok := other.(Bn254PubKey); ok { 51 | return bytes.Equal(pubKey[:], otherEd[:]) 52 | } 53 | 54 | return false 55 | } 56 | -------------------------------------------------------------------------------- /pkg/crypto/utils.go: -------------------------------------------------------------------------------- 1 | package crypto 2 | 3 | import ( 4 | "strings" 5 | 6 | "github.com/cometbft/cometbft/libs/bytes" 7 | types1 "github.com/cosmos/cosmos-sdk/codec/types" 8 | "github.com/cosmos/cosmos-sdk/crypto/keys/ed25519" 9 | "github.com/cosmos/cosmos-sdk/crypto/keys/secp256k1" 10 | "github.com/cosmos/cosmos-sdk/types/bech32" 11 | ) 12 | 13 | func PubKeyAddressHelper(consensusPubkey *types1.Any) bytes.HexBytes { 14 | switch consensusPubkey.TypeUrl { 15 | case "/cosmos.crypto.ed25519.PubKey": 16 | key := ed25519.PubKey{Key: consensusPubkey.Value[2:]} 17 | return key.Address() 18 | 19 | case "/cosmos.crypto.secp256k1.PubKey": 20 | key := secp256k1.PubKey{Key: consensusPubkey.Value[2:]} 21 | return key.Address() 22 | } 23 | panic("unknown pubkey type: " + consensusPubkey.TypeUrl) 24 | } 25 | 26 | func PubKeyAddress(consensusPubkey *types1.Any) string { 27 | key := PubKeyAddressHelper(consensusPubkey) 28 | return key.String() 29 | } 30 | 31 | func PubKeyBech32Address(consensusPubkey *types1.Any, prefix string) string { 32 | key := PubKeyAddressHelper(consensusPubkey) 33 | address, _ := bech32.ConvertAndEncode(prefix, key) 34 | return address 35 | } 36 | 37 | // GetHrpPrefix returns the human-readable prefix for a given address. 38 | // Examples of valid address HRPs are "cosmosvalcons", "cosmosvaloper". 39 | // So this will return "cosmos" as the prefix 40 | func GetHrpPrefix(a string) string { 41 | 42 | hrp, _, err := bech32.DecodeAndConvert(a) 43 | if err != nil { 44 | return err.Error() 45 | } 46 | 47 | for _, v := range []string{"valoper", "cncl", "valcons"} { 48 | hrp = strings.TrimSuffix(hrp, v) 49 | } 50 | return hrp 51 | } 52 | -------------------------------------------------------------------------------- /pkg/metrics/metrics.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "github.com/prometheus/client_golang/prometheus" 5 | "github.com/prometheus/client_golang/prometheus/collectors" 6 | ) 7 | 8 | type Metrics struct { 9 | Registry *prometheus.Registry 10 | 11 | // Global metrics 12 | ActiveSet *prometheus.GaugeVec 13 | BlockHeight *prometheus.GaugeVec 14 | ProposalEndTime *prometheus.GaugeVec 15 | SeatPrice *prometheus.GaugeVec 16 | SkippedBlocks *prometheus.CounterVec 17 | TrackedBlocks *prometheus.CounterVec 18 | Transactions *prometheus.CounterVec 19 | UpgradePlan *prometheus.GaugeVec 20 | SignedBlocksWindow *prometheus.GaugeVec 21 | MinSignedBlocksPerWindow *prometheus.GaugeVec 22 | DowntimeJailDuration *prometheus.GaugeVec 23 | SlashFractionDoubleSign *prometheus.GaugeVec 24 | SlashFractionDowntime *prometheus.GaugeVec 25 | 26 | // Validator metrics 27 | Rank *prometheus.GaugeVec 28 | ProposedBlocks *prometheus.CounterVec 29 | ValidatedBlocks *prometheus.CounterVec 30 | MissedBlocks *prometheus.CounterVec 31 | SoloMissedBlocks *prometheus.CounterVec 32 | ConsecutiveMissedBlocks *prometheus.GaugeVec 33 | MissedBlocksWindow *prometheus.GaugeVec 34 | EmptyBlocks *prometheus.CounterVec 35 | Tokens *prometheus.GaugeVec 36 | IsBonded *prometheus.GaugeVec 37 | IsJailed *prometheus.GaugeVec 38 | Commission *prometheus.GaugeVec 39 | Vote *prometheus.GaugeVec 40 | 41 | // Babylon metrics 42 | BabylonEpoch *prometheus.GaugeVec 43 | BabylonCheckpointVote *prometheus.CounterVec 44 | BabylonCommittedCheckpointVote *prometheus.CounterVec 45 | BabylonMissedCheckpointVote *prometheus.CounterVec 46 | BabylonConsecutiveMissedCheckpointVote *prometheus.GaugeVec 47 | BabylonFinalityVotes *prometheus.CounterVec 48 | BabylonCommittedFinalityVotes *prometheus.CounterVec 49 | BabylonMissedFinalityVotes *prometheus.CounterVec 50 | BabylonConsecutiveMissedFinalityVotes *prometheus.GaugeVec 51 | 52 | // Node metrics 53 | NodeBlockHeight *prometheus.GaugeVec 54 | NodeSynced *prometheus.GaugeVec 55 | } 56 | 57 | func New(namespace string) *Metrics { 58 | metrics := &Metrics{ 59 | Registry: prometheus.NewRegistry(), 60 | BlockHeight: prometheus.NewGaugeVec( 61 | prometheus.GaugeOpts{ 62 | Namespace: namespace, 63 | Name: "block_height", 64 | Help: "Latest known block height (all nodes mixed up)", 65 | }, 66 | []string{"chain_id"}, 67 | ), 68 | ActiveSet: prometheus.NewGaugeVec( 69 | prometheus.GaugeOpts{ 70 | Namespace: namespace, 71 | Name: "active_set", 72 | Help: "Number of validators in the active set", 73 | }, 74 | []string{"chain_id"}, 75 | ), 76 | SeatPrice: prometheus.NewGaugeVec( 77 | prometheus.GaugeOpts{ 78 | Namespace: namespace, 79 | Name: "seat_price", 80 | Help: "Min seat price to be in the active set (ie. bonded tokens of the latest validator)", 81 | }, 82 | []string{"chain_id", "denom"}, 83 | ), 84 | Rank: prometheus.NewGaugeVec( 85 | prometheus.GaugeOpts{ 86 | Namespace: namespace, 87 | Name: "rank", 88 | Help: "Rank of the validator", 89 | }, 90 | []string{"chain_id", "address", "name"}, 91 | ), 92 | ProposedBlocks: prometheus.NewCounterVec( 93 | prometheus.CounterOpts{ 94 | Namespace: namespace, 95 | Name: "proposed_blocks", 96 | Help: "Number of proposed blocks per validator (for a bonded validator)", 97 | }, 98 | []string{"chain_id", "address", "name"}, 99 | ), 100 | ValidatedBlocks: prometheus.NewCounterVec( 101 | prometheus.CounterOpts{ 102 | Namespace: namespace, 103 | Name: "validated_blocks", 104 | Help: "Number of validated blocks per validator (for a bonded validator)", 105 | }, 106 | []string{"chain_id", "address", "name"}, 107 | ), 108 | MissedBlocks: prometheus.NewCounterVec( 109 | prometheus.CounterOpts{ 110 | Namespace: namespace, 111 | Name: "missed_blocks", 112 | Help: "Number of missed blocks per validator (for a bonded validator)", 113 | }, 114 | []string{"chain_id", "address", "name"}, 115 | ), 116 | SoloMissedBlocks: prometheus.NewCounterVec( 117 | prometheus.CounterOpts{ 118 | Namespace: namespace, 119 | Name: "solo_missed_blocks", 120 | Help: "Number of missed blocks per validator, unless block is missed by many other validators", 121 | }, 122 | []string{"chain_id", "address", "name"}, 123 | ), 124 | ConsecutiveMissedBlocks: prometheus.NewGaugeVec( 125 | prometheus.GaugeOpts{ 126 | Namespace: namespace, 127 | Name: "consecutive_missed_blocks", 128 | Help: "Number of consecutive missed blocks per validator (for a bonded validator)", 129 | }, 130 | []string{"chain_id", "address", "name"}, 131 | ), 132 | MissedBlocksWindow: prometheus.NewGaugeVec( 133 | prometheus.GaugeOpts{ 134 | Namespace: namespace, 135 | Name: "missed_blocks_window", 136 | Help: "Number of missed blocks per validator for the current signing window (for a bonded validator)", 137 | }, 138 | []string{"chain_id", "address", "name"}, 139 | ), 140 | EmptyBlocks: prometheus.NewCounterVec( 141 | prometheus.CounterOpts{ 142 | Namespace: namespace, 143 | Name: "empty_blocks", 144 | Help: "Number of empty blocks proposed by validator", 145 | }, 146 | []string{"chain_id", "address", "name"}, 147 | ), 148 | TrackedBlocks: prometheus.NewCounterVec( 149 | prometheus.CounterOpts{ 150 | Namespace: namespace, 151 | Name: "tracked_blocks", 152 | Help: "Number of blocks tracked since start", 153 | }, 154 | []string{"chain_id"}, 155 | ), 156 | Transactions: prometheus.NewCounterVec( 157 | prometheus.CounterOpts{ 158 | Namespace: namespace, 159 | Name: "transactions_total", 160 | Help: "Number of transactions since start", 161 | }, 162 | []string{"chain_id"}, 163 | ), 164 | SkippedBlocks: prometheus.NewCounterVec( 165 | prometheus.CounterOpts{ 166 | Namespace: namespace, 167 | Name: "skipped_blocks", 168 | Help: "Number of blocks skipped (ie. not tracked) since start", 169 | }, 170 | []string{"chain_id"}, 171 | ), 172 | Tokens: prometheus.NewGaugeVec( 173 | prometheus.GaugeOpts{ 174 | Namespace: namespace, 175 | Name: "tokens", 176 | Help: "Number of staked tokens per validator", 177 | }, 178 | []string{"chain_id", "address", "name", "denom"}, 179 | ), 180 | IsBonded: prometheus.NewGaugeVec( 181 | prometheus.GaugeOpts{ 182 | Namespace: namespace, 183 | Name: "is_bonded", 184 | Help: "Set to 1 if the validator is bonded", 185 | }, 186 | []string{"chain_id", "address", "name"}, 187 | ), 188 | IsJailed: prometheus.NewGaugeVec( 189 | prometheus.GaugeOpts{ 190 | Namespace: namespace, 191 | Name: "is_jailed", 192 | Help: "Set to 1 if the validator is jailed", 193 | }, 194 | []string{"chain_id", "address", "name"}, 195 | ), 196 | Commission: prometheus.NewGaugeVec( 197 | prometheus.GaugeOpts{ 198 | Namespace: namespace, 199 | Name: "commission", 200 | Help: "Earned validator commission", 201 | }, 202 | []string{"chain_id", "address", "name", "denom"}, 203 | ), 204 | Vote: prometheus.NewGaugeVec( 205 | prometheus.GaugeOpts{ 206 | Namespace: namespace, 207 | Name: "vote", 208 | Help: "Set to 1 if the validator has voted on a proposal", 209 | }, 210 | []string{"chain_id", "address", "name", "proposal_id"}, 211 | ), 212 | NodeBlockHeight: prometheus.NewGaugeVec( 213 | prometheus.GaugeOpts{ 214 | Namespace: namespace, 215 | Name: "node_block_height", 216 | Help: "Latest fetched block height for each node", 217 | }, 218 | []string{"chain_id", "node"}, 219 | ), 220 | NodeSynced: prometheus.NewGaugeVec( 221 | prometheus.GaugeOpts{ 222 | Namespace: namespace, 223 | Name: "node_synced", 224 | Help: "Set to 1 is the node is synced (ie. not catching-up)", 225 | }, 226 | []string{"chain_id", "node"}, 227 | ), 228 | UpgradePlan: prometheus.NewGaugeVec( 229 | prometheus.GaugeOpts{ 230 | Namespace: namespace, 231 | Name: "upgrade_plan", 232 | Help: "Block height of the upcoming upgrade (hard fork)", 233 | }, 234 | []string{"chain_id", "version", "block"}, 235 | ), 236 | ProposalEndTime: prometheus.NewGaugeVec( 237 | prometheus.GaugeOpts{ 238 | Namespace: namespace, 239 | Name: "proposal_end_time", 240 | Help: "Timestamp of the voting end time of a proposal", 241 | }, 242 | []string{"chain_id", "proposal_id"}, 243 | ), 244 | SignedBlocksWindow: prometheus.NewGaugeVec( 245 | prometheus.GaugeOpts{ 246 | Namespace: namespace, 247 | Name: "signed_blocks_window", 248 | Help: "Number of blocks per signing window", 249 | }, 250 | []string{"chain_id"}, 251 | ), 252 | MinSignedBlocksPerWindow: prometheus.NewGaugeVec( 253 | prometheus.GaugeOpts{ 254 | Namespace: namespace, 255 | Name: "min_signed_blocks_per_window", 256 | Help: "Minimum number of blocks required to be signed per signing window", 257 | }, 258 | []string{"chain_id"}, 259 | ), 260 | DowntimeJailDuration: prometheus.NewGaugeVec( 261 | prometheus.GaugeOpts{ 262 | Namespace: namespace, 263 | Name: "downtime_jail_duration", 264 | Help: "Duration of the jail period for a validator in seconds", 265 | }, 266 | []string{"chain_id"}, 267 | ), 268 | SlashFractionDoubleSign: prometheus.NewGaugeVec( 269 | prometheus.GaugeOpts{ 270 | Namespace: namespace, 271 | Name: "slash_fraction_double_sign", 272 | Help: "Slash penaltiy for double-signing", 273 | }, 274 | []string{"chain_id"}, 275 | ), 276 | SlashFractionDowntime: prometheus.NewGaugeVec( 277 | prometheus.GaugeOpts{ 278 | Namespace: namespace, 279 | Name: "slash_fraction_downtime", 280 | Help: "Slash penaltiy for downtime", 281 | }, 282 | []string{"chain_id"}, 283 | ), 284 | BabylonEpoch: prometheus.NewGaugeVec( 285 | prometheus.GaugeOpts{ 286 | Namespace: namespace, 287 | Name: "babylon_epoch", 288 | Help: "Babylon epoch", 289 | }, 290 | []string{"chain_id"}, 291 | ), 292 | BabylonCheckpointVote: prometheus.NewCounterVec( 293 | prometheus.CounterOpts{ 294 | Namespace: namespace, 295 | Name: "babylon_checkpoint_vote", 296 | Help: "Count of checkpoint votes since start (equal to number of epochs)", 297 | }, 298 | []string{"chain_id"}, 299 | ), 300 | BabylonCommittedCheckpointVote: prometheus.NewCounterVec( 301 | prometheus.CounterOpts{ 302 | Namespace: namespace, 303 | Name: "babylon_committed_checkpoint_vote", 304 | Help: "Number of committed checkpoint votes for a validator", 305 | }, 306 | []string{"chain_id", "address", "name"}, 307 | ), 308 | BabylonMissedCheckpointVote: prometheus.NewCounterVec( 309 | prometheus.CounterOpts{ 310 | Namespace: namespace, 311 | Name: "babylon_missed_checkpoint_vote", 312 | Help: "Number of missed checkpoint votes for a validator", 313 | }, 314 | []string{"chain_id", "address", "name"}, 315 | ), 316 | BabylonConsecutiveMissedCheckpointVote: prometheus.NewGaugeVec( 317 | prometheus.GaugeOpts{ 318 | Namespace: namespace, 319 | Name: "babylon_consecutive_missed_checkpoint_vote", 320 | Help: "Number of consecutive missed checkpoint votes for a validator", 321 | }, 322 | []string{"chain_id", "address", "name"}, 323 | ), 324 | BabylonFinalityVotes: prometheus.NewCounterVec( 325 | prometheus.CounterOpts{ 326 | Namespace: namespace, 327 | Name: "babylon_finality_votes", 328 | Help: "Count of total finality provider slots since start", 329 | }, 330 | []string{"chain_id"}, 331 | ), 332 | BabylonCommittedFinalityVotes: prometheus.NewCounterVec( 333 | prometheus.CounterOpts{ 334 | Namespace: namespace, 335 | Name: "babylon_committed_finality_votes", 336 | Help: "Number of votes for a finality provider", 337 | }, 338 | []string{"chain_id", "address", "name"}, 339 | ), 340 | BabylonMissedFinalityVotes: prometheus.NewCounterVec( 341 | prometheus.CounterOpts{ 342 | Namespace: namespace, 343 | Name: "babylon_missed_finality_votes", 344 | Help: "Number of missed votes for a finality provider", 345 | }, 346 | []string{"chain_id", "address", "name"}, 347 | ), 348 | BabylonConsecutiveMissedFinalityVotes: prometheus.NewGaugeVec( 349 | prometheus.GaugeOpts{ 350 | Namespace: namespace, 351 | Name: "babylon_consecutive_missed_finality_votes", 352 | Help: "Number of consecutive missed votes for a finality provider", 353 | }, 354 | []string{"chain_id", "address", "name"}, 355 | ), 356 | } 357 | 358 | return metrics 359 | } 360 | 361 | func (m *Metrics) Register() { 362 | m.Registry.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{})) 363 | m.Registry.MustRegister(collectors.NewGoCollector()) 364 | 365 | m.Registry.MustRegister(m.BlockHeight) 366 | m.Registry.MustRegister(m.ActiveSet) 367 | m.Registry.MustRegister(m.SeatPrice) 368 | m.Registry.MustRegister(m.Rank) 369 | m.Registry.MustRegister(m.ProposedBlocks) 370 | m.Registry.MustRegister(m.ValidatedBlocks) 371 | m.Registry.MustRegister(m.MissedBlocks) 372 | m.Registry.MustRegister(m.SoloMissedBlocks) 373 | m.Registry.MustRegister(m.ConsecutiveMissedBlocks) 374 | m.Registry.MustRegister(m.MissedBlocksWindow) 375 | m.Registry.MustRegister(m.EmptyBlocks) 376 | m.Registry.MustRegister(m.TrackedBlocks) 377 | m.Registry.MustRegister(m.Transactions) 378 | m.Registry.MustRegister(m.SkippedBlocks) 379 | m.Registry.MustRegister(m.Tokens) 380 | m.Registry.MustRegister(m.IsBonded) 381 | m.Registry.MustRegister(m.Commission) 382 | m.Registry.MustRegister(m.IsJailed) 383 | m.Registry.MustRegister(m.Vote) 384 | m.Registry.MustRegister(m.NodeBlockHeight) 385 | m.Registry.MustRegister(m.NodeSynced) 386 | m.Registry.MustRegister(m.UpgradePlan) 387 | m.Registry.MustRegister(m.ProposalEndTime) 388 | m.Registry.MustRegister(m.SignedBlocksWindow) 389 | m.Registry.MustRegister(m.MinSignedBlocksPerWindow) 390 | m.Registry.MustRegister(m.DowntimeJailDuration) 391 | m.Registry.MustRegister(m.SlashFractionDoubleSign) 392 | m.Registry.MustRegister(m.SlashFractionDowntime) 393 | m.Registry.MustRegister(m.BabylonEpoch) 394 | m.Registry.MustRegister(m.BabylonCheckpointVote) 395 | m.Registry.MustRegister(m.BabylonCommittedCheckpointVote) 396 | m.Registry.MustRegister(m.BabylonMissedCheckpointVote) 397 | m.Registry.MustRegister(m.BabylonConsecutiveMissedCheckpointVote) 398 | m.Registry.MustRegister(m.BabylonFinalityVotes) 399 | m.Registry.MustRegister(m.BabylonCommittedFinalityVotes) 400 | m.Registry.MustRegister(m.BabylonMissedFinalityVotes) 401 | m.Registry.MustRegister(m.BabylonConsecutiveMissedFinalityVotes) 402 | } 403 | -------------------------------------------------------------------------------- /pkg/metrics/metrics_test.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import "testing" 4 | 5 | func TestMetrics(t *testing.T) { 6 | m := New("cosmos_validator_watcher") 7 | m.Register() 8 | } 9 | -------------------------------------------------------------------------------- /pkg/metrics/utils.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | func BoolToFloat64(b bool) float64 { 4 | if b { 5 | return 1 6 | } 7 | return 0 8 | } 9 | -------------------------------------------------------------------------------- /pkg/metrics/utils_test.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "testing" 5 | 6 | "gotest.tools/assert" 7 | ) 8 | 9 | func TestBoolToFloat64(t *testing.T) { 10 | assert.Equal(t, float64(1), BoolToFloat64(true)) 11 | assert.Equal(t, float64(0), BoolToFloat64(false)) 12 | } 13 | -------------------------------------------------------------------------------- /pkg/rpc/node.go: -------------------------------------------------------------------------------- 1 | package rpc 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/url" 7 | "sync" 8 | "sync/atomic" 9 | "time" 10 | 11 | "github.com/avast/retry-go/v4" 12 | "github.com/cometbft/cometbft/rpc/client/http" 13 | ctypes "github.com/cometbft/cometbft/rpc/core/types" 14 | "github.com/cometbft/cometbft/types" 15 | "github.com/rs/zerolog/log" 16 | ) 17 | 18 | const ( 19 | EventCompleteProposal = "CompleteProposal" 20 | EventNewBlock = "NewBlock" 21 | EventNewRound = "NewRound" 22 | EventRoundState = "RoundState" 23 | EventValidatorSetUpdates = "ValidatorSetUpdates" 24 | EventVote = "Vote" 25 | ) 26 | 27 | type OnNodeEvent func(ctx context.Context, n *Node, event *ctypes.ResultEvent) error 28 | type OnNodeStart func(ctx context.Context, n *Node) error 29 | type OnNodeStatus func(ctx context.Context, n *Node, status *ctypes.ResultStatus) error 30 | 31 | type NodeOption func(*Node) 32 | 33 | func DisableWebsocket() NodeOption { 34 | return func(n *Node) { 35 | n.disableWebsocket = true 36 | } 37 | } 38 | 39 | type Node struct { 40 | Client *http.HTTP 41 | 42 | // Save endpoint url for redacted logging 43 | endpoint *url.URL 44 | 45 | disableWebsocket bool 46 | 47 | onStart []OnNodeStart 48 | onStatus []OnNodeStatus 49 | onEvent map[string][]OnNodeEvent 50 | 51 | chainID string 52 | status atomic.Value 53 | latestBlock atomic.Value 54 | started chan struct{} 55 | startedOnce sync.Once 56 | subscriptions map[string]<-chan ctypes.ResultEvent 57 | } 58 | 59 | func NewNode(client *http.HTTP, options ...NodeOption) *Node { 60 | endpoint, _ := url.Parse(client.Remote()) 61 | 62 | node := &Node{ 63 | Client: client, 64 | endpoint: endpoint, 65 | started: make(chan struct{}), 66 | startedOnce: sync.Once{}, 67 | subscriptions: make(map[string]<-chan ctypes.ResultEvent), 68 | onEvent: make(map[string][]OnNodeEvent), 69 | } 70 | 71 | for _, opt := range options { 72 | opt(node) 73 | } 74 | 75 | return node 76 | } 77 | 78 | func (n *Node) Endpoint() string { 79 | if n.endpoint == nil { 80 | return n.Client.Remote() 81 | } 82 | ep := *n.endpoint 83 | if _, has := ep.User.Password(); has { 84 | ep.User = nil 85 | } 86 | return ep.String() 87 | } 88 | 89 | func (n *Node) Redacted() string { 90 | if n.endpoint == nil { 91 | return n.Client.Remote() 92 | } 93 | return n.endpoint.Redacted() 94 | } 95 | 96 | func (n *Node) OnStart(callback OnNodeStart) { 97 | n.onStart = append(n.onStart, callback) 98 | } 99 | 100 | func (n *Node) OnStatus(callback OnNodeStatus) { 101 | n.onStatus = append(n.onStatus, callback) 102 | } 103 | 104 | func (n *Node) OnEvent(eventType string, callback OnNodeEvent) { 105 | n.onEvent[eventType] = append(n.onEvent[eventType], callback) 106 | } 107 | 108 | func (n *Node) Started() chan struct{} { 109 | return n.started 110 | } 111 | 112 | func (n *Node) IsRunning() bool { 113 | return n.Client.IsRunning() 114 | } 115 | 116 | func (n *Node) IsSynced() bool { 117 | status := n.loadStatus() 118 | if status == nil { 119 | return false 120 | } 121 | 122 | return !status.SyncInfo.CatchingUp && 123 | status.SyncInfo.LatestBlockTime.After(time.Now().Add(-120*time.Second)) 124 | } 125 | 126 | func (n *Node) ChainID() string { 127 | return n.chainID 128 | } 129 | 130 | func (n *Node) Start(ctx context.Context) error { 131 | log := log.With().Str("node", n.Redacted()).Logger() 132 | 133 | // Wait for the node to be ready 134 | initTicker := time.NewTicker(30 * time.Second) 135 | for { 136 | status, err := n.syncStatus(ctx) 137 | if err != nil { 138 | log.Error().Err(err).Msg("failed to sync status") 139 | } 140 | if status != nil && !status.SyncInfo.CatchingUp { 141 | break 142 | } 143 | 144 | select { 145 | case <-ctx.Done(): 146 | return nil 147 | case <-initTicker.C: 148 | continue 149 | } 150 | } 151 | initTicker.Stop() 152 | 153 | // Start the websocket process 154 | if !n.disableWebsocket { 155 | if err := n.Client.Start(); err != nil { 156 | return fmt.Errorf("failed to start client: %w", err) 157 | } 158 | } 159 | 160 | blocksEvents, err := n.Subscribe(ctx, EventNewBlock) 161 | if err != nil { 162 | return fmt.Errorf("failed to get subscription: %w", err) 163 | } 164 | 165 | // Subscribe to validator set changes 166 | validatorEvents, err := n.Subscribe(ctx, EventValidatorSetUpdates) 167 | if err != nil { 168 | return fmt.Errorf("failed to subscribe to events: %w", err) 169 | } 170 | 171 | // Call the start callbacks 172 | n.startedOnce.Do(func() { 173 | n.handleStart(ctx) 174 | }) 175 | 176 | // Start the status loop 177 | statusTicker := time.NewTicker(30 * time.Second) 178 | blocksTicker := time.NewTicker(10 * time.Second) 179 | for { 180 | select { 181 | case <-ctx.Done(): 182 | log.Debug().Err(ctx.Err()).Msgf("stopping node status loop") 183 | return nil 184 | 185 | case evt := <-blocksEvents: 186 | log.Debug().Msg("got new block event") 187 | n.saveLatestBlock(evt.Data.(types.EventDataNewBlock).Block) 188 | n.handleEvent(ctx, EventNewBlock, &evt) 189 | blocksTicker.Reset(10 * time.Second) 190 | 191 | case evt := <-validatorEvents: 192 | log.Debug().Msg("got validator set update event") 193 | n.handleEvent(ctx, EventValidatorSetUpdates, &evt) 194 | 195 | case <-blocksTicker.C: 196 | log.Debug().Msg("syncing latest blocks") 197 | n.syncBlocks(ctx) 198 | 199 | case <-statusTicker.C: 200 | log.Debug().Msg("syncing status") 201 | n.syncStatus(ctx) 202 | } 203 | } 204 | } 205 | 206 | func (n *Node) Status(ctx context.Context) (*ctypes.ResultStatus, error) { 207 | status := n.loadStatus() 208 | 209 | if status == nil || status.SyncInfo.LatestBlockTime.Before(time.Now().Add(-10*time.Second)) { 210 | return n.syncStatus(ctx) 211 | } 212 | 213 | return status, nil 214 | } 215 | 216 | func (n *Node) loadStatus() *ctypes.ResultStatus { 217 | status := n.status.Load() 218 | if status == nil { 219 | return nil 220 | } 221 | return status.(*ctypes.ResultStatus) 222 | } 223 | 224 | func (n *Node) syncStatus(ctx context.Context) (*ctypes.ResultStatus, error) { 225 | log := log.With().Str("node", n.Redacted()).Logger() 226 | 227 | retryOpts := []retry.Option{ 228 | retry.Context(ctx), 229 | retry.Delay(1 * time.Second), 230 | retry.Attempts(2), 231 | // retry.OnRetry(func(_ uint, err error) { 232 | // log.Warn().Err(err).Msgf("retrying status on %s", n.Redacted()) 233 | // }), 234 | } 235 | 236 | status, err := retry.DoWithData(func() (*ctypes.ResultStatus, error) { 237 | return n.Client.Status(ctx) 238 | }, retryOpts...) 239 | 240 | n.status.Store(status) 241 | 242 | if err != nil { 243 | return status, fmt.Errorf("failed to get status of %s: %w", n.Redacted(), err) 244 | } 245 | 246 | if status.SyncInfo.CatchingUp { 247 | // We're catching up, not synced 248 | log.Warn().Int64("block", status.SyncInfo.LatestBlockHeight).Msgf("node is catching up") 249 | return status, nil 250 | } 251 | 252 | n.chainID = status.NodeInfo.Network 253 | 254 | for _, onStatus := range n.onStatus { 255 | if err := onStatus(ctx, n, status); err != nil { 256 | log.Error().Err(err).Msgf("failed to call status callback") 257 | } 258 | } 259 | 260 | return status, nil 261 | } 262 | 263 | func (n *Node) handleStart(ctx context.Context) { 264 | log := log.With().Str("node", n.Redacted()).Logger() 265 | 266 | for _, onStart := range n.onStart { 267 | if err := onStart(ctx, n); err != nil { 268 | log.Error().Err(err).Msgf("failed to call start node callback") 269 | return 270 | } 271 | } 272 | 273 | // Mark the node as ready 274 | close(n.started) 275 | } 276 | 277 | func (n *Node) handleEvent(ctx context.Context, eventType string, event *ctypes.ResultEvent) { 278 | for _, onEvent := range n.onEvent[eventType] { 279 | if err := onEvent(ctx, n, event); err != nil { 280 | log.Error().Err(err).Msgf("failed to call event callback") 281 | } 282 | } 283 | } 284 | 285 | func (n *Node) getLatestBlock() *types.Block { 286 | block := n.latestBlock.Load() 287 | if block == nil { 288 | return nil 289 | } 290 | return block.(*types.Block) 291 | } 292 | 293 | func (n *Node) saveLatestBlock(block *types.Block) { 294 | n.latestBlock.Store(block) 295 | } 296 | 297 | func (n *Node) syncBlocks(ctx context.Context) { 298 | log := log.With().Str("node", n.Redacted()).Logger() 299 | 300 | // Fetch latest block 301 | currentBlockResp, err := n.Client.Block(ctx, nil) 302 | if err != nil { 303 | log.Error().Err(err).Msgf("failed to sync with latest block") 304 | return 305 | } 306 | 307 | currentBlock := currentBlockResp.Block 308 | if currentBlock == nil { 309 | log.Error().Err(err).Msgf("no block returned when requesting latest block") 310 | return 311 | } 312 | 313 | // Check the latest known block height 314 | latestBlockHeight := int64(0) 315 | latestBlock := n.getLatestBlock() 316 | if latestBlock != nil { 317 | latestBlockHeight = latestBlock.Height 318 | } 319 | 320 | // Go back to a maximum of 20 blocks 321 | if currentBlock.Height-latestBlockHeight > 20 { 322 | latestBlockHeight = currentBlock.Height - 20 323 | } 324 | 325 | // Fetch all skipped blocks since latest known block 326 | for height := latestBlockHeight + 1; height < currentBlock.Height; height++ { 327 | blockResp, err := n.Client.Block(ctx, &height) 328 | if err != nil { 329 | log.Error().Err(err).Msgf("failed to sync with latest block") 330 | continue 331 | } 332 | 333 | n.handleEvent(ctx, EventNewBlock, &ctypes.ResultEvent{ 334 | Query: "", 335 | Data: types.EventDataNewBlock{ 336 | Block: blockResp.Block, 337 | }, 338 | Events: make(map[string][]string), 339 | }) 340 | } 341 | 342 | n.handleEvent(ctx, EventNewBlock, &ctypes.ResultEvent{ 343 | Query: "", 344 | Data: types.EventDataNewBlock{ 345 | Block: currentBlockResp.Block, 346 | }, 347 | Events: make(map[string][]string), 348 | }) 349 | 350 | n.saveLatestBlock(currentBlockResp.Block) 351 | } 352 | 353 | func (n *Node) Stop(ctx context.Context) error { 354 | if !n.IsRunning() { 355 | return nil 356 | } 357 | 358 | // if err := n.Client.UnsubscribeAll(ctx, "cosmos-validator-watcher"); err != nil { 359 | // return fmt.Errorf("failed to unsubscribe from all events: %w", err) 360 | // } 361 | 362 | // if err := n.Client.Stop(); err != nil { 363 | // return fmt.Errorf("failed to stop client: %w", err) 364 | // } 365 | 366 | return nil 367 | } 368 | 369 | func (n *Node) Subscribe(ctx context.Context, eventType string) (<-chan ctypes.ResultEvent, error) { 370 | if n.disableWebsocket { 371 | return make(chan ctypes.ResultEvent), nil 372 | } 373 | 374 | if res, ok := n.subscriptions[eventType]; ok { 375 | return res, nil 376 | } 377 | 378 | out, err := n.Client.Subscribe(ctx, "cosmos-validator-watcher", fmt.Sprintf("tm.event='%s'", eventType), 10) 379 | if err != nil { 380 | return nil, fmt.Errorf("failed to subscribe to %s: %w", eventType, err) 381 | } 382 | 383 | n.subscriptions[eventType] = out 384 | 385 | return n.subscriptions[eventType], nil 386 | } 387 | -------------------------------------------------------------------------------- /pkg/rpc/pool.go: -------------------------------------------------------------------------------- 1 | package rpc 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "sync" 7 | 8 | "github.com/rs/zerolog/log" 9 | "golang.org/x/sync/errgroup" 10 | ) 11 | 12 | type Pool struct { 13 | ChainID string 14 | Nodes []*Node 15 | 16 | started chan struct{} 17 | startedOnce sync.Once 18 | } 19 | 20 | func NewPool(chainID string, nodes []*Node) *Pool { 21 | return &Pool{ 22 | ChainID: chainID, 23 | Nodes: nodes, 24 | started: make(chan struct{}), 25 | startedOnce: sync.Once{}, 26 | } 27 | } 28 | 29 | func (p *Pool) Start(ctx context.Context) error { 30 | errg, ctx := errgroup.WithContext(ctx) 31 | for _, node := range p.Nodes { 32 | node := node 33 | // Start node 34 | errg.Go(func() error { 35 | if err := node.Start(ctx); err != nil { 36 | log.Error().Err(err).Msg("node error") 37 | } 38 | return nil 39 | }) 40 | 41 | // Mark pool as started when the first node is started 42 | go func() { 43 | select { 44 | case <-ctx.Done(): 45 | case <-node.Started(): 46 | // Mark the pool as started 47 | p.startedOnce.Do(func() { 48 | close(p.started) 49 | }) 50 | } 51 | }() 52 | } 53 | return errg.Wait() 54 | } 55 | 56 | func (p *Pool) Stop(ctx context.Context) error { 57 | errg, ctx := errgroup.WithContext(ctx) 58 | 59 | for _, node := range p.Nodes { 60 | node := node 61 | errg.Go(func() error { 62 | if err := node.Stop(ctx); err != nil { 63 | return fmt.Errorf("failed to stop node: %w", err) 64 | } 65 | 66 | return nil 67 | }) 68 | } 69 | 70 | return errg.Wait() 71 | } 72 | 73 | func (p *Pool) Started() chan struct{} { 74 | return p.started 75 | } 76 | 77 | func (p *Pool) GetSyncedNode() *Node { 78 | for _, node := range p.Nodes { 79 | if node.IsSynced() { 80 | return node 81 | } 82 | } 83 | return nil 84 | } 85 | 86 | func (p *Pool) OnNodeStart(callback OnNodeStart) { 87 | for _, n := range p.Nodes { 88 | n.onStart = append(n.onStart, callback) 89 | } 90 | } 91 | 92 | func (p *Pool) OnNodeStatus(callback OnNodeStatus) { 93 | for _, n := range p.Nodes { 94 | n.onStatus = append(n.onStatus, callback) 95 | } 96 | } 97 | 98 | func (p *Pool) OnNodeEvent(eventType string, callback OnNodeEvent) { 99 | for _, n := range p.Nodes { 100 | n.onEvent[eventType] = append(n.onEvent[eventType], callback) 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /pkg/watcher/babylon.go: -------------------------------------------------------------------------------- 1 | package watcher 2 | 3 | import ( 4 | "context" 5 | "encoding/hex" 6 | "fmt" 7 | "io" 8 | "strings" 9 | "time" 10 | 11 | checkpointing "github.com/babylonlabs-io/babylon/x/checkpointing/types" 12 | epoching "github.com/babylonlabs-io/babylon/x/epoching/types" 13 | finality "github.com/babylonlabs-io/babylon/x/finality/types" 14 | abcitypes "github.com/cometbft/cometbft/abci/types" 15 | tmtypes "github.com/cometbft/cometbft/proto/tendermint/types" 16 | ctypes "github.com/cometbft/cometbft/rpc/core/types" 17 | "github.com/cometbft/cometbft/types" 18 | "github.com/cosmos/cosmos-sdk/client" 19 | "github.com/cosmos/cosmos-sdk/codec" 20 | codectypes "github.com/cosmos/cosmos-sdk/codec/types" 21 | "github.com/cosmos/cosmos-sdk/std" 22 | "github.com/cosmos/cosmos-sdk/types/query" 23 | "github.com/cosmos/cosmos-sdk/types/tx" 24 | "github.com/fatih/color" 25 | "github.com/kilnfi/cosmos-validator-watcher/pkg/metrics" 26 | "github.com/kilnfi/cosmos-validator-watcher/pkg/rpc" 27 | "github.com/rs/zerolog/log" 28 | "github.com/samber/lo" 29 | ) 30 | 31 | type BabylonFinalityProvider struct { 32 | Address string 33 | Label string 34 | } 35 | 36 | func ParseBabylonFinalityProvider(val string) BabylonFinalityProvider { 37 | parts := strings.Split(val, ":") 38 | if len(parts) > 1 { 39 | return BabylonFinalityProvider{ 40 | Address: parts[0], 41 | Label: parts[1], 42 | } 43 | } 44 | 45 | return BabylonFinalityProvider{ 46 | Address: parts[0], 47 | Label: parts[0], 48 | } 49 | } 50 | 51 | type BabylonWatcher struct { 52 | trackedValidators []TrackedValidator 53 | finalityProviders []BabylonFinalityProvider 54 | pool *rpc.Pool 55 | metrics *metrics.Metrics 56 | writer io.Writer 57 | blockChan chan *types.Block 58 | latestBlockHeight int64 59 | protoCodec *codec.ProtoCodec 60 | epochInterval int64 61 | } 62 | 63 | func NewBabylonWatcher(validators []TrackedValidator, finalityProviders []BabylonFinalityProvider, pool *rpc.Pool, metrics *metrics.Metrics, writer io.Writer) *BabylonWatcher { 64 | // Create a new Protobuf codec to decode babylon messages 65 | interfaceRegistry := codectypes.NewInterfaceRegistry() 66 | std.RegisterInterfaces(interfaceRegistry) 67 | checkpointing.RegisterInterfaces(interfaceRegistry) 68 | finality.RegisterInterfaces(interfaceRegistry) 69 | protoCodec := codec.NewProtoCodec(interfaceRegistry) 70 | 71 | return &BabylonWatcher{ 72 | trackedValidators: validators, 73 | finalityProviders: finalityProviders, 74 | pool: pool, 75 | metrics: metrics, 76 | writer: writer, 77 | protoCodec: protoCodec, 78 | blockChan: make(chan *types.Block), 79 | epochInterval: 360, 80 | } 81 | } 82 | 83 | func (w *BabylonWatcher) Start(ctx context.Context) error { 84 | if err := w.syncEpochParams(ctx); err != nil { 85 | return err 86 | } 87 | 88 | ticker := time.NewTicker(5 * time.Minute) 89 | defer ticker.Stop() 90 | 91 | for { 92 | select { 93 | case <-ctx.Done(): 94 | return nil 95 | case block := <-w.blockChan: 96 | w.handleBlock(block) 97 | case <-ticker.C: 98 | if err := w.syncEpochParams(ctx); err != nil { 99 | log.Error().Err(err).Msg("failed to sync epoch params") 100 | } 101 | } 102 | } 103 | } 104 | 105 | func (w *BabylonWatcher) OnNewBlock(ctx context.Context, node *rpc.Node, evt *ctypes.ResultEvent) error { 106 | // Ignore blocks if node is catching up 107 | if !node.IsSynced() { 108 | return nil 109 | } 110 | 111 | blockEvent := evt.Data.(types.EventDataNewBlock) 112 | w.blockChan <- blockEvent.Block 113 | 114 | return nil 115 | } 116 | 117 | func (w *BabylonWatcher) syncEpochParams(ctx context.Context) error { 118 | node := w.pool.GetSyncedNode() 119 | if node == nil { 120 | return fmt.Errorf("no node available to fetch upgrade plan") 121 | } 122 | 123 | chainID := node.ChainID() 124 | w.metrics.BabylonCheckpointVote.WithLabelValues(chainID).Add(0) 125 | w.metrics.BabylonFinalityVotes.WithLabelValues(chainID).Add(0) 126 | 127 | clientCtx := (client.Context{}).WithClient(node.Client) 128 | queryClient := epoching.NewQueryClient(clientCtx) 129 | resp, err := queryClient.EpochsInfo(ctx, &epoching.QueryEpochsInfoRequest{ 130 | Pagination: &query.PageRequest{ 131 | Limit: 1, 132 | Reverse: true, 133 | }, 134 | }) 135 | if err != nil { 136 | return fmt.Errorf("failed to fetch epoch params: %w", err) 137 | } 138 | if len(resp.Epochs) == 0 { 139 | return fmt.Errorf("no epoch params found") 140 | } 141 | 142 | w.epochInterval = int64(resp.Epochs[0].CurrentEpochInterval) 143 | w.metrics.BabylonEpoch.WithLabelValues(chainID).Set(float64(resp.Epochs[0].EpochNumber)) 144 | 145 | log.Debug(). 146 | Uint64("epoch", resp.Epochs[0].EpochNumber). 147 | Int64("epoch-interval", w.epochInterval). 148 | Msg("babylon epoch interval") 149 | 150 | return nil 151 | } 152 | 153 | func (w *BabylonWatcher) handleBlock(block *types.Block) { 154 | var ( 155 | // chainID = block.Header.ChainID 156 | blockHeight = block.Height 157 | ) 158 | 159 | // Skip if the block is already known 160 | if w.latestBlockHeight >= blockHeight { 161 | return 162 | } 163 | w.latestBlockHeight = blockHeight 164 | 165 | chainID := block.Header.ChainID 166 | 167 | // Handle finality signature for each block 168 | msgs := w.findMsgAddFinalitySig(block.Txs) 169 | w.handleMsgAddFinalitySig(chainID, blockHeight, msgs) 170 | 171 | // Handle injectied checkpoint on the first block of an epoch 172 | if blockHeight%w.epochInterval == 1 { 173 | msg := w.findMsgInjectedCheckpoint(block.Txs) 174 | w.handleMsgInjectedCheckpoint(chainID, blockHeight, msg) 175 | } 176 | } 177 | 178 | func (w *BabylonWatcher) findMsgAddFinalitySig(txs types.Txs) []*finality.MsgAddFinalitySig { 179 | var msgs []*finality.MsgAddFinalitySig 180 | 181 | for _, txBytes := range txs { 182 | var tx tx.Tx 183 | 184 | // Unmarshal the transaction and ignore non-decodable txs 185 | if err := w.protoCodec.Unmarshal(txBytes, &tx); err != nil { 186 | continue 187 | } 188 | 189 | for _, msg := range tx.Body.Messages { 190 | if msg.TypeUrl != "/babylon.finality.v1.MsgAddFinalitySig" { 191 | continue 192 | } 193 | 194 | var cpTx finality.MsgAddFinalitySig 195 | if err := w.protoCodec.Unmarshal(msg.Value, &cpTx); err != nil { 196 | log.Warn().Msgf("failed to decode msg type MsgAddFinalitySig: %v", err) 197 | return nil 198 | } 199 | msgs = append(msgs, &cpTx) 200 | } 201 | } 202 | 203 | return msgs 204 | } 205 | 206 | func (w *BabylonWatcher) handleMsgAddFinalitySig(chainID string, blockHeight int64, msgs []*finality.MsgAddFinalitySig) { 207 | if len(msgs) == 0 { 208 | log.Warn().Int64("height", blockHeight).Msgf("babylon finality provider signature not found") 209 | return 210 | } 211 | 212 | w.metrics.BabylonFinalityVotes.WithLabelValues(chainID).Inc() 213 | 214 | validatorStatus := []string{} 215 | for _, fp := range w.finalityProviders { 216 | _, hasSigned := lo.Find(msgs, func(msg *finality.MsgAddFinalitySig) bool { 217 | return msg.Signer == fp.Address 218 | }) 219 | icon := "??" 220 | if hasSigned { 221 | icon = "✅" 222 | w.metrics.BabylonCommittedFinalityVotes.WithLabelValues(chainID, fp.Address, fp.Label).Inc() 223 | w.metrics.BabylonConsecutiveMissedFinalityVotes.WithLabelValues(chainID, fp.Address, fp.Label).Set(0) 224 | } else { 225 | icon = "❌" 226 | w.metrics.BabylonMissedFinalityVotes.WithLabelValues(chainID, fp.Address, fp.Label).Inc() 227 | w.metrics.BabylonConsecutiveMissedFinalityVotes.WithLabelValues(chainID, fp.Address, fp.Label).Inc() 228 | } 229 | validatorStatus = append(validatorStatus, fmt.Sprintf("%s %s", icon, fp.Label)) 230 | } 231 | 232 | fmt.Fprintln( 233 | w.writer, 234 | color.YellowString(fmt.Sprintf("#%d", blockHeight-1)), 235 | color.MagentaString(fmt.Sprintf("%3d finality prvds", len(msgs))), 236 | strings.Join(validatorStatus, " "), 237 | ) 238 | } 239 | 240 | func (w *BabylonWatcher) findMsgInjectedCheckpoint(txs types.Txs) *checkpointing.MsgInjectedCheckpoint { 241 | for _, txBytes := range txs { 242 | var tx tx.Tx 243 | 244 | // Unmarshal the transaction and ignore non-decodable txs 245 | if err := w.protoCodec.Unmarshal(txBytes, &tx); err != nil { 246 | continue 247 | } 248 | 249 | for _, msg := range tx.Body.Messages { 250 | if msg.TypeUrl != "/babylon.checkpointing.v1.MsgInjectedCheckpoint" { 251 | continue 252 | } 253 | 254 | var cpTx checkpointing.MsgInjectedCheckpoint 255 | if err := w.protoCodec.Unmarshal(msg.Value, &cpTx); err != nil { 256 | log.Warn().Msgf("failed to decode msg type MsgInjectedCheckpoint: %v", err) 257 | return nil 258 | } 259 | return &cpTx 260 | } 261 | } 262 | 263 | return nil 264 | } 265 | 266 | func (w *BabylonWatcher) handleMsgInjectedCheckpoint(chainID string, blockHeight int64, msg *checkpointing.MsgInjectedCheckpoint) { 267 | if msg == nil { 268 | log.Warn().Int64("height", blockHeight).Msgf("babylon checkpoint not found on new epoch") 269 | return 270 | } 271 | 272 | var ( 273 | epoch = msg.Ckpt.Ckpt.EpochNum 274 | totalValidators = len(msg.ExtendedCommitInfo.GetVotes()) 275 | ) 276 | 277 | w.metrics.BabylonEpoch.WithLabelValues(chainID).Set(float64(epoch)) 278 | w.metrics.BabylonCheckpointVote.WithLabelValues(chainID).Inc() 279 | 280 | validatorVotes := make(map[string]bool) 281 | for _, val := range w.trackedValidators { 282 | validatorVotes[val.Address] = false 283 | } 284 | 285 | // Check if validator has voted 286 | validatorStatus := []string{} 287 | for _, val := range w.trackedValidators { 288 | _, hasVoted := lo.Find(msg.ExtendedCommitInfo.Votes, func(vote abcitypes.ExtendedVoteInfo) bool { 289 | return strings.ToUpper(hex.EncodeToString(vote.Validator.Address)) == val.Address 290 | }) 291 | icon := "??" 292 | if hasVoted { 293 | icon = "✅" 294 | w.metrics.BabylonCommittedCheckpointVote.WithLabelValues(chainID, val.Address, val.Name).Inc() 295 | w.metrics.BabylonConsecutiveMissedCheckpointVote.WithLabelValues(chainID, val.Address, val.Name).Set(0) 296 | } else { 297 | icon = "❌" 298 | w.metrics.BabylonMissedCheckpointVote.WithLabelValues(chainID, val.Address, val.Name).Inc() 299 | w.metrics.BabylonConsecutiveMissedCheckpointVote.WithLabelValues(chainID, val.Address, val.Name).Inc() 300 | } 301 | validatorStatus = append(validatorStatus, fmt.Sprintf("%s %s", icon, val.Name)) 302 | } 303 | 304 | votedValidators := lo.CountBy(msg.ExtendedCommitInfo.Votes, func(vote abcitypes.ExtendedVoteInfo) bool { 305 | return vote.BlockIdFlag == tmtypes.BlockIDFlagCommit 306 | }) 307 | 308 | fmt.Fprintln( 309 | w.writer, 310 | color.BlueString(fmt.Sprintf("Epoch #%d", epoch)), 311 | color.CyanString(fmt.Sprintf("%3d/%d validators", votedValidators, totalValidators)), 312 | strings.Join(validatorStatus, " "), 313 | ) 314 | } 315 | -------------------------------------------------------------------------------- /pkg/watcher/block.go: -------------------------------------------------------------------------------- 1 | package watcher 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | "strings" 8 | "sync/atomic" 9 | "time" 10 | 11 | ctypes "github.com/cometbft/cometbft/rpc/core/types" 12 | "github.com/cometbft/cometbft/types" 13 | "github.com/fatih/color" 14 | "github.com/kilnfi/cosmos-validator-watcher/pkg/metrics" 15 | "github.com/kilnfi/cosmos-validator-watcher/pkg/rpc" 16 | "github.com/kilnfi/cosmos-validator-watcher/pkg/webhook" 17 | "github.com/rs/zerolog/log" 18 | "github.com/shopspring/decimal" 19 | ) 20 | 21 | type BlockWebhook struct { 22 | Height int64 `json:"height"` 23 | Metadata map[string]string `json:"metadata"` 24 | } 25 | 26 | type BlockWatcher struct { 27 | trackedValidators []TrackedValidator 28 | metrics *metrics.Metrics 29 | writer io.Writer 30 | blockChan chan *BlockInfo 31 | validatorSet atomic.Value // []*types.Validator 32 | latestBlock BlockInfo 33 | webhook *webhook.Webhook 34 | customWebhooks []BlockWebhook 35 | } 36 | 37 | func NewBlockWatcher(validators []TrackedValidator, metrics *metrics.Metrics, writer io.Writer, webhook *webhook.Webhook, customWebhooks []BlockWebhook) *BlockWatcher { 38 | return &BlockWatcher{ 39 | trackedValidators: validators, 40 | metrics: metrics, 41 | writer: writer, 42 | blockChan: make(chan *BlockInfo), 43 | webhook: webhook, 44 | customWebhooks: customWebhooks, 45 | } 46 | } 47 | 48 | func (w *BlockWatcher) Start(ctx context.Context) error { 49 | for { 50 | select { 51 | case <-ctx.Done(): 52 | return nil 53 | case block := <-w.blockChan: 54 | w.handleBlockInfo(ctx, block) 55 | } 56 | } 57 | } 58 | 59 | func (w *BlockWatcher) OnNodeStart(ctx context.Context, node *rpc.Node) error { 60 | if err := w.syncValidatorSet(ctx, node); err != nil { 61 | return fmt.Errorf("failed to sync validator set: %w", err) 62 | } 63 | 64 | blockResp, err := node.Client.Block(ctx, nil) 65 | if err != nil { 66 | log.Warn().Err(err). 67 | Str("node", node.Redacted()). 68 | Msg("failed to get latest block") 69 | } else { 70 | w.handleNodeBlock(node, blockResp.Block) 71 | } 72 | 73 | // Ticker to sync validator set 74 | ticker := time.NewTicker(time.Minute) 75 | 76 | go func() { 77 | for { 78 | select { 79 | case <-ctx.Done(): 80 | log.Debug().Err(ctx.Err()).Str("node", node.Redacted()).Msgf("stopping block watcher loop") 81 | return 82 | case <-ticker.C: 83 | if err := w.syncValidatorSet(ctx, node); err != nil { 84 | log.Error().Err(err).Str("node", node.Redacted()).Msg("failed to sync validator set") 85 | } 86 | } 87 | } 88 | }() 89 | 90 | return nil 91 | } 92 | 93 | func (w *BlockWatcher) OnNewBlock(ctx context.Context, node *rpc.Node, evt *ctypes.ResultEvent) error { 94 | // Ignore blocks if node is catching up 95 | if !node.IsSynced() { 96 | return nil 97 | } 98 | 99 | blockEvent := evt.Data.(types.EventDataNewBlock) 100 | block := blockEvent.Block 101 | 102 | w.handleNodeBlock(node, block) 103 | 104 | return nil 105 | } 106 | 107 | func (w *BlockWatcher) OnValidatorSetUpdates(ctx context.Context, node *rpc.Node, evt *ctypes.ResultEvent) error { 108 | // Ignore blocks if node is catching up 109 | if !node.IsSynced() { 110 | return nil 111 | } 112 | 113 | w.syncValidatorSet(ctx, node) 114 | 115 | return nil 116 | } 117 | 118 | func (w *BlockWatcher) handleNodeBlock(node *rpc.Node, block *types.Block) { 119 | validatorSet := w.getValidatorSet() 120 | 121 | if len(validatorSet) != block.LastCommit.Size() { 122 | log.Warn().Msgf("validator set size mismatch: %d vs %d", len(validatorSet), block.LastCommit.Size()) 123 | } 124 | 125 | // Set node block height 126 | w.metrics.NodeBlockHeight.WithLabelValues(node.ChainID(), node.Endpoint()).Set(float64(block.Height)) 127 | 128 | // Extract block info 129 | w.blockChan <- NewBlockInfo(block, w.computeValidatorStatus(block)) 130 | } 131 | 132 | func (w *BlockWatcher) getValidatorSet() []*types.Validator { 133 | validatorSet := w.validatorSet.Load() 134 | if validatorSet == nil { 135 | return nil 136 | } 137 | 138 | return validatorSet.([]*types.Validator) 139 | } 140 | 141 | func (w *BlockWatcher) syncValidatorSet(ctx context.Context, n *rpc.Node) error { 142 | validators := make([]*types.Validator, 0) 143 | 144 | for i := 0; i < 5; i++ { 145 | var ( 146 | page int = i + 1 147 | perPage int = 100 148 | ) 149 | 150 | result, err := n.Client.Validators(ctx, nil, &page, &perPage) 151 | if err != nil { 152 | return fmt.Errorf("failed to get validators: %w", err) 153 | } 154 | validators = append(validators, result.Validators...) 155 | 156 | if len(validators) >= int(result.Total) { 157 | break 158 | } 159 | } 160 | 161 | log.Debug(). 162 | Str("node", n.Redacted()). 163 | Int("validators", len(validators)). 164 | Msgf("validator set") 165 | 166 | w.validatorSet.Store(validators) 167 | 168 | return nil 169 | } 170 | 171 | func (w *BlockWatcher) handleBlockInfo(ctx context.Context, block *BlockInfo) { 172 | chainId := block.ChainID 173 | 174 | if w.latestBlock.Height >= block.Height { 175 | // Skip already processed blocks 176 | return 177 | } 178 | 179 | // Ensure to initialize counters for each validator 180 | for _, val := range w.trackedValidators { 181 | w.metrics.ValidatedBlocks.WithLabelValues(chainId, val.Address, val.Name) 182 | w.metrics.MissedBlocks.WithLabelValues(chainId, val.Address, val.Name) 183 | w.metrics.SoloMissedBlocks.WithLabelValues(chainId, val.Address, val.Name) 184 | w.metrics.ConsecutiveMissedBlocks.WithLabelValues(chainId, val.Address, val.Name) 185 | w.metrics.EmptyBlocks.WithLabelValues(chainId, val.Address, val.Name) 186 | } 187 | w.metrics.SkippedBlocks.WithLabelValues(chainId) 188 | 189 | blockDiff := block.Height - w.latestBlock.Height 190 | if w.latestBlock.Height > 0 && blockDiff > 1 { 191 | log.Warn().Msgf("skipped %d unknown blocks", blockDiff-1) 192 | w.metrics.SkippedBlocks.WithLabelValues(chainId).Add(float64(blockDiff)) 193 | } 194 | 195 | w.metrics.BlockHeight.WithLabelValues(chainId).Set(float64(block.Height)) 196 | w.metrics.ActiveSet.WithLabelValues(chainId).Set(float64(block.TotalValidators)) 197 | w.metrics.TrackedBlocks.WithLabelValues(chainId).Inc() 198 | w.metrics.Transactions.WithLabelValues(chainId).Add(float64(block.Transactions)) 199 | 200 | // Print block result & update metrics 201 | validatorStatus := []string{} 202 | for _, res := range block.ValidatorStatus { 203 | icon := "⚪️" 204 | if w.latestBlock.ProposerAddress == res.Address { 205 | // Check if this is an empty block 206 | if w.latestBlock.Transactions == 0 { 207 | icon = "🟡" 208 | w.metrics.EmptyBlocks.WithLabelValues(block.ChainID, res.Address, res.Label).Inc() 209 | } else { 210 | icon = "👑" 211 | } 212 | w.metrics.ProposedBlocks.WithLabelValues(block.ChainID, res.Address, res.Label).Inc() 213 | w.metrics.ValidatedBlocks.WithLabelValues(block.ChainID, res.Address, res.Label).Inc() 214 | w.metrics.ConsecutiveMissedBlocks.WithLabelValues(block.ChainID, res.Address, res.Label).Set(0) 215 | } else if res.Signed { 216 | icon = "✅" 217 | w.metrics.ValidatedBlocks.WithLabelValues(block.ChainID, res.Address, res.Label).Inc() 218 | w.metrics.ConsecutiveMissedBlocks.WithLabelValues(block.ChainID, res.Address, res.Label).Set(0) 219 | } else if res.Bonded { 220 | icon = "❌" 221 | w.metrics.MissedBlocks.WithLabelValues(block.ChainID, res.Address, res.Label).Inc() 222 | w.metrics.ConsecutiveMissedBlocks.WithLabelValues(block.ChainID, res.Address, res.Label).Inc() 223 | 224 | // Check if solo missed block 225 | if block.SignedRatio().GreaterThan(decimal.NewFromFloat(0.66)) { 226 | w.metrics.SoloMissedBlocks.WithLabelValues(block.ChainID, res.Address, res.Label).Inc() 227 | } 228 | } 229 | validatorStatus = append(validatorStatus, fmt.Sprintf("%s %s", icon, res.Label)) 230 | } 231 | 232 | fmt.Fprintln( 233 | w.writer, 234 | color.YellowString(fmt.Sprintf("#%d", block.Height-1)), 235 | color.CyanString(fmt.Sprintf("%3d/%d validators", block.SignedValidators, block.TotalValidators)), 236 | strings.Join(validatorStatus, " "), 237 | ) 238 | 239 | // Handle webhooks 240 | w.handleWebhooks(ctx, block) 241 | 242 | // Save latest block 243 | w.latestBlock = *block 244 | } 245 | 246 | func (w *BlockWatcher) computeValidatorStatus(block *types.Block) []ValidatorStatus { 247 | validatorStatus := []ValidatorStatus{} 248 | 249 | for _, val := range w.trackedValidators { 250 | bonded := w.isValidatorActive(val.Address) 251 | signed := false 252 | rank := 0 253 | for i, sig := range block.LastCommit.Signatures { 254 | if val.Address == sig.ValidatorAddress.String() { 255 | bonded = true 256 | signed = (sig.BlockIDFlag == types.BlockIDFlagCommit) 257 | rank = i + 1 258 | } 259 | if signed { 260 | break 261 | } 262 | } 263 | validatorStatus = append(validatorStatus, ValidatorStatus{ 264 | Address: val.Address, 265 | Label: val.Name, 266 | Bonded: bonded, 267 | Signed: signed, 268 | Rank: rank, 269 | }) 270 | } 271 | 272 | return validatorStatus 273 | } 274 | 275 | func (w *BlockWatcher) isValidatorActive(address string) bool { 276 | for _, val := range w.getValidatorSet() { 277 | if val.Address.String() == address { 278 | return true 279 | } 280 | } 281 | return false 282 | } 283 | 284 | func (w *BlockWatcher) handleWebhooks(ctx context.Context, block *BlockInfo) { 285 | if len(w.customWebhooks) == 0 { 286 | return 287 | } 288 | 289 | newWebhooks := []BlockWebhook{} 290 | 291 | for _, webhook := range w.customWebhooks { 292 | // If webhook block height is passed 293 | if webhook.Height <= block.Height { 294 | w.triggerWebhook(ctx, block.ChainID, webhook) 295 | } else { 296 | newWebhooks = append(newWebhooks, webhook) 297 | } 298 | } 299 | 300 | w.customWebhooks = newWebhooks 301 | } 302 | 303 | func (w *BlockWatcher) triggerWebhook(ctx context.Context, chainID string, wh BlockWebhook) { 304 | msg := make(map[string]string) 305 | msg["type"] = "custom" 306 | msg["block"] = fmt.Sprintf("%d", wh.Height) 307 | msg["chain_id"] = chainID 308 | for k, v := range wh.Metadata { 309 | msg[k] = v 310 | } 311 | 312 | go func() { 313 | if err := w.webhook.Send(context.Background(), msg); err != nil { 314 | log.Error().Err(err).Msg("failed to send upgrade webhook") 315 | } 316 | }() 317 | } 318 | -------------------------------------------------------------------------------- /pkg/watcher/block_test.go: -------------------------------------------------------------------------------- 1 | package watcher 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "net/url" 7 | "strings" 8 | "testing" 9 | 10 | "github.com/kilnfi/cosmos-validator-watcher/pkg/metrics" 11 | "github.com/kilnfi/cosmos-validator-watcher/pkg/webhook" 12 | "github.com/prometheus/client_golang/prometheus/testutil" 13 | "gotest.tools/assert" 14 | ) 15 | 16 | func TestBlockWatcher(t *testing.T) { 17 | var ( 18 | kilnAddress = "3DC4DD610817606AD4A8F9D762A068A81E8741E2" 19 | kilnName = "Kiln" 20 | chainID = "chain-42" 21 | ) 22 | 23 | blockWatcher := NewBlockWatcher( 24 | []TrackedValidator{ 25 | { 26 | Address: kilnAddress, 27 | Name: kilnName, 28 | }, 29 | }, 30 | metrics.New("cosmos_validator_watcher"), 31 | &bytes.Buffer{}, 32 | webhook.New(url.URL{}), 33 | []BlockWebhook{}, 34 | ) 35 | 36 | t.Run("Handle BlockInfo", func(t *testing.T) { 37 | blocks := []BlockInfo{ 38 | { 39 | ChainID: chainID, 40 | Height: 36, 41 | Transactions: 4, 42 | TotalValidators: 1, 43 | SignedValidators: 0, 44 | ValidatorStatus: []ValidatorStatus{ 45 | { 46 | Address: kilnAddress, 47 | Label: kilnName, 48 | Bonded: false, 49 | Signed: false, 50 | Rank: 0, 51 | }, 52 | }, 53 | }, 54 | { 55 | ChainID: chainID, 56 | Height: 41, 57 | Transactions: 5, 58 | TotalValidators: 1, 59 | SignedValidators: 0, 60 | ValidatorStatus: []ValidatorStatus{ 61 | { 62 | Address: kilnAddress, 63 | Label: kilnName, 64 | Bonded: true, 65 | Signed: false, 66 | Rank: 1, 67 | }, 68 | }, 69 | }, 70 | { 71 | ChainID: chainID, 72 | Height: 42, 73 | Transactions: 6, 74 | TotalValidators: 2, 75 | SignedValidators: 1, 76 | ValidatorStatus: []ValidatorStatus{ 77 | { 78 | Address: kilnAddress, 79 | Label: kilnName, 80 | Bonded: true, 81 | Signed: true, 82 | Rank: 2, 83 | }, 84 | }, 85 | }, 86 | { 87 | ChainID: chainID, 88 | Height: 43, 89 | Transactions: 7, 90 | TotalValidators: 2, 91 | SignedValidators: 2, 92 | ProposerAddress: kilnAddress, 93 | ValidatorStatus: []ValidatorStatus{ 94 | { 95 | Address: kilnAddress, 96 | Label: kilnName, 97 | Bonded: true, 98 | Signed: true, 99 | Rank: 2, 100 | }, 101 | }, 102 | }, 103 | { 104 | ChainID: chainID, 105 | Height: 44, 106 | Transactions: 0, 107 | TotalValidators: 2, 108 | SignedValidators: 2, 109 | ProposerAddress: kilnAddress, 110 | ValidatorStatus: []ValidatorStatus{ 111 | { 112 | Address: kilnAddress, 113 | Label: kilnName, 114 | Bonded: true, 115 | Signed: true, 116 | Rank: 2, 117 | }, 118 | }, 119 | }, 120 | { 121 | ChainID: chainID, 122 | Height: 45, 123 | Transactions: 7, 124 | TotalValidators: 2, 125 | SignedValidators: 2, 126 | ValidatorStatus: []ValidatorStatus{ 127 | { 128 | Address: kilnAddress, 129 | Label: kilnName, 130 | Bonded: true, 131 | Signed: true, 132 | Rank: 2, 133 | }, 134 | }, 135 | }, 136 | } 137 | 138 | for _, block := range blocks { 139 | blockWatcher.handleBlockInfo(context.Background(), &block) 140 | } 141 | 142 | assert.Equal(t, 143 | strings.Join([]string{ 144 | `#35 0/1 validators ⚪️ Kiln`, 145 | `#40 0/1 validators ❌ Kiln`, 146 | `#41 1/2 validators ✅ Kiln`, 147 | `#42 2/2 validators ✅ Kiln`, 148 | `#43 2/2 validators 👑 Kiln`, 149 | `#44 2/2 validators 🟡 Kiln`, 150 | }, "\n")+"\n", 151 | blockWatcher.writer.(*bytes.Buffer).String(), 152 | ) 153 | 154 | assert.Equal(t, float64(45), testutil.ToFloat64(blockWatcher.metrics.BlockHeight.WithLabelValues(chainID))) 155 | assert.Equal(t, float64(29), testutil.ToFloat64(blockWatcher.metrics.Transactions.WithLabelValues(chainID))) 156 | assert.Equal(t, float64(2), testutil.ToFloat64(blockWatcher.metrics.ActiveSet.WithLabelValues(chainID))) 157 | assert.Equal(t, float64(6), testutil.ToFloat64(blockWatcher.metrics.TrackedBlocks.WithLabelValues(chainID))) 158 | assert.Equal(t, float64(5), testutil.ToFloat64(blockWatcher.metrics.SkippedBlocks.WithLabelValues(chainID))) 159 | 160 | assert.Equal(t, 1, testutil.CollectAndCount(blockWatcher.metrics.ValidatedBlocks)) 161 | assert.Equal(t, 1, testutil.CollectAndCount(blockWatcher.metrics.MissedBlocks)) 162 | assert.Equal(t, 1, testutil.CollectAndCount(blockWatcher.metrics.SoloMissedBlocks)) 163 | assert.Equal(t, 1, testutil.CollectAndCount(blockWatcher.metrics.ConsecutiveMissedBlocks)) 164 | assert.Equal(t, 1, testutil.CollectAndCount(blockWatcher.metrics.EmptyBlocks)) 165 | assert.Equal(t, float64(2), testutil.ToFloat64(blockWatcher.metrics.ProposedBlocks.WithLabelValues(chainID, kilnAddress, kilnName))) 166 | assert.Equal(t, float64(4), testutil.ToFloat64(blockWatcher.metrics.ValidatedBlocks.WithLabelValues(chainID, kilnAddress, kilnName))) 167 | assert.Equal(t, float64(1), testutil.ToFloat64(blockWatcher.metrics.MissedBlocks.WithLabelValues(chainID, kilnAddress, kilnName))) 168 | assert.Equal(t, float64(0), testutil.ToFloat64(blockWatcher.metrics.SoloMissedBlocks.WithLabelValues(chainID, kilnAddress, kilnName))) 169 | assert.Equal(t, float64(0), testutil.ToFloat64(blockWatcher.metrics.ConsecutiveMissedBlocks.WithLabelValues(chainID, kilnAddress, kilnName))) 170 | assert.Equal(t, float64(1), testutil.ToFloat64(blockWatcher.metrics.EmptyBlocks.WithLabelValues(chainID, kilnAddress, kilnName))) 171 | }) 172 | } 173 | -------------------------------------------------------------------------------- /pkg/watcher/block_types.go: -------------------------------------------------------------------------------- 1 | package watcher 2 | 3 | import ( 4 | "github.com/cometbft/cometbft/types" 5 | "github.com/shopspring/decimal" 6 | ) 7 | 8 | type BlockInfo struct { 9 | ChainID string 10 | Height int64 11 | Transactions int 12 | TotalValidators int 13 | SignedValidators int 14 | ProposerAddress string 15 | ValidatorStatus []ValidatorStatus 16 | } 17 | 18 | func NewBlockInfo(block *types.Block, validatorStatus []ValidatorStatus) *BlockInfo { 19 | // Compute total signed validators 20 | signedValidators := 0 21 | for _, sig := range block.LastCommit.Signatures { 22 | if sig.BlockIDFlag == types.BlockIDFlagCommit { 23 | signedValidators++ 24 | } 25 | } 26 | 27 | return &BlockInfo{ 28 | ChainID: block.Header.ChainID, 29 | Height: block.Header.Height, 30 | Transactions: block.Txs.Len(), 31 | TotalValidators: len(block.LastCommit.Signatures), 32 | SignedValidators: signedValidators, 33 | ValidatorStatus: validatorStatus, 34 | ProposerAddress: block.Header.ProposerAddress.String(), 35 | } 36 | } 37 | 38 | func (b *BlockInfo) SignedRatio() decimal.Decimal { 39 | if b.TotalValidators == 0 { 40 | return decimal.Zero 41 | } 42 | 43 | return decimal.NewFromInt(int64(b.SignedValidators)). 44 | Div(decimal.NewFromInt(int64(b.TotalValidators))) 45 | } 46 | 47 | type ValidatorStatus struct { 48 | Address string 49 | Label string 50 | Bonded bool 51 | Signed bool 52 | Rank int 53 | } 54 | -------------------------------------------------------------------------------- /pkg/watcher/commissions.go: -------------------------------------------------------------------------------- 1 | package watcher 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "time" 7 | 8 | "github.com/cosmos/cosmos-sdk/client" 9 | "github.com/cosmos/cosmos-sdk/types" 10 | distribution "github.com/cosmos/cosmos-sdk/x/distribution/types" 11 | "github.com/kilnfi/cosmos-validator-watcher/pkg/metrics" 12 | "github.com/kilnfi/cosmos-validator-watcher/pkg/rpc" 13 | "github.com/rs/zerolog/log" 14 | ) 15 | 16 | type CommissionWatcher struct { 17 | validators []TrackedValidator 18 | metrics *metrics.Metrics 19 | pool *rpc.Pool 20 | } 21 | 22 | func NewCommissionsWatcher(validators []TrackedValidator, metrics *metrics.Metrics, pool *rpc.Pool) *CommissionWatcher { 23 | return &CommissionWatcher{ 24 | validators: validators, 25 | metrics: metrics, 26 | pool: pool, 27 | } 28 | } 29 | 30 | func (w *CommissionWatcher) Start(ctx context.Context) error { 31 | ticker := time.NewTicker(1 * time.Minute) 32 | 33 | for { 34 | node := w.pool.GetSyncedNode() 35 | if node == nil { 36 | log.Warn().Msg("no node available to fetch validators commissions") 37 | } else if err := w.fetchCommissions(ctx, node); err != nil { 38 | log.Error().Err(err).Msg("failed to fetch validators commissions") 39 | } 40 | 41 | select { 42 | case <-ctx.Done(): 43 | return nil 44 | case <-ticker.C: 45 | } 46 | } 47 | } 48 | 49 | func (w *CommissionWatcher) fetchCommissions(ctx context.Context, node *rpc.Node) error { 50 | for _, validator := range w.validators { 51 | if err := w.fetchValidatorCommission(ctx, node, validator); err != nil { 52 | log.Error().Err(err).Msgf("failed to fetch commission for validator %s", validator.OperatorAddress) 53 | } 54 | } 55 | return nil 56 | } 57 | 58 | func (w *CommissionWatcher) fetchValidatorCommission(ctx context.Context, node *rpc.Node, validator TrackedValidator) error { 59 | clientCtx := (client.Context{}).WithClient(node.Client) 60 | queryClient := distribution.NewQueryClient(clientCtx) 61 | 62 | commissionResq, err := queryClient.ValidatorCommission(ctx, &distribution.QueryValidatorCommissionRequest{ 63 | ValidatorAddress: validator.OperatorAddress, 64 | }) 65 | if err != nil { 66 | return fmt.Errorf("failed to get validators: %w", err) 67 | } 68 | 69 | w.handleValidatorCommission(node.ChainID(), validator, commissionResq.Commission.Commission) 70 | 71 | return nil 72 | } 73 | 74 | func (w *CommissionWatcher) handleValidatorCommission(chainID string, validator TrackedValidator, coins types.DecCoins) { 75 | for _, commission := range coins { 76 | w.metrics.Commission. 77 | WithLabelValues(chainID, validator.Address, validator.Name, commission.Denom). 78 | Set(commission.Amount.MustFloat64()) 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /pkg/watcher/commissions_test.go: -------------------------------------------------------------------------------- 1 | package watcher 2 | 3 | import ( 4 | "testing" 5 | 6 | sdkmath "cosmossdk.io/math" 7 | "github.com/cosmos/cosmos-sdk/types" 8 | "github.com/kilnfi/cosmos-validator-watcher/pkg/metrics" 9 | "github.com/prometheus/client_golang/prometheus/testutil" 10 | "gotest.tools/assert" 11 | ) 12 | 13 | func TestCommissionsWatcher(t *testing.T) { 14 | chainID := "chain-42" 15 | 16 | kilnValidator := TrackedValidator{ 17 | Address: "3DC4DD610817606AD4A8F9D762A068A81E8741E2", 18 | Name: "Kiln", 19 | OperatorAddress: "cosmosvaloper1uxlf7mvr8nep3gm7udf2u9remms2jyjqvwdul2", 20 | } 21 | 22 | watcher := NewCommissionsWatcher( 23 | []TrackedValidator{kilnValidator}, 24 | metrics.New("cosmos_validator_watcher"), 25 | nil, 26 | ) 27 | 28 | t.Run("Handle Commissions", func(t *testing.T) { 29 | watcher.handleValidatorCommission(chainID, kilnValidator, []types.DecCoin{ 30 | { 31 | Denom: "uatom", 32 | Amount: sdkmath.LegacyNewDec(123), 33 | }, 34 | { 35 | Denom: "ibc/0025F8A87464A471E66B234C4F93AEC5B4DA3D42D7986451A059273426290DD5", 36 | Amount: sdkmath.LegacyNewDec(42), 37 | }, 38 | }) 39 | 40 | assert.Equal(t, float64(123), testutil.ToFloat64(watcher.metrics.Commission.WithLabelValues(chainID, kilnValidator.Address, kilnValidator.Name, "uatom"))) 41 | assert.Equal(t, float64(42), testutil.ToFloat64(watcher.metrics.Commission.WithLabelValues(chainID, kilnValidator.Address, kilnValidator.Name, "ibc/0025F8A87464A471E66B234C4F93AEC5B4DA3D42D7986451A059273426290DD5"))) 42 | }) 43 | } 44 | -------------------------------------------------------------------------------- /pkg/watcher/slashing.go: -------------------------------------------------------------------------------- 1 | package watcher 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "time" 7 | 8 | "github.com/cosmos/cosmos-sdk/client" 9 | slashing "github.com/cosmos/cosmos-sdk/x/slashing/types" 10 | "github.com/kilnfi/cosmos-validator-watcher/pkg/metrics" 11 | "github.com/kilnfi/cosmos-validator-watcher/pkg/rpc" 12 | "github.com/rs/zerolog/log" 13 | ) 14 | 15 | type SlashingWatcher struct { 16 | metrics *metrics.Metrics 17 | pool *rpc.Pool 18 | 19 | signedBlocksWindow int64 20 | minSignedPerWindow float64 21 | downtimeJailDuration float64 22 | slashFractionDoubleSign float64 23 | slashFractionDowntime float64 24 | } 25 | 26 | func NewSlashingWatcher(metrics *metrics.Metrics, pool *rpc.Pool) *SlashingWatcher { 27 | return &SlashingWatcher{ 28 | metrics: metrics, 29 | pool: pool, 30 | } 31 | } 32 | 33 | func (w *SlashingWatcher) Start(ctx context.Context) error { 34 | // update metrics every 30 minutes 35 | ticker := time.NewTicker(30 * time.Minute) 36 | 37 | for { 38 | node := w.pool.GetSyncedNode() 39 | if node == nil { 40 | log.Warn().Msg("no node available to fetch slashing parameters") 41 | } else if err := w.fetchSlashingParameters(ctx, node); err != nil { 42 | log.Error().Err(err). 43 | Str("node", node.Redacted()). 44 | Msg("failed to fetch slashing parameters") 45 | } 46 | 47 | select { 48 | case <-ctx.Done(): 49 | return nil 50 | case <-ticker.C: 51 | } 52 | } 53 | } 54 | 55 | func (w *SlashingWatcher) fetchSlashingParameters(ctx context.Context, node *rpc.Node) error { 56 | clientCtx := (client.Context{}).WithClient(node.Client) 57 | queryClient := slashing.NewQueryClient(clientCtx) 58 | sigininParams, err := queryClient.Params(ctx, &slashing.QueryParamsRequest{}) 59 | if err != nil { 60 | return fmt.Errorf("failed to get slashing parameters: %w", err) 61 | } 62 | 63 | w.handleSlashingParams(node.ChainID(), sigininParams.Params) 64 | 65 | return nil 66 | 67 | } 68 | 69 | func (w *SlashingWatcher) handleSlashingParams(chainID string, params slashing.Params) { 70 | log.Debug(). 71 | Str("chainID", chainID). 72 | Str("downtimeJailDuration", params.DowntimeJailDuration.String()). 73 | Str("minSignedPerWindow", fmt.Sprintf("%.2f", params.MinSignedPerWindow.MustFloat64())). 74 | Str("signedBlocksWindow", fmt.Sprint(params.SignedBlocksWindow)). 75 | Str("slashFractionDoubleSign", fmt.Sprintf("%.2f", params.SlashFractionDoubleSign.MustFloat64())). 76 | Str("slashFractionDowntime", fmt.Sprintf("%.2f", params.SlashFractionDowntime.MustFloat64())). 77 | Msgf("updating slashing metrics") 78 | 79 | w.signedBlocksWindow = params.SignedBlocksWindow 80 | w.minSignedPerWindow, _ = params.MinSignedPerWindow.Float64() 81 | w.downtimeJailDuration = params.DowntimeJailDuration.Seconds() 82 | w.slashFractionDoubleSign, _ = params.SlashFractionDoubleSign.Float64() 83 | w.slashFractionDowntime, _ = params.SlashFractionDowntime.Float64() 84 | 85 | w.metrics.SignedBlocksWindow.WithLabelValues(chainID).Set(float64(w.signedBlocksWindow)) 86 | w.metrics.MinSignedBlocksPerWindow.WithLabelValues(chainID).Set(w.minSignedPerWindow) 87 | w.metrics.DowntimeJailDuration.WithLabelValues(chainID).Set(w.downtimeJailDuration) 88 | w.metrics.SlashFractionDoubleSign.WithLabelValues(chainID).Set(w.slashFractionDoubleSign) 89 | w.metrics.SlashFractionDowntime.WithLabelValues(chainID).Set(w.slashFractionDowntime) 90 | } 91 | -------------------------------------------------------------------------------- /pkg/watcher/slashing_test.go: -------------------------------------------------------------------------------- 1 | package watcher 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | cosmossdk_io_math "cosmossdk.io/math" 8 | slashing "github.com/cosmos/cosmos-sdk/x/slashing/types" 9 | "github.com/kilnfi/cosmos-validator-watcher/pkg/metrics" 10 | "github.com/prometheus/client_golang/prometheus/testutil" 11 | "gotest.tools/assert" 12 | ) 13 | 14 | func TestSlashingWatcher(t *testing.T) { 15 | var chainID = "test-chain" 16 | 17 | watcher := NewSlashingWatcher( 18 | metrics.New("cosmos_validator_watcher"), 19 | nil, 20 | ) 21 | 22 | t.Run("Handle Slashing Parameters", func(t *testing.T) { 23 | 24 | minSignedPerWindow := cosmossdk_io_math.LegacyMustNewDecFromStr("0.1") 25 | slashFractionDoubleSign := cosmossdk_io_math.LegacyMustNewDecFromStr("0.01") 26 | slashFractionDowntime := cosmossdk_io_math.LegacyMustNewDecFromStr("0.001") 27 | 28 | params := slashing.Params{ 29 | SignedBlocksWindow: int64(1000), 30 | MinSignedPerWindow: minSignedPerWindow, 31 | DowntimeJailDuration: time.Duration(10) * time.Second, 32 | SlashFractionDoubleSign: slashFractionDoubleSign, 33 | SlashFractionDowntime: slashFractionDowntime, 34 | } 35 | 36 | watcher.handleSlashingParams(chainID, params) 37 | 38 | assert.Equal(t, float64(1000), testutil.ToFloat64(watcher.metrics.SignedBlocksWindow.WithLabelValues(chainID))) 39 | assert.Equal(t, float64(0.1), testutil.ToFloat64(watcher.metrics.MinSignedBlocksPerWindow.WithLabelValues(chainID))) 40 | assert.Equal(t, float64(10), testutil.ToFloat64(watcher.metrics.DowntimeJailDuration.WithLabelValues(chainID))) 41 | assert.Equal(t, float64(0.01), testutil.ToFloat64(watcher.metrics.SlashFractionDoubleSign.WithLabelValues(chainID))) 42 | assert.Equal(t, float64(0.001), testutil.ToFloat64(watcher.metrics.SlashFractionDowntime.WithLabelValues(chainID))) 43 | }) 44 | 45 | } 46 | -------------------------------------------------------------------------------- /pkg/watcher/status.go: -------------------------------------------------------------------------------- 1 | package watcher 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | ctypes "github.com/cometbft/cometbft/rpc/core/types" 8 | "github.com/kilnfi/cosmos-validator-watcher/pkg/metrics" 9 | "github.com/kilnfi/cosmos-validator-watcher/pkg/rpc" 10 | "github.com/rs/zerolog/log" 11 | ) 12 | 13 | type StatusWatcher struct { 14 | metrics *metrics.Metrics 15 | chainID string 16 | statusChan chan *ctypes.ResultStatus 17 | } 18 | 19 | func NewStatusWatcher(chainID string, metrics *metrics.Metrics) *StatusWatcher { 20 | return &StatusWatcher{ 21 | metrics: metrics, 22 | chainID: chainID, 23 | statusChan: make(chan *ctypes.ResultStatus), 24 | } 25 | } 26 | 27 | func (w *StatusWatcher) Start(ctx context.Context) error { 28 | for { 29 | select { 30 | case <-ctx.Done(): 31 | return nil 32 | case status := <-w.statusChan: 33 | if status == nil { 34 | continue 35 | } else if w.chainID == "" { 36 | w.chainID = status.NodeInfo.Network 37 | } else if w.chainID != status.NodeInfo.Network { 38 | return fmt.Errorf("chain ID mismatch: %s != %s", w.chainID, status.NodeInfo.Network) 39 | } 40 | } 41 | } 42 | } 43 | 44 | func (w *StatusWatcher) OnNodeStatus(ctx context.Context, n *rpc.Node, status *ctypes.ResultStatus) error { 45 | synced := false 46 | blockHeight := int64(0) 47 | chainID := "" 48 | 49 | if status != nil { 50 | synced = !status.SyncInfo.CatchingUp 51 | blockHeight = status.SyncInfo.LatestBlockHeight 52 | chainID = status.NodeInfo.Network 53 | } 54 | 55 | log.Debug(). 56 | Str("node", n.Redacted()). 57 | Int64("height", blockHeight). 58 | Bool("synced", synced). 59 | Msgf("node status") 60 | 61 | w.metrics.NodeSynced.WithLabelValues(chainID, n.Endpoint()).Set( 62 | metrics.BoolToFloat64(synced), 63 | ) 64 | 65 | w.statusChan <- status 66 | 67 | return nil 68 | } 69 | -------------------------------------------------------------------------------- /pkg/watcher/types.go: -------------------------------------------------------------------------------- 1 | package watcher 2 | 3 | import ( 4 | "strings" 5 | 6 | "github.com/cosmos/cosmos-sdk/types/bech32" 7 | utils "github.com/kilnfi/cosmos-validator-watcher/pkg/crypto" 8 | ) 9 | 10 | type TrackedValidator struct { 11 | Address string 12 | Name string 13 | Moniker string 14 | OperatorAddress string 15 | ConsensusAddress string 16 | } 17 | 18 | func ParseValidator(val string) TrackedValidator { 19 | parts := strings.Split(val, ":") 20 | if len(parts) > 1 { 21 | return TrackedValidator{ 22 | Address: parts[0], 23 | Name: parts[1], 24 | } 25 | } 26 | 27 | return TrackedValidator{ 28 | Address: parts[0], 29 | Name: parts[0], 30 | } 31 | } 32 | 33 | func (t TrackedValidator) AccountAddress() string { 34 | _, bytes, err := bech32.DecodeAndConvert(t.OperatorAddress) 35 | if err != nil { 36 | return err.Error() 37 | } 38 | 39 | prefix := utils.GetHrpPrefix(t.OperatorAddress) 40 | 41 | conv, err := bech32.ConvertAndEncode(prefix, bytes) 42 | if err != nil { 43 | return err.Error() 44 | } 45 | 46 | return conv 47 | } 48 | -------------------------------------------------------------------------------- /pkg/watcher/types_test.go: -------------------------------------------------------------------------------- 1 | package watcher 2 | 3 | import ( 4 | "testing" 5 | 6 | "gotest.tools/assert" 7 | ) 8 | 9 | func TestTrackedValidator(t *testing.T) { 10 | 11 | t.Run("AccountAddress", func(t *testing.T) { 12 | testdata := []struct { 13 | Address string 14 | Account string 15 | }{ 16 | { 17 | Address: "cosmosvaloper1uxlf7mvr8nep3gm7udf2u9remms2jyjqvwdul2", 18 | Account: "cosmos1uxlf7mvr8nep3gm7udf2u9remms2jyjqf6efne", 19 | }, 20 | { 21 | Address: "cosmosvaloper1n229vhepft6wnkt5tjpwmxdmcnfz55jv3vp77d", 22 | Account: "cosmos1n229vhepft6wnkt5tjpwmxdmcnfz55jv5c4tj7", 23 | }, 24 | } 25 | 26 | for _, td := range testdata { 27 | 28 | v := TrackedValidator{ 29 | OperatorAddress: td.Address, 30 | } 31 | 32 | assert.Equal(t, v.AccountAddress(), td.Account) 33 | } 34 | }) 35 | } 36 | -------------------------------------------------------------------------------- /pkg/watcher/upgrade.go: -------------------------------------------------------------------------------- 1 | package watcher 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "time" 7 | 8 | upgrade "cosmossdk.io/x/upgrade/types" 9 | ctypes "github.com/cometbft/cometbft/rpc/core/types" 10 | comettypes "github.com/cometbft/cometbft/types" 11 | "github.com/cosmos/cosmos-sdk/client" 12 | codectypes "github.com/cosmos/cosmos-sdk/codec/types" 13 | query "github.com/cosmos/cosmos-sdk/types/query" 14 | gov "github.com/cosmos/cosmos-sdk/x/gov/types/v1" 15 | govbeta "github.com/cosmos/cosmos-sdk/x/gov/types/v1beta1" 16 | "github.com/gogo/protobuf/codec" 17 | "github.com/kilnfi/cosmos-validator-watcher/pkg/metrics" 18 | "github.com/kilnfi/cosmos-validator-watcher/pkg/rpc" 19 | "github.com/kilnfi/cosmos-validator-watcher/pkg/webhook" 20 | "github.com/rs/zerolog/log" 21 | ) 22 | 23 | type UpgradeWatcher struct { 24 | metrics *metrics.Metrics 25 | pool *rpc.Pool 26 | webhook *webhook.Webhook 27 | options UpgradeWatcherOptions 28 | 29 | nextUpgradePlan *upgrade.Plan // known upgrade plan 30 | latestBlockHeight int64 // latest block received 31 | latestWebhookSent int64 // latest block for which webhook has been sent 32 | } 33 | 34 | type UpgradeWatcherOptions struct { 35 | CheckPendingProposals bool 36 | GovModuleVersion string 37 | } 38 | 39 | func NewUpgradeWatcher(metrics *metrics.Metrics, pool *rpc.Pool, webhook *webhook.Webhook, options UpgradeWatcherOptions) *UpgradeWatcher { 40 | return &UpgradeWatcher{ 41 | metrics: metrics, 42 | pool: pool, 43 | webhook: webhook, 44 | options: options, 45 | } 46 | } 47 | 48 | func (w *UpgradeWatcher) Start(ctx context.Context) error { 49 | ticker := time.NewTicker(1 * time.Minute) 50 | 51 | for { 52 | node := w.pool.GetSyncedNode() 53 | if node == nil { 54 | log.Warn().Msg("no node available to fetch upgrade plan") 55 | } else if err := w.fetchUpgrade(ctx, node); err != nil { 56 | log.Error().Err(err). 57 | Str("node", node.Redacted()). 58 | Msg("failed to fetch upgrade plan") 59 | } 60 | 61 | select { 62 | case <-ctx.Done(): 63 | return nil 64 | case <-ticker.C: 65 | } 66 | } 67 | } 68 | 69 | func (w *UpgradeWatcher) OnNewBlock(ctx context.Context, node *rpc.Node, evt *ctypes.ResultEvent) error { 70 | blockEvent := evt.Data.(comettypes.EventDataNewBlock) 71 | block := blockEvent.Block 72 | 73 | // Skip already processed blocks 74 | if w.latestBlockHeight >= block.Height { 75 | return nil 76 | } 77 | 78 | w.latestBlockHeight = block.Height 79 | 80 | // Ignore is webhook is not configured 81 | if w.webhook == nil { 82 | return nil 83 | } 84 | 85 | // Ignore if no upgrade plan 86 | if w.nextUpgradePlan == nil { 87 | return nil 88 | } 89 | 90 | // Ignore blocks if node is catching up 91 | if !node.IsSynced() { 92 | return nil 93 | } 94 | 95 | // Ignore if upgrade plan is for a future block 96 | if w.latestBlockHeight < w.nextUpgradePlan.Height-1 { 97 | return nil 98 | } 99 | 100 | // Ignore if webhook has already been sent 101 | if w.latestWebhookSent >= w.nextUpgradePlan.Height { 102 | return nil 103 | } 104 | 105 | // Upgrade plan is for this block 106 | go w.triggerWebhook(ctx, node.ChainID(), *w.nextUpgradePlan) 107 | w.latestWebhookSent = w.nextUpgradePlan.Height 108 | w.nextUpgradePlan = nil 109 | 110 | return nil 111 | } 112 | 113 | func (w *UpgradeWatcher) triggerWebhook(ctx context.Context, chainID string, plan upgrade.Plan) { 114 | msg := struct { 115 | Type string `json:"type"` 116 | Block int64 `json:"block"` 117 | ChainID string `json:"chain_id"` 118 | Version string `json:"version"` 119 | }{ 120 | Type: "upgrade", 121 | Block: plan.Height, 122 | ChainID: chainID, 123 | Version: plan.Name, 124 | } 125 | 126 | if err := w.webhook.Send(ctx, msg); err != nil { 127 | log.Error().Err(err).Msg("failed to send upgrade webhook") 128 | } 129 | } 130 | 131 | func (w *UpgradeWatcher) fetchUpgrade(ctx context.Context, node *rpc.Node) error { 132 | clientCtx := (client.Context{}).WithClient(node.Client) 133 | queryClient := upgrade.NewQueryClient(clientCtx) 134 | 135 | resp, err := queryClient.CurrentPlan(ctx, &upgrade.QueryCurrentPlanRequest{}) 136 | if err != nil { 137 | return err 138 | } 139 | 140 | plan := resp.Plan 141 | 142 | if plan == nil && w.options.CheckPendingProposals { 143 | switch w.options.GovModuleVersion { 144 | case "v1beta1": 145 | plan, err = w.checkUpgradeProposalsV1Beta1(ctx, node) 146 | default: // v1 147 | plan, err = w.checkUpgradeProposalsV1(ctx, node) 148 | } 149 | if err != nil { 150 | log.Error().Err(err).Msg("failed to check upgrade proposals") 151 | } 152 | } 153 | 154 | w.handleUpgradePlan(node.ChainID(), plan) 155 | 156 | return nil 157 | } 158 | 159 | func (w *UpgradeWatcher) checkUpgradeProposalsV1(ctx context.Context, node *rpc.Node) (*upgrade.Plan, error) { 160 | clientCtx := (client.Context{}).WithClient(node.Client) 161 | queryClient := gov.NewQueryClient(clientCtx) 162 | 163 | // Fetch all proposals in voting period 164 | proposalsResp, err := queryClient.Proposals(ctx, &gov.QueryProposalsRequest{ 165 | ProposalStatus: gov.StatusVotingPeriod, 166 | Pagination: &query.PageRequest{ 167 | Reverse: true, 168 | Limit: 100, 169 | }, 170 | }) 171 | if err != nil { 172 | return nil, fmt.Errorf("failed to get proposals: %w", err) 173 | } 174 | 175 | for _, proposal := range proposalsResp.GetProposals() { 176 | for _, message := range proposal.Messages { 177 | plan, err := extractUpgradePlan(message) 178 | if err != nil { 179 | return nil, fmt.Errorf("failed to extract upgrade plan: %w", err) 180 | } 181 | if plan != nil && plan.Height > w.latestBlockHeight { 182 | return plan, nil 183 | } 184 | } 185 | } 186 | 187 | return nil, nil 188 | } 189 | 190 | func (w *UpgradeWatcher) checkUpgradeProposalsV1Beta1(ctx context.Context, node *rpc.Node) (*upgrade.Plan, error) { 191 | clientCtx := (client.Context{}).WithClient(node.Client) 192 | queryClient := govbeta.NewQueryClient(clientCtx) 193 | 194 | // Fetch all proposals in voting period 195 | proposalsResp, err := queryClient.Proposals(ctx, &govbeta.QueryProposalsRequest{ 196 | ProposalStatus: govbeta.StatusVotingPeriod, 197 | }) 198 | if err != nil { 199 | return nil, fmt.Errorf("failed to get proposals: %w", err) 200 | } 201 | 202 | for _, proposal := range proposalsResp.GetProposals() { 203 | plan, err := extractUpgradePlan(proposal.Content) 204 | if err != nil { 205 | return nil, fmt.Errorf("failed to extract upgrade plan: %w", err) 206 | } 207 | if plan != nil && plan.Height > w.latestBlockHeight { 208 | return plan, nil 209 | } 210 | } 211 | 212 | return nil, nil 213 | } 214 | 215 | func extractUpgradePlan(content *codectypes.Any) (*upgrade.Plan, error) { 216 | if content == nil { 217 | return nil, nil 218 | } 219 | 220 | cdc := codec.New(1) 221 | 222 | switch content.TypeUrl { 223 | case "/cosmos.upgrade.v1beta1.SoftwareUpgradeProposal": 224 | var upgrade upgrade.SoftwareUpgradeProposal 225 | err := cdc.Unmarshal(content.Value, &upgrade) 226 | if err != nil { 227 | return nil, fmt.Errorf("failed to unmarshal software upgrade proposal: %w", err) 228 | } 229 | return &upgrade.Plan, nil 230 | 231 | case "/cosmos.upgrade.v1beta1.MsgSoftwareUpgrade": 232 | var upgrade upgrade.MsgSoftwareUpgrade 233 | err := cdc.Unmarshal(content.Value, &upgrade) 234 | if err != nil { 235 | return nil, fmt.Errorf("failed to unmarshal software upgrade proposal: %w", err) 236 | } 237 | return &upgrade.Plan, nil 238 | } 239 | 240 | return nil, nil 241 | } 242 | 243 | func (w *UpgradeWatcher) handleUpgradePlan(chainID string, plan *upgrade.Plan) { 244 | w.nextUpgradePlan = plan 245 | 246 | if plan == nil { 247 | w.metrics.UpgradePlan.Reset() 248 | } else { 249 | w.metrics.UpgradePlan.WithLabelValues(chainID, plan.Name, fmt.Sprintf("%d", plan.Height)).Set(float64(plan.Height)) 250 | } 251 | } 252 | -------------------------------------------------------------------------------- /pkg/watcher/upgrade_test.go: -------------------------------------------------------------------------------- 1 | package watcher 2 | 3 | import ( 4 | "testing" 5 | 6 | upgrade "cosmossdk.io/x/upgrade/types" 7 | "github.com/kilnfi/cosmos-validator-watcher/pkg/metrics" 8 | "github.com/prometheus/client_golang/prometheus/testutil" 9 | "gotest.tools/assert" 10 | ) 11 | 12 | func TestUpgradeWatcher(t *testing.T) { 13 | var chainID = "chain-42" 14 | 15 | watcher := NewUpgradeWatcher( 16 | metrics.New("cosmos_validator_watcher"), 17 | nil, 18 | nil, 19 | UpgradeWatcherOptions{}, 20 | ) 21 | 22 | t.Run("Handle Upgrade Plan", func(t *testing.T) { 23 | var ( 24 | version = "v42.0.0" 25 | blockHeight = int64(123456789) 26 | ) 27 | 28 | watcher.handleUpgradePlan(chainID, &upgrade.Plan{ 29 | Name: version, 30 | Height: blockHeight, 31 | }) 32 | 33 | assert.Equal(t, float64(123456789), testutil.ToFloat64(watcher.metrics.UpgradePlan.WithLabelValues(chainID, version, "123456789"))) 34 | }) 35 | 36 | t.Run("Handle No Upgrade Plan", func(t *testing.T) { 37 | watcher.handleUpgradePlan(chainID, nil) 38 | 39 | assert.Equal(t, 0, testutil.CollectAndCount(watcher.metrics.UpgradePlan)) 40 | }) 41 | } 42 | -------------------------------------------------------------------------------- /pkg/watcher/validators.go: -------------------------------------------------------------------------------- 1 | package watcher 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "sort" 7 | "time" 8 | 9 | "github.com/cosmos/cosmos-sdk/client" 10 | "github.com/cosmos/cosmos-sdk/types/query" 11 | slashing "github.com/cosmos/cosmos-sdk/x/slashing/types" 12 | staking "github.com/cosmos/cosmos-sdk/x/staking/types" 13 | "github.com/kilnfi/cosmos-validator-watcher/pkg/crypto" 14 | "github.com/kilnfi/cosmos-validator-watcher/pkg/metrics" 15 | "github.com/kilnfi/cosmos-validator-watcher/pkg/rpc" 16 | "github.com/rs/zerolog/log" 17 | "github.com/shopspring/decimal" 18 | ) 19 | 20 | type ValidatorsWatcher struct { 21 | metrics *metrics.Metrics 22 | validators []TrackedValidator 23 | pool *rpc.Pool 24 | opts ValidatorsWatcherOptions 25 | } 26 | 27 | type ValidatorsWatcherOptions struct { 28 | Denom string 29 | DenomExponent uint 30 | NoSlashing bool 31 | } 32 | 33 | func NewValidatorsWatcher(validators []TrackedValidator, metrics *metrics.Metrics, pool *rpc.Pool, opts ValidatorsWatcherOptions) *ValidatorsWatcher { 34 | return &ValidatorsWatcher{ 35 | metrics: metrics, 36 | validators: validators, 37 | pool: pool, 38 | opts: opts, 39 | } 40 | } 41 | 42 | func (w *ValidatorsWatcher) Start(ctx context.Context) error { 43 | ticker := time.NewTicker(30 * time.Second) 44 | 45 | for { 46 | node := w.pool.GetSyncedNode() 47 | if node == nil { 48 | log.Warn().Msg("no node available to fetch validators") 49 | } else if err := w.fetchValidators(ctx, node); err != nil { 50 | log.Error().Err(err). 51 | Str("node", node.Redacted()). 52 | Msg("failed to fetch staking validators") 53 | } else if err := w.fetchSigningInfos(ctx, node); err != nil { 54 | log.Error().Err(err). 55 | Str("node", node.Redacted()). 56 | Msg("failed to fetch signing infos") 57 | } 58 | select { 59 | case <-ctx.Done(): 60 | return nil 61 | case <-ticker.C: 62 | } 63 | } 64 | } 65 | 66 | func (w *ValidatorsWatcher) fetchSigningInfos(ctx context.Context, node *rpc.Node) error { 67 | if !w.opts.NoSlashing { 68 | clientCtx := (client.Context{}).WithClient(node.Client) 69 | queryClient := slashing.NewQueryClient(clientCtx) 70 | signingInfos, err := queryClient.SigningInfos(ctx, &slashing.QuerySigningInfosRequest{ 71 | Pagination: &query.PageRequest{ 72 | Limit: 3000, 73 | }, 74 | }) 75 | if err != nil { 76 | return fmt.Errorf("failed to get signing infos: %w", err) 77 | } 78 | 79 | w.handleSigningInfos(node.ChainID(), signingInfos.Info) 80 | 81 | return nil 82 | } else { 83 | return nil 84 | } 85 | 86 | } 87 | 88 | func (w *ValidatorsWatcher) fetchValidators(ctx context.Context, node *rpc.Node) error { 89 | clientCtx := (client.Context{}).WithClient(node.Client) 90 | queryClient := staking.NewQueryClient(clientCtx) 91 | 92 | validators, err := queryClient.Validators(ctx, &staking.QueryValidatorsRequest{ 93 | Pagination: &query.PageRequest{ 94 | Limit: 3000, 95 | }, 96 | }) 97 | if err != nil { 98 | return fmt.Errorf("failed to get validators: %w", err) 99 | } 100 | 101 | w.handleValidators(node.ChainID(), validators.Validators) 102 | 103 | return nil 104 | } 105 | 106 | func (w *ValidatorsWatcher) handleSigningInfos(chainID string, signingInfos []slashing.ValidatorSigningInfo) { 107 | for _, tracked := range w.validators { 108 | 109 | for _, val := range signingInfos { 110 | 111 | if tracked.ConsensusAddress == val.Address { 112 | w.metrics.MissedBlocksWindow.WithLabelValues(chainID, tracked.Address, tracked.Name).Set(float64(val.MissedBlocksCounter)) 113 | break 114 | } 115 | } 116 | 117 | } 118 | } 119 | 120 | func (w *ValidatorsWatcher) handleValidators(chainID string, validators []staking.Validator) { 121 | // Sort validators by tokens & status (bonded, unbonded, jailed) 122 | sort.Sort(RankedValidators(validators)) 123 | 124 | denomExponent := w.opts.DenomExponent 125 | if denomExponent == 0 { 126 | denomExponent = 1 127 | } 128 | 129 | seatPrice := decimal.Zero 130 | for _, val := range validators { 131 | tokens := decimal.NewFromBigInt(val.Tokens.BigInt(), -int32(denomExponent)) 132 | if val.Status == staking.Bonded && (seatPrice.IsZero() || seatPrice.GreaterThan(tokens)) { 133 | seatPrice = tokens 134 | } 135 | w.metrics.SeatPrice.WithLabelValues(chainID, w.opts.Denom).Set(seatPrice.InexactFloat64()) 136 | } 137 | 138 | for _, tracked := range w.validators { 139 | name := tracked.Name 140 | 141 | for i, val := range validators { 142 | address := crypto.PubKeyAddress(val.ConsensusPubkey) 143 | if tracked.Address == address { 144 | var ( 145 | rank = i + 1 146 | isBonded = val.Status == staking.Bonded 147 | isJailed = val.Jailed 148 | tokens = decimal.NewFromBigInt(val.Tokens.BigInt(), -int32(denomExponent)) 149 | ) 150 | 151 | w.metrics.Rank.WithLabelValues(chainID, address, name).Set(float64(rank)) 152 | w.metrics.Tokens.WithLabelValues(chainID, address, name, w.opts.Denom).Set(tokens.InexactFloat64()) 153 | w.metrics.IsBonded.WithLabelValues(chainID, address, name).Set(metrics.BoolToFloat64(isBonded)) 154 | w.metrics.IsJailed.WithLabelValues(chainID, address, name).Set(metrics.BoolToFloat64(isJailed)) 155 | break 156 | } 157 | } 158 | } 159 | } 160 | 161 | type RankedValidators []staking.Validator 162 | 163 | func (p RankedValidators) Len() int { return len(p) } 164 | func (p RankedValidators) Swap(i, j int) { p[i], p[j] = p[j], p[i] } 165 | func (s RankedValidators) Less(i, j int) bool { 166 | // Jailed validators are always last 167 | if s[i].Jailed && !s[j].Jailed { 168 | return false 169 | } else if !s[i].Jailed && s[j].Jailed { 170 | return true 171 | } 172 | 173 | // Not bonded validators are after bonded validators 174 | if s[i].Status == staking.Bonded && s[j].Status != staking.Bonded { 175 | return true 176 | } else if s[i].Status != staking.Bonded && s[j].Status == staking.Bonded { 177 | return false 178 | } 179 | 180 | return s[i].Tokens.BigInt().Cmp(s[j].Tokens.BigInt()) > 0 181 | } 182 | -------------------------------------------------------------------------------- /pkg/watcher/validators_test.go: -------------------------------------------------------------------------------- 1 | package watcher 2 | 3 | import ( 4 | "encoding/hex" 5 | "testing" 6 | 7 | "cosmossdk.io/math" 8 | codectypes "github.com/cosmos/cosmos-sdk/codec/types" 9 | slashing "github.com/cosmos/cosmos-sdk/x/slashing/types" 10 | staking "github.com/cosmos/cosmos-sdk/x/staking/types" 11 | utils "github.com/kilnfi/cosmos-validator-watcher/pkg/crypto" 12 | "github.com/kilnfi/cosmos-validator-watcher/pkg/metrics" 13 | "github.com/prometheus/client_golang/prometheus/testutil" 14 | "github.com/stretchr/testify/require" 15 | "gotest.tools/assert" 16 | ) 17 | 18 | func TestValidatorsWatcher(t *testing.T) { 19 | var ( 20 | kilnAddress = "3DC4DD610817606AD4A8F9D762A068A81E8741E2" 21 | kilnName = "Kiln" 22 | chainID = "chain-42" 23 | ) 24 | 25 | validatorsWatcher := NewValidatorsWatcher( 26 | []TrackedValidator{ 27 | { 28 | Address: kilnAddress, 29 | Name: kilnName, 30 | ConsensusAddress: "cosmosvalcons18hzd6cggzasx449gl8tk9grg4q0gws0z52nvvy", 31 | }, 32 | }, 33 | metrics.New("cosmos_validator_watcher"), 34 | nil, 35 | ValidatorsWatcherOptions{ 36 | Denom: "denom", 37 | DenomExponent: 6, 38 | NoSlashing: false, 39 | }, 40 | ) 41 | 42 | t.Run("Handle Validators", func(t *testing.T) { 43 | createAddress := func(pubkey string) *codectypes.Any { 44 | prefix := "0000" 45 | ba, err := hex.DecodeString(prefix + pubkey) 46 | require.NoError(t, err) 47 | 48 | return &codectypes.Any{ 49 | TypeUrl: "/cosmos.crypto.ed25519.PubKey", 50 | Value: ba, 51 | } 52 | } 53 | 54 | createConsAddress := func(pubkey codectypes.Any) string { 55 | prefix := "cosmosvalcons" 56 | consensusAddress := utils.PubKeyBech32Address(&pubkey, prefix) 57 | return consensusAddress 58 | } 59 | 60 | validators := []staking.Validator{ 61 | { 62 | OperatorAddress: "", 63 | ConsensusPubkey: createAddress("915dea44121fbceb01452f98ca005b457fe8360c5e191b6601ee01b8a8d407a0"), // 3DC4DD610817606AD4A8F9D762A068A81E8741E2 64 | Jailed: false, 65 | Status: staking.Bonded, 66 | Tokens: math.NewInt(42000000), 67 | }, 68 | { 69 | OperatorAddress: "", 70 | ConsensusPubkey: createAddress("0000000000000000000000000000000000000000000000000000000000000001"), 71 | Jailed: false, 72 | Status: staking.Bonded, 73 | Tokens: math.NewInt(43000000), 74 | }, 75 | { 76 | OperatorAddress: "", 77 | ConsensusPubkey: createAddress("0000000000000000000000000000000000000000000000000000000000000002"), 78 | Jailed: false, 79 | Status: staking.Unbonded, 80 | Tokens: math.NewInt(1000000), 81 | }, 82 | { 83 | OperatorAddress: "", 84 | ConsensusPubkey: createAddress("0000000000000000000000000000000000000000000000000000000000000003"), 85 | Jailed: true, 86 | Status: staking.Bonded, 87 | Tokens: math.NewInt(99000000), 88 | }, 89 | } 90 | 91 | validatorSigningInfo := []slashing.ValidatorSigningInfo{ 92 | { 93 | Address: createConsAddress(*createAddress("915dea44121fbceb01452f98ca005b457fe8360c5e191b6601ee01b8a8d407a0")), 94 | MissedBlocksCounter: 3, 95 | }} 96 | 97 | validatorsWatcher.handleValidators(chainID, validators) 98 | validatorsWatcher.handleSigningInfos(chainID, validatorSigningInfo) 99 | 100 | assert.Equal(t, float64(42), testutil.ToFloat64(validatorsWatcher.metrics.Tokens.WithLabelValues(chainID, kilnAddress, kilnName, "denom"))) 101 | assert.Equal(t, float64(2), testutil.ToFloat64(validatorsWatcher.metrics.Rank.WithLabelValues(chainID, kilnAddress, kilnName))) 102 | assert.Equal(t, float64(1), testutil.ToFloat64(validatorsWatcher.metrics.IsBonded.WithLabelValues(chainID, kilnAddress, kilnName))) 103 | assert.Equal(t, float64(0), testutil.ToFloat64(validatorsWatcher.metrics.IsJailed.WithLabelValues(chainID, kilnAddress, kilnName))) 104 | 105 | assert.Equal(t, float64(3), testutil.ToFloat64(validatorsWatcher.metrics.MissedBlocksWindow.WithLabelValues(chainID, kilnAddress, kilnName))) 106 | }) 107 | } 108 | -------------------------------------------------------------------------------- /pkg/watcher/votes.go: -------------------------------------------------------------------------------- 1 | package watcher 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "time" 7 | 8 | "github.com/cosmos/cosmos-sdk/client" 9 | gov "github.com/cosmos/cosmos-sdk/x/gov/types/v1" 10 | govbeta "github.com/cosmos/cosmos-sdk/x/gov/types/v1beta1" 11 | "github.com/kilnfi/cosmos-validator-watcher/pkg/metrics" 12 | "github.com/kilnfi/cosmos-validator-watcher/pkg/rpc" 13 | "github.com/rs/zerolog/log" 14 | "google.golang.org/grpc/codes" 15 | "google.golang.org/grpc/status" 16 | ) 17 | 18 | type VotesWatcher struct { 19 | metrics *metrics.Metrics 20 | validators []TrackedValidator 21 | pool *rpc.Pool 22 | options VotesWatcherOptions 23 | } 24 | 25 | type VotesWatcherOptions struct { 26 | GovModuleVersion string 27 | } 28 | 29 | func NewVotesWatcher(validators []TrackedValidator, metrics *metrics.Metrics, pool *rpc.Pool, options VotesWatcherOptions) *VotesWatcher { 30 | return &VotesWatcher{ 31 | metrics: metrics, 32 | validators: validators, 33 | pool: pool, 34 | options: options, 35 | } 36 | } 37 | 38 | func (w *VotesWatcher) Start(ctx context.Context) error { 39 | ticker := time.NewTicker(1 * time.Minute) 40 | 41 | for { 42 | node := w.pool.GetSyncedNode() 43 | if node == nil { 44 | log.Warn().Msg("no node available to fetch proposals") 45 | } else if err := w.fetchProposals(ctx, node); err != nil { 46 | log.Error().Err(err). 47 | Str("node", node.Redacted()). 48 | Msg("failed to fetch pending proposals") 49 | } 50 | 51 | select { 52 | case <-ctx.Done(): 53 | return nil 54 | case <-ticker.C: 55 | } 56 | } 57 | } 58 | 59 | func (w *VotesWatcher) fetchProposals(ctx context.Context, node *rpc.Node) error { 60 | var ( 61 | votes map[uint64]map[TrackedValidator]bool 62 | err error 63 | ) 64 | 65 | switch w.options.GovModuleVersion { 66 | case "v1beta1": 67 | votes, err = w.fetchProposalsV1Beta1(ctx, node) 68 | default: // v1 69 | votes, err = w.fetchProposalsV1(ctx, node) 70 | } 71 | 72 | if err != nil { 73 | return err 74 | } 75 | 76 | w.metrics.Vote.Reset() 77 | for proposalId, votes := range votes { 78 | for validator, voted := range votes { 79 | w.metrics.Vote. 80 | WithLabelValues(node.ChainID(), validator.Address, validator.Name, fmt.Sprintf("%d", proposalId)). 81 | Set(metrics.BoolToFloat64(voted)) 82 | } 83 | } 84 | 85 | return nil 86 | } 87 | 88 | func (w *VotesWatcher) fetchProposalsV1(ctx context.Context, node *rpc.Node) (map[uint64]map[TrackedValidator]bool, error) { 89 | votes := make(map[uint64]map[TrackedValidator]bool) 90 | 91 | clientCtx := (client.Context{}).WithClient(node.Client) 92 | queryClient := gov.NewQueryClient(clientCtx) 93 | 94 | // Fetch all proposals in voting period 95 | proposalsResp, err := queryClient.Proposals(ctx, &gov.QueryProposalsRequest{ 96 | ProposalStatus: gov.StatusVotingPeriod, 97 | }) 98 | if err != nil { 99 | return votes, fmt.Errorf("failed to fetch proposals in voting period: %w", err) 100 | } 101 | 102 | chainID := node.ChainID() 103 | 104 | // For each proposal, fetch validators vote 105 | for _, proposal := range proposalsResp.GetProposals() { 106 | votes[proposal.Id] = make(map[TrackedValidator]bool) 107 | w.metrics.ProposalEndTime.WithLabelValues(chainID, fmt.Sprintf("%d", proposal.Id)).Set(float64(proposal.VotingEndTime.Unix())) 108 | 109 | for _, validator := range w.validators { 110 | voter := validator.AccountAddress() 111 | if voter == "" { 112 | log.Warn().Str("validator", validator.Name).Msg("no account address for validator") 113 | continue 114 | } 115 | voteResp, err := queryClient.Vote(ctx, &gov.QueryVoteRequest{ 116 | ProposalId: proposal.Id, 117 | Voter: voter, 118 | }) 119 | 120 | if isInvalidArgumentError(err) { 121 | votes[proposal.Id][validator] = false 122 | } else if err != nil { 123 | votes[proposal.Id][validator] = false 124 | log.Warn(). 125 | Str("validator", validator.Name). 126 | Str("proposal", fmt.Sprintf("%d", proposal.Id)). 127 | Err(err).Msg("failed to get validator vote for proposal") 128 | } else { 129 | vote := voteResp.GetVote() 130 | voted := false 131 | for _, option := range vote.Options { 132 | if option.Option != gov.OptionEmpty { 133 | voted = true 134 | break 135 | } 136 | } 137 | votes[proposal.Id][validator] = voted 138 | } 139 | } 140 | } 141 | 142 | return votes, nil 143 | } 144 | 145 | func (w *VotesWatcher) fetchProposalsV1Beta1(ctx context.Context, node *rpc.Node) (map[uint64]map[TrackedValidator]bool, error) { 146 | votes := make(map[uint64]map[TrackedValidator]bool) 147 | 148 | clientCtx := (client.Context{}).WithClient(node.Client) 149 | queryClient := govbeta.NewQueryClient(clientCtx) 150 | 151 | // Fetch all proposals in voting period 152 | proposalsResp, err := queryClient.Proposals(ctx, &govbeta.QueryProposalsRequest{ 153 | ProposalStatus: govbeta.StatusVotingPeriod, 154 | }) 155 | if err != nil { 156 | return votes, fmt.Errorf("failed to fetch proposals in voting period: %w", err) 157 | } 158 | 159 | chainID := node.ChainID() 160 | 161 | // For each proposal, fetch validators vote 162 | for _, proposal := range proposalsResp.GetProposals() { 163 | votes[proposal.ProposalId] = make(map[TrackedValidator]bool) 164 | w.metrics.ProposalEndTime.WithLabelValues(chainID, fmt.Sprintf("%d", proposal.ProposalId)).Set(float64(proposal.VotingEndTime.Unix())) 165 | 166 | for _, validator := range w.validators { 167 | voter := validator.AccountAddress() 168 | if voter == "" { 169 | log.Warn().Str("validator", validator.Name).Msg("no account address for validator") 170 | continue 171 | } 172 | voteResp, err := queryClient.Vote(ctx, &govbeta.QueryVoteRequest{ 173 | ProposalId: proposal.ProposalId, 174 | Voter: voter, 175 | }) 176 | 177 | if isInvalidArgumentError(err) { 178 | votes[proposal.ProposalId][validator] = false 179 | } else if err != nil { 180 | votes[proposal.ProposalId][validator] = false 181 | log.Warn(). 182 | Str("validator", validator.Name). 183 | Str("proposal", fmt.Sprintf("%d", proposal.ProposalId)). 184 | Err(err).Msg("failed to get validator vote for proposal") 185 | } else { 186 | vote := voteResp.GetVote() 187 | voted := false 188 | for _, option := range vote.Options { 189 | if option.Option != govbeta.OptionEmpty { 190 | voted = true 191 | break 192 | } 193 | } 194 | votes[proposal.ProposalId][validator] = voted 195 | } 196 | } 197 | } 198 | 199 | return votes, nil 200 | } 201 | 202 | func (w *VotesWatcher) handleVoteV1Beta1(chainID string, validator TrackedValidator, proposalId uint64, votes []govbeta.WeightedVoteOption) { 203 | voted := false 204 | for _, option := range votes { 205 | if option.Option != govbeta.OptionEmpty { 206 | voted = true 207 | break 208 | } 209 | } 210 | 211 | w.metrics.Vote. 212 | WithLabelValues(chainID, validator.Address, validator.Name, fmt.Sprintf("%d", proposalId)). 213 | Set(metrics.BoolToFloat64(voted)) 214 | } 215 | 216 | func isInvalidArgumentError(err error) bool { 217 | st, ok := status.FromError(err) 218 | if !ok { 219 | return false 220 | } 221 | return st.Code() == codes.InvalidArgument 222 | } 223 | -------------------------------------------------------------------------------- /pkg/watcher/votes_test.go: -------------------------------------------------------------------------------- 1 | package watcher 2 | 3 | import ( 4 | "testing" 5 | 6 | gov "github.com/cosmos/cosmos-sdk/x/gov/types/v1beta1" 7 | "github.com/kilnfi/cosmos-validator-watcher/pkg/metrics" 8 | "github.com/prometheus/client_golang/prometheus/testutil" 9 | "gotest.tools/assert" 10 | ) 11 | 12 | func TestVotesWatcher(t *testing.T) { 13 | var ( 14 | kilnAddress = "3DC4DD610817606AD4A8F9D762A068A81E8741E2" 15 | kilnName = "Kiln" 16 | chainID = "chain-42" 17 | validators = []TrackedValidator{ 18 | { 19 | Address: kilnAddress, 20 | Name: kilnName, 21 | }, 22 | } 23 | ) 24 | 25 | votesWatcher := NewVotesWatcher( 26 | validators, 27 | metrics.New("cosmos_validator_watcher"), 28 | nil, 29 | VotesWatcherOptions{ 30 | GovModuleVersion: "v1beta1", 31 | }, 32 | ) 33 | 34 | t.Run("Handle Votes", func(t *testing.T) { 35 | votesWatcher.handleVoteV1Beta1(chainID, validators[0], 40, nil) 36 | votesWatcher.handleVoteV1Beta1(chainID, validators[0], 41, []gov.WeightedVoteOption{{Option: gov.OptionEmpty}}) 37 | votesWatcher.handleVoteV1Beta1(chainID, validators[0], 42, []gov.WeightedVoteOption{{Option: gov.OptionYes}}) 38 | 39 | assert.Equal(t, float64(0), testutil.ToFloat64(votesWatcher.metrics.Vote.WithLabelValues(chainID, kilnAddress, kilnName, "40"))) 40 | assert.Equal(t, float64(0), testutil.ToFloat64(votesWatcher.metrics.Vote.WithLabelValues(chainID, kilnAddress, kilnName, "41"))) 41 | assert.Equal(t, float64(1), testutil.ToFloat64(votesWatcher.metrics.Vote.WithLabelValues(chainID, kilnAddress, kilnName, "42"))) 42 | }) 43 | } 44 | -------------------------------------------------------------------------------- /pkg/webhook/webhook.go: -------------------------------------------------------------------------------- 1 | package webhook 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/json" 7 | "fmt" 8 | "net/http" 9 | "net/url" 10 | "time" 11 | 12 | "github.com/avast/retry-go/v4" 13 | "github.com/rs/zerolog/log" 14 | ) 15 | 16 | type Webhook struct { 17 | endpoint url.URL 18 | client *http.Client 19 | } 20 | 21 | func New(endpoint url.URL) *Webhook { 22 | return &Webhook{ 23 | endpoint: endpoint, 24 | client: &http.Client{}, 25 | } 26 | } 27 | 28 | func (w *Webhook) Send(ctx context.Context, message interface{}) error { 29 | body, err := json.Marshal(message) 30 | if err != nil { 31 | return fmt.Errorf("failed to marshal message: %w", err) 32 | } 33 | 34 | log.Info().Msgf("sending webhook: %s", body) 35 | 36 | req, err := http.NewRequestWithContext(ctx, "POST", w.endpoint.String(), bytes.NewBuffer(body)) 37 | if err != nil { 38 | return fmt.Errorf("failed to create request: %w", err) 39 | } 40 | 41 | req.Header.Set("Content-Type", "application/json") 42 | 43 | retryOpts := []retry.Option{ 44 | retry.Context(ctx), 45 | retry.Delay(1 * time.Second), 46 | retry.Attempts(3), 47 | retry.OnRetry(func(_ uint, err error) { 48 | log.Warn().Err(err).Msgf("retrying webhook on %s", w.endpoint.String()) 49 | }), 50 | } 51 | 52 | return retry.Do(func() error { 53 | return w.postRequest(ctx, req) 54 | }, retryOpts...) 55 | } 56 | 57 | func (w *Webhook) postRequest(_ context.Context, req *http.Request) error { 58 | resp, err := w.client.Do(req) 59 | if err != nil { 60 | return fmt.Errorf("failed to send request: %w", err) 61 | } 62 | defer resp.Body.Close() 63 | 64 | // Check if response is not 4xx or 5xx 65 | if resp.StatusCode >= 400 { 66 | return fmt.Errorf("unexpected response status: %s", resp.Status) 67 | } 68 | 69 | return nil 70 | } 71 | -------------------------------------------------------------------------------- /tests/e2e.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "io" 8 | "net/http" 9 | "strings" 10 | "time" 11 | 12 | "github.com/kilnfi/cosmos-validator-watcher/pkg/app" 13 | "github.com/rs/zerolog/log" 14 | "github.com/urfave/cli/v2" 15 | "golang.org/x/sync/errgroup" 16 | ) 17 | 18 | var data = []struct { 19 | node string 20 | validator string 21 | }{ 22 | {"https://cosmos-rpc.publicnode.com:443", "3DC4DD610817606AD4A8F9D762A068A81E8741E2"}, 23 | {"https://evmos-rpc.publicnode.com:443", "C28576ECA1802CD7046913848784D8147F6BAB22"}, 24 | {"https://injective-rpc.publicnode.com:443", "37CE71AD8CD4DC01D8798E103A70978591708169"}, 25 | {"https://juno-rpc.publicnode.com:443", "C55563E7FAD03561899AC6638E3E1E07DCB68D99"}, 26 | {"https://kava-rpc.publicnode.com:443", "C669FD6A91DD9771BCEAE988E2C58356E8B33383"}, 27 | // {"https://neutron-rpc.publicnode.com:443", "D2C7578217BA3ACEE64120FBCABD1B47EA51F9CE"}, 28 | {"https://osmosis-rpc.publicnode.com:443", "17386B308EF9670CDD412FB2F3C09C5B875FB8B8"}, 29 | {"https://persistence-rpc.publicnode.com:443", "C016A67FFD0E91FF9F0A81CF639F05B9161D90C8"}, 30 | {"https://quicksilver-rpc.publicnode.com:443", "72A46AB01C191843223BF54B29201D87D7CEC44E"}, 31 | {"https://stride-rpc.publicnode.com:443", "6F5D7AB6943BE2D01DD910CCD7B06D9E30A0FEFF"}, 32 | {"https://rpc-sei-ia.cosmosia.notional.ventures:443", "908A25DA02880FAA7C1BE016A5ABFD165D9D9087"}, 33 | } 34 | 35 | func main() { 36 | ctx := context.Background() 37 | 38 | app := &cli.App{ 39 | Name: "cosmos-validator-watcher", 40 | Flags: app.Flags, 41 | Action: app.RunFunc, 42 | } 43 | 44 | for _, d := range data { 45 | args := []string{ 46 | "cosmos-validator-watcher", 47 | "--node", d.node, 48 | "--validator", d.validator, 49 | } 50 | 51 | // Allow 3 seconds timeout 52 | ctx, cancel := context.WithTimeout(ctx, 3*time.Second) 53 | defer cancel() 54 | 55 | errg, ctx := errgroup.WithContext(ctx) 56 | errg.Go(func() error { 57 | return app.RunContext(ctx, args) 58 | }) 59 | errg.Go(func() error { 60 | time.Sleep(1500 * time.Millisecond) 61 | return checkMetrics(ctx) 62 | }) 63 | 64 | if err := errg.Wait(); err != nil && !errors.Is(err, context.Canceled) { 65 | log.Fatal().Err(err).Msg("") 66 | } 67 | } 68 | } 69 | 70 | // checkMetrics ensures if the given metric is present 71 | func checkMetrics(ctx context.Context) error { 72 | metric := "cosmos_validator_watcher_block_height" 73 | 74 | req, err := http.NewRequestWithContext(ctx, "GET", "http://localhost:8080/metrics", nil) 75 | if err != nil { 76 | return fmt.Errorf("error creating request: %w", err) 77 | } 78 | 79 | client := http.DefaultClient 80 | resp, err := client.Do(req) 81 | if err != nil { 82 | return fmt.Errorf("error sending request: %w", err) 83 | } 84 | defer resp.Body.Close() 85 | 86 | body, err := io.ReadAll(resp.Body) 87 | if err != nil { 88 | return fmt.Errorf("error reading response body: %w", err) 89 | } 90 | 91 | if !strings.Contains(string(body), metric) { 92 | return fmt.Errorf("metric '%s' is not present", metric) 93 | } 94 | 95 | log.Info().Msg("metrics are present") 96 | 97 | return nil 98 | } 99 | --------------------------------------------------------------------------------