├── .editorconfig ├── .github ├── issue_template.md └── workflows │ ├── release-cli.yml │ ├── release-plugin-agave.yml │ ├── release-richat.yml │ └── test.yml ├── .gitignore ├── CHANGELOG.md ├── Cargo.lock ├── Cargo.toml ├── Makefile ├── README.md ├── cli ├── Cargo.toml ├── richat-track.yml └── src │ ├── bin │ └── richat-cli.rs │ ├── lib.rs │ ├── pubsub.rs │ ├── stream.rs │ ├── stream_grpc.rs │ ├── stream_richat.rs │ └── track.rs ├── client ├── Cargo.toml ├── build.rs └── src │ ├── error.rs │ ├── grpc.rs │ ├── lib.rs │ ├── quic.rs │ └── stream.rs ├── deny.toml ├── filter ├── Cargo.toml └── src │ ├── config.rs │ ├── filter.rs │ ├── lib.rs │ ├── message.rs │ └── protobuf │ ├── decode.rs │ ├── encode.rs │ └── mod.rs ├── plugin-agave ├── Cargo.toml ├── README.md ├── benches │ └── encode │ │ ├── account.rs │ │ ├── block_meta.rs │ │ ├── entry.rs │ │ ├── main.rs │ │ ├── slot.rs │ │ └── transaction.rs ├── build.rs ├── config.json ├── fixtures │ ├── blocks │ │ ├── 18144001.bincode │ │ ├── 43200000.bincode │ │ └── 64800004.bincode │ └── target ├── fuzz │ ├── Cargo.lock │ ├── Cargo.toml │ └── fuzz_targets │ │ ├── account.rs │ │ ├── blockmeta.rs │ │ ├── entry.rs │ │ ├── slot.rs │ │ └── transaction.rs └── src │ ├── bin │ └── config-check.rs │ ├── channel.rs │ ├── config.rs │ ├── lib.rs │ ├── metrics.rs │ ├── plugin.rs │ ├── protobuf │ ├── encoding │ │ ├── account.rs │ │ ├── block_meta.rs │ │ ├── entry.rs │ │ ├── mod.rs │ │ ├── slot.rs │ │ └── transaction.rs │ ├── message.rs │ └── mod.rs │ └── version.rs ├── proto ├── Cargo.toml ├── build.rs ├── proto │ └── richat.proto └── src │ └── lib.rs ├── richat ├── Cargo.toml ├── build.rs ├── config.yml └── src │ ├── bin │ └── richat.rs │ ├── channel.rs │ ├── config.rs │ ├── grpc │ ├── block_meta.rs │ ├── config.rs │ ├── mod.rs │ └── server.rs │ ├── lib.rs │ ├── log.rs │ ├── metrics.rs │ ├── pubsub │ ├── config.rs │ ├── filter.rs │ ├── mod.rs │ ├── notification.rs │ ├── server.rs │ ├── solana.rs │ └── tracker.rs │ ├── richat │ ├── config.rs │ ├── mod.rs │ └── server.rs │ ├── source.rs │ └── version.rs ├── rust-toolchain.toml ├── rustfmt.toml └── shared ├── Cargo.toml ├── build.rs └── src ├── config.rs ├── five8.rs ├── jsonrpc ├── helpers.rs ├── metrics.rs ├── mod.rs └── requests.rs ├── lib.rs ├── metrics.rs ├── shutdown.rs ├── transports ├── grpc.rs ├── mod.rs └── quic.rs └── version.rs /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | end_of_line = lf 5 | charset = utf-8 6 | trim_trailing_whitespace = true 7 | insert_final_newline = true 8 | 9 | [*.{diff,md}] 10 | trim_trailing_whitespace = false 11 | 12 | [*.{js,json,proto,yaml,yml}] 13 | indent_style = space 14 | indent_size = 2 15 | 16 | [*.{rs,toml}] 17 | indent_style = space 18 | indent_size = 4 19 | 20 | [{Makefile,**.mk}] 21 | indent_style = tab 22 | -------------------------------------------------------------------------------- /.github/issue_template.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Default issue 3 | --- 4 | 5 | Please use issues only for reporting bugs or discussing feature-related topics. If you're having trouble loading a plugin or need guidance on how to use crates, please post your question in the Telegram group: [https://t.me/lamportsdev](https://t.me/lamportsdev) 6 | -------------------------------------------------------------------------------- /.github/workflows/release-cli.yml: -------------------------------------------------------------------------------- 1 | permissions: 2 | contents: write 3 | 4 | concurrency: 5 | group: ${{ github.workflow }}-${{ github.ref }} 6 | cancel-in-progress: true 7 | 8 | on: 9 | pull_request: 10 | paths: 11 | - '.github/workflows/release-cli.yml' 12 | push: 13 | branches: 14 | - master 15 | - agave-v2.0 16 | - agave-v2.1 17 | - agave-v2.2 18 | tags: 19 | - 'cli-v*' 20 | workflow_dispatch: 21 | 22 | jobs: 23 | test: 24 | strategy: 25 | matrix: 26 | os: 27 | - ubuntu-22.04 28 | - ubuntu-24.04 29 | runs-on: ["${{ matrix.os }}"] 30 | steps: 31 | - uses: actions/checkout@v4 32 | 33 | - name: install dependencies 34 | run: | 35 | sudo apt-get update 36 | sudo apt-get install libudev-dev 37 | 38 | - uses: fanatid/rust-github-ci-prepare@master 39 | with: 40 | cache-version: v0002-cli 41 | 42 | - name: Build richat-cli 43 | run: | 44 | cargo build -p richat-cli --release 45 | cd target/release && \ 46 | mv richat-cli richat-cli-${{ matrix.os }} 47 | 48 | - name: Upload artifact 49 | uses: actions/upload-artifact@v4 50 | with: 51 | name: richat-cli-${{ matrix.os }}-${{ github.sha }} 52 | path: | 53 | target/release/richat-cli-${{ matrix.os }} 54 | 55 | - name: Upload release 56 | if: startsWith(github.ref, 'refs/tags/') 57 | uses: softprops/action-gh-release@v2 58 | with: 59 | files: | 60 | target/release/richat-cli-${{ matrix.os }} 61 | -------------------------------------------------------------------------------- /.github/workflows/release-plugin-agave.yml: -------------------------------------------------------------------------------- 1 | permissions: 2 | contents: write 3 | 4 | concurrency: 5 | group: ${{ github.workflow }}-${{ github.ref }} 6 | cancel-in-progress: true 7 | 8 | on: 9 | pull_request: 10 | paths: 11 | - '.github/workflows/release-plugin-agave.yml' 12 | push: 13 | branches: 14 | - master 15 | - agave-v2.0 16 | - agave-v2.1 17 | - agave-v2.2 18 | tags: 19 | - 'plugin-agave-v*' 20 | workflow_dispatch: 21 | 22 | jobs: 23 | test: 24 | strategy: 25 | matrix: 26 | os: 27 | - ubuntu-22.04 28 | - ubuntu-24.04 29 | runs-on: ["${{ matrix.os }}"] 30 | steps: 31 | - uses: actions/checkout@v4 32 | 33 | - name: install dependencies 34 | run: | 35 | sudo apt-get update 36 | sudo apt-get install libudev-dev 37 | 38 | - uses: fanatid/rust-github-ci-prepare@master 39 | with: 40 | cache-version: v0002-plugin-agave 41 | 42 | - name: Build richat plugin agave 43 | run: | 44 | cargo build -p richat-plugin-agave --release 45 | cd target/release && \ 46 | mv richat-plugin-agave-config-check richat-plugin-agave-config-check-${{ matrix.os }} && \ 47 | mv librichat_plugin_agave.so librichat_plugin_agave_${{ matrix.os }}.so 48 | 49 | - name: Upload artifact 50 | uses: actions/upload-artifact@v4 51 | with: 52 | name: richat-plugin-agave-${{ matrix.os }}-${{ github.sha }} 53 | path: | 54 | target/release/richat-plugin-agave-config-check-${{ matrix.os }} 55 | target/release/librichat_plugin_agave_${{ matrix.os }}.so 56 | 57 | - name: Upload release 58 | if: startsWith(github.ref, 'refs/tags/') 59 | uses: softprops/action-gh-release@v2 60 | with: 61 | files: | 62 | target/release/richat-plugin-agave-config-check-${{ matrix.os }} 63 | target/release/librichat_plugin_agave_${{ matrix.os }}.so 64 | -------------------------------------------------------------------------------- /.github/workflows/release-richat.yml: -------------------------------------------------------------------------------- 1 | permissions: 2 | contents: write 3 | 4 | concurrency: 5 | group: ${{ github.workflow }}-${{ github.ref }} 6 | cancel-in-progress: true 7 | 8 | on: 9 | pull_request: 10 | paths: 11 | - '.github/workflows/release-richat.yml' 12 | push: 13 | branches: 14 | - master 15 | - agave-v2.0 16 | - agave-v2.1 17 | - agave-v2.2 18 | tags: 19 | - 'richat-v*' 20 | workflow_dispatch: 21 | 22 | jobs: 23 | test: 24 | strategy: 25 | matrix: 26 | os: 27 | - ubuntu-22.04 28 | - ubuntu-24.04 29 | runs-on: ["${{ matrix.os }}"] 30 | steps: 31 | - uses: actions/checkout@v4 32 | 33 | - name: install dependencies 34 | run: | 35 | sudo apt-get update 36 | sudo apt-get install libudev-dev 37 | 38 | - uses: fanatid/rust-github-ci-prepare@master 39 | with: 40 | cache-version: v0002-richat 41 | 42 | - name: Build richat 43 | run: | 44 | cargo build -p richat --release 45 | cd target/release && \ 46 | mv richat richat-${{ matrix.os }} 47 | 48 | - name: Upload artifact 49 | uses: actions/upload-artifact@v4 50 | with: 51 | name: richat-${{ matrix.os }}-${{ github.sha }} 52 | path: | 53 | target/release/richat-${{ matrix.os }} 54 | 55 | - name: Upload release 56 | if: startsWith(github.ref, 'refs/tags/') 57 | uses: softprops/action-gh-release@v2 58 | with: 59 | files: | 60 | target/release/richat-${{ matrix.os }} 61 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | concurrency: 2 | group: ${{ github.workflow }}-${{ github.ref }} 3 | cancel-in-progress: true 4 | 5 | on: 6 | pull_request: 7 | push: 8 | branches: 9 | - master 10 | - agave-v2.0 11 | - agave-v2.1 12 | - agave-v2.2 13 | workflow_dispatch: 14 | 15 | env: 16 | CARGO_TERM_COLOR: always 17 | 18 | jobs: 19 | test: 20 | strategy: 21 | matrix: 22 | os: 23 | - ubuntu-22.04 24 | - ubuntu-24.04 25 | runs-on: ["${{ matrix.os }}"] 26 | steps: 27 | - uses: actions/checkout@v4 28 | 29 | - name: install dependencies 30 | run: | 31 | sudo apt-get update 32 | sudo apt-get install libudev-dev 33 | 34 | - uses: fanatid/rust-github-ci-prepare@master 35 | with: 36 | cache-version: v0003 37 | 38 | - name: cargo deny check advisories 39 | uses: EmbarkStudios/cargo-deny-action@v1 40 | with: 41 | command: check advisories 42 | 43 | - name: run clippy 44 | run: cargo clippy --workspace --all-targets -- -Dwarnings 45 | 46 | - name: run clippy in plugin-fuzz 47 | run: cd plugin-agave/fuzz && cargo clippy --workspace --all-targets -- -Dwarnings 48 | 49 | - name: check features in `richat-cli` 50 | run: cargo check -p richat-cli --all-targets 51 | 52 | - name: check features in `richat-client` 53 | run: cargo check -p richat-client --all-targets 54 | 55 | - name: check features in `richat-filter` 56 | run: cargo check -p richat-filter --all-targets 57 | 58 | - name: check features in `richat-plugin-agave` 59 | run: cargo check -p richat-plugin-agave --all-targets 60 | 61 | - name: check features in `richat-plugin-agave` with all features 62 | run: cargo check -p richat-plugin-agave --all-targets --all-features 63 | 64 | - name: check features in `richat` 65 | run: cargo check -p richat --all-targets 66 | 67 | - name: check features in `richat-shared` 68 | run: cargo check -p richat-shared --all-targets 69 | 70 | - name: check features in `richat-shared` 71 | run: cargo check -p richat-shared --all-targets --no-default-features --features="config" 72 | 73 | - name: check features in `richat-shared` 74 | run: cargo check -p richat-shared --all-targets --no-default-features --features="five8" 75 | 76 | - name: check features in `richat-shared` 77 | run: cargo check -p richat-shared --all-targets --no-default-features --features="jsonrpc" 78 | 79 | - name: check features in `richat-shared` 80 | run: cargo check -p richat-shared --all-targets --no-default-features --features="metrics" 81 | 82 | - name: check features in `richat-shared` 83 | run: cargo check -p richat-shared --all-targets --no-default-features --features="shutdown" 84 | 85 | - name: check features in `richat-shared` 86 | run: cargo check -p richat-shared --all-targets --no-default-features --features="transports" 87 | 88 | - name: check features in `richat-shared` 89 | run: cargo check -p richat-shared --all-targets --no-default-features --features="version" 90 | 91 | - name: run test 92 | run: cargo test --all-targets 93 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Project 2 | plugin-agave/config.dev.json 3 | plugin-agave/fixtures/blocks 4 | richat/config.dev.yml 5 | 6 | # Solana 7 | test-ledger 8 | 9 | # Rust 10 | target 11 | 12 | # IDE 13 | .idea 14 | .vscode 15 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | All notable changes to this project will be documented in this file. 4 | 5 | The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 6 | and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 7 | 8 | **Note:** Version 0 of Semantic Versioning is handled differently from version 1 and above. 9 | The minor version will be incremented upon a breaking change and the patch version will be incremented for features. 10 | 11 | ## [Unreleased] 12 | 13 | ### Fixes 14 | 15 | - filter: fix memcmp decode data in the config ([#118](https://github.com/lamports-dev/richat/pull/118)) 16 | 17 | ### Features 18 | 19 | - proto: add version to quic response ([#115](https://github.com/lamports-dev/richat/pull/115)) 20 | - richat: use jemalloc ([#117](https://github.com/lamports-dev/richat/pull/117)) 21 | 22 | ### Breaking 23 | 24 | - richat: replay only processed commitment ([#116](https://github.com/lamports-dev/richat/pull/116)) 25 | 26 | ## 2025-05-30 27 | 28 | - richat-cli-v4.5.0 29 | - richat-client-v3.5.0 30 | - richat-filter-v3.5.0 31 | - richat-plugin-agave-v3.5.0 32 | - richat-proto-v3.1.0 33 | - richat-v3.6.0 34 | - richat-shared-v3.5.0 35 | 36 | ### Fixes 37 | 38 | - richat: receive all slot statuses for confirmed / finalized ([#113](https://github.com/lamports-dev/richat/pull/113)) 39 | 40 | ### Features 41 | 42 | - richat: impl `SubscribeReplayInfo` ([#109](https://github.com/lamports-dev/richat/pull/109)) 43 | - richat: faster account messages deduper ([#110](https://github.com/lamports-dev/richat/pull/110)) 44 | - richat: remove tcp ([#111](https://github.com/lamports-dev/richat/pull/111)) 45 | - richat: use foldhash for performance ([#112](https://github.com/lamports-dev/richat/pull/112)) 46 | - shared: affinity by relative cores ([#114](https://github.com/lamports-dev/richat/pull/114)) 47 | 48 | ## 2025-05-03 49 | 50 | - richat-cli-v4.4.0 51 | - richat-client-v3.4.0 52 | - richat-filter-v3.4.0 53 | - richat-plugin-agave-v3.4.0 54 | - richat-v3.5.0 55 | - richat-shared-v3.4.0 56 | 57 | ### Features 58 | 59 | - richat: detailed mismatch error message ([#107](https://github.com/lamports-dev/richat/pull/107)) 60 | - shared: `accept Vec` instead of `serde_json::Value` ([#107](https://github.com/lamports-dev/richat/pull/107)) 61 | 62 | ## 2025-04-21 63 | 64 | - richat-cli-v4.3.0 65 | - richat-client-v3.3.0 66 | - richat-filter-v3.3.0 67 | - richat-plugin-agave-v3.3.0 68 | - richat-v3.4.0 69 | - richat-shared-v3.3.0 70 | 71 | ### Features 72 | 73 | - shared: add jsonrpc feature ([#104](https://github.com/lamports-dev/richat/pull/104)) 74 | 75 | ## 2025-04-12 76 | 77 | - richat-cli-v4.2.0 78 | - richat-client-v3.2.0 79 | - richat-filter-v3.2.0 80 | - richat-plugin-agave-v3.2.0 81 | - richat-v3.3.0 82 | - richat-shared-v3.2.0 83 | 84 | ### Features 85 | 86 | - richat: use metrics.rs ([#101](https://github.com/lamports-dev/richat/pull/101)) 87 | 88 | ## 2025-04-09 89 | 90 | - richat-cli-v4.1.0 91 | - richat-client-v3.1.0 92 | - richat-filter-v3.1.0 93 | - richat-plugin-agave-v3.1.0 94 | - richat-v3.2.0 95 | - richat-shared-v3.1.0 96 | 97 | ### Features 98 | 99 | - shared: support ready and health endpoints on metrics server ([#97](https://github.com/lamports-dev/richat/pull/97)) 100 | - shared: use affinity only on linux ([#99](https://github.com/lamports-dev/richat/pull/99)) 101 | 102 | ## 2025-03-19 103 | 104 | - richat-v3.1.0 105 | 106 | ### Fixes 107 | 108 | - richat: do not update head on new filter with same commitment ([#88](https://github.com/lamports-dev/richat/pull/88)) 109 | - richat: push messages even after SlotStatus ([#89](https://github.com/lamports-dev/richat/pull/89)) 110 | 111 | ### Features 112 | 113 | - richat: add metrics ([#90](https://github.com/lamports-dev/richat/pull/90)) 114 | 115 | ## 2025-03-12 116 | 117 | - cli-v4.0.0 118 | - client-v3.0.0 119 | - filter-v3.0.0 120 | - plugin-agave-v3.0.0 121 | - proto-v3.0.0 122 | - richat-v3.0.0 123 | - shared-v3.0.0 124 | 125 | ### Breaking 126 | 127 | - solana: upgrade to v2.2 ([#85](https://github.com/lamports-dev/richat/pull/85)) 128 | 129 | ## 2025-03-12 130 | 131 | - richat-cli-v3.0.0 132 | - richat-filter-v2.4.1 133 | 134 | ### Breaking 135 | 136 | - cli: add dragon's mouth support ([#83](https://github.com/lamports-dev/richat/pull/83)) 137 | 138 | ## 2025-03-06 139 | 140 | - richat-cli-v2.2.2 141 | - richat-plugin-agave-v2.1.1 142 | - richat-v2.3.1 143 | 144 | ### Fixes 145 | 146 | - rustls: install CryptoProvider ([#82](https://github.com/lamports-dev/richat/pull/82)) 147 | 148 | ## 2025-02-22 149 | 150 | - richat-shared-v2.5.0 151 | 152 | ### Features 153 | 154 | - shared: add features ([#77](https://github.com/lamports-dev/richat/pull/77)) 155 | 156 | ## 2025-02-20 157 | 158 | - richat-cli-v2.2.1 159 | - richat-filter-v2.4.0 160 | - richat-v2.3.0 161 | - richat-shared-v2.4.0 162 | 163 | ### Features 164 | 165 | - shared: use `five8` to encode/decode ([#70](https://github.com/lamports-dev/richat/pull/70)) 166 | - shared: use `slab` for `Shutdown` ([#75](https://github.com/lamports-dev/richat/pull/75)) 167 | 168 | ### Breaking 169 | 170 | - filter: encode with slices ([#73](https://github.com/lamports-dev/richat/pull/73)) 171 | - richat: remove parser thread ([#74](https://github.com/lamports-dev/richat/pull/74)) 172 | - richat: support multiple sources ([#76](https://github.com/lamports-dev/richat/pull/76)) 173 | 174 | ## 2025-02-11 175 | 176 | - richat-cli-v2.2.0 177 | - richat-client-v2.2.0 178 | - richat-filter-v2.3.0 179 | - richat-plugin-agave-v2.1.0 180 | - richat-proto-v2.1.0 181 | - richat-v2.2.0 182 | - richat-shared-v2.3.0 183 | 184 | ### Fixes 185 | 186 | - richat: remove extra lock on clients queue ([#49](https://github.com/lamports-dev/richat/pull/49)) 187 | - plugin-agave: set `nodelay` correctly for Tcp ([#53](https://github.com/lamports-dev/richat/pull/53)) 188 | - richat: add minimal sleep ([#54](https://github.com/lamports-dev/richat/pull/54)) 189 | - richat: consume dragons mouth stream ([#62](https://github.com/lamports-dev/richat/pull/62)) 190 | - richat: fix slot status ([#66](https://github.com/lamports-dev/richat/pull/66)) 191 | 192 | ### Features 193 | 194 | - cli: add bin `richat-track` ([#51](https://github.com/lamports-dev/richat/pull/51)) 195 | - richat: change logs and metrics in the config ([#64](https://github.com/lamports-dev/richat/pull/64)) 196 | - richat: add solana pubsub ([#65](https://github.com/lamports-dev/richat/pull/65)) 197 | - richat: add metrics (backport of [private#5](https://github.com/lamports-dev/richat-private/pull/5)) ([#69](https://github.com/lamports-dev/richat/pull/69)) 198 | 199 | ### Breaking 200 | 201 | - plugin-agave,richat: remove `max_slots` ([#68](https://github.com/lamports-dev/richat/pull/68)) 202 | 203 | ## 2025-01-22 204 | 205 | - filter-v2.2.0 206 | - richat-v2.1.0 207 | 208 | ### Features 209 | 210 | - richat: add metrics (backport of [private#1](https://github.com/lamports-dev/richat-private/pull/1)) ([#44](https://github.com/lamports-dev/richat/pull/44)) 211 | - richat: add downstream server (backport of [private#3](https://github.com/lamports-dev/richat-private/pull/3)) ([#46](https://github.com/lamports-dev/richat/pull/46)) 212 | 213 | ## 2025-01-19 214 | 215 | - cli-v2.1.0 216 | - client-v2.1.0 217 | - filter-v2.1.0 218 | - richat-v2.0.0 219 | - shared-v2.1.0 220 | 221 | ### Features 222 | 223 | - richat: impl gRPC dragon's mouth ([#42](https://github.com/lamports-dev/richat/pull/42)) 224 | 225 | ## 2025-01-14 226 | 227 | - cli-v2.0.0 228 | - cli-v1.0.0 229 | - client-v2.0.0 230 | - client-v1.0.0 231 | - filter-v2.0.0 232 | - filter-v1.0.0 233 | - plugin-agave-v2.0.0 234 | - plugin-agave-v1.0.0 235 | - proto-v2.0.0 236 | - proto-v1.0.0 237 | - shared-v2.0.0 238 | - shared-v1.0.0 239 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | resolver = "2" 3 | members = [ 4 | "cli", 5 | "client", 6 | "filter", 7 | "plugin-agave", 8 | "proto", 9 | "richat", 10 | "shared", 11 | ] 12 | 13 | [workspace.package] 14 | authors = ["Lamports Dev", "Triton One"] 15 | edition = "2021" 16 | homepage = "https://lamports.dev" 17 | repository = "https://github.com/lamports-dev/richat" 18 | license = "Apache-2.0" 19 | keywords = ["solana"] 20 | publish = false 21 | 22 | [workspace.dependencies] 23 | affinity-linux = "1.0.0" 24 | agave-geyser-plugin-interface = "~2.2.1" 25 | anyhow = "1.0.62" 26 | arrayvec = "0.7.6" 27 | base64 = "0.22.1" 28 | bincode = "1.3.3" 29 | bs58 = "0.5.1" 30 | bytes = "1.9.0" 31 | cargo-lock = "10.0.1" 32 | clap = "4.5.23" 33 | const-hex = "1.14.0" 34 | criterion = "0.5" 35 | env_logger = "0.11.5" 36 | fastwebsockets = "0.10.0" 37 | five8 = "0.2.1" 38 | foldhash = "0.1.5" 39 | futures = "0.3.31" 40 | git-version = "0.3.9" 41 | hostname = "0.4.0" 42 | http = "1.1.0" 43 | http-body-util = "0.1.2" 44 | humantime-serde = "1.1.1" 45 | hyper = "1.4.1" 46 | hyper-util = "0.1.7" 47 | indicatif = "0.17.9" 48 | json5 = "0.4.1" 49 | jsonrpc-core = "18.0.0" 50 | jsonrpsee-types = "0.24.8" 51 | log = "0.4.22" 52 | maplit = "1.0.2" 53 | metrics = "0.24.1" 54 | metrics-exporter-prometheus = { version = "0.16.2", default-features = false } 55 | pin-project-lite = "0.2.15" 56 | prost = "0.13.4" 57 | prost-types = "0.13.4" 58 | prost_011 = { package = "prost", version = "0.11.9" } 59 | protobuf-src = "1.1.0" 60 | quanta = "0.12.5" 61 | quinn = "0.11.6" 62 | rayon = "1.10.0" 63 | rcgen = "0.13.1" 64 | richat-client = { path = "client", version = "3.5.0" } 65 | richat-filter = { path = "filter", version = "3.5.0" } 66 | richat-plugin-agave = { path = "plugin-agave", version = "3.5.0" } 67 | richat-proto = { path = "proto", version = "3.1.0" } 68 | richat-shared = { path = "shared", version = "3.5.0", default-features = false } 69 | rustls = { version = "0.23.20", default-features = false } 70 | rustls-native-certs = "0.8.1" 71 | rustls-pemfile = "2.2.0" 72 | serde = "1.0.145" 73 | serde_json = "1.0.86" 74 | serde_yaml = "0.9.33" 75 | signal-hook = "0.3.17" 76 | slab = "0.4.9" 77 | smallvec = "1.13.2" 78 | socket2 = "0.5.8" 79 | solana-account = "~2.2.1" 80 | solana-account-decoder = "~2.2.1" 81 | solana-client = "~2.2.1" 82 | solana-logger = "~2.2.1" 83 | solana-rpc-client-api = "~2.2.1" 84 | solana-sdk = "~2.2.1" 85 | solana-storage-proto = "~2.2.1" 86 | solana-transaction-status = "~2.2.1" 87 | solana-version = "~2.2.1" 88 | spl-token-2022 = "7.0.0" 89 | thiserror = "2.0.7" 90 | tikv-jemallocator = { version = "0.6.0", features = ["unprefixed_malloc_on_supported_platforms"] } 91 | tokio = "1.42.0" 92 | tokio-rustls = "0.26.1" 93 | tokio-tungstenite = "0.26.1" 94 | tonic = "0.12.3" 95 | tonic-build = "0.12.3" 96 | tracing = "0.1.41" 97 | tracing-subscriber = "0.3.19" 98 | vergen = "9.0.2" 99 | webpki-roots = "0.26.7" 100 | yellowstone-grpc-proto = "6.1.0" 101 | 102 | [workspace.lints.clippy] 103 | clone_on_ref_ptr = "deny" 104 | missing_const_for_fn = "deny" 105 | trivially_copy_pass_by_ref = "deny" 106 | 107 | [profile.release] 108 | lto = true 109 | codegen-units = 1 110 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | ci-all: \ 2 | ci-fmt \ 3 | ci-cargo-deny \ 4 | ci-clippy \ 5 | ci-clippy-fuzz \ 6 | ci-check \ 7 | ci-test 8 | 9 | ci-fmt: 10 | cargo +nightly fmt --check 11 | 12 | ci-cargo-deny: 13 | cargo deny check advisories 14 | 15 | ci-clippy: 16 | cargo clippy --workspace --all-targets -- -Dwarnings 17 | 18 | ci-clippy-fuzz: 19 | cd plugin-agave/fuzz && cargo clippy --workspace --all-targets -- -Dwarnings 20 | 21 | PACKAGES=richat-cli richat-client richat-filter richat-plugin-agave richat richat-shared 22 | ci-check: 23 | for package in $(PACKAGES) ; do \ 24 | echo cargo check -p $$package --all-targets ; \ 25 | cargo check -p $$package --all-targets ; \ 26 | done 27 | cargo check -p richat-plugin-agave --all-targets --all-features 28 | cargo check -p richat-shared --all-targets --no-default-features --features="config" 29 | cargo check -p richat-shared --all-targets --no-default-features --features="five8" 30 | cargo check -p richat-shared --all-targets --no-default-features --features="jsonrpc" 31 | cargo check -p richat-shared --all-targets --no-default-features --features="metrics" 32 | cargo check -p richat-shared --all-targets --no-default-features --features="shutdown" 33 | cargo check -p richat-shared --all-targets --no-default-features --features="transports" 34 | cargo check -p richat-shared --all-targets --no-default-features --features="version" 35 | 36 | ci-test: 37 | cargo test --all-targets 38 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # richat 2 | 3 | Next iteration of [Yellowstone Dragon's Mouth / Geyser gRPC](https://github.com/rpcpool/yellowstone-grpc) that was originally developed and currently maintained by [Triton One](https://triton.one/). `Richat` includes code derived from `Dragon's Mouth` (copyright `Triton One Limited`) with significant architecture changes. 4 | 5 | In addition to `Yellowstone Drangon's Mouth / Geyser gRPC` richat includes Solana PubSub implementation. 6 | 7 | Please use issues only for reporting bugs or discussing feature-related topics. If you're having trouble loading a plugin or need guidance on how to use crates, please post your question in the Telegram group: [https://t.me/lamportsdev](https://t.me/lamportsdev) 8 | 9 | ## Sponsored by 10 | 11 | ## Blueprint 12 | 13 | ```mermaid 14 | flowchart LR 15 | P[plugin] -->|full stream| R1(richat) 16 | R1 -->|full stream| R2(richat) 17 | R2 -->|filtered stream| C1(client) 18 | R1 -->|filtered stream| C2(client) 19 | R1 -->|filtered stream| C3(client) 20 | R2 -->|filtered stream| C4(client) 21 | ``` 22 | 23 | ```mermaid 24 | flowchart LR 25 | subgraph agave1 [**agave**] 26 | subgraph geyser1 [richat-plugin-agave] 27 | end 28 | end 29 | 30 | subgraph agave2 [**agave**] 31 | subgraph geyser2 [richat-plugin-agave] 32 | end 33 | end 34 | 35 | subgraph richat0 [**richat**] 36 | subgraph richat0_server [richat-server] 37 | end 38 | end 39 | 40 | subgraph richat1 [**richat**] 41 | subgraph tokio1 [Tokio Runtime] 42 | richat1_tokio1_receiver(receiver) 43 | richat1_channel[(messages
storage)] 44 | end 45 | 46 | subgraph tokio2 [Tokio Runtime] 47 | subgraph grpc1 [gRPC] 48 | richat1_grpc1_streaming1(streaming) 49 | richat1_grpc1_unary(unary) 50 | 51 | richat1_grpc1_blockmeta[(block meta
storage)] 52 | richat1_grpc1_subscriptions[(clients
subscriptions)] 53 | end 54 | 55 | subgraph pubsub1 [Solana PubSub] 56 | richat1_pubsub1_server(server) 57 | end 58 | 59 | subgraph richat_server1 [Richat] 60 | richat_server1_sender(server) 61 | end 62 | end 63 | 64 | subgraph pubsub1_pool [Filters Thread Pool] 65 | richat1_pubsub1_pool_worker1(worker 1) 66 | richat1_pubsub1_pool_worker2(worker N) 67 | end 68 | 69 | subgraph pubsub1_main [Subscriptions Thread] 70 | richat1_pubsub1_subscriptions[(clients
subscriptions)] 71 | end 72 | 73 | subgraph blockmeta_recv [BlockMeta Thread] 74 | richat1_blockmeta_recv_thread(blockmeta receiver) 75 | end 76 | 77 | subgraph grpc_workers [gRPC Filters Thread Pool] 78 | richat1_grpc_worker1(worker 1) 79 | richat1_grpc_worker2(worker N) 80 | end 81 | end 82 | 83 | client1(client) 84 | client2(client) 85 | client3(client) 86 | 87 | geyser1 -->|gRPC / Quic
full stream| richat1_tokio1_receiver 88 | geyser2 -->|gRPC / Quic
full stream| richat1_tokio1_receiver 89 | richat0_server -->|gRPC / Quic
full stream| richat1_tokio1_receiver 90 | richat1_tokio1_receiver --> richat1_channel 91 | richat1_channel --> richat1_blockmeta_recv_thread 92 | richat1_channel --> richat1_grpc_worker1 93 | richat1_channel --> richat1_grpc_worker2 94 | richat1_blockmeta_recv_thread --> richat1_grpc1_blockmeta 95 | richat1_grpc1_blockmeta <--> richat1_grpc1_unary 96 | richat1_grpc_worker1 <--> richat1_grpc1_subscriptions 97 | richat1_grpc_worker2 <--> richat1_grpc1_subscriptions 98 | richat1_grpc1_subscriptions <--> richat1_grpc1_streaming1 99 | client1 <--> |gRPC
filtered stream| richat1_grpc1_streaming1 100 | client1 --> richat1_grpc1_unary 101 | richat1_channel --> richat_server1_sender 102 | richat_server1_sender -->|gRPC / Quic
full stream| client2 103 | richat1_channel --> richat1_pubsub1_subscriptions 104 | richat1_pubsub1_subscriptions <--> richat1_pubsub1_pool_worker1 105 | richat1_pubsub1_subscriptions <--> richat1_pubsub1_pool_worker2 106 | richat1_pubsub1_subscriptions <--> richat1_pubsub1_server 107 | richat1_pubsub1_server <-->|WebSocket| client3 108 | ``` 109 | 110 | ## Components 111 | 112 | - `cli` — CLI client for full stream, gRPC stream with filters, simple Solana PubSub 113 | - `client` — library for building consumers 114 | - `filter` — library for filtering geyser messages 115 | - `plugin-agave` — Agave validator geyser plugin https://docs.anza.xyz/validator/geyser 116 | - `proto` — library with proto files, re-imports structs from crate `yellowstone-grpc-proto` 117 | - `richat` — app with full stream consumer and producers: gRPC (`Dragon's Mouth`), Solana PubSub 118 | - `shared` — shared code between components (except `client`) 119 | 120 | ## Releases 121 | 122 | #### Branches 123 | 124 | - `master` — development branch 125 | - `agave-v2.0` — development branch for agave v2.0 126 | - `agave-v2.1` — development branch for agave v2.1 127 | - `agave-v2.2` — development branch for agave v2.2 128 | 129 | #### Tags 130 | 131 | - `cli-v0.0.0` 132 | - `client-v0.0.0` 133 | - `filter-v0.0.0` 134 | - `plugin-agave-v0.0.0` 135 | - `plugin-agave-v0.0.0+solana.2.1.5` 136 | - `proto-v0.0.0` 137 | - `richat-v0.0.0` 138 | - `shared-v0.0.0` 139 | 140 | At one moment of time we can support more than one agave version (like v2.0 and v2.1), as result we can have two different major supported versions of every component, for example: `cli-v1.y.z` for `agave-v2.0` and `cli-v2.y.z` for `agave-v2.1`. In addition to standard version, `plugin-agave` can have one or more tags with pinned solana version. 141 | 142 | ## List of RPC providers with Dragon's Mouth support 143 | 144 | - `Allnodes` — https://www.allnodes.com/ 145 | - `ERPC` — https://erpc.global/en/ 146 | - `Gadfly Node` — https://gadflynode.com/ 147 | - `Geeks Labs` — https://discord.gg/geekslabs 148 | - `GetBlock` — https://getblock.io/ 149 | - `Helius` — https://www.helius.dev/ 150 | - `InstantNodes` — https://instantnodes.io/ 151 | - `OrbitFlare` — https://orbitflare.com/ 152 | - `PixelLabz` — https://pixellabz.io/ 153 | - `Platinum Node` — https://www.platinumnode.io/ 154 | - `PublicNode` — https://solana-rpc.publicnode.com/ 155 | - `QuickNode` — https://www.quicknode.com/ 156 | - `Shyft` — https://shyft.to/ 157 | - `Solana Tracker` — https://www.solanatracker.io/solana-rpc 158 | - `Solana Vibe Station` — https://www.solanavibestation.com/ 159 | - `SolSqueezer` — https://solsqueezer.io/ 160 | - `Triton One` — https://triton.one/ 161 | - `Urban Node` — https://urbannode.io/ 162 | 163 | If your RPC provider not in the list, please open Issue / PR! 164 | -------------------------------------------------------------------------------- /cli/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "richat-cli" 3 | version = "4.5.0" 4 | authors = { workspace = true } 5 | edition = { workspace = true } 6 | description = "Richat Cli Tool" 7 | homepage = { workspace = true } 8 | repository = { workspace = true } 9 | license = { workspace = true } 10 | keywords = { workspace = true } 11 | publish = { workspace = true } 12 | 13 | [dependencies] 14 | agave-geyser-plugin-interface = { workspace = true } 15 | anyhow = { workspace = true } 16 | clap = { workspace = true, features = ["derive"] } 17 | const-hex = { workspace = true } 18 | env_logger = { workspace = true } 19 | futures = { workspace = true } 20 | indicatif = { workspace = true } 21 | jsonrpsee-types = { workspace = true } 22 | maplit = { workspace = true } 23 | prost = { workspace = true } 24 | richat-client = { workspace = true } 25 | richat-plugin-agave = { workspace = true } 26 | richat-proto = { workspace = true } 27 | richat-shared = { workspace = true } 28 | rustls = { workspace = true, features = ["aws_lc_rs"] } 29 | serde = { workspace = true } 30 | serde_json = { workspace = true } 31 | serde_yaml = { workspace = true } 32 | solana-account-decoder = { workspace = true } 33 | solana-client = { workspace = true } 34 | solana-rpc-client-api = { workspace = true } 35 | solana-sdk = { workspace = true, features = ["dev-context-only-utils"] } 36 | solana-transaction-status = { workspace = true } 37 | tikv-jemallocator = { workspace = true } 38 | tokio = { workspace = true, features = ["fs"] } 39 | tokio-tungstenite = { workspace = true, features = ["rustls"] } 40 | tonic = { workspace = true, features = ["tls-native-roots"] } 41 | tracing = { workspace = true } 42 | 43 | [lints] 44 | workspace = true 45 | -------------------------------------------------------------------------------- /cli/richat-track.yml: -------------------------------------------------------------------------------- 1 | --- 2 | accounts: true 3 | transactions: true 4 | sources: 5 | richat-plugin-agave-grpc: 6 | source: richat-plugin-agave 7 | transport: grpc 8 | endpoint: http://127.0.0.1:10100 9 | connect_timeout: 3s 10 | timeout: 3s 11 | max_decoding_message_size: 134_217_728 12 | # richat-plugin-agave-quic: 13 | # source: richat-plugin-agave 14 | # transport: quic 15 | # endpoint: 127.0.0.1:10101 16 | # local_addr: "[::]:0" 17 | # server_name: "localhost" 18 | # insecure: true 19 | # richat-grpc: 20 | # source: richat-grpc 21 | # endpoint: http://127.0.0.1:10200 22 | # max_decoding_message_size: 134_217_728 23 | # yellowstone-grpc: 24 | # source: yellowstone-grpc 25 | # endpoint: http://127.0.0.1:10000 26 | # max_decoding_message_size: 134_217_728 27 | tracks: 28 | - event: BlockMeta 29 | - event: Transaction 30 | index: 0 31 | -------------------------------------------------------------------------------- /cli/src/bin/richat-cli.rs: -------------------------------------------------------------------------------- 1 | use { 2 | clap::{Parser, Subcommand}, 3 | richat_cli::{ 4 | pubsub::ArgsAppPubSub, stream_grpc::ArgsAppStreamGrpc, stream_richat::ArgsAppStreamRichat, 5 | track::ArgsAppTrack, 6 | }, 7 | std::{ 8 | env, 9 | sync::atomic::{AtomicU64, Ordering}, 10 | }, 11 | }; 12 | 13 | #[cfg(not(target_env = "msvc"))] 14 | #[global_allocator] 15 | static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; 16 | 17 | #[derive(Debug, Parser)] 18 | #[clap(author, version, about = "Richat Cli Tool: pubsub, stream, track")] 19 | struct Args { 20 | #[command(subcommand)] 21 | action: ArgsAppSelect, 22 | } 23 | 24 | #[derive(Debug, Subcommand)] 25 | enum ArgsAppSelect { 26 | /// Subscribe on updates over WebSocket (Solana PubSub) 27 | Pubsub(ArgsAppPubSub), 28 | 29 | /// Stream data from Yellowstone gRPC / Dragon's Mouth 30 | StreamGrpc(ArgsAppStreamGrpc), 31 | 32 | /// Stream data directly from the richat-plugin 33 | StreamRichat(ArgsAppStreamRichat), 34 | 35 | /// Events tracker 36 | Track(ArgsAppTrack), 37 | } 38 | 39 | async fn main2() -> anyhow::Result<()> { 40 | anyhow::ensure!( 41 | rustls::crypto::aws_lc_rs::default_provider() 42 | .install_default() 43 | .is_ok(), 44 | "failed to call CryptoProvider::install_default()" 45 | ); 46 | 47 | env::set_var( 48 | env_logger::DEFAULT_FILTER_ENV, 49 | env::var_os(env_logger::DEFAULT_FILTER_ENV).unwrap_or_else(|| "info".into()), 50 | ); 51 | env_logger::init(); 52 | 53 | let args = Args::parse(); 54 | match args.action { 55 | ArgsAppSelect::Pubsub(action) => action.run().await, 56 | ArgsAppSelect::StreamGrpc(action) => action.run().await, 57 | ArgsAppSelect::StreamRichat(action) => action.run().await, 58 | ArgsAppSelect::Track(action) => action.run().await, 59 | } 60 | } 61 | 62 | fn main() -> anyhow::Result<()> { 63 | tokio::runtime::Builder::new_multi_thread() 64 | .thread_name_fn(move || { 65 | static ATOMIC_ID: AtomicU64 = AtomicU64::new(0); 66 | let id = ATOMIC_ID.fetch_add(1, Ordering::Relaxed); 67 | format!("richatCli{id:02}") 68 | }) 69 | .enable_all() 70 | .build()? 71 | .block_on(main2()) 72 | } 73 | -------------------------------------------------------------------------------- /cli/src/lib.rs: -------------------------------------------------------------------------------- 1 | pub mod pubsub; 2 | pub mod stream; 3 | pub mod stream_grpc; 4 | pub mod stream_richat; 5 | pub mod track; 6 | -------------------------------------------------------------------------------- /client/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "richat-client" 3 | version = "3.5.0" 4 | authors = { workspace = true } 5 | edition = { workspace = true } 6 | description = "Richat Client Library" 7 | homepage = { workspace = true } 8 | repository = { workspace = true } 9 | license = { workspace = true } 10 | keywords = { workspace = true } 11 | publish = true 12 | 13 | [dependencies] 14 | bytes = { workspace = true } 15 | foldhash = { workspace = true } 16 | futures = { workspace = true } 17 | humantime-serde = { workspace = true } 18 | pin-project-lite = { workspace = true } 19 | prost = { workspace = true } 20 | quinn = { workspace = true } 21 | richat-proto = { workspace = true } 22 | richat-shared = { workspace = true, features = ["config", "transports"] } 23 | rustls = { workspace = true } 24 | rustls-native-certs = { workspace = true } 25 | rustls-pemfile = { workspace = true } 26 | serde = { workspace = true } 27 | solana-sdk = { workspace = true } 28 | thiserror = { workspace = true } 29 | tokio = { workspace = true, features = ["rt-multi-thread", "fs"] } 30 | tonic = { workspace = true, features = ["tls-native-roots"] } 31 | tracing = { workspace = true } 32 | webpki-roots = { workspace = true } 33 | 34 | [build-dependencies] 35 | anyhow = { workspace = true } 36 | tonic-build = { workspace = true } 37 | 38 | [lints] 39 | workspace = true 40 | -------------------------------------------------------------------------------- /client/build.rs: -------------------------------------------------------------------------------- 1 | use tonic_build::manual::{Builder, Method, Service}; 2 | 3 | fn main() -> anyhow::Result<()> { 4 | generate_grpc_geyser() 5 | } 6 | 7 | fn generate_grpc_geyser() -> anyhow::Result<()> { 8 | let geyser_service = Service::builder() 9 | .name("Geyser") 10 | .package("geyser") 11 | .method( 12 | Method::builder() 13 | .name("subscribe") 14 | .route_name("Subscribe") 15 | .input_type("richat_proto::geyser::SubscribeRequest") 16 | .output_type("Vec") 17 | .codec_path("crate::grpc::SubscribeCodec") 18 | .client_streaming() 19 | .server_streaming() 20 | .build(), 21 | ) 22 | .method( 23 | Method::builder() 24 | .name("subscribe_richat") 25 | .route_name("Subscribe") 26 | .input_type("richat_proto::richat::GrpcSubscribeRequest") 27 | .output_type("Vec") 28 | .codec_path("crate::grpc::SubscribeCodec") 29 | .client_streaming() 30 | .server_streaming() 31 | .build(), 32 | ) 33 | .method( 34 | Method::builder() 35 | .name("subscribe_replay_info") 36 | .route_name("SubscribeReplayInfo") 37 | .input_type("richat_proto::geyser::SubscribeReplayInfoRequest") 38 | .output_type("richat_proto::geyser::SubscribeReplayInfoResponse") 39 | .codec_path("tonic::codec::ProstCodec") 40 | .build(), 41 | ) 42 | .method( 43 | Method::builder() 44 | .name("ping") 45 | .route_name("Ping") 46 | .input_type("richat_proto::geyser::PingRequest") 47 | .output_type("richat_proto::geyser::PongResponse") 48 | .codec_path("tonic::codec::ProstCodec") 49 | .build(), 50 | ) 51 | .method( 52 | Method::builder() 53 | .name("get_latest_blockhash") 54 | .route_name("GetLatestBlockhash") 55 | .input_type("richat_proto::geyser::GetLatestBlockhashRequest") 56 | .output_type("richat_proto::geyser::GetLatestBlockhashResponse") 57 | .codec_path("tonic::codec::ProstCodec") 58 | .build(), 59 | ) 60 | .method( 61 | Method::builder() 62 | .name("get_block_height") 63 | .route_name("GetBlockHeight") 64 | .input_type("richat_proto::geyser::GetBlockHeightRequest") 65 | .output_type("richat_proto::geyser::GetBlockHeightResponse") 66 | .codec_path("tonic::codec::ProstCodec") 67 | .build(), 68 | ) 69 | .method( 70 | Method::builder() 71 | .name("get_slot") 72 | .route_name("GetSlot") 73 | .input_type("richat_proto::geyser::GetSlotRequest") 74 | .output_type("richat_proto::geyser::GetSlotResponse") 75 | .codec_path("tonic::codec::ProstCodec") 76 | .build(), 77 | ) 78 | .method( 79 | Method::builder() 80 | .name("is_blockhash_valid") 81 | .route_name("IsBlockhashValid") 82 | .input_type("richat_proto::geyser::IsBlockhashValidRequest") 83 | .output_type("richat_proto::geyser::IsBlockhashValidResponse") 84 | .codec_path("tonic::codec::ProstCodec") 85 | .build(), 86 | ) 87 | .method( 88 | Method::builder() 89 | .name("get_version") 90 | .route_name("GetVersion") 91 | .input_type("richat_proto::geyser::GetVersionRequest") 92 | .output_type("richat_proto::geyser::GetVersionResponse") 93 | .codec_path("tonic::codec::ProstCodec") 94 | .build(), 95 | ) 96 | .build(); 97 | 98 | Builder::new() 99 | .build_server(false) 100 | .compile(&[geyser_service]); 101 | 102 | Ok(()) 103 | } 104 | -------------------------------------------------------------------------------- /client/src/error.rs: -------------------------------------------------------------------------------- 1 | use { 2 | prost::{DecodeError, Message}, 3 | richat_proto::richat::{ 4 | QuicSubscribeClose, QuicSubscribeCloseError, QuicSubscribeResponse, 5 | QuicSubscribeResponseError, 6 | }, 7 | std::io, 8 | thiserror::Error, 9 | tokio::io::{AsyncRead, AsyncReadExt}, 10 | }; 11 | 12 | #[derive(Debug, Error)] 13 | pub enum SubscribeError { 14 | #[error("failed to send/recv data: {0}")] 15 | Io(#[from] io::Error), 16 | #[error("failed to send data: {0}")] 17 | QuicWrite(#[from] quinn::WriteError), 18 | #[error("connection lost: {0}")] 19 | QuicConnection(#[from] quinn::ConnectionError), 20 | #[error("failed to decode response: {0}")] 21 | Decode(#[from] DecodeError), 22 | #[error("unknown subscribe response error: {0}")] 23 | Unknown(i32), 24 | #[error("recv stream should be greater than zero")] 25 | ZeroRecvStreams, 26 | #[error("exceed max number of recv streams: {0}")] 27 | ExceedRecvStreams(u32), 28 | #[error("stream not initialized yet")] 29 | NotInitialized, 30 | #[error("replay from slot is not available, lowest available: {0}")] 31 | ReplayFromSlotNotAvailable(u64), 32 | #[error("request is too large")] 33 | RequestSizeTooLarge, 34 | #[error("x-token required")] 35 | XTokenRequired, 36 | #[error("x-token invalid")] 37 | XTokenInvalid, 38 | } 39 | 40 | impl SubscribeError { 41 | pub(crate) async fn parse_quic_response( 42 | recv: &mut R, 43 | ) -> Result { 44 | let size = recv.read_u64().await?; 45 | let mut buf = vec![0; size as usize]; 46 | recv.read_exact(buf.as_mut_slice()).await?; 47 | 48 | let response = QuicSubscribeResponse::decode(buf.as_slice())?; 49 | if let Some(error) = response.error { 50 | Err(match QuicSubscribeResponseError::try_from(error) { 51 | Ok(QuicSubscribeResponseError::ZeroRecvStreams) => SubscribeError::ZeroRecvStreams, 52 | Ok(QuicSubscribeResponseError::ExceedRecvStreams) => { 53 | SubscribeError::ExceedRecvStreams(response.max_recv_streams()) 54 | } 55 | Ok(QuicSubscribeResponseError::NotInitialized) => SubscribeError::NotInitialized, 56 | Ok(QuicSubscribeResponseError::SlotNotAvailable) => { 57 | SubscribeError::ReplayFromSlotNotAvailable(response.first_available_slot()) 58 | } 59 | Ok(QuicSubscribeResponseError::RequestSizeTooLarge) => { 60 | SubscribeError::RequestSizeTooLarge 61 | } 62 | Ok(QuicSubscribeResponseError::XTokenRequired) => SubscribeError::XTokenRequired, 63 | Ok(QuicSubscribeResponseError::XTokenInvalid) => SubscribeError::XTokenInvalid, 64 | Err(_error) => SubscribeError::Unknown(error), 65 | }) 66 | } else { 67 | Ok(response.version) 68 | } 69 | } 70 | } 71 | 72 | #[derive(Debug, Error)] 73 | pub enum ReceiveError { 74 | #[error("failed to recv data: {0}")] 75 | Io(#[from] io::Error), 76 | #[error("failed to recv data: {0}")] 77 | QuicRecv(#[from] quinn::ReadExactError), 78 | #[error("failed to decode response: {0}")] 79 | Decode(#[from] DecodeError), 80 | #[error("stream failed: {0}")] 81 | Status(#[from] tonic::Status), 82 | #[error("unknown close error: {0}")] 83 | Unknown(i32), 84 | #[error("stream lagged")] 85 | Lagged, 86 | #[error("internal geyser stream is closed")] 87 | Closed, 88 | } 89 | 90 | impl From for ReceiveError { 91 | fn from(close: QuicSubscribeClose) -> Self { 92 | match QuicSubscribeCloseError::try_from(close.error) { 93 | Ok(QuicSubscribeCloseError::Lagged) => Self::Lagged, 94 | Ok(QuicSubscribeCloseError::Closed) => Self::Closed, 95 | Err(_error) => Self::Unknown(close.error), 96 | } 97 | } 98 | } 99 | 100 | impl ReceiveError { 101 | pub fn is_eof(&self) -> bool { 102 | if let Self::Io(error) = self { 103 | error.kind() == io::ErrorKind::UnexpectedEof 104 | } else { 105 | false 106 | } 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /client/src/lib.rs: -------------------------------------------------------------------------------- 1 | pub mod error; 2 | pub mod grpc; 3 | pub mod quic; 4 | pub mod stream; 5 | -------------------------------------------------------------------------------- /client/src/stream.rs: -------------------------------------------------------------------------------- 1 | use { 2 | crate::error::ReceiveError, 3 | futures::stream::{BoxStream, Stream}, 4 | pin_project_lite::pin_project, 5 | prost::Message, 6 | richat_proto::geyser::SubscribeUpdate, 7 | std::{ 8 | fmt, 9 | pin::Pin, 10 | task::{ready, Context, Poll}, 11 | }, 12 | }; 13 | 14 | type InputStream = BoxStream<'static, Result, ReceiveError>>; 15 | 16 | pin_project! { 17 | pub struct SubscribeStream { 18 | stream: InputStream, 19 | } 20 | } 21 | 22 | impl SubscribeStream { 23 | pub(crate) fn new(stream: InputStream) -> Self { 24 | Self { stream } 25 | } 26 | } 27 | 28 | impl fmt::Debug for SubscribeStream { 29 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 30 | f.debug_struct("SubscribeStream").finish() 31 | } 32 | } 33 | 34 | impl Stream for SubscribeStream { 35 | type Item = Result; 36 | 37 | fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { 38 | let mut me = self.project(); 39 | Poll::Ready(match ready!(Pin::new(&mut me.stream).poll_next(cx)) { 40 | Some(Ok(slice)) => Some(SubscribeUpdate::decode(slice.as_slice()).map_err(Into::into)), 41 | Some(Err(error)) => Some(Err(error)), 42 | None => None, 43 | }) 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /deny.toml: -------------------------------------------------------------------------------- 1 | [graph] 2 | all-features = true 3 | 4 | [advisories] 5 | ignore = [ 6 | # atty 0.2.14 7 | # Advisory: https://rustsec.org/advisories/RUSTSEC-2021-0145 8 | "RUSTSEC-2021-0145", 9 | 10 | # ed25519-dalek 1.0.1 11 | # Advisory: https://rustsec.org/advisories/RUSTSEC-2022-0093 12 | "RUSTSEC-2022-0093", 13 | 14 | # curve25519-dalek 3.2.0 15 | # Advisory: https://rustsec.org/advisories/RUSTSEC-2024-0344 16 | "RUSTSEC-2024-0344", 17 | 18 | # atty 0.2.14 19 | # Advisory: https://rustsec.org/advisories/RUSTSEC-2024-0375 20 | "RUSTSEC-2024-0375", 21 | 22 | # derivative 2.2.0 23 | # Advisory: https://rustsec.org/advisories/RUSTSEC-2024-0388 24 | "RUSTSEC-2024-0388", 25 | 26 | # paste 1.0.15 27 | # Advisory: https://rustsec.org/advisories/RUSTSEC-2024-0436 28 | "RUSTSEC-2024-0436", 29 | ] 30 | -------------------------------------------------------------------------------- /filter/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "richat-filter" 3 | version = "3.5.0" 4 | authors = { workspace = true } 5 | edition = { workspace = true } 6 | description = "Richat Filters" 7 | homepage = { workspace = true } 8 | repository = { workspace = true } 9 | license = { workspace = true } 10 | keywords = { workspace = true } 11 | publish = true 12 | 13 | [dependencies] 14 | arrayvec = { workspace = true } 15 | base64 = { workspace = true } 16 | bs58 = { workspace = true } 17 | prost = { workspace = true } 18 | prost-types = { workspace = true } 19 | richat-proto = { workspace = true } 20 | richat-shared = { workspace = true, features = ["config", "five8"] } 21 | serde = { workspace = true, features = ["derive"] } 22 | smallvec = { workspace = true } 23 | solana-account = { workspace = true } 24 | solana-sdk = { workspace = true } 25 | solana-transaction-status = { workspace = true } 26 | spl-token-2022 = { workspace = true } 27 | thiserror = { workspace = true } 28 | 29 | [lints] 30 | workspace = true 31 | -------------------------------------------------------------------------------- /filter/src/lib.rs: -------------------------------------------------------------------------------- 1 | pub mod config; 2 | pub mod filter; 3 | pub mod message; 4 | pub mod protobuf; 5 | -------------------------------------------------------------------------------- /filter/src/protobuf/mod.rs: -------------------------------------------------------------------------------- 1 | use prost::{ 2 | bytes::BufMut, 3 | encoding::{encode_key, encode_varint, encoded_len_varint, key_len, WireType}, 4 | }; 5 | 6 | pub mod decode; 7 | pub mod encode; 8 | 9 | #[inline] 10 | pub fn bytes_encode(tag: u32, value: &[u8], buf: &mut impl BufMut) { 11 | encode_key(tag, WireType::LengthDelimited, buf); 12 | encode_varint(value.len() as u64, buf); 13 | buf.put(value) 14 | } 15 | 16 | #[inline] 17 | pub const fn bytes_encoded_len(tag: u32, value: &[u8]) -> usize { 18 | key_len(tag) + encoded_len_varint(value.len() as u64) + value.len() 19 | } 20 | -------------------------------------------------------------------------------- /plugin-agave/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "richat-plugin-agave" 3 | version = "3.5.0" 4 | authors = { workspace = true } 5 | edition = { workspace = true } 6 | description = "Richat Agave Geyser Plugin" 7 | homepage = { workspace = true } 8 | repository = { workspace = true } 9 | license = { workspace = true } 10 | keywords = { workspace = true } 11 | publish = { workspace = true } 12 | 13 | [lib] 14 | crate-type = ["cdylib", "rlib"] 15 | 16 | [[bin]] 17 | name = "richat-plugin-agave-config-check" 18 | path = "src/bin/config-check.rs" 19 | 20 | [[bench]] 21 | name = "encode" 22 | harness = false 23 | required-features = ["fixtures"] 24 | 25 | [dependencies] 26 | agave-geyser-plugin-interface = { workspace = true } 27 | anyhow = { workspace = true } 28 | bincode = { workspace = true } 29 | bytes = { workspace = true } 30 | clap = { workspace = true, features = ["derive"] } 31 | futures = { workspace = true } 32 | log = { workspace = true } 33 | metrics = { workspace = true } 34 | metrics-exporter-prometheus = { workspace = true } 35 | prost = { workspace = true } 36 | prost_011 = { workspace = true, optional = true } 37 | prost-types = { workspace = true } 38 | richat-proto = { workspace = true } 39 | richat-shared = { workspace = true, features = ["metrics", "shutdown", "transports", "version"] } 40 | rustls = { workspace = true, features = ["aws_lc_rs"] } 41 | serde = { workspace = true, features = ["derive"] } 42 | serde_json = { workspace = true } 43 | smallvec = { workspace = true } 44 | solana-account-decoder = { workspace = true } 45 | solana-logger = { workspace = true } 46 | solana-sdk = { workspace = true } 47 | solana-storage-proto = { workspace = true, optional = true } 48 | solana-transaction-status = { workspace = true } 49 | tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } 50 | 51 | [dev-dependencies] 52 | criterion = { workspace = true } 53 | prost_011 = { workspace = true } 54 | richat-proto = { workspace = true, features = ["yellowstone-grpc-plugin"] } 55 | solana-storage-proto = { workspace = true } 56 | 57 | [build-dependencies] 58 | anyhow = { workspace = true } 59 | cargo-lock = { workspace = true } 60 | git-version = { workspace = true } 61 | vergen = { workspace = true, features = ["build", "rustc"] } 62 | 63 | [features] 64 | fixtures = ["dep:prost_011", "dep:solana-storage-proto"] 65 | test-validator = [] 66 | 67 | [lints] 68 | workspace = true 69 | -------------------------------------------------------------------------------- /plugin-agave/README.md: -------------------------------------------------------------------------------- 1 | # Development 2 | 3 | To run plugin with `solana-test-validator`: 4 | 5 | ``` 6 | cp config.json config.dev.json && vim config.dev.json 7 | cargo build -p richat-plugin-agave --lib --release --features="test-validator" && solana-test-validator --geyser-plugin-config plugin-agave/config.dev.json 8 | ``` 9 | 10 | If you run plugin on mainnet validator do not try to do it in `debug` mode, validator would start fall behind. 11 | -------------------------------------------------------------------------------- /plugin-agave/benches/encode/account.rs: -------------------------------------------------------------------------------- 1 | use { 2 | criterion::{black_box, Criterion}, 3 | prost::Message, 4 | prost_types::Timestamp, 5 | richat_plugin_agave::protobuf::{ 6 | fixtures::generate_accounts, ProtobufEncoder, ProtobufMessage, 7 | }, 8 | richat_proto::plugin::{ 9 | filter::{ 10 | message::{FilteredUpdate, FilteredUpdateFilters, FilteredUpdateOneof}, 11 | FilterAccountsDataSlice, 12 | }, 13 | message::MessageAccount, 14 | }, 15 | std::time::SystemTime, 16 | }; 17 | 18 | pub fn bench_encode_accounts(criterion: &mut Criterion) { 19 | let accounts = generate_accounts(); 20 | 21 | let accounts_replica = accounts 22 | .iter() 23 | .map(|acc| acc.to_replica()) 24 | .collect::>(); 25 | 26 | let grpc_replicas = accounts_replica 27 | .iter() 28 | .cloned() 29 | .map(|account| { 30 | ( 31 | account, 32 | FilterAccountsDataSlice::new(&[], usize::MAX).unwrap(), 33 | ) 34 | }) 35 | .collect::>(); 36 | let grpc_messages = grpc_replicas 37 | .iter() 38 | .map(|((slot, account), data_slice)| { 39 | ( 40 | MessageAccount::from_geyser(account, *slot, false), 41 | data_slice.clone(), 42 | ) 43 | }) 44 | .collect::>(); 45 | 46 | criterion 47 | .benchmark_group("encode_accounts") 48 | .bench_with_input("richat/prost", &accounts_replica, |criterion, accounts| { 49 | let created_at = SystemTime::now(); 50 | criterion.iter(|| { 51 | #[allow(clippy::unit_arg)] 52 | black_box({ 53 | for (slot, account) in accounts { 54 | let message = ProtobufMessage::Account { 55 | slot: *slot, 56 | account, 57 | }; 58 | message.encode_with_timestamp(ProtobufEncoder::Prost, created_at); 59 | } 60 | }) 61 | }) 62 | }) 63 | .bench_with_input("richat/raw", &accounts_replica, |criterion, accounts| { 64 | let created_at = SystemTime::now(); 65 | criterion.iter(|| { 66 | #[allow(clippy::unit_arg)] 67 | black_box({ 68 | for (slot, account) in accounts { 69 | let message = ProtobufMessage::Account { 70 | slot: *slot, 71 | account, 72 | }; 73 | message.encode_with_timestamp(ProtobufEncoder::Raw, created_at); 74 | } 75 | }) 76 | }) 77 | }) 78 | .bench_with_input( 79 | "dragons-mouth/encoding-only", 80 | &grpc_messages, 81 | |criterion, grpc_messages| { 82 | let created_at = Timestamp::from(SystemTime::now()); 83 | criterion.iter(|| { 84 | #[allow(clippy::unit_arg)] 85 | black_box({ 86 | for (message, data_slice) in grpc_messages { 87 | let update = FilteredUpdate { 88 | filters: FilteredUpdateFilters::new(), 89 | message: FilteredUpdateOneof::account(message, data_slice.clone()), 90 | created_at, 91 | }; 92 | update.encode_to_vec(); 93 | } 94 | }) 95 | }); 96 | }, 97 | ) 98 | .bench_with_input( 99 | "dragons-mouth/full-pipeline", 100 | &grpc_replicas, 101 | |criterion, grpc_replicas| { 102 | let created_at = Timestamp::from(SystemTime::now()); 103 | criterion.iter(|| { 104 | #[allow(clippy::unit_arg)] 105 | black_box(for ((slot, account), data_slice) in grpc_replicas { 106 | let message = MessageAccount::from_geyser(account, *slot, false); 107 | let update = FilteredUpdate { 108 | filters: FilteredUpdateFilters::new(), 109 | message: FilteredUpdateOneof::account(&message, data_slice.clone()), 110 | created_at, 111 | }; 112 | update.encode_to_vec(); 113 | }) 114 | }); 115 | }, 116 | ); 117 | } 118 | -------------------------------------------------------------------------------- /plugin-agave/benches/encode/block_meta.rs: -------------------------------------------------------------------------------- 1 | use { 2 | criterion::{black_box, BatchSize, Criterion}, 3 | prost::Message, 4 | prost_types::Timestamp, 5 | richat_plugin_agave::protobuf::{ 6 | fixtures::generate_block_metas, ProtobufEncoder, ProtobufMessage, 7 | }, 8 | richat_proto::plugin::{ 9 | filter::message::{FilteredUpdate, FilteredUpdateFilters, FilteredUpdateOneof}, 10 | message::MessageBlockMeta, 11 | }, 12 | std::{sync::Arc, time::SystemTime}, 13 | }; 14 | 15 | pub fn bench_encode_block_metas(criterion: &mut Criterion) { 16 | let blocks_meta = generate_block_metas(); 17 | 18 | let blocks_meta_replica = blocks_meta 19 | .iter() 20 | .map(|b| b.to_replica()) 21 | .collect::>(); 22 | 23 | let blocks_meta_grpc = blocks_meta_replica 24 | .iter() 25 | .map(MessageBlockMeta::from_geyser) 26 | .map(Arc::new) 27 | .collect::>(); 28 | 29 | criterion 30 | .benchmark_group("encode_block_meta") 31 | .bench_with_input( 32 | "richat/prost", 33 | &blocks_meta_replica, 34 | |criterion, block_metas| { 35 | let created_at = SystemTime::now(); 36 | criterion.iter(|| { 37 | #[allow(clippy::unit_arg)] 38 | black_box({ 39 | for blockinfo in block_metas { 40 | let message = ProtobufMessage::BlockMeta { blockinfo }; 41 | message.encode_with_timestamp(ProtobufEncoder::Prost, created_at); 42 | } 43 | }) 44 | }) 45 | }, 46 | ) 47 | .bench_with_input( 48 | "richat/raw", 49 | &blocks_meta_replica, 50 | |criterion, block_metas| { 51 | let created_at = SystemTime::now(); 52 | criterion.iter(|| { 53 | #[allow(clippy::unit_arg)] 54 | black_box({ 55 | for blockinfo in block_metas { 56 | let message = ProtobufMessage::BlockMeta { blockinfo }; 57 | message.encode_with_timestamp(ProtobufEncoder::Raw, created_at); 58 | } 59 | }) 60 | }) 61 | }, 62 | ) 63 | .bench_with_input( 64 | "dragons-mouth/encoding-only", 65 | &blocks_meta_grpc, 66 | |criterion, messages| { 67 | let created_at = Timestamp::from(SystemTime::now()); 68 | criterion.iter_batched( 69 | || messages.to_owned(), 70 | |messages| { 71 | #[allow(clippy::unit_arg)] 72 | black_box({ 73 | for message in messages { 74 | let update = FilteredUpdate { 75 | filters: FilteredUpdateFilters::new(), 76 | message: FilteredUpdateOneof::block_meta(message), 77 | created_at, 78 | }; 79 | update.encode_to_vec(); 80 | } 81 | }) 82 | }, 83 | BatchSize::LargeInput, 84 | ); 85 | }, 86 | ) 87 | .bench_with_input( 88 | "dragons-mouth/full-pipeline", 89 | &blocks_meta_replica, 90 | |criterion, block_metas| { 91 | let created_at = Timestamp::from(SystemTime::now()); 92 | criterion.iter(|| { 93 | #[allow(clippy::unit_arg)] 94 | black_box(for blockinfo in block_metas { 95 | let message = MessageBlockMeta::from_geyser(blockinfo); 96 | let update = FilteredUpdate { 97 | filters: FilteredUpdateFilters::new(), 98 | message: FilteredUpdateOneof::block_meta(Arc::new(message)), 99 | created_at, 100 | }; 101 | update.encode_to_vec(); 102 | }) 103 | }); 104 | }, 105 | ); 106 | } 107 | -------------------------------------------------------------------------------- /plugin-agave/benches/encode/entry.rs: -------------------------------------------------------------------------------- 1 | use { 2 | criterion::{black_box, BatchSize, Criterion}, 3 | prost::Message, 4 | prost_types::Timestamp, 5 | richat_plugin_agave::protobuf::{fixtures::generate_entries, ProtobufEncoder, ProtobufMessage}, 6 | richat_proto::plugin::{ 7 | filter::message::{FilteredUpdate, FilteredUpdateFilters, FilteredUpdateOneof}, 8 | message::MessageEntry, 9 | }, 10 | std::{sync::Arc, time::SystemTime}, 11 | }; 12 | 13 | pub fn bench_encode_entries(criterion: &mut Criterion) { 14 | let entries = generate_entries(); 15 | 16 | let entries_replica = entries.iter().map(|e| e.to_replica()).collect::>(); 17 | 18 | let entries_grpc = entries_replica 19 | .iter() 20 | .map(MessageEntry::from_geyser) 21 | .map(Arc::new) 22 | .collect::>(); 23 | 24 | criterion 25 | .benchmark_group("encode_entry") 26 | .bench_with_input("richat/prost", &entries_replica, |criterion, entries| { 27 | let created_at = SystemTime::now(); 28 | criterion.iter(|| { 29 | #[allow(clippy::unit_arg)] 30 | black_box({ 31 | for entry in entries { 32 | let message = ProtobufMessage::Entry { entry }; 33 | message.encode_with_timestamp(ProtobufEncoder::Prost, created_at); 34 | } 35 | }) 36 | }); 37 | }) 38 | .bench_with_input("richat/raw", &entries_replica, |criterion, entries| { 39 | let created_at = SystemTime::now(); 40 | criterion.iter(|| { 41 | #[allow(clippy::unit_arg)] 42 | black_box({ 43 | for entry in entries { 44 | let message = ProtobufMessage::Entry { entry }; 45 | message.encode_with_timestamp(ProtobufEncoder::Raw, created_at); 46 | } 47 | }) 48 | }); 49 | }) 50 | .bench_with_input( 51 | "dragons-mouth/encoding-only", 52 | &entries_grpc, 53 | |criterion, entry_messages| { 54 | let created_at = Timestamp::from(SystemTime::now()); 55 | criterion.iter_batched( 56 | || entry_messages.to_owned(), 57 | |entry_messages| { 58 | #[allow(clippy::unit_arg)] 59 | black_box({ 60 | for message in entry_messages { 61 | let update = FilteredUpdate { 62 | filters: FilteredUpdateFilters::new(), 63 | message: FilteredUpdateOneof::entry(message), 64 | created_at, 65 | }; 66 | update.encode_to_vec(); 67 | } 68 | }) 69 | }, 70 | BatchSize::LargeInput, 71 | ); 72 | }, 73 | ) 74 | .bench_with_input( 75 | "dragons-mouth/full-pipeline", 76 | &entries_replica, 77 | |criterion, entries| { 78 | let created_at = Timestamp::from(SystemTime::now()); 79 | criterion.iter(|| { 80 | #[allow(clippy::unit_arg)] 81 | black_box({ 82 | for entry in entries { 83 | let message = MessageEntry::from_geyser(entry); 84 | let update = FilteredUpdate { 85 | filters: FilteredUpdateFilters::new(), 86 | message: FilteredUpdateOneof::entry(Arc::new(message)), 87 | created_at, 88 | }; 89 | update.encode_to_vec(); 90 | } 91 | }) 92 | }); 93 | }, 94 | ); 95 | } 96 | -------------------------------------------------------------------------------- /plugin-agave/benches/encode/main.rs: -------------------------------------------------------------------------------- 1 | use criterion::{criterion_group, criterion_main}; 2 | 3 | mod account; 4 | mod block_meta; 5 | mod entry; 6 | mod slot; 7 | mod transaction; 8 | 9 | criterion_group!( 10 | benches, 11 | account::bench_encode_accounts, 12 | slot::bench_encode_slot, 13 | entry::bench_encode_entries, 14 | block_meta::bench_encode_block_metas, 15 | transaction::bench_encode_transactions 16 | ); 17 | 18 | criterion_main!(benches); 19 | -------------------------------------------------------------------------------- /plugin-agave/benches/encode/slot.rs: -------------------------------------------------------------------------------- 1 | use { 2 | criterion::{black_box, Criterion}, 3 | prost::Message, 4 | prost_types::Timestamp, 5 | richat_plugin_agave::protobuf::{fixtures::generate_slots, ProtobufEncoder, ProtobufMessage}, 6 | richat_proto::plugin::{ 7 | filter::message::{FilteredUpdate, FilteredUpdateFilters, FilteredUpdateOneof}, 8 | message::MessageSlot, 9 | }, 10 | std::time::SystemTime, 11 | }; 12 | 13 | pub fn bench_encode_slot(criterion: &mut Criterion) { 14 | let slots = generate_slots(); 15 | 16 | let slots_replica = slots.iter().map(|s| s.to_replica()).collect::>(); 17 | 18 | criterion 19 | .benchmark_group("encode_slot") 20 | .bench_with_input("richat/prost", &slots_replica, |criterion, slots| { 21 | let created_at = SystemTime::now(); 22 | criterion.iter(|| { 23 | #[allow(clippy::unit_arg)] 24 | black_box({ 25 | for (slot, parent, status) in slots { 26 | let message = ProtobufMessage::Slot { 27 | slot: *slot, 28 | parent: *parent, 29 | status, 30 | }; 31 | message.encode_with_timestamp(ProtobufEncoder::Prost, created_at); 32 | } 33 | }) 34 | }); 35 | }) 36 | .bench_with_input("richat/raw", &slots_replica, |criterion, slots| { 37 | let created_at = SystemTime::now(); 38 | criterion.iter(|| { 39 | #[allow(clippy::unit_arg)] 40 | black_box({ 41 | for (slot, parent, status) in slots { 42 | let message = ProtobufMessage::Slot { 43 | slot: *slot, 44 | parent: *parent, 45 | status, 46 | }; 47 | message.encode_with_timestamp(ProtobufEncoder::Raw, created_at); 48 | } 49 | }) 50 | }); 51 | }) 52 | .bench_with_input( 53 | "dragons-mouth/full-pipeline", 54 | &slots_replica, 55 | |criterion, slots| { 56 | let created_at = Timestamp::from(SystemTime::now()); 57 | criterion.iter(|| { 58 | #[allow(clippy::unit_arg)] 59 | black_box({ 60 | for (slot, parent, status) in slots { 61 | let message = MessageSlot::from_geyser(*slot, *parent, status); 62 | let update = FilteredUpdate { 63 | filters: FilteredUpdateFilters::new(), 64 | message: FilteredUpdateOneof::slot(message), 65 | created_at, 66 | }; 67 | update.encode_to_vec(); 68 | } 69 | }) 70 | }); 71 | }, 72 | ); 73 | } 74 | -------------------------------------------------------------------------------- /plugin-agave/benches/encode/transaction.rs: -------------------------------------------------------------------------------- 1 | use { 2 | criterion::{black_box, Criterion}, 3 | prost::Message, 4 | prost_types::Timestamp, 5 | richat_plugin_agave::protobuf::{ 6 | fixtures::generate_transactions, ProtobufEncoder, ProtobufMessage, 7 | }, 8 | richat_proto::plugin::{ 9 | filter::message::{FilteredUpdate, FilteredUpdateFilters, FilteredUpdateOneof}, 10 | message::MessageTransaction, 11 | }, 12 | std::time::SystemTime, 13 | }; 14 | 15 | pub fn bench_encode_transactions(criterion: &mut Criterion) { 16 | let transactions = generate_transactions(); 17 | 18 | let transactions_replica = transactions 19 | .iter() 20 | .map(|tx| tx.to_replica()) 21 | .collect::>(); 22 | 23 | let transactions_grpc = transactions_replica 24 | .iter() 25 | .map(|(slot, transaction)| MessageTransaction::from_geyser(transaction, *slot)) 26 | .collect::>(); 27 | 28 | criterion 29 | .benchmark_group("encode_transaction") 30 | .bench_with_input( 31 | "richat/prost", 32 | &transactions_replica, 33 | |criterion, transactions| { 34 | let created_at = SystemTime::now(); 35 | criterion.iter(|| { 36 | #[allow(clippy::unit_arg)] 37 | black_box({ 38 | for (slot, transaction) in transactions { 39 | let message = ProtobufMessage::Transaction { 40 | slot: *slot, 41 | transaction, 42 | }; 43 | message.encode_with_timestamp(ProtobufEncoder::Prost, created_at); 44 | } 45 | }) 46 | }); 47 | }, 48 | ) 49 | .bench_with_input( 50 | "richat/raw", 51 | &transactions_replica, 52 | |criterion, transactions| { 53 | let created_at = SystemTime::now(); 54 | criterion.iter(|| { 55 | #[allow(clippy::unit_arg)] 56 | black_box({ 57 | for (slot, transaction) in transactions { 58 | let message = ProtobufMessage::Transaction { 59 | slot: *slot, 60 | transaction, 61 | }; 62 | message.encode_with_timestamp(ProtobufEncoder::Raw, created_at); 63 | } 64 | }) 65 | }); 66 | }, 67 | ) 68 | .bench_with_input( 69 | "dragons-mouth/encoding-only", 70 | &transactions_grpc, 71 | |criterion, transaction_messages| { 72 | let created_at = Timestamp::from(SystemTime::now()); 73 | criterion.iter(|| { 74 | #[allow(clippy::unit_arg)] 75 | black_box({ 76 | for message in transaction_messages { 77 | let update = FilteredUpdate { 78 | filters: FilteredUpdateFilters::new(), 79 | message: FilteredUpdateOneof::transaction(message), 80 | created_at, 81 | }; 82 | update.encode_to_vec(); 83 | } 84 | }) 85 | }); 86 | }, 87 | ) 88 | .bench_with_input( 89 | "dragons-mouth/full-pipeline", 90 | &transactions_replica, 91 | |criterion, transactions| { 92 | let created_at = Timestamp::from(SystemTime::now()); 93 | criterion.iter(|| { 94 | #[allow(clippy::unit_arg)] 95 | black_box({ 96 | for (slot, transaction) in transactions { 97 | let message = MessageTransaction::from_geyser(transaction, *slot); 98 | let update = FilteredUpdate { 99 | filters: FilteredUpdateFilters::new(), 100 | message: FilteredUpdateOneof::transaction(&message), 101 | created_at, 102 | }; 103 | update.encode_to_vec(); 104 | } 105 | }) 106 | }); 107 | }, 108 | ); 109 | } 110 | -------------------------------------------------------------------------------- /plugin-agave/build.rs: -------------------------------------------------------------------------------- 1 | use {cargo_lock::Lockfile, std::collections::HashSet}; 2 | 3 | fn main() -> anyhow::Result<()> { 4 | emit_version() 5 | } 6 | 7 | fn emit_version() -> anyhow::Result<()> { 8 | vergen::Emitter::default() 9 | .add_instructions(&vergen::BuildBuilder::all_build()?)? 10 | .add_instructions(&vergen::RustcBuilder::all_rustc()?)? 11 | .emit()?; 12 | 13 | // vergen git version does not looks cool 14 | println!( 15 | "cargo:rustc-env=GIT_VERSION={}", 16 | git_version::git_version!() 17 | ); 18 | 19 | // Extract packages version 20 | let lockfile = Lockfile::load("../Cargo.lock")?; 21 | println!( 22 | "cargo:rustc-env=SOLANA_SDK_VERSION={}", 23 | get_pkg_version(&lockfile, "solana-sdk") 24 | ); 25 | println!( 26 | "cargo:rustc-env=YELLOWSTONE_GRPC_PROTO_VERSION={}", 27 | get_pkg_version(&lockfile, "yellowstone-grpc-proto") 28 | ); 29 | println!( 30 | "cargo:rustc-env=RICHAT_PROTO_VERSION={}", 31 | get_pkg_version(&lockfile, "richat-proto") 32 | ); 33 | 34 | Ok(()) 35 | } 36 | 37 | fn get_pkg_version(lockfile: &Lockfile, pkg_name: &str) -> String { 38 | lockfile 39 | .packages 40 | .iter() 41 | .filter(|pkg| pkg.name.as_str() == pkg_name) 42 | .map(|pkg| pkg.version.to_string()) 43 | .collect::>() 44 | .into_iter() 45 | .collect::>() 46 | .join(",") 47 | } 48 | -------------------------------------------------------------------------------- /plugin-agave/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "libpath": "../target/release/librichat_plugin_agave.so", 3 | "logs": { 4 | "level": "info" 5 | }, 6 | "metrics": { 7 | "endpoint": "127.0.0.1:10123" 8 | }, 9 | "tokio": { 10 | "worker_threads": 4, 11 | "affinity": "0-3" 12 | }, 13 | "channel": { 14 | "encoder": "raw", 15 | "max_messages": "2_097_152", 16 | "max_bytes": "16_106_127_360" 17 | }, 18 | "grpc": { 19 | "endpoint": "127.0.0.1:10100", 20 | "tls_config": { 21 | "cert": "/path/to/cert.cert", 22 | "key": "/path/to/key.key" 23 | }, 24 | "compression": { 25 | "accept": ["gzip", "zstd"], 26 | "send": ["gzip", "zstd"] 27 | }, 28 | "max_decoding_message_size": "4_194_304", 29 | "server_tcp_keepalive": "20s", 30 | "server_tcp_nodelay": true, 31 | "server_http2_adaptive_window": null, 32 | "server_http2_keepalive_interval": null, 33 | "server_http2_keepalive_timeout": null, 34 | "server_initial_connection_window_size": null, 35 | "server_initial_stream_window_size": null, 36 | "x_tokens": [] 37 | }, 38 | "quic": { 39 | "endpoint": "127.0.0.1:10101", 40 | "tls_config": { 41 | // "cert": "/path/to/cert.cert", 42 | // "key": "/path/to/key.key", 43 | "self_signed_alt_names": ["localhost"] 44 | }, 45 | "expected_rtt": 100, 46 | "max_stream_bandwidth": 12500000, 47 | "max_recv_streams": 16, 48 | "x_tokens": [] 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /plugin-agave/fixtures/blocks/18144001.bincode: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lamports-dev/richat/8750ace2888e341403909df7b6686b4e6644e7b7/plugin-agave/fixtures/blocks/18144001.bincode -------------------------------------------------------------------------------- /plugin-agave/fixtures/blocks/43200000.bincode: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lamports-dev/richat/8750ace2888e341403909df7b6686b4e6644e7b7/plugin-agave/fixtures/blocks/43200000.bincode -------------------------------------------------------------------------------- /plugin-agave/fixtures/blocks/64800004.bincode: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lamports-dev/richat/8750ace2888e341403909df7b6686b4e6644e7b7/plugin-agave/fixtures/blocks/64800004.bincode -------------------------------------------------------------------------------- /plugin-agave/fixtures/target: -------------------------------------------------------------------------------- 1 | ../../target -------------------------------------------------------------------------------- /plugin-agave/fuzz/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "richat-plugin-agave-fuzz" 3 | version = "0.0.0" 4 | authors = ["Lamports Dev"] 5 | edition = "2021" 6 | description = "Richat Agave Geyser Plugin Fuzz Tests" 7 | publish = false 8 | 9 | [package.metadata] 10 | cargo-fuzz = true 11 | 12 | [workspace] 13 | members = ["."] 14 | 15 | [[bin]] 16 | name = "account" 17 | path = "fuzz_targets/account.rs" 18 | test = true 19 | doc = false 20 | bench = false 21 | 22 | [[bin]] 23 | name = "blockmeta" 24 | path = "fuzz_targets/blockmeta.rs" 25 | test = true 26 | doc = false 27 | bench = false 28 | 29 | [[bin]] 30 | name = "entry" 31 | path = "fuzz_targets/entry.rs" 32 | test = true 33 | doc = false 34 | bench = false 35 | 36 | [[bin]] 37 | name = "slot" 38 | path = "fuzz_targets/slot.rs" 39 | test = true 40 | doc = false 41 | bench = false 42 | 43 | [[bin]] 44 | name = "transaction" 45 | path = "fuzz_targets/transaction.rs" 46 | test = true 47 | doc = false 48 | bench = false 49 | 50 | [dependencies] 51 | agave-geyser-plugin-interface = "~2.2.1" 52 | arbitrary = { version = "1.4.1", features = ["derive"] } 53 | const-hex = "1.14.0" 54 | libfuzzer-sys = "0.4.8" 55 | prost = "0.11.9" 56 | richat-plugin-agave = { path = ".." } 57 | solana-account-decoder = "~2.2.1" 58 | solana-sdk = { version = "~2.2.1", features = ["dev-context-only-utils"] } 59 | solana-storage-proto = "~2.2.1" 60 | solana-transaction-status = "~2.2.1" 61 | -------------------------------------------------------------------------------- /plugin-agave/fuzz/fuzz_targets/account.rs: -------------------------------------------------------------------------------- 1 | #![no_main] 2 | 3 | use { 4 | agave_geyser_plugin_interface::geyser_plugin_interface::ReplicaAccountInfoV3, 5 | arbitrary::Arbitrary, 6 | richat_plugin_agave::protobuf::ProtobufMessage, 7 | solana_sdk::{ 8 | message::{LegacyMessage, Message, SanitizedMessage}, 9 | pubkey::PUBKEY_BYTES, 10 | signature::SIGNATURE_BYTES, 11 | transaction::SanitizedTransaction, 12 | }, 13 | std::{collections::HashSet, time::SystemTime}, 14 | }; 15 | 16 | #[derive(Debug, Arbitrary)] 17 | pub struct FuzzAccount<'a> { 18 | pubkey: [u8; PUBKEY_BYTES], 19 | lamports: u64, 20 | owner: [u8; PUBKEY_BYTES], 21 | executable: bool, 22 | rent_epoch: u64, 23 | data: &'a [u8], 24 | write_version: u64, 25 | txn: Option<[u8; SIGNATURE_BYTES]>, 26 | } 27 | 28 | #[derive(Debug, Arbitrary)] 29 | pub struct FuzzAccountMessage<'a> { 30 | slot: u64, 31 | account: FuzzAccount<'a>, 32 | } 33 | 34 | libfuzzer_sys::fuzz_target!(|fuzz_message: FuzzAccountMessage| { 35 | let txn = fuzz_message.account.txn.map(|signature| { 36 | SanitizedTransaction::new_for_tests( 37 | SanitizedMessage::Legacy(LegacyMessage::new(Message::default(), &HashSet::new())), 38 | vec![signature.as_slice().try_into().unwrap()], 39 | false, 40 | ) 41 | }); 42 | 43 | let message = ProtobufMessage::Account { 44 | account: &ReplicaAccountInfoV3 { 45 | pubkey: &fuzz_message.account.pubkey, 46 | lamports: fuzz_message.account.lamports, 47 | owner: &fuzz_message.account.owner, 48 | executable: fuzz_message.account.executable, 49 | rent_epoch: fuzz_message.account.rent_epoch, 50 | data: fuzz_message.account.data, 51 | write_version: fuzz_message.account.write_version, 52 | txn: txn.as_ref(), 53 | }, 54 | slot: fuzz_message.slot, 55 | }; 56 | let created_at = SystemTime::now(); 57 | 58 | let vec_prost = message.encode_prost(created_at); 59 | let vec_raw = message.encode_raw(created_at); 60 | 61 | assert_eq!( 62 | vec_prost, 63 | vec_raw, 64 | "prost hex: {}", 65 | const_hex::encode(&vec_prost) 66 | ); 67 | }); 68 | -------------------------------------------------------------------------------- /plugin-agave/fuzz/fuzz_targets/blockmeta.rs: -------------------------------------------------------------------------------- 1 | #![no_main] 2 | 3 | use { 4 | agave_geyser_plugin_interface::geyser_plugin_interface::ReplicaBlockInfoV4, 5 | arbitrary::Arbitrary, 6 | richat_plugin_agave::protobuf::ProtobufMessage, 7 | solana_transaction_status::{RewardType, RewardsAndNumPartitions}, 8 | std::time::SystemTime, 9 | }; 10 | 11 | #[derive(Debug, Clone, Copy, Arbitrary)] 12 | #[repr(i32)] 13 | pub enum FuzzRewardType { 14 | Fee = 1, 15 | Rent = 2, 16 | Staking = 3, 17 | Voting = 4, 18 | } 19 | 20 | impl From for RewardType { 21 | fn from(fuzz: FuzzRewardType) -> Self { 22 | match fuzz { 23 | FuzzRewardType::Fee => RewardType::Fee, 24 | FuzzRewardType::Rent => RewardType::Rent, 25 | FuzzRewardType::Staking => RewardType::Staking, 26 | FuzzRewardType::Voting => RewardType::Voting, 27 | } 28 | } 29 | } 30 | 31 | #[derive(Debug, Arbitrary)] 32 | pub struct FuzzReward { 33 | pubkey: String, 34 | lamports: i64, 35 | post_balance: u64, 36 | reward_type: Option, 37 | commission: Option, 38 | } 39 | 40 | #[derive(Debug, Arbitrary)] 41 | pub struct FuzzBlockMeta<'a> { 42 | parent_slot: u64, 43 | parent_blockhash: &'a str, 44 | slot: u64, 45 | blockhash: &'a str, 46 | rewards: Vec, 47 | num_partitions: Option, 48 | block_time: Option, 49 | block_height: Option, 50 | executed_transaction_count: u64, 51 | entry_count: u64, 52 | } 53 | 54 | libfuzzer_sys::fuzz_target!(|fuzz_blockmeta: FuzzBlockMeta| { 55 | let rewards_and_num_partitions = RewardsAndNumPartitions { 56 | rewards: fuzz_blockmeta 57 | .rewards 58 | .iter() 59 | .map(|reward| solana_transaction_status::Reward { 60 | pubkey: reward.pubkey.to_owned(), 61 | lamports: reward.lamports, 62 | post_balance: reward.post_balance, 63 | reward_type: reward.reward_type.map(Into::into), 64 | commission: reward.commission, 65 | }) 66 | .collect(), 67 | num_partitions: fuzz_blockmeta.num_partitions, 68 | }; 69 | let blockinfo = ReplicaBlockInfoV4 { 70 | parent_slot: fuzz_blockmeta.parent_slot, 71 | parent_blockhash: fuzz_blockmeta.parent_blockhash, 72 | slot: fuzz_blockmeta.slot, 73 | blockhash: fuzz_blockmeta.blockhash, 74 | rewards: &rewards_and_num_partitions, 75 | block_time: fuzz_blockmeta.block_time, 76 | block_height: fuzz_blockmeta.block_height, 77 | executed_transaction_count: fuzz_blockmeta.executed_transaction_count, 78 | entry_count: fuzz_blockmeta.entry_count, 79 | }; 80 | 81 | let message = ProtobufMessage::BlockMeta { 82 | blockinfo: &blockinfo, 83 | }; 84 | let created_at = SystemTime::now(); 85 | 86 | let vec_prost = message.encode_prost(created_at); 87 | let vec_raw = message.encode_raw(created_at); 88 | 89 | assert_eq!( 90 | vec_prost, 91 | vec_raw, 92 | "prost hex: {}", 93 | const_hex::encode(&vec_prost) 94 | ); 95 | }); 96 | -------------------------------------------------------------------------------- /plugin-agave/fuzz/fuzz_targets/entry.rs: -------------------------------------------------------------------------------- 1 | #![no_main] 2 | 3 | use { 4 | agave_geyser_plugin_interface::geyser_plugin_interface::ReplicaEntryInfoV2, 5 | arbitrary::Arbitrary, richat_plugin_agave::protobuf::ProtobufMessage, std::time::SystemTime, 6 | }; 7 | 8 | #[derive(Debug, Arbitrary)] 9 | pub struct FuzzEntry { 10 | pub slot: u64, 11 | pub index: usize, 12 | pub num_hashes: u64, 13 | pub hash: Vec, 14 | pub executed_transaction_count: u64, 15 | pub starting_transaction_index: usize, 16 | } 17 | 18 | libfuzzer_sys::fuzz_target!(|fuzz_entry: FuzzEntry| { 19 | let message = ProtobufMessage::Entry { 20 | entry: &ReplicaEntryInfoV2 { 21 | slot: fuzz_entry.slot, 22 | index: fuzz_entry.index, 23 | num_hashes: fuzz_entry.num_hashes, 24 | hash: &fuzz_entry.hash, 25 | executed_transaction_count: fuzz_entry.executed_transaction_count, 26 | starting_transaction_index: fuzz_entry.starting_transaction_index, 27 | }, 28 | }; 29 | let created_at = SystemTime::now(); 30 | 31 | let vec_prost = message.encode_prost(created_at); 32 | let vec_raw = message.encode_raw(created_at); 33 | 34 | assert_eq!( 35 | vec_prost, 36 | vec_raw, 37 | "prost hex: {}", 38 | const_hex::encode(&vec_prost) 39 | ); 40 | }); 41 | -------------------------------------------------------------------------------- /plugin-agave/fuzz/fuzz_targets/slot.rs: -------------------------------------------------------------------------------- 1 | #![no_main] 2 | 3 | use { 4 | agave_geyser_plugin_interface::geyser_plugin_interface::SlotStatus, arbitrary::Arbitrary, 5 | richat_plugin_agave::protobuf::ProtobufMessage, std::time::SystemTime, 6 | }; 7 | 8 | #[derive(Debug, Clone, Copy, Arbitrary)] 9 | #[repr(i32)] 10 | pub enum FuzzSlotStatus { 11 | Processed = 0, 12 | Rooted = 1, 13 | Confirmed = 2, 14 | FirstShredReceived = 3, 15 | Completed = 4, 16 | CreatedBank = 5, 17 | Dead = 6, 18 | } 19 | 20 | impl From for SlotStatus { 21 | fn from(fuzz: FuzzSlotStatus) -> Self { 22 | match fuzz { 23 | FuzzSlotStatus::Processed => SlotStatus::Processed, 24 | FuzzSlotStatus::Rooted => SlotStatus::Rooted, 25 | FuzzSlotStatus::Confirmed => SlotStatus::Confirmed, 26 | FuzzSlotStatus::FirstShredReceived => SlotStatus::FirstShredReceived, 27 | FuzzSlotStatus::Completed => SlotStatus::Completed, 28 | FuzzSlotStatus::CreatedBank => SlotStatus::CreatedBank, 29 | FuzzSlotStatus::Dead => SlotStatus::Dead(String::new()), 30 | } 31 | } 32 | } 33 | 34 | #[derive(Arbitrary, Debug)] 35 | pub struct FuzzSlot { 36 | slot: u64, 37 | parent: Option, 38 | status: FuzzSlotStatus, 39 | } 40 | 41 | libfuzzer_sys::fuzz_target!(|fuzz_slot: FuzzSlot| { 42 | let message = ProtobufMessage::Slot { 43 | slot: fuzz_slot.slot, 44 | parent: fuzz_slot.parent, 45 | status: &fuzz_slot.status.into(), 46 | }; 47 | let created_at = SystemTime::now(); 48 | 49 | let vec_prost = message.encode_prost(created_at); 50 | let vec_raw = message.encode_raw(created_at); 51 | 52 | assert_eq!( 53 | vec_prost, 54 | vec_raw, 55 | "prost hex: {}", 56 | const_hex::encode(&vec_prost) 57 | ); 58 | }); 59 | -------------------------------------------------------------------------------- /plugin-agave/src/bin/config-check.rs: -------------------------------------------------------------------------------- 1 | use {clap::Parser, richat_plugin_agave::config::Config}; 2 | 3 | #[derive(Debug, Parser)] 4 | #[clap( 5 | author, 6 | version, 7 | about = "Richat Agave Geyser Plugin Config Check Cli Tool" 8 | )] 9 | struct Args { 10 | #[clap(short, long, default_value_t = String::from("config.json"))] 11 | /// Path to config 12 | config: String, 13 | } 14 | 15 | fn main() -> anyhow::Result<()> { 16 | anyhow::ensure!( 17 | rustls::crypto::aws_lc_rs::default_provider() 18 | .install_default() 19 | .is_ok(), 20 | "failed to call CryptoProvider::install_default()" 21 | ); 22 | 23 | let args = Args::parse(); 24 | let _config = Config::load_from_file(args.config)?; 25 | println!("Config is OK!"); 26 | Ok(()) 27 | } 28 | -------------------------------------------------------------------------------- /plugin-agave/src/config.rs: -------------------------------------------------------------------------------- 1 | use { 2 | crate::protobuf::ProtobufEncoder, 3 | agave_geyser_plugin_interface::geyser_plugin_interface::{ 4 | GeyserPluginError, Result as PluginResult, 5 | }, 6 | richat_shared::{ 7 | config::{deserialize_num_str, ConfigMetrics, ConfigTokio}, 8 | transports::{grpc::ConfigGrpcServer, quic::ConfigQuicServer}, 9 | }, 10 | serde::{ 11 | de::{self, Deserializer}, 12 | Deserialize, 13 | }, 14 | std::{fs, path::Path}, 15 | }; 16 | 17 | #[derive(Debug, Clone, Default, Deserialize)] 18 | #[serde(deny_unknown_fields, default)] 19 | pub struct Config { 20 | pub libpath: String, 21 | pub logs: ConfigLogs, 22 | pub metrics: Option, 23 | pub tokio: ConfigTokio, 24 | pub channel: ConfigChannel, 25 | pub quic: Option, 26 | pub grpc: Option, 27 | } 28 | 29 | impl Config { 30 | fn load_from_str(config: &str) -> PluginResult { 31 | serde_json::from_str(config).map_err(|error| GeyserPluginError::ConfigFileReadError { 32 | msg: error.to_string(), 33 | }) 34 | } 35 | 36 | pub fn load_from_file>(file: P) -> PluginResult { 37 | let config = fs::read_to_string(file).map_err(GeyserPluginError::ConfigFileOpenError)?; 38 | Self::load_from_str(&config) 39 | } 40 | } 41 | 42 | #[derive(Debug, Clone, Deserialize)] 43 | #[serde(deny_unknown_fields, default)] 44 | pub struct ConfigLogs { 45 | /// Log level 46 | pub level: String, 47 | } 48 | 49 | impl Default for ConfigLogs { 50 | fn default() -> Self { 51 | Self { 52 | level: "info".to_owned(), 53 | } 54 | } 55 | } 56 | 57 | #[derive(Debug, Clone, Copy, Deserialize)] 58 | #[serde(deny_unknown_fields, default)] 59 | pub struct ConfigChannel { 60 | #[serde(deserialize_with = "ConfigChannel::deserialize_encoder")] 61 | pub encoder: ProtobufEncoder, 62 | #[serde(deserialize_with = "deserialize_num_str")] 63 | pub max_messages: usize, 64 | #[serde(deserialize_with = "deserialize_num_str")] 65 | pub max_bytes: usize, 66 | } 67 | 68 | impl Default for ConfigChannel { 69 | fn default() -> Self { 70 | Self { 71 | encoder: ProtobufEncoder::Raw, 72 | max_messages: 2_097_152, // aligned to power of 2, ~20k/slot should give us ~100 slots 73 | max_bytes: 15 * 1024 * 1024 * 1024, // 15GiB with ~150MiB/slot should give us ~100 slots 74 | } 75 | } 76 | } 77 | 78 | impl ConfigChannel { 79 | pub fn deserialize_encoder<'de, D>(deserializer: D) -> Result 80 | where 81 | D: Deserializer<'de>, 82 | { 83 | match Deserialize::deserialize(deserializer)? { 84 | "prost" => Ok(ProtobufEncoder::Prost), 85 | "raw" => Ok(ProtobufEncoder::Raw), 86 | value => Err(de::Error::custom(format!( 87 | "failed to decode encoder: {value}" 88 | ))), 89 | } 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /plugin-agave/src/lib.rs: -------------------------------------------------------------------------------- 1 | pub mod channel; 2 | pub mod config; 3 | pub mod metrics; 4 | pub mod plugin; 5 | pub mod protobuf; 6 | pub mod version; 7 | -------------------------------------------------------------------------------- /plugin-agave/src/metrics.rs: -------------------------------------------------------------------------------- 1 | use { 2 | crate::version::VERSION as VERSION_INFO, 3 | bytes::Bytes, 4 | metrics::{counter, describe_counter, describe_gauge}, 5 | metrics_exporter_prometheus::{BuildError, PrometheusBuilder, PrometheusHandle}, 6 | richat_shared::config::ConfigMetrics, 7 | std::{future::Future, io}, 8 | tokio::{ 9 | task::JoinError, 10 | time::{sleep, Duration}, 11 | }, 12 | }; 13 | 14 | pub const GEYSER_SLOT_STATUS: &str = "geyser_slot_status"; // status 15 | pub const GEYSER_MISSED_SLOT_STATUS: &str = "geyser_missed_slot_status_total"; // status 16 | pub const CHANNEL_MESSAGES_TOTAL: &str = "channel_messages_total"; 17 | pub const CHANNEL_SLOTS_TOTAL: &str = "channel_slots_total"; 18 | pub const CHANNEL_BYTES_TOTAL: &str = "channel_bytes_total"; 19 | pub const CONNECTIONS_TOTAL: &str = "connections_total"; // transport 20 | 21 | pub fn setup() -> Result { 22 | let handle = PrometheusBuilder::new().install_recorder()?; 23 | 24 | describe_counter!("version", "Richat Plugin version info"); 25 | counter!( 26 | "version", 27 | "buildts" => VERSION_INFO.buildts, 28 | "git" => VERSION_INFO.git, 29 | "package" => VERSION_INFO.package, 30 | "proto" => VERSION_INFO.proto, 31 | "rustc" => VERSION_INFO.rustc, 32 | "solana" => VERSION_INFO.solana, 33 | "version" => VERSION_INFO.version, 34 | ) 35 | .absolute(1); 36 | 37 | describe_gauge!(GEYSER_SLOT_STATUS, "Latest slot received from Geyser"); 38 | describe_counter!( 39 | GEYSER_MISSED_SLOT_STATUS, 40 | "Number of missed slot status updates" 41 | ); 42 | describe_gauge!( 43 | CHANNEL_MESSAGES_TOTAL, 44 | "Total number of messages in channel" 45 | ); 46 | describe_gauge!(CHANNEL_SLOTS_TOTAL, "Total number of slots in channel"); 47 | describe_gauge!(CHANNEL_BYTES_TOTAL, "Total size of all messages in channel"); 48 | describe_gauge!(CONNECTIONS_TOTAL, "Total number of connections"); 49 | 50 | Ok(handle) 51 | } 52 | 53 | pub async fn spawn_server( 54 | config: ConfigMetrics, 55 | handle: PrometheusHandle, 56 | shutdown: impl Future + Send + 'static, 57 | ) -> io::Result>> { 58 | let recorder_handle = handle.clone(); 59 | tokio::spawn(async move { 60 | loop { 61 | sleep(Duration::from_secs(1)).await; 62 | recorder_handle.run_upkeep(); 63 | } 64 | }); 65 | 66 | richat_shared::metrics::spawn_server( 67 | config, 68 | move || Bytes::from(handle.render()), // metrics 69 | || true, // health 70 | || true, // ready 71 | shutdown, 72 | ) 73 | .await 74 | } 75 | -------------------------------------------------------------------------------- /plugin-agave/src/protobuf/encoding/account.rs: -------------------------------------------------------------------------------- 1 | use { 2 | super::{bytes_encode, bytes_encoded_len}, 3 | agave_geyser_plugin_interface::geyser_plugin_interface::ReplicaAccountInfoV3, 4 | prost::encoding::{self, WireType}, 5 | solana_sdk::clock::Slot, 6 | std::ops::Deref, 7 | }; 8 | 9 | #[derive(Debug)] 10 | struct ReplicaWrapper<'a>(&'a ReplicaAccountInfoV3<'a>); 11 | 12 | impl<'a> Deref for ReplicaWrapper<'a> { 13 | type Target = &'a ReplicaAccountInfoV3<'a>; 14 | 15 | fn deref(&self) -> &Self::Target { 16 | &self.0 17 | } 18 | } 19 | 20 | impl prost::Message for ReplicaWrapper<'_> { 21 | fn encode_raw(&self, buf: &mut impl bytes::BufMut) 22 | where 23 | Self: Sized, 24 | { 25 | bytes_encode(1, self.pubkey, buf); 26 | if self.lamports != 0 { 27 | encoding::uint64::encode(2, &self.lamports, buf); 28 | }; 29 | bytes_encode(3, self.owner, buf); 30 | if self.executable { 31 | encoding::bool::encode(4, &self.executable, buf); 32 | } 33 | if self.rent_epoch != 0 { 34 | encoding::uint64::encode(5, &self.rent_epoch, buf); 35 | } 36 | if !self.data.is_empty() { 37 | bytes_encode(6, self.data, buf); 38 | } 39 | if self.write_version != 0 { 40 | encoding::uint64::encode(7, &self.write_version, buf); 41 | } 42 | if let Some(txn) = self.txn { 43 | bytes_encode(8, txn.signature().as_ref(), buf); 44 | } 45 | } 46 | 47 | fn encoded_len(&self) -> usize { 48 | bytes_encoded_len(1, self.pubkey) 49 | + if self.lamports != 0 { 50 | encoding::uint64::encoded_len(2, &self.lamports) 51 | } else { 52 | 0 53 | } 54 | + bytes_encoded_len(3, self.owner) 55 | + if self.executable { 56 | encoding::bool::encoded_len(4, &self.executable) 57 | } else { 58 | 0 59 | } 60 | + if self.rent_epoch != 0 { 61 | encoding::uint64::encoded_len(5, &self.rent_epoch) 62 | } else { 63 | 0 64 | } 65 | + if !self.data.is_empty() { 66 | bytes_encoded_len(6, self.data) 67 | } else { 68 | 0 69 | } 70 | + if self.write_version != 0 { 71 | encoding::uint64::encoded_len(7, &self.write_version) 72 | } else { 73 | 0 74 | } 75 | + self 76 | .0 77 | .txn 78 | .map_or(0, |txn| bytes_encoded_len(8, txn.signature().as_ref())) 79 | } 80 | 81 | fn clear(&mut self) { 82 | unimplemented!() 83 | } 84 | 85 | fn merge_field( 86 | &mut self, 87 | _tag: u32, 88 | _wire_type: WireType, 89 | _buf: &mut impl bytes::Buf, 90 | _ctx: encoding::DecodeContext, 91 | ) -> Result<(), prost::DecodeError> 92 | where 93 | Self: Sized, 94 | { 95 | unimplemented!() 96 | } 97 | } 98 | 99 | #[derive(Debug)] 100 | pub struct Account<'a> { 101 | account: &'a ReplicaAccountInfoV3<'a>, 102 | slot: Slot, 103 | } 104 | 105 | impl<'a> Account<'a> { 106 | pub const fn new(slot: Slot, account: &'a ReplicaAccountInfoV3<'a>) -> Self { 107 | Self { slot, account } 108 | } 109 | } 110 | 111 | impl prost::Message for Account<'_> { 112 | fn encode_raw(&self, buf: &mut impl bytes::BufMut) { 113 | let wrapper = ReplicaWrapper(self.account); 114 | encoding::message::encode(1, &wrapper, buf); 115 | if self.slot != 0 { 116 | encoding::uint64::encode(2, &self.slot, buf) 117 | } 118 | } 119 | 120 | fn encoded_len(&self) -> usize { 121 | let wrapper = ReplicaWrapper(self.account); 122 | encoding::message::encoded_len(1, &wrapper) 123 | + if self.slot != 0 { 124 | encoding::uint64::encoded_len(2, &self.slot) 125 | } else { 126 | 0 127 | } 128 | } 129 | 130 | fn merge_field( 131 | &mut self, 132 | _tag: u32, 133 | _wire_type: WireType, 134 | _buf: &mut impl bytes::Buf, 135 | _ctx: encoding::DecodeContext, 136 | ) -> Result<(), prost::DecodeError> 137 | where 138 | Self: Sized, 139 | { 140 | unimplemented!() 141 | } 142 | 143 | fn clear(&mut self) { 144 | unimplemented!() 145 | } 146 | } 147 | -------------------------------------------------------------------------------- /plugin-agave/src/protobuf/encoding/block_meta.rs: -------------------------------------------------------------------------------- 1 | use { 2 | super::{bytes_encode, bytes_encoded_len, RewardWrapper}, 3 | agave_geyser_plugin_interface::geyser_plugin_interface::ReplicaBlockInfoV4, 4 | prost::{ 5 | bytes::BufMut, 6 | encoding::{self, WireType}, 7 | }, 8 | richat_proto::convert_to, 9 | solana_transaction_status::RewardsAndNumPartitions, 10 | std::ops::Deref, 11 | }; 12 | 13 | #[derive(Debug)] 14 | pub struct BlockMeta<'a> { 15 | blockinfo: &'a ReplicaBlockInfoV4<'a>, 16 | } 17 | 18 | impl<'a> BlockMeta<'a> { 19 | pub const fn new(blockinfo: &'a ReplicaBlockInfoV4<'a>) -> Self { 20 | Self { blockinfo } 21 | } 22 | } 23 | 24 | impl prost::Message for BlockMeta<'_> { 25 | fn encode_raw(&self, buf: &mut impl prost::bytes::BufMut) { 26 | let rewards = RewardsAndNumPartitionsWrapper(self.blockinfo.rewards); 27 | 28 | if self.blockinfo.slot != 0 { 29 | encoding::uint64::encode(1, &self.blockinfo.slot, buf); 30 | } 31 | if !self.blockinfo.blockhash.is_empty() { 32 | bytes_encode(2, self.blockinfo.blockhash.as_ref(), buf); 33 | } 34 | encoding::message::encode(3, &rewards, buf); 35 | if let Some(block_time) = self.blockinfo.block_time { 36 | encoding::message::encode(4, &convert_to::create_timestamp(block_time), buf); 37 | } 38 | if let Some(block_height) = self.blockinfo.block_height { 39 | encoding::message::encode(5, &convert_to::create_block_height(block_height), buf); 40 | } 41 | if self.blockinfo.parent_slot != 0 { 42 | encoding::uint64::encode(6, &self.blockinfo.parent_slot, buf); 43 | } 44 | if !self.blockinfo.parent_blockhash.is_empty() { 45 | bytes_encode(7, self.blockinfo.parent_blockhash.as_ref(), buf); 46 | } 47 | if self.blockinfo.executed_transaction_count != 0 { 48 | encoding::uint64::encode(8, &self.blockinfo.executed_transaction_count, buf); 49 | } 50 | if self.blockinfo.entry_count != 0 { 51 | encoding::uint64::encode(9, &self.blockinfo.entry_count, buf); 52 | } 53 | } 54 | 55 | fn encoded_len(&self) -> usize { 56 | let rewards = RewardsAndNumPartitionsWrapper(self.blockinfo.rewards); 57 | 58 | (if self.blockinfo.slot != 0 { 59 | encoding::uint64::encoded_len(1, &self.blockinfo.slot) 60 | } else { 61 | 0 62 | }) + if !self.blockinfo.blockhash.is_empty() { 63 | bytes_encoded_len(2, self.blockinfo.blockhash.as_ref()) 64 | } else { 65 | 0 66 | } + encoding::message::encoded_len(3, &rewards) 67 | + if let Some(block_time) = self.blockinfo.block_time { 68 | encoding::message::encoded_len(4, &convert_to::create_timestamp(block_time)) 69 | } else { 70 | 0 71 | } 72 | + if let Some(block_height) = self.blockinfo.block_height { 73 | encoding::message::encoded_len(5, &convert_to::create_block_height(block_height)) 74 | } else { 75 | 0 76 | } 77 | + if self.blockinfo.parent_slot != 0 { 78 | encoding::uint64::encoded_len(6, &self.blockinfo.parent_slot) 79 | } else { 80 | 0 81 | } 82 | + if !self.blockinfo.parent_blockhash.is_empty() { 83 | bytes_encoded_len(7, self.blockinfo.parent_blockhash.as_ref()) 84 | } else { 85 | 0 86 | } 87 | + if self.blockinfo.executed_transaction_count != 0 { 88 | encoding::uint64::encoded_len(8, &self.blockinfo.executed_transaction_count) 89 | } else { 90 | 0 91 | } 92 | + if self.blockinfo.entry_count != 0 { 93 | encoding::uint64::encoded_len(9, &self.blockinfo.entry_count) 94 | } else { 95 | 0 96 | } 97 | } 98 | 99 | fn merge_field( 100 | &mut self, 101 | _tag: u32, 102 | _wire_type: encoding::WireType, 103 | _buf: &mut impl bytes::Buf, 104 | _ctx: encoding::DecodeContext, 105 | ) -> Result<(), prost::DecodeError> 106 | where 107 | Self: Sized, 108 | { 109 | unimplemented!() 110 | } 111 | 112 | fn clear(&mut self) { 113 | unimplemented!() 114 | } 115 | } 116 | 117 | #[derive(Debug)] 118 | struct RewardsAndNumPartitionsWrapper<'a>(&'a RewardsAndNumPartitions); 119 | 120 | impl<'a> Deref for RewardsAndNumPartitionsWrapper<'a> { 121 | type Target = &'a RewardsAndNumPartitions; 122 | 123 | fn deref(&self) -> &Self::Target { 124 | &self.0 125 | } 126 | } 127 | 128 | impl prost::Message for RewardsAndNumPartitionsWrapper<'_> { 129 | fn encode_raw(&self, buf: &mut impl BufMut) 130 | where 131 | Self: Sized, 132 | { 133 | encoding::message::encode_repeated(1, RewardWrapper::new(&self.rewards), buf); 134 | if let Some(num_partitions) = self.num_partitions { 135 | encoding::message::encode(2, &convert_to::create_num_partitions(num_partitions), buf); 136 | } 137 | } 138 | 139 | fn encoded_len(&self) -> usize { 140 | encoding::message::encoded_len_repeated(1, RewardWrapper::new(&self.rewards)) 141 | + if let Some(num_partitions) = self.num_partitions { 142 | encoding::message::encoded_len( 143 | 2, 144 | &convert_to::create_num_partitions(num_partitions), 145 | ) 146 | } else { 147 | 0 148 | } 149 | } 150 | 151 | fn clear(&mut self) { 152 | unimplemented!() 153 | } 154 | 155 | fn merge_field( 156 | &mut self, 157 | _tag: u32, 158 | _wire_type: WireType, 159 | _buf: &mut impl bytes::Buf, 160 | _ctx: encoding::DecodeContext, 161 | ) -> Result<(), prost::DecodeError> 162 | where 163 | Self: Sized, 164 | { 165 | unimplemented!() 166 | } 167 | } 168 | -------------------------------------------------------------------------------- /plugin-agave/src/protobuf/encoding/entry.rs: -------------------------------------------------------------------------------- 1 | use { 2 | super::{bytes_encode, bytes_encoded_len}, 3 | agave_geyser_plugin_interface::geyser_plugin_interface::ReplicaEntryInfoV2, 4 | prost::encoding, 5 | }; 6 | 7 | #[derive(Debug)] 8 | pub struct Entry<'a> { 9 | entry: &'a ReplicaEntryInfoV2<'a>, 10 | } 11 | 12 | impl<'a> Entry<'a> { 13 | pub const fn new(entry: &'a ReplicaEntryInfoV2<'a>) -> Self { 14 | Self { entry } 15 | } 16 | } 17 | 18 | impl prost::Message for Entry<'_> { 19 | fn encode_raw(&self, buf: &mut impl bytes::BufMut) { 20 | let index = self.entry.index as u64; 21 | let starting_transaction_index = self.entry.starting_transaction_index as u64; 22 | 23 | if self.entry.slot != 0 { 24 | encoding::uint64::encode(1, &self.entry.slot, buf); 25 | } 26 | if index != 0 { 27 | encoding::uint64::encode(2, &index, buf); 28 | } 29 | if self.entry.num_hashes != 0 { 30 | encoding::uint64::encode(3, &self.entry.num_hashes, buf); 31 | } 32 | bytes_encode(4, self.entry.hash, buf); 33 | if self.entry.executed_transaction_count != 0 { 34 | encoding::uint64::encode(5, &self.entry.executed_transaction_count, buf); 35 | } 36 | if starting_transaction_index != 0 { 37 | encoding::uint64::encode(6, &starting_transaction_index, buf); 38 | } 39 | } 40 | 41 | fn encoded_len(&self) -> usize { 42 | let index = self.entry.index as u64; 43 | let starting_transaction_index = self.entry.starting_transaction_index as u64; 44 | 45 | (if self.entry.slot != 0 { 46 | encoding::uint64::encoded_len(1, &self.entry.slot) 47 | } else { 48 | 0 49 | }) + if index != 0 { 50 | encoding::uint64::encoded_len(2, &index) 51 | } else { 52 | 0 53 | } + if self.entry.num_hashes != 0 { 54 | encoding::uint64::encoded_len(3, &self.entry.num_hashes) 55 | } else { 56 | 0 57 | } + bytes_encoded_len(4, self.entry.hash) 58 | + if self.entry.executed_transaction_count != 0 { 59 | encoding::uint64::encoded_len(5, &self.entry.executed_transaction_count) 60 | } else { 61 | 0 62 | } 63 | + if starting_transaction_index != 0 { 64 | encoding::uint64::encoded_len(6, &starting_transaction_index) 65 | } else { 66 | 0 67 | } 68 | } 69 | 70 | fn merge_field( 71 | &mut self, 72 | _tag: u32, 73 | _wire_type: encoding::WireType, 74 | _buf: &mut impl bytes::Buf, 75 | _ctx: encoding::DecodeContext, 76 | ) -> Result<(), prost::DecodeError> 77 | where 78 | Self: Sized, 79 | { 80 | unimplemented!() 81 | } 82 | 83 | fn clear(&mut self) { 84 | unimplemented!() 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /plugin-agave/src/protobuf/encoding/mod.rs: -------------------------------------------------------------------------------- 1 | pub use self::{ 2 | account::Account, block_meta::BlockMeta, entry::Entry, slot::Slot, transaction::Transaction, 3 | }; 4 | use { 5 | prost::{ 6 | bytes::BufMut, 7 | encoding::{self, encode_key, encode_varint, encoded_len_varint, key_len, WireType}, 8 | }, 9 | solana_transaction_status::{Reward, RewardType}, 10 | std::{marker::PhantomData, ops::Deref}, 11 | }; 12 | 13 | mod account; 14 | mod block_meta; 15 | mod entry; 16 | mod slot; 17 | mod transaction; 18 | 19 | const NUM_STRINGS: [&str; 256] = [ 20 | "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", 21 | "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", 22 | "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", 23 | "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61", "62", "63", "64", 24 | "65", "66", "67", "68", "69", "70", "71", "72", "73", "74", "75", "76", "77", "78", "79", "80", 25 | "81", "82", "83", "84", "85", "86", "87", "88", "89", "90", "91", "92", "93", "94", "95", "96", 26 | "97", "98", "99", "100", "101", "102", "103", "104", "105", "106", "107", "108", "109", "110", 27 | "111", "112", "113", "114", "115", "116", "117", "118", "119", "120", "121", "122", "123", 28 | "124", "125", "126", "127", "128", "129", "130", "131", "132", "133", "134", "135", "136", 29 | "137", "138", "139", "140", "141", "142", "143", "144", "145", "146", "147", "148", "149", 30 | "150", "151", "152", "153", "154", "155", "156", "157", "158", "159", "160", "161", "162", 31 | "163", "164", "165", "166", "167", "168", "169", "170", "171", "172", "173", "174", "175", 32 | "176", "177", "178", "179", "180", "181", "182", "183", "184", "185", "186", "187", "188", 33 | "189", "190", "191", "192", "193", "194", "195", "196", "197", "198", "199", "200", "201", 34 | "202", "203", "204", "205", "206", "207", "208", "209", "210", "211", "212", "213", "214", 35 | "215", "216", "217", "218", "219", "220", "221", "222", "223", "224", "225", "226", "227", 36 | "228", "229", "230", "231", "232", "233", "234", "235", "236", "237", "238", "239", "240", 37 | "241", "242", "243", "244", "245", "246", "247", "248", "249", "250", "251", "252", "253", 38 | "254", "255", 39 | ]; 40 | 41 | const fn u8_to_static_str(num: u8) -> &'static str { 42 | NUM_STRINGS[num as usize] 43 | } 44 | 45 | #[repr(transparent)] 46 | #[derive(Debug)] 47 | struct RewardWrapper<'a>(Reward, PhantomData<&'a ()>); 48 | 49 | impl RewardWrapper<'_> { 50 | const fn new(rewards: &[Reward]) -> &[Self] { 51 | // SAFETY: the compiler guarantees that 52 | // `align_of::() == align_of::()`, 53 | // `size_of::() == size_of::()`, 54 | // the alignment of `RewardWrapper` and `Reward` are identical. 55 | unsafe { std::mem::transmute(rewards) } 56 | } 57 | } 58 | 59 | impl Deref for RewardWrapper<'_> { 60 | type Target = Reward; 61 | 62 | fn deref(&self) -> &Self::Target { 63 | &self.0 64 | } 65 | } 66 | 67 | impl prost::Message for RewardWrapper<'_> { 68 | fn encode_raw(&self, buf: &mut impl BufMut) 69 | where 70 | Self: Sized, 71 | { 72 | if !self.pubkey.is_empty() { 73 | encoding::string::encode(1, &self.pubkey, buf); 74 | } 75 | if self.lamports != 0 { 76 | encoding::int64::encode(2, &self.lamports, buf); 77 | } 78 | if self.post_balance != 0 { 79 | encoding::uint64::encode(3, &self.post_balance, buf); 80 | } 81 | if self.reward_type.is_some() { 82 | encoding::int32::encode(4, &reward_type_as_i32(self.reward_type), buf); 83 | } 84 | if let Some(commission) = self.commission { 85 | bytes_encode(5, u8_to_static_str(commission).as_ref(), buf); 86 | } 87 | } 88 | 89 | fn encoded_len(&self) -> usize { 90 | (if !self.pubkey.is_empty() { 91 | encoding::string::encoded_len(1, &self.pubkey) 92 | } else { 93 | 0 94 | }) + if self.lamports != 0 { 95 | encoding::int64::encoded_len(2, &self.lamports) 96 | } else { 97 | 0 98 | } + if self.post_balance != 0 { 99 | encoding::uint64::encoded_len(3, &self.post_balance) 100 | } else { 101 | 0 102 | } + if self.reward_type.is_some() { 103 | encoding::int32::encoded_len(4, &reward_type_as_i32(self.reward_type)) 104 | } else { 105 | 0 106 | } + self.commission.map_or(0, |commission| { 107 | bytes_encoded_len(5, u8_to_static_str(commission).as_ref()) 108 | }) 109 | } 110 | 111 | fn clear(&mut self) { 112 | unimplemented!() 113 | } 114 | 115 | fn merge_field( 116 | &mut self, 117 | _tag: u32, 118 | _wire_type: WireType, 119 | _buf: &mut impl bytes::Buf, 120 | _ctx: encoding::DecodeContext, 121 | ) -> Result<(), prost::DecodeError> 122 | where 123 | Self: Sized, 124 | { 125 | unimplemented!() 126 | } 127 | } 128 | 129 | pub const fn reward_type_as_i32(reward_type: Option) -> i32 { 130 | match reward_type { 131 | None => 0, 132 | Some(RewardType::Fee) => 1, 133 | Some(RewardType::Rent) => 2, 134 | Some(RewardType::Staking) => 3, 135 | Some(RewardType::Voting) => 4, 136 | } 137 | } 138 | 139 | #[inline] 140 | pub fn bytes_encode(tag: u32, value: &[u8], buf: &mut impl BufMut) { 141 | encode_key(tag, WireType::LengthDelimited, buf); 142 | encode_varint(value.len() as u64, buf); 143 | buf.put(value) 144 | } 145 | 146 | #[inline] 147 | pub const fn bytes_encoded_len(tag: u32, value: &[u8]) -> usize { 148 | key_len(tag) + encoded_len_varint(value.len() as u64) + value.len() 149 | } 150 | -------------------------------------------------------------------------------- /plugin-agave/src/protobuf/encoding/slot.rs: -------------------------------------------------------------------------------- 1 | use { 2 | agave_geyser_plugin_interface::geyser_plugin_interface::SlotStatus, prost::encoding, 3 | richat_proto::geyser::CommitmentLevel, 4 | }; 5 | 6 | const fn slot_status_as_i32(status: &SlotStatus) -> i32 { 7 | match status { 8 | SlotStatus::Processed => 0, 9 | SlotStatus::Rooted => 2, 10 | SlotStatus::Confirmed => 1, 11 | SlotStatus::FirstShredReceived => 3, 12 | SlotStatus::Completed => 4, 13 | SlotStatus::CreatedBank => 5, 14 | SlotStatus::Dead(_) => 6, 15 | } 16 | } 17 | 18 | const fn slot_status_as_dead_error(status: &SlotStatus) -> Option<&String> { 19 | if let SlotStatus::Dead(dead) = status { 20 | Some(dead) 21 | } else { 22 | None 23 | } 24 | } 25 | 26 | #[derive(Debug)] 27 | pub struct Slot<'a> { 28 | slot: solana_sdk::clock::Slot, 29 | parent: Option, 30 | status: &'a SlotStatus, 31 | } 32 | 33 | impl<'a> Slot<'a> { 34 | pub const fn new( 35 | slot: solana_sdk::clock::Slot, 36 | parent: Option, 37 | status: &'a SlotStatus, 38 | ) -> Self { 39 | Self { 40 | slot, 41 | parent, 42 | status, 43 | } 44 | } 45 | } 46 | 47 | impl prost::Message for Slot<'_> { 48 | fn encode_raw(&self, buf: &mut impl bytes::BufMut) { 49 | let status = slot_status_as_i32(self.status); 50 | let dead_error = slot_status_as_dead_error(self.status); 51 | 52 | if self.slot != 0u64 { 53 | encoding::uint64::encode(1u32, &self.slot, buf); 54 | } 55 | if let Some(value) = &self.parent { 56 | encoding::uint64::encode(2u32, value, buf); 57 | } 58 | if status != CommitmentLevel::default() as i32 { 59 | encoding::int32::encode(3u32, &status, buf); 60 | } 61 | if let Some(value) = dead_error { 62 | encoding::string::encode(4u32, value, buf); 63 | } 64 | } 65 | 66 | fn encoded_len(&self) -> usize { 67 | let status = slot_status_as_i32(self.status); 68 | let dead_error = slot_status_as_dead_error(self.status); 69 | 70 | (if self.slot != 0u64 { 71 | encoding::uint64::encoded_len(1u32, &self.slot) 72 | } else { 73 | 0 74 | }) + self 75 | .parent 76 | .as_ref() 77 | .map_or(0, |value| encoding::uint64::encoded_len(2u32, value)) 78 | + if status != CommitmentLevel::default() as i32 { 79 | encoding::int32::encoded_len(3u32, &status) 80 | } else { 81 | 0 82 | } 83 | + dead_error 84 | .as_ref() 85 | .map_or(0, |value| encoding::string::encoded_len(4u32, value)) 86 | } 87 | 88 | fn merge_field( 89 | &mut self, 90 | _tag: u32, 91 | _wire_type: encoding::WireType, 92 | _buf: &mut impl bytes::Buf, 93 | _ctx: encoding::DecodeContext, 94 | ) -> Result<(), prost::DecodeError> 95 | where 96 | Self: Sized, 97 | { 98 | unimplemented!() 99 | } 100 | 101 | fn clear(&mut self) { 102 | unimplemented!() 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /plugin-agave/src/protobuf/message.rs: -------------------------------------------------------------------------------- 1 | use { 2 | crate::{plugin::PluginNotification, protobuf::encoding}, 3 | agave_geyser_plugin_interface::geyser_plugin_interface::{ 4 | ReplicaAccountInfoV3, ReplicaBlockInfoV4, ReplicaEntryInfoV2, ReplicaTransactionInfoV2, 5 | SlotStatus as GeyserSlotStatus, 6 | }, 7 | prost::encoding::message, 8 | prost_types::Timestamp, 9 | solana_sdk::clock::Slot, 10 | std::time::SystemTime, 11 | }; 12 | 13 | #[derive(Debug, Clone, Copy, PartialEq, Eq)] 14 | pub enum ProtobufEncoder { 15 | Prost, 16 | Raw, 17 | } 18 | 19 | #[derive(Debug)] 20 | pub enum ProtobufMessage<'a> { 21 | Account { 22 | slot: Slot, 23 | account: &'a ReplicaAccountInfoV3<'a>, 24 | }, 25 | Slot { 26 | slot: Slot, 27 | parent: Option, 28 | status: &'a GeyserSlotStatus, 29 | }, 30 | Transaction { 31 | slot: Slot, 32 | transaction: &'a ReplicaTransactionInfoV2<'a>, 33 | }, 34 | Entry { 35 | entry: &'a ReplicaEntryInfoV2<'a>, 36 | }, 37 | BlockMeta { 38 | blockinfo: &'a ReplicaBlockInfoV4<'a>, 39 | }, 40 | } 41 | 42 | impl ProtobufMessage<'_> { 43 | pub const fn get_plugin_notification(&self) -> PluginNotification { 44 | match self { 45 | Self::Account { .. } => PluginNotification::Account, 46 | Self::Slot { .. } => PluginNotification::Slot, 47 | Self::Transaction { .. } => PluginNotification::Transaction, 48 | Self::Entry { .. } => PluginNotification::Entry, 49 | Self::BlockMeta { .. } => PluginNotification::BlockMeta, 50 | } 51 | } 52 | 53 | pub const fn get_slot(&self) -> Slot { 54 | match self { 55 | Self::Account { slot, .. } => *slot, 56 | Self::Slot { slot, .. } => *slot, 57 | Self::Transaction { slot, .. } => *slot, 58 | Self::Entry { entry } => entry.slot, 59 | Self::BlockMeta { blockinfo } => blockinfo.slot, 60 | } 61 | } 62 | 63 | pub fn encode(&self, encoder: ProtobufEncoder) -> Vec { 64 | self.encode_with_timestamp(encoder, SystemTime::now()) 65 | } 66 | 67 | pub fn encode_with_timestamp( 68 | &self, 69 | encoder: ProtobufEncoder, 70 | created_at: impl Into, 71 | ) -> Vec { 72 | match encoder { 73 | ProtobufEncoder::Prost => self.encode_prost(created_at), 74 | ProtobufEncoder::Raw => self.encode_raw(created_at), 75 | } 76 | } 77 | 78 | pub fn encode_prost(&self, created_at: impl Into) -> Vec { 79 | use { 80 | prost::Message, 81 | richat_proto::{ 82 | convert_to, 83 | geyser::{ 84 | subscribe_update::UpdateOneof, SlotStatus, SubscribeUpdate, 85 | SubscribeUpdateAccount, SubscribeUpdateAccountInfo, SubscribeUpdateBlockMeta, 86 | SubscribeUpdateEntry, SubscribeUpdateSlot, SubscribeUpdateTransaction, 87 | SubscribeUpdateTransactionInfo, 88 | }, 89 | }, 90 | }; 91 | 92 | SubscribeUpdate { 93 | filters: Vec::new(), 94 | update_oneof: Some(match self { 95 | Self::Account { slot, account } => UpdateOneof::Account(SubscribeUpdateAccount { 96 | account: Some(SubscribeUpdateAccountInfo { 97 | pubkey: account.pubkey.as_ref().to_vec(), 98 | lamports: account.lamports, 99 | owner: account.owner.as_ref().to_vec(), 100 | executable: account.executable, 101 | rent_epoch: account.rent_epoch, 102 | data: account.data.to_vec(), 103 | write_version: account.write_version, 104 | txn_signature: account 105 | .txn 106 | .as_ref() 107 | .map(|transaction| transaction.signature().as_ref().to_vec()), 108 | }), 109 | slot: *slot, 110 | is_startup: false, 111 | }), 112 | Self::Slot { 113 | slot, 114 | parent, 115 | status, 116 | } => UpdateOneof::Slot(SubscribeUpdateSlot { 117 | slot: *slot, 118 | parent: *parent, 119 | status: match status { 120 | GeyserSlotStatus::Processed => SlotStatus::SlotProcessed, 121 | GeyserSlotStatus::Rooted => SlotStatus::SlotFinalized, 122 | GeyserSlotStatus::Confirmed => SlotStatus::SlotConfirmed, 123 | GeyserSlotStatus::FirstShredReceived => SlotStatus::SlotFirstShredReceived, 124 | GeyserSlotStatus::Completed => SlotStatus::SlotCompleted, 125 | GeyserSlotStatus::CreatedBank => SlotStatus::SlotCreatedBank, 126 | GeyserSlotStatus::Dead(_) => SlotStatus::SlotDead, 127 | } as i32, 128 | dead_error: if let GeyserSlotStatus::Dead(error) = status { 129 | Some(error.clone()) 130 | } else { 131 | None 132 | }, 133 | }), 134 | Self::Transaction { slot, transaction } => { 135 | UpdateOneof::Transaction(SubscribeUpdateTransaction { 136 | transaction: Some(SubscribeUpdateTransactionInfo { 137 | signature: transaction.signature.as_ref().to_vec(), 138 | is_vote: transaction.is_vote, 139 | transaction: Some(convert_to::create_transaction( 140 | transaction.transaction, 141 | )), 142 | meta: Some(convert_to::create_transaction_meta( 143 | transaction.transaction_status_meta, 144 | )), 145 | index: transaction.index as u64, 146 | }), 147 | slot: *slot, 148 | }) 149 | } 150 | Self::BlockMeta { blockinfo } => UpdateOneof::BlockMeta(SubscribeUpdateBlockMeta { 151 | slot: blockinfo.slot, 152 | blockhash: blockinfo.blockhash.to_string(), 153 | rewards: Some(convert_to::create_rewards_obj( 154 | &blockinfo.rewards.rewards, 155 | blockinfo.rewards.num_partitions, 156 | )), 157 | block_time: blockinfo.block_time.map(convert_to::create_timestamp), 158 | block_height: blockinfo.block_height.map(convert_to::create_block_height), 159 | parent_slot: blockinfo.parent_slot, 160 | parent_blockhash: blockinfo.parent_blockhash.to_string(), 161 | executed_transaction_count: blockinfo.executed_transaction_count, 162 | entries_count: blockinfo.entry_count, 163 | }), 164 | Self::Entry { entry } => UpdateOneof::Entry(SubscribeUpdateEntry { 165 | slot: entry.slot, 166 | index: entry.index as u64, 167 | num_hashes: entry.num_hashes, 168 | hash: entry.hash.as_ref().to_vec(), 169 | executed_transaction_count: entry.executed_transaction_count, 170 | starting_transaction_index: entry.starting_transaction_index as u64, 171 | }), 172 | }), 173 | created_at: Some(created_at.into()), 174 | } 175 | .encode_to_vec() 176 | } 177 | 178 | pub fn encode_raw(&self, created_at: impl Into) -> Vec { 179 | let created_at = created_at.into(); 180 | 181 | let size = match self { 182 | Self::Account { slot, account } => { 183 | let account = encoding::Account::new(*slot, account); 184 | message::encoded_len(2, &account) 185 | } 186 | Self::Slot { 187 | slot, 188 | parent, 189 | status, 190 | } => { 191 | let slot = encoding::Slot::new(*slot, *parent, status); 192 | message::encoded_len(3, &slot) 193 | } 194 | Self::Transaction { slot, transaction } => { 195 | let transaction = encoding::Transaction::new(*slot, transaction); 196 | message::encoded_len(4, &transaction) 197 | } 198 | Self::BlockMeta { blockinfo } => { 199 | let blockmeta = encoding::BlockMeta::new(blockinfo); 200 | message::encoded_len(7, &blockmeta) 201 | } 202 | Self::Entry { entry } => { 203 | let entry = encoding::Entry::new(entry); 204 | message::encoded_len(8, &entry) 205 | } 206 | } + message::encoded_len(11, &created_at); 207 | 208 | let mut vec = Vec::with_capacity(size); 209 | let buffer = &mut vec; 210 | 211 | match self { 212 | Self::Account { slot, account } => { 213 | let account = encoding::Account::new(*slot, account); 214 | message::encode(2, &account, buffer) 215 | } 216 | Self::Slot { 217 | slot, 218 | parent, 219 | status, 220 | } => { 221 | let slot = encoding::Slot::new(*slot, *parent, status); 222 | message::encode(3, &slot, buffer) 223 | } 224 | Self::Transaction { slot, transaction } => { 225 | let transaction = encoding::Transaction::new(*slot, transaction); 226 | message::encode(4, &transaction, buffer) 227 | } 228 | Self::BlockMeta { blockinfo } => { 229 | let blockmeta = encoding::BlockMeta::new(blockinfo); 230 | message::encode(7, &blockmeta, buffer) 231 | } 232 | Self::Entry { entry } => { 233 | let entry = encoding::Entry::new(entry); 234 | message::encode(8, &entry, buffer) 235 | } 236 | } 237 | message::encode(11, &created_at, buffer); 238 | 239 | vec 240 | } 241 | } 242 | -------------------------------------------------------------------------------- /plugin-agave/src/version.rs: -------------------------------------------------------------------------------- 1 | use {richat_shared::version::Version, std::env}; 2 | 3 | pub const VERSION: Version = Version { 4 | package: env!("CARGO_PKG_NAME"), 5 | version: env!("CARGO_PKG_VERSION"), 6 | proto: env!("YELLOWSTONE_GRPC_PROTO_VERSION"), 7 | proto_richat: env!("RICHAT_PROTO_VERSION"), 8 | solana: env!("SOLANA_SDK_VERSION"), 9 | git: env!("GIT_VERSION"), 10 | rustc: env!("VERGEN_RUSTC_SEMVER"), 11 | buildts: env!("VERGEN_BUILD_TIMESTAMP"), 12 | }; 13 | -------------------------------------------------------------------------------- /proto/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "richat-proto" 3 | version = "3.1.0" 4 | authors = { workspace = true } 5 | edition = { workspace = true } 6 | description = "Richat Proto" 7 | homepage = { workspace = true } 8 | repository = { workspace = true } 9 | license = { workspace = true } 10 | keywords = { workspace = true } 11 | publish = true 12 | 13 | [dependencies] 14 | prost = { workspace = true } 15 | yellowstone-grpc-proto = { workspace = true } 16 | 17 | [build-dependencies] 18 | anyhow = { workspace = true } 19 | cargo-lock = { workspace = true } 20 | protobuf-src = { workspace = true } 21 | tonic-build = { workspace = true } 22 | 23 | [features] 24 | default = [] 25 | yellowstone-grpc-plugin = ["yellowstone-grpc-proto/plugin"] 26 | 27 | [lints] 28 | workspace = true 29 | -------------------------------------------------------------------------------- /proto/build.rs: -------------------------------------------------------------------------------- 1 | fn main() -> anyhow::Result<()> { 2 | // build protos 3 | std::env::set_var("PROTOC", protobuf_src::protoc()); 4 | generate_transport() 5 | } 6 | 7 | fn generate_transport() -> anyhow::Result<()> { 8 | tonic_build::configure() 9 | .build_client(false) 10 | .build_server(false) 11 | .compile_protos(&["proto/richat.proto"], &["proto"])?; 12 | 13 | Ok(()) 14 | } 15 | -------------------------------------------------------------------------------- /proto/proto/richat.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package richat; 4 | 5 | message RichatFilter { 6 | bool disable_accounts = 1; 7 | bool disable_transactions = 2; 8 | bool disable_entries = 3; 9 | } 10 | 11 | message GrpcSubscribeRequest { 12 | optional uint64 replay_from_slot = 11; // Same tag as in Yellowstone gRPC SubscribeRequest 13 | RichatFilter filter = 100; 14 | } 15 | 16 | message QuicSubscribeRequest { 17 | optional bytes x_token = 1; 18 | uint32 recv_streams = 2; 19 | optional uint32 max_backlog = 3; 20 | optional uint64 replay_from_slot = 4; 21 | RichatFilter filter = 5; 22 | } 23 | 24 | message QuicSubscribeResponse { 25 | optional QuicSubscribeResponseError error = 1; 26 | optional uint32 max_recv_streams = 2; 27 | optional uint64 first_available_slot = 3; 28 | string version = 4; 29 | } 30 | 31 | enum QuicSubscribeResponseError { 32 | ZERO_RECV_STREAMS = 0; 33 | EXCEED_RECV_STREAMS = 1; 34 | NOT_INITIALIZED = 2; 35 | SLOT_NOT_AVAILABLE = 3; 36 | REQUEST_SIZE_TOO_LARGE = 4; 37 | X_TOKEN_REQUIRED = 5; 38 | X_TOKEN_INVALID = 6; 39 | } 40 | 41 | message QuicSubscribeClose { 42 | QuicSubscribeCloseError error = 1; 43 | } 44 | 45 | enum QuicSubscribeCloseError { 46 | LAGGED = 0; 47 | CLOSED = 1; 48 | } 49 | -------------------------------------------------------------------------------- /proto/src/lib.rs: -------------------------------------------------------------------------------- 1 | #[cfg(feature = "yellowstone-grpc-plugin")] 2 | pub use yellowstone_grpc_proto::plugin; 3 | pub use yellowstone_grpc_proto::{convert_from, convert_to, geyser, solana}; 4 | 5 | pub mod richat { 6 | #![allow(clippy::missing_const_for_fn)] 7 | include!(concat!(env!("OUT_DIR"), "/richat.rs")); 8 | } 9 | -------------------------------------------------------------------------------- /richat/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "richat" 3 | version = "3.6.0" 4 | authors = { workspace = true } 5 | edition = { workspace = true } 6 | description = "Richat App" 7 | homepage = { workspace = true } 8 | repository = { workspace = true } 9 | license = { workspace = true } 10 | keywords = { workspace = true } 11 | publish = { workspace = true } 12 | 13 | [dependencies] 14 | affinity-linux = { workspace = true } 15 | anyhow = { workspace = true } 16 | arrayvec = { workspace = true } 17 | clap = { workspace = true, features = ["derive"] } 18 | fastwebsockets = { workspace = true, features = ["upgrade", "unstable-split"] } 19 | foldhash = { workspace = true } 20 | futures = { workspace = true } 21 | http-body-util = { workspace = true } 22 | humantime-serde = { workspace = true } 23 | hyper = { workspace = true } 24 | hyper-util = { workspace = true } 25 | json5 = { workspace = true } 26 | jsonrpsee-types = { workspace = true } 27 | maplit = { workspace = true } 28 | metrics = { workspace = true } 29 | metrics-exporter-prometheus = { workspace = true } 30 | prost = { workspace = true } 31 | prost-types = { workspace = true } 32 | quanta = { workspace = true } 33 | rayon = { workspace = true } 34 | richat-client = { workspace = true } 35 | richat-filter = { workspace = true } 36 | richat-proto = { workspace = true } 37 | richat-shared = { workspace = true, features = ["jsonrpc", "metrics"] } 38 | rustls = { workspace = true, features = ["aws_lc_rs"] } 39 | serde = { workspace = true, features = ["derive"] } 40 | serde_json = { workspace = true, features = ["raw_value"] } 41 | serde_yaml = { workspace = true } 42 | signal-hook = { workspace = true } 43 | smallvec = { workspace = true } 44 | solana-account = { workspace = true } 45 | solana-account-decoder = { workspace = true } 46 | solana-rpc-client-api = { workspace = true } 47 | solana-sdk = { workspace = true } 48 | solana-transaction-status = { workspace = true } 49 | solana-version = { workspace = true } 50 | spl-token-2022 = { workspace = true } 51 | thiserror = { workspace = true } 52 | tikv-jemallocator = { workspace = true } 53 | tokio = { workspace = true } 54 | tokio-rustls = { workspace = true } 55 | tonic = { workspace = true } 56 | tracing = { workspace = true } 57 | tracing-subscriber = { workspace = true, features = ["ansi", "env-filter", "json"] } 58 | 59 | [dev-dependencies] 60 | const-hex = { workspace = true } 61 | 62 | [build-dependencies] 63 | anyhow = { workspace = true } 64 | cargo-lock = { workspace = true } 65 | git-version = { workspace = true } 66 | tonic-build = { workspace = true } 67 | vergen = { workspace = true, features = ["build", "rustc"] } 68 | 69 | [lints] 70 | workspace = true 71 | -------------------------------------------------------------------------------- /richat/build.rs: -------------------------------------------------------------------------------- 1 | use { 2 | cargo_lock::Lockfile, 3 | std::collections::HashSet, 4 | tonic_build::manual::{Builder, Method, Service}, 5 | }; 6 | 7 | fn main() -> anyhow::Result<()> { 8 | emit_version()?; 9 | generate_grpc_geyser() 10 | } 11 | 12 | fn emit_version() -> anyhow::Result<()> { 13 | vergen::Emitter::default() 14 | .add_instructions(&vergen::BuildBuilder::all_build()?)? 15 | .add_instructions(&vergen::RustcBuilder::all_rustc()?)? 16 | .emit()?; 17 | 18 | // vergen git version does not looks cool 19 | println!( 20 | "cargo:rustc-env=GIT_VERSION={}", 21 | git_version::git_version!() 22 | ); 23 | 24 | // Extract packages version 25 | let lockfile = Lockfile::load("../Cargo.lock")?; 26 | println!( 27 | "cargo:rustc-env=SOLANA_SDK_VERSION={}", 28 | get_pkg_version(&lockfile, "solana-sdk") 29 | ); 30 | println!( 31 | "cargo:rustc-env=YELLOWSTONE_GRPC_PROTO_VERSION={}", 32 | get_pkg_version(&lockfile, "yellowstone-grpc-proto") 33 | ); 34 | println!( 35 | "cargo:rustc-env=RICHAT_PROTO_VERSION={}", 36 | get_pkg_version(&lockfile, "richat-proto") 37 | ); 38 | 39 | Ok(()) 40 | } 41 | 42 | fn get_pkg_version(lockfile: &Lockfile, pkg_name: &str) -> String { 43 | lockfile 44 | .packages 45 | .iter() 46 | .filter(|pkg| pkg.name.as_str() == pkg_name) 47 | .map(|pkg| pkg.version.to_string()) 48 | .collect::>() 49 | .into_iter() 50 | .collect::>() 51 | .join(",") 52 | } 53 | 54 | fn generate_grpc_geyser() -> anyhow::Result<()> { 55 | let geyser_service = Service::builder() 56 | .name("Geyser") 57 | .package("geyser") 58 | .method( 59 | Method::builder() 60 | .name("subscribe") 61 | .route_name("Subscribe") 62 | .input_type("richat_proto::geyser::SubscribeRequest") 63 | .output_type("Vec") 64 | .codec_path("richat_shared::transports::grpc::SubscribeCodec") 65 | .client_streaming() 66 | .server_streaming() 67 | .build(), 68 | ) 69 | .method( 70 | Method::builder() 71 | .name("subscribe_replay_info") 72 | .route_name("SubscribeReplayInfo") 73 | .input_type("richat_proto::geyser::SubscribeReplayInfoRequest") 74 | .output_type("richat_proto::geyser::SubscribeReplayInfoResponse") 75 | .codec_path("tonic::codec::ProstCodec") 76 | .build(), 77 | ) 78 | .method( 79 | Method::builder() 80 | .name("ping") 81 | .route_name("Ping") 82 | .input_type("richat_proto::geyser::PingRequest") 83 | .output_type("richat_proto::geyser::PongResponse") 84 | .codec_path("tonic::codec::ProstCodec") 85 | .build(), 86 | ) 87 | .method( 88 | Method::builder() 89 | .name("get_latest_blockhash") 90 | .route_name("GetLatestBlockhash") 91 | .input_type("richat_proto::geyser::GetLatestBlockhashRequest") 92 | .output_type("richat_proto::geyser::GetLatestBlockhashResponse") 93 | .codec_path("tonic::codec::ProstCodec") 94 | .build(), 95 | ) 96 | .method( 97 | Method::builder() 98 | .name("get_block_height") 99 | .route_name("GetBlockHeight") 100 | .input_type("richat_proto::geyser::GetBlockHeightRequest") 101 | .output_type("richat_proto::geyser::GetBlockHeightResponse") 102 | .codec_path("tonic::codec::ProstCodec") 103 | .build(), 104 | ) 105 | .method( 106 | Method::builder() 107 | .name("get_slot") 108 | .route_name("GetSlot") 109 | .input_type("richat_proto::geyser::GetSlotRequest") 110 | .output_type("richat_proto::geyser::GetSlotResponse") 111 | .codec_path("tonic::codec::ProstCodec") 112 | .build(), 113 | ) 114 | .method( 115 | Method::builder() 116 | .name("is_blockhash_valid") 117 | .route_name("IsBlockhashValid") 118 | .input_type("richat_proto::geyser::IsBlockhashValidRequest") 119 | .output_type("richat_proto::geyser::IsBlockhashValidResponse") 120 | .codec_path("tonic::codec::ProstCodec") 121 | .build(), 122 | ) 123 | .method( 124 | Method::builder() 125 | .name("get_version") 126 | .route_name("GetVersion") 127 | .input_type("richat_proto::geyser::GetVersionRequest") 128 | .output_type("richat_proto::geyser::GetVersionResponse") 129 | .codec_path("tonic::codec::ProstCodec") 130 | .build(), 131 | ) 132 | .build(); 133 | 134 | Builder::new() 135 | .build_client(false) 136 | .compile(&[geyser_service]); 137 | 138 | Ok(()) 139 | } 140 | -------------------------------------------------------------------------------- /richat/config.yml: -------------------------------------------------------------------------------- 1 | --- 2 | logs: 3 | json: false 4 | metrics: 5 | endpoint: 127.0.0.1:10124 6 | channel: 7 | tokio: 8 | worker_threads: 2 9 | affinity: 0-1 10 | sources: 11 | - name: plugin-grpc 12 | parser: prost # valid: prost, limited 13 | disable_accounts: false 14 | reconnect: null 15 | source: richat # valid: richat, dragons_mouth 16 | transport: grpc 17 | endpoint: http://127.0.0.1:10100 18 | ca_certificate: null 19 | connect_timeout: null 20 | buffer_size: null 21 | http2_adaptive_window: null 22 | http2_keep_alive_interval: null 23 | initial_connection_window_size: null 24 | initial_stream_window_size: null 25 | keep_alive_timeout: null 26 | keep_alive_while_idle: false 27 | tcp_keepalive: 15s 28 | tcp_nodelay: true 29 | timeout: null 30 | max_decoding_message_size: 16_777_216 # 16MiB 31 | compression: 32 | accept: ["gzip", "zstd"] 33 | send: ["gzip", "zstd"] 34 | x_token: null 35 | # - name: plugin-quic 36 | # parser: prost 37 | # disable_accounts: false 38 | # reconnect: 39 | # initial_interval: 1s 40 | # max_interval: 30s 41 | # multiplier: 2 42 | # transport: quic 43 | # source: richat # valid: richat, dragons_mouth 44 | # endpoint: 127.0.0.1:10101 45 | # local_addr: "[::]:0" 46 | # expected_rtt: 100 47 | # max_stream_bandwidth: 12_500_000 # 100Mbits with 100ms latency 48 | # max_idle_timeout: 30ms 49 | # server_name: null # localhost 50 | # recv_streams: 1 51 | # max_backlog: null 52 | # insecure: false 53 | # cert: null 54 | # x_token: null 55 | config: 56 | max_messages: 2_097_152 57 | max_bytes: 16_106_127_360 58 | apps: 59 | tokio: 60 | worker_threads: 2 61 | affinity: 2-3 62 | richat: 63 | grpc: 64 | endpoint: '127.0.0.1:10100' 65 | tls_config: 66 | cert: /path/to/cert.cert 67 | key: /path/to/key.key 68 | compression: 69 | accept: 70 | - gzip 71 | - zstd 72 | send: 73 | - gzip 74 | - zstd 75 | max_decoding_message_size: '4_194_304' 76 | server_tcp_keepalive: 20s 77 | server_tcp_nodelay: true 78 | server_http2_adaptive_window: null 79 | server_http2_keepalive_interval: null 80 | server_http2_keepalive_timeout: null 81 | server_initial_connection_window_size: null 82 | server_initial_stream_window_size: null 83 | quic: 84 | endpoint: '127.0.0.1:10101' 85 | tls_config: 86 | # cert: /path/to/cert.cert 87 | # key: /path/to/key.key 88 | self_signed_alt_names: 89 | - localhost 90 | expected_rtt: 100 91 | max_stream_bandwidth: 12500000 92 | max_recv_streams: 16 93 | x_tokens: [] 94 | grpc: 95 | server: 96 | endpoint: 127.0.0.1:10000 97 | tls_config: 98 | cert: /path/to/cert.cert 99 | key: /path/to/key.key 100 | compression: 101 | accept: 102 | - gzip 103 | - zstd 104 | send: 105 | - gzip 106 | - zstd 107 | max_decoding_message_size: 4_194_304 # 4MiB 108 | server_tcp_keepalive: 20s 109 | server_tcp_nodelay: true 110 | server_http2_adaptive_window: 111 | server_http2_keepalive_interval: 112 | server_http2_keepalive_timeout: 113 | server_initial_connection_window_size: 114 | server_initial_stream_window_size: 115 | workers: 116 | threads: 2 117 | affinity: 4-5 118 | messages_cached_max: 1_024 119 | stream: 120 | messages_len_max: 16_777_216 121 | messages_max_per_tick: 100 122 | ping_iterval: 15s 123 | unary: 124 | enabled: true 125 | affinity: 4-5 126 | requests_queue_size: 100 127 | filters: 128 | name_max: 128 129 | accounts: 130 | max: 1 131 | any: false 132 | account_max: 10 133 | account_reject: 134 | - TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA 135 | owner_max: 10 136 | owner_reject: 137 | - '11111111111111111111111111111111' 138 | data_slice_max: 2 139 | slots: 140 | max: 1 141 | transactions: 142 | max: 1 143 | any: false 144 | account_include_max: 10 145 | account_include_reject: 146 | - TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA 147 | account_exclude_max: 10 148 | account_required_max: 10 149 | transactions_status: 150 | max: 1 151 | any: false 152 | account_include_max: 10 153 | account_include_reject: 154 | - TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA 155 | account_exclude_max: 10 156 | account_required_max: 10 157 | entries: 158 | max: 1 159 | blocks_meta: 160 | max: 1 161 | blocks: 162 | max: 1 163 | account_include_max: 10 164 | account_include_any: false 165 | account_include_reject: 166 | - TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA 167 | include_transactions: true 168 | include_accounts: false 169 | include_entries: false 170 | x_token: [] 171 | pubsub: 172 | endpoint: 0.0.0.0:8000 173 | tcp_nodelay: true 174 | tls_config: 175 | # cert: /path/to/cert.cert 176 | # key: /path/to/key.key 177 | self_signed_alt_names: 178 | - localhost 179 | recv_max_message_size: 4_096 180 | enable_block_subscription: false 181 | enable_transaction_subscription: false 182 | clients_requests_channel_size: 8_192 183 | subscriptions_worker_affinity: 0 184 | subscriptions_workers_count: 2 185 | subscriptions_workers_affinity: 0-1 186 | subscriptions_max_clients_request_per_tick: 32 187 | subscriptions_max_messages_per_commitment_per_tick: 256 188 | notifications_messages_max_count: 10_000_000 189 | notifications_messages_max_bytes: 34_359_738_368 190 | signatures_cache_max: 1_228_800 191 | signatures_cache_slots_max: 150 192 | -------------------------------------------------------------------------------- /richat/src/bin/richat.rs: -------------------------------------------------------------------------------- 1 | use { 2 | anyhow::Context, 3 | clap::Parser, 4 | futures::{ 5 | future::{ready, try_join_all, FutureExt, TryFutureExt}, 6 | stream::StreamExt, 7 | }, 8 | richat::{ 9 | channel::Messages, config::Config, grpc::server::GrpcServer, pubsub::server::PubSubServer, 10 | richat::server::RichatServer, source::Subscriptions, 11 | }, 12 | richat_shared::shutdown::Shutdown, 13 | signal_hook::{consts::SIGINT, iterator::Signals}, 14 | std::{ 15 | thread::{self, sleep}, 16 | time::Duration, 17 | }, 18 | tracing::{info, warn}, 19 | }; 20 | 21 | #[cfg(not(target_env = "msvc"))] 22 | #[global_allocator] 23 | static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; 24 | 25 | #[derive(Debug, Parser)] 26 | #[clap(author, version, about = "Richat App")] 27 | struct Args { 28 | /// Path to config 29 | #[clap(short, long, default_value_t = String::from("config.json"))] 30 | pub config: String, 31 | 32 | /// Only check config and exit 33 | #[clap(long, default_value_t = false)] 34 | pub check: bool, 35 | } 36 | 37 | fn main() -> anyhow::Result<()> { 38 | anyhow::ensure!( 39 | rustls::crypto::aws_lc_rs::default_provider() 40 | .install_default() 41 | .is_ok(), 42 | "failed to call CryptoProvider::install_default()" 43 | ); 44 | 45 | let args = Args::parse(); 46 | let config = Config::load_from_file(&args.config) 47 | .with_context(|| format!("failed to load config from {}", args.config))?; 48 | if args.check { 49 | info!("Config is OK!"); 50 | return Ok(()); 51 | } 52 | 53 | let metrics_handle = if config.metrics.is_some() { 54 | Some(richat::metrics::setup().context("failed to setup metrics")?) 55 | } else { 56 | None 57 | }; 58 | 59 | // Setup logs 60 | richat::log::setup(config.logs.json)?; 61 | 62 | // Shutdown channel/flag 63 | let shutdown = Shutdown::new(); 64 | 65 | // Create channel runtime (receive messages from solana node / richat) 66 | let messages = Messages::new( 67 | config.channel.config, 68 | config.apps.richat.is_some(), 69 | config.apps.grpc.is_some(), 70 | config.apps.pubsub.is_some(), 71 | ); 72 | let source_jh = thread::Builder::new() 73 | .name("richatSource".to_owned()) 74 | .spawn({ 75 | let shutdown = shutdown.clone(); 76 | let mut messages = messages.to_sender(); 77 | || { 78 | let runtime = config.channel.tokio.build_runtime("richatSource")?; 79 | runtime.block_on(async move { 80 | let streams_total = config.channel.sources.len(); 81 | let mut stream = Subscriptions::new(config.channel.sources).await?; 82 | tokio::pin!(shutdown); 83 | loop { 84 | let (index, message) = tokio::select! { 85 | biased; 86 | message = stream.next() => match message { 87 | Some(Ok(value)) => value, 88 | Some(Err(error)) => return Err(anyhow::Error::new(error)), 89 | None => anyhow::bail!("source stream finished"), 90 | }, 91 | () = &mut shutdown => return Ok(()), 92 | }; 93 | 94 | let index_info = if streams_total == 1 { 95 | None 96 | } else { 97 | Some((index, streams_total)) 98 | }; 99 | messages.push(message, index_info); 100 | } 101 | }) 102 | } 103 | })?; 104 | 105 | // Create runtime for incoming connections 106 | let apps_jh = thread::Builder::new().name("richatApp".to_owned()).spawn({ 107 | let shutdown = shutdown.clone(); 108 | move || { 109 | let runtime = config.apps.tokio.build_runtime("richatApp")?; 110 | runtime.block_on(async move { 111 | let richat_fut = if let Some(config) = config.apps.richat { 112 | RichatServer::spawn(config, messages.clone(), shutdown.clone()) 113 | .await? 114 | .boxed() 115 | } else { 116 | ready(Ok(())).boxed() 117 | }; 118 | 119 | let grpc_fut = if let Some(config) = config.apps.grpc { 120 | GrpcServer::spawn(config, messages.clone(), shutdown.clone())?.boxed() 121 | } else { 122 | ready(Ok(())).boxed() 123 | }; 124 | 125 | let pubsub_fut = if let Some(config) = config.apps.pubsub { 126 | PubSubServer::spawn(config, messages, shutdown.clone())?.boxed() 127 | } else { 128 | ready(Ok(())).boxed() 129 | }; 130 | 131 | let metrics_fut = if let (Some(config), Some(metrics_handle)) = 132 | (config.metrics, metrics_handle) 133 | { 134 | richat::metrics::spawn_server(config, metrics_handle, shutdown) 135 | .await? 136 | .map_err(anyhow::Error::from) 137 | .boxed() 138 | } else { 139 | ready(Ok(())).boxed() 140 | }; 141 | 142 | try_join_all(vec![richat_fut, grpc_fut, pubsub_fut, metrics_fut]) 143 | .await 144 | .map(|_| ()) 145 | }) 146 | } 147 | })?; 148 | 149 | let mut signals = Signals::new([SIGINT])?; 150 | let mut threads = [("source", Some(source_jh)), ("apps", Some(apps_jh))]; 151 | 'outer: while threads.iter().any(|th| th.1.is_some()) { 152 | for signal in signals.pending() { 153 | match signal { 154 | SIGINT => { 155 | if shutdown.is_set() { 156 | warn!("SIGINT received again, shutdown now"); 157 | break 'outer; 158 | } 159 | info!("SIGINT received..."); 160 | shutdown.shutdown(); 161 | } 162 | _ => unreachable!(), 163 | } 164 | } 165 | 166 | for (name, tjh) in threads.iter_mut() { 167 | if let Some(jh) = tjh.take() { 168 | if jh.is_finished() { 169 | jh.join() 170 | .unwrap_or_else(|_| panic!("{name} thread join failed"))?; 171 | info!("thread {name} finished"); 172 | } else { 173 | *tjh = Some(jh); 174 | } 175 | } 176 | } 177 | 178 | sleep(Duration::from_millis(25)); 179 | } 180 | 181 | Ok(()) 182 | } 183 | -------------------------------------------------------------------------------- /richat/src/config.rs: -------------------------------------------------------------------------------- 1 | use { 2 | crate::{ 3 | grpc::config::ConfigAppsGrpc, pubsub::config::ConfigAppsPubsub, 4 | richat::config::ConfigAppsRichat, 5 | }, 6 | futures::future::{ready, try_join_all, TryFutureExt}, 7 | richat_client::{grpc::ConfigGrpcClient, quic::ConfigQuicClient}, 8 | richat_filter::message::MessageParserEncoding, 9 | richat_shared::{ 10 | config::{deserialize_affinity, deserialize_num_str, ConfigMetrics, ConfigTokio}, 11 | shutdown::Shutdown, 12 | }, 13 | serde::Deserialize, 14 | std::{fs, path::Path, thread::Builder}, 15 | tokio::time::{sleep, Duration}, 16 | }; 17 | 18 | #[derive(Debug, Clone, Deserialize)] 19 | #[serde(deny_unknown_fields)] 20 | pub struct Config { 21 | #[serde(default)] 22 | pub logs: ConfigLogs, 23 | #[serde(default)] 24 | pub metrics: Option, 25 | pub channel: ConfigChannel, 26 | #[serde(default)] 27 | pub apps: ConfigApps, 28 | } 29 | 30 | impl Config { 31 | pub fn load_from_file>(file: P) -> anyhow::Result { 32 | let config = fs::read_to_string(&file)?; 33 | if matches!( 34 | file.as_ref().extension().and_then(|e| e.to_str()), 35 | Some("yml") | Some("yaml") 36 | ) { 37 | serde_yaml::from_str(&config).map_err(Into::into) 38 | } else { 39 | json5::from_str(&config).map_err(Into::into) 40 | } 41 | } 42 | } 43 | 44 | #[derive(Debug, Clone, Default, Deserialize)] 45 | #[serde(deny_unknown_fields, default)] 46 | pub struct ConfigLogs { 47 | pub json: bool, 48 | } 49 | 50 | #[derive(Debug, Clone, Deserialize)] 51 | #[serde(deny_unknown_fields)] 52 | pub struct ConfigChannel { 53 | /// Runtime for receiving plugin messages 54 | #[serde(default)] 55 | pub tokio: ConfigTokio, 56 | pub sources: Vec, 57 | #[serde(default)] 58 | pub config: ConfigChannelInner, 59 | } 60 | 61 | #[derive(Debug, Clone, Deserialize)] 62 | #[serde(deny_unknown_fields, tag = "transport")] 63 | pub enum ConfigChannelSource { 64 | #[serde(rename = "quic")] 65 | Quic { 66 | #[serde(flatten)] 67 | general: ConfigChannelSourceGeneral, 68 | #[serde(flatten)] 69 | config: ConfigQuicClient, 70 | }, 71 | #[serde(rename = "grpc")] 72 | Grpc { 73 | #[serde(flatten)] 74 | general: ConfigChannelSourceGeneral, 75 | source: ConfigGrpcClientSource, 76 | #[serde(flatten)] 77 | config: ConfigGrpcClient, 78 | }, 79 | } 80 | 81 | #[derive(Debug, Clone, Deserialize)] 82 | pub struct ConfigChannelSourceGeneral { 83 | pub name: String, 84 | /// Messages parser: `prost` or `limited` 85 | pub parser: MessageParserEncoding, 86 | #[serde(default)] 87 | pub disable_accounts: bool, 88 | #[serde(default)] 89 | pub reconnect: Option, 90 | } 91 | 92 | #[derive(Debug, Clone, Deserialize)] 93 | pub struct ConfigChannelSourceReconnect { 94 | #[serde( 95 | with = "humantime_serde", 96 | default = "ConfigChannelSourceReconnect::default_initial_interval" 97 | )] 98 | pub initial_interval: Duration, 99 | #[serde( 100 | with = "humantime_serde", 101 | default = "ConfigChannelSourceReconnect::default_max_interval" 102 | )] 103 | pub max_interval: Duration, 104 | #[serde(default = "ConfigChannelSourceReconnect::default_multiplier")] 105 | pub multiplier: f64, 106 | } 107 | 108 | impl ConfigChannelSourceReconnect { 109 | const fn default_initial_interval() -> Duration { 110 | Duration::from_secs(1) 111 | } 112 | 113 | const fn default_max_interval() -> Duration { 114 | Duration::from_secs(15) 115 | } 116 | 117 | const fn default_multiplier() -> f64 { 118 | 2.0 119 | } 120 | } 121 | 122 | #[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Deserialize)] 123 | #[serde(deny_unknown_fields, rename_all = "snake_case")] 124 | pub enum ConfigGrpcClientSource { 125 | DragonsMouth, 126 | #[default] 127 | Richat, 128 | } 129 | 130 | #[derive(Debug, Clone, Deserialize)] 131 | #[serde(deny_unknown_fields, default)] 132 | pub struct ConfigChannelInner { 133 | #[serde(deserialize_with = "deserialize_num_str")] 134 | pub max_messages: usize, 135 | #[serde(deserialize_with = "deserialize_num_str")] 136 | pub max_bytes: usize, 137 | } 138 | 139 | impl Default for ConfigChannelInner { 140 | fn default() -> Self { 141 | Self { 142 | max_messages: 2_097_152, // aligned to power of 2, ~20k/slot should give us ~100 slots 143 | max_bytes: 15 * 1024 * 1024 * 1024, // 15GiB with ~150MiB/slot should give us ~100 slots 144 | } 145 | } 146 | } 147 | 148 | #[derive(Debug, Clone, Default, Deserialize)] 149 | #[serde(deny_unknown_fields, default)] 150 | pub struct ConfigApps { 151 | /// Runtime for incoming connections 152 | pub tokio: ConfigTokio, 153 | /// downstream richat 154 | pub richat: Option, 155 | /// gRPC app (fully compatible with Yellowstone Dragon's Mouth) 156 | pub grpc: Option, 157 | /// WebSocket app (fully compatible with Solana PubSub) 158 | pub pubsub: Option, 159 | } 160 | 161 | #[derive(Debug, Clone, Deserialize)] 162 | #[serde(deny_unknown_fields, default)] 163 | pub struct ConfigAppsWorkers { 164 | /// Number of worker threads 165 | pub threads: usize, 166 | /// Threads affinity 167 | #[serde(deserialize_with = "deserialize_affinity")] 168 | pub affinity: Option>, 169 | } 170 | 171 | impl Default for ConfigAppsWorkers { 172 | fn default() -> Self { 173 | Self { 174 | threads: 1, 175 | affinity: None, 176 | } 177 | } 178 | } 179 | 180 | impl ConfigAppsWorkers { 181 | pub async fn run( 182 | self, 183 | get_name: impl Fn(usize) -> String, 184 | spawn_fn: impl FnOnce(usize) -> anyhow::Result<()> + Clone + Send + 'static, 185 | shutdown: Shutdown, 186 | ) -> anyhow::Result<()> { 187 | anyhow::ensure!(self.threads > 0, "number of threads can be zero"); 188 | 189 | let mut jhs = Vec::with_capacity(self.threads); 190 | for index in 0..self.threads { 191 | let cpus = self.affinity.as_ref().map(|affinity| { 192 | if self.threads == affinity.len() { 193 | vec![affinity[index]] 194 | } else { 195 | affinity.clone() 196 | } 197 | }); 198 | 199 | jhs.push(Self::run_once( 200 | index, 201 | get_name(index), 202 | cpus, 203 | spawn_fn.clone(), 204 | shutdown.clone(), 205 | )?); 206 | } 207 | 208 | try_join_all(jhs).await.map(|_| ()) 209 | } 210 | 211 | pub fn run_once( 212 | index: usize, 213 | name: String, 214 | cpus: Option>, 215 | spawn_fn: impl FnOnce(usize) -> anyhow::Result<()> + Send + 'static, 216 | shutdown: Shutdown, 217 | ) -> anyhow::Result>> { 218 | let th = Builder::new().name(name).spawn(move || { 219 | if let Some(cpus) = cpus { 220 | affinity_linux::set_thread_affinity(cpus.into_iter()) 221 | .expect("failed to set affinity"); 222 | } 223 | spawn_fn(index) 224 | })?; 225 | 226 | let jh = tokio::spawn(async move { 227 | while !th.is_finished() { 228 | let ms = if shutdown.is_set() { 10 } else { 2_000 }; 229 | sleep(Duration::from_millis(ms)).await; 230 | } 231 | th.join().expect("failed to join thread") 232 | }); 233 | 234 | Ok(jh.map_err(anyhow::Error::new).and_then(ready)) 235 | } 236 | } 237 | -------------------------------------------------------------------------------- /richat/src/grpc/block_meta.rs: -------------------------------------------------------------------------------- 1 | use { 2 | crate::{channel::ParsedMessage, metrics}, 3 | ::metrics::gauge, 4 | foldhash::quality::RandomState, 5 | futures::future::TryFutureExt, 6 | richat_proto::geyser::{CommitmentLevel as CommitmentLevelProto, SlotStatus}, 7 | solana_sdk::clock::{Slot, MAX_PROCESSING_AGE}, 8 | std::{collections::HashMap, future::Future, sync::Arc}, 9 | tokio::sync::{mpsc, oneshot}, 10 | tonic::Status, 11 | }; 12 | 13 | #[derive(Debug, Default, Clone)] 14 | pub struct BlockMeta { 15 | pub slot: Slot, 16 | pub blockhash: Arc, 17 | pub block_height: Slot, 18 | 19 | processed: bool, // flag, means that we received block meta message 20 | confirmed: bool, 21 | finalized: bool, 22 | } 23 | 24 | #[derive(Debug, Default)] 25 | struct BlockStatus { 26 | last_valid_block_height: Slot, 27 | 28 | processed: bool, // flag, means that we received block meta message 29 | confirmed: bool, 30 | finalized: bool, 31 | } 32 | 33 | #[derive(Debug, Clone)] 34 | pub struct BlockMetaStorage { 35 | messages_tx: mpsc::UnboundedSender, 36 | requests_tx: mpsc::Sender, 37 | } 38 | 39 | impl BlockMetaStorage { 40 | pub fn new(request_queue_size: usize) -> (Self, impl Future>) { 41 | let (messages_tx, messages_rx) = mpsc::unbounded_channel(); 42 | let (requests_tx, requests_rx) = mpsc::channel(request_queue_size); 43 | 44 | let me = Self { 45 | messages_tx, 46 | requests_tx, 47 | }; 48 | let fut = tokio::spawn(Self::work(messages_rx, requests_rx)).map_err(anyhow::Error::new); 49 | 50 | (me, fut) 51 | } 52 | 53 | async fn work( 54 | mut messages_rx: mpsc::UnboundedReceiver, 55 | mut requests_rx: mpsc::Receiver, 56 | ) { 57 | let mut blocks = HashMap::::default(); 58 | let mut blockhashes = HashMap::, BlockStatus, RandomState>::default(); 59 | let mut processed = 0; 60 | let mut confirmed = 0; 61 | let mut finalized = 0; 62 | 63 | loop { 64 | tokio::select! { 65 | biased; 66 | message = messages_rx.recv() => match message { 67 | Some(ParsedMessage::Slot(msg)) => { 68 | let slot = msg.slot(); 69 | let status = msg.status(); 70 | if status == SlotStatus::SlotConfirmed { 71 | let entry = blocks.entry(slot).or_default(); 72 | entry.confirmed = true; 73 | blockhashes.entry(Arc::clone(&entry.blockhash)).or_default().confirmed = true; 74 | confirmed = slot; 75 | gauge!(metrics::GRPC_BLOCK_META_SLOT, "commitment" => "confirmed").set(slot as f64); 76 | } else if status == SlotStatus::SlotFinalized { 77 | let entry = blocks.entry(slot).or_default(); 78 | entry.finalized = true; 79 | blockhashes.entry(Arc::clone(&entry.blockhash)).or_default().finalized = true; 80 | finalized = slot; 81 | gauge!(metrics::GRPC_BLOCK_META_SLOT, "commitment" => "finalized").set(slot as f64); 82 | 83 | // cleanup 84 | blockhashes.retain(|_blockhash, bentry| bentry.last_valid_block_height < entry.block_height); 85 | blocks.retain(|bslot, _block| *bslot >= slot); 86 | } 87 | } 88 | Some(ParsedMessage::BlockMeta(msg)) => { 89 | let slot = msg.slot(); 90 | let entry = blocks.entry(slot).or_default(); 91 | entry.slot = slot; 92 | entry.blockhash = Arc::new(msg.blockhash().to_owned()); 93 | entry.block_height = msg.block_height(); 94 | entry.processed = true; 95 | let bentry = blockhashes.entry(Arc::clone(&entry.blockhash)).or_default(); 96 | bentry.last_valid_block_height = entry.block_height + MAX_PROCESSING_AGE as u64; 97 | bentry.processed = true; 98 | processed = processed.max(slot); 99 | gauge!(metrics::GRPC_BLOCK_META_SLOT, "commitment" => "processed").set(slot as f64); 100 | } 101 | Some(_) => {} 102 | None => break, 103 | }, 104 | request = requests_rx.recv() => { 105 | gauge!(metrics::GRPC_BLOCK_META_QUEUE_SIZE).decrement(1); 106 | match request { 107 | Some(Request::GetBlock(tx, commitment)) => { 108 | let slot = match commitment { 109 | CommitmentLevelProto::Processed => processed, 110 | CommitmentLevelProto::Confirmed => confirmed, 111 | CommitmentLevelProto::Finalized => finalized, 112 | }; 113 | let block = blocks.get(&slot).cloned(); 114 | let _ = tx.send(block); 115 | } 116 | Some(Request::IsBlockhashValid(tx, blockhash, commitment)) => { 117 | let slot = match commitment { 118 | CommitmentLevelProto::Processed => processed, 119 | CommitmentLevelProto::Confirmed => confirmed, 120 | CommitmentLevelProto::Finalized => finalized, 121 | }; 122 | let block = blocks.get(&slot).cloned(); 123 | let value = if let (Some(block), Some(entry)) = (block, blockhashes.get(&blockhash)) { 124 | let valid = block.block_height < entry.last_valid_block_height; 125 | Some((valid, block.slot)) 126 | } else { 127 | None 128 | }; 129 | let _ = tx.send(value); 130 | } 131 | None => break, 132 | } 133 | } 134 | }; 135 | } 136 | } 137 | 138 | pub fn push(&self, message: ParsedMessage) { 139 | let _ = self.messages_tx.send(message); 140 | } 141 | 142 | async fn send_request( 143 | &self, 144 | request: Request, 145 | rx: oneshot::Receiver>, 146 | ) -> tonic::Result { 147 | if self.requests_tx.try_send(request).is_err() { 148 | return Err(tonic::Status::resource_exhausted("queue channel is full")); 149 | } 150 | 151 | gauge!(metrics::GRPC_BLOCK_META_QUEUE_SIZE).increment(1); 152 | match rx.await { 153 | Ok(Some(block)) => Ok(block), 154 | Ok(None) => Err(Status::aborted("failed to get result")), 155 | Err(_) => Err(Status::aborted("failed to wait response")), 156 | } 157 | } 158 | 159 | pub async fn get_block(&self, commitment: CommitmentLevelProto) -> tonic::Result { 160 | let (tx, rx) = oneshot::channel(); 161 | let request = Request::GetBlock(tx, commitment); 162 | self.send_request(request, rx).await 163 | } 164 | 165 | pub async fn is_blockhash_valid( 166 | &self, 167 | blockhash: String, 168 | commitment: CommitmentLevelProto, 169 | ) -> tonic::Result<(bool, Slot)> { 170 | let (tx, rx) = oneshot::channel(); 171 | let request = Request::IsBlockhashValid(tx, blockhash, commitment); 172 | self.send_request(request, rx).await 173 | } 174 | } 175 | 176 | enum Request { 177 | GetBlock(oneshot::Sender>, CommitmentLevelProto), 178 | IsBlockhashValid( 179 | oneshot::Sender>, 180 | String, 181 | CommitmentLevelProto, 182 | ), 183 | } 184 | -------------------------------------------------------------------------------- /richat/src/grpc/config.rs: -------------------------------------------------------------------------------- 1 | use { 2 | crate::config::ConfigAppsWorkers, 3 | richat_filter::config::ConfigLimits as ConfigFilterLimits, 4 | richat_shared::{ 5 | config::{deserialize_affinity, deserialize_num_str, deserialize_x_token_set}, 6 | transports::grpc::ConfigGrpcServer as ConfigAppGrpcServer, 7 | }, 8 | serde::Deserialize, 9 | std::{collections::HashSet, time::Duration}, 10 | }; 11 | 12 | #[derive(Debug, Default, Clone, Deserialize)] 13 | #[serde(deny_unknown_fields, default)] 14 | pub struct ConfigAppsGrpc { 15 | pub server: ConfigAppGrpcServer, 16 | pub workers: ConfigAppsGrpcWorkers, 17 | pub stream: ConfigAppsGrpcStream, 18 | pub unary: ConfigAppsGrpcUnary, 19 | pub filter_limits: ConfigFilterLimits, 20 | #[serde(deserialize_with = "deserialize_x_token_set")] 21 | pub x_token: HashSet>, 22 | } 23 | 24 | #[derive(Debug, Clone, Deserialize)] 25 | #[serde(deny_unknown_fields, default)] 26 | pub struct ConfigAppsGrpcWorkers { 27 | #[serde(flatten)] 28 | pub threads: ConfigAppsWorkers, 29 | #[serde(deserialize_with = "deserialize_num_str")] 30 | pub messages_cached_max: usize, 31 | } 32 | 33 | impl Default for ConfigAppsGrpcWorkers { 34 | fn default() -> Self { 35 | Self { 36 | threads: ConfigAppsWorkers::default(), 37 | messages_cached_max: 1_024, 38 | } 39 | } 40 | } 41 | 42 | #[derive(Debug, Clone, Copy, Deserialize)] 43 | #[serde(deny_unknown_fields, default)] 44 | pub struct ConfigAppsGrpcStream { 45 | #[serde(deserialize_with = "deserialize_num_str")] 46 | pub messages_len_max: usize, 47 | #[serde(deserialize_with = "deserialize_num_str")] 48 | pub messages_max_per_tick: usize, 49 | #[serde(with = "humantime_serde")] 50 | pub ping_iterval: Duration, 51 | } 52 | 53 | impl Default for ConfigAppsGrpcStream { 54 | fn default() -> Self { 55 | Self { 56 | messages_len_max: 16 * 1024 * 1024, 57 | messages_max_per_tick: 100, 58 | ping_iterval: Duration::from_secs(15), 59 | } 60 | } 61 | } 62 | 63 | #[derive(Debug, Clone, Deserialize)] 64 | #[serde(deny_unknown_fields, default)] 65 | pub struct ConfigAppsGrpcUnary { 66 | pub enabled: bool, 67 | #[serde(deserialize_with = "deserialize_affinity")] 68 | pub affinity: Option>, 69 | #[serde(deserialize_with = "deserialize_num_str")] 70 | pub requests_queue_size: usize, 71 | } 72 | 73 | impl Default for ConfigAppsGrpcUnary { 74 | fn default() -> Self { 75 | Self { 76 | enabled: true, 77 | affinity: None, 78 | requests_queue_size: 100, 79 | } 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /richat/src/grpc/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod block_meta; 2 | pub mod config; 3 | pub mod server; 4 | -------------------------------------------------------------------------------- /richat/src/lib.rs: -------------------------------------------------------------------------------- 1 | pub mod channel; 2 | pub mod config; 3 | pub mod grpc; 4 | pub mod log; 5 | pub mod metrics; 6 | pub mod pubsub; 7 | pub mod richat; 8 | pub mod source; 9 | pub mod version; 10 | -------------------------------------------------------------------------------- /richat/src/log.rs: -------------------------------------------------------------------------------- 1 | use { 2 | std::io::{self, IsTerminal}, 3 | tracing::Subscriber, 4 | tracing_subscriber::{ 5 | filter::{EnvFilter, LevelFilter}, 6 | fmt::layer, 7 | layer::{Layer, SubscriberExt}, 8 | registry::LookupSpan, 9 | util::SubscriberInitExt, 10 | }, 11 | }; 12 | 13 | pub fn setup(json: bool) -> anyhow::Result<()> { 14 | let env = EnvFilter::builder() 15 | .with_default_directive(LevelFilter::INFO.into()) 16 | .from_env()?; 17 | 18 | tracing_subscriber::registry() 19 | .with(env) 20 | .with(create_io_layer(json)) 21 | .try_init()?; 22 | 23 | Ok(()) 24 | } 25 | 26 | fn create_io_layer(json: bool) -> Box + Send + Sync + 'static> 27 | where 28 | S: Subscriber, 29 | for<'a> S: LookupSpan<'a>, 30 | { 31 | let is_atty = io::stdout().is_terminal() && io::stderr().is_terminal(); 32 | let io_layer = layer().with_ansi(is_atty).with_line_number(true); 33 | 34 | if json { 35 | Box::new(io_layer.json()) 36 | } else { 37 | Box::new(io_layer) 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /richat/src/metrics.rs: -------------------------------------------------------------------------------- 1 | use { 2 | crate::version::VERSION as VERSION_INFO, 3 | hyper::body::Bytes, 4 | metrics::{counter, describe_counter, describe_gauge}, 5 | metrics_exporter_prometheus::{BuildError, PrometheusBuilder, PrometheusHandle}, 6 | richat_filter::filter::FilteredUpdateType, 7 | richat_shared::config::ConfigMetrics, 8 | solana_sdk::clock::Slot, 9 | std::{borrow::Cow, future::Future}, 10 | tokio::{ 11 | task::JoinError, 12 | time::{sleep, Duration}, 13 | }, 14 | tracing::error, 15 | }; 16 | 17 | pub const BLOCK_MESSAGE_FAILED: &str = "block_message_failed"; // reason 18 | pub const CHANNEL_SLOT: &str = "channel_slot"; // commitment 19 | pub const CHANNEL_MESSAGES_TOTAL: &str = "channel_messages_total"; 20 | pub const CHANNEL_SLOTS_TOTAL: &str = "channel_slots_total"; 21 | pub const CHANNEL_BYTES_TOTAL: &str = "channel_bytes_total"; 22 | pub const GRPC_BLOCK_META_SLOT: &str = "grpc_block_meta_slot"; // commitment 23 | pub const GRPC_BLOCK_META_QUEUE_SIZE: &str = "grpc_block_meta_queue_size"; 24 | pub const GRPC_REQUESTS_TOTAL: &str = "grpc_requests_total"; // x_subscription_id, method 25 | pub const GRPC_SUBSCRIBE_TOTAL: &str = "grpc_subscribe_total"; // x_subscription_id 26 | pub const GRPC_SUBSCRIBE_MESSAGES_COUNT_TOTAL: &str = "grpc_subscribe_messages_count_total"; // x_subscription_id, message 27 | pub const GRPC_SUBSCRIBE_MESSAGES_BYTES_TOTAL: &str = "grpc_subscribe_messages_bytes_total"; // x_subscription_id, message 28 | pub const GRPC_SUBSCRIBE_CPU_SECONDS_TOTAL: &str = "grpc_subscribe_cpu_seconds_total"; // x_subscription_id 29 | pub const PUBSUB_SLOT: &str = "pubsub_slot"; // commitment 30 | pub const PUBSUB_CACHED_SIGNATURES_TOTAL: &str = "pubsub_cached_signatures_total"; 31 | pub const PUBSUB_STORED_MESSAGES_COUNT_TOTAL: &str = "pubsub_stored_messages_count_total"; 32 | pub const PUBSUB_STORED_MESSAGES_BYTES_TOTAL: &str = "pubsub_stored_messages_bytes_total"; 33 | pub const PUBSUB_CONNECTIONS_TOTAL: &str = "pubsub_connections_total"; // x_subscription_id 34 | pub const PUBSUB_SUBSCRIPTIONS_TOTAL: &str = "pubsub_subscriptions_total"; // x_subscription_id, subscription 35 | pub const PUBSUB_MESSAGES_SENT_COUNT_TOTAL: &str = "pubsub_messages_sent_count_total"; // x_subscription_id, subscription 36 | pub const PUBSUB_MESSAGES_SENT_BYTES_TOTAL: &str = "pubsub_messages_sent_bytes_total"; // x_subscription_id, subscription 37 | pub const RICHAT_CONNECTIONS_TOTAL: &str = "richat_connections_total"; // transport 38 | 39 | pub fn setup() -> Result { 40 | let handle = PrometheusBuilder::new().install_recorder()?; 41 | 42 | describe_counter!("version", "Richat App version info"); 43 | counter!( 44 | "version", 45 | "buildts" => VERSION_INFO.buildts, 46 | "git" => VERSION_INFO.git, 47 | "package" => VERSION_INFO.package, 48 | "proto" => VERSION_INFO.proto, 49 | "rustc" => VERSION_INFO.rustc, 50 | "solana" => VERSION_INFO.solana, 51 | "version" => VERSION_INFO.version, 52 | ) 53 | .absolute(1); 54 | 55 | describe_counter!(BLOCK_MESSAGE_FAILED, "Block message reconstruction errors"); 56 | describe_gauge!(CHANNEL_SLOT, "Latest slot in channel by commitment"); 57 | describe_gauge!( 58 | CHANNEL_MESSAGES_TOTAL, 59 | "Total number of messages in channel" 60 | ); 61 | describe_gauge!(CHANNEL_SLOTS_TOTAL, "Total number of slots in channel"); 62 | describe_gauge!(CHANNEL_BYTES_TOTAL, "Total size of all messages in channel"); 63 | describe_gauge!(GRPC_BLOCK_META_SLOT, "Latest slot in gRPC block meta"); 64 | describe_gauge!( 65 | GRPC_BLOCK_META_QUEUE_SIZE, 66 | "Number of gRPC requests to block meta data" 67 | ); 68 | describe_counter!(GRPC_REQUESTS_TOTAL, "Number of gRPC requests per method"); 69 | describe_gauge!(GRPC_SUBSCRIBE_TOTAL, "Number of gRPC subscriptions"); 70 | describe_counter!( 71 | GRPC_SUBSCRIBE_MESSAGES_COUNT_TOTAL, 72 | "Number of gRPC messages in subscriptions by type" 73 | ); 74 | describe_counter!( 75 | GRPC_SUBSCRIBE_MESSAGES_BYTES_TOTAL, 76 | "Total size of gRPC messages in subscriptions by type" 77 | ); 78 | describe_gauge!( 79 | GRPC_SUBSCRIBE_CPU_SECONDS_TOTAL, 80 | "CPU consumption of gRPC filters in subscriptions" 81 | ); 82 | describe_gauge!(PUBSUB_SLOT, "Latest slot handled in PubSub by commitment"); 83 | describe_gauge!( 84 | PUBSUB_CACHED_SIGNATURES_TOTAL, 85 | "Number of cached signatures" 86 | ); 87 | describe_gauge!( 88 | PUBSUB_STORED_MESSAGES_COUNT_TOTAL, 89 | "Number of stored filtered messages in cache" 90 | ); 91 | describe_gauge!( 92 | PUBSUB_STORED_MESSAGES_BYTES_TOTAL, 93 | "Total size of stored filtered messages in cache" 94 | ); 95 | describe_gauge!(PUBSUB_CONNECTIONS_TOTAL, "Number of connections to PubSub"); 96 | describe_gauge!( 97 | PUBSUB_SUBSCRIPTIONS_TOTAL, 98 | "Number of subscriptions by type" 99 | ); 100 | describe_counter!( 101 | PUBSUB_MESSAGES_SENT_COUNT_TOTAL, 102 | "Number of sent filtered messages by type" 103 | ); 104 | describe_counter!( 105 | PUBSUB_MESSAGES_SENT_BYTES_TOTAL, 106 | "Total size of sent filtered messages by type" 107 | ); 108 | describe_gauge!( 109 | RICHAT_CONNECTIONS_TOTAL, 110 | "Total number of connections to Richat" 111 | ); 112 | 113 | Ok(handle) 114 | } 115 | 116 | pub async fn spawn_server( 117 | config: ConfigMetrics, 118 | handle: PrometheusHandle, 119 | shutdown: impl Future + Send + 'static, 120 | ) -> anyhow::Result>> { 121 | let recorder_handle = handle.clone(); 122 | tokio::spawn(async move { 123 | loop { 124 | sleep(Duration::from_secs(1)).await; 125 | recorder_handle.run_upkeep(); 126 | } 127 | }); 128 | 129 | richat_shared::metrics::spawn_server( 130 | config, 131 | move || Bytes::from(handle.render()), // metrics 132 | || true, // health 133 | || true, // ready 134 | shutdown, 135 | ) 136 | .await 137 | .map_err(Into::into) 138 | } 139 | 140 | #[derive(Debug, Clone, Copy, PartialEq, Eq)] 141 | pub enum BlockMessageFailedReason { 142 | MissedBlockMeta, 143 | MismatchTransactions { actual: usize, expected: usize }, 144 | MismatchEntries { actual: usize, expected: usize }, 145 | ExtraAccount, 146 | ExtraTransaction, 147 | ExtraEntry, 148 | ExtraBlockMeta, 149 | } 150 | 151 | pub fn block_message_failed_inc(slot: Slot, reasons: &[BlockMessageFailedReason]) { 152 | if !reasons.is_empty() { 153 | error!( 154 | "failed to build block ({slot}): {}", 155 | reasons 156 | .iter() 157 | .map(|reason| match reason { 158 | BlockMessageFailedReason::MissedBlockMeta => Cow::Borrowed("MissedBlockMeta"), 159 | BlockMessageFailedReason::MismatchTransactions { actual, expected } => 160 | Cow::Owned(format!("MismatchTransactions({actual}/{expected})")), 161 | BlockMessageFailedReason::MismatchEntries { actual, expected } => 162 | Cow::Owned(format!("MismatchEntries({actual}/{expected})")), 163 | BlockMessageFailedReason::ExtraAccount => Cow::Borrowed("ExtraAccount"), 164 | BlockMessageFailedReason::ExtraTransaction => Cow::Borrowed("ExtraTransaction"), 165 | BlockMessageFailedReason::ExtraEntry => Cow::Borrowed("ExtraEntry"), 166 | BlockMessageFailedReason::ExtraBlockMeta => Cow::Borrowed("ExtraBlockMeta"), 167 | }) 168 | .collect::>() 169 | .join(",") 170 | ); 171 | 172 | for reason in reasons { 173 | let reason = match reason { 174 | BlockMessageFailedReason::MissedBlockMeta => "MissedBlockMeta", 175 | BlockMessageFailedReason::MismatchTransactions { .. } => "MismatchTransactions", 176 | BlockMessageFailedReason::MismatchEntries { .. } => "MismatchEntries", 177 | BlockMessageFailedReason::ExtraAccount => "ExtraAccount", 178 | BlockMessageFailedReason::ExtraTransaction => "ExtraTransaction", 179 | BlockMessageFailedReason::ExtraEntry => "ExtraEntry", 180 | BlockMessageFailedReason::ExtraBlockMeta => "ExtraBlockMeta", 181 | }; 182 | counter!(BLOCK_MESSAGE_FAILED, "reason" => reason).increment(1); 183 | } 184 | counter!(BLOCK_MESSAGE_FAILED, "reason" => "Total").increment(reasons.len() as u64); 185 | } 186 | } 187 | 188 | #[derive(Debug, Clone, Copy, PartialEq, Eq)] 189 | pub enum GrpcSubscribeMessage { 190 | Slot, 191 | Account, 192 | Transaction, 193 | TransactionStatus, 194 | Entry, 195 | BlockMeta, 196 | Block, 197 | Ping, 198 | Pong, 199 | } 200 | 201 | impl<'a> From<&FilteredUpdateType<'a>> for GrpcSubscribeMessage { 202 | fn from(value: &FilteredUpdateType<'a>) -> Self { 203 | match value { 204 | FilteredUpdateType::Slot { .. } => Self::Slot, 205 | FilteredUpdateType::Account { .. } => Self::Account, 206 | FilteredUpdateType::Transaction { .. } => Self::Transaction, 207 | FilteredUpdateType::TransactionStatus { .. } => Self::TransactionStatus, 208 | FilteredUpdateType::Entry { .. } => Self::Entry, 209 | FilteredUpdateType::BlockMeta { .. } => Self::BlockMeta, 210 | FilteredUpdateType::Block { .. } => Self::Block, 211 | } 212 | } 213 | } 214 | 215 | impl GrpcSubscribeMessage { 216 | pub const fn as_str(self) -> &'static str { 217 | match self { 218 | GrpcSubscribeMessage::Slot => "slot", 219 | GrpcSubscribeMessage::Account => "account", 220 | GrpcSubscribeMessage::Transaction => "transaction", 221 | GrpcSubscribeMessage::TransactionStatus => "transactionstatus", 222 | GrpcSubscribeMessage::Entry => "entry", 223 | GrpcSubscribeMessage::BlockMeta => "blockmeta", 224 | GrpcSubscribeMessage::Block => "block", 225 | GrpcSubscribeMessage::Ping => "ping", 226 | GrpcSubscribeMessage::Pong => "pong", 227 | } 228 | } 229 | } 230 | -------------------------------------------------------------------------------- /richat/src/pubsub/config.rs: -------------------------------------------------------------------------------- 1 | use { 2 | richat_shared::config::{ 3 | deserialize_affinity, deserialize_maybe_rustls_server_config, deserialize_num_str, 4 | }, 5 | serde::Deserialize, 6 | std::{ 7 | io, 8 | net::{IpAddr, Ipv4Addr, SocketAddr}, 9 | }, 10 | tokio::net::TcpStream, 11 | }; 12 | 13 | #[derive(Debug, Clone, Deserialize)] 14 | #[serde(deny_unknown_fields, default)] 15 | pub struct ConfigAppsPubsub { 16 | pub endpoint: SocketAddr, 17 | pub tcp_nodelay: Option, 18 | #[serde(deserialize_with = "deserialize_maybe_rustls_server_config")] 19 | pub tls_config: Option, 20 | #[serde(deserialize_with = "deserialize_num_str")] 21 | pub recv_max_message_size: usize, 22 | pub enable_block_subscription: bool, 23 | pub enable_transaction_subscription: bool, 24 | #[serde(deserialize_with = "deserialize_num_str")] 25 | pub clients_requests_channel_size: usize, 26 | #[serde(deserialize_with = "deserialize_affinity")] 27 | pub subscriptions_worker_affinity: Option>, 28 | #[serde(deserialize_with = "deserialize_num_str")] 29 | pub subscriptions_workers_count: usize, 30 | #[serde(deserialize_with = "deserialize_affinity")] 31 | pub subscriptions_workers_affinity: Option>, 32 | #[serde(deserialize_with = "deserialize_num_str")] 33 | pub subscriptions_max_clients_request_per_tick: usize, 34 | #[serde(deserialize_with = "deserialize_num_str")] 35 | pub subscriptions_max_messages_per_commitment_per_tick: usize, 36 | #[serde(deserialize_with = "deserialize_num_str")] 37 | pub notifications_messages_max_count: usize, 38 | #[serde(deserialize_with = "deserialize_num_str")] 39 | pub notifications_messages_max_bytes: usize, 40 | #[serde(deserialize_with = "deserialize_num_str")] 41 | pub signatures_cache_max: usize, 42 | #[serde(deserialize_with = "deserialize_num_str")] 43 | pub signatures_cache_slots_max: usize, 44 | } 45 | 46 | impl Default for ConfigAppsPubsub { 47 | fn default() -> Self { 48 | Self { 49 | endpoint: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8000), 50 | tcp_nodelay: None, 51 | tls_config: None, 52 | recv_max_message_size: 4 * 1024, // 4KiB 53 | enable_block_subscription: false, 54 | enable_transaction_subscription: false, 55 | clients_requests_channel_size: 8_192, 56 | subscriptions_worker_affinity: None, 57 | subscriptions_workers_count: 2, 58 | subscriptions_workers_affinity: None, 59 | subscriptions_max_clients_request_per_tick: 32, 60 | subscriptions_max_messages_per_commitment_per_tick: 256, 61 | notifications_messages_max_count: 10_000_000, 62 | notifications_messages_max_bytes: 32 * 1024 * 1024 * 1024, 63 | signatures_cache_max: 150 * 8_192, // 8k more than enough per slot, should be about 300MiB 64 | signatures_cache_slots_max: 150, 65 | } 66 | } 67 | } 68 | 69 | impl ConfigAppsPubsub { 70 | pub fn set_accepted_socket_options(&self, stream: &TcpStream) -> io::Result<()> { 71 | if let Some(nodelay) = self.tcp_nodelay { 72 | stream.set_nodelay(nodelay)?; 73 | } 74 | Ok(()) 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /richat/src/pubsub/filter.rs: -------------------------------------------------------------------------------- 1 | use { 2 | richat_filter::message::MessageTransaction, 3 | solana_sdk::{pubkey::Pubkey, signature::Signature}, 4 | std::{ 5 | collections::HashSet, 6 | hash::{Hash, Hasher}, 7 | }, 8 | }; 9 | 10 | #[derive(Debug)] 11 | pub struct TransactionFilter { 12 | pub vote: Option, 13 | pub failed: Option, 14 | pub signature: Option, 15 | pub account_include: HashSet, 16 | pub account_exclude: HashSet, 17 | pub account_required: HashSet, 18 | } 19 | 20 | impl Hash for TransactionFilter { 21 | fn hash(&self, state: &mut H) { 22 | self.vote.hash(state); 23 | self.failed.hash(state); 24 | self.signature.hash(state); 25 | for pubkeys in &[ 26 | &self.account_include, 27 | &self.account_exclude, 28 | &self.account_required, 29 | ] { 30 | let mut pubkeys = pubkeys.iter().copied().collect::>(); 31 | pubkeys.sort_unstable(); 32 | pubkeys.hash(state) 33 | } 34 | } 35 | } 36 | 37 | impl TransactionFilter { 38 | pub fn matches(&self, message: &MessageTransaction) -> bool { 39 | if let Some(vote) = self.vote { 40 | if vote != message.vote() { 41 | return false; 42 | } 43 | } 44 | 45 | if let Some(failed) = self.failed { 46 | if failed != message.failed() { 47 | return false; 48 | } 49 | } 50 | 51 | if let Some(filter_signature) = &self.signature { 52 | if filter_signature != message.signature() { 53 | return false; 54 | } 55 | } 56 | 57 | if !self.account_include.is_empty() 58 | && self 59 | .account_include 60 | .intersection(message.account_keys()) 61 | .next() 62 | .is_none() 63 | { 64 | return false; 65 | } 66 | 67 | if !self.account_exclude.is_empty() 68 | && self 69 | .account_exclude 70 | .intersection(message.account_keys()) 71 | .next() 72 | .is_some() 73 | { 74 | return false; 75 | } 76 | 77 | if !self.account_required.is_empty() 78 | && !self.account_required.is_subset(message.account_keys()) 79 | { 80 | return false; 81 | } 82 | 83 | true 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /richat/src/pubsub/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod config; 2 | pub mod filter; 3 | pub mod notification; 4 | pub mod server; 5 | pub mod solana; 6 | pub mod tracker; 7 | 8 | pub type ClientId = u64; 9 | pub type SubscriptionId = u64; 10 | -------------------------------------------------------------------------------- /richat/src/richat/config.rs: -------------------------------------------------------------------------------- 1 | use { 2 | richat_shared::transports::{grpc::ConfigGrpcServer, quic::ConfigQuicServer}, 3 | serde::Deserialize, 4 | }; 5 | 6 | #[derive(Debug, Clone, Default, Deserialize)] 7 | #[serde(deny_unknown_fields, default)] 8 | pub struct ConfigAppsRichat { 9 | pub quic: Option, 10 | pub grpc: Option, 11 | } 12 | -------------------------------------------------------------------------------- /richat/src/richat/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod config; 2 | pub mod server; 3 | -------------------------------------------------------------------------------- /richat/src/richat/server.rs: -------------------------------------------------------------------------------- 1 | use { 2 | crate::{channel::Messages, metrics, richat::config::ConfigAppsRichat, version::VERSION}, 3 | ::metrics::gauge, 4 | futures::future::{try_join_all, FutureExt, TryFutureExt}, 5 | richat_shared::{ 6 | shutdown::Shutdown, 7 | transports::{grpc::GrpcServer, quic::QuicServer}, 8 | }, 9 | std::future::Future, 10 | }; 11 | 12 | #[derive(Debug)] 13 | pub struct RichatServer; 14 | 15 | impl RichatServer { 16 | pub async fn spawn( 17 | config: ConfigAppsRichat, 18 | messages: Messages, 19 | shutdown: Shutdown, 20 | ) -> anyhow::Result>> { 21 | let mut tasks = Vec::with_capacity(3); 22 | 23 | // Start Quic 24 | if let Some(config) = config.quic { 25 | let connections_inc = gauge!(metrics::RICHAT_CONNECTIONS_TOTAL, "transport" => "quic"); 26 | let connections_dec = connections_inc.clone(); 27 | tasks.push( 28 | QuicServer::spawn( 29 | config, 30 | messages.clone(), 31 | move || connections_inc.increment(1), // on_conn_new_cb 32 | move || connections_dec.decrement(1), // on_conn_drop_cb 33 | VERSION, 34 | shutdown.clone(), 35 | ) 36 | .await? 37 | .boxed(), 38 | ); 39 | } 40 | 41 | // Start gRPC 42 | if let Some(config) = config.grpc { 43 | let connections_inc = gauge!(metrics::RICHAT_CONNECTIONS_TOTAL, "transport" => "grpc"); 44 | let connections_dec = connections_inc.clone(); 45 | tasks.push( 46 | GrpcServer::spawn( 47 | config, 48 | messages.clone(), 49 | move || connections_inc.increment(1), // on_conn_new_cb 50 | move || connections_dec.decrement(1), // on_conn_drop_cb 51 | VERSION, 52 | shutdown.clone(), 53 | ) 54 | .await? 55 | .boxed(), 56 | ); 57 | } 58 | 59 | Ok(try_join_all(tasks).map_ok(|_| ()).map_err(Into::into)) 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /richat/src/version.rs: -------------------------------------------------------------------------------- 1 | use {richat_shared::version::Version, std::env}; 2 | 3 | pub const VERSION: Version = Version { 4 | package: env!("CARGO_PKG_NAME"), 5 | version: env!("CARGO_PKG_VERSION"), 6 | proto: env!("YELLOWSTONE_GRPC_PROTO_VERSION"), 7 | proto_richat: env!("RICHAT_PROTO_VERSION"), 8 | solana: env!("SOLANA_SDK_VERSION"), 9 | git: env!("GIT_VERSION"), 10 | rustc: env!("VERGEN_RUSTC_SEMVER"), 11 | buildts: env!("VERGEN_BUILD_TIMESTAMP"), 12 | }; 13 | -------------------------------------------------------------------------------- /rust-toolchain.toml: -------------------------------------------------------------------------------- 1 | [toolchain] 2 | channel = "1.84.1" # agave v2.2 3 | components = ["clippy", "rustfmt"] 4 | targets = [] 5 | profile = "minimal" 6 | -------------------------------------------------------------------------------- /rustfmt.toml: -------------------------------------------------------------------------------- 1 | edition = "2021" 2 | imports_granularity = "One" 3 | group_imports = "One" 4 | -------------------------------------------------------------------------------- /shared/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "richat-shared" 3 | version = "3.5.0" 4 | authors = { workspace = true } 5 | edition = { workspace = true } 6 | description = "Richat Shared code" 7 | homepage = { workspace = true } 8 | repository = { workspace = true } 9 | license = { workspace = true } 10 | keywords = { workspace = true } 11 | publish = true 12 | 13 | [dependencies] 14 | affinity-linux = { workspace = true, optional = true } 15 | anyhow = { workspace = true, optional = true } 16 | base64 = { workspace = true, optional = true } 17 | bs58 = { workspace = true, optional = true } 18 | five8 = { workspace = true, optional = true } 19 | futures = { workspace = true, optional = true } 20 | hostname = { workspace = true, optional = true } 21 | humantime-serde = { workspace = true, optional = true } 22 | http = { workspace = true, optional = true } 23 | http-body-util = { workspace = true, optional = true } 24 | hyper = { workspace = true, optional = true } 25 | hyper-util = { workspace = true, features = ["server-auto", "tokio"], optional = true } 26 | jsonrpc-core = { workspace = true, optional = true } 27 | jsonrpsee-types = { workspace = true, optional = true } 28 | metrics = { workspace = true, optional = true } 29 | prost = { workspace = true, optional = true } 30 | quanta = { workspace = true, optional = true } 31 | quinn = { workspace = true, optional = true } 32 | rcgen = { workspace = true, optional = true } 33 | richat-proto = { workspace = true, optional = true } 34 | rustls = { workspace = true, features = ["std", "tls12"], optional = true } 35 | rustls-pemfile = { workspace = true, optional = true } 36 | serde = { workspace = true, features = ["derive"], optional = true } 37 | serde_json = { workspace = true, optional = true } 38 | slab = { workspace = true, optional = true } 39 | socket2 = { workspace = true, optional = true } 40 | solana-rpc-client-api = { workspace = true, optional = true } 41 | solana-sdk = { workspace = true, optional = true } 42 | thiserror = { workspace = true, optional = true } 43 | tokio = { workspace = true, features = ["rt-multi-thread", "macros"], optional = true } 44 | tonic = { workspace = true, features = ["tls", "gzip", "zstd"], optional = true } 45 | tracing = { workspace = true, optional = true } 46 | 47 | [build-dependencies] 48 | anyhow = { workspace = true, optional = true } 49 | protobuf-src = { workspace = true, optional = true } 50 | tonic-build = { workspace = true, optional = true } 51 | 52 | [features] 53 | default = [ 54 | "config", 55 | "five8", 56 | "jsonrpc", 57 | "metrics", 58 | "shutdown", 59 | "transports", 60 | "version" 61 | ] 62 | config = [ 63 | "dep:affinity-linux", 64 | "dep:base64", 65 | "dep:bs58", 66 | "dep:rcgen", 67 | "dep:rustls", 68 | "dep:rustls-pemfile", 69 | "dep:serde", 70 | "dep:solana-sdk", 71 | "dep:thiserror", 72 | "dep:tokio", 73 | "five8", 74 | ] 75 | five8 = ["dep:five8", "dep:solana-sdk"] 76 | jsonrpc = [ 77 | "dep:anyhow", 78 | "dep:futures", 79 | "dep:jsonrpc-core", 80 | "dep:jsonrpsee-types", 81 | "dep:metrics", 82 | "dep:quanta", 83 | "dep:serde", 84 | "dep:serde_json", 85 | "dep:solana-rpc-client-api", 86 | "metrics", 87 | ] 88 | metrics = [ 89 | "dep:http", 90 | "dep:http-body-util", 91 | "dep:hyper", 92 | "dep:hyper-util", 93 | "dep:tracing", 94 | "config", 95 | ] 96 | shutdown = ["dep:slab"] 97 | transports = [ 98 | "dep:anyhow", 99 | "dep:futures", 100 | "dep:humantime-serde", 101 | "dep:prost", 102 | "dep:protobuf-src", 103 | "dep:quinn", 104 | "dep:richat-proto", 105 | "dep:socket2", 106 | "dep:tonic", 107 | "dep:tonic-build", 108 | "dep:tracing", 109 | "config", 110 | "shutdown", 111 | "version", 112 | ] 113 | version = ["dep:hostname", "dep:serde", "dep:serde_json"] 114 | 115 | [lints] 116 | workspace = true 117 | -------------------------------------------------------------------------------- /shared/build.rs: -------------------------------------------------------------------------------- 1 | #[cfg(not(feature = "transports"))] 2 | fn main() {} 3 | 4 | #[cfg(feature = "transports")] 5 | fn main() -> anyhow::Result<()> { 6 | // build protos 7 | std::env::set_var("PROTOC", protobuf_src::protoc()); 8 | generate_grpc_geyser() 9 | } 10 | 11 | #[cfg(feature = "transports")] 12 | fn generate_grpc_geyser() -> anyhow::Result<()> { 13 | use tonic_build::manual::{Builder, Method, Service}; 14 | 15 | let geyser_service = Service::builder() 16 | .name("Geyser") 17 | .package("geyser") 18 | .method( 19 | Method::builder() 20 | .name("subscribe") 21 | .route_name("Subscribe") 22 | .input_type("crate::transports::grpc::GrpcSubscribeRequest") 23 | .output_type("Arc>") 24 | .codec_path("crate::transports::grpc::SubscribeCodec") 25 | .client_streaming() 26 | .server_streaming() 27 | .build(), 28 | ) 29 | .method( 30 | Method::builder() 31 | .name("get_version") 32 | .route_name("GetVersion") 33 | .input_type("richat_proto::geyser::GetVersionRequest") 34 | .output_type("richat_proto::geyser::GetVersionResponse") 35 | .codec_path("tonic::codec::ProstCodec") 36 | .build(), 37 | ) 38 | .build(); 39 | 40 | Builder::new() 41 | .build_client(false) 42 | .compile(&[geyser_service]); 43 | 44 | Ok(()) 45 | } 46 | -------------------------------------------------------------------------------- /shared/src/five8.rs: -------------------------------------------------------------------------------- 1 | use { 2 | five8::DecodeError, 3 | solana_sdk::{ 4 | pubkey::{ParsePubkeyError, Pubkey}, 5 | signature::{ParseSignatureError, Signature}, 6 | }, 7 | }; 8 | 9 | pub fn pubkey_decode>(encoded: I) -> Result { 10 | let mut out = [0; 32]; 11 | match five8::decode_32(encoded, &mut out) { 12 | Ok(()) => Ok(Pubkey::new_from_array(out)), 13 | Err(DecodeError::InvalidChar(_)) => Err(ParsePubkeyError::Invalid), 14 | Err(DecodeError::TooLong) => Err(ParsePubkeyError::WrongSize), 15 | Err(DecodeError::TooShort) => Err(ParsePubkeyError::WrongSize), 16 | Err(DecodeError::LargestTermTooHigh) => Err(ParsePubkeyError::WrongSize), 17 | Err(DecodeError::OutputTooLong) => Err(ParsePubkeyError::WrongSize), 18 | } 19 | } 20 | 21 | pub fn pubkey_encode(bytes: &[u8; 32]) -> String { 22 | let mut out = [0; 44]; 23 | let len = five8::encode_32(bytes, &mut out) as usize; 24 | out[0..len].iter().copied().map(char::from).collect() 25 | } 26 | 27 | pub fn signature_decode>(encoded: I) -> Result { 28 | let mut out = [0; 64]; 29 | match five8::decode_64(encoded, &mut out) { 30 | Ok(()) => Ok(Signature::from(out)), 31 | Err(DecodeError::InvalidChar(_)) => Err(ParseSignatureError::Invalid), 32 | Err(DecodeError::TooLong) => Err(ParseSignatureError::WrongSize), 33 | Err(DecodeError::TooShort) => Err(ParseSignatureError::WrongSize), 34 | Err(DecodeError::LargestTermTooHigh) => Err(ParseSignatureError::WrongSize), 35 | Err(DecodeError::OutputTooLong) => Err(ParseSignatureError::WrongSize), 36 | } 37 | } 38 | 39 | pub fn signature_encode(bytes: &[u8; 64]) -> String { 40 | let mut out = [0; 88]; 41 | let len = five8::encode_64(bytes, &mut out) as usize; 42 | out[0..len].iter().copied().map(char::from).collect() 43 | } 44 | -------------------------------------------------------------------------------- /shared/src/jsonrpc/helpers.rs: -------------------------------------------------------------------------------- 1 | use { 2 | http_body_util::{combinators::BoxBody, BodyExt, Full as BodyFull}, 3 | hyper::{body::Bytes, header::CONTENT_TYPE, http::Result as HttpResult, HeaderMap, StatusCode}, 4 | jsonrpsee_types::{ 5 | ErrorCode, ErrorObject, ErrorObjectOwned, Id, Response, ResponsePayload, TwoPointZero, 6 | }, 7 | serde::Serialize, 8 | solana_rpc_client_api::custom_error::RpcCustomError, 9 | std::{fmt, sync::Arc}, 10 | }; 11 | 12 | pub const X_SUBSCRIPTION_ID: &str = "x-subscription-id"; 13 | pub const X_BIGTABLE: &str = "x-bigtable"; // https://github.com/anza-xyz/agave/blob/v2.2.10/rpc/src/rpc_service.rs#L554 14 | 15 | pub type RpcResponse = hyper::Response>; 16 | 17 | pub fn get_x_subscription_id(headers: &HeaderMap) -> Arc { 18 | headers 19 | .get(X_SUBSCRIPTION_ID) 20 | .and_then(|value| value.to_str().ok().map(ToOwned::to_owned)) 21 | .unwrap_or_default() 22 | .into() 23 | } 24 | 25 | pub fn get_x_bigtable_disabled(headers: &HeaderMap) -> bool { 26 | headers.get(X_BIGTABLE).is_some_and(|v| v == "disabled") 27 | } 28 | 29 | pub fn response_200>(data: D) -> HttpResult { 30 | hyper::Response::builder() 31 | .header(CONTENT_TYPE, "application/json; charset=utf-8") 32 | .body(BodyFull::from(data.into()).boxed()) 33 | } 34 | 35 | pub fn response_400(error: E) -> HttpResult { 36 | hyper::Response::builder() 37 | .status(StatusCode::BAD_REQUEST) 38 | .body(format!("{error}\n").boxed()) 39 | } 40 | 41 | pub fn response_500(error: E) -> HttpResult { 42 | hyper::Response::builder() 43 | .status(StatusCode::INTERNAL_SERVER_ERROR) 44 | .body(format!("{error}\n").boxed()) 45 | } 46 | 47 | pub fn jsonrpc_response_success(id: Id<'_>, payload: T) -> Vec { 48 | to_vec(&Response { 49 | jsonrpc: Some(TwoPointZero), 50 | payload: ResponsePayload::success(payload), 51 | id, 52 | }) 53 | } 54 | 55 | pub fn jsonrpc_response_error(id: Id<'_>, error: ErrorObjectOwned) -> Vec { 56 | to_vec(&Response { 57 | jsonrpc: Some(TwoPointZero), 58 | payload: ResponsePayload::<()>::error(error), 59 | id, 60 | }) 61 | } 62 | 63 | pub fn jsonrpc_response_error_custom(id: Id<'_>, error: RpcCustomError) -> Vec { 64 | let error = jsonrpc_core::Error::from(error); 65 | jsonrpc_response_error( 66 | id, 67 | ErrorObject::owned(error.code.code() as i32, error.message, error.data), 68 | ) 69 | } 70 | 71 | pub fn jsonrpc_error_invalid_params( 72 | message: impl Into, 73 | data: Option, 74 | ) -> ErrorObjectOwned { 75 | ErrorObject::owned(ErrorCode::InvalidParams.code(), message, data) 76 | } 77 | 78 | pub fn to_vec(value: &T) -> Vec { 79 | serde_json::to_vec(value).expect("json serialization never fail") 80 | } 81 | -------------------------------------------------------------------------------- /shared/src/jsonrpc/metrics.rs: -------------------------------------------------------------------------------- 1 | use metrics::{describe_counter, describe_histogram}; 2 | 3 | pub const RPC_REQUESTS_TOTAL: &str = "rpc_requests_total"; // x_subscription_id, method 4 | pub const RPC_REQUESTS_DURATION_SECONDS: &str = "rpc_requests_duration_seconds"; // x_subscription_id, method 5 | pub const RPC_REQUESTS_GENERATED_BYTES_TOTAL: &str = "rpc_requests_generated_bytes_total"; // x_subscription_id 6 | 7 | pub fn describe() { 8 | describe_counter!( 9 | RPC_REQUESTS_TOTAL, 10 | "Number of RPC requests by x-subscription-id and method" 11 | ); 12 | describe_histogram!( 13 | RPC_REQUESTS_DURATION_SECONDS, 14 | "RPC request time by x-subscription-id and method" 15 | ); 16 | describe_counter!( 17 | RPC_REQUESTS_GENERATED_BYTES_TOTAL, 18 | "Number of bytes generated by RPC requests by x-subscription-id" 19 | ); 20 | } 21 | -------------------------------------------------------------------------------- /shared/src/jsonrpc/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod helpers; 2 | pub mod metrics; 3 | pub mod requests; 4 | -------------------------------------------------------------------------------- /shared/src/jsonrpc/requests.rs: -------------------------------------------------------------------------------- 1 | use { 2 | crate::{ 3 | jsonrpc::{ 4 | helpers::{ 5 | get_x_bigtable_disabled, get_x_subscription_id, response_200, response_400, 6 | response_500, to_vec, RpcResponse, 7 | }, 8 | metrics::{ 9 | RPC_REQUESTS_DURATION_SECONDS, RPC_REQUESTS_GENERATED_BYTES_TOTAL, 10 | RPC_REQUESTS_TOTAL, 11 | }, 12 | }, 13 | metrics::duration_to_seconds, 14 | }, 15 | futures::{ 16 | future::BoxFuture, 17 | stream::{FuturesOrdered, StreamExt}, 18 | }, 19 | http_body_util::{BodyExt, Limited}, 20 | hyper::{ 21 | body::{Bytes, Incoming as BodyIncoming}, 22 | http::Result as HttpResult, 23 | }, 24 | jsonrpsee_types::{error::ErrorCode, Request, Response, ResponsePayload, TwoPointZero}, 25 | metrics::{counter, histogram}, 26 | quanta::Instant, 27 | std::{collections::HashMap, fmt, sync::Arc}, 28 | }; 29 | 30 | pub type RpcRequestResult = anyhow::Result>; 31 | 32 | pub type RpcRequestHandler = 33 | Box, bool, Request<'_>) -> BoxFuture<'_, RpcRequestResult> + Send + Sync>; 34 | 35 | #[derive(Debug)] 36 | enum RpcRequests<'a> { 37 | Single(Request<'a>), 38 | Batch(Vec>), 39 | } 40 | 41 | impl<'a> RpcRequests<'a> { 42 | fn parse(bytes: &'a Bytes) -> serde_json::Result { 43 | for i in 0..bytes.len() { 44 | if bytes[i] == b'[' { 45 | return serde_json::from_slice::>>(bytes).map(Self::Batch); 46 | } else if bytes[i] == b'{' { 47 | break; 48 | } 49 | } 50 | serde_json::from_slice::>(bytes).map(Self::Single) 51 | } 52 | } 53 | 54 | pub struct RpcRequestsProcessor { 55 | body_limit: usize, 56 | state: S, 57 | methods: HashMap<&'static str, RpcRequestHandler>, 58 | } 59 | 60 | impl fmt::Debug for RpcRequestsProcessor { 61 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 62 | f.debug_struct("RpcRequestsProcessor").finish() 63 | } 64 | } 65 | 66 | impl RpcRequestsProcessor { 67 | pub fn new(body_limit: usize, state: S) -> Self { 68 | Self { 69 | body_limit, 70 | state, 71 | methods: HashMap::new(), 72 | } 73 | } 74 | 75 | pub fn add_handler( 76 | &mut self, 77 | method: &'static str, 78 | handler: RpcRequestHandler, 79 | ) -> &mut Self { 80 | self.methods.insert(method, handler); 81 | self 82 | } 83 | 84 | pub async fn on_request(&self, req: hyper::Request) -> HttpResult { 85 | let (parts, body) = req.into_parts(); 86 | 87 | let x_subscription_id = get_x_subscription_id(&parts.headers); 88 | let upstream_disabled = get_x_bigtable_disabled(&parts.headers); 89 | 90 | let bytes = match Limited::new(body, self.body_limit).collect().await { 91 | Ok(body) => body.to_bytes(), 92 | Err(error) => return response_400(error), 93 | }; 94 | let requests = match RpcRequests::parse(&bytes) { 95 | Ok(requests) => requests, 96 | Err(error) => return response_400(error), 97 | }; 98 | 99 | let mut buffer = match requests { 100 | RpcRequests::Single(request) => { 101 | match self 102 | .process(Arc::clone(&x_subscription_id), upstream_disabled, request) 103 | .await 104 | { 105 | Ok(response) => response, 106 | Err(error) => return response_500(error), 107 | } 108 | } 109 | RpcRequests::Batch(requests) => { 110 | let mut futures = FuturesOrdered::new(); 111 | for request in requests { 112 | let x_subscription_id = Arc::clone(&x_subscription_id); 113 | futures.push_back(self.process( 114 | Arc::clone(&x_subscription_id), 115 | upstream_disabled, 116 | request, 117 | )); 118 | } 119 | 120 | let mut buffer = Vec::new(); 121 | buffer.push(b'['); 122 | while let Some(result) = futures.next().await { 123 | match result { 124 | Ok(mut response) => { 125 | buffer.append(&mut response); 126 | } 127 | Err(error) => return response_500(error), 128 | } 129 | if !futures.is_empty() { 130 | buffer.push(b','); 131 | } 132 | } 133 | buffer.push(b']'); 134 | buffer 135 | } 136 | }; 137 | buffer.push(b'\n'); 138 | counter!( 139 | RPC_REQUESTS_GENERATED_BYTES_TOTAL, 140 | "x_subscription_id" => x_subscription_id, 141 | ) 142 | .increment(buffer.len() as u64); 143 | response_200(buffer) 144 | } 145 | 146 | async fn process<'a>( 147 | &'a self, 148 | x_subscription_id: Arc, 149 | upstream_disabled: bool, 150 | request: Request<'a>, 151 | ) -> anyhow::Result> { 152 | let Some((method, handle)) = self.methods.get_key_value(request.method.as_ref()) else { 153 | return Ok(to_vec(&Response { 154 | jsonrpc: Some(TwoPointZero), 155 | payload: ResponsePayload::<()>::error(ErrorCode::MethodNotFound), 156 | id: request.id.into_owned(), 157 | })); 158 | }; 159 | 160 | let ts = Instant::now(); 161 | let result = handle( 162 | self.state.clone(), 163 | Arc::clone(&x_subscription_id), 164 | upstream_disabled, 165 | request, 166 | ) 167 | .await; 168 | counter!( 169 | RPC_REQUESTS_TOTAL, 170 | "x_subscription_id" => Arc::clone(&x_subscription_id), 171 | "method" => *method, 172 | ) 173 | .increment(1); 174 | histogram!( 175 | RPC_REQUESTS_DURATION_SECONDS, 176 | "x_subscription_id" => x_subscription_id, 177 | "method" => *method, 178 | ) 179 | .record(duration_to_seconds(ts.elapsed())); 180 | result 181 | } 182 | } 183 | -------------------------------------------------------------------------------- /shared/src/lib.rs: -------------------------------------------------------------------------------- 1 | #[cfg(feature = "config")] 2 | pub mod config; 3 | #[cfg(feature = "five8")] 4 | pub mod five8; 5 | #[cfg(feature = "jsonrpc")] 6 | pub mod jsonrpc; 7 | #[cfg(feature = "metrics")] 8 | pub mod metrics; 9 | #[cfg(feature = "shutdown")] 10 | pub mod shutdown; 11 | #[cfg(feature = "transports")] 12 | pub mod transports; 13 | #[cfg(feature = "version")] 14 | pub mod version; 15 | -------------------------------------------------------------------------------- /shared/src/metrics.rs: -------------------------------------------------------------------------------- 1 | use { 2 | crate::config::ConfigMetrics, 3 | http_body_util::{BodyExt, Full as BodyFull}, 4 | hyper::{ 5 | body::{Bytes, Incoming as BodyIncoming}, 6 | service::service_fn, 7 | Request, Response, StatusCode, 8 | }, 9 | hyper_util::{ 10 | rt::tokio::{TokioExecutor, TokioIo}, 11 | server::conn::auto::Builder as ServerBuilder, 12 | }, 13 | std::future::Future, 14 | tokio::{net::TcpListener, task::JoinError, time::Duration}, 15 | tracing::{error, info}, 16 | }; 17 | 18 | #[inline] 19 | pub fn duration_to_seconds(d: Duration) -> f64 { 20 | d.as_secs() as f64 + d.subsec_nanos() as f64 / 1e9 21 | } 22 | 23 | pub async fn spawn_server( 24 | ConfigMetrics { endpoint }: ConfigMetrics, 25 | gather_metrics: impl Fn() -> Bytes + Clone + Send + 'static, 26 | is_health_check: impl Fn() -> bool + Clone + Send + 'static, 27 | is_ready_check: impl Fn() -> bool + Clone + Send + 'static, 28 | shutdown: impl Future + Send + 'static, 29 | ) -> std::io::Result>> { 30 | let listener = TcpListener::bind(endpoint).await?; 31 | info!("start server at: {endpoint}"); 32 | 33 | Ok(tokio::spawn(async move { 34 | tokio::pin!(shutdown); 35 | loop { 36 | let stream = tokio::select! { 37 | maybe_conn = listener.accept() => { 38 | match maybe_conn { 39 | Ok((stream, _addr)) => stream, 40 | Err(error) => { 41 | error!("failed to accept new connection: {error}"); 42 | break; 43 | } 44 | } 45 | } 46 | () = &mut shutdown => { 47 | info!("shutdown"); 48 | break 49 | }, 50 | }; 51 | let gather_metrics = gather_metrics.clone(); 52 | let is_health_check = is_health_check.clone(); 53 | let is_ready_check = is_ready_check.clone(); 54 | tokio::spawn(async move { 55 | if let Err(error) = ServerBuilder::new(TokioExecutor::new()) 56 | .serve_connection( 57 | TokioIo::new(stream), 58 | service_fn(move |req: Request| { 59 | let gather_metrics = gather_metrics.clone(); 60 | let is_health_check = is_health_check.clone(); 61 | let is_ready_check = is_ready_check.clone(); 62 | async move { 63 | let (status, bytes) = match req.uri().path() { 64 | "/health" => { 65 | if is_health_check() { 66 | (StatusCode::OK, Bytes::from("OK")) 67 | } else { 68 | ( 69 | StatusCode::INTERNAL_SERVER_ERROR, 70 | Bytes::from("Service is unhealthy"), 71 | ) 72 | } 73 | } 74 | "/metrics" => (StatusCode::OK, gather_metrics()), 75 | "/ready" => { 76 | if is_ready_check() { 77 | (StatusCode::OK, Bytes::from("OK")) 78 | } else { 79 | ( 80 | StatusCode::INTERNAL_SERVER_ERROR, 81 | Bytes::from("Service is not ready"), 82 | ) 83 | } 84 | } 85 | _ => (StatusCode::NOT_FOUND, Bytes::new()), 86 | }; 87 | 88 | Response::builder() 89 | .status(status) 90 | .body(BodyFull::new(bytes).boxed()) 91 | } 92 | }), 93 | ) 94 | .await 95 | { 96 | error!("failed to handle request: {error}"); 97 | } 98 | }); 99 | } 100 | })) 101 | } 102 | -------------------------------------------------------------------------------- /shared/src/shutdown.rs: -------------------------------------------------------------------------------- 1 | use { 2 | slab::Slab, 3 | std::{ 4 | future::Future, 5 | pin::Pin, 6 | sync::{Arc, Mutex, MutexGuard}, 7 | task::{Context, Poll, Waker}, 8 | }, 9 | }; 10 | 11 | #[derive(Debug)] 12 | pub struct Shutdown { 13 | state: Arc>, 14 | index: usize, 15 | } 16 | 17 | impl Shutdown { 18 | pub fn new() -> Self { 19 | let mut state = State { 20 | shutdown: false, 21 | wakers: Slab::with_capacity(64), 22 | }; 23 | let index = state.wakers.insert(None); 24 | 25 | Self { 26 | state: Arc::new(Mutex::new(state)), 27 | index, 28 | } 29 | } 30 | 31 | fn state_lock(&self) -> MutexGuard<'_, State> { 32 | match self.state.lock() { 33 | Ok(guard) => guard, 34 | Err(error) => error.into_inner(), 35 | } 36 | } 37 | 38 | pub fn shutdown(&self) { 39 | let mut state = self.state_lock(); 40 | state.shutdown = true; 41 | for (_index, value) in state.wakers.iter_mut() { 42 | if let Some(waker) = value.take() { 43 | waker.wake(); 44 | } 45 | } 46 | } 47 | 48 | pub fn is_set(&self) -> bool { 49 | self.state_lock().shutdown 50 | } 51 | } 52 | 53 | impl Default for Shutdown { 54 | fn default() -> Self { 55 | Self::new() 56 | } 57 | } 58 | 59 | impl Clone for Shutdown { 60 | fn clone(&self) -> Self { 61 | let mut state = self.state_lock(); 62 | let index = state.wakers.insert(None); 63 | 64 | Self { 65 | state: Arc::clone(&self.state), 66 | index, 67 | } 68 | } 69 | } 70 | 71 | impl Drop for Shutdown { 72 | fn drop(&mut self) { 73 | let mut state = self.state_lock(); 74 | state.wakers.remove(self.index); 75 | } 76 | } 77 | 78 | impl Future for Shutdown { 79 | type Output = (); 80 | 81 | fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { 82 | let me = self.as_ref().get_ref(); 83 | let mut state = me.state_lock(); 84 | 85 | if state.shutdown { 86 | return Poll::Ready(()); 87 | } 88 | 89 | state.wakers[self.index] = Some(cx.waker().clone()); 90 | Poll::Pending 91 | } 92 | } 93 | 94 | #[derive(Debug)] 95 | struct State { 96 | shutdown: bool, 97 | wakers: Slab>, 98 | } 99 | -------------------------------------------------------------------------------- /shared/src/transports/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod grpc; 2 | pub mod quic; 3 | 4 | use { 5 | futures::stream::BoxStream, 6 | richat_proto::richat::RichatFilter, 7 | solana_sdk::clock::Slot, 8 | std::{ 9 | future::Future, 10 | io::{self, IoSlice}, 11 | pin::Pin, 12 | sync::Arc, 13 | task::{ready, Context, Poll}, 14 | }, 15 | thiserror::Error, 16 | tokio::io::AsyncWrite, 17 | }; 18 | 19 | pub type RecvItem = Arc>; 20 | 21 | pub type RecvStream = BoxStream<'static, Result>; 22 | 23 | #[derive(Debug, Clone, Copy, PartialEq, Eq, Error)] 24 | pub enum RecvError { 25 | #[error("channel lagged")] 26 | Lagged, 27 | #[error("channel closed")] 28 | Closed, 29 | } 30 | 31 | #[derive(Debug, Error)] 32 | pub enum SubscribeError { 33 | #[error("channel is not initialized yet")] 34 | NotInitialized, 35 | #[error("only available from slot {first_available}")] 36 | SlotNotAvailable { first_available: Slot }, 37 | } 38 | 39 | pub trait Subscribe { 40 | fn subscribe( 41 | &self, 42 | replay_from_slot: Option, 43 | filter: Option, 44 | ) -> Result; 45 | } 46 | 47 | #[derive(Debug)] 48 | pub struct WriteVectored<'a, W: ?Sized> { 49 | writer: &'a mut W, 50 | buffers: &'a mut [IoSlice<'a>], 51 | offset: usize, 52 | } 53 | 54 | impl<'a, W> WriteVectored<'a, W> { 55 | pub fn new(writer: &'a mut W, buffers: &'a mut [IoSlice<'a>]) -> Self { 56 | Self { 57 | writer, 58 | buffers, 59 | offset: 0, 60 | } 61 | } 62 | } 63 | 64 | impl Future for WriteVectored<'_, W> 65 | where 66 | W: AsyncWrite + Unpin + ?Sized, 67 | { 68 | type Output = io::Result<()>; 69 | 70 | fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { 71 | let me = unsafe { self.get_unchecked_mut() }; 72 | while me.offset < me.buffers.len() { 73 | let bufs = &me.buffers[me.offset..]; 74 | let mut n = ready!(Pin::new(&mut *me.writer).poll_write_vectored(cx, bufs))?; 75 | if n == 0 { 76 | return Poll::Ready(Err(io::ErrorKind::WriteZero.into())); 77 | } 78 | 79 | while n > 0 { 80 | if n >= me.buffers[me.offset].len() { 81 | n -= me.buffers[me.offset].len(); 82 | me.offset += 1; 83 | continue; 84 | } 85 | 86 | me.buffers[me.offset].advance(n); 87 | n = 0; 88 | } 89 | } 90 | Poll::Ready(Ok(())) 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /shared/src/version.rs: -------------------------------------------------------------------------------- 1 | use serde::{Deserialize, Serialize}; 2 | 3 | #[derive(Debug, Clone, Copy, Deserialize, Serialize)] 4 | pub struct Version<'a> { 5 | pub package: &'a str, 6 | pub version: &'a str, 7 | pub proto: &'a str, 8 | pub proto_richat: &'a str, 9 | pub solana: &'a str, 10 | pub git: &'a str, 11 | pub rustc: &'a str, 12 | pub buildts: &'a str, 13 | } 14 | 15 | impl<'a> Version<'a> { 16 | pub fn create_grpc_version_info(self) -> GrpcVersionInfo<'a> { 17 | GrpcVersionInfo::new(self) 18 | } 19 | } 20 | 21 | #[derive(Debug, Clone, Deserialize, Serialize)] 22 | pub struct GrpcVersionInfoExtra { 23 | hostname: Option, 24 | } 25 | 26 | #[derive(Debug, Clone, Deserialize, Serialize)] 27 | pub struct GrpcVersionInfo<'a> { 28 | #[serde(borrow)] 29 | version: Version<'a>, 30 | extra: GrpcVersionInfoExtra, 31 | } 32 | 33 | impl<'a> GrpcVersionInfo<'a> { 34 | pub fn new(version: Version<'a>) -> Self { 35 | Self { 36 | version, 37 | extra: GrpcVersionInfoExtra { 38 | hostname: hostname::get() 39 | .ok() 40 | .and_then(|name| name.into_string().ok()), 41 | }, 42 | } 43 | } 44 | 45 | pub fn json(&self) -> String { 46 | serde_json::to_string(self).expect("json serialization never fail") 47 | } 48 | 49 | pub fn value(&self) -> serde_json::Value { 50 | serde_json::to_value(self).expect("json serialization never fail") 51 | } 52 | } 53 | --------------------------------------------------------------------------------