├── .dockerignore ├── .gitattributes ├── .github ├── actions │ └── setup-rust │ │ └── action.yaml └── workflows │ ├── build.yml │ ├── release.yml │ └── test.yml ├── .gitignore ├── .gitmodules ├── Cargo.lock ├── Cargo.toml ├── Dockerfile ├── LICENSE ├── README.md ├── b ├── bins └── serialized_shreds.bin ├── examples ├── Cargo.toml ├── README.md └── deshred.rs ├── jito_protos ├── Cargo.toml ├── build.rs └── src │ └── lib.rs ├── p ├── proxy ├── Cargo.toml └── src │ ├── deshred.rs │ ├── forwarder.rs │ ├── heartbeat.rs │ ├── main.rs │ ├── server.rs │ └── token_authenticator.rs ├── rust-toolchain.toml ├── rustfmt.toml └── scripts ├── create_test_listeners.sh └── get_tvu_port.sh /.dockerignore: -------------------------------------------------------------------------------- 1 | *.json 2 | .dockerignore 3 | .git/ 4 | .github/ 5 | .gitignore 6 | .idea/ 7 | README.md 8 | b 9 | docker-compose.yml 10 | p 11 | target/ -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | *.bin filter=lfs diff=lfs merge=lfs -text 2 | -------------------------------------------------------------------------------- /.github/actions/setup-rust/action.yaml: -------------------------------------------------------------------------------- 1 | name: Setup Rust 2 | description: Installs rust in a bare metal fashion 3 | inputs: 4 | caller-workflow-name: 5 | description: 'Name of workflow used for creating a cache key in ASCII format.' 6 | required: true 7 | default: '' 8 | 9 | runs: 10 | using: composite 11 | steps: 12 | - name: Generate cache key 13 | id: cache-key-generator 14 | run: echo "cache-key=$(echo ${{inputs.caller-workflow-name}} | tr ' ' '_')" >> $GITHUB_OUTPUT 15 | shell: bash 16 | 17 | - name: Check cache key 18 | run: "echo Cache key: ${{ steps.cache-key-generator.outputs.cache-key }}" 19 | shell: bash 20 | 21 | - name: Install Protobuf 22 | run: | 23 | export PROTOC_VERSION=23.4 && \ 24 | export PROTOC_ZIP=protoc-$PROTOC_VERSION-linux-x86_64.zip && \ 25 | curl -Ss -OL https://github.com/google/protobuf/releases/download/v$PROTOC_VERSION/$PROTOC_ZIP \ 26 | && sudo unzip -o $PROTOC_ZIP -d /usr/local bin/protoc \ 27 | && sudo unzip -o $PROTOC_ZIP -d /usr/local include/* \ 28 | && rm -f $PROTOC_ZIP 29 | shell: bash 30 | 31 | - name: cache dependencies 32 | uses: actions/cache@v4 33 | with: 34 | path: | 35 | ~/.cargo 36 | key: ${{ runner.os }}-cargo-${{ steps.cache-key-generator.outputs.cache-key }}-${{ hashFiles('**/Cargo.lock') }} 37 | 38 | - name: cache rust files 39 | uses: actions/cache@v4 40 | with: 41 | path: | 42 | target/ 43 | key: ${{ runner.os }}-cargo-${{ steps.cache-key-generator.outputs.cache-key }}-${{ hashFiles('**/*.rs') }} 44 | 45 | - name: Check rust version 46 | run: | 47 | rustc --version || true; 48 | cargo --version || true; 49 | cargo clippy --version || true; 50 | cargo fmt --version || true; 51 | shell: bash 52 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: build 2 | 3 | on: 4 | pull_request: 5 | 6 | jobs: 7 | build: 8 | runs-on: ubuntu-22.04-16c-64g-public 9 | steps: 10 | - name: Checkout 11 | uses: actions/checkout@v4 12 | with: 13 | submodules: recursive 14 | lfs: true 15 | 16 | - name: Sanity Check 17 | run: | 18 | cat /proc/cpuinfo 19 | 20 | - name: Setup Rust 21 | uses: ./.github/actions/setup-rust 22 | with: 23 | caller-workflow-name: test 24 | 25 | - name: Clippy check 26 | run: cargo clippy --all-features --all-targets --tests -- -D warnings 27 | 28 | - name: Set up Docker Buildx 29 | uses: docker/setup-buildx-action@v3 30 | 31 | # see https://docs.docker.com/build/ci/github-actions/cache/#cache-backend-api 32 | - name: Login to Docker Hub 33 | uses: docker/login-action@v2 34 | with: 35 | username: ${{ secrets.DOCKERHUB_USER }} 36 | password: ${{ secrets.DOCKERHUB_PWD }} 37 | 38 | - name: Build and push 39 | uses: docker/build-push-action@v5 40 | with: 41 | context: . 42 | push: true 43 | tags: jitolabs/jito-shredstream-proxy:${{github.sha}} 44 | cache-from: type=gha 45 | cache-to: type=gha,mode=max 46 | platforms: linux/arm64,linux/x86_64 47 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: release 2 | 3 | on: 4 | push: 5 | # branches: 6 | # - master 7 | tags: 8 | - 'v*' 9 | 10 | jobs: 11 | release: 12 | runs-on: ubuntu-22.04-16c-64g-public 13 | steps: 14 | - name: Checkout 15 | uses: actions/checkout@v4 16 | with: 17 | submodules: recursive 18 | lfs: true 19 | 20 | - name: Sanity Check 21 | run: | 22 | cat /proc/cpuinfo 23 | 24 | - name: Setup Rust 25 | uses: ./.github/actions/setup-rust 26 | with: 27 | caller-workflow-name: test 28 | 29 | - name: Clippy check 30 | run: cargo clippy --all-features --all-targets --tests -- -D warnings 31 | 32 | - name: Set up Docker Buildx 33 | uses: docker/setup-buildx-action@v3 34 | 35 | # see https://docs.docker.com/build/ci/github-actions/cache/#cache-backend-api 36 | - name: Login to Docker Hub 37 | uses: docker/login-action@v2 38 | with: 39 | username: ${{ secrets.DOCKERHUB_USER }} 40 | password: ${{ secrets.DOCKERHUB_PWD }} 41 | 42 | - name: Build and push 43 | uses: docker/build-push-action@v4 44 | with: 45 | context: . 46 | push: true 47 | tags: jitolabs/jito-shredstream-proxy:${{github.ref_name}} 48 | cache-from: type=gha 49 | cache-to: type=gha,mode=max 50 | platforms: linux/arm64,linux/x86_64 51 | 52 | - name: Copy artifact from container 53 | run: | 54 | docker run --rm --platform linux/x86_64 --entrypoint cat jitolabs/jito-shredstream-proxy:${{github.ref_name}} /app/jito-shredstream-proxy > ./jito-shredstream-proxy-x86_64-unknown-linux-gnu 55 | ls -lh . 56 | file ./jito-shredstream-proxy-x86_64-unknown-linux-gnu 57 | 58 | - name: Release 59 | uses: softprops/action-gh-release@v2 60 | if: startsWith(github.ref, 'refs/tags/') 61 | with: 62 | files: | 63 | ./jito-shredstream-proxy-x86_64-unknown-linux-gnu 64 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: test 2 | 3 | on: 4 | pull_request: 5 | 6 | jobs: 7 | test: 8 | runs-on: ubuntu-22.04-16c-64g-public 9 | steps: 10 | - name: Checkout 11 | uses: actions/checkout@v4 12 | with: 13 | submodules: recursive 14 | lfs: true 15 | 16 | - name: Setup Rust 17 | uses: ./.github/actions/setup-rust 18 | with: 19 | caller-workflow-name: test 20 | 21 | - name: Run tests 22 | run: cargo test --all-features --locked 23 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Generated by Cargo 2 | # will have compiled files and executables 3 | **/target/ 4 | 5 | # These are backup files generated by rustfmt 6 | **/*.rs.bk 7 | 8 | .idea/ 9 | 10 | .env -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "jito_protos/protos"] 2 | path = jito_protos/protos 3 | url = https://github.com/jito-labs/mev-protos.git 4 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | members = [ 3 | "examples", 4 | "jito_protos", 5 | "proxy", 6 | ] 7 | resolver = "2" 8 | 9 | [workspace.package] 10 | version = "0.2.5" 11 | description = "Fast path to receive shreds from Jito, forwarding to local consumers. See https://jito-labs.gitbook.io/mev/searcher-services/shredstream for details." 12 | authors = ["Jito Team "] 13 | homepage = "https://jito.wtf/" 14 | edition = "2021" 15 | 16 | [profile.release] 17 | # thin has minimal overhead vs none (default): https://blog.llvm.org/2016/06/thinlto-scalable-and-incremental-lto.html 18 | lto = "thin" 19 | 20 | [workspace.dependencies] 21 | arc-swap = "1.6" 22 | clap = { version = "4", features = ["derive", "env"] } 23 | crossbeam-channel = "0.5.8" 24 | dashmap = "5" 25 | env_logger = "0.11" 26 | hostname = "0.4.0" 27 | itertools = "0.13.0" 28 | jito-protos = { path = "jito_protos" } 29 | log = "0.4" 30 | prost = "0.12" 31 | prost-types = "0.12" 32 | rand = "0.8" 33 | borsh = "1.5.3" 34 | bincode = "1.3.3" 35 | reqwest = { version = "0.11", features = ["blocking", "json"] } 36 | serde_json = "1" 37 | signal-hook = "0.3" 38 | solana-client = "=2.2.1" 39 | solana-metrics = "=2.2.1" 40 | solana-net-utils = "=2.2.1" 41 | solana-perf = "=2.2.1" 42 | solana-sdk = "=2.2.1" 43 | solana-streamer = "=2.2.1" 44 | solana-ledger = { git = "https://github.com/jito-foundation/jito-solana.git", branch = "eric/v2.2-merkle-recovery", package = "solana-ledger" } 45 | solana-entry = "=2.2.1" 46 | thiserror = "1" 47 | tokio = "1" 48 | tonic = { version = "0.10", features = ["tls", "tls-roots", "tls-webpki-roots"] } 49 | tonic-build = "0.10" 50 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # syntax=docker/dockerfile:1.4.0 2 | FROM --platform=linux/amd64 rust:1.84-slim-bullseye as builder 3 | 4 | RUN apt-get -qq update && apt-get install -qq -y ca-certificates libssl-dev protobuf-compiler pkg-config libudev-dev zlib1g-dev llvm clang cmake make libprotobuf-dev g++ 5 | RUN rustup component add rustfmt && update-ca-certificates 6 | 7 | ENV HOME=/home/root 8 | WORKDIR $HOME/app 9 | COPY . . 10 | 11 | # with buildkit, you need to copy the binary to the main folder 12 | # w/o buildkit, you can remove the cp 13 | RUN --mount=type=cache,mode=0777,target=/home/root/app/target \ 14 | --mount=type=cache,mode=0777,target=/usr/local/cargo/registry \ 15 | --mount=type=cache,mode=0777,target=/usr/local/cargo/git \ 16 | cargo build --release && cp target/release/jito-* ./ 17 | 18 | ################################################################################ 19 | FROM --platform=linux/amd64 debian:bullseye-slim as base_image 20 | RUN apt-get -qq update && apt-get install -qq -y ca-certificates libssl1.1 && rm -rf /var/lib/apt/lists/* 21 | 22 | ################################################################################ 23 | FROM base_image as shredstream_proxy 24 | ENV APP="jito-shredstream-proxy" 25 | 26 | WORKDIR /app 27 | # with buildkit, the binary is placed in the git root folder 28 | # w/o buildkit, the binary will be in target/release 29 | COPY --from=builder /home/root/app/${APP} ./ 30 | ENTRYPOINT ["/app/jito-shredstream-proxy"] 31 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright (c) 2025 Jito Labs 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Jito Shredstream Proxy 2 | 3 | ShredStream provides the lowest latency to shreds from leaders on Solana. 4 | 5 | See more at https://docs.jito.wtf/lowlatencytxnfeed/ 6 | 7 | ## Disclaimer 8 | Use this at your own risk. 9 | -------------------------------------------------------------------------------- /b: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -eux 3 | 4 | # Some container vars 5 | TAG=$(git describe --match=NeVeRmAtCh --always --abbrev=8 --dirty) 6 | ORG="jitolabs" 7 | 8 | DOCKER_BUILDKIT=1 docker build -t "$ORG/jito-shredstream-proxy:${TAG}" . 9 | 10 | docker run "$ORG/jito-shredstream-proxy:${TAG}" 11 | -------------------------------------------------------------------------------- /bins/serialized_shreds.bin: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:d6645ec7f298f5189659605819bf1eff705dcf5a5c674fe3eebfd3a1b2c4b14e 3 | size 61000624 4 | -------------------------------------------------------------------------------- /examples/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "examples" 3 | edition = "2021" 4 | publish = false 5 | 6 | [dependencies] 7 | bincode = { workspace = true } 8 | jito-protos = { path = "../jito_protos" } 9 | solana-entry = { workspace = true } 10 | tokio = { version = "1", features = ["full"] } 11 | 12 | [[example]] 13 | name = "deshred" 14 | path = "deshred.rs" -------------------------------------------------------------------------------- /examples/README.md: -------------------------------------------------------------------------------- 1 | # Deshred Example 2 | 3 | This example demonstrates how to consume transactions from 4 | the shredstream proxy in real-time. 5 | 6 | ## Prerequisites 7 | 8 | - Running shredstream proxy with flag: `--grpc-service-port 9999` 9 | 10 | ## Usage 11 | 12 | ```bash 13 | cargo run --example deshred 14 | ``` 15 | -------------------------------------------------------------------------------- /examples/deshred.rs: -------------------------------------------------------------------------------- 1 | use jito_protos::shredstream::{ 2 | shredstream_proxy_client::ShredstreamProxyClient, SubscribeEntriesRequest, 3 | }; 4 | 5 | #[tokio::main] 6 | async fn main() -> Result<(), std::io::Error> { 7 | let mut client = ShredstreamProxyClient::connect("http://127.0.0.1:9999") 8 | .await 9 | .unwrap(); 10 | let mut stream = client 11 | .subscribe_entries(SubscribeEntriesRequest {}) 12 | .await 13 | .unwrap() 14 | .into_inner(); 15 | 16 | while let Some(slot_entry) = stream.message().await.unwrap() { 17 | let entries = 18 | match bincode::deserialize::>(&slot_entry.entries) { 19 | Ok(e) => e, 20 | Err(e) => { 21 | println!("Deserialization failed with err: {e}"); 22 | continue; 23 | } 24 | }; 25 | println!( 26 | "slot {}, entries: {}, transactions: {}", 27 | slot_entry.slot, 28 | entries.len(), 29 | entries.iter().map(|e| e.transactions.len()).sum::() 30 | ); 31 | } 32 | Ok(()) 33 | } 34 | -------------------------------------------------------------------------------- /jito_protos/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "jito-protos" 3 | version = { workspace = true } 4 | description = "Protobufs for working with block engine" 5 | authors = { workspace = true } 6 | homepage = { workspace = true } 7 | edition = { workspace = true } 8 | publish = false 9 | 10 | [dependencies] 11 | prost = { workspace = true } 12 | prost-types = { workspace = true } 13 | tonic = { workspace = true } 14 | 15 | [build-dependencies] 16 | protobuf-src = "1" 17 | tonic-build = { workspace = true } 18 | -------------------------------------------------------------------------------- /jito_protos/build.rs: -------------------------------------------------------------------------------- 1 | use tonic_build::configure; 2 | 3 | fn main() { 4 | const PROTOC_ENVAR: &str = "PROTOC"; 5 | if std::env::var(PROTOC_ENVAR).is_err() { 6 | #[cfg(not(windows))] 7 | std::env::set_var(PROTOC_ENVAR, protobuf_src::protoc()); 8 | } 9 | 10 | configure() 11 | .compile( 12 | &[ 13 | "protos/auth.proto", 14 | "protos/shared.proto", 15 | "protos/shredstream.proto", 16 | ], 17 | &["protos"], 18 | ) 19 | .unwrap(); 20 | } 21 | -------------------------------------------------------------------------------- /jito_protos/src/lib.rs: -------------------------------------------------------------------------------- 1 | pub mod shared { 2 | tonic::include_proto!("shared"); 3 | } 4 | 5 | pub mod auth { 6 | tonic::include_proto!("auth"); 7 | } 8 | 9 | pub mod shredstream { 10 | tonic::include_proto!("shredstream"); 11 | } 12 | -------------------------------------------------------------------------------- /p: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -eux 3 | 4 | # Some container vars 5 | TAG=${TAG:-${USER}-dev} # READ tag in from env var, defaulting to ${USER}-dev 6 | ORG="jitolabs" 7 | 8 | DOCKER_BUILDKIT=1 docker build -t "$ORG/jito-shredstream-proxy:${TAG}" . 9 | 10 | docker push "${ORG}/jito-shredstream-proxy:${TAG}" 11 | 12 | # deploy 13 | #VERSION=v0.2.0 14 | #git tag $VERSION -f; git push --tags -f 15 | #docker tag jitolabs/jito-shredstream-proxy:$VERSION jitolabs/shredstream-proxy:latest 16 | #docker push jitolabs/shredstream-proxy:latest -------------------------------------------------------------------------------- /proxy/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "jito-shredstream-proxy" 3 | version = { workspace = true } 4 | description = "Fast path to receive shreds from Jito, forwarding to local consumers. See https://docs.jito.wtf/lowlatencytxnfeed/ for details." 5 | authors = { workspace = true } 6 | homepage = { workspace = true } 7 | edition = { workspace = true } 8 | 9 | [dependencies] 10 | arc-swap = { workspace = true } 11 | bincode = { workspace = true } 12 | borsh = { workspace = true } 13 | clap = { workspace = true } 14 | crossbeam-channel = { workspace = true } 15 | dashmap = { workspace = true } 16 | env_logger = { workspace = true } 17 | hostname = { workspace = true } 18 | itertools = { workspace = true } 19 | jito-protos = { workspace = true } 20 | log = { workspace = true } 21 | prost = { workspace = true } 22 | prost-types = { workspace = true } 23 | rand = { workspace = true } 24 | reqwest = { workspace = true } 25 | serde_json = { workspace = true } 26 | signal-hook = { workspace = true } 27 | solana-client = { workspace = true } 28 | solana-entry = { workspace = true } 29 | solana-ledger = { workspace = true } 30 | solana-metrics = { workspace = true } 31 | solana-net-utils = { workspace = true } 32 | solana-perf = { workspace = true } 33 | solana-sdk = { workspace = true } 34 | solana-streamer = { workspace = true } 35 | thiserror = { workspace = true } 36 | tokio = { workspace = true } 37 | tonic = { workspace = true } 38 | -------------------------------------------------------------------------------- /proxy/src/deshred.rs: -------------------------------------------------------------------------------- 1 | use std::{ 2 | collections::{HashMap, HashSet}, 3 | hash::Hash, 4 | sync::atomic::Ordering, 5 | }; 6 | 7 | use itertools::Itertools; 8 | use jito_protos::shredstream::TraceShred; 9 | use log::{debug, warn}; 10 | use prost::Message; 11 | use solana_ledger::shred::{ 12 | merkle::{Shred, ShredCode}, 13 | ReedSolomonCache, Shredder, 14 | }; 15 | use solana_sdk::clock::{Slot, MAX_PROCESSING_AGE}; 16 | 17 | use crate::forwarder::ShredMetrics; 18 | 19 | /// Returns the number of shreds reconstructed 20 | /// Updates all_shreds with current state, and deshredded_entries with returned values 21 | pub fn reconstruct_shreds<'a, I: Iterator>( 22 | packet_batch_vec: I, 23 | all_shreds: &mut HashMap< 24 | Slot, 25 | HashMap)>, 26 | >, 27 | deshredded_entries: &mut Vec<(Slot, Vec, Vec)>, 28 | rs_cache: &ReedSolomonCache, 29 | metrics: &ShredMetrics, 30 | ) -> usize { 31 | deshredded_entries.clear(); 32 | let mut slot_fec_index_to_iterate = HashSet::new(); 33 | for data in packet_batch_vec { 34 | match solana_ledger::shred::Shred::new_from_serialized_shred(data.to_vec()) 35 | .and_then(Shred::try_from) 36 | { 37 | Ok(shred) => { 38 | let slot = shred.common_header().slot; 39 | let fec_set_index = shred.fec_set_index(); 40 | all_shreds 41 | .entry(slot) 42 | .or_default() 43 | .entry(fec_set_index) 44 | .or_default() 45 | .1 46 | .insert(ComparableShred(shred)); 47 | slot_fec_index_to_iterate.insert((slot, fec_set_index)); 48 | } 49 | Err(e) => { 50 | if TraceShred::decode(data).is_ok() { 51 | continue; 52 | } 53 | warn!("Failed to decode shred. Err: {e:?}"); 54 | } 55 | } 56 | } 57 | 58 | let mut recovered_count = 0; 59 | let mut highest_slot_seen = 0; 60 | for (slot, fec_set_index) in slot_fec_index_to_iterate { 61 | highest_slot_seen = highest_slot_seen.max(slot); 62 | let Some((already_deshredded, shreds)) = all_shreds 63 | .get_mut(&slot) 64 | .and_then(|fec_set_indexes| fec_set_indexes.get_mut(&fec_set_index)) 65 | else { 66 | continue; 67 | }; 68 | if *already_deshredded { 69 | debug!("already completed slot {slot}"); 70 | continue; 71 | } 72 | 73 | let (num_expected_data_shreds, num_data_shreds) = can_recover(shreds); 74 | 75 | // haven't received last data shred, haven't seen any coding shreds, so wait until more arrive 76 | if num_expected_data_shreds == 0 77 | || (num_data_shreds < num_expected_data_shreds 78 | && shreds.len() < num_data_shreds as usize) 79 | { 80 | // not enough data shreds, not enough shreds to recover 81 | continue; 82 | } 83 | 84 | // try to recover if we have enough coding shreds 85 | let mut recovered_shreds = Vec::new(); 86 | if num_data_shreds < num_expected_data_shreds 87 | && shreds.len() as u16 >= num_expected_data_shreds 88 | { 89 | let merkle_shreds = shreds 90 | .iter() 91 | .sorted_by_key(|s| (u8::MAX - s.shred_type() as u8, s.index())) 92 | .map(|s| s.0.clone()) 93 | .collect_vec(); 94 | let recovered = match solana_ledger::shred::merkle::recover(merkle_shreds, rs_cache) { 95 | Ok(r) => r, 96 | Err(e) => { 97 | warn!("Failed to recover shreds for slot: {slot}, fec set: {fec_set_index}. Err: {e}"); 98 | continue; 99 | } 100 | }; 101 | 102 | for shred in recovered { 103 | match shred { 104 | Ok(shred) => { 105 | recovered_count += 1; 106 | recovered_shreds.push(ComparableShred(shred)); 107 | // can also just insert into hashmap, but kept separate for ease of debug 108 | } 109 | Err(e) => warn!( 110 | "Failed to recover shred for slot {slot}, fec set: {fec_set_index}. Err: {e}" 111 | ), 112 | } 113 | } 114 | } 115 | 116 | let sorted_deduped_data_payloads = shreds 117 | .iter() 118 | .chain(recovered_shreds.iter()) 119 | .filter_map(|s| Some((s, solana_ledger::shred::layout::get_data(s.payload()).ok()?))) 120 | .sorted_by_key(|(s, _data)| s.index()) 121 | .collect_vec(); 122 | 123 | if (sorted_deduped_data_payloads.len() as u16) < num_expected_data_shreds { 124 | continue; 125 | } 126 | 127 | let deshred_payload = match Shredder::deshred_unchecked( 128 | sorted_deduped_data_payloads 129 | .iter() 130 | .map(|(s, _data)| s.payload()), 131 | ) { 132 | Ok(v) => v, 133 | Err(e) => { 134 | warn!( 135 | "slot {slot} failed to deshred fec_set_index {fec_set_index}. num_expected_data_shreds: {num_expected_data_shreds}, num_data_shreds: {num_data_shreds}. shred set len: {}, recovered shred set len: {}, Err: {e}.", 136 | shreds.len(), 137 | recovered_shreds.len(), 138 | ); 139 | metrics 140 | .fec_recovery_error_count 141 | .fetch_add(1, Ordering::Relaxed); 142 | continue; 143 | } 144 | }; 145 | let entries = match bincode::deserialize::>( 146 | &deshred_payload, 147 | ) { 148 | Ok(e) => e, 149 | Err(e) => { 150 | warn!( 151 | "slot {slot} fec_set_index {fec_set_index} failed to deserialize bincode payload of size {}. Err: {e}", 152 | deshred_payload.len() 153 | ); 154 | metrics 155 | .bincode_deserialize_error_count 156 | .fetch_add(1, Ordering::Relaxed); 157 | continue; 158 | } 159 | }; 160 | metrics 161 | .entry_count 162 | .fetch_add(entries.len() as u64, Ordering::Relaxed); 163 | let txn_count = entries.iter().map(|e| e.transactions.len() as u64).sum(); 164 | metrics.txn_count.fetch_add(txn_count, Ordering::Relaxed); 165 | debug!( 166 | "Successfully decoded slot: {slot} fec_index: {fec_set_index} with entry count: {}, txn count: {txn_count}", 167 | entries.len(), 168 | ); 169 | 170 | deshredded_entries.push((slot, entries, deshred_payload)); 171 | if let Some(fec_set) = all_shreds.get_mut(&slot) { 172 | // done with this fec set index 173 | let _ = fec_set 174 | .get_mut(&fec_set_index) 175 | .map(|(is_completed, fec_set_shreds)| { 176 | *is_completed = true; 177 | fec_set_shreds.clear(); 178 | }); 179 | } 180 | } 181 | 182 | if all_shreds.len() > MAX_PROCESSING_AGE && highest_slot_seen > SLOT_LOOKBACK { 183 | let threshold = highest_slot_seen - SLOT_LOOKBACK; 184 | all_shreds.retain(|slot, _fec_set_index| *slot >= threshold); 185 | } 186 | 187 | if recovered_count > 0 { 188 | metrics 189 | .recovered_count 190 | .fetch_add(recovered_count as u64, Ordering::Relaxed); 191 | } 192 | 193 | recovered_count 194 | } 195 | 196 | const SLOT_LOOKBACK: Slot = 50; 197 | 198 | /// check if we can reconstruct (having minimum number of data + coding shreds) 199 | fn can_recover( 200 | shreds: &HashSet, 201 | ) -> ( 202 | u16, /* num_expected_data_shreds */ 203 | u16, /* num_data_shreds */ 204 | ) { 205 | let mut num_expected_data_shreds = 0; 206 | let mut data_shred_count = 0; 207 | for shred in shreds { 208 | match &shred.0 { 209 | Shred::ShredCode(s) => { 210 | num_expected_data_shreds = s.coding_header.num_data_shreds; 211 | } 212 | Shred::ShredData(s) => { 213 | data_shred_count += 1; 214 | if num_expected_data_shreds == 0 && (s.data_complete() || s.last_in_slot()) { 215 | num_expected_data_shreds = 216 | (shred.0.index() - shred.0.fec_set_index()) as u16 + 1; 217 | } 218 | } 219 | } 220 | } 221 | (num_expected_data_shreds, data_shred_count) 222 | } 223 | 224 | /// Issue: datashred equality comparison is wrong due to data size being smaller than the 1203 bytes allocated 225 | #[derive(Clone, Debug, Eq)] 226 | pub struct ComparableShred(Shred); 227 | 228 | impl std::ops::Deref for ComparableShred { 229 | type Target = Shred; 230 | 231 | fn deref(&self) -> &Self::Target { 232 | &self.0 233 | } 234 | } 235 | 236 | impl Hash for ComparableShred { 237 | fn hash(&self, state: &mut H) { 238 | match &self.0 { 239 | Shred::ShredCode(s) => { 240 | s.common_header.hash(state); 241 | s.coding_header.hash(state); 242 | } 243 | Shred::ShredData(s) => { 244 | s.common_header.hash(state); 245 | s.data_header.hash(state); 246 | } 247 | } 248 | } 249 | } 250 | 251 | impl PartialEq for ComparableShred { 252 | // Custom comparison to avoid random bytes that are part of payload 253 | fn eq(&self, other: &Self) -> bool { 254 | match &self.0 { 255 | Shred::ShredCode(s1) => match &other.0 { 256 | Shred::ShredCode(s2) => { 257 | let solana_ledger::shred::ShredVariant::MerkleCode { 258 | proof_size, 259 | chained: _, 260 | resigned, 261 | } = s1.common_header.shred_variant 262 | else { 263 | return false; 264 | }; 265 | 266 | // see https://github.com/jito-foundation/jito-solana/blob/d6c73374e3b4f863436e4b7d4d1ce5eea01cd262/ledger/src/shred/merkle.rs#L346, and re-add the proof component 267 | let comparison_len = 268 | ::SIZE_OF_PAYLOAD 269 | .saturating_sub( 270 | usize::from(proof_size) 271 | * solana_ledger::shred::merkle::SIZE_OF_MERKLE_PROOF_ENTRY 272 | + if resigned { 273 | solana_ledger::shred::SIZE_OF_SIGNATURE 274 | } else { 275 | 0 276 | }, 277 | ); 278 | 279 | s1.coding_header == s2.coding_header 280 | && s1.common_header == s2.common_header 281 | && s1.payload[..comparison_len] == s2.payload[..comparison_len] 282 | } 283 | Shred::ShredData(_) => false, 284 | }, 285 | Shred::ShredData(s1) => match &other.0 { 286 | Shred::ShredCode(_) => false, 287 | Shred::ShredData(s2) => { 288 | let Ok(s1_data) = solana_ledger::shred::layout::get_data(self.payload()) else { 289 | return false; 290 | }; 291 | let Ok(s2_data) = solana_ledger::shred::layout::get_data(other.payload()) 292 | else { 293 | return false; 294 | }; 295 | s1.data_header == s2.data_header 296 | && s1.common_header == s2.common_header 297 | && s1_data == s2_data 298 | } 299 | }, 300 | } 301 | } 302 | } 303 | #[cfg(test)] 304 | mod tests { 305 | use std::{ 306 | collections::{hash_map, HashMap, HashSet}, 307 | io::{Read, Write}, 308 | net::UdpSocket, 309 | sync::Arc, 310 | }; 311 | 312 | use borsh::BorshDeserialize; 313 | use itertools::Itertools; 314 | use rand::Rng; 315 | use solana_ledger::{ 316 | blockstore::make_slot_entries_with_transactions, 317 | shred::{merkle::Shred, ProcessShredsStats, ReedSolomonCache, ShredCommonHeader, Shredder}, 318 | }; 319 | use solana_perf::packet::Packet; 320 | use solana_sdk::{clock::Slot, hash::Hash, signature::Keypair}; 321 | 322 | use crate::{ 323 | deshred::{reconstruct_shreds, ComparableShred}, 324 | forwarder::ShredMetrics, 325 | }; 326 | 327 | /// For serializing packets to disk 328 | #[derive(borsh::BorshSerialize, borsh::BorshDeserialize, PartialEq, Debug)] 329 | struct Packets { 330 | pub packets: Vec>, 331 | } 332 | 333 | #[allow(unused)] 334 | fn listen_and_write_shreds() -> std::io::Result<()> { 335 | let socket = UdpSocket::bind("127.0.0.1:5000")?; 336 | println!("Listening on {}", socket.local_addr()?); 337 | 338 | let mut map = HashMap::::new(); 339 | let mut buf = [0u8; 1500]; 340 | let mut vec = Packets { 341 | packets: Vec::new(), 342 | }; 343 | 344 | let mut i = 0; 345 | loop { 346 | i += 1; 347 | match socket.recv_from(&mut buf) { 348 | Ok((amt, _src)) => { 349 | vec.packets.push(buf[..amt].to_vec()); 350 | match map.entry(amt) { 351 | hash_map::Entry::Occupied(mut e) => { 352 | *e.get_mut() += 1; 353 | } 354 | hash_map::Entry::Vacant(e) => { 355 | e.insert(1); 356 | } 357 | } 358 | *map.get_mut(&amt).unwrap_or(&mut 0) += 1; 359 | } 360 | Err(e) => { 361 | eprintln!("Error receiving data: {}", e); 362 | } 363 | } 364 | if i % 50000 == 0 { 365 | dbg!(&map); 366 | // size 1203 are data shreds: https://github.com/jito-foundation/jito-solana/blob/1742826fca975bd6d17daa5693abda861bbd2adf/ledger/src/shred/merkle.rs#L42 367 | // size 1228 are coding shreds: https://github.com/jito-foundation/jito-solana/blob/1742826fca975bd6d17daa5693abda861bbd2adf/ledger/src/shred/shred_code.rs#L16 368 | let mut file = std::fs::File::create("serialized_shreds.bin")?; 369 | file.write_all(&borsh::to_vec(&vec)?)?; 370 | return Ok(()); 371 | } 372 | } 373 | } 374 | 375 | #[test] 376 | fn test_reconstruct_live_shreds() { 377 | let packets = { 378 | let mut file = std::fs::File::open("../bins/serialized_shreds.bin").unwrap(); 379 | let mut buffer = Vec::new(); 380 | file.read_to_end(&mut buffer).unwrap(); 381 | Packets::try_from_slice(&buffer).unwrap() 382 | }; 383 | assert_eq!(packets.packets.len(), 50_000); 384 | 385 | let shreds = packets 386 | .packets 387 | .iter() 388 | .filter_map(|p| Shred::from_payload(p.clone()).ok()) 389 | .collect::>(); 390 | assert_eq!(shreds.len(), 49989); 391 | 392 | let unique_shreds = packets 393 | .packets 394 | .iter() 395 | .filter_map(|p| Shred::from_payload(p.clone()).ok().map(ComparableShred)) 396 | .collect::>(); 397 | assert_eq!(unique_shreds.len(), 44900); 398 | 399 | let unique_slot_fec_shreds = packets 400 | .packets 401 | .iter() 402 | .filter_map(|p| { 403 | Shred::from_payload(p.clone()) 404 | .ok() 405 | .map(|s| *s.common_header()) 406 | }) 407 | .collect::>(); 408 | assert_eq!(unique_slot_fec_shreds.len(), 44900); 409 | 410 | let rs_cache = ReedSolomonCache::default(); 411 | let metrics = Arc::new(ShredMetrics::default()); 412 | 413 | // Test 1: all shreds provided 414 | let mut deshredded_entries = Vec::new(); 415 | let mut all_shreds: HashMap< 416 | Slot, 417 | HashMap)>, 418 | > = HashMap::new(); 419 | let recovered_count = reconstruct_shreds( 420 | shreds.iter().map(|shred| shred.payload().iter().as_slice()), 421 | &mut all_shreds, 422 | &mut deshredded_entries, 423 | &rs_cache, 424 | &metrics, 425 | ); 426 | assert!(recovered_count < deshredded_entries.len()); 427 | assert_eq!( 428 | deshredded_entries 429 | .iter() 430 | .map(|(_slot, entries, _entries_bytes)| entries.len()) 431 | .sum::(), 432 | 13580 433 | ); 434 | assert_eq!(all_shreds.len(), 30); 435 | 436 | let slot_to_entry = deshredded_entries 437 | .iter() 438 | .into_group_map_by(|(slot, _entries, _entries_bytes)| *slot); 439 | // slot_to_entry 440 | // .iter() 441 | // .sorted_by_key(|(slot, _)| *slot) 442 | // .for_each(|(slot, entry)| { 443 | // println!( 444 | // "slot {slot} entry count: {:?}, txn count: {}", 445 | // entry.len(), 446 | // entry 447 | // .iter() 448 | // .map(|(_slot, entry)| entry.transactions.len()) 449 | // .sum::() 450 | // ); 451 | // }); 452 | assert_eq!(slot_to_entry.len(), 29); 453 | 454 | // Test 2: 33% of shreds missing 455 | let mut deshredded_entries = Vec::new(); 456 | let mut all_shreds: HashMap< 457 | Slot, 458 | HashMap)>, 459 | > = HashMap::new(); 460 | let recovered_count = reconstruct_shreds( 461 | shreds 462 | .iter() 463 | .enumerate() 464 | .filter(|(index, _)| (index + 1) % 3 != 0) 465 | .map(|(_, shred)| shred.payload().iter().as_slice()), 466 | &mut all_shreds, 467 | &mut deshredded_entries, 468 | &rs_cache, 469 | &metrics, 470 | ); 471 | 472 | assert!(recovered_count > (deshredded_entries.len() / 4)); 473 | assert_eq!( 474 | deshredded_entries 475 | .iter() 476 | .map(|(_slot, entries, _entries_bytes)| entries.len()) 477 | .sum::(), 478 | 13580 479 | ); 480 | assert!(all_shreds.len() > 15); 481 | 482 | let slot_to_entry = deshredded_entries 483 | .iter() 484 | .into_group_map_by(|(slot, _entries, _entries_bytes)| *slot); 485 | assert_eq!(slot_to_entry.len(), 29); 486 | } 487 | #[test] 488 | fn test_recover_shreds() { 489 | let mut rng = rand::thread_rng(); 490 | let slot = 11_111; 491 | let leader_keypair = Arc::new(Keypair::new()); 492 | let reed_solomon_cache = ReedSolomonCache::default(); 493 | let shredder = Shredder::new(slot, slot - 1, 0, 0).unwrap(); 494 | let chained_merkle_root = Some(Hash::new_from_array(rng.gen())); 495 | let num_entry_groups = 10; 496 | let num_entries = 10; 497 | let mut entries = Vec::new(); 498 | let mut data_shreds = Vec::new(); 499 | let mut coding_shreds = Vec::new(); 500 | 501 | let mut index = 0; 502 | (0..num_entry_groups).for_each(|_i| { 503 | let _entries = make_slot_entries_with_transactions(num_entries); 504 | let (_data_shreds, _coding_shreds) = shredder.entries_to_shreds( 505 | &leader_keypair, 506 | _entries.as_slice(), 507 | true, 508 | chained_merkle_root, 509 | index as u32, // next_shred_index 510 | index as u32, // next_code_index, 511 | true, // merkle_variant 512 | &reed_solomon_cache, 513 | &mut ProcessShredsStats::default(), 514 | ); 515 | index += _data_shreds.len(); 516 | entries.extend(_entries); 517 | data_shreds.extend(_data_shreds); 518 | coding_shreds.extend(_coding_shreds); 519 | }); 520 | 521 | let packets = data_shreds 522 | .iter() 523 | .chain(coding_shreds.iter()) 524 | .map(|s| { 525 | let mut p = Packet::default(); 526 | s.copy_to_packet(&mut p); 527 | p 528 | }) 529 | .collect_vec(); 530 | assert_eq!(data_shreds.len(), 320); 531 | assert_eq!( 532 | data_shreds 533 | .iter() 534 | .map(|s| s.fec_set_index()) 535 | .dedup() 536 | .count(), 537 | num_entry_groups 538 | ); 539 | 540 | let metrics = Arc::new(ShredMetrics::default()); 541 | let rs_cache = ReedSolomonCache::default(); 542 | 543 | // Test 1: all shreds provided 544 | let mut deshredded_entries = Vec::new(); 545 | let mut all_shreds: HashMap< 546 | Slot, 547 | HashMap)>, 548 | > = HashMap::new(); 549 | let recovered_count = reconstruct_shreds( 550 | packets.iter().map(|s| s.data(..).unwrap()), 551 | &mut all_shreds, 552 | &mut deshredded_entries, 553 | &rs_cache, 554 | &metrics, 555 | ); 556 | assert_eq!(recovered_count, 0); 557 | assert_eq!( 558 | deshredded_entries 559 | .iter() 560 | .map(|(_slot, entries, _entries_bytes)| entries.len()) 561 | .sum::(), 562 | entries.len() 563 | ); 564 | assert_eq!( 565 | all_shreds.len(), 566 | 1, // slot 11111 567 | ); 568 | 569 | // Test 2: 33% of shreds missing 570 | let mut deshredded_entries = Vec::new(); 571 | let mut all_shreds: HashMap< 572 | Slot, 573 | HashMap)>, 574 | > = HashMap::new(); 575 | let recovered_count = reconstruct_shreds( 576 | packets 577 | .iter() 578 | .enumerate() 579 | .filter(|(index, _)| (index + 1) % 3 != 0) 580 | .map(|(_, s)| s.data(..).unwrap()), 581 | &mut all_shreds, 582 | &mut deshredded_entries, 583 | &rs_cache, 584 | &metrics, 585 | ); 586 | assert!(recovered_count > 0); 587 | assert_eq!( 588 | deshredded_entries 589 | .iter() 590 | .map(|(_slot, entries, _entries_bytes)| entries.len()) 591 | .sum::(), 592 | entries.len() 593 | ); 594 | assert_eq!( 595 | all_shreds.len(), 596 | 1, // slot 11111 597 | ); 598 | } 599 | } 600 | -------------------------------------------------------------------------------- /proxy/src/forwarder.rs: -------------------------------------------------------------------------------- 1 | use std::{ 2 | collections::{HashMap, HashSet}, 3 | net::{IpAddr, Ipv6Addr, SocketAddr, UdpSocket}, 4 | sync::{ 5 | atomic::{AtomicBool, AtomicU64, Ordering}, 6 | Arc, RwLock, 7 | }, 8 | thread::{Builder, JoinHandle}, 9 | time::{Duration, SystemTime}, 10 | }; 11 | 12 | use arc_swap::ArcSwap; 13 | use crossbeam_channel::{Receiver, RecvError}; 14 | use dashmap::DashMap; 15 | use itertools::Itertools; 16 | use jito_protos::shredstream::{Entry as PbEntry, TraceShred}; 17 | use log::{debug, error, info, warn}; 18 | use prost::Message; 19 | use solana_ledger::shred::ReedSolomonCache; 20 | use solana_metrics::{datapoint_info, datapoint_warn}; 21 | use solana_net_utils::SocketConfig; 22 | use solana_perf::{ 23 | deduper::Deduper, 24 | packet::{PacketBatch, PacketBatchRecycler}, 25 | recycler::Recycler, 26 | }; 27 | use solana_sdk::clock::{Slot, MAX_PROCESSING_AGE}; 28 | use solana_streamer::{ 29 | sendmmsg::{batch_send, SendPktsError}, 30 | streamer::{self, StreamerReceiveStats}, 31 | }; 32 | use tokio::sync::broadcast::Sender; 33 | 34 | use crate::{deshred, deshred::ComparableShred, resolve_hostname_port, ShredstreamProxyError}; 35 | 36 | // values copied from https://github.com/solana-labs/solana/blob/33bde55bbdde13003acf45bb6afe6db4ab599ae4/core/src/sigverify_shreds.rs#L20 37 | pub const DEDUPER_FALSE_POSITIVE_RATE: f64 = 0.001; 38 | pub const DEDUPER_NUM_BITS: u64 = 637_534_199; // 76MB 39 | pub const DEDUPER_RESET_CYCLE: Duration = Duration::from_secs(5 * 60); 40 | 41 | /// Bind to ports and start forwarding shreds 42 | #[allow(clippy::too_many_arguments)] 43 | pub fn start_forwarder_threads( 44 | unioned_dest_sockets: Arc>>, /* sockets shared between endpoint discovery thread and forwarders */ 45 | src_addr: IpAddr, 46 | src_port: u16, 47 | num_threads: Option, 48 | deduper: Arc>>, 49 | should_reconstruct_shreds: bool, 50 | entry_sender: Arc>, 51 | debug_trace_shred: bool, 52 | use_discovery_service: bool, 53 | forward_stats: Arc, 54 | metrics: Arc, 55 | shutdown_receiver: Receiver<()>, 56 | exit: Arc, 57 | ) -> Vec> { 58 | let num_threads = num_threads 59 | .unwrap_or_else(|| usize::from(std::thread::available_parallelism().unwrap()).min(4)); 60 | 61 | let recycler: PacketBatchRecycler = Recycler::warmed(100, 1024); 62 | 63 | // multi_bind_in_range returns (port, Vec) 64 | let sockets = solana_net_utils::multi_bind_in_range_with_config( 65 | src_addr, 66 | (src_port, src_port + 1), 67 | SocketConfig::default().reuseport(true), 68 | num_threads, 69 | ) 70 | .unwrap_or_else(|_| { 71 | panic!("Failed to bind listener sockets. Check that port {src_port} is not in use.") 72 | }); 73 | 74 | sockets 75 | .1 76 | .into_iter() 77 | .enumerate() 78 | .flat_map(|(thread_id, incoming_shred_socket)| { 79 | let (packet_sender, packet_receiver) = crossbeam_channel::unbounded(); 80 | let listen_thread = streamer::receiver( 81 | format!("ssListen{thread_id}"), 82 | Arc::new(incoming_shred_socket), 83 | exit.clone(), 84 | packet_sender, 85 | recycler.clone(), 86 | forward_stats.clone(), 87 | Duration::default(), 88 | false, 89 | None, 90 | false, 91 | ); 92 | 93 | let deduper = deduper.clone(); 94 | let unioned_dest_sockets = unioned_dest_sockets.clone(); 95 | let metrics = metrics.clone(); 96 | let shutdown_receiver = shutdown_receiver.clone(); 97 | let mut deshredded_entries = Vec::new(); 98 | let rs_cache = ReedSolomonCache::default(); 99 | let entry_sender = entry_sender.clone(); 100 | let exit = exit.clone(); 101 | 102 | let send_thread = Builder::new() 103 | .name(format!("ssPxyTx_{thread_id}")) 104 | .spawn(move || { 105 | let send_socket = 106 | UdpSocket::bind(SocketAddr::new(IpAddr::V6(Ipv6Addr::UNSPECIFIED), 0)) 107 | .expect("to bind to udp port for forwarding"); 108 | let mut local_dest_sockets = unioned_dest_sockets.load(); 109 | 110 | let refresh_subscribers_tick = if use_discovery_service { 111 | crossbeam_channel::tick(Duration::from_secs(30)) 112 | } else { 113 | crossbeam_channel::tick(Duration::MAX) 114 | }; 115 | 116 | // Track parsed Shred as reconstructed_shreds[ slot ][ fec_set_index ] -> Vec 117 | let mut all_shreds: HashMap< 118 | Slot, 119 | HashMap< 120 | u32, /* fec_set_index */ 121 | (bool /* completed */, HashSet), 122 | >, 123 | > = HashMap::with_capacity(MAX_PROCESSING_AGE); 124 | 125 | while !exit.load(Ordering::Relaxed) { 126 | crossbeam_channel::select! { 127 | // forward packets 128 | recv(packet_receiver) -> maybe_packet_batch => { 129 | let res = recv_from_channel_and_send_multiple_dest( 130 | maybe_packet_batch, 131 | &deduper, 132 | &mut all_shreds, 133 | &mut deshredded_entries, 134 | &rs_cache, 135 | &send_socket, 136 | &local_dest_sockets, 137 | should_reconstruct_shreds, 138 | &entry_sender, 139 | debug_trace_shred, 140 | &metrics, 141 | ); 142 | 143 | // If the channel is closed or error, break out 144 | if res.is_err() { 145 | break; 146 | } 147 | } 148 | 149 | // refresh thread-local subscribers 150 | recv(refresh_subscribers_tick) -> _ => { 151 | local_dest_sockets = unioned_dest_sockets.load(); 152 | } 153 | 154 | // handle shutdown (avoid using sleep since it can hang) 155 | recv(shutdown_receiver) -> _ => { 156 | break; 157 | } 158 | } 159 | } 160 | info!("Exiting forwarder thread {thread_id}."); 161 | }) 162 | .unwrap(); 163 | 164 | vec![listen_thread, send_thread] 165 | }) 166 | .collect::>>() 167 | } 168 | 169 | /// Broadcasts the same packet to multiple recipients, parses it into a Shred if possible, 170 | /// and stores that shred in `all_shreds`. 171 | #[allow(clippy::too_many_arguments)] 172 | fn recv_from_channel_and_send_multiple_dest( 173 | maybe_packet_batch: Result, 174 | deduper: &RwLock>, 175 | all_shreds: &mut HashMap< 176 | Slot, 177 | HashMap)>, 178 | >, 179 | deshredded_entries: &mut Vec<(Slot, Vec, Vec)>, 180 | rs_cache: &ReedSolomonCache, 181 | send_socket: &UdpSocket, 182 | local_dest_sockets: &[SocketAddr], 183 | should_reconstruct_shreds: bool, 184 | entry_sender: &Sender, 185 | debug_trace_shred: bool, 186 | metrics: &ShredMetrics, 187 | ) -> Result<(), ShredstreamProxyError> { 188 | let packet_batch = maybe_packet_batch.map_err(ShredstreamProxyError::RecvError)?; 189 | let trace_shred_received_time = SystemTime::now(); 190 | metrics 191 | .received 192 | .fetch_add(packet_batch.len() as u64, Ordering::Relaxed); 193 | debug!( 194 | "Got batch of {} packets, total size in bytes: {}", 195 | packet_batch.len(), 196 | packet_batch.iter().map(|x| x.meta().size).sum::() 197 | ); 198 | 199 | let mut packet_batch_vec = vec![packet_batch]; 200 | 201 | let num_deduped = solana_perf::deduper::dedup_packets_and_count_discards( 202 | &deduper.read().unwrap(), 203 | &mut packet_batch_vec, 204 | ); 205 | 206 | // Store stats for each Packet 207 | packet_batch_vec.iter().for_each(|batch| { 208 | batch.iter().for_each(|packet| { 209 | metrics 210 | .packets_received 211 | .entry(packet.meta().addr) 212 | .and_modify(|(discarded, not_discarded)| { 213 | *discarded += packet.meta().discard() as u64; 214 | *not_discarded += (!packet.meta().discard()) as u64; 215 | }) 216 | .or_insert_with(|| { 217 | ( 218 | packet.meta().discard() as u64, 219 | (!packet.meta().discard()) as u64, 220 | ) 221 | }); 222 | }); 223 | }); 224 | 225 | // send out to RPCs 226 | local_dest_sockets.iter().for_each(|outgoing_socketaddr| { 227 | let packets_with_dest = packet_batch_vec[0] 228 | .iter() 229 | .filter_map(|pkt| { 230 | let data = pkt.data(..)?; 231 | let addr = outgoing_socketaddr; 232 | Some((data, addr)) 233 | }) 234 | .collect::>(); 235 | 236 | match batch_send(send_socket, &packets_with_dest) { 237 | Ok(_) => { 238 | metrics 239 | .success_forward 240 | .fetch_add(packets_with_dest.len() as u64, Ordering::Relaxed); 241 | metrics.duplicate.fetch_add(num_deduped, Ordering::Relaxed); 242 | } 243 | Err(SendPktsError::IoError(err, num_failed)) => { 244 | metrics 245 | .fail_forward 246 | .fetch_add(packets_with_dest.len() as u64, Ordering::Relaxed); 247 | metrics 248 | .duplicate 249 | .fetch_add(num_failed as u64, Ordering::Relaxed); 250 | error!( 251 | "Failed to send batch of size {} to {outgoing_socketaddr:?}. \ 252 | {num_failed} packets failed. Error: {err}", 253 | packets_with_dest.len() 254 | ); 255 | } 256 | } 257 | }); 258 | 259 | if should_reconstruct_shreds { 260 | deshred::reconstruct_shreds( 261 | packet_batch_vec 262 | .iter() 263 | .flat_map(|x| x.iter()) 264 | .filter_map(|x| x.data(..)), 265 | all_shreds, 266 | deshredded_entries, 267 | rs_cache, 268 | metrics, 269 | ); 270 | 271 | deshredded_entries 272 | .drain(..) 273 | .for_each(|(slot, _entries, entries_bytes)| { 274 | let _ = entry_sender.send(PbEntry { 275 | slot, 276 | entries: entries_bytes, 277 | }); 278 | }); 279 | } 280 | 281 | // Count TraceShred shreds 282 | if debug_trace_shred { 283 | packet_batch_vec[0] 284 | .iter() 285 | .filter_map(|p| TraceShred::decode(p.data(..)?).ok()) 286 | .filter(|t| t.created_at.is_some()) 287 | .for_each(|trace_shred| { 288 | let elapsed = trace_shred_received_time 289 | .duration_since(SystemTime::try_from(trace_shred.created_at.unwrap()).unwrap()) 290 | .unwrap_or_default(); 291 | 292 | datapoint_info!( 293 | "shredstream_proxy-trace_shred_latency", 294 | "trace_region" => trace_shred.region, 295 | ("trace_seq_num", trace_shred.seq_num as i64, i64), 296 | ("elapsed_micros", elapsed.as_micros(), i64), 297 | ); 298 | }); 299 | } 300 | 301 | Ok(()) 302 | } 303 | 304 | /// Starts a thread that updates our destinations used by the forwarder threads 305 | pub fn start_destination_refresh_thread( 306 | endpoint_discovery_url: String, 307 | discovered_endpoints_port: u16, 308 | static_dest_sockets: Vec<(SocketAddr, String)>, 309 | unioned_dest_sockets: Arc>>, 310 | shutdown_receiver: Receiver<()>, 311 | exit: Arc, 312 | ) -> JoinHandle<()> { 313 | Builder::new().name("ssPxyDstRefresh".to_string()).spawn(move || { 314 | let fetch_socket_tick = crossbeam_channel::tick(Duration::from_secs(30)); 315 | let metrics_tick = crossbeam_channel::tick(Duration::from_secs(30)); 316 | let mut socket_count = static_dest_sockets.len(); 317 | while !exit.load(Ordering::Relaxed) { 318 | crossbeam_channel::select! { 319 | recv(fetch_socket_tick) -> _ => { 320 | let fetched = fetch_unioned_destinations( 321 | &endpoint_discovery_url, 322 | discovered_endpoints_port, 323 | &static_dest_sockets, 324 | ); 325 | let new_sockets = match fetched { 326 | Ok(s) => { 327 | info!("Sending shreds to {} destinations: {s:?}", s.len()); 328 | s 329 | } 330 | Err(e) => { 331 | warn!("Failed to fetch from discovery service, retrying. Error: {e}"); 332 | datapoint_warn!("shredstream_proxy-destination_refresh_error", 333 | ("prev_unioned_dest_count", socket_count, i64), 334 | ("errors", 1, i64), 335 | ("error_str", e.to_string(), String), 336 | ); 337 | continue; 338 | } 339 | }; 340 | socket_count = new_sockets.len(); 341 | unioned_dest_sockets.store(Arc::new(new_sockets)); 342 | } 343 | recv(metrics_tick) -> _ => { 344 | datapoint_info!("shredstream_proxy-destination_refresh_stats", 345 | ("destination_count", socket_count, i64), 346 | ); 347 | } 348 | recv(shutdown_receiver) -> _ => { 349 | break; 350 | } 351 | } 352 | } 353 | }).unwrap() 354 | } 355 | 356 | /// Returns dynamically discovered endpoints with CLI arg defined endpoints 357 | fn fetch_unioned_destinations( 358 | endpoint_discovery_url: &str, 359 | discovered_endpoints_port: u16, 360 | static_dest_sockets: &[(SocketAddr, String)], 361 | ) -> Result, ShredstreamProxyError> { 362 | let bytes = reqwest::blocking::get(endpoint_discovery_url)?.bytes()?; 363 | 364 | let sockets_json = match serde_json::from_slice::>(&bytes) { 365 | Ok(s) => s, 366 | Err(e) => { 367 | warn!( 368 | "Failed to parse json from: {:?}", 369 | std::str::from_utf8(&bytes) 370 | ); 371 | return Err(ShredstreamProxyError::from(e)); 372 | } 373 | }; 374 | 375 | // resolve again since ip address could change 376 | let static_dest_sockets = static_dest_sockets 377 | .iter() 378 | .filter_map(|(_socketaddr, hostname_port)| { 379 | Some(resolve_hostname_port(hostname_port).ok()?.0) 380 | }) 381 | .collect::>(); 382 | 383 | let unioned_dest_sockets = sockets_json 384 | .into_iter() 385 | .map(|ip| SocketAddr::new(ip, discovered_endpoints_port)) 386 | .chain(static_dest_sockets) 387 | .unique() 388 | .collect::>(); 389 | Ok(unioned_dest_sockets) 390 | } 391 | 392 | /// Reset dedup + send metrics to influx 393 | pub fn start_forwarder_accessory_thread( 394 | deduper: Arc>>, 395 | metrics: Arc, 396 | metrics_update_interval_ms: u64, 397 | shutdown_receiver: Receiver<()>, 398 | exit: Arc, 399 | ) -> JoinHandle<()> { 400 | Builder::new() 401 | .name("ssPxyAccessory".to_string()) 402 | .spawn(move || { 403 | let metrics_tick = 404 | crossbeam_channel::tick(Duration::from_millis(metrics_update_interval_ms)); 405 | let deduper_reset_tick = crossbeam_channel::tick(Duration::from_secs(2)); 406 | let mut rng = rand::thread_rng(); 407 | while !exit.load(Ordering::Relaxed) { 408 | crossbeam_channel::select! { 409 | // reset deduper to avoid false positives 410 | recv(deduper_reset_tick) -> _ => { 411 | deduper 412 | .write() 413 | .unwrap() 414 | .maybe_reset(&mut rng, DEDUPER_FALSE_POSITIVE_RATE, DEDUPER_RESET_CYCLE); 415 | } 416 | 417 | // send metrics to influx 418 | recv(metrics_tick) -> _ => { 419 | metrics.report(); 420 | metrics.reset(); 421 | } 422 | 423 | // handle SIGINT shutdown 424 | recv(shutdown_receiver) -> _ => { 425 | break; 426 | } 427 | } 428 | } 429 | }) 430 | .unwrap() 431 | } 432 | 433 | pub struct ShredMetrics { 434 | // receive stats 435 | /// Total number of shreds received. Includes duplicates when receiving shreds from multiple regions 436 | pub received: AtomicU64, 437 | /// Total number of shreds successfully forwarded, accounting for all destinations 438 | pub success_forward: AtomicU64, 439 | /// Total number of shreds failed to forward, accounting for all destinations 440 | pub fail_forward: AtomicU64, 441 | /// Number of duplicate shreds received 442 | pub duplicate: AtomicU64, 443 | /// (discarded, not discarded, from other shredstream instances) 444 | pub packets_received: DashMap, 445 | 446 | // service metrics 447 | pub enabled_grpc_service: bool, 448 | /// Number of data shreds recovered using coding shreds 449 | pub recovered_count: AtomicU64, 450 | /// Number of Solana entries decoded from shreds 451 | pub entry_count: AtomicU64, 452 | /// Number of transactions decoded from shreds 453 | pub txn_count: AtomicU64, 454 | /// Number of FEC recovery errors 455 | pub fec_recovery_error_count: AtomicU64, 456 | /// Number of bincode Entry deserialization errors 457 | pub bincode_deserialize_error_count: AtomicU64, 458 | 459 | // cumulative metrics (persist after reset) 460 | pub agg_received_cumulative: AtomicU64, 461 | pub agg_success_forward_cumulative: AtomicU64, 462 | pub agg_fail_forward_cumulative: AtomicU64, 463 | pub duplicate_cumulative: AtomicU64, 464 | } 465 | 466 | impl Default for ShredMetrics { 467 | fn default() -> Self { 468 | Self::new(false) 469 | } 470 | } 471 | 472 | impl ShredMetrics { 473 | pub fn new(enabled_grpc_service: bool) -> Self { 474 | Self { 475 | enabled_grpc_service, 476 | received: Default::default(), 477 | success_forward: Default::default(), 478 | fail_forward: Default::default(), 479 | duplicate: Default::default(), 480 | packets_received: DashMap::with_capacity(10), 481 | recovered_count: Default::default(), 482 | entry_count: Default::default(), 483 | txn_count: Default::default(), 484 | fec_recovery_error_count: Default::default(), 485 | bincode_deserialize_error_count: Default::default(), 486 | agg_received_cumulative: Default::default(), 487 | agg_success_forward_cumulative: Default::default(), 488 | agg_fail_forward_cumulative: Default::default(), 489 | duplicate_cumulative: Default::default(), 490 | } 491 | } 492 | 493 | pub fn report(&self) { 494 | datapoint_info!( 495 | "shredstream_proxy-connection_metrics", 496 | ("received", self.received.load(Ordering::Relaxed), i64), 497 | ( 498 | "success_forward", 499 | self.success_forward.load(Ordering::Relaxed), 500 | i64 501 | ), 502 | ( 503 | "fail_forward", 504 | self.fail_forward.load(Ordering::Relaxed), 505 | i64 506 | ), 507 | ("duplicate", self.duplicate.load(Ordering::Relaxed), i64), 508 | ); 509 | 510 | if self.enabled_grpc_service { 511 | datapoint_info!( 512 | "shredstream_proxy-service_metrics", 513 | ( 514 | "recovered_count", 515 | self.recovered_count.swap(0, Ordering::Relaxed), 516 | i64 517 | ), 518 | ( 519 | "entry_count", 520 | self.entry_count.swap(0, Ordering::Relaxed), 521 | i64 522 | ), 523 | ("txn_count", self.txn_count.swap(0, Ordering::Relaxed), i64), 524 | ( 525 | "fec_recovery_error_count", 526 | self.fec_recovery_error_count.swap(0, Ordering::Relaxed), 527 | i64 528 | ), 529 | ( 530 | "bincode_deserialize_error_count", 531 | self.bincode_deserialize_error_count 532 | .swap(0, Ordering::Relaxed), 533 | i64 534 | ), 535 | ); 536 | } 537 | 538 | self.packets_received 539 | .retain(|addr, (discarded_packets, not_discarded_packets)| { 540 | datapoint_info!("shredstream_proxy-receiver_stats", 541 | "addr" => addr.to_string(), 542 | ("discarded_packets", *discarded_packets, i64), 543 | ("not_discarded_packets", *not_discarded_packets, i64), 544 | ); 545 | false 546 | }); 547 | } 548 | 549 | /// resets current values, increments cumulative values 550 | pub fn reset(&self) { 551 | self.agg_received_cumulative 552 | .fetch_add(self.received.swap(0, Ordering::Relaxed), Ordering::Relaxed); 553 | self.agg_success_forward_cumulative.fetch_add( 554 | self.success_forward.swap(0, Ordering::Relaxed), 555 | Ordering::Relaxed, 556 | ); 557 | self.agg_fail_forward_cumulative.fetch_add( 558 | self.fail_forward.swap(0, Ordering::Relaxed), 559 | Ordering::Relaxed, 560 | ); 561 | self.duplicate_cumulative 562 | .fetch_add(self.duplicate.swap(0, Ordering::Relaxed), Ordering::Relaxed); 563 | } 564 | } 565 | 566 | #[cfg(test)] 567 | mod tests { 568 | use std::{ 569 | collections::{HashMap, HashSet}, 570 | net::{IpAddr, Ipv4Addr, SocketAddr, UdpSocket}, 571 | str::FromStr, 572 | sync::{Arc, Mutex, RwLock}, 573 | thread, 574 | thread::sleep, 575 | time::Duration, 576 | }; 577 | 578 | use solana_ledger::shred::ReedSolomonCache; 579 | use solana_perf::{ 580 | deduper::Deduper, 581 | packet::{Meta, Packet, PacketBatch}, 582 | }; 583 | use solana_sdk::{ 584 | clock::Slot, 585 | packet::{PacketFlags, PACKET_DATA_SIZE}, 586 | }; 587 | use tokio::sync::broadcast::Sender as BroadcastSender; 588 | 589 | use crate::{ 590 | deshred::ComparableShred, 591 | forwarder::{recv_from_channel_and_send_multiple_dest, ShredMetrics}, 592 | }; 593 | 594 | fn listen_and_collect(listen_socket: UdpSocket, received_packets: Arc>>>) { 595 | let mut buf = [0u8; PACKET_DATA_SIZE]; 596 | loop { 597 | listen_socket.recv(&mut buf).unwrap(); 598 | received_packets.lock().unwrap().push(Vec::from(buf)); 599 | } 600 | } 601 | 602 | #[test] 603 | fn test_2shreds_3destinations() { 604 | let packet_batch = PacketBatch::new(vec![ 605 | Packet::new( 606 | [1; PACKET_DATA_SIZE], 607 | Meta { 608 | size: PACKET_DATA_SIZE, 609 | addr: IpAddr::V4(Ipv4Addr::UNSPECIFIED), 610 | port: 48289, // received on random port 611 | flags: PacketFlags::empty(), 612 | }, 613 | ), 614 | Packet::new( 615 | [2; PACKET_DATA_SIZE], 616 | Meta { 617 | size: PACKET_DATA_SIZE, 618 | addr: IpAddr::V4(Ipv4Addr::UNSPECIFIED), 619 | port: 9999, 620 | flags: PacketFlags::empty(), 621 | }, 622 | ), 623 | ]); 624 | let (packet_sender, packet_receiver) = crossbeam_channel::unbounded::(); 625 | packet_sender.send(packet_batch).unwrap(); 626 | 627 | let dest_socketaddrs = vec![ 628 | SocketAddr::from_str("0.0.0.0:32881").unwrap(), 629 | SocketAddr::from_str("0.0.0.0:33881").unwrap(), 630 | SocketAddr::from_str("0.0.0.0:34881").unwrap(), 631 | ]; 632 | 633 | let test_listeners = dest_socketaddrs 634 | .iter() 635 | .map(|socketaddr| { 636 | ( 637 | UdpSocket::bind(socketaddr).unwrap(), 638 | *socketaddr, 639 | // store results in vec of packet, where packet is Vec 640 | Arc::new(Mutex::new(vec![])), 641 | ) 642 | }) 643 | .collect::>(); 644 | 645 | let udp_sender = UdpSocket::bind("0.0.0.0:10000").unwrap(); 646 | 647 | // spawn listeners 648 | test_listeners 649 | .iter() 650 | .for_each(|(listen_socket, _socketaddr, to_receive)| { 651 | let socket = listen_socket.try_clone().unwrap(); 652 | let to_receive = to_receive.to_owned(); 653 | thread::spawn(move || listen_and_collect(socket, to_receive)); 654 | }); 655 | 656 | let entry_sender = Arc::new(BroadcastSender::new(1_000)); 657 | let mut all_shreds: HashMap< 658 | Slot, 659 | HashMap)>, 660 | > = HashMap::new(); 661 | // send packets 662 | recv_from_channel_and_send_multiple_dest( 663 | packet_receiver.recv(), 664 | &Arc::new(RwLock::new(Deduper::<2, [u8]>::new( 665 | &mut rand::thread_rng(), 666 | crate::forwarder::DEDUPER_NUM_BITS, 667 | ))), 668 | &mut all_shreds, 669 | &mut Vec::new(), 670 | &ReedSolomonCache::default(), 671 | &udp_sender, 672 | &Arc::new(dest_socketaddrs), 673 | true, 674 | &entry_sender, 675 | false, 676 | &Arc::new(ShredMetrics::default()), 677 | ) 678 | .unwrap(); 679 | 680 | // allow packets to be received 681 | sleep(Duration::from_millis(500)); 682 | 683 | let received = test_listeners 684 | .iter() 685 | .map(|(_, _, results)| results.clone()) 686 | .collect::>(); 687 | 688 | // check results 689 | for received in received.iter() { 690 | let received = received.lock().unwrap(); 691 | assert_eq!(received.len(), 2); 692 | assert!(received 693 | .iter() 694 | .all(|packet| packet.len() == PACKET_DATA_SIZE)); 695 | assert_eq!(received[0], [1; PACKET_DATA_SIZE]); 696 | assert_eq!(received[1], [2; PACKET_DATA_SIZE]); 697 | } 698 | 699 | assert_eq!( 700 | received 701 | .iter() 702 | .fold(0, |acc, elem| acc + elem.lock().unwrap().len()), 703 | 6 704 | ); 705 | } 706 | } 707 | -------------------------------------------------------------------------------- /proxy/src/heartbeat.rs: -------------------------------------------------------------------------------- 1 | use std::{ 2 | net::SocketAddr, 3 | sync::{ 4 | atomic::{AtomicBool, Ordering}, 5 | Arc, 6 | }, 7 | thread::{sleep, Builder, JoinHandle}, 8 | time::Duration, 9 | }; 10 | 11 | use crossbeam_channel::Receiver; 12 | use jito_protos::{ 13 | auth::{auth_service_client::AuthServiceClient, Role}, 14 | shredstream::{shredstream_client::ShredstreamClient, Heartbeat}, 15 | }; 16 | use log::{info, warn}; 17 | use solana_metrics::{datapoint_info, datapoint_warn}; 18 | use solana_sdk::signature::Keypair; 19 | use tokio::runtime::Runtime; 20 | use tonic::{codegen::InterceptedService, transport::Channel, Code}; 21 | 22 | use crate::{ 23 | forwarder::ShredMetrics, 24 | token_authenticator::{create_grpc_channel, ClientInterceptor}, 25 | ShredstreamProxyError, 26 | }; 27 | /* 28 | This is a wrapper around AtomicBool that allows us to scope the lifetime of the AtomicBool to the heartbeat loop. 29 | This is useful because we want to ensure that the AtomicBool is set to true when the heartbeat loop exits. 30 | */ 31 | struct ScopedAtomicBool { 32 | inner: Arc, 33 | } 34 | 35 | impl ScopedAtomicBool { 36 | fn get_inner_clone(&self) -> Arc { 37 | self.inner.clone() 38 | } 39 | } 40 | 41 | impl Default for ScopedAtomicBool { 42 | fn default() -> Self { 43 | Self { 44 | inner: Arc::new(AtomicBool::new(false)), 45 | } 46 | } 47 | } 48 | 49 | impl Drop for ScopedAtomicBool { 50 | fn drop(&mut self) { 51 | self.inner.store(true, Ordering::Relaxed); 52 | } 53 | } 54 | 55 | #[allow(clippy::too_many_arguments)] 56 | pub fn heartbeat_loop_thread( 57 | block_engine_url: String, 58 | auth_url: String, 59 | auth_keypair: Arc, 60 | desired_regions: Vec, 61 | recv_socket: SocketAddr, 62 | runtime: Runtime, 63 | service_name: String, 64 | metrics: Arc, 65 | shutdown_receiver: Receiver<()>, 66 | exit: Arc, 67 | ) -> JoinHandle<()> { 68 | Builder::new().name("ssPxyHbeatLoop".to_string()).spawn(move || { 69 | let heartbeat_socket = jito_protos::shared::Socket { 70 | ip: recv_socket.ip().to_string(), 71 | port: recv_socket.port() as i64, 72 | }; 73 | let mut heartbeat_interval = Duration::from_secs(1); //start with 1s, change based on server suggestion 74 | // use tick() since we want to avoid thread::sleep(), as it's not interruptible. want to be interruptible for exiting quickly 75 | let mut heartbeat_tick = crossbeam_channel::tick(heartbeat_interval); 76 | let metrics_tick = crossbeam_channel::tick(Duration::from_secs(30)); 77 | let mut last_cumulative_received_shred_count = 0; 78 | let mut client_restart_count = 0u64; 79 | let mut successful_heartbeat_count = 0u64; 80 | let mut failed_heartbeat_count = 0u64; 81 | let mut client_restart_count_cumulative = 0u64; 82 | let mut successful_heartbeat_count_cumulative = 0u64; 83 | let mut failed_heartbeat_count_cumulative = 0u64; 84 | 85 | while !exit.load(Ordering::Relaxed) { 86 | // We want to scope the grpc shredstream client to the heartbeat loop. This way shredstream client exits when the heartbeat loop exits 87 | let per_con_exit = ScopedAtomicBool::default(); 88 | info!("Starting heartbeat client"); 89 | let shredstream_client_res = runtime.block_on( 90 | get_grpc_client( 91 | block_engine_url.clone(), 92 | auth_url.clone(), 93 | auth_keypair.clone(), 94 | service_name.clone(), 95 | per_con_exit.get_inner_clone(), 96 | ) 97 | ); 98 | // Shredstream client lives here -- so it has the same scope as per_con_exit 99 | let (mut shredstream_client , refresh_thread_hdl) = match shredstream_client_res { 100 | Ok(c) => c, 101 | Err(e) => { 102 | warn!("Failed to connect to block engine, retrying. Error: {e}"); 103 | client_restart_count += 1; 104 | datapoint_warn!( 105 | "shredstream_proxy-heartbeat_client_error", 106 | "block_engine_url" => block_engine_url, 107 | ("errors", 1, i64), 108 | ("error_str", e.to_string(), String), 109 | ); 110 | sleep(Duration::from_secs(5)); 111 | continue; // avoid sending heartbeat, try acquiring grpc client again 112 | } 113 | }; 114 | while !exit.load(Ordering::Relaxed) { 115 | crossbeam_channel::select! { 116 | // send heartbeat 117 | recv(heartbeat_tick) -> _ => { 118 | let heartbeat_result = runtime.block_on(shredstream_client 119 | .send_heartbeat(Heartbeat { 120 | socket: Some(heartbeat_socket.clone()), 121 | regions: desired_regions.clone(), 122 | })); 123 | 124 | match heartbeat_result { 125 | Ok(hb) => { 126 | // retry sooner in case a heartbeat fails 127 | let new_interval = Duration::from_millis((hb.get_ref().ttl_ms / 3) as u64); 128 | if heartbeat_interval != new_interval { 129 | info!("Sending heartbeat every {new_interval:?}."); 130 | heartbeat_interval = new_interval; 131 | heartbeat_tick = crossbeam_channel::tick(new_interval); 132 | } 133 | successful_heartbeat_count += 1; 134 | } 135 | Err(err) => { 136 | if err.code() == Code::InvalidArgument { 137 | panic!("Invalid arguments: {err}."); 138 | }; 139 | warn!("Error sending heartbeat: {err}"); 140 | datapoint_warn!( 141 | "shredstream_proxy-heartbeat_send_error", 142 | "block_engine_url" => block_engine_url, 143 | ("errors", 1, i64), 144 | ("error_str", err.to_string(), String), 145 | ); 146 | failed_heartbeat_count += 1; 147 | } 148 | } 149 | } 150 | 151 | // send metrics and handle grpc connection failing 152 | recv(metrics_tick) -> _ => { 153 | datapoint_info!( 154 | "shredstream_proxy-heartbeat_stats", 155 | "block_engine_url" => block_engine_url, 156 | ("successful_heartbeat_count", successful_heartbeat_count, i64), 157 | ("failed_heartbeat_count", failed_heartbeat_count, i64), 158 | ("client_restart_count", client_restart_count, i64), 159 | ); 160 | 161 | // handle scenario when grpc connection is open, but backend doesn't receive heartbeat 162 | // possibly due to envoy losing track of the pod when backend restarts. 163 | // we restart our grpc connection to work around the stale connection 164 | // if no shreds received, then restart 165 | let new_received_count = metrics.agg_received_cumulative.load(Ordering::Relaxed); 166 | if new_received_count == last_cumulative_received_shred_count { 167 | warn!("No shreds received recently, restarting heartbeat client."); 168 | datapoint_warn!( 169 | "shredstream_proxy-heartbeat_restart_signal", 170 | "block_engine_url" => block_engine_url, 171 | ("desired_regions", format!("{desired_regions:?}"), String), 172 | ); 173 | refresh_thread_hdl.abort(); 174 | break; 175 | } 176 | last_cumulative_received_shred_count = new_received_count; 177 | 178 | 179 | successful_heartbeat_count_cumulative += successful_heartbeat_count; 180 | failed_heartbeat_count_cumulative += failed_heartbeat_count; 181 | client_restart_count_cumulative += client_restart_count; 182 | successful_heartbeat_count = 0; 183 | failed_heartbeat_count = 0; 184 | client_restart_count = 0; 185 | } 186 | 187 | // handle SIGINT shutdown 188 | recv(shutdown_receiver) -> _ => { 189 | // exit should be true 190 | break; 191 | } 192 | } 193 | } 194 | } 195 | info!("Exiting heartbeat thread, sent {successful_heartbeat_count_cumulative} successful, {failed_heartbeat_count_cumulative} failed heartbeats. Client restarted {client_restart_count_cumulative} times."); 196 | }).unwrap() 197 | } 198 | 199 | pub async fn get_grpc_client( 200 | block_engine_url: String, 201 | auth_url: String, 202 | auth_keypair: Arc, 203 | service_name: String, 204 | exit: Arc, 205 | ) -> Result< 206 | ( 207 | ShredstreamClient>, 208 | tokio::task::JoinHandle<()>, 209 | ), 210 | ShredstreamProxyError, 211 | > { 212 | let auth_channel = create_grpc_channel(auth_url).await?; 213 | let searcher_channel = create_grpc_channel(block_engine_url).await?; 214 | let (client_interceptor, thread_handle) = ClientInterceptor::new( 215 | AuthServiceClient::new(auth_channel), 216 | auth_keypair, 217 | Role::ShredstreamSubscriber, 218 | service_name, 219 | exit, 220 | ) 221 | .await?; 222 | let searcher_client = ShredstreamClient::with_interceptor(searcher_channel, client_interceptor); 223 | Ok((searcher_client, thread_handle)) 224 | } 225 | -------------------------------------------------------------------------------- /proxy/src/main.rs: -------------------------------------------------------------------------------- 1 | use std::{ 2 | collections::HashMap, 3 | io, 4 | io::{Error, ErrorKind}, 5 | net::{IpAddr, Ipv4Addr, SocketAddr, ToSocketAddrs}, 6 | panic, 7 | path::{Path, PathBuf}, 8 | str::FromStr, 9 | sync::{ 10 | atomic::{AtomicBool, Ordering}, 11 | Arc, RwLock, 12 | }, 13 | thread, 14 | thread::{sleep, spawn, JoinHandle}, 15 | time::Duration, 16 | }; 17 | 18 | use arc_swap::ArcSwap; 19 | use clap::{arg, Parser}; 20 | use crossbeam_channel::{Receiver, RecvError, Sender}; 21 | use log::*; 22 | use signal_hook::consts::{SIGINT, SIGTERM}; 23 | use solana_client::client_error::{reqwest, ClientError}; 24 | use solana_ledger::shred::Shred; 25 | use solana_metrics::set_host_id; 26 | use solana_perf::deduper::Deduper; 27 | use solana_sdk::{clock::Slot, signature::read_keypair_file}; 28 | use solana_streamer::streamer::StreamerReceiveStats; 29 | use thiserror::Error; 30 | use tokio::{runtime::Runtime, sync::broadcast::Sender as BroadcastSender}; 31 | use tonic::Status; 32 | 33 | use crate::{forwarder::ShredMetrics, token_authenticator::BlockEngineConnectionError}; 34 | mod deshred; 35 | pub mod forwarder; 36 | mod heartbeat; 37 | mod server; 38 | mod token_authenticator; 39 | 40 | #[derive(Clone, Debug, Parser)] 41 | #[clap(author, version, about, long_about = None)] 42 | // https://docs.rs/clap/latest/clap/_derive/_cookbook/git_derive/index.html 43 | struct Args { 44 | #[command(subcommand)] 45 | shredstream_args: ProxySubcommands, 46 | } 47 | 48 | #[derive(Clone, Debug, clap::Subcommand)] 49 | enum ProxySubcommands { 50 | /// Requests shreds from Jito and sends to all destinations. 51 | Shredstream(ShredstreamArgs), 52 | 53 | /// Does not request shreds from Jito. Sends anything received on `src-bind-addr`:`src-bind-port` to all destinations. 54 | ForwardOnly(CommonArgs), 55 | } 56 | 57 | #[derive(clap::Args, Clone, Debug)] 58 | struct ShredstreamArgs { 59 | /// Address for Jito Block Engine. 60 | /// See https://jito-labs.gitbook.io/mev/searcher-resources/block-engine#connection-details 61 | #[arg(long, env)] 62 | block_engine_url: String, 63 | 64 | /// Manual override for auth service address. For internal use. 65 | #[arg(long, env)] 66 | auth_url: Option, 67 | 68 | /// Path to keypair file used to authenticate with the backend. 69 | #[arg(long, env)] 70 | auth_keypair: PathBuf, 71 | 72 | /// Desired regions to receive heartbeats from. 73 | /// Receives `n` different streams. Requires at least 1 region, comma separated. 74 | #[arg(long, env, value_delimiter = ',', required(true))] 75 | desired_regions: Vec, 76 | 77 | #[clap(flatten)] 78 | common_args: CommonArgs, 79 | } 80 | 81 | #[derive(clap::Args, Clone, Debug)] 82 | struct CommonArgs { 83 | /// Address where Shredstream proxy listens. 84 | #[arg(long, env, default_value_t = IpAddr::V4(std::net::Ipv4Addr::new(0, 0, 0, 0)))] 85 | src_bind_addr: IpAddr, 86 | 87 | /// Port where Shredstream proxy listens. Use `0` for random ephemeral port. 88 | #[arg(long, env, default_value_t = 20_000)] 89 | src_bind_port: u16, 90 | 91 | /// Static set of IP:Port where Shredstream proxy forwards shreds to, comma separated. 92 | /// Eg. `127.0.0.1:8001,10.0.0.1:8001`. 93 | // Note: store the original string, so we can do hostname resolution when refreshing destinations 94 | #[arg(long, env, value_delimiter = ',', value_parser = resolve_hostname_port)] 95 | dest_ip_ports: Vec<(SocketAddr, String)>, 96 | 97 | /// Http JSON endpoint to dynamically get IPs for Shredstream proxy to forward shreds. 98 | /// Endpoints are then set-union with `dest-ip-ports`. 99 | #[arg(long, env)] 100 | endpoint_discovery_url: Option, 101 | 102 | /// Port to send shreds to for hosts fetched via `endpoint-discovery-url`. 103 | /// Port can be found using `scripts/get_tvu_port.sh`. 104 | /// See https://jito-labs.gitbook.io/mev/searcher-services/shredstream#running-shredstream 105 | #[arg(long, env)] 106 | discovered_endpoints_port: Option, 107 | 108 | /// Interval between logging stats to stdout and influx 109 | #[arg(long, env, default_value_t = 15_000)] 110 | metrics_report_interval_ms: u64, 111 | 112 | /// Logs trace shreds to stdout and influx 113 | #[arg(long, env, default_value_t = false)] 114 | debug_trace_shred: bool, 115 | 116 | /// GRPC port for serving decoded shreds as Solana entries 117 | #[arg(long, env)] 118 | grpc_service_port: Option, 119 | 120 | /// Public IP address to use. 121 | /// Overrides value fetched from `ifconfig.me`. 122 | #[arg(long, env)] 123 | public_ip: Option, 124 | 125 | /// Number of threads to use. Defaults to use up to 4. 126 | #[arg(long, env)] 127 | num_threads: Option, 128 | } 129 | 130 | #[derive(Debug, Error)] 131 | pub enum ShredstreamProxyError { 132 | #[error("TonicError {0}")] 133 | TonicError(#[from] tonic::transport::Error), 134 | #[error("GrpcError {0}")] 135 | GrpcError(#[from] Status), 136 | #[error("ReqwestError {0}")] 137 | ReqwestError(#[from] reqwest::Error), 138 | #[error("SerdeJsonError {0}")] 139 | SerdeJsonError(#[from] serde_json::Error), 140 | #[error("RpcError {0}")] 141 | RpcError(#[from] ClientError), 142 | #[error("BlockEngineConnectionError {0}")] 143 | BlockEngineConnectionError(#[from] BlockEngineConnectionError), 144 | #[error("RecvError {0}")] 145 | RecvError(#[from] RecvError), 146 | #[error("IoError {0}")] 147 | IoError(#[from] io::Error), 148 | #[error("Shutdown")] 149 | Shutdown, 150 | } 151 | 152 | fn resolve_hostname_port(hostname_port: &str) -> io::Result<(SocketAddr, String)> { 153 | let socketaddr = hostname_port.to_socket_addrs()?.next().ok_or_else(|| { 154 | Error::new( 155 | ErrorKind::AddrNotAvailable, 156 | format!("Could not find destination {hostname_port}"), 157 | ) 158 | })?; 159 | 160 | Ok((socketaddr, hostname_port.to_string())) 161 | } 162 | 163 | /// Returns public-facing IPV4 address 164 | pub fn get_public_ip() -> reqwest::Result { 165 | info!("Requesting public ip from ifconfig.me..."); 166 | let client = reqwest::blocking::Client::builder() 167 | .local_address(IpAddr::V4(Ipv4Addr::UNSPECIFIED)) 168 | .build()?; 169 | let response = client.get("https://ifconfig.me/ip").send()?.text()?; 170 | let public_ip = IpAddr::from_str(&response).unwrap(); 171 | info!("Retrieved public ip: {public_ip:?}"); 172 | 173 | Ok(public_ip) 174 | } 175 | 176 | // Creates a channel that gets a message every time `SIGINT` is signalled. 177 | fn shutdown_notifier(exit: Arc) -> io::Result<(Sender<()>, Receiver<()>)> { 178 | let (s, r) = crossbeam_channel::bounded(256); 179 | let mut signals = signal_hook::iterator::Signals::new([SIGINT, SIGTERM])?; 180 | 181 | let s_thread = s.clone(); 182 | thread::spawn(move || { 183 | for _ in signals.forever() { 184 | exit.store(true, Ordering::SeqCst); 185 | // send shutdown signal multiple times since crossbeam doesn't have broadcast channels 186 | // each thread will consume a shutdown signal 187 | for _ in 0..256 { 188 | if s_thread.send(()).is_err() { 189 | break; 190 | } 191 | } 192 | } 193 | }); 194 | 195 | Ok((s, r)) 196 | } 197 | 198 | pub type ReconstructedShredsMap = HashMap>>; 199 | fn main() -> Result<(), ShredstreamProxyError> { 200 | env_logger::builder().init(); 201 | 202 | let all_args: Args = Args::parse(); 203 | 204 | let shredstream_args = all_args.shredstream_args.clone(); 205 | // common args 206 | let args = match all_args.shredstream_args { 207 | ProxySubcommands::Shredstream(x) => x.common_args, 208 | ProxySubcommands::ForwardOnly(x) => x, 209 | }; 210 | set_host_id(hostname::get()?.into_string().unwrap()); 211 | if (args.endpoint_discovery_url.is_none() && args.discovered_endpoints_port.is_some()) 212 | || (args.endpoint_discovery_url.is_some() && args.discovered_endpoints_port.is_none()) 213 | { 214 | panic!("Invalid arguments provided, dynamic endpoints requires both --endpoint-discovery-url and --discovered-endpoints-port.") 215 | } 216 | if args.endpoint_discovery_url.is_none() 217 | && args.discovered_endpoints_port.is_none() 218 | && args.dest_ip_ports.is_empty() 219 | { 220 | panic!("No destinations found. You must provide values for --dest-ip-ports or --endpoint-discovery-url.") 221 | } 222 | 223 | let exit = Arc::new(AtomicBool::new(false)); 224 | let (shutdown_sender, shutdown_receiver) = 225 | shutdown_notifier(exit.clone()).expect("Failed to set up signal handler"); 226 | let panic_hook = panic::take_hook(); 227 | { 228 | let exit = exit.clone(); 229 | panic::set_hook(Box::new(move |panic_info| { 230 | exit.store(true, Ordering::SeqCst); 231 | let _ = shutdown_sender.send(()); 232 | error!("exiting process"); 233 | sleep(Duration::from_secs(1)); 234 | // invoke the default handler and exit the process 235 | panic_hook(panic_info); 236 | })); 237 | } 238 | 239 | let metrics = Arc::new(ShredMetrics::new(args.grpc_service_port.is_some())); 240 | 241 | let runtime = Runtime::new()?; 242 | let mut thread_handles = vec![]; 243 | if let ProxySubcommands::Shredstream(args) = shredstream_args { 244 | let heartbeat_hdl = 245 | start_heartbeat(args, &exit, &shutdown_receiver, runtime, metrics.clone()); 246 | thread_handles.push(heartbeat_hdl); 247 | } 248 | 249 | // share sockets between refresh and forwarder thread 250 | let unioned_dest_sockets = Arc::new(ArcSwap::from_pointee( 251 | args.dest_ip_ports 252 | .iter() 253 | .map(|x| x.0) 254 | .collect::>(), 255 | )); 256 | 257 | // share deduper + metrics between forwarder <-> accessory thread 258 | // use mutex since metrics are write heavy. cheaper than rwlock 259 | let deduper = Arc::new(RwLock::new(Deduper::<2, [u8]>::new( 260 | &mut rand::thread_rng(), 261 | forwarder::DEDUPER_NUM_BITS, 262 | ))); 263 | 264 | let entry_sender = Arc::new(BroadcastSender::new(100)); 265 | let forward_stats = Arc::new(StreamerReceiveStats::new("shredstream_proxy-listen_thread")); 266 | let use_discovery_service = 267 | args.endpoint_discovery_url.is_some() && args.discovered_endpoints_port.is_some(); 268 | let forwarder_hdls = forwarder::start_forwarder_threads( 269 | unioned_dest_sockets.clone(), 270 | args.src_bind_addr, 271 | args.src_bind_port, 272 | args.num_threads, 273 | deduper.clone(), 274 | args.grpc_service_port.is_some(), 275 | entry_sender.clone(), 276 | args.debug_trace_shred, 277 | use_discovery_service, 278 | forward_stats.clone(), 279 | metrics.clone(), 280 | shutdown_receiver.clone(), 281 | exit.clone(), 282 | ); 283 | thread_handles.extend(forwarder_hdls); 284 | 285 | let report_metrics_thread = { 286 | let exit = exit.clone(); 287 | spawn(move || { 288 | while !exit.load(Ordering::Relaxed) { 289 | sleep(Duration::from_secs(1)); 290 | forward_stats.report(); 291 | } 292 | }) 293 | }; 294 | thread_handles.push(report_metrics_thread); 295 | 296 | let metrics_hdl = forwarder::start_forwarder_accessory_thread( 297 | deduper, 298 | metrics.clone(), 299 | args.metrics_report_interval_ms, 300 | shutdown_receiver.clone(), 301 | exit.clone(), 302 | ); 303 | thread_handles.push(metrics_hdl); 304 | if use_discovery_service { 305 | let refresh_handle = forwarder::start_destination_refresh_thread( 306 | args.endpoint_discovery_url.unwrap(), 307 | args.discovered_endpoints_port.unwrap(), 308 | args.dest_ip_ports, 309 | unioned_dest_sockets, 310 | shutdown_receiver.clone(), 311 | exit.clone(), 312 | ); 313 | thread_handles.push(refresh_handle); 314 | } 315 | 316 | if let Some(port) = args.grpc_service_port { 317 | let server_hdl = server::start_server_thread( 318 | SocketAddr::new(IpAddr::V4(Ipv4Addr::UNSPECIFIED), port), 319 | entry_sender.clone(), 320 | exit.clone(), 321 | shutdown_receiver.clone(), 322 | ); 323 | thread_handles.push(server_hdl); 324 | } 325 | 326 | info!( 327 | "Shredstream started, listening on {}:{}/udp.", 328 | args.src_bind_addr, args.src_bind_port 329 | ); 330 | 331 | for thread in thread_handles { 332 | thread.join().expect("thread panicked"); 333 | } 334 | 335 | info!( 336 | "Exiting Shredstream, {} received , {} sent successfully, {} failed, {} duplicate shreds.", 337 | metrics.agg_received_cumulative.load(Ordering::Relaxed), 338 | metrics 339 | .agg_success_forward_cumulative 340 | .load(Ordering::Relaxed), 341 | metrics.agg_fail_forward_cumulative.load(Ordering::Relaxed), 342 | metrics.duplicate_cumulative.load(Ordering::Relaxed), 343 | ); 344 | Ok(()) 345 | } 346 | 347 | fn start_heartbeat( 348 | args: ShredstreamArgs, 349 | exit: &Arc, 350 | shutdown_receiver: &Receiver<()>, 351 | runtime: Runtime, 352 | metrics: Arc, 353 | ) -> JoinHandle<()> { 354 | let auth_keypair = Arc::new( 355 | read_keypair_file(Path::new(&args.auth_keypair)).unwrap_or_else(|e| { 356 | panic!( 357 | "Unable to parse keypair file. Ensure that file {:?} is readable. Error: {e}", 358 | args.auth_keypair 359 | ) 360 | }), 361 | ); 362 | 363 | heartbeat::heartbeat_loop_thread( 364 | args.block_engine_url.clone(), 365 | args.auth_url.unwrap_or(args.block_engine_url), 366 | auth_keypair, 367 | args.desired_regions, 368 | SocketAddr::new( 369 | args.common_args 370 | .public_ip 371 | .unwrap_or_else(|| get_public_ip().unwrap()), 372 | args.common_args.src_bind_port, 373 | ), 374 | runtime, 375 | "shredstream_proxy".to_string(), 376 | metrics, 377 | shutdown_receiver.clone(), 378 | exit.clone(), 379 | ) 380 | } 381 | -------------------------------------------------------------------------------- /proxy/src/server.rs: -------------------------------------------------------------------------------- 1 | use std::{ 2 | net::SocketAddr, 3 | sync::{atomic::AtomicBool, Arc}, 4 | thread::JoinHandle, 5 | time::Duration, 6 | }; 7 | 8 | use crossbeam_channel::Receiver; 9 | use jito_protos::shredstream::{ 10 | shredstream_proxy_server::{ShredstreamProxy, ShredstreamProxyServer}, 11 | Entry as PbEntry, SubscribeEntriesRequest, 12 | }; 13 | use log::{debug, info}; 14 | use tokio::sync::broadcast::{Receiver as BroadcastReceiver, Sender}; 15 | use tonic::codegen::tokio_stream::wrappers::ReceiverStream; 16 | 17 | #[derive(Debug)] 18 | pub struct ShredstreamProxyService { 19 | entry_sender: Arc>, 20 | } 21 | 22 | pub fn start_server_thread( 23 | addr: SocketAddr, 24 | entry_sender: Arc>, 25 | exit: Arc, 26 | shutdown_receiver: Receiver<()>, 27 | ) -> JoinHandle<()> { 28 | std::thread::spawn(move || { 29 | let runtime = tokio::runtime::Runtime::new().unwrap(); 30 | 31 | let server_handle = runtime.spawn(async move { 32 | info!("starting server on {:?}", addr); 33 | tonic::transport::Server::builder() 34 | .add_service(ShredstreamProxyServer::new(ShredstreamProxyService { 35 | entry_sender, 36 | })) 37 | .serve(addr) 38 | .await 39 | .unwrap(); 40 | }); 41 | 42 | while !exit.load(std::sync::atomic::Ordering::Relaxed) { 43 | if shutdown_receiver 44 | .recv_timeout(Duration::from_secs(1)) 45 | .is_ok() 46 | { 47 | server_handle.abort(); 48 | info!("shutting down entries server"); 49 | break; 50 | } 51 | } 52 | }) 53 | } 54 | #[tonic::async_trait] 55 | impl ShredstreamProxy for ShredstreamProxyService { 56 | type SubscribeEntriesStream = ReceiverStream>; 57 | 58 | async fn subscribe_entries( 59 | &self, 60 | _request: tonic::Request, 61 | ) -> Result, tonic::Status> { 62 | let (tx, rx) = tokio::sync::mpsc::channel(100); 63 | let mut entry_receiver: BroadcastReceiver = self.entry_sender.subscribe(); 64 | 65 | tokio::spawn(async move { 66 | while let Ok(entry) = entry_receiver.recv().await { 67 | match tx.send(Ok(entry)).await { 68 | Ok(_) => (), 69 | Err(_e) => { 70 | debug!("client disconnected"); 71 | break; 72 | } 73 | } 74 | } 75 | }); 76 | 77 | Ok(tonic::Response::new(ReceiverStream::new(rx))) 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /proxy/src/token_authenticator.rs: -------------------------------------------------------------------------------- 1 | use std::{ 2 | sync::{ 3 | atomic::{AtomicBool, Ordering}, 4 | Arc, 5 | }, 6 | time::{Duration, Instant, SystemTime}, 7 | }; 8 | 9 | use arc_swap::{ArcSwap, ArcSwapAny}; 10 | use jito_protos::auth::{ 11 | auth_service_client::AuthServiceClient, GenerateAuthChallengeRequest, 12 | GenerateAuthTokensRequest, RefreshAccessTokenRequest, Role, Token, 13 | }; 14 | use prost_types::Timestamp; 15 | use solana_metrics::datapoint_info; 16 | use solana_sdk::signature::{Keypair, Signer}; 17 | use thiserror::Error; 18 | use tokio::{task::JoinHandle, time::sleep}; 19 | use tonic::{ 20 | metadata::errors::InvalidMetadataValue, 21 | service::Interceptor, 22 | transport::{Channel, Endpoint}, 23 | Request, Status, 24 | }; 25 | 26 | /// Adds the token to each requests' authorization header. 27 | #[derive(Debug, Error)] 28 | pub enum BlockEngineConnectionError { 29 | #[error("transport error: {0}")] 30 | Transport(#[from] tonic::transport::Error), 31 | 32 | #[error("client error: {0}")] 33 | Client(#[from] Status), 34 | 35 | #[error("deserializing error")] 36 | Deserialization, 37 | } 38 | 39 | pub type BlockEngineConnectionResult = Result; 40 | 41 | /// Manages refreshing the token in a separate thread. 42 | #[derive(Clone)] 43 | pub struct ClientInterceptor { 44 | /// The access token from jito that added to each request header. 45 | bearer_token: Arc>, 46 | } 47 | 48 | impl ClientInterceptor { 49 | pub async fn new( 50 | mut auth_service_client: AuthServiceClient, 51 | keypair: Arc, 52 | role: Role, 53 | service_name: String, 54 | exit: Arc, 55 | ) -> BlockEngineConnectionResult<(Self, JoinHandle<()>)> { 56 | let ( 57 | Token { 58 | value: access_token, 59 | expires_at_utc: access_token_expiration, 60 | }, 61 | refresh_token, 62 | ) = Self::auth(&mut auth_service_client, &keypair, role).await?; 63 | let bearer_token = Arc::new(ArcSwap::from_pointee(access_token)); 64 | 65 | let refresh_thread_handle = Self::spawn_token_refresh_thread( 66 | auth_service_client, 67 | bearer_token.clone(), 68 | refresh_token, 69 | access_token_expiration.ok_or(BlockEngineConnectionError::Deserialization)?, 70 | keypair, 71 | role, 72 | service_name, 73 | exit, 74 | ); 75 | 76 | Ok((Self { bearer_token }, refresh_thread_handle)) 77 | } 78 | 79 | /// Returns (access token, refresh token) 80 | async fn auth( 81 | auth_service_client: &mut AuthServiceClient, 82 | keypair: &Keypair, 83 | role: Role, 84 | ) -> BlockEngineConnectionResult<(Token, Token)> { 85 | let pubkey_vec = keypair.pubkey().as_ref().to_vec(); 86 | let challenge_resp = auth_service_client 87 | .generate_auth_challenge(GenerateAuthChallengeRequest { 88 | role: role as i32, 89 | pubkey: pubkey_vec.clone(), 90 | }) 91 | .await? 92 | .into_inner(); 93 | let challenge = format!("{}-{}", keypair.pubkey(), challenge_resp.challenge); 94 | let signed_challenge = keypair.sign_message(challenge.as_bytes()).as_ref().to_vec(); 95 | 96 | let tokens = auth_service_client 97 | .generate_auth_tokens(GenerateAuthTokensRequest { 98 | challenge, 99 | client_pubkey: pubkey_vec, 100 | signed_challenge, 101 | }) 102 | .await? 103 | .into_inner(); 104 | 105 | Ok(( 106 | tokens 107 | .access_token 108 | .ok_or(BlockEngineConnectionError::Deserialization)?, 109 | tokens 110 | .refresh_token 111 | .ok_or(BlockEngineConnectionError::Deserialization)?, 112 | )) 113 | } 114 | 115 | /// Periodically updates `bearer_token` 116 | #[allow(clippy::too_many_arguments)] 117 | fn spawn_token_refresh_thread( 118 | mut auth_service_client: AuthServiceClient, 119 | bearer_token: Arc>, 120 | initial_refresh_token: Token, 121 | initial_access_token_expiration: Timestamp, 122 | keypair: Arc, 123 | role: Role, 124 | service_name: String, 125 | exit: Arc, 126 | ) -> JoinHandle<()> { 127 | tokio::spawn(async move { 128 | // refresh token gets us an access token. access token is short-lived 129 | let mut refresh_token = initial_refresh_token; 130 | let mut access_token_expiration = initial_access_token_expiration; 131 | 132 | while !exit.load(Ordering::Relaxed) { 133 | let now = SystemTime::now(); 134 | 135 | let refresh_token_ttl = 136 | SystemTime::try_from(refresh_token.expires_at_utc.as_ref().unwrap().clone()) 137 | .unwrap() 138 | .duration_since(now) 139 | .unwrap_or_default(); 140 | // re-run entire auth workflow if refresh token expiring soon 141 | if refresh_token_ttl < Duration::from_secs(5 * 60) { 142 | let start = Instant::now(); 143 | let is_error = { 144 | if let Ok((new_access_token, new_refresh_token)) = 145 | Self::auth(&mut auth_service_client, &keypair, role).await 146 | { 147 | bearer_token.store(Arc::new(new_access_token.value)); 148 | access_token_expiration = new_access_token.expires_at_utc.unwrap(); 149 | refresh_token = new_refresh_token; 150 | false 151 | } else { 152 | true 153 | } 154 | }; 155 | datapoint_info!( 156 | "token_auth", 157 | ("auth_type", "full_auth", String), 158 | ("service", service_name, String), 159 | ("is_error", is_error, bool), 160 | ("latency_us", start.elapsed().as_micros(), i64), 161 | ); 162 | continue; 163 | } 164 | 165 | let access_token_ttl = SystemTime::try_from(access_token_expiration.clone()) 166 | .unwrap() 167 | .duration_since(now) 168 | .unwrap_or_default(); 169 | // re-up the access token if it expires soon 170 | if access_token_ttl < Duration::from_secs(5 * 60) { 171 | let start = Instant::now(); 172 | let is_error = { 173 | if let Ok(refresh_resp) = auth_service_client 174 | .refresh_access_token(RefreshAccessTokenRequest { 175 | refresh_token: refresh_token.value.clone(), 176 | }) 177 | .await 178 | { 179 | let access_token = refresh_resp.into_inner().access_token.unwrap(); 180 | bearer_token.store(Arc::new(access_token.value.clone())); 181 | access_token_expiration = access_token.expires_at_utc.unwrap(); 182 | false 183 | } else { 184 | true 185 | } 186 | }; 187 | 188 | datapoint_info!( 189 | "token_auth", 190 | ("auth_type", "access_token", String), 191 | ("service", service_name, String), 192 | ("is_error", is_error, bool), 193 | ("latency_us", start.elapsed().as_micros(), i64), 194 | ); 195 | continue; 196 | } 197 | 198 | sleep(Duration::from_secs(5)).await; 199 | } 200 | }) 201 | } 202 | } 203 | 204 | impl Interceptor for ClientInterceptor { 205 | fn call(&mut self, mut request: Request<()>) -> Result, Status> { 206 | let l_token = ArcSwapAny::load(&self.bearer_token); 207 | if l_token.is_empty() { 208 | return Err(Status::invalid_argument("missing bearer token")); 209 | } 210 | request.metadata_mut().insert( 211 | "authorization", 212 | format!("Bearer {l_token}") 213 | .parse() 214 | .map_err(|e: InvalidMetadataValue| Status::invalid_argument(e.to_string()))?, 215 | ); 216 | 217 | Ok(request) 218 | } 219 | } 220 | 221 | pub async fn create_grpc_channel(url: String) -> BlockEngineConnectionResult { 222 | let endpoint = match url.starts_with("https") { 223 | true => Endpoint::from_shared(url) 224 | .map_err(BlockEngineConnectionError::Transport)? 225 | .tls_config(tonic::transport::ClientTlsConfig::new())?, 226 | false => Endpoint::from_shared(url).map_err(BlockEngineConnectionError::Transport)?, 227 | }; 228 | Ok(endpoint.connect().await?) 229 | } 230 | -------------------------------------------------------------------------------- /rust-toolchain.toml: -------------------------------------------------------------------------------- 1 | [toolchain] 2 | channel = "1.84" 3 | components = ["rustfmt", "rustc-dev", "clippy", "cargo"] 4 | -------------------------------------------------------------------------------- /rustfmt.toml: -------------------------------------------------------------------------------- 1 | edition = "2021" # required by rust-analyzer 2 | imports_granularity="Crate" 3 | format_code_in_doc_comments = true 4 | error_on_unformatted = true 5 | group_imports = "StdExternalCrate" 6 | -------------------------------------------------------------------------------- /scripts/create_test_listeners.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -eu 3 | 4 | # get busybox here: https://busybox.net/downloads/binaries/ 5 | 6 | echo "Rerun this script everytime shredstream-proxy is killed, as busybox nc will ignore subsequent connections" 7 | 8 | # kill any previous instances 9 | pkill busybox || true 10 | 11 | # start new listeners 12 | nohup ./busybox nc -v -u -l -p 9900 > 9900.out & 13 | nohup ./busybox nc -v -u -l -p 9901 > 9901.out & 14 | nohup ./busybox nc -v -u -l -p 9902 > 9902.out & 15 | 16 | # let socket show up 17 | sleep 0.5 18 | sudo ss --all -upn | grep 990 19 | -------------------------------------------------------------------------------- /scripts/get_tvu_port.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -eu 3 | 4 | LEDGER_DIR=${LEDGER_DIR:-"/solana/ledger"} 5 | 6 | # fetch and print port using solana tooling 7 | get_tvu_solana() { 8 | echo "Getting shred listen port using solana cli with \$LEDGER_DIR=$LEDGER_DIR" 9 | 10 | # get solana cli version 11 | SOLANA_VERSION=$(solana --version) 12 | 13 | # check the solana cli version 14 | if [[ $SOLANA_VERSION == solana-cli\ 2.* ]]; then 15 | # use agave-validator for solana cli version 2.x 16 | echo "Using agave-validator for solana cli version 2.x" 17 | agave-validator --ledger "$LEDGER_DIR" contact-info | grep "TVU:" | cut -d ':' -f 3 18 | elif [[ $SOLANA_VERSION == solana-cli\ 1.* ]]; then 19 | # use solana-validator for solana cli version 1.x 20 | echo "Using solana-validator for solana cli version 1.x" 21 | solana-validator --ledger "$LEDGER_DIR" contact-info | grep "TVU:" | cut -d ':' -f 3 22 | else 23 | # unsupported solana cli version 24 | echo "Unsupported solana cli version: $SOLANA_VERSION" 25 | fi 26 | } 27 | 28 | # fetch port using curl. not guaranteed to be accurate as we assume it uses the default port allocation order 29 | get_tvu_curl() { 30 | HOST=${HOST:-"http://localhost:8899"} 31 | echo "Getting shred listen port from \$HOST=$HOST using curl" 32 | IDENTITY_PUBKEY=$(curl --show-error --silent "$HOST" -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","id":1, "method":"getIdentity"}' | jq -r .result.identity) 33 | GOSSIP_SOCKETADDR=$(curl --show-error --silent "$HOST" -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0", "id":1, "method":"getClusterNodes"}' | jq -r ".result | map(select(.pubkey == \"$IDENTITY_PUBKEY\")) | .[0].gossip") 34 | GOSSIP_PORT=$(echo "$GOSSIP_SOCKETADDR" | cut -d ':' -f 2) 35 | 36 | # offset by 2: https://github.com/jito-foundation/jito-solana/blob/efc5f1af5442fbd6645b2debcacd555c7c4b955b/gossip/src/cluster_info.rs#L2942 37 | echo $(("$GOSSIP_PORT" + 1)) 38 | } 39 | 40 | # check solana cli and ledger directory exists 41 | if [[ -x "$(command -v solana)" && -d $LEDGER_DIR ]]; then 42 | get_tvu_solana 43 | exit 0 44 | fi 45 | 46 | # exit if jq not exists 47 | if [[ ! -x "$(command -v jq)" ]]; then 48 | echo "'jq' not found" 49 | exit 1 50 | fi 51 | 52 | # exit if curl not exists 53 | if [[ ! -x "$(command -v curl)" ]]; then 54 | echo "'curl' not found" 55 | exit 1 56 | fi 57 | 58 | get_tvu_curl 59 | --------------------------------------------------------------------------------