├── .dockerignore ├── .editorconfig ├── .env.example ├── .github ├── pull_request_template.md ├── release.yml └── workflows │ ├── release.yml │ └── test.yml ├── .gitignore ├── Cargo.lock ├── Cargo.toml ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── Solana.Dockerfile ├── accounts-selector-config.json ├── ci ├── cargo-build-test.sh ├── cargo-install-all.sh ├── create-tarball.sh ├── env.sh └── rust-version.sh ├── docker-compose.yml ├── docker ├── build.sh ├── fetch-spl.sh ├── runs.sh └── wait.sh ├── geyser-config.json ├── plerkle ├── .dockerignore ├── Cargo.toml ├── Readme.md ├── rustfmt.toml └── src │ ├── accounts_selector.rs │ ├── config.rs │ ├── error.rs │ ├── example-config.json │ ├── geyser_plugin_nft.rs │ ├── lib.rs │ ├── metrics.rs │ └── transaction_selector.rs ├── plerkle_messenger ├── Cargo.toml ├── Readme.md ├── rustfmt.toml └── src │ ├── error.rs │ ├── lib.rs │ ├── metrics.rs │ ├── plerkle_messenger.rs │ └── redis │ ├── mod.rs │ ├── redis_messenger.rs │ └── redis_pool_messenger.rs ├── plerkle_serialization ├── Cargo.toml ├── Readme.md ├── account_info.fbs ├── block_info.fbs ├── common.fbs ├── compiled_instruction.fbs ├── generate_flatbuffers_code.sh ├── rustfmt.toml ├── slot_status_info.fbs ├── src │ ├── account_info_generated.rs │ ├── block_info_generated.rs │ ├── common_generated.rs │ ├── compiled_instruction_generated.rs │ ├── deserializer │ │ ├── mod.rs │ │ └── solana.rs │ ├── error.rs │ ├── lib.rs │ ├── serializer │ │ ├── mod.rs │ │ ├── serializer_common.rs │ │ └── serializer_stable.rs │ ├── slot_status_info_generated.rs │ ├── solana_geyser_plugin_interface_shims.rs │ └── transaction_info_generated.rs └── transaction_info.fbs ├── plerkle_snapshot ├── Cargo.toml ├── LICENSE.md ├── README.md └── src │ ├── append_vec.rs │ ├── archived.rs │ ├── bin │ └── solana-snapshot-etl │ │ ├── accounts_selector.rs │ │ ├── geyser.rs │ │ ├── main.rs │ │ └── mpl_metadata.rs │ ├── lib.rs │ ├── parallel.rs │ ├── solana.rs │ └── unpacked.rs ├── rust-toolchain.toml ├── rustfmt.toml └── scripts └── docker-entrypoint.sh /.dockerignore: -------------------------------------------------------------------------------- 1 | **/target 2 | */target 3 | target 4 | db-data 5 | node-modules 6 | ledger 7 | .anchor 8 | docker/geyser-outputs 9 | docker/solana-outputs 10 | snapshot/ 11 | .env 12 | Dockerfile 13 | Solana.Dockerfile 14 | Makefile 15 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | [*] 2 | charset = utf-8 3 | end_of_line = lf 4 | indent_style = space 5 | trim_trailing_whitespace = true 6 | insert_final_newline = true 7 | 8 | [*.sh] 9 | indent_size = 2 10 | -------------------------------------------------------------------------------- /.env.example: -------------------------------------------------------------------------------- 1 | SNAPSHOTDIR=./snapshot 2 | # if running locally, the program will fail parsing .env, since the value below must be enclosed in quotes. 3 | # make sure to do so if not using Docker. 4 | PLUGIN_MESSENGER_CONFIG={messenger_type="Redis", connection_config={redis_connection_str="redis://:pass@redis.app:6379"}} 5 | 6 | # please do not remove the following line from this example 7 | # remove when creating the actual env file 8 | # vim: set ft=bash 9 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | 2 | Thank you for taking the time to make an enhancement or bug fix to the digital asset validator plugin repo. 3 | To ensure a quality PR experience this call may be recorded. Just Kidding, but we do expect a few things. 4 | 5 | 1. The Name of the PR will show up in the changelog so make it a good one, we will rename PRs or reject based on the name. 6 | 2. Please make the PR as small as possible to achieve the bugfix or feature. Big prs often are scary and hard to review. 7 | 3. Be kind to your reviewers 🤓 8 | 4. Add a good description, so we can see what the PR is all about without investing the time in the code review. We will often review code at a certain time in the day and having a list if important prs to review helps us move that along. 9 | -------------------------------------------------------------------------------- /.github/release.yml: -------------------------------------------------------------------------------- 1 | changelog: 2 | exclude: 3 | labels: 4 | - ignore-for-release 5 | categories: 6 | - title: Breaking Changes 🛠 7 | labels: 8 | - major 9 | - title: Bug Fix 🐛 10 | labels: 11 | - bug 12 | - title: Exciting New Features 🎉 13 | labels: 14 | - minor 15 | - title: Other Changes 📋 16 | labels: 17 | - "*" -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | on: 2 | push: 3 | tags: 4 | - "v*" 5 | 6 | env: 7 | CARGO_TERM_COLOR: always 8 | IMAGE_NAME: plerkle-test-validator 9 | RUST_VERSION: 1.83.0 10 | SOLANA_VERSION_STABLE: v2.1.11 11 | jobs: 12 | release-stable: 13 | runs-on: ubuntu-20-04-8-cores 14 | steps: 15 | - uses: actions/checkout@v4 16 | - name: Set env vars 17 | run: | 18 | source ci/env.sh 19 | echo "GEYSER_PLUGIN_NAME=$plugin_name" | tee -a $GITHUB_ENV 20 | echo "GEYSER_PLUGIN_LIB=lib${plugin_lib_name}" | tee -a $GITHUB_ENV 21 | - if: runner.os == 'Linux' 22 | run: | 23 | sudo apt-get update 24 | sudo apt-get upgrade 25 | wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - 26 | sudo apt-add-repository "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-10 main" 27 | sudo apt-get update 28 | sudo apt-get install -y libudev-dev libssl-dev libsasl2-dev libzstd-dev 29 | sudo apt-get install -y openssl --allow-unauthenticated 30 | sudo apt-get install -y libssl1.1 --allow-unauthenticated 31 | - uses: actions-rs/toolchain@v1 32 | with: 33 | toolchain: ${{ env.RUST_VERSION }} 34 | override: true 35 | profile: minimal 36 | components: rustfmt 37 | - name: Build Plugin 38 | run: | 39 | echo "CI_TAG=${GITHUB_REF#refs/*/}" >> "$GITHUB_ENV" 40 | echo "CI_OS_NAME=linux" >> "$GITHUB_ENV" 41 | cargo build --release --locked 42 | - name: Build release tarball 43 | run: ./ci/create-tarball.sh 44 | - name: Publish to crates registry 45 | run: | 46 | cargo publish -p plerkle_serialization --token $CARGO_TOKEN --no-verify || true 47 | sleep 30 48 | cargo publish -p plerkle_messenger --token $CARGO_TOKEN --no-verify || true 49 | shell: bash 50 | env: 51 | CARGO_TOKEN: ${{ secrets.CARGO_TOKEN }} 52 | - name: Release 53 | uses: softprops/action-gh-release@v1 54 | if: startsWith(github.ref, 'refs/tags/') 55 | with: 56 | name: STABLE ${{ env.CI_TAG }} 57 | body: | 58 | ## STABLE VERSION: 59 | Reccomended for Mainnet 60 | ${{ env.GEYSER_PLUGIN_NAME }} ${{ env.CI_TAG }} 61 | solana ${{ env.SOLANA_VERSION_STABLE }} 62 | rust ${{ env.RUST_VERSION }} 63 | files: | 64 | ${{ env.GEYSER_PLUGIN_NAME }}-release-* 65 | push-stable: 66 | runs-on: ubuntu-latest 67 | needs: release-stable 68 | permissions: 69 | packages: write 70 | contents: read 71 | steps: 72 | - uses: actions/checkout@v4 73 | - name: Build image 74 | run: docker build . --file Solana.Dockerfile --tag $IMAGE_NAME --label 'runnumber=${GITHUB_RUN_ID}' 75 | - name: Log in to registry 76 | run: echo '${{ secrets.GITHUB_TOKEN }}' | docker login ghcr.io -u $ --password-stdin 77 | - name: Push image Stable 78 | if: startsWith(github.ref, 'refs/tags/') 79 | run: | 80 | CI_TAG=${GITHUB_REF#refs/*/} 81 | IMAGE_ID=ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME 82 | # Change all uppercase to lowercase 83 | IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]') 84 | # Strip git ref prefix from version 85 | VERSION=$CI_TAG 86 | echo IMAGE_ID=$IMAGE_ID 87 | export TAG=$VERSION-${{ env.RUST_VERSION }}-${{ env.SOLANA_VERSION_STABLE }} 88 | echo VERSION=$TAG 89 | docker tag $IMAGE_NAME $IMAGE_ID:$TAG 90 | docker push $IMAGE_ID:$TAG 91 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | # Source: 2 | # https://github.com/solana-labs/solana-accountsdb-plugin-postgres/blob/master/.github/workflows/test.yml 3 | 4 | on: 5 | push: 6 | branches: 7 | - main 8 | pull_request: 9 | branches: 10 | - main 11 | 12 | env: 13 | CARGO_TERM_COLOR: always 14 | IMAGE_NAME: plerkle-test-validator 15 | RUST_VERSION: 1.83.0 16 | SOLANA_VERSION_STABLE: v2.1.11 17 | 18 | jobs: 19 | test-stable: 20 | runs-on: ubuntu-20-04-8-cores 21 | steps: 22 | - uses: actions/checkout@v4 23 | - name: Set env vars 24 | run: | 25 | source ci/env.sh 26 | echo "GEYSER_PLUGIN_NAME=$plugin_name" | tee -a $GITHUB_ENV 27 | echo "GEYSER_PLUGIN_LIB=lib${plugin_lib_name}" | tee -a $GITHUB_ENV 28 | - if: runner.os == 'Linux' 29 | run: | 30 | sudo apt-get update 31 | sudo apt-get upgrade 32 | wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - 33 | sudo apt-add-repository "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-10 main" 34 | sudo apt-get update 35 | sudo apt-get install -y libudev-dev libssl-dev libsasl2-dev libzstd-dev 36 | sudo apt-get install -y openssl --allow-unauthenticated 37 | sudo apt-get install -y libssl1.1 --allow-unauthenticated 38 | sudo apt-get install -y protobuf-compiler 39 | - uses: actions-rs/toolchain@v1 40 | with: 41 | toolchain: ${{ env.RUST_VERSION }} 42 | override: true 43 | profile: minimal 44 | components: rustfmt, clippy 45 | - uses: actions/cache@v4 46 | with: 47 | path: | 48 | ~/.cargo/registry 49 | ~/.cargo/git 50 | key: cargo-build-${{ hashFiles('**/Cargo.lock') }}-${{ env.RUST_STABLE}} 51 | - name: cargo clippy 52 | uses: actions-rs/cargo@v1 53 | with: 54 | command: clippy 55 | args: --workspace --all-targets #-- --deny=warnings 56 | - name: Build 57 | run: ./ci/cargo-build-test.sh 58 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Generated by Cargo 2 | # will have compiled files and executables 3 | /target/ 4 | db-data 5 | .idea 6 | **/target/ 7 | 8 | # These are backup files generated by rustfmt 9 | **/*.rs.bk 10 | ledger 11 | package-lock.json 12 | node_modules 13 | .anchor 14 | target 15 | *.so 16 | .vscode 17 | dist 18 | test-ledger 19 | *.swp 20 | *error.log 21 | *iml 22 | programs 23 | docker/geyser-outputs 24 | docker/solana-outputs 25 | .env 26 | snapshot 27 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | members = [ 3 | "plerkle_messenger", 4 | "plerkle", 5 | "plerkle_serialization", 6 | "plerkle_snapshot" 7 | ] 8 | resolver = "2" -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # This Dockerfile describes the build process for the snapshot 2 | # ETL tool. 3 | 4 | FROM rust:1.83.0-bookworm AS chef 5 | RUN cargo install cargo-chef 6 | WORKDIR /app 7 | 8 | FROM chef AS planner 9 | COPY . . 10 | RUN cargo chef prepare --recipe-path recipe.json 11 | 12 | FROM chef AS builder 13 | COPY --from=planner /app/recipe.json recipe.json 14 | RUN apt update && apt install -y git curl protobuf-compiler 15 | # Build dependencies - this is the caching Docker layer! 16 | RUN cargo chef cook --release --recipe-path recipe.json 17 | # Build application 18 | COPY . . 19 | RUN cargo build --release --bin solana-snapshot-etl -F standalone 20 | 21 | FROM debian:trixie-slim AS runtime 22 | WORKDIR /app 23 | COPY --from=builder /app/target/release/solana-snapshot-etl . 24 | COPY --from=builder /app/scripts/docker-entrypoint.sh . 25 | COPY ./accounts-selector-config.json . 26 | RUN groupadd -g 10001 appgroup && useradd -u 10001 -r -g appgroup -m -d /home/appuser -s /bin/bash appuser 27 | USER appuser 28 | 29 | ENTRYPOINT ["./docker-entrypoint.sh"] 30 | STOPSIGNAL SIGINT 31 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: build start dev stop test mocks lint 2 | 3 | SHELL := /bin/bash 4 | 5 | ifneq (,$(wildcard .env)) 6 | include .env 7 | export $(shell sed 's/=.*//' .env) 8 | endif 9 | 10 | export IMAGE_NAME=solana-snapshot-etl 11 | 12 | build: 13 | @docker build -f Dockerfile . -t ${IMAGE_NAME} 14 | 15 | # NOTE: make sure that Redis is reachable from the container. 16 | # If Redis is running locally, it might be feasible to run the container 17 | # with `--net=host` option specified. 18 | stream: 19 | @export SNAPSHOT_MOUNT="$$(realpath $$SNAPSHOTDIR)" && echo $$SNAPSHOT_MOUNT && docker run --env-file .env --rm -it --mount type=bind,source=$$SNAPSHOT_MOUNT,target=/app/snapshot,ro $$IMAGE_NAME 20 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## Digital Asset Validator Plugin 2 | 3 | This repo houses a validator plugin that is a key part of the Metaplex's Digital Asset RPC API. It is responsible for getting 4 | information out of the Solana validator and sending it to a message bus in a binary format. While this component was 5 | built to serve the APIs, it was designed to allow any message bus tech to be used. That being said, it can be used for many use cases. 6 | 7 | ## WARNING 8 | 9 | ``` 10 | Solana 1.10.41 or greater is required. Your calling components must support V2 GeyserPlugin types 11 | ``` 12 | 13 | It is built on the following principles. 14 | 15 | - Do as little work in the validator process as possible. 16 | - Allow any message bus tech to work. 17 | - Opinionated and efficient Wire format as a standard. 18 | - Async first 19 | 20 | ### Components 21 | 22 | 1. Plerkle -> Geyser Plugin that sends raw information to a message bus using Messenger 23 | 2. Messenger -> A message bus agnostic Messaging Library that sends Transaction, Account, Block and Slot updates in the Plerkle Serialization format. 24 | 3. Plerkle Serialization -> FlatBuffers based serialization code and schemas. This is the wire-format of Plerkle. 25 | 4. Plerkle Snapshot -> ETL for Solana accounts snapshot. 26 | 27 | ## Developing 28 | 29 | If you are building the Metaplex RPC API infrastructure please follow the instructions in [Metaplex RPC API infrastructure](https://github.com/metaplex-foundation/digital-asset-rpc-infrastructure). 30 | 31 | If you are using this plugin for your bespoke use case then the build steps are below. 32 | 33 | #### A note on formatting 34 | 35 | Since `rustfmt.toml` uses unstable configuration options, it is required to run formatting with the nightly toolchain: `cargo +nightly fmt`. 36 | 37 | ### Building Locally 38 | 39 | #### Linux 40 | 41 | `cargo build` for debug or 42 | `cargo build --release` for a release build. 43 | 44 | You will now have a libplerkle.so file in the target folder. 45 | 46 | #### Mac 47 | 48 | Building is similar to Linux, except for the extension of the library produced. 49 | Instead of a `.so` file, look for `libplerkle.dylib`. The loader does not really care what extension to link, as long as it's a proper dynamically linked object, such as a `dylib`. 50 | 51 | ### Configuration 52 | 53 | ```bash 54 | --geyser-plugin-config plugin-config.json 55 | ``` 56 | 57 | The plugin config for plerkle must have this format, but you can put whatever keys you want 58 | 59 | ```json 60 | EXAMPLE PLEASE DONT CONSIDER THIS THE PERFECT CONFIG 61 | { 62 | "libpath": "/.../libplerkle.so", 63 | "enable_metrics": false, 64 | "env": "local", 65 | "handle_startup": true, // set to false if you dont want initial account flush 66 | "accounts_selector": { 67 | "owners": [ 68 | "metaqbxxUerdq28cj1RbAWkYQm3ybzjb6a8bt518x1s" 69 | ] 70 | }, 71 | "transaction_selector": { 72 | "mentions": [ 73 | "BGUMAp9Gq7iTEuizy4pqaxsTyUCBK68MDfK752saRPUY" 74 | ] 75 | } 76 | } 77 | ``` 78 | 79 | This config file points to where your plugin library file is, and what programs it is listening to. 80 | This is the standard Solana geyser plugin config file that the validator reads. 81 | 82 | There are some other bits of configuration needed. Environment Variables. 83 | The process running the validator must have access to environment variables. Those variables are as follows 84 | 85 | ```bash 86 | RUST_LOG=warn 87 | PLUGIN_MESSENGER_CONFIG={ messenger_type="Redis", connection_config={ redis_connection_str="redis://redis" } } 88 | ``` 89 | 90 | The PLUGIN_MESSENGER_CONFIG determines which compiled messenger to select and a specific configuration for the messenger. 91 | 92 | #### Additional Configuration Examples 93 | 94 | **_Producer Configuration_** 95 | 96 | - "pipeline_size_bytes" - Maximum command size, roughly equates to the payload size. This setting locally buffers bytes in a queue to be flushed when the buffer grows past the desired amount. Default is 512MB (max redis command size) / 1000, maximum is 512MB (max redis command size) / 1000. You should test your optimal size to avoid high send latency and avoid RTT. 97 | - "local_buffer_max_window" - Maximum time to wait for the buffer to fill be for flushing. For lower traffic you dont want to be waiting around so set a max window and it will send at a minumum of every X milliseconds . Default 10 98 | - "confirmation_level" - Can be one of "Processed", "Confirmed", "Rooted". Defaults to Processed which is the level we wait for before sending. "Processed" is essentially when we first see it which can on rare cases be reverted. "Confirmed" has extremley low likley hood of being reverted but takes longer (~1k ms in our testing) to show up. "Rooted" is impossible to revert but takes the longest. 99 | - "num_workers" - This is the number of workers who will pickup notifications from the plugin and send them to the messenger. Default is 5 100 | - "account_stream_size" - default value 100_000_000 101 | - "slot_stream_size" - default value 100_000 102 | - "transaction_stream_size" - default value 10_000_000 103 | - "block_stream_size" - default value 100_000 104 | 105 | ``` 106 | Lower Scale Low network latency 107 | 108 | PLUGIN_MESSENGER_CONFIG={pipeline_size_bytes=1000000,local_buffer_max_window=10, messenger_type="Redis", connection_config={ redis_connection_str="redis://redis" } } 109 | 110 | High Scale Higher latency 111 | 112 | PLUGIN_MESSENGER_CONFIG={pipeline_size_bytes=50000000,local_buffer_max_window=500, messenger_type="Redis", connection_config={ redis_connection_str="redis://redis" } } 113 | 114 | 115 | ``` 116 | 117 | **_Consumer Configuration_** 118 | 119 | - "retries" - Amount of times to deliver the message. If delivered this many times and not ACKed, then it is deleted 120 | - "batch_size" - Max Amout of messages to grab within the wait timeout window. 121 | - "message_wait_timeout" - Amount of time the consumer will keep the stream open and wait for messages 122 | - "idle_timeout" - Amount of time a consumer can have the message before it goes back on the queue 123 | - "consumer_id" - VERY important. This is used to scale horizontally so messages arent duplicated over instances.Make sure this is different per instance 124 | 125 | ``` 126 | 127 | PLUGIN_MESSENGER_CONFIG={batch_size=1000, message_wait_timeout=5, retries=5, consumer_id="random_string",messenger_type="Redis", connection_config={ redis_connection_str="redis://redis" } } 128 | PLUGIN_ACCOUNT_STREAM_SIZE=250000000 129 | PLUGIN_SLOT_STREAM_SIZE=250000 130 | PLUGIN_TRANSACTION_STREAM_SIZE=25000000 131 | PLUGIN_BLOCK_STREAM_SIZE=250000 132 | 133 | ``` 134 | 135 | NOTE: in 1.4.0 we are not sending to slot status. 136 | 137 | ### Metrics 138 | 139 | The plugin exposes the following statsd metrics 140 | 141 | - count plugin.startup -> times the plugin started 142 | - time message_send_queue_time -> time spent on messenger internal buffer 143 | - time message_send_latency -> rtt time to messenger bus 144 | - count account_seen_event , tags: owner , is_startup -> number of account events filtered and seen 145 | - time startup.timer -> startup flush timer 146 | - count transaction_seen_event tags slot-idx -> number of filtered txns seen 147 | 148 | ### Building With Docker 149 | 150 | This repo contains a docker file that allows you to run and test the plerkle plugin using a test validator. 151 | To test it you can build the container with`docker compose build` and run it with `docker compose up`. 152 | 153 | You will want to change the programs you are listening to in `./docker/runs.sh`. Once you spin up the validator send your transactions to the docker host as you would a normal RPC. 154 | 155 | Any program .so files you add to the /so/ file upon running the docker compose system will be added to the local validator. 156 | 157 | You need to name the so file what you want the public key to be: 158 | 159 | ```bash 160 | metaqbxxUerdq28cj1RbAWkYQm3ybzjb6a8bt518x1s.so 161 | ``` 162 | 163 | This is because of this line in the `docker-compose.yml` file. 164 | 165 | ```yaml 166 | - ./programs:/so/:ro 167 | ``` 168 | 169 | You can comment this out if you dont want it. 170 | 171 | ### Using docker output for solana and geyser artifacts 172 | 173 | You can run `./docker/build.sh` to create a build of the geyser plugin and solana, then output the artifacts 174 | to your local filesystem so you can run the validator. 175 | 176 | Note that differences in image distro could cause incompatible GLibc versions. 177 | 178 | ### Crates 179 | 180 | NOTE WE DO NOT PUBLISH THE PLUGIN ANY MORE: 181 | 182 | plerkle_messenger-https://crates.io/crates/plerkle_messenger 183 | plerkle_serialization-https://crates.io/crates/plerkle_serialization 184 | 185 | ## Snapshot ETL 186 | 187 | The Plerkle snapshot tool can be used for parsing Solana account snapshots. The repository already includes a pre-configured `accounts-selector-config.json` file, which is ready to use. The only thing you might want to modify is the list of programs in `accounts-selector-config.json`; otherwise, you can leave the configurations as they are. 188 | 189 | Before running the tool, it's important to create an .env file, modeled after .env.example. In this file, you should specify the path to the directory containing the snapshots as well as the snapshot redis connection details. 190 | 191 | Once everything is set up, you can build the Docker container for ETL by running: 192 | 193 | ``` 194 | make build 195 | ``` 196 | 197 | This will create a Docker container with the Geyser plugin and ETL fully built and ready to use. 198 | 199 | The next step is to run the ETL: 200 | 201 | ``` 202 | make stream 203 | ``` 204 | 205 | This command will launch the ETL Docker container. It will load the snapshot archives, the Geyser plugin binary, and stream all the accounts from the snapshot to the plugin. 206 | -------------------------------------------------------------------------------- /Solana.Dockerfile: -------------------------------------------------------------------------------- 1 | ARG SOLANA_VERSION=v2.1.11 2 | ARG RUST_VERSION=1.83.0 3 | FROM rust:$RUST_VERSION-bullseye AS builder 4 | RUN apt-get update \ 5 | && apt-get -y install \ 6 | wget \ 7 | curl \ 8 | build-essential \ 9 | software-properties-common \ 10 | lsb-release \ 11 | libelf-dev \ 12 | linux-headers-generic \ 13 | pkg-config \ 14 | curl \ 15 | cmake \ 16 | protobuf-compiler 17 | RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y 18 | ENV PATH="/root/.cargo/bin:${PATH}" 19 | WORKDIR /rust/ 20 | COPY plerkle_serialization /rust/plerkle_serialization 21 | COPY plerkle_snapshot /rust/plerkle_snapshot 22 | COPY plerkle_messenger /rust/plerkle_messenger 23 | COPY plerkle /rust/plerkle 24 | COPY Cargo.toml /rust/ 25 | COPY Cargo.lock /rust/ 26 | WORKDIR /rust 27 | RUN cargo build --release --locked 28 | 29 | FROM anzaxyz/agave:$SOLANA_VERSION 30 | COPY --from=builder /rust/target/release/libplerkle.so /plugin/plugin.so 31 | COPY ./docker . 32 | RUN chmod +x ./*.sh 33 | ENTRYPOINT [ "./runs.sh" ] 34 | CMD [""] 35 | -------------------------------------------------------------------------------- /accounts-selector-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "owners": [ 3 | "metaqbxxUerdq28cj1RbAWkYQm3ybzjb6a8bt518x1s", 4 | "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", 5 | "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", 6 | "CoREENxT6tW1HoK8ypY1SxRMZTcVPm7R94rH4PZNhX7d" 7 | ], 8 | "accounts": [], 9 | "select_all_accounts": false 10 | } 11 | -------------------------------------------------------------------------------- /ci/cargo-build-test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Source: 4 | # https://github.com/solana-labs/solana-accountsdb-plugin-postgres/blob/master/ci/cargo-build-test.sh 5 | 6 | set -e 7 | cd "$(dirname "$0")/.." 8 | 9 | source ./ci/rust-version.sh stable 10 | 11 | export RUSTFLAGS="" 12 | export RUSTBACKTRACE=1 13 | 14 | set -x 15 | 16 | # Build/test all host crates 17 | cargo +"$rust_stable" build --locked 18 | cargo +"$rust_stable" test --locked -- --nocapture 19 | 20 | exit 0 21 | -------------------------------------------------------------------------------- /ci/cargo-install-all.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | 5 | 6 | usage() { 7 | exitcode=0 8 | if [[ -n "$1" ]]; then 9 | exitcode=1 10 | echo "Error: $*" 11 | fi 12 | cat <] [--debug] 14 | EOF 15 | exit $exitcode 16 | } 17 | 18 | case "$CI_OS_NAME" in 19 | osx) 20 | libExt=dylib 21 | ;; 22 | linux) 23 | libExt=so 24 | ;; 25 | *) 26 | echo CI_OS_NAME unsupported 27 | exit 1 28 | ;; 29 | esac 30 | 31 | maybeRustVersion= 32 | installDir= 33 | buildVariant=release 34 | maybeReleaseFlag=--release 35 | 36 | while [[ -n $1 ]]; do 37 | if [[ ${1:0:1} = - ]]; then 38 | if [[ $1 = --debug ]]; then 39 | maybeReleaseFlag= 40 | buildVariant=debug 41 | shift 42 | else 43 | usage "Unknown option: $1" 44 | fi 45 | elif [[ ${1:0:1} = \+ ]]; then 46 | maybeRustVersion=$1 47 | shift 48 | else 49 | installDir=$1 50 | shift 51 | fi 52 | done 53 | 54 | if [[ -z "$installDir" ]]; then 55 | usage "Install directory not specified" 56 | exit 1 57 | fi 58 | 59 | installDir="$(mkdir -p "$installDir"; cd "$installDir"; pwd)" 60 | 61 | echo "Install location: $installDir ($buildVariant)" 62 | 63 | cd "$(dirname "$0")"/.. 64 | 65 | SECONDS=0 66 | 67 | mkdir -p "$installDir/lib" 68 | 69 | ( 70 | set -x 71 | # shellcheck disable=SC2086 # Don't want to double quote $rust_version 72 | cargo $maybeRustVersion build $maybeReleaseFlag --lib 73 | ) 74 | 75 | cp -fv "target/$buildVariant/${GEYSER_PLUGIN_LIB}.$libExt" "$installDir"/lib/ 76 | 77 | echo "Done after $SECONDS seconds" 78 | 79 | -------------------------------------------------------------------------------- /ci/create-tarball.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | 4 | cd "$(dirname "$0")/.." 5 | 6 | case "$CI_OS_NAME" in 7 | osx) 8 | _cputype="$(uname -m)" 9 | if [[ $_cputype = arm64 ]]; then 10 | _cputype=aarch64 11 | fi 12 | TARGET=${_cputype}-apple-darwin 13 | ;; 14 | linux) 15 | TARGET=x86_64-unknown-linux-gnu 16 | ;; 17 | *) 18 | echo CI_OS_NAME unsupported 19 | exit 1 20 | ;; 21 | esac 22 | 23 | RELEASE_BASENAME="${RELEASE_BASENAME:="${GEYSER_PLUGIN_NAME}-release"}" 24 | TARBALL_BASENAME="${TARBALL_BASENAME:="$RELEASE_BASENAME"}" 25 | 26 | echo --- Creating release tarball 27 | ( 28 | set -x 29 | rm -rf "${RELEASE_BASENAME:?}"/ 30 | mkdir "${RELEASE_BASENAME}"/ 31 | 32 | COMMIT="$(git rev-parse HEAD)" 33 | 34 | ( 35 | echo "channel: $CI_TAG" 36 | echo "commit: $COMMIT" 37 | echo "target: $TARGET" 38 | ) > "${RELEASE_BASENAME}"/version.yml 39 | 40 | # Make CHANNEL available to include in the software version information 41 | export CHANNEL 42 | 43 | ci/cargo-install-all.sh stable "${RELEASE_BASENAME}" 44 | 45 | tar cvf "${TARBALL_BASENAME}"-$TARGET.tar "${RELEASE_BASENAME}" 46 | bzip2 "${TARBALL_BASENAME}"-$TARGET.tar 47 | cp "${RELEASE_BASENAME}"/version.yml "${TARBALL_BASENAME}"-$TARGET.yml 48 | ) 49 | 50 | echo --- ok 51 | -------------------------------------------------------------------------------- /ci/env.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | plugin_name=plerkle 3 | plugin_lib_name=plerkle 4 | -------------------------------------------------------------------------------- /ci/rust-version.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Source: 4 | # https://github.com/solana-labs/solana-accountsdb-plugin-postgres/blob/master/ci/rust-version.sh 5 | 6 | # 7 | # This file maintains the rust versions for use by CI. 8 | # 9 | # Obtain the environment variables without any automatic toolchain updating: 10 | # $ source ci/rust-version.sh 11 | # 12 | # Obtain the environment variables updating both stable and nightly, only stable, or 13 | # only nightly: 14 | # $ source ci/rust-version.sh all 15 | # $ source ci/rust-version.sh stable 16 | # $ source ci/rust-version.sh nightly 17 | 18 | # Then to build with either stable or nightly: 19 | # $ cargo +"$rust_stable" build 20 | # $ cargo +"$rust_nightly" build 21 | # 22 | 23 | if [[ -n $RUST_VERSION ]]; then 24 | stable_version="$RUST_VERSION" 25 | else 26 | stable_version=latest 27 | fi 28 | 29 | if [[ -n $RUST_NIGHTLY_VERSION ]]; then 30 | nightly_version="$RUST_NIGHTLY_VERSION" 31 | else 32 | nightly_version=latest 33 | fi 34 | 35 | 36 | export rust_stable="$stable_version" 37 | export rust_stable_docker_image=rust:"$stable_version" 38 | 39 | export rust_nightly_docker_image=shepmaster/rust-nightly:"$nightly_version" 40 | 41 | [[ -z $1 ]] || ( 42 | rustup_install() { 43 | declare toolchain=$1 44 | if ! cargo +"$toolchain" -V > /dev/null; then 45 | echo "$0: Missing toolchain? Installing...: $toolchain" >&2 46 | rustup install "$toolchain" 47 | cargo +"$toolchain" -V 48 | fi 49 | } 50 | 51 | set -e 52 | cd "$(dirname "${BASH_SOURCE[0]}")" 53 | case $1 in 54 | stable) 55 | rustup_install "$rust_stable" 56 | ;; 57 | nightly) 58 | rustup_install "nightly" 59 | ;; 60 | all) 61 | rustup_install "$rust_stable" 62 | rustup_install "nightly" 63 | ;; 64 | *) 65 | echo "$0: Note: ignoring unknown argument: $1" >&2 66 | ;; 67 | esac 68 | ) 69 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3.9" 2 | services: 3 | redis: 4 | image: "redis:alpine" 5 | ports: 6 | - "6379:6379" 7 | solana: 8 | build: 9 | context: . 10 | dockerfile: Solana.Dockerfile 11 | volumes: 12 | - ./ledger:/config:rw 13 | - ./programs:/so/:ro 14 | environment: 15 | RUST_LOG: warn 16 | PLUGIN_CONFIG_RELOAD_TTL: 300 17 | PLUGIN_MESSENGER_CONFIG.messenger_type: "Redis" 18 | PLUGIN_MESSENGER_CONFIG.connection_config: '{redis_connection_str="redis://redis"}' 19 | ports: 20 | - "8900:8900" 21 | - "8001:8001" 22 | - "8899:8899" 23 | - "9900:9900" 24 | -------------------------------------------------------------------------------- /docker/build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | # builds the Solana.Dockerfile image and outputs build artefacts to docker-output directory, you can use this 3 | # to get the geyser plugin build and solana validator build to run on a node 4 | set -eux 5 | 6 | SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" 7 | 8 | docker build -t das-geyser/build -f Solana.Dockerfile . 9 | docker create --name temp das-geyser/build 10 | mkdir -p $SCRIPT_DIR/geyser-outputs 11 | mkdir -p $SCRIPT_DIR/solana-outputs 12 | # copy plugin .so file 13 | docker container cp temp:/plugin $SCRIPT_DIR/geyser-outputs 14 | # copy solana executables 15 | docker container cp temp:/usr/bin $SCRIPT_DIR/solana-outputs 16 | docker rm temp 17 | -------------------------------------------------------------------------------- /docker/fetch-spl.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Fetches the latest SPL programs and produces the solana-genesis command-line 4 | # arguments needed to install them 5 | # 6 | 7 | set -e 8 | 9 | fetch_program() { 10 | declare name=$1 11 | declare version=$2 12 | declare address=$3 13 | declare loader=$4 14 | 15 | declare so=spl_$name-$version.so 16 | 17 | genesis_args+=(--bpf-program "$address" "$loader" "$so") 18 | 19 | if [[ -r $so ]]; then 20 | return 21 | fi 22 | 23 | if [[ -r ~/.cache/solana-spl/$so ]]; then 24 | cp ~/.cache/solana-spl/"$so" "$so" 25 | else 26 | echo "Downloading $name $version" 27 | so_name="spl_${name//-/_}.so" 28 | ( 29 | set -x 30 | curl -L --retry 5 --retry-delay 2 --retry-connrefused \ 31 | -o "$so" \ 32 | "https://github.com/solana-labs/solana-program-library/releases/download/$name-v$version/$so_name" 33 | ) 34 | 35 | mkdir -p ~/.cache/solana-spl 36 | cp "$so" ~/.cache/solana-spl/"$so" 37 | fi 38 | 39 | } 40 | 41 | fetch_program token 3.2.0 TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA BPFLoader2111111111111111111111111111111111 42 | fetch_program memo 1.0.0 Memo1UhkJRfHyvLMcVucJwxXeuD728EqVDDwQDxFMNo BPFLoader1111111111111111111111111111111111 43 | fetch_program memo 3.0.0 MemoSq4gqABAXKb96qnH8TysNcWxMyWCqXgDLGmfcHr BPFLoader2111111111111111111111111111111111 44 | fetch_program associated-token-account 1.0.3 ATokenGPvbdGVxr1b2hvZbsiqW5xWH25efTNsLJA8knL BPFLoader2111111111111111111111111111111111 45 | fetch_program feature-proposal 1.0.0 Feat1YXHhH6t1juaWF74WLcfv4XoNocjXA6sPWHNgAse BPFLoader2111111111111111111111111111111111 46 | 47 | echo "${genesis_args[@]}" > spl-genesis-args.sh 48 | 49 | echo 50 | echo "Available SPL programs:" 51 | ls -l spl_*.so 52 | 53 | echo 54 | echo "solana-genesis command-line arguments (spl-genesis-args.sh):" 55 | cat spl-genesis-args.sh -------------------------------------------------------------------------------- /docker/runs.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Run a minimal Solana cluster. Ctrl-C to exit. 4 | # 5 | # Before running this script ensure standard Solana programs are available 6 | # in the PATH, or that `cargo build` ran successfully 7 | # change 8 | # 9 | 10 | set -e 11 | cat << EOL > config.yaml 12 | json_rpc_url: http://localhost:8899 13 | websocket_url: ws://localhost:8899 14 | commitment: finalized 15 | EOL 16 | 17 | mkdir plugin-config && true 18 | if [[ ! -f /plugin-config/accountsdb-plugin-config.json ]] 19 | then 20 | cat << EOL > /plugin-config/accountsdb-plugin-config.json 21 | { 22 | "libpath": "/plugin/plugin.so", 23 | "accounts_selector" : { 24 | "owners" : [ 25 | "metaqbxxUerdq28cj1RbAWkYQm3ybzjb6a8bt518x1s", 26 | "GRoLLMza82AiYN7W9S9KCCtCyyPRAQP2ifBy4v4D5RMD", 27 | "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", 28 | "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", 29 | "ATokenGPvbdGVxr1b2hvZbsiqW5xWH25efTNsLJA8knL", 30 | "BGUMAp9Gq7iTEuizy4pqaxsTyUCBK68MDfK752saRPUY", 31 | "cndy3Z4yapfJBmL3ShUp5exZKqR3z33thTzeNMm2gRZ", 32 | "CndyV3LdqHUfDLmE5naZjVN8rBZz4tqhdefbAnjHG3JR", 33 | "Guard1JwRhJkVH6XZhzoYxeBVQe872VH6QggF4BWmS9g" 34 | ] 35 | }, 36 | "transaction_selector" : { 37 | "mentions" : [ 38 | "metaqbxxUerdq28cj1RbAWkYQm3ybzjb6a8bt518x1s", 39 | "GRoLLMza82AiYN7W9S9KCCtCyyPRAQP2ifBy4v4D5RMD", 40 | "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", 41 | "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", 42 | "ATokenGPvbdGVxr1b2hvZbsiqW5xWH25efTNsLJA8knL", 43 | "BGUMAp9Gq7iTEuizy4pqaxsTyUCBK68MDfK752saRPUY", 44 | "cndy3Z4yapfJBmL3ShUp5exZKqR3z33thTzeNMm2gRZ", 45 | "CndyV3LdqHUfDLmE5naZjVN8rBZz4tqhdefbAnjHG3JR", 46 | "Guard1JwRhJkVH6XZhzoYxeBVQe872VH6QggF4BWmS9g" 47 | ] 48 | } 49 | } 50 | EOL 51 | fi 52 | 53 | 54 | programs=() 55 | if [ "$(ls -A /so)" ]; then 56 | for prog in /so/*; do 57 | programs+=("--bpf-program" "$(basename $prog .so)" "$prog") 58 | done 59 | fi 60 | 61 | export RUST_BACKTRACE=1 62 | dataDir=$PWD/config/"$(basename "$0" .sh)" 63 | ledgerDir=$PWD/config/ledger 64 | mkdir -p "$dataDir" "$ledgerDir" 65 | echo $ledgerDir 66 | echo $dataDir 67 | ls -la /so/ 68 | args=( 69 | --config config.yaml 70 | --log 71 | --reset 72 | --limit-ledger-size 10000000000000000 73 | --rpc-port 8899 74 | --geyser-plugin-config /plugin-config/accountsdb-plugin-config.json 75 | ) 76 | # shellcheck disable=SC2086 77 | cat /plugin-config/accountsdb-plugin-config.json 78 | ls -la /so/ 79 | solana-test-validator "${programs[@]}" "${args[@]}" $SOLANA_RUN_SH_VALIDATOR_ARGS 80 | -------------------------------------------------------------------------------- /docker/wait.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Use this script to test if a given TCP host/port are available 3 | 4 | WAITFORIT_cmdname=${0##*/} 5 | 6 | echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi } 7 | 8 | usage() 9 | { 10 | cat << USAGE >&2 11 | Usage: 12 | $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args] 13 | -h HOST | --host=HOST Host or IP under test 14 | -p PORT | --port=PORT TCP port under test 15 | Alternatively, you specify the host and port as host:port 16 | -s | --strict Only execute subcommand if the test succeeds 17 | -q | --quiet Don't output any status messages 18 | -t TIMEOUT | --timeout=TIMEOUT 19 | Timeout in seconds, zero for no timeout 20 | -- COMMAND ARGS Execute command with args after the test finishes 21 | USAGE 22 | exit 1 23 | } 24 | 25 | wait_for() 26 | { 27 | if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then 28 | echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" 29 | else 30 | echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout" 31 | fi 32 | WAITFORIT_start_ts=$(date +%s) 33 | while : 34 | do 35 | if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then 36 | nc -z $WAITFORIT_HOST $WAITFORIT_PORT 37 | WAITFORIT_result=$? 38 | else 39 | (echo -n > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1 40 | WAITFORIT_result=$? 41 | fi 42 | if [[ $WAITFORIT_result -eq 0 ]]; then 43 | WAITFORIT_end_ts=$(date +%s) 44 | echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds" 45 | break 46 | fi 47 | sleep 1 48 | done 49 | return $WAITFORIT_result 50 | } 51 | 52 | wait_for_wrapper() 53 | { 54 | # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692 55 | if [[ $WAITFORIT_QUIET -eq 1 ]]; then 56 | timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & 57 | else 58 | timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & 59 | fi 60 | WAITFORIT_PID=$! 61 | trap "kill -INT -$WAITFORIT_PID" INT 62 | wait $WAITFORIT_PID 63 | WAITFORIT_RESULT=$? 64 | if [[ $WAITFORIT_RESULT -ne 0 ]]; then 65 | echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" 66 | fi 67 | return $WAITFORIT_RESULT 68 | } 69 | 70 | # process arguments 71 | while [[ $# -gt 0 ]] 72 | do 73 | case "$1" in 74 | *:* ) 75 | WAITFORIT_hostport=(${1//:/ }) 76 | WAITFORIT_HOST=${WAITFORIT_hostport[0]} 77 | WAITFORIT_PORT=${WAITFORIT_hostport[1]} 78 | shift 1 79 | ;; 80 | --child) 81 | WAITFORIT_CHILD=1 82 | shift 1 83 | ;; 84 | -q | --quiet) 85 | WAITFORIT_QUIET=1 86 | shift 1 87 | ;; 88 | -s | --strict) 89 | WAITFORIT_STRICT=1 90 | shift 1 91 | ;; 92 | -h) 93 | WAITFORIT_HOST="$2" 94 | if [[ $WAITFORIT_HOST == "" ]]; then break; fi 95 | shift 2 96 | ;; 97 | --host=*) 98 | WAITFORIT_HOST="${1#*=}" 99 | shift 1 100 | ;; 101 | -p) 102 | WAITFORIT_PORT="$2" 103 | if [[ $WAITFORIT_PORT == "" ]]; then break; fi 104 | shift 2 105 | ;; 106 | --port=*) 107 | WAITFORIT_PORT="${1#*=}" 108 | shift 1 109 | ;; 110 | -t) 111 | WAITFORIT_TIMEOUT="$2" 112 | if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi 113 | shift 2 114 | ;; 115 | --timeout=*) 116 | WAITFORIT_TIMEOUT="${1#*=}" 117 | shift 1 118 | ;; 119 | --) 120 | shift 121 | WAITFORIT_CLI=("$@") 122 | break 123 | ;; 124 | --help) 125 | usage 126 | ;; 127 | *) 128 | echoerr "Unknown argument: $1" 129 | usage 130 | ;; 131 | esac 132 | done 133 | 134 | if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then 135 | echoerr "Error: you need to provide a host and port to test." 136 | usage 137 | fi 138 | 139 | WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15} 140 | WAITFORIT_STRICT=${WAITFORIT_STRICT:-0} 141 | WAITFORIT_CHILD=${WAITFORIT_CHILD:-0} 142 | WAITFORIT_QUIET=${WAITFORIT_QUIET:-0} 143 | 144 | # Check to see if timeout is from busybox? 145 | WAITFORIT_TIMEOUT_PATH=$(type -p timeout) 146 | WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH) 147 | 148 | WAITFORIT_BUSYTIMEFLAG="" 149 | if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then 150 | WAITFORIT_ISBUSY=1 151 | # Check if busybox timeout uses -t flag 152 | # (recent Alpine versions don't support -t anymore) 153 | if timeout &>/dev/stdout | grep -q -e '-t '; then 154 | WAITFORIT_BUSYTIMEFLAG="-t" 155 | fi 156 | else 157 | WAITFORIT_ISBUSY=0 158 | fi 159 | 160 | if [[ $WAITFORIT_CHILD -gt 0 ]]; then 161 | wait_for 162 | WAITFORIT_RESULT=$? 163 | exit $WAITFORIT_RESULT 164 | else 165 | if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then 166 | wait_for_wrapper 167 | WAITFORIT_RESULT=$? 168 | else 169 | wait_for 170 | WAITFORIT_RESULT=$? 171 | fi 172 | fi 173 | 174 | if [[ $WAITFORIT_CLI != "" ]]; then 175 | if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then 176 | echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess" 177 | exit $WAITFORIT_RESULT 178 | fi 179 | exec "${WAITFORIT_CLI[@]}" 180 | else 181 | exit $WAITFORIT_RESULT 182 | fi 183 | -------------------------------------------------------------------------------- /geyser-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "libpath": "./plerkle/target/release/libplerkle.so", 3 | "enable_metrics": false, 4 | "env": "local", 5 | "handle_startup": true, 6 | "accounts_selector": { 7 | "owners": [ 8 | "metaqbxxUerdq28cj1RbAWkYQm3ybzjb6a8bt518x1s", 9 | "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", 10 | "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", 11 | "CoREENxT6tW1HoK8ypY1SxRMZTcVPm7R94rH4PZNhX7d" 12 | ] 13 | }, 14 | "transaction_selector": { 15 | "mentions": [] 16 | } 17 | } -------------------------------------------------------------------------------- /plerkle/.dockerignore: -------------------------------------------------------------------------------- 1 | target -------------------------------------------------------------------------------- /plerkle/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "plerkle" 3 | description = "Geyser plugin with dynamic config reloading, message bus agnostic abstractions and a whole lot of fun." 4 | version = "1.12.0" 5 | authors = ["Metaplex Developers "] 6 | repository = "https://github.com/metaplex-foundation/digital-asset-validator-plugin" 7 | license = "AGPL-3.0" 8 | edition = "2021" 9 | readme = "Readme.md" 10 | 11 | [lib] 12 | crate-type = ["cdylib", "rlib"] 13 | 14 | [dependencies] 15 | log = "0.4.11" 16 | async-trait = "0.1.53" 17 | solana-sdk = { version = "2.1" } 18 | solana-transaction-status = { version = "2.1" } 19 | agave-geyser-plugin-interface = { version = "2.1" } 20 | solana-logger = { version = "2.1" } 21 | thiserror = "1.0.30" 22 | base64 = "0.21.0" 23 | lazy_static = "1.4.0" 24 | bs58 = "0.4.0" 25 | bytemuck = "1.7.2" 26 | serde = "1.0.133" 27 | serde_derive = "1.0.103" 28 | serde_json = "1.0.74" 29 | cadence = "0.29.0" 30 | cadence-macros = "0.29.0" 31 | chrono = "0.4.19" 32 | tracing = "0.1.37" 33 | tracing-subscriber = { version = "0.3.16", features = [ 34 | "json", 35 | "env-filter", 36 | "ansi", 37 | ] } 38 | hex = "0.4.3" 39 | plerkle_messenger = { path = "../plerkle_messenger", version = "2.0.0" } 40 | flatbuffers = "23.1.21" 41 | plerkle_serialization = { path = "../plerkle_serialization", version = "2.0.0" } 42 | tokio = { version = "1.23.0", features = ["full"] } 43 | figment = { version = "0.10.6", features = ["env", "test"] } 44 | dashmap = {version = "5.4.0"} 45 | 46 | [dependencies.num-integer] 47 | version = "0.1.44" 48 | default-features = false 49 | 50 | [package.metadata.docs.rs] 51 | targets = ["x86_64-unknown-linux-gnu"] 52 | 53 | -------------------------------------------------------------------------------- /plerkle/Readme.md: -------------------------------------------------------------------------------- 1 | # Plerkle 2 | 3 | Geyser Plugin that sends raw information to a message bus using Messenger -------------------------------------------------------------------------------- /plerkle/rustfmt.toml: -------------------------------------------------------------------------------- 1 | edition = "2021" 2 | imports_granularity="Crate" 3 | reorder_imports = true 4 | -------------------------------------------------------------------------------- /plerkle/src/accounts_selector.rs: -------------------------------------------------------------------------------- 1 | use log::*; 2 | use std::collections::HashSet; 3 | 4 | #[derive(Debug)] 5 | pub(crate) struct AccountsSelector { 6 | pub accounts: HashSet>, 7 | pub owners: HashSet>, 8 | pub select_all_accounts: bool, 9 | } 10 | 11 | impl AccountsSelector { 12 | pub fn default() -> Self { 13 | AccountsSelector { 14 | accounts: HashSet::default(), 15 | owners: HashSet::default(), 16 | select_all_accounts: true, 17 | } 18 | } 19 | 20 | pub fn new(accounts: &[String], owners: &[String]) -> Self { 21 | info!( 22 | "Creating AccountsSelector from accounts: {:?}, owners: {:?}", 23 | accounts, owners 24 | ); 25 | 26 | let select_all_accounts = accounts.iter().any(|key| key == "*"); 27 | if select_all_accounts { 28 | return AccountsSelector { 29 | accounts: HashSet::default(), 30 | owners: HashSet::default(), 31 | select_all_accounts, 32 | }; 33 | } 34 | let accounts = accounts 35 | .iter() 36 | .map(|key| bs58::decode(key).into_vec().unwrap()) 37 | .collect(); 38 | let owners = owners 39 | .iter() 40 | .map(|key| bs58::decode(key).into_vec().unwrap()) 41 | .collect(); 42 | AccountsSelector { 43 | accounts, 44 | owners, 45 | select_all_accounts, 46 | } 47 | } 48 | 49 | pub fn is_account_selected(&self, account: &[u8], owner: &[u8]) -> bool { 50 | self.select_all_accounts || self.accounts.contains(account) || self.owners.contains(owner) 51 | } 52 | 53 | /// Check if any account is of interested at all 54 | pub fn is_enabled(&self) -> bool { 55 | self.select_all_accounts || !self.accounts.is_empty() || !self.owners.is_empty() 56 | } 57 | } 58 | 59 | #[cfg(test)] 60 | pub(crate) mod tests { 61 | use super::*; 62 | 63 | #[test] 64 | fn test_create_accounts_selector() { 65 | AccountsSelector::new( 66 | &["9xQeWvG816bUx9EPjHmaT23yvVM2ZWbrrpZb9PusVFin".to_string()], 67 | &[], 68 | ); 69 | 70 | AccountsSelector::new( 71 | &[], 72 | &["9xQeWvG816bUx9EPjHmaT23yvVM2ZWbrrpZb9PusVFin".to_string()], 73 | ); 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /plerkle/src/config.rs: -------------------------------------------------------------------------------- 1 | use std::env; 2 | 3 | use tracing_subscriber::fmt; 4 | 5 | pub fn init_logger() { 6 | // tracing doesn't seem to load RUST_LOG even though its supposed to, set it 7 | // manually 8 | let env_filter = env::var("RUST_LOG").unwrap_or_else(|_| "info".to_string()); 9 | fmt() 10 | .with_env_filter(env_filter) 11 | .event_format(fmt::format::json()) 12 | .init(); 13 | } 14 | -------------------------------------------------------------------------------- /plerkle/src/error.rs: -------------------------------------------------------------------------------- 1 | use thiserror::Error; 2 | 3 | use agave_geyser_plugin_interface::geyser_plugin_interface::GeyserPluginError; 4 | 5 | #[derive(Error, Debug)] 6 | pub enum PlerkleError { 7 | #[error("General Plugin Config Error ({msg})")] 8 | GeneralPluginConfigError { msg: String }, 9 | 10 | #[error("Error connecting to the backend data store. Error message: ({msg})")] 11 | DataStoreConnectionError { msg: String }, 12 | 13 | #[error("Error preparing data store schema. Error message: ({msg})")] 14 | DataSchemaError { msg: String }, 15 | 16 | #[error("Error preparing data store schema. Error message: ({msg})")] 17 | ConfigurationError { msg: String }, 18 | 19 | #[error("Malformed Anchor Event")] 20 | EventError {}, 21 | 22 | #[error("Unable to Send Event to Stream ({msg})")] 23 | EventStreamError { msg: String }, 24 | 25 | #[error("Unable to acquire lock for updating slots seen. Error message: ({msg})")] 26 | SlotsSeenLockError { msg: String }, 27 | } 28 | 29 | // Implement the From trait for the PlerkleError to convert it into GeyserPluginError 30 | impl From for GeyserPluginError { 31 | fn from(err: PlerkleError) -> Self { 32 | GeyserPluginError::Custom(Box::new(err)) 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /plerkle/src/example-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "libpath": "/plugin/plugin.so", 3 | "enable_metrics": false, 4 | "env": "local", 5 | "handle_startup": true, 6 | "accounts_selector": { 7 | "owners": [ 8 | "metaqbxxUerdq28cj1RbAWkYQm3ybzjb6a8bt518x1s", 9 | "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", 10 | "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", 11 | "BGUMAp9Gq7iTEuizy4pqaxsTyUCBK68MDfK752saRPUY" 12 | ] 13 | }, 14 | "transaction_selector": { 15 | "mentions": [ 16 | "BGUMAp9Gq7iTEuizy4pqaxsTyUCBK68MDfK752saRPUY" 17 | ] 18 | } 19 | } -------------------------------------------------------------------------------- /plerkle/src/lib.rs: -------------------------------------------------------------------------------- 1 | pub mod accounts_selector; 2 | pub mod config; 3 | pub mod error; 4 | pub mod geyser_plugin_nft; 5 | pub mod metrics; 6 | pub mod transaction_selector; 7 | -------------------------------------------------------------------------------- /plerkle/src/metrics.rs: -------------------------------------------------------------------------------- 1 | use cadence_macros::is_global_default_set; 2 | 3 | pub fn safe_metric(f: F) { 4 | if is_global_default_set() { 5 | f() 6 | } 7 | } 8 | 9 | #[macro_export] 10 | macro_rules! metric { 11 | {$($block:stmt;)*} => { 12 | 13 | if is_global_default_set() { 14 | $( 15 | $block 16 | )* 17 | } 18 | 19 | }; 20 | } 21 | -------------------------------------------------------------------------------- /plerkle/src/transaction_selector.rs: -------------------------------------------------------------------------------- 1 | /// The transaction selector is responsible for filtering transactions 2 | /// in the plugin framework. 3 | use {log::*, solana_sdk::pubkey::Pubkey, std::collections::HashSet}; 4 | 5 | pub(crate) struct TransactionSelector { 6 | pub mentioned_addresses: HashSet>, 7 | pub select_all_transactions: bool, 8 | pub select_all_vote_transactions: bool, 9 | } 10 | 11 | #[allow(dead_code)] 12 | impl TransactionSelector { 13 | pub fn default() -> Self { 14 | Self { 15 | mentioned_addresses: HashSet::default(), 16 | select_all_transactions: false, 17 | select_all_vote_transactions: false, 18 | } 19 | } 20 | 21 | /// Create a selector based on the mentioned addresses 22 | /// To select all transactions use ["*"] or ["all"] 23 | /// To select all vote transactions, use ["all_votes"] 24 | /// To select transactions mentioning specific addresses use ["", "", ...] 25 | pub fn new(mentioned_addresses: &[String]) -> Self { 26 | info!( 27 | "Creating TransactionSelector from addresses: {:?}", 28 | mentioned_addresses 29 | ); 30 | 31 | let select_all_transactions = mentioned_addresses 32 | .iter() 33 | .any(|key| key == "*" || key == "all"); 34 | if select_all_transactions { 35 | return Self { 36 | mentioned_addresses: HashSet::default(), 37 | select_all_transactions, 38 | select_all_vote_transactions: true, 39 | }; 40 | } 41 | let select_all_vote_transactions = mentioned_addresses.iter().any(|key| key == "all_votes"); 42 | if select_all_vote_transactions { 43 | return Self { 44 | mentioned_addresses: HashSet::default(), 45 | select_all_transactions, 46 | select_all_vote_transactions: true, 47 | }; 48 | } 49 | 50 | let mentioned_addresses = mentioned_addresses 51 | .iter() 52 | .map(|key| bs58::decode(key).into_vec().unwrap()) 53 | .collect(); 54 | 55 | Self { 56 | mentioned_addresses, 57 | select_all_transactions: false, 58 | select_all_vote_transactions: false, 59 | } 60 | } 61 | 62 | /// Check if a transaction is of interest. 63 | pub fn is_transaction_selected( 64 | &self, 65 | is_vote: bool, 66 | mentioned_addresses: Box + '_>, 67 | ) -> bool { 68 | if !self.is_enabled() { 69 | return false; 70 | } 71 | 72 | if self.select_all_transactions || (self.select_all_vote_transactions && is_vote) { 73 | return true; 74 | } 75 | for address in mentioned_addresses { 76 | if self.mentioned_addresses.contains(address.as_ref()) { 77 | return true; 78 | } 79 | } 80 | false 81 | } 82 | 83 | /// Check if any transaction is of interest at all 84 | pub fn is_enabled(&self) -> bool { 85 | self.select_all_transactions 86 | || self.select_all_vote_transactions 87 | || !self.mentioned_addresses.is_empty() 88 | } 89 | } 90 | 91 | #[cfg(test)] 92 | pub(crate) mod tests { 93 | use super::*; 94 | 95 | #[test] 96 | fn test_select_transaction() { 97 | let pubkey1 = Pubkey::new_unique(); 98 | let pubkey2 = Pubkey::new_unique(); 99 | 100 | let selector = TransactionSelector::new(&[pubkey1.to_string()]); 101 | 102 | assert!(selector.is_enabled()); 103 | 104 | let addresses = [pubkey1]; 105 | 106 | assert!(selector.is_transaction_selected(false, Box::new(addresses.iter()))); 107 | 108 | let addresses = [pubkey2]; 109 | assert!(!selector.is_transaction_selected(false, Box::new(addresses.iter()))); 110 | 111 | let addresses = [pubkey1, pubkey2]; 112 | assert!(selector.is_transaction_selected(false, Box::new(addresses.iter()))); 113 | } 114 | 115 | #[test] 116 | fn test_select_all_transaction_using_wildcard() { 117 | let pubkey1 = Pubkey::new_unique(); 118 | let pubkey2 = Pubkey::new_unique(); 119 | 120 | let selector = TransactionSelector::new(&["*".to_string()]); 121 | 122 | assert!(selector.is_enabled()); 123 | 124 | let addresses = [pubkey1]; 125 | 126 | assert!(selector.is_transaction_selected(false, Box::new(addresses.iter()))); 127 | 128 | let addresses = [pubkey2]; 129 | assert!(selector.is_transaction_selected(false, Box::new(addresses.iter()))); 130 | 131 | let addresses = [pubkey1, pubkey2]; 132 | assert!(selector.is_transaction_selected(false, Box::new(addresses.iter()))); 133 | } 134 | 135 | #[test] 136 | fn test_select_all_transaction_all() { 137 | let pubkey1 = Pubkey::new_unique(); 138 | let pubkey2 = Pubkey::new_unique(); 139 | 140 | let selector = TransactionSelector::new(&["all".to_string()]); 141 | 142 | assert!(selector.is_enabled()); 143 | 144 | let addresses = [pubkey1]; 145 | 146 | assert!(selector.is_transaction_selected(false, Box::new(addresses.iter()))); 147 | 148 | let addresses = [pubkey2]; 149 | assert!(selector.is_transaction_selected(false, Box::new(addresses.iter()))); 150 | 151 | let addresses = [pubkey1, pubkey2]; 152 | assert!(selector.is_transaction_selected(false, Box::new(addresses.iter()))); 153 | } 154 | 155 | #[test] 156 | fn test_select_all_vote_transaction() { 157 | let pubkey1 = Pubkey::new_unique(); 158 | let pubkey2 = Pubkey::new_unique(); 159 | 160 | let selector = TransactionSelector::new(&["all_votes".to_string()]); 161 | 162 | assert!(selector.is_enabled()); 163 | 164 | let addresses = [pubkey1]; 165 | 166 | assert!(!selector.is_transaction_selected(false, Box::new(addresses.iter()))); 167 | 168 | let addresses = [pubkey2]; 169 | assert!(selector.is_transaction_selected(true, Box::new(addresses.iter()))); 170 | 171 | let addresses = [pubkey1, pubkey2]; 172 | assert!(selector.is_transaction_selected(true, Box::new(addresses.iter()))); 173 | } 174 | 175 | #[test] 176 | fn test_select_no_transaction() { 177 | let pubkey1 = Pubkey::new_unique(); 178 | let pubkey2 = Pubkey::new_unique(); 179 | 180 | let selector = TransactionSelector::new(&[]); 181 | 182 | assert!(!selector.is_enabled()); 183 | 184 | let addresses = [pubkey1]; 185 | 186 | assert!(!selector.is_transaction_selected(false, Box::new(addresses.iter()))); 187 | 188 | let addresses = [pubkey2]; 189 | assert!(!selector.is_transaction_selected(true, Box::new(addresses.iter()))); 190 | 191 | let addresses = [pubkey1, pubkey2]; 192 | assert!(!selector.is_transaction_selected(true, Box::new(addresses.iter()))); 193 | } 194 | } 195 | -------------------------------------------------------------------------------- /plerkle_messenger/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "plerkle_messenger" 3 | description = "Metaplex Messenger trait for Geyser plugin producer/consumer patterns." 4 | version = "2.0.0" 5 | authors = ["Metaplex Developers "] 6 | repository = "https://github.com/metaplex-foundation/digital-asset-validator-plugin" 7 | license = "AGPL-3.0" 8 | edition = "2021" 9 | readme = "Readme.md" 10 | 11 | [dependencies] 12 | redis = { version = "0.27.2", features = ["aio", "tokio-comp", "streams", "tokio-native-tls-comp", "connection-manager"]} 13 | tokio = "1.40.0" 14 | log = "0.4.11" 15 | thiserror = "1.0.30" 16 | async-trait = "0.1.53" 17 | figment = "0.10.6" 18 | futures = "0.3" 19 | async-mutex = "1.4.0" 20 | serde = {version = "1.0.137", features = ["derive"] } 21 | blake3 = "1.3.3" 22 | cadence = "0.29.0" 23 | cadence-macros = "0.29.0" 24 | 25 | [package.metadata.docs.rs] 26 | targets = ["x86_64-unknown-linux-gnu"] 27 | 28 | [dev-dependencies] 29 | tokio = {version = "1.20.1", features = ["full"]} 30 | -------------------------------------------------------------------------------- /plerkle_messenger/Readme.md: -------------------------------------------------------------------------------- 1 | # Messenger 2 | 3 | A message bus agnostic Messaging Library that sends Transaction, Account, Block and Slot updates in the Plerkle Serialization format. 4 | 5 | ## Note on 1.0.0 6 | 7 | The plerkle serialization API changes at 1.0.0 which is a breaking change. 8 | This method removes confusion around the Recv data lifetime being tied back to the messenger interface. Now the data is owned. 9 | 10 | # Env example 11 | 12 | The Messenger can operate in two modes: a single Redis instance or multiple Redis instances. 13 | 14 | Just to clarify, the multiple Redis instances setup doesn't create a clustered connection. It's designed to work with separate, independent instances. 15 | 16 | You can configure the Redis client type via environment variables. 17 | 18 | Example environment configuration for a single Redis instance: 19 | 20 | ``` 21 | export PLUGIN_MESSENGER_CONFIG='{ 22 | messenger_type="Redis", 23 | redis_connection_str="redis://:pass@redis.app:6379" 24 | }' 25 | ``` 26 | 27 | Example environment configuration for multiple Redis instances: 28 | 29 | ``` 30 | export PLUGIN_MESSENGER_CONFIG='{ 31 | messenger_type="RedisPool", 32 | redis_connection_str=[ 33 | "redis://:pass@redis1.app:6379", 34 | "redis://:pass@redis2.app:6379" 35 | ] 36 | }' 37 | ``` 38 | 39 | To switch between modes, you'll need to update both the `messenger_type` and `redis_connection_str` values. -------------------------------------------------------------------------------- /plerkle_messenger/rustfmt.toml: -------------------------------------------------------------------------------- 1 | edition = "2021" 2 | imports_granularity="Crate" 3 | reorder_imports = true 4 | -------------------------------------------------------------------------------- /plerkle_messenger/src/error.rs: -------------------------------------------------------------------------------- 1 | use thiserror::Error; 2 | 3 | #[derive(Error, Debug)] 4 | pub enum MessengerError { 5 | #[error("Missing or invalid configuration: ({msg})")] 6 | ConfigurationError { msg: String }, 7 | 8 | #[error("Error creating connection: ({msg})")] 9 | ConnectionError { msg: String }, 10 | 11 | #[error("Error sending data: ({msg})")] 12 | SendError { msg: String }, 13 | 14 | #[error("Error receiving data: ({msg})")] 15 | ReceiveError { msg: String }, 16 | 17 | #[error("Error ACKing message: ({msg})")] 18 | AckError { msg: String }, 19 | 20 | #[error("Error autoclaiming message: ({msg})")] 21 | AutoclaimError { msg: String }, 22 | } 23 | -------------------------------------------------------------------------------- /plerkle_messenger/src/lib.rs: -------------------------------------------------------------------------------- 1 | mod error; 2 | mod metrics; 3 | mod plerkle_messenger; 4 | 5 | pub mod redis; 6 | pub use redis::*; 7 | 8 | pub use crate::error::*; 9 | pub use plerkle_messenger::*; 10 | -------------------------------------------------------------------------------- /plerkle_messenger/src/metrics.rs: -------------------------------------------------------------------------------- 1 | #[macro_export] 2 | macro_rules! metric { 3 | {$($block:stmt;)*} => { 4 | if cadence_macros::is_global_default_set() { 5 | $( 6 | $block 7 | )* 8 | } 9 | }; 10 | } 11 | -------------------------------------------------------------------------------- /plerkle_messenger/src/plerkle_messenger.rs: -------------------------------------------------------------------------------- 1 | use crate::{error::MessengerError, redis_pool_messenger::RedisPoolMessenger}; 2 | use async_trait::async_trait; 3 | use blake3::OUT_LEN; 4 | use figment::value::{Dict, Value}; 5 | use serde::Deserialize; 6 | use std::collections::BTreeMap; 7 | 8 | use crate::redis_messenger::RedisMessenger; 9 | 10 | /// Some constants that can be used as stream key values. 11 | pub const ACCOUNT_STREAM: &str = "ACC"; 12 | pub const ACCOUNT_BACKFILL_STREAM: &str = "ACCFILL"; 13 | pub const SLOT_STREAM: &str = "SLT"; 14 | pub const TRANSACTION_STREAM: &str = "TXN"; 15 | pub const TRANSACTION_BACKFILL_STREAM: &str = "TXNFILL"; 16 | pub const BLOCK_STREAM: &str = "BLK"; 17 | 18 | #[derive(Clone, Debug, PartialEq, Eq)] 19 | pub struct RecvData { 20 | pub id: String, 21 | pub tries: usize, 22 | pub data: Vec, 23 | } 24 | 25 | impl RecvData { 26 | pub fn new(id: String, data: Vec) -> Self { 27 | RecvData { id, data, tries: 0 } 28 | } 29 | 30 | pub fn new_retry(id: String, data: Vec, tries: usize) -> Self { 31 | RecvData { id, data, tries } 32 | } 33 | 34 | pub fn hash(&mut self) -> [u8; OUT_LEN] { 35 | let mut hasher = blake3::Hasher::new(); 36 | hasher.update(&self.data); 37 | let hash = hasher.finalize(); 38 | hash.as_bytes().to_owned() 39 | } 40 | } 41 | 42 | #[derive(Clone, Debug, PartialEq, Eq)] 43 | pub enum ConsumptionType { 44 | New, 45 | Redeliver, 46 | All, 47 | } 48 | 49 | #[async_trait] 50 | pub trait Messenger: Sync + Send { 51 | fn messenger_type(&self) -> MessengerType; 52 | async fn recv( 53 | &mut self, 54 | stream_key: &'static str, 55 | consumption_type: ConsumptionType, 56 | ) -> Result, MessengerError>; 57 | async fn stream_size(&mut self, stream_key: &'static str) -> Result; 58 | 59 | // Ack-ing messages is made a bit awkward by the current interface layout because 60 | // the sequence of msgs returned by `recv` will mutably borrow `self`, and calling 61 | // `ack_msg` need to do the same thing, which isn't possible while that returned `Vec` 62 | // is alive or the borrow checker complains. We can do stuff like making `recv` and `ack` 63 | // require interior mutability, but that or other alternatives are non-trivial refactoring 64 | // efforts best applied after we get more data about how the system performs and what 65 | // changes we'd like to do overall. 66 | // 67 | // For now, the flow is that `recv` returns a `Vec` of items where ids are owned `Strings` 68 | // for convenience, which can be kept until going through all data items, and then 69 | // passed to `ack_msg` together. Right now, we're reading a single messages via `recv` 70 | // anyway, but at some point we might want to get more in a single shot if talking 71 | // to the backing channel becomes a bottleneck. 72 | async fn ack_msg( 73 | &mut self, 74 | stream_key: &'static str, 75 | ids: &[String], 76 | ) -> Result<(), MessengerError>; 77 | } 78 | 79 | #[async_trait] 80 | pub trait MessageStreamer: Sync + Send { 81 | fn messenger_type(&self) -> MessengerType; 82 | async fn add_stream(&mut self, stream_key: &'static str) -> Result<(), MessengerError>; 83 | async fn set_buffer_size(&mut self, stream_key: &'static str, max_buffer_size: usize); 84 | async fn send(&mut self, stream_key: &'static str, bytes: &[u8]) -> Result<(), MessengerError>; 85 | } 86 | 87 | pub async fn select_messenger_read( 88 | config: MessengerConfig, 89 | ) -> Result, MessengerError> { 90 | match config.messenger_type { 91 | MessengerType::Redis => RedisMessenger::new(config) 92 | .await 93 | .map(|a| Box::new(a) as Box), 94 | _ => Err(MessengerError::ConfigurationError { 95 | msg: "This Messenger type is not valid or not unimplemented.".to_string(), 96 | }), 97 | } 98 | } 99 | 100 | pub async fn select_messenger_stream( 101 | config: MessengerConfig, 102 | ) -> Result, MessengerError> { 103 | match config.messenger_type { 104 | MessengerType::Redis => { 105 | RedisMessenger::new(config).await.map(|a| Box::new(a) as Box) 106 | } 107 | MessengerType::RedisPool => { 108 | RedisPoolMessenger::new(config).await.map(|a| Box::new(a) as Box) 109 | } 110 | _ => Err(MessengerError::ConfigurationError { 111 | msg: "This Messenger type is not valid, unimplemented or you don't have the right crate features on.".to_string() 112 | }) 113 | } 114 | } 115 | 116 | #[derive(Deserialize, Debug, PartialEq, Eq, Clone)] 117 | pub enum MessengerType { 118 | // Connect to one Redis instance 119 | Redis, 120 | // Connect to few different Redis instances 121 | // Not a cluster 122 | RedisPool, 123 | Invalid, 124 | } 125 | 126 | impl Default for MessengerType { 127 | fn default() -> Self { 128 | MessengerType::Redis 129 | } 130 | } 131 | 132 | #[derive(Deserialize, Debug, Default, PartialEq)] 133 | pub struct MessengerConfig { 134 | pub messenger_type: MessengerType, 135 | pub connection_config: Dict, 136 | } 137 | 138 | impl Clone for MessengerConfig { 139 | fn clone(&self) -> Self { 140 | let mut d: BTreeMap = BTreeMap::new(); 141 | for (k, i) in self.connection_config.iter() { 142 | d.insert(k.clone(), i.clone()); 143 | } 144 | MessengerConfig { 145 | messenger_type: self.messenger_type.clone(), 146 | connection_config: d, 147 | } 148 | } 149 | } 150 | 151 | impl MessengerConfig { 152 | pub fn get(&self, key: &str) -> Option<&Value> { 153 | self.connection_config.get(key) 154 | } 155 | } 156 | 157 | #[cfg(test)] 158 | mod tests { 159 | use crate::{MessengerConfig, MessengerType}; 160 | use figment::{providers::Env, value::Dict, Figment, Jail}; 161 | use serde::Deserialize; 162 | 163 | #[derive(Deserialize, Debug, PartialEq)] 164 | struct Container { 165 | messenger_config: MessengerConfig, 166 | } 167 | 168 | #[test] 169 | fn test_config_deser() { 170 | Jail::expect_with(|jail| { 171 | jail.set_env("MESSENGER_CONFIG.messenger_type", "Redis"); 172 | jail.set_env( 173 | "MESSENGER_CONFIG.connection_config", 174 | r#"{redis_connection_str="redis://redis"}"#, 175 | ); 176 | 177 | let config: Container = Figment::from(Env::raw()).extract()?; 178 | let mut expected_dict = Dict::new(); 179 | expected_dict.insert("redis_connection_str".to_string(), "redis://redis".into()); 180 | assert_eq!( 181 | config.messenger_config, 182 | MessengerConfig { 183 | messenger_type: MessengerType::Redis, 184 | connection_config: expected_dict, 185 | } 186 | ); 187 | Ok(()) 188 | }); 189 | } 190 | } 191 | -------------------------------------------------------------------------------- /plerkle_messenger/src/redis/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod redis_messenger; 2 | pub mod redis_pool_messenger; 3 | 4 | // Redis stream values. 5 | pub const GROUP_NAME: &str = "plerkle"; 6 | pub const DATA_KEY: &str = "data"; 7 | pub const DEFAULT_RETRIES: usize = 3; 8 | pub const DEFAULT_MSG_BATCH_SIZE: usize = 10; 9 | pub const MESSAGE_WAIT_TIMEOUT: usize = 10; 10 | pub const IDLE_TIMEOUT: usize = 5000; 11 | pub const REDIS_MAX_BYTES_COMMAND: usize = 536870912; 12 | pub const PIPELINE_SIZE_BYTES: usize = REDIS_MAX_BYTES_COMMAND / 100; 13 | pub const PIPELINE_MAX_TIME: u64 = 10; 14 | 15 | pub(crate) const REDIS_CON_STR: &str = "redis_connection_str"; 16 | -------------------------------------------------------------------------------- /plerkle_messenger/src/redis/redis_messenger.rs: -------------------------------------------------------------------------------- 1 | use crate::{ 2 | error::MessengerError, metric, ConsumptionType, MessageStreamer, Messenger, MessengerConfig, 3 | MessengerType, RecvData, 4 | }; 5 | use async_trait::async_trait; 6 | 7 | use cadence_macros::statsd_count; 8 | use log::*; 9 | use redis::{ 10 | aio::ConnectionManager, 11 | cmd, 12 | streams::{ 13 | StreamId, StreamKey, StreamMaxlen, StreamPendingCountReply, StreamReadOptions, 14 | StreamReadReply, 15 | }, 16 | AsyncCommands, RedisResult, Value, 17 | }; 18 | 19 | use redis::streams::StreamRangeReply; 20 | use std::{ 21 | collections::{HashMap, LinkedList}, 22 | fmt::{Debug, Formatter}, 23 | time::{Duration, Instant}, 24 | }; 25 | 26 | use super::{ 27 | DATA_KEY, DEFAULT_MSG_BATCH_SIZE, DEFAULT_RETRIES, GROUP_NAME, IDLE_TIMEOUT, 28 | MESSAGE_WAIT_TIMEOUT, PIPELINE_MAX_TIME, PIPELINE_SIZE_BYTES, REDIS_CON_STR, 29 | }; 30 | 31 | pub struct RedisMessenger { 32 | connection: ConnectionManager, 33 | streams: HashMap<&'static str, RedisMessengerStream>, 34 | consumer_id: String, 35 | retries: usize, 36 | batch_size: usize, 37 | idle_timeout: usize, 38 | _message_wait_timeout: usize, 39 | consumer_group_name: String, 40 | pipeline_size: usize, 41 | pipeline_max_time: u64, 42 | } 43 | 44 | pub struct RedisMessengerStream { 45 | pub max_len: Option, 46 | pub local_buffer: LinkedList>, 47 | pub local_buffer_total: usize, 48 | pub local_buffer_last_flush: Instant, 49 | } 50 | 51 | impl RedisMessenger { 52 | pub async fn new(config: MessengerConfig) -> Result { 53 | let uri = config 54 | .get(REDIS_CON_STR) 55 | .and_then(|u| u.clone().into_string()) 56 | .ok_or(MessengerError::ConfigurationError { 57 | msg: format!("Connection String Missing: {}", REDIS_CON_STR), 58 | })?; 59 | // Setup Redis client. 60 | let client = redis::Client::open(uri).unwrap(); 61 | 62 | // Get connection. 63 | let connection = client.get_connection_manager().await.map_err(|e| { 64 | error!("{}", e.to_string()); 65 | MessengerError::ConnectionError { msg: e.to_string() } 66 | })?; 67 | 68 | let consumer_id = config 69 | .get("consumer_id") 70 | .and_then(|id| id.clone().into_string()) 71 | // Using the previous default name when the configuration does not 72 | // specify any particular consumer_id. 73 | .unwrap_or_else(|| String::from("ingester")); 74 | 75 | let retries = config 76 | .get("retries") 77 | .and_then(|r| r.clone().to_u128().map(|n| n as usize)) 78 | .unwrap_or(DEFAULT_RETRIES); 79 | 80 | let batch_size = config 81 | .get("batch_size") 82 | .and_then(|r| r.clone().to_u128().map(|n| n as usize)) 83 | .unwrap_or(DEFAULT_MSG_BATCH_SIZE); 84 | 85 | let idle_timeout = config 86 | .get("idle_timeout") 87 | .and_then(|r| r.clone().to_u128().map(|n| n as usize)) 88 | .unwrap_or(IDLE_TIMEOUT); 89 | let message_wait_timeout = config 90 | .get("message_wait_timeout") 91 | .and_then(|r| r.clone().to_u128().map(|n| n as usize)) 92 | .unwrap_or(MESSAGE_WAIT_TIMEOUT); 93 | 94 | let consumer_group_name = config 95 | .get("consumer_group_name") 96 | .and_then(|r| r.clone().into_string()) 97 | .unwrap_or_else(|| GROUP_NAME.to_string()); 98 | 99 | let pipeline_size = config 100 | .get("pipeline_size_bytes") 101 | .and_then(|r| r.clone().to_u128().map(|n| n as usize)) 102 | .unwrap_or(PIPELINE_SIZE_BYTES); 103 | 104 | let pipeline_max_time = config 105 | .get("local_buffer_max_window") 106 | .and_then(|r| r.clone().to_u128().map(|n| n as u64)) 107 | .unwrap_or(PIPELINE_MAX_TIME); 108 | 109 | Ok(Self { 110 | connection, 111 | streams: HashMap::<&'static str, RedisMessengerStream>::default(), 112 | consumer_id, 113 | retries, 114 | batch_size, 115 | idle_timeout, 116 | _message_wait_timeout: message_wait_timeout, 117 | consumer_group_name, 118 | pipeline_size, 119 | pipeline_max_time, 120 | }) 121 | } 122 | 123 | async fn xautoclaim( 124 | &mut self, 125 | stream_key: &'static str, 126 | ) -> Result, MessengerError> { 127 | let id = "0-0".to_owned(); 128 | let mut xauto = cmd("XAUTOCLAIM"); 129 | xauto 130 | .arg(stream_key) 131 | .arg(self.consumer_group_name.clone()) 132 | .arg(self.consumer_id.as_str()) 133 | // We only reclaim items that have been idle for at least 2 sec. 134 | .arg(self.idle_timeout) 135 | .arg(id.as_str()) 136 | .arg("COUNT") 137 | .arg(self.batch_size); 138 | 139 | let result: (String, StreamRangeReply, Vec) = xauto 140 | .query_async(&mut self.connection) 141 | .await 142 | .map_err(|e| MessengerError::AutoclaimError { msg: e.to_string() })?; 143 | 144 | let range_reply = result.1; 145 | if range_reply.ids.is_empty() { 146 | // We've reached the end of the PEL. 147 | return Ok(Vec::new()); 148 | } 149 | 150 | let mut retained_ids = Vec::new(); 151 | let f = range_reply.ids.first().unwrap(); 152 | let l = range_reply.ids.last().unwrap(); 153 | 154 | // We need to use `xpending_count` to get a `StreamPendingCountReply` which 155 | // // contains information about the number of times a message has been 156 | // // delivered. 157 | let pending_result: StreamPendingCountReply = self 158 | .connection 159 | .xpending_count( 160 | stream_key, 161 | self.consumer_group_name.clone(), 162 | &f.id.clone(), 163 | &l.id.clone(), 164 | range_reply.ids.len(), 165 | ) 166 | .await 167 | .map_err(|e| { 168 | error!("Redis receive error: {e}"); 169 | MessengerError::ReceiveError { msg: e.to_string() } 170 | })?; 171 | let mut pending = HashMap::new(); 172 | let mut ack_list = Vec::new(); 173 | let prs = pending_result.ids.into_iter(); 174 | for pr in prs { 175 | pending.insert(pr.id.clone(), pr); 176 | } 177 | for sid in range_reply.ids { 178 | let StreamId { id, map } = sid; 179 | let info = if let Some(info) = pending.get(&id) { 180 | info 181 | } else { 182 | warn!("No pending info for ID {id}"); 183 | continue; 184 | }; 185 | let data = if let Some(data) = map.get(DATA_KEY) { 186 | data 187 | } else { 188 | info!("No Data was stored in Redis for ID {id}"); 189 | continue; 190 | }; 191 | // Get data from map. 192 | 193 | let bytes = match data { 194 | Value::BulkString(bytes) => bytes, 195 | _ => { 196 | error!("Redis data for ID {id} in wrong format"); 197 | continue; 198 | } 199 | }; 200 | 201 | if info.times_delivered > self.retries { 202 | metric! { 203 | statsd_count!("plerkle.messenger.retries.exceeded", 1); 204 | } 205 | error!("Message has reached maximum retries {} for id", id); 206 | ack_list.push(id.clone()); 207 | continue; 208 | } 209 | retained_ids.push(RecvData::new_retry( 210 | id, 211 | bytes.to_vec(), 212 | info.times_delivered, 213 | )); 214 | } 215 | if let Err(e) = self.ack_msg(stream_key, &ack_list).await { 216 | error!("Error acking pending messages: {}", e); 217 | } 218 | Ok(retained_ids) 219 | } 220 | 221 | pub async fn force_flush(mut self) -> Result<(), MessengerError> { 222 | for (stream_key, mut stream) in self.streams.into_iter() { 223 | // Get max length for the stream. 224 | let maxlen = if let Some(maxlen) = stream.max_len { 225 | maxlen 226 | } else { 227 | error!("Cannot send data for stream key {stream_key}, buffer size not set."); 228 | return Ok(()); 229 | }; 230 | let mut pipe = redis::pipe(); 231 | for bytes in stream.local_buffer.iter() { 232 | pipe.xadd_maxlen(stream_key, maxlen, "*", &[(DATA_KEY, &bytes)]); 233 | } 234 | let result: Result, redis::RedisError> = 235 | pipe.query_async(&mut self.connection).await; 236 | if let Err(e) = result { 237 | error!("Redis send error: {e}"); 238 | return Err(MessengerError::SendError { msg: e.to_string() }); 239 | } else { 240 | debug!("Data Sent to {}", stream_key); 241 | stream.local_buffer.clear(); 242 | stream.local_buffer_total = 0; 243 | stream.local_buffer_last_flush = Instant::now(); 244 | } 245 | } 246 | Ok(()) 247 | } 248 | 249 | pub async fn stream_len(&mut self, stream_key: &'static str) -> Result { 250 | Ok(self.connection.xlen(stream_key).await.map_err(|e| { 251 | error!("Failed to read stream length: {}", e); 252 | MessengerError::ConnectionError { msg: e.to_string() } 253 | })?) 254 | } 255 | } 256 | 257 | #[async_trait] 258 | impl Messenger for RedisMessenger { 259 | fn messenger_type(&self) -> MessengerType { 260 | MessengerType::Redis 261 | } 262 | 263 | async fn stream_size(&mut self, stream_key: &'static str) -> Result { 264 | let result: RedisResult = self.connection.xlen(stream_key).await; 265 | match result { 266 | Ok(reply) => Ok(reply), 267 | Err(e) => Err(MessengerError::ConnectionError { msg: e.to_string() }), 268 | } 269 | } 270 | 271 | // is used only on client side 272 | // Geyser does not call this method 273 | async fn recv( 274 | &mut self, 275 | stream_key: &'static str, 276 | consumption_type: ConsumptionType, 277 | ) -> Result, MessengerError> { 278 | let mut data_vec = Vec::with_capacity(self.batch_size * 2); 279 | if consumption_type == ConsumptionType::New || consumption_type == ConsumptionType::All { 280 | let opts = StreamReadOptions::default() 281 | //.block(self.message_wait_timeout) 282 | .count(self.batch_size) 283 | .group(self.consumer_group_name.as_str(), self.consumer_id.as_str()); 284 | 285 | // Read on stream key and save the reply. Log but do not return errors. 286 | let reply: StreamReadReply = self 287 | .connection 288 | .xread_options(&[stream_key], &[">"], &opts) 289 | .await 290 | .map_err(|e| { 291 | error!("Redis receive error: {e}"); 292 | MessengerError::ReceiveError { msg: e.to_string() } 293 | })?; 294 | // Parse data in stream read reply and store in Vec to return to caller. 295 | for StreamKey { key: _, ids } in reply.keys.into_iter() { 296 | for StreamId { id, map } in ids { 297 | // Get data from map. 298 | let data = if let Some(data) = map.get(DATA_KEY) { 299 | data 300 | } else { 301 | error!("No Data was stored in Redis for ID {id}"); 302 | continue; 303 | }; 304 | let bytes = match data { 305 | Value::BulkString(bytes) => bytes, 306 | _ => { 307 | error!("Redis data for ID {id} in wrong format"); 308 | continue; 309 | } 310 | }; 311 | 312 | data_vec.push(RecvData::new(id, bytes.to_vec())); 313 | } 314 | } 315 | } 316 | if consumption_type == ConsumptionType::Redeliver 317 | || consumption_type == ConsumptionType::All 318 | { 319 | let xauto_reply = self.xautoclaim(stream_key).await; 320 | match xauto_reply { 321 | Ok(reply) => { 322 | let mut pending_messages = reply; 323 | data_vec.append(&mut pending_messages); 324 | } 325 | Err(e) => { 326 | error!("XPENDING ERROR {e}"); 327 | } 328 | } 329 | } 330 | 331 | Ok(data_vec) 332 | } 333 | 334 | async fn ack_msg( 335 | &mut self, 336 | stream_key: &'static str, 337 | ids: &[String], 338 | ) -> Result<(), MessengerError> { 339 | if ids.is_empty() { 340 | return Ok(()); 341 | } 342 | let mut pipe = redis::pipe(); 343 | pipe.xack(stream_key, self.consumer_group_name.as_str(), ids); 344 | pipe.xdel(stream_key, ids); 345 | 346 | pipe.query_async(&mut self.connection) 347 | .await 348 | .map_err(|e| MessengerError::AckError { msg: e.to_string() }) 349 | } 350 | } 351 | 352 | #[async_trait] 353 | impl MessageStreamer for RedisMessenger { 354 | fn messenger_type(&self) -> MessengerType { 355 | MessengerType::Redis 356 | } 357 | 358 | async fn add_stream(&mut self, stream_key: &'static str) -> Result<(), MessengerError> { 359 | // Add to streams hashmap. 360 | let _result = self.streams.insert( 361 | stream_key, 362 | RedisMessengerStream { 363 | max_len: None, 364 | local_buffer: LinkedList::new(), 365 | local_buffer_total: 0, 366 | local_buffer_last_flush: Instant::now(), 367 | }, 368 | ); 369 | 370 | // Add stream to Redis. 371 | let result: RedisResult<()> = self 372 | .connection 373 | .xgroup_create_mkstream(stream_key, self.consumer_group_name.as_str(), "$") 374 | .await; 375 | 376 | if let Err(e) = result { 377 | info!("Group already exists: {:?}", e) 378 | } 379 | Ok(()) 380 | } 381 | 382 | async fn set_buffer_size(&mut self, stream_key: &'static str, max_buffer_size: usize) { 383 | // Set max length for the stream. 384 | if let Some(stream) = self.streams.get_mut(stream_key) { 385 | stream.max_len = Some(StreamMaxlen::Approx(max_buffer_size)); 386 | } else { 387 | error!("Stream key {stream_key} not configured"); 388 | } 389 | } 390 | 391 | async fn send(&mut self, stream_key: &'static str, bytes: &[u8]) -> Result<(), MessengerError> { 392 | // Check if stream is configured. 393 | let stream = if let Some(stream) = self.streams.get_mut(stream_key) { 394 | stream 395 | } else { 396 | error!("Cannot send data for stream key {stream_key}, it is not configured"); 397 | return Ok(()); 398 | }; 399 | 400 | // Get max length for the stream. 401 | let maxlen = if let Some(maxlen) = stream.max_len { 402 | maxlen 403 | } else { 404 | error!("Cannot send data for stream key {stream_key}, buffer size not set."); 405 | return Ok(()); 406 | }; 407 | stream.local_buffer.push_back(bytes.to_vec()); 408 | stream.local_buffer_total += bytes.len(); 409 | // Put serialized data into Redis. 410 | if stream.local_buffer_total < self.pipeline_size 411 | && stream.local_buffer_last_flush.elapsed() 412 | <= Duration::from_millis(self.pipeline_max_time as u64) 413 | { 414 | debug!( 415 | "Redis local buffer bytes {} and message pipeline size {} elapsed time {}ms", 416 | stream.local_buffer_total, 417 | stream.local_buffer.len(), 418 | stream.local_buffer_last_flush.elapsed().as_millis() 419 | ); 420 | return Ok(()); 421 | } else { 422 | let mut pipe = redis::pipe(); 423 | for bytes in stream.local_buffer.iter() { 424 | pipe.xadd_maxlen(stream_key, maxlen, "*", &[(DATA_KEY, &bytes)]); 425 | } 426 | let result: Result, redis::RedisError> = 427 | pipe.query_async(&mut self.connection).await; 428 | if let Err(e) = result { 429 | error!("Redis send error: {e}"); 430 | return Err(MessengerError::SendError { msg: e.to_string() }); 431 | } else { 432 | debug!("Data Sent to {}", stream_key); 433 | stream.local_buffer.clear(); 434 | stream.local_buffer_total = 0; 435 | stream.local_buffer_last_flush = Instant::now(); 436 | } 437 | } 438 | Ok(()) 439 | } 440 | } 441 | 442 | impl Debug for RedisMessenger { 443 | fn fmt(&self, _f: &mut Formatter<'_>) -> std::fmt::Result { 444 | Ok(()) 445 | } 446 | } 447 | -------------------------------------------------------------------------------- /plerkle_messenger/src/redis/redis_pool_messenger.rs: -------------------------------------------------------------------------------- 1 | use crate::{ 2 | error::MessengerError, redis_messenger::RedisMessengerStream, MessageStreamer, MessengerConfig, 3 | MessengerType, DATA_KEY, 4 | }; 5 | use async_trait::async_trait; 6 | 7 | use log::*; 8 | use redis::{aio::ConnectionManager, streams::StreamMaxlen, AsyncCommands, RedisResult}; 9 | 10 | use std::{ 11 | collections::{HashMap, LinkedList}, 12 | time::{Duration, Instant}, 13 | }; 14 | use tokio::task::JoinSet; 15 | 16 | use super::{ 17 | GROUP_NAME, MESSAGE_WAIT_TIMEOUT, PIPELINE_MAX_TIME, PIPELINE_SIZE_BYTES, REDIS_CON_STR, 18 | }; 19 | 20 | /// A Redis Messenger capable of streaming data to multiple separate Redis instances. 21 | pub struct RedisPoolMessenger { 22 | connections_pool: Vec, 23 | streams: HashMap<&'static str, RedisMessengerStream>, 24 | _message_wait_timeout: usize, 25 | consumer_group_name: String, 26 | pipeline_size: usize, 27 | pipeline_max_time: u64, 28 | } 29 | 30 | impl RedisPoolMessenger { 31 | pub async fn new(config: MessengerConfig) -> Result { 32 | let uris = config 33 | .get(REDIS_CON_STR) 34 | .and_then(|u| u.clone().into_array()) 35 | .ok_or(MessengerError::ConfigurationError { 36 | msg: format!("Connection String Missing: {}", REDIS_CON_STR), 37 | })?; 38 | 39 | let mut connections_pool = vec![]; 40 | 41 | for uri in uris { 42 | // Setup Redis client. 43 | let client = redis::Client::open(uri.into_string().ok_or( 44 | MessengerError::ConfigurationError { 45 | msg: format!("Connection String Missing: {}", REDIS_CON_STR), 46 | }, 47 | )?) 48 | .unwrap(); 49 | 50 | // Get connection. 51 | connections_pool.push(client.get_connection_manager().await.map_err(|e| { 52 | error!("{}", e.to_string()); 53 | MessengerError::ConnectionError { msg: e.to_string() } 54 | })?); 55 | } 56 | 57 | let message_wait_timeout = config 58 | .get("message_wait_timeout") 59 | .and_then(|r| r.clone().to_u128().map(|n| n as usize)) 60 | .unwrap_or(MESSAGE_WAIT_TIMEOUT); 61 | 62 | let consumer_group_name = config 63 | .get("consumer_group_name") 64 | .and_then(|r| r.clone().into_string()) 65 | .unwrap_or_else(|| GROUP_NAME.to_string()); 66 | 67 | let pipeline_size = config 68 | .get("pipeline_size_bytes") 69 | .and_then(|r| r.clone().to_u128().map(|n| n as usize)) 70 | .unwrap_or(PIPELINE_SIZE_BYTES); 71 | 72 | let pipeline_max_time = config 73 | .get("local_buffer_max_window") 74 | .and_then(|r| r.clone().to_u128().map(|n| n as u64)) 75 | .unwrap_or(PIPELINE_MAX_TIME); 76 | 77 | Ok(Self { 78 | connections_pool, 79 | streams: HashMap::<&'static str, RedisMessengerStream>::default(), 80 | _message_wait_timeout: message_wait_timeout, 81 | consumer_group_name, 82 | pipeline_size, 83 | pipeline_max_time, 84 | }) 85 | } 86 | } 87 | 88 | #[async_trait] 89 | impl MessageStreamer for RedisPoolMessenger { 90 | fn messenger_type(&self) -> MessengerType { 91 | MessengerType::RedisPool 92 | } 93 | 94 | async fn add_stream(&mut self, stream_key: &'static str) -> Result<(), MessengerError> { 95 | // Add to streams hashmap. 96 | let _result = self.streams.insert( 97 | stream_key, 98 | RedisMessengerStream { 99 | max_len: None, 100 | local_buffer: LinkedList::new(), 101 | local_buffer_total: 0, 102 | local_buffer_last_flush: Instant::now(), 103 | }, 104 | ); 105 | 106 | for connection in &mut self.connections_pool { 107 | // Add stream to Redis. 108 | let result: RedisResult<()> = connection 109 | .xgroup_create_mkstream(stream_key, self.consumer_group_name.as_str(), "$") 110 | .await; 111 | 112 | if let Err(e) = result { 113 | info!("Group already exists: {:?}", e) 114 | } 115 | } 116 | 117 | Ok(()) 118 | } 119 | 120 | async fn set_buffer_size(&mut self, stream_key: &'static str, max_buffer_size: usize) { 121 | // Set max length for the stream. 122 | if let Some(stream) = self.streams.get_mut(stream_key) { 123 | stream.max_len = Some(StreamMaxlen::Approx(max_buffer_size)); 124 | } else { 125 | error!("Stream key {stream_key} not configured"); 126 | } 127 | } 128 | 129 | async fn send(&mut self, stream_key: &'static str, bytes: &[u8]) -> Result<(), MessengerError> { 130 | // Check if stream is configured. 131 | let stream = if let Some(stream) = self.streams.get_mut(stream_key) { 132 | stream 133 | } else { 134 | error!("Cannot send data for stream key {stream_key}, it is not configured"); 135 | return Ok(()); 136 | }; 137 | 138 | // Get max length for the stream. 139 | let maxlen = if let Some(maxlen) = stream.max_len { 140 | maxlen 141 | } else { 142 | error!("Cannot send data for stream key {stream_key}, buffer size not set."); 143 | return Ok(()); 144 | }; 145 | stream.local_buffer.push_back(bytes.to_vec()); 146 | stream.local_buffer_total += bytes.len(); 147 | // Put serialized data into Redis. 148 | if stream.local_buffer_total < self.pipeline_size 149 | && stream.local_buffer_last_flush.elapsed() 150 | <= Duration::from_millis(self.pipeline_max_time as u64) 151 | { 152 | debug!( 153 | "Redis local buffer bytes {} and message pipeline size {} elapsed time {}ms", 154 | stream.local_buffer_total, 155 | stream.local_buffer.len(), 156 | stream.local_buffer_last_flush.elapsed().as_millis() 157 | ); 158 | } else { 159 | let mut pipe = redis::pipe(); 160 | pipe.atomic(); 161 | for bytes in stream.local_buffer.iter() { 162 | pipe.xadd_maxlen(stream_key, maxlen, "*", &[(DATA_KEY, &bytes)]); 163 | } 164 | 165 | let mut tasks = JoinSet::new(); 166 | 167 | for connection in &self.connections_pool { 168 | let mut connection = connection.clone(); 169 | let pipe = pipe.clone(); 170 | tasks.spawn(async move { 171 | let result: Result, redis::RedisError> = 172 | pipe.query_async(&mut connection).await; 173 | if let Err(e) = result { 174 | error!("Redis send error: {e}"); 175 | return Err(MessengerError::SendError { msg: e.to_string() }); 176 | } 177 | 178 | Ok(()) 179 | }); 180 | } 181 | 182 | while let Some(task) = tasks.join_next().await { 183 | match task { 184 | Ok(_) => { 185 | debug!("One of the message send tasks was finished") 186 | } 187 | Err(err) if err.is_panic() => { 188 | let msg = err.to_string(); 189 | error!("Task panic during sending message to Redis: {:?}", err); 190 | return Err(MessengerError::SendError { msg }); 191 | } 192 | Err(err) => { 193 | let msg = err.to_string(); 194 | return Err(MessengerError::SendError { msg }); 195 | } 196 | } 197 | } 198 | 199 | debug!("Data Sent to {}", stream_key); 200 | stream.local_buffer.clear(); 201 | stream.local_buffer_total = 0; 202 | stream.local_buffer_last_flush = Instant::now(); 203 | } 204 | Ok(()) 205 | } 206 | } 207 | -------------------------------------------------------------------------------- /plerkle_serialization/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "plerkle_serialization" 3 | description = "Metaplex Flatbuffers Plerkle Serialization for Geyser plugin producer/consumer patterns." 4 | version = "2.0.0" 5 | authors = ["Metaplex Developers "] 6 | repository = "https://github.com/metaplex-foundation/digital-asset-validator-plugin" 7 | license = "AGPL-3.0" 8 | edition = "2021" 9 | readme = "Readme.md" 10 | 11 | [dependencies] 12 | flatbuffers = "23.1.21" 13 | chrono = "0.4.22" 14 | serde = { version = "1.0.152" } 15 | solana-sdk = { version = "2.1" } 16 | solana-transaction-status = { version = "2.1" } 17 | bs58 = "0.4.0" 18 | thiserror = "1.0.38" 19 | [package.metadata.docs.rs] 20 | targets = ["x86_64-unknown-linux-gnu"] 21 | -------------------------------------------------------------------------------- /plerkle_serialization/Readme.md: -------------------------------------------------------------------------------- 1 | # Plerkle Serialization 2 | 3 | FlatBuffers based serialization code and schemas. This is the wire-format of Plerkle. 4 | 5 | -------------------------------------------------------------------------------- /plerkle_serialization/account_info.fbs: -------------------------------------------------------------------------------- 1 | // Flatbuffer IDL for Account Info schema. 2 | include "./common.fbs"; 3 | 4 | table AccountInfo { 5 | pubkey:Pubkey; 6 | lamports:uint64; 7 | owner:Pubkey; 8 | executable:bool; 9 | rent_epoch:uint64; 10 | data:[uint8]; 11 | write_version:uint64; 12 | slot:uint64; 13 | is_startup:bool; 14 | seen_at: int64; 15 | } 16 | 17 | root_type AccountInfo; 18 | -------------------------------------------------------------------------------- /plerkle_serialization/block_info.fbs: -------------------------------------------------------------------------------- 1 | // Flatbuffer IDL for Block Info schema. 2 | 3 | 4 | 5 | enum RewardType:uint8 { 6 | Fee, 7 | Rent, 8 | Staking, 9 | Voting, 10 | } 11 | 12 | table Reward { 13 | pubkey:[uint8]; 14 | lamports:int64; 15 | // Account balance in lamports after `lamports` was applied. 16 | post_balance:uint64; 17 | reward_type:RewardType = null; 18 | // Vote account commission when the reward was credited, only present for voting and staking rewards. 19 | commission:uint8 = null; 20 | } 21 | 22 | table BlockInfo { 23 | slot:uint64; 24 | blockhash:string; 25 | rewards:[Reward]; 26 | block_time:int64 = null; 27 | block_height:uint64 = null; 28 | seen_at: int64; 29 | } 30 | 31 | root_type BlockInfo; 32 | -------------------------------------------------------------------------------- /plerkle_serialization/common.fbs: -------------------------------------------------------------------------------- 1 | struct Pubkey { 2 | key:[uint8:32]; 3 | } -------------------------------------------------------------------------------- /plerkle_serialization/compiled_instruction.fbs: -------------------------------------------------------------------------------- 1 | include "./common.fbs"; 2 | 3 | table CompiledInstruction { 4 | // Index into the transaction keys array indicating the program account that executes this instruction. 5 | program_id_index:uint8; 6 | // Ordered indices into the transaction keys array indicating which accounts to pass to the program. 7 | accounts:[uint8]; 8 | // The program input data. 9 | data:[uint8]; 10 | } 11 | 12 | table CompiledInnerInstruction { 13 | compiled_instruction: CompiledInstruction; 14 | stack_height:uint8; 15 | } 16 | 17 | root_type CompiledInstruction; 18 | -------------------------------------------------------------------------------- /plerkle_serialization/generate_flatbuffers_code.sh: -------------------------------------------------------------------------------- 1 | #flatc --rust -o src/ account_info.fbs 2 | #flatc --rust -o src/ block_info.fbs 3 | #flatc --rust -o src/ slot_status_info.fbs 4 | #flatc --rust -o src/ transaction_info.fbs 5 | #flatc --rust -o src/ block_info.fbs 6 | flatc --rust --reflect-names --reflect-types --gen-mutable -o src *.fbs 7 | -------------------------------------------------------------------------------- /plerkle_serialization/rustfmt.toml: -------------------------------------------------------------------------------- 1 | edition = "2021" 2 | imports_granularity="Crate" 3 | reorder_imports = true 4 | -------------------------------------------------------------------------------- /plerkle_serialization/slot_status_info.fbs: -------------------------------------------------------------------------------- 1 | // Flatbuffer IDL for Slot Status Info schema. 2 | 3 | 4 | 5 | enum Status:byte { Processed, Rooted, Confirmed } 6 | 7 | table SlotStatusInfo { 8 | slot:uint64; 9 | parent:uint64 = null; 10 | status:Status; 11 | seen_at: int64; 12 | } 13 | 14 | root_type SlotStatusInfo; 15 | -------------------------------------------------------------------------------- /plerkle_serialization/src/account_info_generated.rs: -------------------------------------------------------------------------------- 1 | // automatically generated by the FlatBuffers compiler, do not modify 2 | 3 | 4 | // @generated 5 | 6 | use crate::common_generated::*; 7 | use core::mem; 8 | use core::cmp::Ordering; 9 | 10 | extern crate flatbuffers; 11 | use self::flatbuffers::{EndianScalar, Follow}; 12 | 13 | pub enum AccountInfoOffset {} 14 | #[derive(Copy, Clone, PartialEq, Eq)] 15 | 16 | pub struct AccountInfo<'a> { 17 | pub _tab: flatbuffers::Table<'a>, 18 | } 19 | 20 | impl<'a> flatbuffers::Follow<'a> for AccountInfo<'a> { 21 | type Inner = AccountInfo<'a>; 22 | #[inline] 23 | unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { 24 | Self { _tab: flatbuffers::Table::new(buf, loc) } 25 | } 26 | } 27 | 28 | impl<'a> AccountInfo<'a> { 29 | pub const VT_PUBKEY: flatbuffers::VOffsetT = 4; 30 | pub const VT_LAMPORTS: flatbuffers::VOffsetT = 6; 31 | pub const VT_OWNER: flatbuffers::VOffsetT = 8; 32 | pub const VT_EXECUTABLE: flatbuffers::VOffsetT = 10; 33 | pub const VT_RENT_EPOCH: flatbuffers::VOffsetT = 12; 34 | pub const VT_DATA: flatbuffers::VOffsetT = 14; 35 | pub const VT_WRITE_VERSION: flatbuffers::VOffsetT = 16; 36 | pub const VT_SLOT: flatbuffers::VOffsetT = 18; 37 | pub const VT_IS_STARTUP: flatbuffers::VOffsetT = 20; 38 | pub const VT_SEEN_AT: flatbuffers::VOffsetT = 22; 39 | 40 | #[inline] 41 | pub unsafe fn init_from_table(table: flatbuffers::Table<'a>) -> Self { 42 | AccountInfo { _tab: table } 43 | } 44 | #[allow(unused_mut)] 45 | pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( 46 | _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, 47 | args: &'args AccountInfoArgs<'args> 48 | ) -> flatbuffers::WIPOffset> { 49 | let mut builder = AccountInfoBuilder::new(_fbb); 50 | builder.add_seen_at(args.seen_at); 51 | builder.add_slot(args.slot); 52 | builder.add_write_version(args.write_version); 53 | builder.add_rent_epoch(args.rent_epoch); 54 | builder.add_lamports(args.lamports); 55 | if let Some(x) = args.data { builder.add_data(x); } 56 | if let Some(x) = args.owner { builder.add_owner(x); } 57 | if let Some(x) = args.pubkey { builder.add_pubkey(x); } 58 | builder.add_is_startup(args.is_startup); 59 | builder.add_executable(args.executable); 60 | builder.finish() 61 | } 62 | 63 | 64 | #[inline] 65 | pub fn pubkey(&self) -> Option<&'a Pubkey> { 66 | // Safety: 67 | // Created from valid Table for this object 68 | // which contains a valid value in this slot 69 | unsafe { self._tab.get::(AccountInfo::VT_PUBKEY, None)} 70 | } 71 | #[inline] 72 | pub fn lamports(&self) -> u64 { 73 | // Safety: 74 | // Created from valid Table for this object 75 | // which contains a valid value in this slot 76 | unsafe { self._tab.get::(AccountInfo::VT_LAMPORTS, Some(0)).unwrap()} 77 | } 78 | #[inline] 79 | pub fn owner(&self) -> Option<&'a Pubkey> { 80 | // Safety: 81 | // Created from valid Table for this object 82 | // which contains a valid value in this slot 83 | unsafe { self._tab.get::(AccountInfo::VT_OWNER, None)} 84 | } 85 | #[inline] 86 | pub fn executable(&self) -> bool { 87 | // Safety: 88 | // Created from valid Table for this object 89 | // which contains a valid value in this slot 90 | unsafe { self._tab.get::(AccountInfo::VT_EXECUTABLE, Some(false)).unwrap()} 91 | } 92 | #[inline] 93 | pub fn rent_epoch(&self) -> u64 { 94 | // Safety: 95 | // Created from valid Table for this object 96 | // which contains a valid value in this slot 97 | unsafe { self._tab.get::(AccountInfo::VT_RENT_EPOCH, Some(0)).unwrap()} 98 | } 99 | #[inline] 100 | pub fn data(&self) -> Option> { 101 | // Safety: 102 | // Created from valid Table for this object 103 | // which contains a valid value in this slot 104 | unsafe { self._tab.get::>>(AccountInfo::VT_DATA, None)} 105 | } 106 | #[inline] 107 | pub fn write_version(&self) -> u64 { 108 | // Safety: 109 | // Created from valid Table for this object 110 | // which contains a valid value in this slot 111 | unsafe { self._tab.get::(AccountInfo::VT_WRITE_VERSION, Some(0)).unwrap()} 112 | } 113 | #[inline] 114 | pub fn slot(&self) -> u64 { 115 | // Safety: 116 | // Created from valid Table for this object 117 | // which contains a valid value in this slot 118 | unsafe { self._tab.get::(AccountInfo::VT_SLOT, Some(0)).unwrap()} 119 | } 120 | #[inline] 121 | pub fn is_startup(&self) -> bool { 122 | // Safety: 123 | // Created from valid Table for this object 124 | // which contains a valid value in this slot 125 | unsafe { self._tab.get::(AccountInfo::VT_IS_STARTUP, Some(false)).unwrap()} 126 | } 127 | #[inline] 128 | pub fn seen_at(&self) -> i64 { 129 | // Safety: 130 | // Created from valid Table for this object 131 | // which contains a valid value in this slot 132 | unsafe { self._tab.get::(AccountInfo::VT_SEEN_AT, Some(0)).unwrap()} 133 | } 134 | } 135 | 136 | impl flatbuffers::Verifiable for AccountInfo<'_> { 137 | #[inline] 138 | fn run_verifier( 139 | v: &mut flatbuffers::Verifier, pos: usize 140 | ) -> Result<(), flatbuffers::InvalidFlatbuffer> { 141 | use self::flatbuffers::Verifiable; 142 | v.visit_table(pos)? 143 | .visit_field::("pubkey", Self::VT_PUBKEY, false)? 144 | .visit_field::("lamports", Self::VT_LAMPORTS, false)? 145 | .visit_field::("owner", Self::VT_OWNER, false)? 146 | .visit_field::("executable", Self::VT_EXECUTABLE, false)? 147 | .visit_field::("rent_epoch", Self::VT_RENT_EPOCH, false)? 148 | .visit_field::>>("data", Self::VT_DATA, false)? 149 | .visit_field::("write_version", Self::VT_WRITE_VERSION, false)? 150 | .visit_field::("slot", Self::VT_SLOT, false)? 151 | .visit_field::("is_startup", Self::VT_IS_STARTUP, false)? 152 | .visit_field::("seen_at", Self::VT_SEEN_AT, false)? 153 | .finish(); 154 | Ok(()) 155 | } 156 | } 157 | pub struct AccountInfoArgs<'a> { 158 | pub pubkey: Option<&'a Pubkey>, 159 | pub lamports: u64, 160 | pub owner: Option<&'a Pubkey>, 161 | pub executable: bool, 162 | pub rent_epoch: u64, 163 | pub data: Option>>, 164 | pub write_version: u64, 165 | pub slot: u64, 166 | pub is_startup: bool, 167 | pub seen_at: i64, 168 | } 169 | impl<'a> Default for AccountInfoArgs<'a> { 170 | #[inline] 171 | fn default() -> Self { 172 | AccountInfoArgs { 173 | pubkey: None, 174 | lamports: 0, 175 | owner: None, 176 | executable: false, 177 | rent_epoch: 0, 178 | data: None, 179 | write_version: 0, 180 | slot: 0, 181 | is_startup: false, 182 | seen_at: 0, 183 | } 184 | } 185 | } 186 | 187 | pub struct AccountInfoBuilder<'a: 'b, 'b> { 188 | fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, 189 | start_: flatbuffers::WIPOffset, 190 | } 191 | impl<'a: 'b, 'b> AccountInfoBuilder<'a, 'b> { 192 | #[inline] 193 | pub fn add_pubkey(&mut self, pubkey: &Pubkey) { 194 | self.fbb_.push_slot_always::<&Pubkey>(AccountInfo::VT_PUBKEY, pubkey); 195 | } 196 | #[inline] 197 | pub fn add_lamports(&mut self, lamports: u64) { 198 | self.fbb_.push_slot::(AccountInfo::VT_LAMPORTS, lamports, 0); 199 | } 200 | #[inline] 201 | pub fn add_owner(&mut self, owner: &Pubkey) { 202 | self.fbb_.push_slot_always::<&Pubkey>(AccountInfo::VT_OWNER, owner); 203 | } 204 | #[inline] 205 | pub fn add_executable(&mut self, executable: bool) { 206 | self.fbb_.push_slot::(AccountInfo::VT_EXECUTABLE, executable, false); 207 | } 208 | #[inline] 209 | pub fn add_rent_epoch(&mut self, rent_epoch: u64) { 210 | self.fbb_.push_slot::(AccountInfo::VT_RENT_EPOCH, rent_epoch, 0); 211 | } 212 | #[inline] 213 | pub fn add_data(&mut self, data: flatbuffers::WIPOffset>) { 214 | self.fbb_.push_slot_always::>(AccountInfo::VT_DATA, data); 215 | } 216 | #[inline] 217 | pub fn add_write_version(&mut self, write_version: u64) { 218 | self.fbb_.push_slot::(AccountInfo::VT_WRITE_VERSION, write_version, 0); 219 | } 220 | #[inline] 221 | pub fn add_slot(&mut self, slot: u64) { 222 | self.fbb_.push_slot::(AccountInfo::VT_SLOT, slot, 0); 223 | } 224 | #[inline] 225 | pub fn add_is_startup(&mut self, is_startup: bool) { 226 | self.fbb_.push_slot::(AccountInfo::VT_IS_STARTUP, is_startup, false); 227 | } 228 | #[inline] 229 | pub fn add_seen_at(&mut self, seen_at: i64) { 230 | self.fbb_.push_slot::(AccountInfo::VT_SEEN_AT, seen_at, 0); 231 | } 232 | #[inline] 233 | pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> AccountInfoBuilder<'a, 'b> { 234 | let start = _fbb.start_table(); 235 | AccountInfoBuilder { 236 | fbb_: _fbb, 237 | start_: start, 238 | } 239 | } 240 | #[inline] 241 | pub fn finish(self) -> flatbuffers::WIPOffset> { 242 | let o = self.fbb_.end_table(self.start_); 243 | flatbuffers::WIPOffset::new(o.value()) 244 | } 245 | } 246 | 247 | impl core::fmt::Debug for AccountInfo<'_> { 248 | fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { 249 | let mut ds = f.debug_struct("AccountInfo"); 250 | ds.field("pubkey", &self.pubkey()); 251 | ds.field("lamports", &self.lamports()); 252 | ds.field("owner", &self.owner()); 253 | ds.field("executable", &self.executable()); 254 | ds.field("rent_epoch", &self.rent_epoch()); 255 | ds.field("data", &self.data()); 256 | ds.field("write_version", &self.write_version()); 257 | ds.field("slot", &self.slot()); 258 | ds.field("is_startup", &self.is_startup()); 259 | ds.field("seen_at", &self.seen_at()); 260 | ds.finish() 261 | } 262 | } 263 | #[inline] 264 | /// Verifies that a buffer of bytes contains a `AccountInfo` 265 | /// and returns it. 266 | /// Note that verification is still experimental and may not 267 | /// catch every error, or be maximally performant. For the 268 | /// previous, unchecked, behavior use 269 | /// `root_as_account_info_unchecked`. 270 | pub fn root_as_account_info(buf: &[u8]) -> Result { 271 | flatbuffers::root::(buf) 272 | } 273 | #[inline] 274 | /// Verifies that a buffer of bytes contains a size prefixed 275 | /// `AccountInfo` and returns it. 276 | /// Note that verification is still experimental and may not 277 | /// catch every error, or be maximally performant. For the 278 | /// previous, unchecked, behavior use 279 | /// `size_prefixed_root_as_account_info_unchecked`. 280 | pub fn size_prefixed_root_as_account_info(buf: &[u8]) -> Result { 281 | flatbuffers::size_prefixed_root::(buf) 282 | } 283 | #[inline] 284 | /// Verifies, with the given options, that a buffer of bytes 285 | /// contains a `AccountInfo` and returns it. 286 | /// Note that verification is still experimental and may not 287 | /// catch every error, or be maximally performant. For the 288 | /// previous, unchecked, behavior use 289 | /// `root_as_account_info_unchecked`. 290 | pub fn root_as_account_info_with_opts<'b, 'o>( 291 | opts: &'o flatbuffers::VerifierOptions, 292 | buf: &'b [u8], 293 | ) -> Result, flatbuffers::InvalidFlatbuffer> { 294 | flatbuffers::root_with_opts::>(opts, buf) 295 | } 296 | #[inline] 297 | /// Verifies, with the given verifier options, that a buffer of 298 | /// bytes contains a size prefixed `AccountInfo` and returns 299 | /// it. Note that verification is still experimental and may not 300 | /// catch every error, or be maximally performant. For the 301 | /// previous, unchecked, behavior use 302 | /// `root_as_account_info_unchecked`. 303 | pub fn size_prefixed_root_as_account_info_with_opts<'b, 'o>( 304 | opts: &'o flatbuffers::VerifierOptions, 305 | buf: &'b [u8], 306 | ) -> Result, flatbuffers::InvalidFlatbuffer> { 307 | flatbuffers::size_prefixed_root_with_opts::>(opts, buf) 308 | } 309 | #[inline] 310 | /// Assumes, without verification, that a buffer of bytes contains a AccountInfo and returns it. 311 | /// # Safety 312 | /// Callers must trust the given bytes do indeed contain a valid `AccountInfo`. 313 | pub unsafe fn root_as_account_info_unchecked(buf: &[u8]) -> AccountInfo { 314 | flatbuffers::root_unchecked::(buf) 315 | } 316 | #[inline] 317 | /// Assumes, without verification, that a buffer of bytes contains a size prefixed AccountInfo and returns it. 318 | /// # Safety 319 | /// Callers must trust the given bytes do indeed contain a valid size prefixed `AccountInfo`. 320 | pub unsafe fn size_prefixed_root_as_account_info_unchecked(buf: &[u8]) -> AccountInfo { 321 | flatbuffers::size_prefixed_root_unchecked::(buf) 322 | } 323 | #[inline] 324 | pub fn finish_account_info_buffer<'a, 'b>( 325 | fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>, 326 | root: flatbuffers::WIPOffset>) { 327 | fbb.finish(root, None); 328 | } 329 | 330 | #[inline] 331 | pub fn finish_size_prefixed_account_info_buffer<'a, 'b>(fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>, root: flatbuffers::WIPOffset>) { 332 | fbb.finish_size_prefixed(root, None); 333 | } 334 | -------------------------------------------------------------------------------- /plerkle_serialization/src/common_generated.rs: -------------------------------------------------------------------------------- 1 | // automatically generated by the FlatBuffers compiler, do not modify 2 | 3 | 4 | // @generated 5 | 6 | use core::mem; 7 | use core::cmp::Ordering; 8 | 9 | extern crate flatbuffers; 10 | use self::flatbuffers::{EndianScalar, Follow}; 11 | 12 | // struct Pubkey, aligned to 1 13 | #[repr(transparent)] 14 | #[derive(Clone, Copy, PartialEq, Eq)] 15 | pub struct Pubkey(pub [u8; 32]); 16 | impl Default for Pubkey { 17 | fn default() -> Self { 18 | Self([0; 32]) 19 | } 20 | } 21 | impl core::fmt::Debug for Pubkey { 22 | fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { 23 | f.debug_struct("Pubkey") 24 | .field("key", &self.key()) 25 | .finish() 26 | } 27 | } 28 | 29 | impl flatbuffers::SimpleToVerifyInSlice for Pubkey {} 30 | impl<'a> flatbuffers::Follow<'a> for Pubkey { 31 | type Inner = &'a Pubkey; 32 | #[inline] 33 | unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { 34 | <&'a Pubkey>::follow(buf, loc) 35 | } 36 | } 37 | impl<'a> flatbuffers::Follow<'a> for &'a Pubkey { 38 | type Inner = &'a Pubkey; 39 | #[inline] 40 | unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { 41 | flatbuffers::follow_cast_ref::(buf, loc) 42 | } 43 | } 44 | impl<'b> flatbuffers::Push for Pubkey { 45 | type Output = Pubkey; 46 | #[inline] 47 | unsafe fn push(&self, dst: &mut [u8], _written_len: usize) { 48 | let src = ::core::slice::from_raw_parts(self as *const Pubkey as *const u8, Self::size()); 49 | dst.copy_from_slice(src); 50 | } 51 | } 52 | 53 | impl<'a> flatbuffers::Verifiable for Pubkey { 54 | #[inline] 55 | fn run_verifier( 56 | v: &mut flatbuffers::Verifier, pos: usize 57 | ) -> Result<(), flatbuffers::InvalidFlatbuffer> { 58 | use self::flatbuffers::Verifiable; 59 | v.in_buffer::(pos) 60 | } 61 | } 62 | 63 | impl<'a> Pubkey { 64 | #[allow(clippy::too_many_arguments)] 65 | pub fn new( 66 | key: &[u8; 32], 67 | ) -> Self { 68 | let mut s = Self([0; 32]); 69 | s.set_key(key); 70 | s 71 | } 72 | 73 | pub fn key(&'a self) -> flatbuffers::Array<'a, u8, 32> { 74 | // Safety: 75 | // Created from a valid Table for this object 76 | // Which contains a valid array in this slot 77 | unsafe { flatbuffers::Array::follow(&self.0, 0) } 78 | } 79 | 80 | pub fn set_key(&mut self, items: &[u8; 32]) { 81 | // Safety: 82 | // Created from a valid Table for this object 83 | // Which contains a valid array in this slot 84 | unsafe { flatbuffers::emplace_scalar_array(&mut self.0, 0, items) }; 85 | } 86 | 87 | } 88 | 89 | -------------------------------------------------------------------------------- /plerkle_serialization/src/compiled_instruction_generated.rs: -------------------------------------------------------------------------------- 1 | // automatically generated by the FlatBuffers compiler, do not modify 2 | 3 | 4 | // @generated 5 | 6 | use crate::common_generated::*; 7 | use core::mem; 8 | use core::cmp::Ordering; 9 | 10 | extern crate flatbuffers; 11 | use self::flatbuffers::{EndianScalar, Follow}; 12 | 13 | pub enum CompiledInstructionOffset {} 14 | #[derive(Copy, Clone, PartialEq, Eq)] 15 | 16 | pub struct CompiledInstruction<'a> { 17 | pub _tab: flatbuffers::Table<'a>, 18 | } 19 | 20 | impl<'a> flatbuffers::Follow<'a> for CompiledInstruction<'a> { 21 | type Inner = CompiledInstruction<'a>; 22 | #[inline] 23 | unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { 24 | Self { _tab: flatbuffers::Table::new(buf, loc) } 25 | } 26 | } 27 | 28 | impl<'a> CompiledInstruction<'a> { 29 | pub const VT_PROGRAM_ID_INDEX: flatbuffers::VOffsetT = 4; 30 | pub const VT_ACCOUNTS: flatbuffers::VOffsetT = 6; 31 | pub const VT_DATA: flatbuffers::VOffsetT = 8; 32 | 33 | #[inline] 34 | pub unsafe fn init_from_table(table: flatbuffers::Table<'a>) -> Self { 35 | CompiledInstruction { _tab: table } 36 | } 37 | #[allow(unused_mut)] 38 | pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( 39 | _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, 40 | args: &'args CompiledInstructionArgs<'args> 41 | ) -> flatbuffers::WIPOffset> { 42 | let mut builder = CompiledInstructionBuilder::new(_fbb); 43 | if let Some(x) = args.data { builder.add_data(x); } 44 | if let Some(x) = args.accounts { builder.add_accounts(x); } 45 | builder.add_program_id_index(args.program_id_index); 46 | builder.finish() 47 | } 48 | 49 | 50 | #[inline] 51 | pub fn program_id_index(&self) -> u8 { 52 | // Safety: 53 | // Created from valid Table for this object 54 | // which contains a valid value in this slot 55 | unsafe { self._tab.get::(CompiledInstruction::VT_PROGRAM_ID_INDEX, Some(0)).unwrap()} 56 | } 57 | #[inline] 58 | pub fn accounts(&self) -> Option> { 59 | // Safety: 60 | // Created from valid Table for this object 61 | // which contains a valid value in this slot 62 | unsafe { self._tab.get::>>(CompiledInstruction::VT_ACCOUNTS, None)} 63 | } 64 | #[inline] 65 | pub fn data(&self) -> Option> { 66 | // Safety: 67 | // Created from valid Table for this object 68 | // which contains a valid value in this slot 69 | unsafe { self._tab.get::>>(CompiledInstruction::VT_DATA, None)} 70 | } 71 | } 72 | 73 | impl flatbuffers::Verifiable for CompiledInstruction<'_> { 74 | #[inline] 75 | fn run_verifier( 76 | v: &mut flatbuffers::Verifier, pos: usize 77 | ) -> Result<(), flatbuffers::InvalidFlatbuffer> { 78 | use self::flatbuffers::Verifiable; 79 | v.visit_table(pos)? 80 | .visit_field::("program_id_index", Self::VT_PROGRAM_ID_INDEX, false)? 81 | .visit_field::>>("accounts", Self::VT_ACCOUNTS, false)? 82 | .visit_field::>>("data", Self::VT_DATA, false)? 83 | .finish(); 84 | Ok(()) 85 | } 86 | } 87 | pub struct CompiledInstructionArgs<'a> { 88 | pub program_id_index: u8, 89 | pub accounts: Option>>, 90 | pub data: Option>>, 91 | } 92 | impl<'a> Default for CompiledInstructionArgs<'a> { 93 | #[inline] 94 | fn default() -> Self { 95 | CompiledInstructionArgs { 96 | program_id_index: 0, 97 | accounts: None, 98 | data: None, 99 | } 100 | } 101 | } 102 | 103 | pub struct CompiledInstructionBuilder<'a: 'b, 'b> { 104 | fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, 105 | start_: flatbuffers::WIPOffset, 106 | } 107 | impl<'a: 'b, 'b> CompiledInstructionBuilder<'a, 'b> { 108 | #[inline] 109 | pub fn add_program_id_index(&mut self, program_id_index: u8) { 110 | self.fbb_.push_slot::(CompiledInstruction::VT_PROGRAM_ID_INDEX, program_id_index, 0); 111 | } 112 | #[inline] 113 | pub fn add_accounts(&mut self, accounts: flatbuffers::WIPOffset>) { 114 | self.fbb_.push_slot_always::>(CompiledInstruction::VT_ACCOUNTS, accounts); 115 | } 116 | #[inline] 117 | pub fn add_data(&mut self, data: flatbuffers::WIPOffset>) { 118 | self.fbb_.push_slot_always::>(CompiledInstruction::VT_DATA, data); 119 | } 120 | #[inline] 121 | pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> CompiledInstructionBuilder<'a, 'b> { 122 | let start = _fbb.start_table(); 123 | CompiledInstructionBuilder { 124 | fbb_: _fbb, 125 | start_: start, 126 | } 127 | } 128 | #[inline] 129 | pub fn finish(self) -> flatbuffers::WIPOffset> { 130 | let o = self.fbb_.end_table(self.start_); 131 | flatbuffers::WIPOffset::new(o.value()) 132 | } 133 | } 134 | 135 | impl core::fmt::Debug for CompiledInstruction<'_> { 136 | fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { 137 | let mut ds = f.debug_struct("CompiledInstruction"); 138 | ds.field("program_id_index", &self.program_id_index()); 139 | ds.field("accounts", &self.accounts()); 140 | ds.field("data", &self.data()); 141 | ds.finish() 142 | } 143 | } 144 | pub enum CompiledInnerInstructionOffset {} 145 | #[derive(Copy, Clone, PartialEq, Eq)] 146 | 147 | pub struct CompiledInnerInstruction<'a> { 148 | pub _tab: flatbuffers::Table<'a>, 149 | } 150 | 151 | impl<'a> flatbuffers::Follow<'a> for CompiledInnerInstruction<'a> { 152 | type Inner = CompiledInnerInstruction<'a>; 153 | #[inline] 154 | unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { 155 | Self { _tab: flatbuffers::Table::new(buf, loc) } 156 | } 157 | } 158 | 159 | impl<'a> CompiledInnerInstruction<'a> { 160 | pub const VT_COMPILED_INSTRUCTION: flatbuffers::VOffsetT = 4; 161 | pub const VT_STACK_HEIGHT: flatbuffers::VOffsetT = 6; 162 | 163 | #[inline] 164 | pub unsafe fn init_from_table(table: flatbuffers::Table<'a>) -> Self { 165 | CompiledInnerInstruction { _tab: table } 166 | } 167 | #[allow(unused_mut)] 168 | pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( 169 | _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, 170 | args: &'args CompiledInnerInstructionArgs<'args> 171 | ) -> flatbuffers::WIPOffset> { 172 | let mut builder = CompiledInnerInstructionBuilder::new(_fbb); 173 | if let Some(x) = args.compiled_instruction { builder.add_compiled_instruction(x); } 174 | builder.add_stack_height(args.stack_height); 175 | builder.finish() 176 | } 177 | 178 | 179 | #[inline] 180 | pub fn compiled_instruction(&self) -> Option> { 181 | // Safety: 182 | // Created from valid Table for this object 183 | // which contains a valid value in this slot 184 | unsafe { self._tab.get::>(CompiledInnerInstruction::VT_COMPILED_INSTRUCTION, None)} 185 | } 186 | #[inline] 187 | pub fn stack_height(&self) -> u8 { 188 | // Safety: 189 | // Created from valid Table for this object 190 | // which contains a valid value in this slot 191 | unsafe { self._tab.get::(CompiledInnerInstruction::VT_STACK_HEIGHT, Some(0)).unwrap()} 192 | } 193 | } 194 | 195 | impl flatbuffers::Verifiable for CompiledInnerInstruction<'_> { 196 | #[inline] 197 | fn run_verifier( 198 | v: &mut flatbuffers::Verifier, pos: usize 199 | ) -> Result<(), flatbuffers::InvalidFlatbuffer> { 200 | use self::flatbuffers::Verifiable; 201 | v.visit_table(pos)? 202 | .visit_field::>("compiled_instruction", Self::VT_COMPILED_INSTRUCTION, false)? 203 | .visit_field::("stack_height", Self::VT_STACK_HEIGHT, false)? 204 | .finish(); 205 | Ok(()) 206 | } 207 | } 208 | pub struct CompiledInnerInstructionArgs<'a> { 209 | pub compiled_instruction: Option>>, 210 | pub stack_height: u8, 211 | } 212 | impl<'a> Default for CompiledInnerInstructionArgs<'a> { 213 | #[inline] 214 | fn default() -> Self { 215 | CompiledInnerInstructionArgs { 216 | compiled_instruction: None, 217 | stack_height: 0, 218 | } 219 | } 220 | } 221 | 222 | pub struct CompiledInnerInstructionBuilder<'a: 'b, 'b> { 223 | fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, 224 | start_: flatbuffers::WIPOffset, 225 | } 226 | impl<'a: 'b, 'b> CompiledInnerInstructionBuilder<'a, 'b> { 227 | #[inline] 228 | pub fn add_compiled_instruction(&mut self, compiled_instruction: flatbuffers::WIPOffset>) { 229 | self.fbb_.push_slot_always::>(CompiledInnerInstruction::VT_COMPILED_INSTRUCTION, compiled_instruction); 230 | } 231 | #[inline] 232 | pub fn add_stack_height(&mut self, stack_height: u8) { 233 | self.fbb_.push_slot::(CompiledInnerInstruction::VT_STACK_HEIGHT, stack_height, 0); 234 | } 235 | #[inline] 236 | pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> CompiledInnerInstructionBuilder<'a, 'b> { 237 | let start = _fbb.start_table(); 238 | CompiledInnerInstructionBuilder { 239 | fbb_: _fbb, 240 | start_: start, 241 | } 242 | } 243 | #[inline] 244 | pub fn finish(self) -> flatbuffers::WIPOffset> { 245 | let o = self.fbb_.end_table(self.start_); 246 | flatbuffers::WIPOffset::new(o.value()) 247 | } 248 | } 249 | 250 | impl core::fmt::Debug for CompiledInnerInstruction<'_> { 251 | fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { 252 | let mut ds = f.debug_struct("CompiledInnerInstruction"); 253 | ds.field("compiled_instruction", &self.compiled_instruction()); 254 | ds.field("stack_height", &self.stack_height()); 255 | ds.finish() 256 | } 257 | } 258 | #[inline] 259 | /// Verifies that a buffer of bytes contains a `CompiledInstruction` 260 | /// and returns it. 261 | /// Note that verification is still experimental and may not 262 | /// catch every error, or be maximally performant. For the 263 | /// previous, unchecked, behavior use 264 | /// `root_as_compiled_instruction_unchecked`. 265 | pub fn root_as_compiled_instruction(buf: &[u8]) -> Result { 266 | flatbuffers::root::(buf) 267 | } 268 | #[inline] 269 | /// Verifies that a buffer of bytes contains a size prefixed 270 | /// `CompiledInstruction` and returns it. 271 | /// Note that verification is still experimental and may not 272 | /// catch every error, or be maximally performant. For the 273 | /// previous, unchecked, behavior use 274 | /// `size_prefixed_root_as_compiled_instruction_unchecked`. 275 | pub fn size_prefixed_root_as_compiled_instruction(buf: &[u8]) -> Result { 276 | flatbuffers::size_prefixed_root::(buf) 277 | } 278 | #[inline] 279 | /// Verifies, with the given options, that a buffer of bytes 280 | /// contains a `CompiledInstruction` and returns it. 281 | /// Note that verification is still experimental and may not 282 | /// catch every error, or be maximally performant. For the 283 | /// previous, unchecked, behavior use 284 | /// `root_as_compiled_instruction_unchecked`. 285 | pub fn root_as_compiled_instruction_with_opts<'b, 'o>( 286 | opts: &'o flatbuffers::VerifierOptions, 287 | buf: &'b [u8], 288 | ) -> Result, flatbuffers::InvalidFlatbuffer> { 289 | flatbuffers::root_with_opts::>(opts, buf) 290 | } 291 | #[inline] 292 | /// Verifies, with the given verifier options, that a buffer of 293 | /// bytes contains a size prefixed `CompiledInstruction` and returns 294 | /// it. Note that verification is still experimental and may not 295 | /// catch every error, or be maximally performant. For the 296 | /// previous, unchecked, behavior use 297 | /// `root_as_compiled_instruction_unchecked`. 298 | pub fn size_prefixed_root_as_compiled_instruction_with_opts<'b, 'o>( 299 | opts: &'o flatbuffers::VerifierOptions, 300 | buf: &'b [u8], 301 | ) -> Result, flatbuffers::InvalidFlatbuffer> { 302 | flatbuffers::size_prefixed_root_with_opts::>(opts, buf) 303 | } 304 | #[inline] 305 | /// Assumes, without verification, that a buffer of bytes contains a CompiledInstruction and returns it. 306 | /// # Safety 307 | /// Callers must trust the given bytes do indeed contain a valid `CompiledInstruction`. 308 | pub unsafe fn root_as_compiled_instruction_unchecked(buf: &[u8]) -> CompiledInstruction { 309 | flatbuffers::root_unchecked::(buf) 310 | } 311 | #[inline] 312 | /// Assumes, without verification, that a buffer of bytes contains a size prefixed CompiledInstruction and returns it. 313 | /// # Safety 314 | /// Callers must trust the given bytes do indeed contain a valid size prefixed `CompiledInstruction`. 315 | pub unsafe fn size_prefixed_root_as_compiled_instruction_unchecked(buf: &[u8]) -> CompiledInstruction { 316 | flatbuffers::size_prefixed_root_unchecked::(buf) 317 | } 318 | #[inline] 319 | pub fn finish_compiled_instruction_buffer<'a, 'b>( 320 | fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>, 321 | root: flatbuffers::WIPOffset>) { 322 | fbb.finish(root, None); 323 | } 324 | 325 | #[inline] 326 | pub fn finish_size_prefixed_compiled_instruction_buffer<'a, 'b>(fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>, root: flatbuffers::WIPOffset>) { 327 | fbb.finish_size_prefixed(root, None); 328 | } 329 | -------------------------------------------------------------------------------- /plerkle_serialization/src/deserializer/mod.rs: -------------------------------------------------------------------------------- 1 | mod solana; 2 | 3 | pub use solana::*; 4 | -------------------------------------------------------------------------------- /plerkle_serialization/src/deserializer/solana.rs: -------------------------------------------------------------------------------- 1 | use std::convert::TryFrom; 2 | 3 | use crate::{ 4 | CompiledInnerInstructions as FBCompiledInnerInstructions, 5 | CompiledInstruction as FBCompiledInstruction, InnerInstructions as FBInnerInstructions, 6 | Pubkey as FBPubkey, 7 | }; 8 | use flatbuffers::{ForwardsUOffset, Vector}; 9 | use solana_sdk::{instruction::CompiledInstruction, pubkey::Pubkey, signature::Signature}; 10 | use solana_transaction_status::{InnerInstruction, InnerInstructions}; 11 | 12 | #[derive(Debug, Clone, PartialEq, thiserror::Error)] 13 | pub enum SolanaDeserializerError { 14 | #[error("Deserialization error")] 15 | DeserializationError, 16 | #[error("Not found")] 17 | NotFound, 18 | #[error("Invalid FlatBuffer key")] 19 | InvalidFlatBufferKey, 20 | } 21 | 22 | pub type SolanaDeserializeResult = Result; 23 | 24 | impl<'a> TryFrom<&FBPubkey> for Pubkey { 25 | type Error = SolanaDeserializerError; 26 | 27 | fn try_from(pubkey: &FBPubkey) -> SolanaDeserializeResult { 28 | Pubkey::try_from(pubkey.0.as_slice()) 29 | .map_err(|_error| SolanaDeserializerError::InvalidFlatBufferKey) 30 | } 31 | } 32 | 33 | pub struct PlerkleOptionalU8Vector<'a>(pub Option>); 34 | 35 | impl<'a> TryFrom> for Vec { 36 | type Error = SolanaDeserializerError; 37 | 38 | fn try_from(data: PlerkleOptionalU8Vector<'a>) -> SolanaDeserializeResult { 39 | Ok(data 40 | .0 41 | .ok_or(SolanaDeserializerError::NotFound)? 42 | .bytes() 43 | .to_vec()) 44 | } 45 | } 46 | 47 | pub struct PlerkleOptionalStr<'a>(pub Option<&'a str>); 48 | 49 | impl<'a> TryFrom> for Signature { 50 | type Error = SolanaDeserializerError; 51 | 52 | fn try_from(data: PlerkleOptionalStr<'a>) -> SolanaDeserializeResult { 53 | data.0 54 | .ok_or(SolanaDeserializerError::NotFound)? 55 | .parse::() 56 | .map_err(|_error| SolanaDeserializerError::DeserializationError) 57 | } 58 | } 59 | 60 | pub struct PlerkleOptionalPubkeyVector<'a>(pub Option>); 61 | 62 | impl<'a> TryFrom> for Vec { 63 | type Error = SolanaDeserializerError; 64 | 65 | fn try_from(public_keys: PlerkleOptionalPubkeyVector<'a>) -> SolanaDeserializeResult { 66 | public_keys 67 | .0 68 | .ok_or(SolanaDeserializerError::NotFound)? 69 | .iter() 70 | .map(|key| { 71 | Pubkey::try_from(key.0.as_slice()) 72 | .map_err(|_error| SolanaDeserializerError::InvalidFlatBufferKey) 73 | }) 74 | .collect::>>() 75 | } 76 | } 77 | 78 | pub struct PlerkleCompiledInstructionVector<'a>( 79 | pub Vector<'a, ForwardsUOffset>>, 80 | ); 81 | 82 | impl<'a> TryFrom> for Vec { 83 | type Error = SolanaDeserializerError; 84 | 85 | fn try_from(vec_cix: PlerkleCompiledInstructionVector<'a>) -> SolanaDeserializeResult { 86 | let mut message_instructions = vec![]; 87 | 88 | for cix in vec_cix.0 { 89 | message_instructions.push(CompiledInstruction { 90 | program_id_index: cix.program_id_index(), 91 | accounts: cix 92 | .accounts() 93 | .ok_or(SolanaDeserializerError::NotFound)? 94 | .bytes() 95 | .to_vec(), 96 | data: cix 97 | .data() 98 | .ok_or(SolanaDeserializerError::NotFound)? 99 | .bytes() 100 | .to_vec(), 101 | }) 102 | } 103 | 104 | Ok(message_instructions) 105 | } 106 | } 107 | 108 | pub struct PlerkleCompiledInnerInstructionVector<'a>( 109 | pub Vector<'a, ForwardsUOffset>>, 110 | ); 111 | impl<'a> TryFrom> for Vec { 112 | type Error = SolanaDeserializerError; 113 | 114 | fn try_from( 115 | vec_ixs: PlerkleCompiledInnerInstructionVector<'a>, 116 | ) -> SolanaDeserializeResult { 117 | let mut meta_inner_instructions = vec![]; 118 | 119 | for ixs in vec_ixs.0 { 120 | let mut instructions = vec![]; 121 | for ix in ixs 122 | .instructions() 123 | .ok_or(SolanaDeserializerError::NotFound)? 124 | { 125 | let cix = ix 126 | .compiled_instruction() 127 | .ok_or(SolanaDeserializerError::NotFound)?; 128 | instructions.push(InnerInstruction { 129 | instruction: CompiledInstruction { 130 | program_id_index: cix.program_id_index(), 131 | accounts: cix 132 | .accounts() 133 | .ok_or(SolanaDeserializerError::NotFound)? 134 | .bytes() 135 | .to_vec(), 136 | data: cix 137 | .data() 138 | .ok_or(SolanaDeserializerError::NotFound)? 139 | .bytes() 140 | .to_vec(), 141 | }, 142 | stack_height: Some(ix.stack_height() as u32), 143 | }); 144 | } 145 | meta_inner_instructions.push(InnerInstructions { 146 | index: ixs.index(), 147 | instructions, 148 | }) 149 | } 150 | 151 | Ok(meta_inner_instructions) 152 | } 153 | } 154 | 155 | pub struct PlerkleInnerInstructionsVector<'a>( 156 | pub Vector<'a, ForwardsUOffset>>, 157 | ); 158 | 159 | impl<'a> TryFrom> for Vec { 160 | type Error = SolanaDeserializerError; 161 | 162 | fn try_from(vec_ixs: PlerkleInnerInstructionsVector<'a>) -> SolanaDeserializeResult { 163 | vec_ixs 164 | .0 165 | .iter() 166 | .map(|iixs| { 167 | let instructions = iixs 168 | .instructions() 169 | .ok_or(SolanaDeserializerError::NotFound)? 170 | .iter() 171 | .map(|cix| { 172 | Ok(InnerInstruction { 173 | instruction: CompiledInstruction { 174 | program_id_index: cix.program_id_index(), 175 | accounts: cix 176 | .accounts() 177 | .ok_or(SolanaDeserializerError::NotFound)? 178 | .bytes() 179 | .to_vec(), 180 | data: cix 181 | .data() 182 | .ok_or(SolanaDeserializerError::NotFound)? 183 | .bytes() 184 | .to_vec(), 185 | }, 186 | stack_height: Some(0), 187 | }) 188 | }) 189 | .collect::>>()?; 190 | Ok(InnerInstructions { 191 | index: iixs.index(), 192 | instructions, 193 | }) 194 | }) 195 | .collect::>>() 196 | } 197 | } 198 | -------------------------------------------------------------------------------- /plerkle_serialization/src/error.rs: -------------------------------------------------------------------------------- 1 | use thiserror::Error; 2 | #[derive(Debug, Clone, PartialEq, Eq, Error)] 3 | pub enum PlerkleSerializationError { 4 | #[error("Serialization error: {0}")] 5 | SerializationError(String), 6 | } 7 | -------------------------------------------------------------------------------- /plerkle_serialization/src/lib.rs: -------------------------------------------------------------------------------- 1 | #[allow(unused_imports)] 2 | #[rustfmt::skip] 3 | mod account_info_generated; 4 | #[allow(unused_imports)] 5 | #[rustfmt::skip] 6 | mod block_info_generated; 7 | #[allow(unused_imports)] 8 | #[rustfmt::skip] 9 | mod common_generated; 10 | #[allow(unused_imports)] 11 | #[rustfmt::skip] 12 | mod compiled_instruction_generated; 13 | #[allow(unused_imports)] 14 | #[rustfmt::skip] 15 | mod slot_status_info_generated; 16 | #[allow(unused_imports)] 17 | #[rustfmt::skip] 18 | mod transaction_info_generated; 19 | 20 | pub mod deserializer; 21 | pub mod error; 22 | pub mod serializer; 23 | pub use account_info_generated::*; 24 | pub use block_info_generated::*; 25 | pub use common_generated::*; 26 | pub use compiled_instruction_generated::*; 27 | pub use slot_status_info_generated::*; 28 | pub use transaction_info_generated::*; 29 | 30 | // ---- SHIMS 31 | #[allow(unused_imports)] 32 | pub mod solana_geyser_plugin_interface_shims; 33 | -------------------------------------------------------------------------------- /plerkle_serialization/src/serializer/mod.rs: -------------------------------------------------------------------------------- 1 | mod serializer_common; 2 | mod serializer_stable; 3 | pub use serializer_stable::*; 4 | -------------------------------------------------------------------------------- /plerkle_serialization/src/serializer/serializer_common.rs: -------------------------------------------------------------------------------- 1 | use crate::Pubkey; 2 | 3 | impl From<&[u8]> for Pubkey { 4 | fn from(slice: &[u8]) -> Self { 5 | let arr = <[u8; 32]>::try_from(slice); 6 | Pubkey::new(&arr.unwrap()) 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /plerkle_serialization/src/slot_status_info_generated.rs: -------------------------------------------------------------------------------- 1 | // automatically generated by the FlatBuffers compiler, do not modify 2 | 3 | 4 | // @generated 5 | 6 | use core::mem; 7 | use core::cmp::Ordering; 8 | 9 | extern crate flatbuffers; 10 | use self::flatbuffers::{EndianScalar, Follow}; 11 | 12 | #[deprecated(since = "2.0.0", note = "Use associated constants instead. This will no longer be generated in 2021.")] 13 | pub const ENUM_MIN_STATUS: i8 = 0; 14 | #[deprecated(since = "2.0.0", note = "Use associated constants instead. This will no longer be generated in 2021.")] 15 | pub const ENUM_MAX_STATUS: i8 = 2; 16 | #[deprecated(since = "2.0.0", note = "Use associated constants instead. This will no longer be generated in 2021.")] 17 | #[allow(non_camel_case_types)] 18 | pub const ENUM_VALUES_STATUS: [Status; 3] = [ 19 | Status::Processed, 20 | Status::Rooted, 21 | Status::Confirmed, 22 | ]; 23 | 24 | #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Default)] 25 | #[repr(transparent)] 26 | pub struct Status(pub i8); 27 | #[allow(non_upper_case_globals)] 28 | impl Status { 29 | pub const Processed: Self = Self(0); 30 | pub const Rooted: Self = Self(1); 31 | pub const Confirmed: Self = Self(2); 32 | 33 | pub const ENUM_MIN: i8 = 0; 34 | pub const ENUM_MAX: i8 = 2; 35 | pub const ENUM_VALUES: &'static [Self] = &[ 36 | Self::Processed, 37 | Self::Rooted, 38 | Self::Confirmed, 39 | ]; 40 | /// Returns the variant's name or "" if unknown. 41 | pub fn variant_name(self) -> Option<&'static str> { 42 | match self { 43 | Self::Processed => Some("Processed"), 44 | Self::Rooted => Some("Rooted"), 45 | Self::Confirmed => Some("Confirmed"), 46 | _ => None, 47 | } 48 | } 49 | } 50 | impl core::fmt::Debug for Status { 51 | fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { 52 | if let Some(name) = self.variant_name() { 53 | f.write_str(name) 54 | } else { 55 | f.write_fmt(format_args!("", self.0)) 56 | } 57 | } 58 | } 59 | impl<'a> flatbuffers::Follow<'a> for Status { 60 | type Inner = Self; 61 | #[inline] 62 | unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { 63 | let b = flatbuffers::read_scalar_at::(buf, loc); 64 | Self(b) 65 | } 66 | } 67 | 68 | impl flatbuffers::Push for Status { 69 | type Output = Status; 70 | #[inline] 71 | unsafe fn push(&self, dst: &mut [u8], _written_len: usize) { 72 | flatbuffers::emplace_scalar::(dst, self.0); 73 | } 74 | } 75 | 76 | impl flatbuffers::EndianScalar for Status { 77 | type Scalar = i8; 78 | #[inline] 79 | fn to_little_endian(self) -> i8 { 80 | self.0.to_le() 81 | } 82 | #[inline] 83 | #[allow(clippy::wrong_self_convention)] 84 | fn from_little_endian(v: i8) -> Self { 85 | let b = i8::from_le(v); 86 | Self(b) 87 | } 88 | } 89 | 90 | impl<'a> flatbuffers::Verifiable for Status { 91 | #[inline] 92 | fn run_verifier( 93 | v: &mut flatbuffers::Verifier, pos: usize 94 | ) -> Result<(), flatbuffers::InvalidFlatbuffer> { 95 | use self::flatbuffers::Verifiable; 96 | i8::run_verifier(v, pos) 97 | } 98 | } 99 | 100 | impl flatbuffers::SimpleToVerifyInSlice for Status {} 101 | pub enum SlotStatusInfoOffset {} 102 | #[derive(Copy, Clone, PartialEq, Eq)] 103 | 104 | pub struct SlotStatusInfo<'a> { 105 | pub _tab: flatbuffers::Table<'a>, 106 | } 107 | 108 | impl<'a> flatbuffers::Follow<'a> for SlotStatusInfo<'a> { 109 | type Inner = SlotStatusInfo<'a>; 110 | #[inline] 111 | unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { 112 | Self { _tab: flatbuffers::Table::new(buf, loc) } 113 | } 114 | } 115 | 116 | impl<'a> SlotStatusInfo<'a> { 117 | pub const VT_SLOT: flatbuffers::VOffsetT = 4; 118 | pub const VT_PARENT: flatbuffers::VOffsetT = 6; 119 | pub const VT_STATUS: flatbuffers::VOffsetT = 8; 120 | pub const VT_SEEN_AT: flatbuffers::VOffsetT = 10; 121 | 122 | #[inline] 123 | pub unsafe fn init_from_table(table: flatbuffers::Table<'a>) -> Self { 124 | SlotStatusInfo { _tab: table } 125 | } 126 | #[allow(unused_mut)] 127 | pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( 128 | _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, 129 | args: &'args SlotStatusInfoArgs 130 | ) -> flatbuffers::WIPOffset> { 131 | let mut builder = SlotStatusInfoBuilder::new(_fbb); 132 | builder.add_seen_at(args.seen_at); 133 | if let Some(x) = args.parent { builder.add_parent(x); } 134 | builder.add_slot(args.slot); 135 | builder.add_status(args.status); 136 | builder.finish() 137 | } 138 | 139 | 140 | #[inline] 141 | pub fn slot(&self) -> u64 { 142 | // Safety: 143 | // Created from valid Table for this object 144 | // which contains a valid value in this slot 145 | unsafe { self._tab.get::(SlotStatusInfo::VT_SLOT, Some(0)).unwrap()} 146 | } 147 | #[inline] 148 | pub fn parent(&self) -> Option { 149 | // Safety: 150 | // Created from valid Table for this object 151 | // which contains a valid value in this slot 152 | unsafe { self._tab.get::(SlotStatusInfo::VT_PARENT, None)} 153 | } 154 | #[inline] 155 | pub fn status(&self) -> Status { 156 | // Safety: 157 | // Created from valid Table for this object 158 | // which contains a valid value in this slot 159 | unsafe { self._tab.get::(SlotStatusInfo::VT_STATUS, Some(Status::Processed)).unwrap()} 160 | } 161 | #[inline] 162 | pub fn seen_at(&self) -> i64 { 163 | // Safety: 164 | // Created from valid Table for this object 165 | // which contains a valid value in this slot 166 | unsafe { self._tab.get::(SlotStatusInfo::VT_SEEN_AT, Some(0)).unwrap()} 167 | } 168 | } 169 | 170 | impl flatbuffers::Verifiable for SlotStatusInfo<'_> { 171 | #[inline] 172 | fn run_verifier( 173 | v: &mut flatbuffers::Verifier, pos: usize 174 | ) -> Result<(), flatbuffers::InvalidFlatbuffer> { 175 | use self::flatbuffers::Verifiable; 176 | v.visit_table(pos)? 177 | .visit_field::("slot", Self::VT_SLOT, false)? 178 | .visit_field::("parent", Self::VT_PARENT, false)? 179 | .visit_field::("status", Self::VT_STATUS, false)? 180 | .visit_field::("seen_at", Self::VT_SEEN_AT, false)? 181 | .finish(); 182 | Ok(()) 183 | } 184 | } 185 | pub struct SlotStatusInfoArgs { 186 | pub slot: u64, 187 | pub parent: Option, 188 | pub status: Status, 189 | pub seen_at: i64, 190 | } 191 | impl<'a> Default for SlotStatusInfoArgs { 192 | #[inline] 193 | fn default() -> Self { 194 | SlotStatusInfoArgs { 195 | slot: 0, 196 | parent: None, 197 | status: Status::Processed, 198 | seen_at: 0, 199 | } 200 | } 201 | } 202 | 203 | pub struct SlotStatusInfoBuilder<'a: 'b, 'b> { 204 | fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, 205 | start_: flatbuffers::WIPOffset, 206 | } 207 | impl<'a: 'b, 'b> SlotStatusInfoBuilder<'a, 'b> { 208 | #[inline] 209 | pub fn add_slot(&mut self, slot: u64) { 210 | self.fbb_.push_slot::(SlotStatusInfo::VT_SLOT, slot, 0); 211 | } 212 | #[inline] 213 | pub fn add_parent(&mut self, parent: u64) { 214 | self.fbb_.push_slot_always::(SlotStatusInfo::VT_PARENT, parent); 215 | } 216 | #[inline] 217 | pub fn add_status(&mut self, status: Status) { 218 | self.fbb_.push_slot::(SlotStatusInfo::VT_STATUS, status, Status::Processed); 219 | } 220 | #[inline] 221 | pub fn add_seen_at(&mut self, seen_at: i64) { 222 | self.fbb_.push_slot::(SlotStatusInfo::VT_SEEN_AT, seen_at, 0); 223 | } 224 | #[inline] 225 | pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> SlotStatusInfoBuilder<'a, 'b> { 226 | let start = _fbb.start_table(); 227 | SlotStatusInfoBuilder { 228 | fbb_: _fbb, 229 | start_: start, 230 | } 231 | } 232 | #[inline] 233 | pub fn finish(self) -> flatbuffers::WIPOffset> { 234 | let o = self.fbb_.end_table(self.start_); 235 | flatbuffers::WIPOffset::new(o.value()) 236 | } 237 | } 238 | 239 | impl core::fmt::Debug for SlotStatusInfo<'_> { 240 | fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { 241 | let mut ds = f.debug_struct("SlotStatusInfo"); 242 | ds.field("slot", &self.slot()); 243 | ds.field("parent", &self.parent()); 244 | ds.field("status", &self.status()); 245 | ds.field("seen_at", &self.seen_at()); 246 | ds.finish() 247 | } 248 | } 249 | #[inline] 250 | /// Verifies that a buffer of bytes contains a `SlotStatusInfo` 251 | /// and returns it. 252 | /// Note that verification is still experimental and may not 253 | /// catch every error, or be maximally performant. For the 254 | /// previous, unchecked, behavior use 255 | /// `root_as_slot_status_info_unchecked`. 256 | pub fn root_as_slot_status_info(buf: &[u8]) -> Result { 257 | flatbuffers::root::(buf) 258 | } 259 | #[inline] 260 | /// Verifies that a buffer of bytes contains a size prefixed 261 | /// `SlotStatusInfo` and returns it. 262 | /// Note that verification is still experimental and may not 263 | /// catch every error, or be maximally performant. For the 264 | /// previous, unchecked, behavior use 265 | /// `size_prefixed_root_as_slot_status_info_unchecked`. 266 | pub fn size_prefixed_root_as_slot_status_info(buf: &[u8]) -> Result { 267 | flatbuffers::size_prefixed_root::(buf) 268 | } 269 | #[inline] 270 | /// Verifies, with the given options, that a buffer of bytes 271 | /// contains a `SlotStatusInfo` and returns it. 272 | /// Note that verification is still experimental and may not 273 | /// catch every error, or be maximally performant. For the 274 | /// previous, unchecked, behavior use 275 | /// `root_as_slot_status_info_unchecked`. 276 | pub fn root_as_slot_status_info_with_opts<'b, 'o>( 277 | opts: &'o flatbuffers::VerifierOptions, 278 | buf: &'b [u8], 279 | ) -> Result, flatbuffers::InvalidFlatbuffer> { 280 | flatbuffers::root_with_opts::>(opts, buf) 281 | } 282 | #[inline] 283 | /// Verifies, with the given verifier options, that a buffer of 284 | /// bytes contains a size prefixed `SlotStatusInfo` and returns 285 | /// it. Note that verification is still experimental and may not 286 | /// catch every error, or be maximally performant. For the 287 | /// previous, unchecked, behavior use 288 | /// `root_as_slot_status_info_unchecked`. 289 | pub fn size_prefixed_root_as_slot_status_info_with_opts<'b, 'o>( 290 | opts: &'o flatbuffers::VerifierOptions, 291 | buf: &'b [u8], 292 | ) -> Result, flatbuffers::InvalidFlatbuffer> { 293 | flatbuffers::size_prefixed_root_with_opts::>(opts, buf) 294 | } 295 | #[inline] 296 | /// Assumes, without verification, that a buffer of bytes contains a SlotStatusInfo and returns it. 297 | /// # Safety 298 | /// Callers must trust the given bytes do indeed contain a valid `SlotStatusInfo`. 299 | pub unsafe fn root_as_slot_status_info_unchecked(buf: &[u8]) -> SlotStatusInfo { 300 | flatbuffers::root_unchecked::(buf) 301 | } 302 | #[inline] 303 | /// Assumes, without verification, that a buffer of bytes contains a size prefixed SlotStatusInfo and returns it. 304 | /// # Safety 305 | /// Callers must trust the given bytes do indeed contain a valid size prefixed `SlotStatusInfo`. 306 | pub unsafe fn size_prefixed_root_as_slot_status_info_unchecked(buf: &[u8]) -> SlotStatusInfo { 307 | flatbuffers::size_prefixed_root_unchecked::(buf) 308 | } 309 | #[inline] 310 | pub fn finish_slot_status_info_buffer<'a, 'b>( 311 | fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>, 312 | root: flatbuffers::WIPOffset>) { 313 | fbb.finish(root, None); 314 | } 315 | 316 | #[inline] 317 | pub fn finish_size_prefixed_slot_status_info_buffer<'a, 'b>(fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>, root: flatbuffers::WIPOffset>) { 318 | fbb.finish_size_prefixed(root, None); 319 | } 320 | -------------------------------------------------------------------------------- /plerkle_serialization/src/solana_geyser_plugin_interface_shims.rs: -------------------------------------------------------------------------------- 1 | use solana_sdk::{clock::UnixTimestamp, signature::Signature, transaction::SanitizedTransaction}; 2 | use solana_transaction_status::TransactionStatusMeta; 3 | #[derive(Debug, Clone, PartialEq, Eq)] 4 | /// Information about an account being updated 5 | /// (extended with transaction signature doing this update) 6 | pub struct ReplicaAccountInfoV2<'a> { 7 | /// The Pubkey for the account 8 | pub pubkey: &'a [u8], 9 | 10 | /// The lamports for the account 11 | pub lamports: u64, 12 | 13 | /// The Pubkey of the owner program account 14 | pub owner: &'a [u8], 15 | 16 | /// This account's data contains a loaded program (and is now read-only) 17 | pub executable: bool, 18 | 19 | /// The epoch at which this account will next owe rent 20 | pub rent_epoch: u64, 21 | 22 | /// The data held in this account. 23 | pub data: &'a [u8], 24 | 25 | /// A global monotonically increasing atomic number, which can be used 26 | /// to tell the order of the account update. For example, when an 27 | /// account is updated in the same slot multiple times, the update 28 | /// with higher write_version should supersede the one with lower 29 | /// write_version. 30 | pub write_version: u64, 31 | 32 | /// First signature of the transaction caused this account modification 33 | pub txn_signature: Option<&'a Signature>, 34 | } 35 | 36 | #[derive(Clone, Debug)] 37 | pub struct ReplicaTransactionInfoV2<'a> { 38 | /// The first signature of the transaction, used for identifying the transaction. 39 | pub signature: &'a Signature, 40 | 41 | /// Indicates if the transaction is a simple vote transaction. 42 | pub is_vote: bool, 43 | 44 | /// The sanitized transaction. 45 | pub transaction: &'a SanitizedTransaction, 46 | 47 | /// Metadata of the transaction status. 48 | pub transaction_status_meta: &'a TransactionStatusMeta, 49 | 50 | /// The transaction's index in the block 51 | pub index: usize, 52 | } 53 | 54 | #[derive(Clone, Debug)] 55 | pub struct ReplicaBlockInfoV2<'a> { 56 | pub parent_slot: u64, 57 | pub parent_blockhash: &'a str, 58 | pub slot: u64, 59 | pub blockhash: &'a str, 60 | pub block_time: Option, 61 | pub block_height: Option, 62 | pub executed_transaction_count: u64, 63 | } 64 | 65 | #[derive(Debug, Clone, Copy, PartialEq, Eq)] 66 | pub enum SlotStatus { 67 | /// The highest slot of the heaviest fork processed by the node. Ledger state at this slot is 68 | /// not derived from a confirmed or finalized block, but if multiple forks are present, is from 69 | /// the fork the validator believes is most likely to finalize. 70 | Processed, 71 | 72 | /// The highest slot having reached max vote lockout. 73 | Rooted, 74 | 75 | /// The highest slot that has been voted on by supermajority of the cluster, ie. is confirmed. 76 | Confirmed, 77 | } 78 | 79 | impl SlotStatus { 80 | pub fn as_str(&self) -> &'static str { 81 | match self { 82 | SlotStatus::Confirmed => "confirmed", 83 | SlotStatus::Processed => "processed", 84 | SlotStatus::Rooted => "rooted", 85 | } 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /plerkle_serialization/transaction_info.fbs: -------------------------------------------------------------------------------- 1 | // Flatbuffer IDL for selected Transaction Info schema. 2 | include "./compiled_instruction.fbs"; 3 | include "./common.fbs"; 4 | 5 | //Legacy is default for unknown 6 | enum TransactionVersion:byte { 7 | Legacy, 8 | V0 9 | } 10 | 11 | table TransactionInfo { 12 | is_vote: bool; 13 | account_keys:[Pubkey]; 14 | log_messages:[string]; 15 | // To be deprecated 16 | inner_instructions: [InnerInstructions]; 17 | outer_instructions: [CompiledInstruction]; 18 | slot: uint64; 19 | slot_index: string; 20 | seen_at: int64; 21 | signature: string; 22 | compiled_inner_instructions: [CompiledInnerInstructions]; 23 | version: TransactionVersion; 24 | } 25 | 26 | table InnerInstructions { 27 | // Transaction instruction index. 28 | index:uint8; 29 | // List of inner instructions. 30 | instructions: [CompiledInstruction]; 31 | } 32 | 33 | table CompiledInnerInstructions { 34 | // Transaction instruction index. 35 | index:uint8; 36 | // List of inner instructions. 37 | instructions: [CompiledInnerInstruction]; 38 | } 39 | 40 | root_type TransactionInfo; 41 | -------------------------------------------------------------------------------- /plerkle_snapshot/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | # Renamed from original "solana-snapshot-etl" 3 | name = "plerkle_snapshot" 4 | version = "0.8.0" 5 | edition = "2021" 6 | license = "Apache-2.0" 7 | documentation = "https://docs.rs/solana-snapshot-etl" 8 | description = "Efficiently unpack Solana snapshots" 9 | authors = ["Richard Patel "] 10 | categories = ["cryptography::cryptocurrencies", "database"] 11 | keywords = ["solana"] 12 | 13 | [dependencies] 14 | clap = { version = "4.4.6", features = ["derive"] } 15 | log = "0.4.17" 16 | solana-runtime = "2.1" 17 | solana-frozen-abi-macro = "2.1" 18 | solana-accounts-db = "2.1" 19 | thiserror = "1.0.31" 20 | bincode = "1.3.3" 21 | serde = { version = "1.0.139", features = ["derive"] } 22 | solana-sdk = "2.1" 23 | memmap2 = "0.9.0" 24 | itertools = "0.11.0" 25 | tar = "0.4.38" 26 | zstd = "0.12.4" 27 | 28 | # Binary deps 29 | borsh = { version = "0.10.3", optional = true } 30 | crossbeam = { version = "0.8.2", optional = true } 31 | csv = { version = "1.1.6", optional = true } 32 | indicatif = { version = "0.17.0-rc.11", optional = true } 33 | libloading = { version = "0.8.1", optional = true } 34 | num_cpus = { version = "1.13.1", optional = true } 35 | reqwest = { version = "0.11.11", features = ["blocking"], optional = true } 36 | rusqlite = { version = "0.29.0", features = ["bundled"], optional = true } 37 | flatbuffers = { version = "23.1.21", optional = true } 38 | bs58 = { version = "0.4.0", optional = true } 39 | serde_json = { version = "1.0.82", optional = true } 40 | agave-geyser-plugin-interface = { version = "2.1", optional = true } 41 | solana-program = { version = "2.1", optional = true } 42 | solana_rbpf = { version = "0.7.2", optional = true } 43 | spl-token = { version = "4.0.0", optional = true } 44 | json5 = { version = "0.4.1", optional = true } 45 | plerkle_serialization = { path = "../plerkle_serialization", optional = true } 46 | plerkle_messenger = { path = "../plerkle_messenger", optional = true } 47 | tokio = { version = "1.23.0", features = ["full"], optional = true } 48 | async-trait = { version = "0.1.53", optional = true } 49 | figment = { version = "0.10.6", features = ["env"], optional = true } 50 | tracing = { version = "0.1.37", optional = true } 51 | tracing-subscriber = { version = "0.3.16", features = [ 52 | "json", 53 | "env-filter", 54 | "ansi", 55 | ], optional = true } 56 | dotenvy = { version = "0.15.7", optional = true } 57 | 58 | [features] 59 | parallel = [] 60 | standalone = [ 61 | "dep:borsh", 62 | "dep:crossbeam", 63 | "dep:csv", 64 | "dep:indicatif", 65 | "dep:libloading", 66 | "dep:num_cpus", 67 | "parallel", 68 | "dep:reqwest", 69 | "dep:rusqlite", 70 | "dep:serde_json", 71 | "dep:agave-geyser-plugin-interface", 72 | "dep:solana-program", 73 | "dep:spl-token", 74 | "dep:json5", 75 | "dep:flatbuffers", 76 | "dep:bs58", 77 | "dep:plerkle_serialization", 78 | "dep:tokio", 79 | "dep:async-trait", 80 | "dep:plerkle_messenger", 81 | "dep:figment", 82 | "dep:tracing", 83 | "dep:tracing-subscriber", 84 | "dep:dotenvy" 85 | ] 86 | opcode_stats = [ 87 | "dep:solana_rbpf", 88 | ] 89 | 90 | [[bin]] 91 | name = "solana-snapshot-etl" 92 | required-features = ["standalone"] 93 | -------------------------------------------------------------------------------- /plerkle_snapshot/LICENSE.md: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /plerkle_snapshot/README.md: -------------------------------------------------------------------------------- 1 | # Solana Snapshot ETL 📸 2 | 3 | [![license](https://img.shields.io/badge/license-Apache--2.0-blue?style=flat-square)](#license) 4 | 5 | **`solana-snapshot-etl` efficiently extracts all accounts in a snapshot** to load them into an external system. 6 | 7 | > [!IMPORTANT] 8 | > This code is a fork of the [original repository](https://github.com/riptl/solana-snapshot-etl.git) and has been modified to diverge from the behavior of the original implementation. 9 | 10 | ## Motivation 11 | 12 | Solana nodes periodically backup their account database into a `.tar.zst` "snapshot" stream. 13 | If you run a node yourself, you've probably seen a snapshot file such as this one already: 14 | 15 | ``` 16 | snapshot-139240745-D17vR2iksG5RoLMfTX7i5NwSsr4VpbybuX1eqzesQfu2.tar.zst 17 | ``` 18 | 19 | A full snapshot file contains a copy of all accounts at a specific slot state (in this case slot `139240745`). 20 | 21 | Historical accounts data is relevant to blockchain analytics use-cases and event tracing. 22 | Despite archives being readily available, the ecosystem was missing an easy-to-use tool to access snapshot data. 23 | 24 | ## Usage 25 | 26 | Instruction of usage we can find in [this section](../README.md#snapshot-etl). 27 | 28 | ## Changes 29 | 30 | The following changes were made to the original Solana Snapshot ETL tool: 31 | 32 | - The solana-opcode-stats binary has been removed. 33 | - The current version of the ETL only streams data to the Geyser plugin. CSV and SQLite support have been removed. 34 | - The ETL now assigns a slot number to the account data, extracting the slot number from the snapshot file name. -------------------------------------------------------------------------------- /plerkle_snapshot/src/append_vec.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Solana Foundation. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // This file contains code vendored from https://github.com/solana-labs/solana 16 | // Source: solana/runtime/src/append_vec.rs 17 | 18 | use std::{ 19 | convert::TryFrom, 20 | fs::OpenOptions, 21 | io::{self, Read}, 22 | mem, 23 | path::Path, 24 | }; 25 | 26 | use log::*; 27 | use memmap2::{Mmap, MmapMut}; 28 | use serde::{Deserialize, Serialize}; 29 | use solana_sdk::{ 30 | account::{Account, AccountSharedData, ReadableAccount}, 31 | clock::Epoch, 32 | hash::Hash, 33 | pubkey::Pubkey, 34 | }; 35 | 36 | // Data placement should be aligned at the next boundary. Without alignment accessing the memory may 37 | // crash on some architectures. 38 | pub const ALIGN_BOUNDARY_OFFSET: usize = mem::size_of::(); 39 | macro_rules! u64_align { 40 | ($addr: expr) => { 41 | ($addr + (ALIGN_BOUNDARY_OFFSET - 1)) & !(ALIGN_BOUNDARY_OFFSET - 1) 42 | }; 43 | } 44 | 45 | pub const MAXIMUM_APPEND_VEC_FILE_SIZE: u64 = 16 * 1024 * 1024 * 1024; // 16 GiB 46 | 47 | pub type StoredMetaWriteVersion = u64; 48 | 49 | /// Meta contains enough context to recover the index from storage itself 50 | /// This struct will be backed by mmaped and snapshotted data files. 51 | /// So the data layout must be stable and consistent across the entire cluster! 52 | #[repr(C)] 53 | #[derive(Clone, PartialEq, Eq, Debug)] 54 | pub struct StoredMeta { 55 | pub write_version: StoredMetaWriteVersion, 56 | pub data_len: u64, 57 | pub pubkey: Pubkey, 58 | } 59 | 60 | /// This struct will be backed by mmaped and snapshotted data files. 61 | /// So the data layout must be stable and consistent across the entire cluster! 62 | #[repr(C)] 63 | #[derive(Serialize, Deserialize, Clone, Debug, Default, Eq, PartialEq)] 64 | pub struct AccountMeta { 65 | /// lamports in the account 66 | pub lamports: u64, 67 | /// the epoch at which this account will next owe rent 68 | pub rent_epoch: Epoch, 69 | /// the program that owns this account. If executable, the program that loads this account. 70 | pub owner: Pubkey, 71 | /// this account's data contains a loaded program (and is now read-only) 72 | pub executable: bool, 73 | } 74 | 75 | impl<'a, T: ReadableAccount> From<&'a T> for AccountMeta { 76 | fn from(account: &'a T) -> Self { 77 | Self { 78 | lamports: account.lamports(), 79 | owner: *account.owner(), 80 | executable: account.executable(), 81 | rent_epoch: account.rent_epoch(), 82 | } 83 | } 84 | } 85 | 86 | impl<'a, T: ReadableAccount> From> for AccountMeta { 87 | fn from(account: Option<&'a T>) -> Self { 88 | match account { 89 | Some(account) => AccountMeta::from(account), 90 | None => AccountMeta::default(), 91 | } 92 | } 93 | } 94 | 95 | /// References to account data stored elsewhere. Getting an `Account` requires cloning 96 | /// (see `StoredAccountMeta::clone_account()`). 97 | #[derive(PartialEq, Eq, Debug)] 98 | pub struct StoredAccountMeta<'a> { 99 | pub meta: &'a StoredMeta, 100 | /// account data 101 | pub account_meta: &'a AccountMeta, 102 | pub data: &'a [u8], 103 | pub offset: usize, 104 | pub stored_size: usize, 105 | pub hash: &'a Hash, 106 | } 107 | 108 | impl<'a> StoredAccountMeta<'a> { 109 | /// Return a new Account by copying all the data referenced by the `StoredAccountMeta`. 110 | pub fn clone_account(&self) -> (StoredMeta, AccountSharedData) { 111 | ( 112 | self.meta.clone(), 113 | AccountSharedData::from(Account { 114 | lamports: self.account_meta.lamports, 115 | owner: self.account_meta.owner, 116 | executable: self.account_meta.executable, 117 | rent_epoch: self.account_meta.rent_epoch, 118 | data: self.data.to_vec(), 119 | }), 120 | ) 121 | } 122 | } 123 | 124 | /// A thread-safe, file-backed block of memory used to store `Account` instances. Append operations 125 | /// are serialized such that only one thread updates the internal `append_lock` at a time. No 126 | /// restrictions are placed on reading. That is, one may read items from one thread while another 127 | /// is appending new items. 128 | pub struct AppendVec { 129 | /// A file-backed block of memory that is used to store the data for each appended item. 130 | map: Mmap, 131 | 132 | /// The number of bytes used to store items, not the number of items. 133 | current_len: usize, 134 | 135 | /// The number of bytes available for storing items. 136 | file_size: u64, 137 | 138 | slot: u64, 139 | } 140 | 141 | impl AppendVec { 142 | fn sanitize_len_and_size(current_len: usize, file_size: usize) -> io::Result<()> { 143 | if file_size == 0 { 144 | Err(std::io::Error::new( 145 | std::io::ErrorKind::Other, 146 | format!("too small file size {} for AppendVec", file_size), 147 | )) 148 | } else if usize::try_from(MAXIMUM_APPEND_VEC_FILE_SIZE) 149 | .map(|max| file_size > max) 150 | .unwrap_or(true) 151 | { 152 | Err(std::io::Error::new( 153 | std::io::ErrorKind::Other, 154 | format!("too large file size {} for AppendVec", file_size), 155 | )) 156 | } else if current_len > file_size { 157 | Err(std::io::Error::new( 158 | std::io::ErrorKind::Other, 159 | format!("current_len is larger than file size ({})", file_size), 160 | )) 161 | } else { 162 | Ok(()) 163 | } 164 | } 165 | 166 | /// how many more bytes can be stored in this append vec 167 | pub fn remaining_bytes(&self) -> u64 { 168 | (self.capacity()).saturating_sub(self.len() as u64) 169 | } 170 | 171 | pub fn len(&self) -> usize { 172 | self.current_len 173 | } 174 | 175 | pub fn is_empty(&self) -> bool { 176 | self.len() == 0 177 | } 178 | 179 | pub fn capacity(&self) -> u64 { 180 | self.file_size 181 | } 182 | 183 | pub fn new_from_file>( 184 | path: P, 185 | current_len: usize, 186 | slot: u64, 187 | ) -> io::Result { 188 | let data = OpenOptions::new().read(true).write(false).create(false).open(&path)?; 189 | 190 | let file_size = std::fs::metadata(&path)?.len(); 191 | AppendVec::sanitize_len_and_size(current_len, file_size as usize)?; 192 | 193 | let map = unsafe { 194 | let result = Mmap::map(&data); 195 | if result.is_err() { 196 | // for vm.max_map_count, error is: {code: 12, kind: Other, message: "Cannot allocate memory"} 197 | info!("memory map error: {:?}. This may be because vm.max_map_count is not set correctly.", result); 198 | } 199 | result? 200 | }; 201 | 202 | let new = AppendVec { map, current_len, file_size, slot }; 203 | 204 | Ok(new) 205 | } 206 | 207 | pub fn new_from_reader( 208 | reader: &mut R, 209 | current_len: usize, 210 | slot: u64, 211 | ) -> io::Result { 212 | let mut map = MmapMut::map_anon(current_len)?; 213 | io::copy(&mut reader.take(current_len as u64), &mut map.as_mut())?; 214 | Ok(AppendVec { 215 | map: map.make_read_only()?, 216 | current_len, 217 | file_size: current_len as u64, 218 | slot, 219 | }) 220 | } 221 | 222 | /// Get a reference to the data at `offset` of `size` bytes if that slice 223 | /// doesn't overrun the internal buffer. Otherwise return None. 224 | /// Also return the offset of the first byte after the requested data that 225 | /// falls on a 64-byte boundary. 226 | fn get_slice(&self, offset: usize, size: usize) -> Option<(&[u8], usize)> { 227 | let (next, overflow) = offset.overflowing_add(size); 228 | if overflow || next > self.len() { 229 | return None; 230 | } 231 | let data = &self.map[offset..next]; 232 | let next = u64_align!(next); 233 | 234 | Some(( 235 | //UNSAFE: This unsafe creates a slice that represents a chunk of self.map memory 236 | //The lifetime of this slice is tied to &self, since it points to self.map memory 237 | unsafe { std::slice::from_raw_parts(data.as_ptr() as *const u8, size) }, 238 | next, 239 | )) 240 | } 241 | 242 | /// Return a reference to the type at `offset` if its data doesn't overrun the internal buffer. 243 | /// Otherwise return None. Also return the offset of the first byte after the requested data 244 | /// that falls on a 64-byte boundary. 245 | fn get_type<'a, T>(&self, offset: usize) -> Option<(&'a T, usize)> { 246 | let (data, next) = self.get_slice(offset, mem::size_of::())?; 247 | let ptr: *const T = data.as_ptr() as *const T; 248 | //UNSAFE: The cast is safe because the slice is aligned and fits into the memory 249 | //and the lifetime of the &T is tied to self, which holds the underlying memory map 250 | Some((unsafe { &*ptr }, next)) 251 | } 252 | 253 | /// Return account metadata for the account at `offset` if its data doesn't overrun 254 | /// the internal buffer. Otherwise return None. Also return the offset of the first byte 255 | /// after the requested data that falls on a 64-byte boundary. 256 | pub fn get_account<'a>(&'a self, offset: usize) -> Option<(StoredAccountMeta<'a>, usize)> { 257 | let (meta, next): (&'a StoredMeta, _) = self.get_type(offset)?; 258 | let (account_meta, next): (&'a AccountMeta, _) = self.get_type(next)?; 259 | let (hash, next): (&'a Hash, _) = self.get_type(next)?; 260 | let (data, next) = self.get_slice(next, meta.data_len as usize)?; 261 | let stored_size = next - offset; 262 | Some((StoredAccountMeta { meta, account_meta, data, offset, stored_size, hash }, next)) 263 | } 264 | 265 | pub fn get_slot(&self) -> u64 { 266 | self.slot 267 | } 268 | } 269 | -------------------------------------------------------------------------------- /plerkle_snapshot/src/archived.rs: -------------------------------------------------------------------------------- 1 | use std::{ 2 | fs::File, 3 | io::{BufReader, Read}, 4 | path::{Component, Path}, 5 | pin::Pin, 6 | time::Instant, 7 | }; 8 | 9 | use log::info; 10 | use tar::{Archive, Entries, Entry}; 11 | 12 | use crate::{ 13 | deserialize_from, parse_append_vec_name, AccountsDbFields, AppendVec, AppendVecIterator, 14 | DeserializableVersionedBank, Result, SerializableAccountStorageEntry, SnapshotError, 15 | SnapshotExtractor, 16 | }; 17 | 18 | /// Extracts account data from a .tar.zst stream. 19 | pub struct ArchiveSnapshotExtractor 20 | where 21 | Source: Read + Unpin + 'static, 22 | { 23 | accounts_db_fields: AccountsDbFields, 24 | _archive: Pin>>>>, 25 | entries: Option>>>, 26 | } 27 | 28 | impl SnapshotExtractor for ArchiveSnapshotExtractor 29 | where 30 | Source: Read + Unpin + 'static, 31 | { 32 | fn iter(&mut self) -> AppendVecIterator<'_> { 33 | Box::new(self.unboxed_iter()) 34 | } 35 | } 36 | 37 | impl ArchiveSnapshotExtractor 38 | where 39 | Source: Read + Unpin + 'static, 40 | { 41 | pub fn from_reader(source: Source) -> Result { 42 | let tar_stream = zstd::stream::read::Decoder::new(source)?; 43 | let mut archive = Box::pin(Archive::new(tar_stream)); 44 | 45 | // This is safe as long as we guarantee that entries never gets accessed past drop. 46 | let archive_static = unsafe { &mut *((&mut *archive) as *mut Archive<_>) }; 47 | let mut entries = archive_static.entries()?; 48 | 49 | // Search for snapshot manifest. 50 | let mut snapshot_file: Option> = None; 51 | for entry in entries.by_ref() { 52 | let entry = entry?; 53 | let path = entry.path()?; 54 | if Self::is_snapshot_manifest_file(&path) { 55 | snapshot_file = Some(entry); 56 | break; 57 | } else if Self::is_appendvec_file(&path) { 58 | // TODO Support archives where AppendVecs precede snapshot manifests 59 | return Err(SnapshotError::UnexpectedAppendVec); 60 | } 61 | } 62 | let snapshot_file = snapshot_file.ok_or(SnapshotError::NoSnapshotManifest)?; 63 | //let snapshot_file_len = snapshot_file.size(); 64 | let snapshot_file_path = snapshot_file.path()?.as_ref().to_path_buf(); 65 | 66 | info!("Opening snapshot manifest: {:?}", &snapshot_file_path); 67 | let mut snapshot_file = BufReader::new(snapshot_file); 68 | 69 | let pre_unpack = Instant::now(); 70 | let versioned_bank: DeserializableVersionedBank = deserialize_from(&mut snapshot_file)?; 71 | drop(versioned_bank); 72 | let versioned_bank_post_time = Instant::now(); 73 | 74 | let accounts_db_fields: AccountsDbFields = 75 | deserialize_from(&mut snapshot_file)?; 76 | let accounts_db_fields_post_time = Instant::now(); 77 | drop(snapshot_file); 78 | 79 | info!("Read bank fields in {:?}", versioned_bank_post_time - pre_unpack); 80 | info!( 81 | "Read accounts DB fields in {:?}", 82 | accounts_db_fields_post_time - versioned_bank_post_time 83 | ); 84 | 85 | Ok(ArchiveSnapshotExtractor { 86 | _archive: archive, 87 | accounts_db_fields, 88 | entries: Some(entries), 89 | }) 90 | } 91 | 92 | fn unboxed_iter(&mut self) -> impl Iterator> + '_ { 93 | self.entries.take().into_iter().flatten().filter_map(|entry| { 94 | let mut entry = match entry { 95 | Ok(x) => x, 96 | Err(e) => return Some(Err(e.into())), 97 | }; 98 | let path = match entry.path() { 99 | Ok(x) => x, 100 | Err(e) => return Some(Err(e.into())), 101 | }; 102 | let (slot, id) = path.file_name().and_then(parse_append_vec_name)?; 103 | Some(self.process_entry(&mut entry, slot, id)) 104 | }) 105 | } 106 | 107 | fn process_entry( 108 | &self, 109 | entry: &mut Entry<'static, zstd::Decoder<'static, BufReader>>, 110 | slot: u64, 111 | id: u64, 112 | ) -> Result { 113 | let known_vecs = self.accounts_db_fields.0.get(&slot).map(|v| &v[..]).unwrap_or(&[]); 114 | let known_vec = known_vecs.iter().find(|entry| entry.id == (id as usize)); 115 | let known_vec = match known_vec { 116 | None => return Err(SnapshotError::UnexpectedAppendVec), 117 | Some(v) => v, 118 | }; 119 | Ok(AppendVec::new_from_reader(entry, known_vec.accounts_current_len, slot)?) 120 | } 121 | 122 | fn is_snapshot_manifest_file(path: &Path) -> bool { 123 | let mut components = path.components(); 124 | if components.next() != Some(Component::Normal("snapshots".as_ref())) { 125 | return false; 126 | } 127 | let slot_number_str_1 = match components.next() { 128 | Some(Component::Normal(slot)) => slot, 129 | _ => return false, 130 | }; 131 | // Check if slot number file is valid u64. 132 | if slot_number_str_1.to_str().and_then(|s| s.parse::().ok()).is_none() { 133 | return false; 134 | } 135 | let slot_number_str_2 = match components.next() { 136 | Some(Component::Normal(slot)) => slot, 137 | _ => return false, 138 | }; 139 | components.next().is_none() && slot_number_str_1 == slot_number_str_2 140 | } 141 | 142 | fn is_appendvec_file(path: &Path) -> bool { 143 | let mut components = path.components(); 144 | if components.next() != Some(Component::Normal("accounts".as_ref())) { 145 | return false; 146 | } 147 | let name = match components.next() { 148 | Some(Component::Normal(c)) => c, 149 | _ => return false, 150 | }; 151 | components.next().is_none() && parse_append_vec_name(name).is_some() 152 | } 153 | } 154 | 155 | impl ArchiveSnapshotExtractor { 156 | pub fn open(path: &Path) -> Result { 157 | Self::from_reader(File::open(path)?) 158 | } 159 | } 160 | -------------------------------------------------------------------------------- /plerkle_snapshot/src/bin/solana-snapshot-etl/accounts_selector.rs: -------------------------------------------------------------------------------- 1 | //! Copied from plerkle/src/accounts_selector.rs with some API-changing improvements. 2 | 3 | use std::collections::HashSet; 4 | 5 | use tracing::*; 6 | 7 | const fn select_all_accounts_by_default() -> bool { 8 | true 9 | } 10 | 11 | #[derive(Debug, serde::Deserialize)] 12 | pub(crate) struct AccountsSelectorConfig { 13 | #[serde(default)] 14 | pub accounts: Vec, 15 | #[serde(default)] 16 | pub owners: Vec, 17 | #[serde(default = "select_all_accounts_by_default")] 18 | pub select_all_accounts: bool, 19 | } 20 | 21 | impl Default for AccountsSelectorConfig { 22 | fn default() -> Self { 23 | Self { 24 | accounts: vec![], 25 | owners: vec![], 26 | select_all_accounts: select_all_accounts_by_default(), 27 | } 28 | } 29 | } 30 | 31 | #[derive(Clone)] 32 | pub(crate) struct AccountsSelector { 33 | pub accounts: HashSet>, 34 | pub owners: HashSet>, 35 | pub select_all_accounts: bool, 36 | } 37 | 38 | impl AccountsSelector { 39 | pub fn new(config: AccountsSelectorConfig) -> Self { 40 | let AccountsSelectorConfig { accounts, owners, select_all_accounts } = config; 41 | info!("Creating AccountsSelector from accounts: {:?}, owners: {:?}", accounts, owners); 42 | 43 | let accounts = accounts.iter().map(|key| bs58::decode(key).into_vec().unwrap()).collect(); 44 | let owners = owners.iter().map(|key| bs58::decode(key).into_vec().unwrap()).collect(); 45 | AccountsSelector { accounts, owners, select_all_accounts } 46 | } 47 | 48 | pub fn is_account_selected(&self, account: &[u8], owner: &[u8]) -> bool { 49 | self.select_all_accounts || self.accounts.contains(account) || self.owners.contains(owner) 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /plerkle_snapshot/src/bin/solana-snapshot-etl/geyser.rs: -------------------------------------------------------------------------------- 1 | // TODO add multi-threading 2 | 3 | use std::{error::Error, sync::Arc}; 4 | 5 | use agave_geyser_plugin_interface::geyser_plugin_interface::ReplicaAccountInfo; 6 | use figment::{providers::Env, Figment}; 7 | use indicatif::{ProgressBar, ProgressStyle}; 8 | use plerkle_messenger::{ 9 | redis_messenger::RedisMessenger, MessageStreamer, MessengerConfig, ACCOUNT_BACKFILL_STREAM, 10 | }; 11 | use plerkle_serialization::serializer::serialize_account; 12 | use plerkle_snapshot::append_vec::StoredMeta; 13 | use serde::Deserialize; 14 | use solana_sdk::account::{Account, AccountSharedData, ReadableAccount}; 15 | use tokio::sync::Mutex; 16 | 17 | use crate::accounts_selector::AccountsSelector; 18 | 19 | // the upper limit of accounts stream length for when the snapshot is in progress 20 | const MAX_INTERMEDIATE_STREAM_LEN: u64 = 50_000_000; 21 | // every PROCESSED_CHECKPOINT we check the stream length and reset the local stream_counter 22 | const PROCESSED_CHECKPOINT: u64 = 20_000_000; 23 | 24 | #[derive(Clone)] 25 | pub(crate) struct GeyserDumper { 26 | messenger: Arc>, 27 | throttle_nanos: u64, 28 | accounts_selector: AccountsSelector, 29 | pub accounts_spinner: ProgressBar, 30 | /// how many accounts were processed in total during the snapshot run. 31 | pub accounts_count: u64, 32 | /// intermediate counter of accounts sent to regulate XLEN checks. 33 | /// the reason for a separate field is that we initialize it as the current 34 | /// stream length, which might be non-zero. 35 | pub stream_counter: u64, 36 | } 37 | 38 | impl GeyserDumper { 39 | pub(crate) async fn new(throttle_nanos: u64, accounts_selector: AccountsSelector) -> Self { 40 | // TODO dedup spinner definitions 41 | let spinner_style = ProgressStyle::with_template( 42 | "{prefix:>10.bold.dim} {spinner} rate={per_sec} total={human_pos}", 43 | ) 44 | .unwrap(); 45 | let accounts_spinner = 46 | ProgressBar::new_spinner().with_style(spinner_style).with_prefix("accs"); 47 | 48 | #[derive(Deserialize)] 49 | struct MessengerConfigWrapper { 50 | pub messenger_config: MessengerConfig, 51 | } 52 | let wrapper: MessengerConfigWrapper = Figment::from(Env::prefixed("PLUGIN_")) 53 | .extract() 54 | .expect("PLUGIN_MESSENGER_CONFIG env variable must be defined to run ETL!"); 55 | let mut messenger = 56 | RedisMessenger::new(wrapper.messenger_config).await.expect("create redis messenger"); 57 | messenger.add_stream(ACCOUNT_BACKFILL_STREAM).await.expect("configure accounts stream"); 58 | messenger.set_buffer_size(ACCOUNT_BACKFILL_STREAM, 100_000_000).await; 59 | let initial_stream_len = messenger 60 | .stream_len(&ACCOUNT_BACKFILL_STREAM) 61 | .await 62 | .expect("get initial stream len of accounts"); 63 | 64 | Self { 65 | messenger: Arc::new(Mutex::new(messenger)), 66 | accounts_spinner, 67 | accounts_selector, 68 | accounts_count: 0, 69 | throttle_nanos, 70 | stream_counter: initial_stream_len, 71 | } 72 | } 73 | 74 | pub async fn dump_account( 75 | &mut self, 76 | (meta, account): (StoredMeta, AccountSharedData), 77 | slot: u64, 78 | ) -> Result<(), Box> { 79 | if self.stream_counter >= PROCESSED_CHECKPOINT { 80 | loop { 81 | let stream_len = 82 | self.messenger.lock().await.stream_len(ACCOUNT_BACKFILL_STREAM).await?; 83 | if stream_len < MAX_INTERMEDIATE_STREAM_LEN { 84 | self.stream_counter = 0; 85 | break; 86 | } 87 | tokio::time::sleep(std::time::Duration::from_secs(1)).await; 88 | } 89 | } 90 | if self 91 | .accounts_selector 92 | .is_account_selected(meta.pubkey.as_ref(), account.owner().as_ref()) 93 | { 94 | let account: Account = account.into(); 95 | // Serialize data. 96 | let ai = &ReplicaAccountInfo { 97 | pubkey: meta.pubkey.as_ref(), 98 | lamports: account.lamports, 99 | owner: account.owner.as_ref(), 100 | executable: account.executable, 101 | rent_epoch: account.rent_epoch, 102 | data: &account.data, 103 | write_version: meta.write_version, 104 | }; 105 | let account = 106 | plerkle_serialization::solana_geyser_plugin_interface_shims::ReplicaAccountInfoV2 { 107 | pubkey: ai.pubkey, 108 | lamports: ai.lamports, 109 | owner: ai.owner, 110 | executable: ai.executable, 111 | rent_epoch: ai.rent_epoch, 112 | data: ai.data, 113 | write_version: ai.write_version, 114 | txn_signature: None, 115 | }; 116 | let builder = flatbuffers::FlatBufferBuilder::new(); 117 | let builder = serialize_account(builder, &account, slot, false); 118 | let data = builder.finished_data(); 119 | 120 | self.messenger.lock().await.send(ACCOUNT_BACKFILL_STREAM, data).await?; 121 | self.stream_counter += 1; 122 | } else { 123 | tracing::trace!(?account, ?meta, "Account filtered out by accounts selector"); 124 | return Ok(()); 125 | } 126 | 127 | self.accounts_count += 1; 128 | self.accounts_spinner.set_position(self.accounts_count); 129 | 130 | if self.throttle_nanos > 0 { 131 | tokio::time::sleep(std::time::Duration::from_nanos(self.throttle_nanos)).await; 132 | } 133 | 134 | Ok(()) 135 | } 136 | 137 | pub async fn force_flush(self) { 138 | self.accounts_spinner.set_position(self.accounts_count); 139 | self.accounts_spinner.finish_with_message("Finished processing snapshot!"); 140 | let messenger_mutex = Arc::into_inner(self.messenger) 141 | .expect("reference count to messenger to be 0 when forcing flush at the end"); 142 | 143 | messenger_mutex.into_inner().force_flush().await.expect("force flush to succeed"); 144 | } 145 | } 146 | -------------------------------------------------------------------------------- /plerkle_snapshot/src/bin/solana-snapshot-etl/main.rs: -------------------------------------------------------------------------------- 1 | mod accounts_selector; 2 | mod geyser; 3 | mod mpl_metadata; 4 | 5 | use std::{ 6 | fs::File, 7 | io::{IoSliceMut, Read}, 8 | path::{Path, PathBuf}, 9 | }; 10 | 11 | use clap::Parser; 12 | use indicatif::{ProgressBar, ProgressBarIter, ProgressStyle}; 13 | use plerkle_snapshot::{ 14 | append_vec_iter, archived::ArchiveSnapshotExtractor, unpacked::UnpackedSnapshotExtractor, 15 | AppendVecIterator, ReadProgressTracking, SnapshotExtractor, 16 | }; 17 | use reqwest::blocking::Response; 18 | use tracing::{info, warn}; 19 | 20 | use self::accounts_selector::{AccountsSelector, AccountsSelectorConfig}; 21 | use crate::geyser::GeyserDumper; 22 | 23 | #[derive(Parser, Debug)] 24 | #[clap(author, version, about, long_about = None)] 25 | struct Args { 26 | #[clap(help = "Snapshot source (unpacked snapshot, archive file, or HTTP link)")] 27 | source: String, 28 | #[clap( 29 | long, 30 | default_value_t = 0, 31 | help = "Throttle nanoseconds between the dumping of individual accounts" 32 | )] 33 | throttle_nanos: u64, 34 | #[clap(long, help = "Path to accounts selector config (optional)")] 35 | accounts_selector_config: Option, 36 | } 37 | 38 | #[tokio::main(flavor = "multi_thread")] 39 | async fn main() -> Result<(), Box> { 40 | let env_filter = std::env::var("RUST_LOG").unwrap_or_else(|_| "info".to_string()); 41 | tracing_subscriber::fmt() 42 | .with_env_filter(env_filter) 43 | .event_format(tracing_subscriber::fmt::format::json()) 44 | .init(); 45 | if let Err(e) = dotenvy::dotenv() { 46 | warn!(error = %e, "Error initializing .env, make sure all required env is available..."); 47 | } 48 | let stop_future = tokio::task::spawn(async move { 49 | match tokio::signal::ctrl_c().await { 50 | Ok(_) => info!("Stop signal received, shutting down..."), 51 | Err(e) => tracing::error!(error = %e, "Error shutting down: {e}"), 52 | } 53 | }); 54 | 55 | let args = Args::parse(); 56 | let accounts_selector_config = args 57 | .accounts_selector_config 58 | .and_then(|path| { 59 | std::fs::read(path).ok().map(|slice| { 60 | serde_json::from_slice::(&slice) 61 | .expect("could not decode accounts selector config!") 62 | }) 63 | }) 64 | .unwrap_or_default(); 65 | let accounts_selector = AccountsSelector::new(accounts_selector_config); 66 | 67 | let mut loader = SupportedLoader::new(&args.source, Box::new(LoadProgressTracking {}))?; 68 | 69 | let process_everything_future = async move { 70 | let mut dumper = GeyserDumper::new(args.throttle_nanos, accounts_selector).await; 71 | for append_vec in loader.iter() { 72 | let append_vec = append_vec.unwrap(); 73 | let slot = append_vec.get_slot(); 74 | 75 | for account in append_vec_iter(append_vec) { 76 | dumper.dump_account(account, slot).await.expect("failed to dump account"); 77 | } 78 | } 79 | info!("Done! Accounts: {}", dumper.accounts_count); 80 | dumper.force_flush().await; 81 | }; 82 | 83 | tokio::select! { 84 | _ = stop_future => {}, 85 | _ = process_everything_future => {}, 86 | } 87 | 88 | Ok(()) 89 | } 90 | 91 | struct LoadProgressTracking {} 92 | 93 | impl ReadProgressTracking for LoadProgressTracking { 94 | fn new_read_progress_tracker( 95 | &self, 96 | _: &Path, 97 | rd: Box, 98 | file_len: u64, 99 | ) -> Box { 100 | let progress_bar = ProgressBar::new(file_len).with_style( 101 | ProgressStyle::with_template( 102 | "{prefix:>10.bold.dim} {spinner:.green} [{bar:.cyan/blue}] {bytes}/{total_bytes} ({percent}%)", 103 | ) 104 | .unwrap() 105 | .progress_chars("#>-"), 106 | ); 107 | progress_bar.set_prefix("manifest"); 108 | Box::new(LoadProgressTracker { rd: progress_bar.wrap_read(rd), progress_bar }) 109 | } 110 | } 111 | 112 | struct LoadProgressTracker { 113 | progress_bar: ProgressBar, 114 | rd: ProgressBarIter>, 115 | } 116 | 117 | impl Drop for LoadProgressTracker { 118 | fn drop(&mut self) { 119 | self.progress_bar.finish() 120 | } 121 | } 122 | 123 | impl Read for LoadProgressTracker { 124 | fn read(&mut self, buf: &mut [u8]) -> std::io::Result { 125 | self.rd.read(buf) 126 | } 127 | 128 | fn read_vectored(&mut self, bufs: &mut [IoSliceMut<'_>]) -> std::io::Result { 129 | self.rd.read_vectored(bufs) 130 | } 131 | 132 | fn read_to_string(&mut self, buf: &mut String) -> std::io::Result { 133 | self.rd.read_to_string(buf) 134 | } 135 | 136 | fn read_exact(&mut self, buf: &mut [u8]) -> std::io::Result<()> { 137 | self.rd.read_exact(buf) 138 | } 139 | } 140 | 141 | pub enum SupportedLoader { 142 | Unpacked(UnpackedSnapshotExtractor), 143 | ArchiveFile(ArchiveSnapshotExtractor), 144 | ArchiveDownload(ArchiveSnapshotExtractor), 145 | } 146 | 147 | impl SupportedLoader { 148 | fn new( 149 | source: &str, 150 | progress_tracking: Box, 151 | ) -> Result> { 152 | if source.starts_with("http://") || source.starts_with("https://") { 153 | Self::new_download(source) 154 | } else { 155 | Self::new_file(source.as_ref(), progress_tracking).map_err(Into::into) 156 | } 157 | } 158 | 159 | fn new_download(url: &str) -> Result> { 160 | let resp = reqwest::blocking::get(url)?; 161 | let loader = ArchiveSnapshotExtractor::from_reader(resp)?; 162 | info!("Streaming snapshot from HTTP"); 163 | Ok(Self::ArchiveDownload(loader)) 164 | } 165 | 166 | fn new_file( 167 | path: &Path, 168 | progress_tracking: Box, 169 | ) -> plerkle_snapshot::Result { 170 | Ok(if path.is_dir() { 171 | info!("Reading unpacked snapshot"); 172 | Self::Unpacked(UnpackedSnapshotExtractor::open(path, progress_tracking)?) 173 | } else { 174 | info!("Reading snapshot archive"); 175 | Self::ArchiveFile(ArchiveSnapshotExtractor::open(path)?) 176 | }) 177 | } 178 | } 179 | 180 | impl SnapshotExtractor for SupportedLoader { 181 | fn iter(&mut self) -> AppendVecIterator<'_> { 182 | match self { 183 | SupportedLoader::Unpacked(loader) => Box::new(loader.iter()), 184 | SupportedLoader::ArchiveFile(loader) => Box::new(loader.iter()), 185 | SupportedLoader::ArchiveDownload(loader) => Box::new(loader.iter()), 186 | } 187 | } 188 | } 189 | -------------------------------------------------------------------------------- /plerkle_snapshot/src/bin/solana-snapshot-etl/mpl_metadata.rs: -------------------------------------------------------------------------------- 1 | use borsh::BorshDeserialize; 2 | use solana_program::pubkey::Pubkey; 3 | 4 | solana_program::declare_id!("metaqbxxUerdq28cj1RbAWkYQm3ybzjb6a8bt518x1s"); 5 | 6 | #[derive(BorshDeserialize)] 7 | pub enum AccountKey { 8 | Uninitialized, 9 | EditionV1, 10 | MasterEditionV1, 11 | ReservationListV1, 12 | MetadataV1, 13 | ReservationListV2, 14 | MasterEditionV2, 15 | EditionMarker, 16 | UseAuthorityRecord, 17 | CollectionAuthorityRecord, 18 | } 19 | 20 | #[derive(BorshDeserialize)] 21 | pub struct Metadata { 22 | pub update_authority: Pubkey, 23 | pub mint: Pubkey, 24 | pub data: Data, 25 | pub primary_sale_happened: bool, 26 | pub is_mutable: bool, 27 | } 28 | 29 | #[derive(BorshDeserialize)] 30 | pub struct MetadataExt { 31 | pub edition_nonce: Option, 32 | } 33 | 34 | #[derive(BorshDeserialize)] 35 | pub struct MetadataExtV1_2 { 36 | pub token_standard: Option, 37 | pub collection: Option, 38 | pub uses: Option, 39 | } 40 | 41 | #[derive(BorshDeserialize)] 42 | pub struct Data { 43 | pub name: String, 44 | pub symbol: String, 45 | pub uri: String, 46 | pub seller_fee_basis_points: u16, 47 | pub creators: Option>, 48 | } 49 | 50 | #[derive(BorshDeserialize)] 51 | pub struct DataV2 { 52 | pub name: String, 53 | pub symbol: String, 54 | pub uri: String, 55 | pub seller_fee_basis_points: u16, 56 | pub creators: Option>, 57 | pub collection: Option, 58 | pub uses: Option, 59 | } 60 | 61 | #[derive(BorshDeserialize)] 62 | pub struct Creator { 63 | pub address: Pubkey, 64 | pub verified: bool, 65 | pub share: u8, 66 | } 67 | 68 | #[derive(BorshDeserialize)] 69 | pub struct Collection { 70 | pub verified: bool, 71 | pub key: Pubkey, 72 | } 73 | 74 | #[derive(BorshDeserialize)] 75 | pub struct Uses { 76 | pub use_method: u8, 77 | pub remaining: u64, 78 | pub total: u64, 79 | } 80 | 81 | #[derive(BorshDeserialize)] 82 | pub enum CollectionDetails { 83 | V1 { size: u64 }, 84 | } 85 | -------------------------------------------------------------------------------- /plerkle_snapshot/src/lib.rs: -------------------------------------------------------------------------------- 1 | use std::{cell::RefCell, ffi::OsStr, io::Read, path::Path, str::FromStr, sync::Arc}; 2 | 3 | use solana_sdk::account::AccountSharedData; 4 | use thiserror::Error; 5 | 6 | pub mod append_vec; 7 | pub mod solana; 8 | 9 | pub mod archived; 10 | pub mod unpacked; 11 | 12 | #[cfg(feature = "parallel")] 13 | pub mod parallel; 14 | 15 | use self::append_vec::StoredMeta; 16 | use crate::{ 17 | append_vec::{AppendVec, StoredAccountMeta}, 18 | solana::{ 19 | deserialize_from, AccountsDbFields, DeserializableVersionedBank, 20 | SerializableAccountStorageEntry, 21 | }, 22 | }; 23 | 24 | const SNAPSHOTS_DIR: &str = "snapshots"; 25 | 26 | #[derive(Error, Debug)] 27 | pub enum SnapshotError { 28 | #[error("{0}")] 29 | IOError(#[from] std::io::Error), 30 | #[error("Failed to deserialize: {0}")] 31 | BincodeError(#[from] bincode::Error), 32 | #[error("Missing status cache")] 33 | NoStatusCache, 34 | #[error("No snapshot manifest file found")] 35 | NoSnapshotManifest, 36 | #[error("Unexpected AppendVec")] 37 | UnexpectedAppendVec, 38 | } 39 | 40 | pub type Result = std::result::Result; 41 | 42 | pub type AppendVecIterator<'a> = Box> + 'a>; 43 | 44 | pub trait SnapshotExtractor: Sized { 45 | fn iter(&mut self) -> AppendVecIterator<'_>; 46 | } 47 | 48 | fn parse_append_vec_name(name: &OsStr) -> Option<(u64, u64)> { 49 | let name = name.to_str()?; 50 | let mut parts = name.splitn(2, '.'); 51 | let slot = u64::from_str(parts.next().unwrap_or("")); 52 | let id = u64::from_str(parts.next().unwrap_or("")); 53 | match (slot, id) { 54 | (Ok(slot), Ok(version)) => Some((slot, version)), 55 | _ => None, 56 | } 57 | } 58 | 59 | pub fn append_vec_iter( 60 | append_vec: AppendVec, 61 | ) -> impl Iterator { 62 | let mut offsets = Vec::::new(); 63 | let mut metas = Vec::new(); 64 | let mut offset = 0usize; 65 | loop { 66 | match append_vec.get_account(offset) { 67 | None => break, 68 | Some((meta, next_offset)) => { 69 | offsets.push(offset); 70 | metas.push(meta.clone_account()); 71 | offset = next_offset; 72 | }, 73 | } 74 | } 75 | 76 | metas.into_iter() 77 | } 78 | 79 | pub struct StoredAccountMetaHandle { 80 | append_vec: Arc, 81 | offset: usize, 82 | } 83 | 84 | impl StoredAccountMetaHandle { 85 | pub fn new(append_vec: Arc, offset: usize) -> StoredAccountMetaHandle { 86 | Self { append_vec, offset } 87 | } 88 | 89 | pub fn access(&self) -> Option> { 90 | Some(self.append_vec.get_account(self.offset)?.0) 91 | } 92 | } 93 | 94 | pub trait ReadProgressTracking { 95 | fn new_read_progress_tracker( 96 | &self, 97 | path: &Path, 98 | rd: Box, 99 | file_len: u64, 100 | ) -> Box; 101 | } 102 | 103 | struct NullReadProgressTracking {} 104 | 105 | impl ReadProgressTracking for NullReadProgressTracking { 106 | fn new_read_progress_tracker(&self, _: &Path, rd: Box, _: u64) -> Box { 107 | rd 108 | } 109 | } 110 | 111 | struct RefCellRead { 112 | rd: RefCell, 113 | } 114 | 115 | impl Read for RefCellRead { 116 | fn read(&mut self, buf: &mut [u8]) -> std::io::Result { 117 | self.rd 118 | .try_borrow_mut() 119 | .map_err(|_| { 120 | std::io::Error::new( 121 | std::io::ErrorKind::Other, 122 | "attempted to read archive concurrently", 123 | ) 124 | }) 125 | .and_then(|mut rd| rd.read(buf)) 126 | } 127 | } 128 | -------------------------------------------------------------------------------- /plerkle_snapshot/src/parallel.rs: -------------------------------------------------------------------------------- 1 | use crossbeam::sync::WaitGroup; 2 | 3 | use crate::{AppendVec, AppendVecIterator}; 4 | 5 | pub type GenericResult = std::result::Result>; 6 | 7 | pub trait AppendVecConsumerFactory { 8 | type Consumer: AppendVecConsumer + Send + 'static; 9 | fn new_consumer(&mut self) -> GenericResult; 10 | } 11 | 12 | pub trait AppendVecConsumer { 13 | fn on_append_vec(&mut self, append_vec: AppendVec) -> GenericResult<()>; 14 | } 15 | 16 | pub fn par_iter_append_vecs( 17 | iterator: AppendVecIterator<'_>, 18 | consumers: &mut A, 19 | num_threads: usize, 20 | ) -> GenericResult<()> 21 | where 22 | A: AppendVecConsumerFactory, 23 | { 24 | let (tx, rx) = crossbeam::channel::bounded::(num_threads); 25 | 26 | let wg = WaitGroup::new(); 27 | let mut consumer_vec = Vec::with_capacity(num_threads); 28 | for _ in 0..num_threads { 29 | consumer_vec.push(consumers.new_consumer()?); 30 | } 31 | 32 | for mut consumer in consumer_vec { 33 | let rx = rx.clone(); 34 | let wg = wg.clone(); 35 | std::thread::spawn(move || { 36 | while let Ok(item) = rx.recv() { 37 | consumer.on_append_vec(item).expect("insert failed") 38 | } 39 | drop(wg); 40 | }); 41 | } 42 | 43 | for append_vec in iterator { 44 | let append_vec = append_vec?; 45 | tx.send(append_vec).expect("failed to send AppendVec"); 46 | } 47 | drop(tx); 48 | wg.wait(); 49 | Ok(()) 50 | } 51 | -------------------------------------------------------------------------------- /plerkle_snapshot/src/solana.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Solana Foundation. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // This file contains code vendored from https://github.com/solana-labs/solana 16 | 17 | use std::{ 18 | collections::{HashMap, HashSet}, 19 | io::Read, 20 | }; 21 | 22 | use bincode::Options; 23 | use serde::{de::DeserializeOwned, Deserialize, Serialize}; 24 | use solana_accounts_db::{ 25 | account_storage::meta::StoredMetaWriteVersion, accounts_db::stats::BankHashStats, 26 | ancestors::AncestorsForSerialization, blockhash_queue::BlockhashQueue, 27 | }; 28 | use solana_frozen_abi_macro::AbiExample; 29 | use solana_runtime::{epoch_stakes::EpochStakes, stakes::Stakes}; 30 | use solana_sdk::{ 31 | clock::{Epoch, UnixTimestamp}, 32 | deserialize_utils::default_on_eof, 33 | epoch_schedule::EpochSchedule, 34 | fee_calculator::{FeeCalculator, FeeRateGovernor}, 35 | hard_forks::HardForks, 36 | hash::Hash, 37 | inflation::Inflation, 38 | pubkey::Pubkey, 39 | rent_collector::RentCollector, 40 | slot_history::Slot, 41 | stake::state::Delegation, 42 | }; 43 | 44 | const MAX_STREAM_SIZE: u64 = 32 * 1024 * 1024 * 1024; 45 | 46 | pub fn deserialize_from(reader: R) -> bincode::Result 47 | where 48 | R: Read, 49 | T: DeserializeOwned, 50 | { 51 | bincode::options() 52 | .with_limit(MAX_STREAM_SIZE) 53 | .with_fixint_encoding() 54 | .allow_trailing_bytes() 55 | .deserialize_from::(reader) 56 | } 57 | 58 | #[derive(Default, PartialEq, Eq, Debug, Deserialize)] 59 | struct UnusedAccounts { 60 | unused1: HashSet, 61 | unused2: HashSet, 62 | unused3: HashMap, 63 | } 64 | 65 | #[derive(Deserialize)] 66 | #[allow(dead_code)] 67 | pub struct DeserializableVersionedBank { 68 | pub blockhash_queue: BlockhashQueue, 69 | pub ancestors: AncestorsForSerialization, 70 | pub hash: Hash, 71 | pub parent_hash: Hash, 72 | pub parent_slot: Slot, 73 | pub hard_forks: HardForks, 74 | pub transaction_count: u64, 75 | pub tick_height: u64, 76 | pub signature_count: u64, 77 | pub capitalization: u64, 78 | pub max_tick_height: u64, 79 | pub hashes_per_tick: Option, 80 | pub ticks_per_slot: u64, 81 | pub ns_per_slot: u128, 82 | pub genesis_creation_time: UnixTimestamp, 83 | pub slots_per_year: f64, 84 | pub accounts_data_len: u64, 85 | pub slot: Slot, 86 | pub epoch: Epoch, 87 | pub block_height: u64, 88 | pub collector_id: Pubkey, 89 | pub collector_fees: u64, 90 | pub fee_calculator: FeeCalculator, 91 | pub fee_rate_governor: FeeRateGovernor, 92 | pub collected_rent: u64, 93 | pub rent_collector: RentCollector, 94 | pub epoch_schedule: EpochSchedule, 95 | pub inflation: Inflation, 96 | pub stakes: Stakes, 97 | #[allow(dead_code)] 98 | unused_accounts: UnusedAccounts, 99 | pub epoch_stakes: HashMap, 100 | pub is_delta: bool, 101 | } 102 | 103 | #[derive(Clone, Default, Debug, Serialize, Deserialize, PartialEq, Eq, AbiExample)] 104 | pub struct BankHashInfo { 105 | pub hash: Hash, 106 | pub snapshot_hash: Hash, 107 | pub stats: BankHashStats, 108 | } 109 | 110 | #[derive(Clone, Debug, Default, Deserialize, PartialEq)] 111 | pub struct AccountsDbFields( 112 | pub HashMap>, 113 | pub StoredMetaWriteVersion, 114 | pub Slot, 115 | pub BankHashInfo, 116 | /// all slots that were roots within the last epoch 117 | #[serde(deserialize_with = "default_on_eof")] 118 | pub Vec, 119 | /// slots that were roots within the last epoch for which we care about the hash value 120 | #[serde(deserialize_with = "default_on_eof")] 121 | pub Vec<(Slot, Hash)>, 122 | ); 123 | 124 | pub type SerializedAppendVecId = usize; 125 | 126 | #[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Deserialize)] 127 | pub struct SerializableAccountStorageEntry { 128 | pub id: SerializedAppendVecId, 129 | pub accounts_current_len: usize, 130 | } 131 | -------------------------------------------------------------------------------- /plerkle_snapshot/src/unpacked.rs: -------------------------------------------------------------------------------- 1 | use std::{ 2 | fs::OpenOptions, 3 | io::BufReader, 4 | path::{Path, PathBuf}, 5 | str::FromStr, 6 | time::Instant, 7 | }; 8 | 9 | use itertools::Itertools; 10 | use log::info; 11 | use solana_runtime::snapshot_utils::SNAPSHOT_STATUS_CACHE_FILENAME; 12 | 13 | use crate::{ 14 | deserialize_from, parse_append_vec_name, AccountsDbFields, AppendVec, AppendVecIterator, 15 | DeserializableVersionedBank, ReadProgressTracking, Result, SerializableAccountStorageEntry, 16 | SnapshotError, SnapshotExtractor, SNAPSHOTS_DIR, 17 | }; 18 | 19 | /// Extracts account data from snapshots that were unarchived to a file system. 20 | pub struct UnpackedSnapshotExtractor { 21 | root: PathBuf, 22 | accounts_db_fields: AccountsDbFields, 23 | } 24 | 25 | impl SnapshotExtractor for UnpackedSnapshotExtractor { 26 | fn iter(&mut self) -> AppendVecIterator<'_> { 27 | Box::new(self.unboxed_iter()) 28 | } 29 | } 30 | 31 | impl UnpackedSnapshotExtractor { 32 | pub fn open(path: &Path, progress_tracking: Box) -> Result { 33 | let snapshots_dir = path.join(SNAPSHOTS_DIR); 34 | let status_cache = snapshots_dir.join(SNAPSHOT_STATUS_CACHE_FILENAME); 35 | if !status_cache.is_file() { 36 | return Err(SnapshotError::NoStatusCache); 37 | } 38 | 39 | let snapshot_files = snapshots_dir.read_dir()?; 40 | 41 | let snapshot_file_path = snapshot_files 42 | .filter_map(|entry| entry.ok()) 43 | .find(|entry| u64::from_str(&entry.file_name().to_string_lossy()).is_ok()) 44 | .map(|entry| entry.path().join(entry.file_name())) 45 | .ok_or(SnapshotError::NoSnapshotManifest)?; 46 | 47 | info!("Opening snapshot manifest: {:?}", snapshot_file_path); 48 | let snapshot_file = OpenOptions::new().read(true).open(&snapshot_file_path)?; 49 | let snapshot_file_len = snapshot_file.metadata()?.len(); 50 | 51 | let snapshot_file = progress_tracking.new_read_progress_tracker( 52 | &snapshot_file_path, 53 | Box::new(snapshot_file), 54 | snapshot_file_len, 55 | ); 56 | let mut snapshot_file = BufReader::new(snapshot_file); 57 | 58 | let pre_unpack = Instant::now(); 59 | let versioned_bank: DeserializableVersionedBank = deserialize_from(&mut snapshot_file)?; 60 | drop(versioned_bank); 61 | let versioned_bank_post_time = Instant::now(); 62 | 63 | let accounts_db_fields: AccountsDbFields = 64 | deserialize_from(&mut snapshot_file)?; 65 | let accounts_db_fields_post_time = Instant::now(); 66 | drop(snapshot_file); 67 | 68 | info!("Read bank fields in {:?}", versioned_bank_post_time - pre_unpack); 69 | info!( 70 | "Read accounts DB fields in {:?}", 71 | accounts_db_fields_post_time - versioned_bank_post_time 72 | ); 73 | 74 | Ok(UnpackedSnapshotExtractor { root: path.to_path_buf(), accounts_db_fields }) 75 | } 76 | 77 | pub fn unboxed_iter(&self) -> impl Iterator> + '_ { 78 | std::iter::once(self.iter_streams()).flatten_ok().flatten_ok() 79 | } 80 | 81 | fn iter_streams(&self) -> Result> + '_> { 82 | let accounts_dir = self.root.join("accounts"); 83 | Ok(accounts_dir 84 | .read_dir()? 85 | .filter_map(|f| f.ok()) 86 | .filter_map(|f| { 87 | let name = f.file_name(); 88 | parse_append_vec_name(&f.file_name()).map(move |parsed| (parsed, name)) 89 | }) 90 | .map(move |((slot, version), name)| { 91 | self.open_append_vec(slot, version, &accounts_dir.join(name)) 92 | })) 93 | } 94 | 95 | fn open_append_vec(&self, slot: u64, id: u64, path: &Path) -> Result { 96 | let known_vecs = self.accounts_db_fields.0.get(&slot).map(|v| &v[..]).unwrap_or(&[]); 97 | let known_vec = known_vecs.iter().find(|entry| entry.id == (id as usize)); 98 | let known_vec = match known_vec { 99 | None => return Err(SnapshotError::UnexpectedAppendVec), 100 | Some(v) => v, 101 | }; 102 | 103 | Ok(AppendVec::new_from_file(path, known_vec.accounts_current_len, slot)?) 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /rust-toolchain.toml: -------------------------------------------------------------------------------- 1 | [toolchain] 2 | channel = "1.83.0" -------------------------------------------------------------------------------- /rustfmt.toml: -------------------------------------------------------------------------------- 1 | # Basic 2 | use_small_heuristics = "Max" 3 | max_width = 100 4 | hard_tabs = false 5 | tab_spaces = 4 6 | newline_style = "Unix" 7 | # Imports 8 | imports_granularity = "Crate" 9 | group_imports = "StdExternalCrate" 10 | # Misc 11 | match_block_trailing_comma = true 12 | use_field_init_shorthand = true 13 | 14 | -------------------------------------------------------------------------------- /scripts/docker-entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -eo pipefail 4 | 5 | TIMESTAMP_FORMAT='+%Y-%m-%d %H:%M:%S' 6 | # This env variable is re-set here, since we expect the snapshots to be mounted at 7 | # /app/snapshot. 8 | SNAPSHOTDIR=/app/snapshot/*.tar.zst 9 | 10 | RED='\033[0;31m' 11 | GREEN='\033[0;32m' 12 | YELLOW='\033[0;33m' 13 | BLUE='\033[0;34m' 14 | NC='\033[0m' 15 | 16 | convert_time() { 17 | local total_seconds=$1 18 | local hours=$((total_seconds / 3600)) 19 | local minutes=$(((total_seconds % 3600) / 60)) 20 | local seconds=$((total_seconds % 60)) 21 | printf "%02d:%02d:%02d\n" $hours $minutes $seconds 22 | } 23 | 24 | if [[ -z "$SNAPSHOTDIR" ]]; then 25 | echo "Snapshot dir not set, please run the script from within the Makefile, or check your .env" && exit 1 26 | fi 27 | 28 | start_time=$(date +%s) 29 | 30 | echo -e "$BLUE$(date "$TIMESTAMP_FORMAT") SNAPSHOTDIR=$SNAPSHOTDIR$NC" 31 | echo -e "\n" 32 | 33 | if [ -z "$(ls -A ${SNAPSHOTDIR%/*})" ]; then 34 | echo -e "$RED$(date "$TIMESTAMP_FORMAT") No snapshot files found in ${SNAPSHOTDIR%/*}$NC" 35 | exit 1 36 | fi 37 | 38 | # Iterate through the files in SNAPSHOTDIR and process each one 39 | for f in $SNAPSHOTDIR; do 40 | if [ -f "$f" ]; then 41 | echo -e "$YELLOW$(date "$TIMESTAMP_FORMAT") Processing $f...$NC" 42 | # note: accounts selector config is copied directly in the Dockerfile. 43 | # due to that we can hardocde the path here. 44 | CMD="./solana-snapshot-etl \"$f\" --accounts-selector-config=accounts-selector-config.json" 45 | echo -e "$BLUE$(date "$TIMESTAMP_FORMAT") Running command: $CMD$NC" 46 | 47 | eval $CMD 48 | 49 | if [ $? -ne 0 ]; then 50 | echo -e "$RED$(date "$TIMESTAMP_FORMAT") Error processing $f$NC" 51 | exit 1 52 | fi 53 | 54 | echo -e "$GREEN$(date "$TIMESTAMP_FORMAT") Completed processing $f$NC" 55 | else 56 | echo -e "$RED$(date "$TIMESTAMP_FORMAT") No valid files found in $SNAPSHOTDIR$NC" 57 | exit 1 58 | fi 59 | done 60 | 61 | end_time=$(date +%s) 62 | execution_time=$((end_time - start_time)) 63 | formatted_time=$(convert_time $execution_time) 64 | 65 | echo -e "$GREEN$(date "$TIMESTAMP_FORMAT") Total execution time: $formatted_time$NC" 66 | echo -e "$GREEN$(date "$TIMESTAMP_FORMAT") All snapshot files processed successfully.$NC" 67 | --------------------------------------------------------------------------------