├── .env ├── proto └── transaction.proto ├── Cargo.toml └── README.md /.env: -------------------------------------------------------------------------------- 1 | RPC_ENDPOINT= 2 | LOOKUP_TABLE_CACHE_TTL_SECONDS=300 3 | UDP_BUFFER_SOCKET=0.0.0.0:8002 4 | GRPC_SERVER_ENDPOINT=0.0.0.0:50051 5 | -------------------------------------------------------------------------------- /proto/transaction.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package transaction; 4 | 5 | service TransactionService { 6 | rpc StreamTransactions (StreamTransactionsRequest) returns (stream TransactionResponse); 7 | } 8 | 9 | message StreamTransactionsRequest { 10 | 11 | } 12 | 13 | message TransactionResponse { 14 | string transaction_json = 1; 15 | uint64 timestamp = 2; 16 | } -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "shredstream-decoder" 3 | version = "0.1.1" 4 | edition = "2024" 5 | 6 | [build-dependencies] 7 | tonic-build = "0.12.3" 8 | 9 | [profile.release] 10 | lto = false 11 | opt-level = 3 12 | codegen-units = 1 13 | debug = false 14 | 15 | [dependencies] 16 | tokio = { version = "1.42.0", features = ["full"] } 17 | bincode = "1.3.3" 18 | reed-solomon-erasure = "6.0.0" 19 | borsh = "1.5.3" 20 | serde_json = "1.0.133" 21 | serde = "1.0.215" 22 | sha2 = "0.10.8" 23 | spl-token = "7.0.0" 24 | solana-client = "2.1.4" 25 | solana-sdk = "2.1.4" 26 | solana-entry = "2.1.4" 27 | solana-ledger = "2.1.4" 28 | bs58 = "0.5.1" 29 | hex = "0.4.3" 30 | chrono = "0.4.38" 31 | rustc-hash = "2.1.0" 32 | ring = "0.17.8" 33 | lazy_static = "1.5.0" 34 | rayon = "1.10.0" 35 | dotenv = "0.15.0" 36 | crossbeam = "0.8.4" 37 | thiserror = "2.0.7" 38 | dashmap = "6.1.0" 39 | tracing = "0.1.41" 40 | tracing-subscriber = { version = "0.3.19", features = ["env-filter"] } 41 | console-subscriber = "0.4.1" 42 | tokio-stream = { version = "0.1.17", features = ["sync"] } 43 | futures-core = "0.3.31" 44 | libc = "0.2.169" 45 | env_logger = "0.11.6" 46 | num_cpus = "1.16.0" 47 | yellowstone-grpc-proto = "4.1.1" 48 | tonic = "0.12.3" 49 | prost = "0.13.4" 50 | futures = "0.3.31" 51 | maplit = "1.0.2" 52 | yellowstone-grpc-client = "4.1.0" 53 | json = "0.12.4" 54 | tokio-tungstenite = "0.26.1" 55 | futures-util = "0.3.31" 56 | mimallocator = "0.1.3" -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Solana Shredstream Decoder - Multi-Protocol Edition 2 | 3 | This is an high-performance Rust client that decodes Solana shred packets directly from the Turbine block propagation layer. Unlike Geyser RPC or standard RPC requests, this decoder ensures you receive transactions the instant they are propagated by the leader validator, providing an unmatched speed advantage for time-sensitive trading operations. 4 | 5 | This advanced version supports **11 major Solana DeFi protocols** with comprehensive instruction decoding, Address Lookup Table (ALT) resolution, and intelligent caching mechanisms. 6 | 7 | # Supported Protocols & Instructions 8 | 9 | ## 1. **Pumpfun** (`6EF8rrecthR5Dkzon8Nwu78hRvfCKubJ14M5uBEwF6P`) 10 | ## 2. **Raydium AMM** (`675kPX9MHTjS2zt1qfr1NYHuzeLXfQM9H24wFSUt1Mp8`) 11 | ## 3. **Raydium CPMM** (`CPMMoo8L3F4NbTegBCKVNunggL7H1ZpdTHKxQB5qKP1C`) 12 | ## 4. **Raydium LaunchLab** (`LanMV9sAd7wArD4vJFi2qDdfnVhFxYSUg6eADduJ3uj`) 13 | ## 5. **Moonit** (`MoonCVVNZFSYkqNXP6bxHLPL6QQJiMagDL3qcqUQTrG`) 14 | ## 6. **Boop** (`boop8hVGQGqehUK2iVEMEnMrL5RbjywRzHKBmBE7ry4`) 15 | ## 7. **PumpAMM** (`pAMMBay6oceH9fJKBRHGP5D4bD4sWpmSwMn52FMfXEA`) 16 | ## 8. **Meteora VCurve** (`dbcij3LWUppWqq96dh6gJWwBifmcGfLSB5D4DuSMaqN`) 17 | ## 9. **Meteora DYN** (`Eo7WjKq67rjJQSZxS6z3YkapzY3eMj6Xy8X5EQVn5UaB`) 18 | ## 10. **Meteora AMM V2** (`cpamdpZCGKUy5JxQXB4dcpGPiikHawvSWAd6mEn1sGG`) 19 | ## 11. **Orca Whirlpool** (`whirLbMiicVdio4qvUfM5KAg6Ct8VwpYzGff3uctyCc`) 20 | 21 | # Advanced Features 22 | 23 | ## Core Performance Optimizations 24 | 25 | - **Jemalloc Memory Allocator** – Memory management with reduced fragmentation and improved allocation performance compared to default allocator 26 | - **Optimized UDP Buffer (256KB)** – Prevents packet loss by configuring `SO_RCVBUF` to handle high-volume shred bursts without dropping packets 27 | - **Physical CPU Thread Pool** – Rayon thread pool configured to use `num_cpus - 1` physical cores, reserving one for system operations 28 | - **Parallel Transaction Processing** – Leverages Rayon for CPU-bound deserialization for maximum throughput 29 | - **Async I/O with Tokio** – Non-blocking network operations for receiving shreds and serving gRPC streams 30 | - **DashMap Concurrent Storage** – Lock-free concurrent hash maps for FEC blocks and processed block tracking 31 | - **Zero-Copy Shred Handling** – Efficient `Box<[u8]>` usage to minimize memory allocations 32 | 33 | ## Reed-Solomon FEC Recovery 34 | 35 | - **Automatic Data Shred Recovery** – When data shreds are missing, the decoder uses coding shreds to reconstruct the complete payload 36 | - **Shredder Integration** – Uses Solana's ReedSolomon recovery erasure code processing 37 | - **Complete Block Reconstruction** – Ensures 99%+ transaction capture rate even with packet loss 38 | - **Intelligent Recovery Logging** – Detailed logs showing recovery attempts and success rates per FEC set 39 | 40 | ## Address Lookup Table (ALT) Cache System 41 | 42 | - **RPC-Backed Cache** – Fetches and caches Address Lookup Tables from RPC endpoint with configurable TTL (default: 300 seconds) 43 | - **Automatic Refresh on Extension** – Detects `ExtendLookupTable` instructions and invalidates cache to prevent stale data 44 | - **Out-of-Bounds Protection** – Automatically refreshes lookup tables when indices exceed current size 45 | - **Fallback Mechanism** – Uses stale cache entries if RPC fetch fails, preventing transaction loss 46 | - **Async Resolution** – Pre-resolves all lookup tables before parallel processing to maintain async context 47 | - **V0 Transaction Support** – Full support for versioned transactions with address table lookups 48 | 49 | ## Intelligent Garbage Collection 50 | 51 | - **Periodic FEC Block Cleanup** – Removes FEC blocks older than 20 seconds every 10 seconds 52 | - **Processed Block Tracking** – DashSet-based tracking prevents duplicate processing 53 | 54 | ## gRPC Streaming Server 55 | 56 | - **Sub-Millisecond Latency** – Broadcasts decoded transactions via gRPC with <0.1ms latency on localhost 57 | - **Broadcast Channel Architecture** – 1000-message buffer with `tokio::sync::broadcast` for multiple subscribers 58 | - **Tonic Framework** – Production-ready gRPC implementation with Protocol Buffers 59 | - **Automatic Stream Management** – Handles client connections, disconnections, and backpressure 60 | - **Timestamp Precision** – Microsecond-precision timestamps for each transaction 61 | 62 | ## Comprehensive Logging & Debugging 63 | 64 | - **Debug Mode Enhancements** – Thread IDs, names, line numbers, and targets in debug builds 65 | - **Performance Metrics** – Detailed timing for shred collection, FEC completion, and transaction extraction 66 | - **Slot Statistics** – Tracks FEC blocks processed, transactions extracted, and completion rates per slot 67 | - **Error Context** – Rich error messages with full context for debugging protocol-specific issues 68 | 69 | ## Transaction Deserialization 70 | 71 | - **Standardized Output Format** – Uniform JSON structure across all protocols with: 72 | - Program ID and instruction name 73 | - Protocol identifier 74 | - Raw instruction data (hex) 75 | - Mapped accounts with signer/writable flags 76 | - Parsed instruction arguments 77 | - **Borsh Deserialization** – Efficient binary deserialization for all instruction types 78 | - **Account Safety Checks** – Validates account indices before access to prevent panics 79 | - **Unknown Account Handling** – Gracefully handles missing accounts with "Unknown" placeholders 80 | 81 | # Architecture Highlights 82 | 83 | ## Shred Processing Pipeline 84 | 85 | 1. **Packet Reception** – Receives raw shred packets from configured source 86 | 2. **Validation & Assembly** – Validates and assembles shreds into complete blocks 87 | 3. **Data Recovery** – Reconstructs missing data using advanced error correction 88 | 4. **Transaction Extraction** – Extracts and deserializes transactions from assembled blocks 89 | 5. **Protocol Decoding** – Identifies and decodes protocol-specific instructions 90 | 6. **Output Streaming** – Broadcasts decoded transactions via gRPC to connected clients 91 | 92 | ## Memory Management 93 | 94 | - **Jemalloc Global Allocator** – Optimized for multi-threaded workloads 95 | - **Arc-Based Sharing** – Shared ownership of FEC blocks and sockets across tasks 96 | - **Scoped Lifetimes** – Automatic cleanup when FEC blocks complete 97 | - **Bounded Channels** – Prevents unbounded memory growth in broadcast queue 98 | 99 | # Performance Characteristics 100 | 101 | - **Throughput** – Handles 1000+ transactions per slot without problems 102 | - **CPU Utilization** – Scales linearly with available physical cores 103 | - **Memory Footprint** – ~200-500MB typical usage with automatic GC 104 | - **Cache Hit Rate** – 95%+ ALT cache hits after warm-up period 105 | 106 | # Who Should Use This? 107 | 108 | - **MEV Searchers** – Fastest possible transaction feed for arbitrage and liquidations 109 | - **Sniper Bots** – Catch token launches across 11 protocols instantly 110 | - **Market Makers** – Real-time pool creation and liquidity events 111 | - **Validators** – Decode shreds locally without external RPC dependencies 112 | - **Analytics Platforms** – Comprehensive multi-protocol transaction indexing 113 | - **Trading Firms** – Production-grade infrastructure for high-frequency strategies 114 | 115 | # Setup & Configuration 116 | 117 | ## Environment Variables 118 | 119 | Create a `.env` file with the following variables: 120 | 121 | ```bash 122 | # RPC endpoint example for Address Lookup Table resolution 123 | RPC_ENDPOINT=https://mainnet.helius-rpc.com/?api-key=YOUR_API_KEY 124 | 125 | # ALT cache TTL in seconds (default: 300) 126 | LOOKUP_TABLE_CACHE_TTL_SECONDS=300 127 | 128 | # UDP socket for receiving shreds 129 | # Use 8001 or 8002 depending on Jito Shredstream Proxy configuration 130 | # Or use your validator's TVU port for direct connection 131 | UDP_BUFFER_SOCKET=0.0.0.0:8002 132 | 133 | # gRPC server endpoint for streaming transactions 134 | GRPC_SERVER_ENDPOINT=0.0.0.0:50051 135 | ``` 136 | 137 | ## Build & Run 138 | 139 | ### Optimized Release Build 140 | 141 | ```bash 142 | RUSTFLAGS="-C target-cpu=native" cargo build --release 143 | ``` 144 | 145 | ### Run with Info Logging 146 | 147 | ```bash 148 | RUST_LOG=info cargo run --release 149 | ``` 150 | 151 | ### Run with Debug Logging 152 | 153 | ```bash 154 | RUST_LOG=debug cargo run --release 155 | ``` 156 | 157 | ## Compiler Optimizations 158 | 159 | The release profile in `Cargo.toml` is configured for maximum performance: 160 | 161 | ```toml 162 | [profile.release] 163 | opt-level = 3 # Maximum optimizations 164 | lto = "fat" # Link-time optimization across all crates 165 | codegen-units = 1 # Single codegen unit for better optimization 166 | panic = "abort" # Smaller binary, faster panics 167 | strip = true # Remove debug symbols 168 | debug = false # No debug info in release 169 | ``` 170 | 171 | # Output Format 172 | 173 | Transactions are streamed as JSON with the following structure: 174 | 175 | ```json 176 | { 177 | "signatures": ["5Xm..."], 178 | "slot": 123456789, 179 | "message": { 180 | "header": { 181 | "numRequiredSignatures": 1, 182 | "numReadonlySignedAccounts": 0, 183 | "numReadonlyUnsignedAccounts": 5 184 | }, 185 | "recentBlockhash": "9Xm...", 186 | "instructions": [ 187 | { 188 | "program_id": "6EF8rrecthR5Dkzon8Nwu78hRvfCKubJ14M5uBEwF6P", 189 | "instruction_name": "Create", 190 | "protocol": "Pumpfun", 191 | "raw_data": "181ec828051c0777...", 192 | "accounts": [ 193 | { 194 | "index": 0, 195 | "pubkey": "7Xm...", 196 | "signer": true, 197 | "writable": true 198 | } 199 | ], 200 | "parsed_data": { 201 | "name": "MyToken", 202 | "symbol": "MTK", 203 | "uri": "https://...", 204 | "creator": "7Xm..." 205 | } 206 | } 207 | ] 208 | } 209 | } 210 | ``` 211 | 212 | # Monitoring & Debugging 213 | 214 | ## Log Levels 215 | 216 | - **ERROR** – Critical failures (RPC errors, deserialization failures) 217 | - **WARN** – Recoverable issues (missing accounts, cache misses) 218 | - **INFO** – Key events (FEC completion, cache initialization, GC stats) 219 | - **DEBUG** – Detailed tracing (shred collection, timing metrics) 220 | 221 | ## Performance Tuning 222 | 223 | - Increase `UDP_BUFFER_SOCKET` buffer size if experiencing packet loss 224 | - Adjust `LOOKUP_TABLE_CACHE_TTL_SECONDS` based on network latency 225 | - Monitor CPU usage and adjust Rayon thread pool if needed 226 | 227 | # Source Code 228 | 229 | If you are really interested in the source code, please contact me for details and demo on Discord: `.xanr` 230 | --------------------------------------------------------------------------------