├── .github └── workflows │ └── rust.yml ├── .gitignore ├── CHANGELOG.md ├── Cargo.toml ├── LICENSE ├── README.md ├── examples ├── pyth_watch_all.rs ├── pyth_watch_ethusd.rs ├── pyth_watch_reconnect.rs └── pyth_watch_updates.rs ├── rust-toolchain.toml ├── rustfmt.toml └── src ├── blockchain.rs ├── error.rs ├── lib.rs ├── message.rs ├── network.rs ├── rpc.rs ├── stubborn.rs └── sync.rs /.github/workflows/rust.yml: -------------------------------------------------------------------------------- 1 | name: Rust 2 | 3 | on: [push] 4 | 5 | env: 6 | CARGO_TERM_COLOR: always 7 | 8 | jobs: 9 | build: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v2 13 | - name: Install dependency 14 | run: sudo apt-get install libudev-dev 15 | - name: Check 16 | run: cargo check 17 | - name: Build 18 | run: cargo build 19 | - name: Examples 20 | run: cargo build --examples 21 | 22 | test: 23 | name: Test Suite 24 | runs-on: ubuntu-latest 25 | steps: 26 | - uses: actions/checkout@v2 27 | - name: Install dependency 28 | run: sudo apt-get install libudev-dev 29 | 30 | - name: Run cargo test 31 | run: cargo test --verbose 32 | 33 | lints: 34 | name: Lints 35 | runs-on: ubuntu-latest 36 | steps: 37 | - name: Checkout sources 38 | uses: actions/checkout@v2 39 | 40 | - name: Install dependency 41 | run: sudo apt-get install libudev-dev 42 | 43 | - name: Install stable toolchain 44 | uses: actions-rs/toolchain@v1 45 | with: 46 | profile: minimal 47 | toolchain: stable 48 | override: true 49 | components: rustfmt, clippy 50 | 51 | - name: Run cargo fmt 52 | uses: actions-rs/cargo@v1 53 | with: 54 | command: fmt 55 | args: --all -- --check 56 | 57 | - name: Run cargo clippy 58 | uses: actions-rs/cargo@v1 59 | with: 60 | command: clippy 61 | args: -- -D warnings 62 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | Cargo.lock 3 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # 0.2.4 2 | 3 | - Set WSS and RPC URLs independently 4 | 5 | # 0.2.1 6 | 7 | - tracking of change on accounts for reconnects 8 | - panic! on too low reconnect values, to avoid unstable behaviour 9 | - instrumentation, tracing_futures RUST_LOG=solana_shadow=debug 10 | 11 | # 0.2.0 12 | 13 | - API change, `add_program`, `add_accounts` and `add_account` now return accounts 14 | - blocking rpc calls now executed in thread 15 | - changed "rpc call -> subscribe" into: "subsribe -> confirmation -> rpc call -> oneshot return" 16 | - added timeout on build_async 17 | 18 | # 0.1.8 19 | 20 | - Setting commitment level on RpcClient calls. 21 | 22 | # 0.1.7 23 | 24 | - Changing commitment config default to "finalized" 25 | - Updated dependencies 26 | 27 | # 0.1.6 28 | 29 | - Commitment Config Option 30 | 31 | # 0.1.5 32 | 33 | - `for_each_account` accepts `FnMut` to allow capturing mut variables 34 | - changed to edition = 2018 to be more compatible 35 | 36 | # 0.1.4 37 | 38 | - Auto-reconnect on connection drop 39 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | authors = ["Karim Agha ", "Marius Ciubotariu "] 3 | description = "Synchronized shadow state of Solana programs available for off-chain processing." 4 | edition = "2018" 5 | homepage = "https://github.com/hubble-markets/solana-shadow" 6 | include = ["examples/**/*", "src/**/*", "LICENSE", "README.md", "CHANGELOG.md"] 7 | keywords = ["blockchain", "solana", "web3"] 8 | license = "Apache-2.0" 9 | name = "solana-shadow" 10 | repository = "https://github.com/hubble-markets/solana-shadow" 11 | version = "0.2.5" 12 | 13 | [dependencies] 14 | base64 = "0.13" 15 | borsh = "0.9" 16 | borsh-derive = "0.9" 17 | bs58 = "0.4" 18 | dashmap = "5.0.0" 19 | futures = "0.3" 20 | hyper-tls = "0.5.0" 21 | serde = {version = "1", features = ["derive"]} 22 | serde_json = "1" 23 | solana-client = "1.8.3" 24 | solana-sdk = "1.8.3" 25 | thiserror = "1" 26 | tokio = {version = "1.13", features = ["full"]} 27 | tokio-tungstenite = {version = "0.16.1", features = ["native-tls"]} 28 | tracing = "0.1.29" 29 | tracing-futures = "0.2.5" 30 | zstd = "0.9" 31 | 32 | [dev-dependencies] 33 | anyhow = "1" 34 | pyth-client = "0.2.2" 35 | rand = "0.8" 36 | tracing-subscriber = {version = "0.3.1", features = ["env-filter"]} 37 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Solana Shadow 2 | 3 | The Solana Shadow crate adds shadows to solana on-chain accounts for off-chain processing. This create synchronises all accounts and their data related to a program in real time and allows off-chain bots to act upon changes to those accounts. 4 | 5 | [![Apache 2.0 licensed](https://img.shields.io/badge/license-Apache--2.0-blue)](./LICENSE) 6 | 7 | ## Usage 8 | 9 | Add this in your `Cargo.toml`: 10 | 11 | ```toml 12 | [dependencies] 13 | solana-shadow = "*" 14 | ``` 15 | 16 | Take a look at the `examples/` directory for usage examples. 17 | 18 | # Mirroring a program id and all its owned accounts: 19 | 20 | ```rust 21 | // this is the prog id that owns all pyth oracles on mainnet 22 | let prog = "FsJ3A3u2vn5cTVofAjvy6y5kwABJAqYWpe4975bi2epH".parse()?; 23 | let network = Network::Mainnet; 24 | let local = BlockchainShadow::new_for_program(&prog, network).await?; 25 | 26 | loop { 27 | local.for_each_account(|pubkey, account| { 28 | println!(" - [{}]: {:?}", pubkey, account); 29 | }); 30 | 31 | sleep(Duration::from_secs(3)).await; 32 | } 33 | 34 | local.worker().await?; 35 | ``` 36 | 37 | # Mirroring few random accounts 38 | 39 | ```rust 40 | // https://pyth.network/developers/accounts/ 41 | let ethusd = "JBu1AL4obBcCMqKBBxhpWCNUt136ijcuMZLFvTP7iWdB".parse()?; 42 | let btcusd = "GVXRSBjFk6e6J3NbVPXohDJetcTjaeeuykUpbQF8UoMU".parse()?; 43 | 44 | let local = BlockchainShadow::new_for_accounts(&vec![ethusd, btcusd], Network::Mainnet).await?; 45 | 46 | loop { 47 | let ethacc = shadow.get_account(ðusd).unwrap(); 48 | let ethprice = cast::(ðacc.data).agg.price; 49 | 50 | let btcacc = shadow.get_account(&btcusd).unwrap(); 51 | let btcprice = cast::(&btcacc.data).agg.price; 52 | 53 | println!("ETH/USD: {}", ethprice); 54 | println!("BTC/USD: {}", btcprice); 55 | 56 | sleep(Duration::from_secs(3)).await; 57 | } 58 | 59 | local.worker().await?; 60 | ``` 61 | 62 | 63 | # Listening on changes to accounts: 64 | 65 | ```rust 66 | 67 | // https://pyth.network/developers/accounts/ 68 | let ethusd = "JBu1AL4obBcCMqKBBxhpWCNUt136ijcuMZLFvTP7iWdB".parse()?; 69 | let btcusd = "GVXRSBjFk6e6J3NbVPXohDJetcTjaeeuykUpbQF8UoMU".parse()?; 70 | let solusd = "H6ARHf6YXhGYeQfUzQNGk6rDNnLBQKrenN712K4AQJEG".parse()?; 71 | 72 | // create an offline shadow of the on-chain data. 73 | // whenever the data change on-chain those changes 74 | // will be reflected immediately in this type. 75 | let shadow = BlockchainShadow::new_for_accounts( 76 | &vec![ethusd, btcusd, solusd], 77 | Network::Mainnet, 78 | ) 79 | .await?; 80 | 81 | tokio::spawn(async move { 82 | // start printing updates only after 5 seconds 83 | tokio::time::sleep(Duration::from_secs(5)).await; 84 | 85 | // now everytime an account changes, its pubkey will be 86 | // broadcasted to all receivers that are waiting on updates. 87 | while let Ok((pubkey, account)) = updates_channel.recv().await { 88 | let price = cast::(&account.data).agg.price; 89 | println!("account updated: {}: {}", &pubkey, price); 90 | } 91 | }); 92 | 93 | shadow.worker().await?; 94 | 95 | ``` -------------------------------------------------------------------------------- /examples/pyth_watch_all.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | use solana_shadow::{BlockchainShadow, SyncOptions}; 3 | use std::time::Duration; 4 | use tracing_subscriber::EnvFilter; 5 | 6 | fn configure_logging() { 7 | tracing_subscriber::fmt::Subscriber::builder() 8 | .with_writer(std::io::stdout) 9 | .with_env_filter( 10 | EnvFilter::try_from_default_env().unwrap_or(EnvFilter::new("info")), 11 | ) 12 | .init(); 13 | } 14 | 15 | #[tokio::main] 16 | async fn main() -> Result<()> { 17 | configure_logging(); 18 | 19 | println!("this example will dump the values of all accounts owned by pyth program every 5 seconds"); 20 | println!(); 21 | 22 | // this is the prog id that owns all pyth oracles on mainnet 23 | let prog = "FsJ3A3u2vn5cTVofAjvy6y5kwABJAqYWpe4975bi2epH".parse()?; 24 | let local = 25 | BlockchainShadow::new_for_program(&prog, SyncOptions::default()).await?; 26 | 27 | for _ in 0.. { 28 | local.for_each_account(|pubkey, account| { 29 | println!(" - [{}]: {:?}", pubkey, account); 30 | }); 31 | 32 | tokio::time::sleep(Duration::from_secs(3)).await; 33 | } 34 | 35 | local.worker().await?; 36 | 37 | println!("I will monitor pyth eth/usd prices"); 38 | 39 | Ok(()) 40 | } 41 | -------------------------------------------------------------------------------- /examples/pyth_watch_ethusd.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | use pyth_client::{cast, Price}; 3 | use solana_shadow::{BlockchainShadow, SyncOptions}; 4 | use std::time::Duration; 5 | use tracing_subscriber::EnvFilter; 6 | 7 | fn configure_logging() { 8 | tracing_subscriber::fmt::Subscriber::builder() 9 | .with_writer(std::io::stdout) 10 | .with_env_filter( 11 | EnvFilter::try_from_default_env().unwrap_or(EnvFilter::new("info")), 12 | ) 13 | .init(); 14 | } 15 | 16 | #[tokio::main] 17 | async fn main() -> Result<()> { 18 | configure_logging(); 19 | 20 | // https://pyth.network/developers/accounts/ 21 | let ethusd = "JBu1AL4obBcCMqKBBxhpWCNUt136ijcuMZLFvTP7iWdB".parse()?; 22 | let btcusd = "GVXRSBjFk6e6J3NbVPXohDJetcTjaeeuykUpbQF8UoMU".parse()?; 23 | let solusd = "H6ARHf6YXhGYeQfUzQNGk6rDNnLBQKrenN712K4AQJEG".parse()?; 24 | 25 | // create an offline shadow of the on-chain data. 26 | // whenever the data change on-chain those changes 27 | // will be reflected immediately in this type. 28 | // here it is "mut" because later on we are adding a new 29 | // account after 15 seconds. 30 | let mut shadow = BlockchainShadow::new_for_accounts( 31 | &vec![ethusd, btcusd], 32 | SyncOptions::default(), 33 | ) 34 | .await?; 35 | 36 | println!("this example will print prices of ETH and BTC every 3 seconds"); 37 | println!("starting on the 15th second it will also print SOL prices"); 38 | println!(); 39 | 40 | // iterate over the offline shadow of the account 41 | // everytime any account is accessed, then its contents 42 | // will reflect the latest version on-chain. 43 | for i in 0.. { 44 | // access the most recent snapshot of an account 45 | let ethacc = shadow.get_account(ðusd).unwrap(); 46 | let ethprice = cast::(ðacc.data).agg.price; 47 | 48 | let btcacc = shadow.get_account(&btcusd).unwrap(); 49 | let btcprice = cast::(&btcacc.data).agg.price; 50 | 51 | println!("ETH/USD: {}", ethprice); 52 | println!("BTC/USD: {}", btcprice); 53 | 54 | if i == 5 { 55 | // dynamically add new accounts to mirror 56 | shadow.add_account(&solusd).await?; 57 | } 58 | 59 | if i > 5 { 60 | let solacc = shadow.get_account(&solusd).unwrap(); 61 | let solprice = cast::(&solacc.data).agg.price; 62 | println!("SOL/USD: {}", solprice); 63 | } 64 | 65 | println!(); 66 | 67 | tokio::time::sleep(Duration::from_secs(3)).await; 68 | } 69 | 70 | shadow.worker().await?; 71 | Ok(()) 72 | } 73 | -------------------------------------------------------------------------------- /examples/pyth_watch_reconnect.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | use pyth_client::{cast, Price}; 3 | use solana_shadow::{BlockchainShadow, Network, SyncOptions}; 4 | use std::time::Duration; 5 | use tracing_subscriber::EnvFilter; 6 | 7 | fn configure_logging() { 8 | tracing_subscriber::fmt::Subscriber::builder() 9 | .with_writer(std::io::stdout) 10 | .with_env_filter( 11 | EnvFilter::try_from_default_env().unwrap_or(EnvFilter::new("info")), 12 | ) 13 | .init(); 14 | } 15 | 16 | #[tokio::main] 17 | async fn main() -> Result<()> { 18 | configure_logging(); 19 | 20 | // https://pyth.network/developers/accounts/ 21 | let ethusd = "JBu1AL4obBcCMqKBBxhpWCNUt136ijcuMZLFvTP7iWdB".parse()?; 22 | let btcusd = "GVXRSBjFk6e6J3NbVPXohDJetcTjaeeuykUpbQF8UoMU".parse()?; 23 | let solusd = "H6ARHf6YXhGYeQfUzQNGk6rDNnLBQKrenN712K4AQJEG".parse()?; 24 | 25 | // create an offline shadow of the on-chain data. 26 | // whenever the data change on-chain those changes 27 | // will be reflected immediately in this type. 28 | // here it is "mut" because later on we are adding a new 29 | // account after 15 seconds. 30 | let mut shadow = BlockchainShadow::new_for_accounts( 31 | &vec![ethusd, btcusd], 32 | SyncOptions { 33 | network: Network::Mainnet, 34 | reconnect_every: Some(Duration::from_secs(30)), 35 | ..Default::default() 36 | }, 37 | ) 38 | .await?; 39 | 40 | println!("this example will print prices of ETH and BTC every 3 seconds"); 41 | println!("starting on the 15th second it will also print SOL prices"); 42 | println!(); 43 | 44 | // iterate over the offline shadow of the account 45 | // everytime any account is accessed, then its contents 46 | // will reflect the latest version on-chain. 47 | for i in 0.. { 48 | // access the most recent snapshot of an account 49 | let ethacc = shadow.get_account(ðusd).unwrap(); 50 | let ethprice = cast::(ðacc.data).agg.price; 51 | 52 | let btcacc = shadow.get_account(&btcusd).unwrap(); 53 | let btcprice = cast::(&btcacc.data).agg.price; 54 | 55 | println!("ETH/USD: {}", ethprice); 56 | println!("BTC/USD: {}", btcprice); 57 | 58 | if i == 5 { 59 | // dynamically add new accounts to mirror 60 | println!("adding account"); 61 | shadow.add_account(&solusd).await?; 62 | } 63 | 64 | if i > 5 { 65 | let solacc = shadow.get_account(&solusd).unwrap(); 66 | let solprice = cast::(&solacc.data).agg.price; 67 | println!("SOL/USD: {}", solprice); 68 | } 69 | 70 | println!(); 71 | 72 | tokio::time::sleep(Duration::from_secs(3)).await; 73 | } 74 | 75 | shadow.worker().await?; 76 | Ok(()) 77 | } 78 | -------------------------------------------------------------------------------- /examples/pyth_watch_updates.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | use pyth_client::{cast, Price}; 3 | use solana_shadow::{BlockchainShadow, SyncOptions}; 4 | use std::time::Duration; 5 | use tokio::sync::broadcast::error::RecvError; 6 | use tracing_subscriber::EnvFilter; 7 | 8 | fn configure_logging() { 9 | tracing_subscriber::fmt::Subscriber::builder() 10 | .with_writer(std::io::stdout) 11 | .with_env_filter( 12 | EnvFilter::try_from_default_env().unwrap_or(EnvFilter::new("info")), 13 | ) 14 | .init(); 15 | } 16 | 17 | #[tokio::main] 18 | async fn main() -> Result<()> { 19 | configure_logging(); 20 | 21 | // https://pyth.network/developers/accounts/ 22 | let ethusd = "JBu1AL4obBcCMqKBBxhpWCNUt136ijcuMZLFvTP7iWdB".parse()?; 23 | let btcusd = "GVXRSBjFk6e6J3NbVPXohDJetcTjaeeuykUpbQF8UoMU".parse()?; 24 | let solusd = "H6ARHf6YXhGYeQfUzQNGk6rDNnLBQKrenN712K4AQJEG".parse()?; 25 | 26 | // create an offline shadow of the on-chain data. 27 | // whenever the data change on-chain those changes 28 | // will be reflected immediately in this type. 29 | let shadow = BlockchainShadow::new_for_accounts( 30 | &vec![ethusd, btcusd, solusd], 31 | SyncOptions::default(), 32 | ) 33 | .await?; 34 | 35 | println!( 36 | "this example will start printing prices of {}", 37 | "ETH and BTC every time they change after 5 seconds" 38 | ); 39 | println!(); 40 | 41 | // get a mpmc receiver end of an updates channel 42 | let mut updates_channel = shadow.updates_channel(); 43 | 44 | tokio::spawn(async move { 45 | // start printing updates only starting from the 5th second 46 | tokio::time::sleep(Duration::from_secs(5)).await; 47 | 48 | // now everytime an account changes, its pubkey will be 49 | // broadcasted to all receivers that are waiting on updates. 50 | loop { 51 | match updates_channel.recv().await { 52 | Ok((pubkey, account)) => { 53 | let price = cast::(&account.data).agg.price; 54 | println!("account updated: {}: {}", &pubkey, price); 55 | } 56 | Err(RecvError::Lagged(n)) => println!("updates channel lagging: {}", n), 57 | Err(RecvError::Closed) => eprintln!("updates channel closed"), 58 | } 59 | } 60 | }); 61 | 62 | shadow.worker().await?; 63 | Ok(()) 64 | } 65 | -------------------------------------------------------------------------------- /rust-toolchain.toml: -------------------------------------------------------------------------------- 1 | [toolchain] 2 | channel = "beta" 3 | -------------------------------------------------------------------------------- /rustfmt.toml: -------------------------------------------------------------------------------- 1 | tab_spaces = 2 2 | edition = "2018" 3 | fn_single_line = false 4 | format_code_in_doc_comments = true 5 | format_strings = true 6 | imports_layout = "HorizontalVertical" 7 | imports_granularity = "Crate" 8 | normalize_comments = true 9 | normalize_doc_attributes = true 10 | reorder_impl_items = true 11 | group_imports = "StdExternalCrate" 12 | use_try_shorthand = true 13 | wrap_comments = true 14 | max_width = 80 15 | merge_imports = true -------------------------------------------------------------------------------- /src/blockchain.rs: -------------------------------------------------------------------------------- 1 | use crate::{ 2 | rpc, 3 | sync::{AccountUpdate, SolanaChangeListener, SubRequest}, 4 | Error, Network, Result, 5 | }; 6 | use dashmap::DashMap; 7 | use futures::future::try_join_all; 8 | use solana_sdk::{ 9 | account::Account, commitment_config::CommitmentLevel, pubkey::Pubkey, 10 | }; 11 | use std::{sync::Arc, time::Duration}; 12 | use tokio::{ 13 | sync::{ 14 | broadcast::{self, Receiver, Sender}, 15 | mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender}, 16 | oneshot, 17 | }, 18 | task::JoinHandle, 19 | time::{interval, interval_at, Instant}, 20 | }; 21 | use tracing::{debug, error}; 22 | use tracing_futures::Instrument; 23 | 24 | pub(crate) type AccountsMap = DashMap; 25 | pub(crate) type SubRequestCall = 26 | (SubRequest, Option>>); 27 | 28 | /// This parameter control how many updates are going to be stored in memory 29 | /// for update receivers (if any subscribed) before it starts returning dropping 30 | /// them and returning RecvError::Lagging. 31 | /// For more info see: https://docs.rs/tokio/1.12.0/tokio/sync/broadcast/index.html#lagging 32 | /// for now it is set to 64, which gives us about 30 seconds on Solana 33 | /// as there can be an update at most once every 400 miliseconds (blocktime) 34 | const MAX_UPDATES_SUBSCRIBER_LAG: usize = 64; 35 | const MIN_RECONNECT_EVERY: u64 = 5; 36 | 37 | #[derive(Clone)] 38 | pub struct SyncOptions { 39 | pub network: Network, 40 | pub max_lag: Option, 41 | pub reconnect_every: Option, 42 | pub rpc_timeout: Duration, 43 | pub ws_connect_timeout: Duration, 44 | pub commitment: CommitmentLevel, 45 | } 46 | 47 | impl Default for SyncOptions { 48 | fn default() -> Self { 49 | Self { 50 | network: Network::Mainnet, 51 | max_lag: None, 52 | reconnect_every: None, 53 | commitment: CommitmentLevel::Finalized, 54 | rpc_timeout: Duration::from_secs(12), 55 | ws_connect_timeout: Duration::from_secs(12), 56 | } 57 | } 58 | } 59 | 60 | /// The entry point to the Solana Blockchain Shadow API 61 | /// 62 | /// This type allows its users to monitor several individual 63 | /// accounts or all accounts of a program, or a combination 64 | /// of both for any changes to those accounts and have the 65 | /// most recent version of those accounts available locally 66 | /// and accessible as if they were stored in a local 67 | /// `hashmap` 68 | pub struct BlockchainShadow { 69 | options: SyncOptions, 70 | accounts: Arc, 71 | sub_req: UnboundedSender, 72 | sync_worker: Option>>, 73 | monitor_worker: Option>>, 74 | ext_updates: Sender<(Pubkey, Account)>, 75 | } 76 | 77 | // public methods 78 | impl BlockchainShadow { 79 | pub async fn new(options: SyncOptions) -> Result { 80 | let max_lag = options.max_lag.unwrap_or(MAX_UPDATES_SUBSCRIBER_LAG); 81 | 82 | // protect our enduser against to quick reconnects which would 83 | // cause unstable behaviour 84 | if options 85 | .reconnect_every 86 | .map(|v| v < Duration::from_secs(MIN_RECONNECT_EVERY)) 87 | .unwrap_or(false) 88 | { 89 | panic!("low reconnect_every duration causes unstable behaviour, minimum reconnect_every value is {} seconds", MIN_RECONNECT_EVERY) 90 | } 91 | 92 | let (subscribe_tx, subscribe_rx) = unbounded_channel::(); 93 | 94 | let mut instance = Self { 95 | options, 96 | accounts: Arc::new(AccountsMap::new()), 97 | sync_worker: None, 98 | monitor_worker: None, 99 | sub_req: subscribe_tx, 100 | ext_updates: broadcast::channel(max_lag).0, 101 | }; 102 | 103 | instance.create_worker(subscribe_rx).await?; 104 | instance.create_monitor_worker().await?; 105 | 106 | Ok(instance) 107 | } 108 | 109 | pub async fn add_accounts( 110 | &mut self, 111 | accounts: &[Pubkey], 112 | ) -> Result>> { 113 | let mut results = Vec::new(); 114 | 115 | for key in accounts { 116 | let (oneshot, result) = oneshot::channel::>(); 117 | 118 | self 119 | .sub_req 120 | .clone() 121 | .send((SubRequest::Account(*key), Some(oneshot))) 122 | .map_err(|_| Error::InternalError)?; 123 | 124 | results.push(result); 125 | } 126 | let results = try_join_all(results).await?; 127 | let results = results.into_iter().map(|mut v| v.pop()).collect(); 128 | 129 | Ok(results) 130 | } 131 | 132 | pub async fn add_account( 133 | &mut self, 134 | account: &Pubkey, 135 | ) -> Result> { 136 | let (oneshot, result) = oneshot::channel::>(); 137 | 138 | self 139 | .sub_req 140 | .clone() 141 | .send((SubRequest::Account(*account), Some(oneshot))) 142 | .map_err(|_| Error::InternalError)?; 143 | 144 | Ok(result.await?.pop()) 145 | } 146 | 147 | pub async fn add_program( 148 | &mut self, 149 | program_id: &Pubkey, 150 | ) -> Result> { 151 | let (oneshot, result) = oneshot::channel(); 152 | 153 | self 154 | .sub_req 155 | .clone() 156 | .send((SubRequest::Program(*program_id), Some(oneshot))) 157 | .map_err(|_| Error::InternalError)?; 158 | 159 | Ok(result.await?) 160 | } 161 | 162 | pub async fn new_for_accounts( 163 | accounts: &[Pubkey], 164 | options: SyncOptions, 165 | ) -> Result { 166 | let mut instance = BlockchainShadow::new(options).await?; 167 | instance.add_accounts(accounts).await?; 168 | Ok(instance) 169 | } 170 | 171 | pub async fn new_for_program( 172 | program: &Pubkey, 173 | options: SyncOptions, 174 | ) -> Result { 175 | let mut instance = BlockchainShadow::new(options).await?; 176 | instance.add_program(program).await?; 177 | Ok(instance) 178 | } 179 | 180 | pub const fn network(&self) -> &Network { 181 | &self.options.network 182 | } 183 | 184 | pub fn len(&self) -> usize { 185 | self.accounts.len() 186 | } 187 | 188 | pub fn is_empty(&self) -> bool { 189 | self.len() == 0 190 | } 191 | 192 | pub fn for_each_account(&self, mut op: impl FnMut(&Pubkey, &Account)) { 193 | for pair in self.accounts.iter() { 194 | let pubkey = pair.pair().0; 195 | let (account, _) = pair.pair().1; 196 | op(pubkey, account); 197 | } 198 | } 199 | 200 | pub fn get_account(&self, key: &Pubkey) -> Option { 201 | self.accounts.get(key).map(|acc| acc.clone().0) 202 | } 203 | 204 | pub async fn worker(mut self) -> Result<()> { 205 | match self.sync_worker.take() { 206 | Some(handle) => Ok(handle.await??), 207 | None => Err(Error::WorkerDead), 208 | } 209 | } 210 | 211 | pub fn updates_channel(&self) -> Receiver<(Pubkey, Account)> { 212 | self.ext_updates.subscribe() 213 | } 214 | } 215 | 216 | impl BlockchainShadow { 217 | async fn create_worker( 218 | &mut self, 219 | mut subscribe_rx: UnboundedReceiver, 220 | ) -> Result<()> { 221 | // subscription requests from blockchain shadow -> listener 222 | let accs_ref = self.accounts.clone(); 223 | let updates_tx = self.ext_updates.clone(); 224 | let options = self.options.clone(); 225 | let client = rpc::ClientBuilder::new( 226 | self.network().rpc_url(), 227 | self.options.rpc_timeout, 228 | self.options.commitment, 229 | ); 230 | self.sync_worker = Some(tokio::spawn( 231 | async move { 232 | let mut listener = 233 | SolanaChangeListener::new(client, accs_ref.clone(), options).await?; 234 | loop { 235 | tokio::select! { 236 | message = listener.recv() => { 237 | let message = match message { 238 | Ok(m) => m, 239 | Err(e) => { 240 | error!("error in the sync worker thread: {:?}", e); 241 | listener.reconnect_all().await?; 242 | continue; 243 | } 244 | }; 245 | 246 | let account_update = listener.process_message(message).await?; 247 | if let Some(AccountUpdate { pubkey, account }) = account_update { 248 | debug!("account {} updated", &pubkey); 249 | accs_ref.insert(pubkey, ( account.clone(), true )); 250 | if updates_tx.receiver_count() != 0 { 251 | updates_tx.send((pubkey, account)).unwrap(); 252 | } 253 | } 254 | }, 255 | Some(subreq) = subscribe_rx.recv() => { 256 | match subreq { 257 | ( SubRequest::Account(pubkey), oneshot ) => { 258 | debug!(?pubkey, "subscribe_account recv"); 259 | listener.subscribe_account(pubkey, oneshot).await? 260 | }, 261 | ( SubRequest::Program(pubkey), oneshot ) => { 262 | debug!(?pubkey, "subscribe_program recv"); 263 | listener.subscribe_program(pubkey, oneshot).await? 264 | }, 265 | ( SubRequest::ReconnectAll, _ ) => { 266 | debug!("reconnect_all recv"); 267 | listener.reconnect_all().await? 268 | } 269 | ( SubRequest::Ping, _ ) => { 270 | debug!("ping recv"); 271 | listener.ping().await? 272 | } 273 | } 274 | } 275 | }; 276 | } 277 | } 278 | .instrument(tracing::debug_span!("worker_loop")), 279 | )); 280 | 281 | Ok(()) 282 | } 283 | 284 | async fn create_monitor_worker(&mut self) -> Result<()> { 285 | let mut ping_timer = interval(Duration::from_secs(5)); 286 | let reconnect_every = self.options.reconnect_every; 287 | 288 | let channel = self.sub_req.clone(); 289 | 290 | let fut = async move { 291 | if let Some(every) = reconnect_every { 292 | let mut reconnect_timer = interval_at(Instant::now() + every, every); 293 | loop { 294 | let req = tokio::select! { 295 | _ = ping_timer.tick() => SubRequest::Ping, 296 | _ = reconnect_timer.tick() => SubRequest::ReconnectAll, 297 | }; 298 | 299 | if let Err(e) = channel.send((req, None)) { 300 | tracing::error!(?e) 301 | } 302 | } 303 | } else { 304 | loop { 305 | ping_timer.tick().await; 306 | if let Err(e) = channel.send((SubRequest::Ping, None)) { 307 | tracing::error!(?e) 308 | } 309 | } 310 | } 311 | }; 312 | 313 | self.monitor_worker = Some(tokio::spawn( 314 | fut.instrument(tracing::debug_span!("monitor_worker_loop")), 315 | )); 316 | 317 | Ok(()) 318 | } 319 | } 320 | -------------------------------------------------------------------------------- /src/error.rs: -------------------------------------------------------------------------------- 1 | use solana_client::client_error::ClientError; 2 | use solana_sdk::pubkey::ParsePubkeyError; 3 | use thiserror::Error; 4 | use tokio::sync::oneshot::error::RecvError; 5 | use tokio::task::JoinError; 6 | use tokio::time::error::Elapsed; 7 | use tokio_tungstenite::tungstenite::Error as WsError; 8 | 9 | #[derive(Debug, Error)] 10 | pub enum Error { 11 | #[error("Expected a program account")] 12 | NotAProgramAccount, 13 | 14 | #[error("Invalid Argument")] 15 | InvalidArguemt, 16 | 17 | #[error("Background worker is dead")] 18 | WorkerDead, 19 | 20 | #[error("Solana RPC error")] 21 | SolanaClientError(#[from] ClientError), 22 | 23 | #[error("WebSocket error")] 24 | WebSocketError(#[from] WsError), 25 | 26 | #[error("AsyncWrap error")] 27 | AsyncWrapError(#[from] RecvError), 28 | 29 | #[error("AsyncTimeout error")] 30 | AsyncTimeoutError(#[from] Elapsed), 31 | 32 | #[error("Notification for an unknown subscription")] 33 | UnknownSubscription, 34 | 35 | #[error("Unsupported RPC message format")] 36 | UnsupportedRpcFormat, 37 | 38 | #[error("Internal error")] 39 | InternalError, 40 | 41 | #[error("Invalid JSON-RPC message")] 42 | InvalidRpcMessage(#[from] serde_json::Error), 43 | 44 | #[error("Failed to parse public key")] 45 | InvalidPublicKey(#[from] ParsePubkeyError), 46 | 47 | #[error("Failed to parse base64 data")] 48 | InvalidBase64Data(#[from] base64::DecodeError), 49 | 50 | #[error("Internal synchronization error")] 51 | InternalSynchronizationError(#[from] JoinError), 52 | 53 | #[error("Io Error")] 54 | StdIoError(#[from] std::io::Error), 55 | } 56 | 57 | pub type Result = std::result::Result; 58 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | mod blockchain; 2 | mod error; 3 | mod message; 4 | mod network; 5 | mod rpc; 6 | mod stubborn; 7 | mod sync; 8 | 9 | pub use blockchain::{BlockchainShadow, SyncOptions}; 10 | pub use error::{Error, Result}; 11 | pub use network::Network; 12 | pub use solana_sdk::pubkey::Pubkey; 13 | -------------------------------------------------------------------------------- /src/message.rs: -------------------------------------------------------------------------------- 1 | use std::{convert::TryFrom, io::Read, str::FromStr}; 2 | 3 | use serde::Deserialize; 4 | use solana_sdk::{account::Account, pubkey::Pubkey}; 5 | 6 | #[derive(Debug, Deserialize)] 7 | pub(crate) struct AccountChangeInfo { 8 | #[allow(dead_code)] 9 | pub value: NotificationValue, 10 | } 11 | 12 | #[derive(Debug, Deserialize)] 13 | pub(crate) struct ProgramChangeInfo {} 14 | 15 | #[derive(Debug, Deserialize)] 16 | pub(crate) struct NotificationContext { 17 | #[allow(dead_code)] 18 | pub slot: u64, 19 | } 20 | 21 | #[derive(Debug, Deserialize)] 22 | #[serde(untagged)] 23 | pub(crate) enum NotificationValue { 24 | Account(AccountRepresentation), 25 | Program(WrappedAccountRepresentation), 26 | } 27 | 28 | #[derive(Debug, Deserialize)] 29 | pub(crate) struct WrappedAccountRepresentation { 30 | pub pubkey: String, 31 | pub account: AccountRepresentation, 32 | } 33 | 34 | #[derive(Debug, Deserialize)] 35 | #[serde(rename_all = "camelCase")] 36 | pub(crate) struct AccountRepresentation { 37 | owner: String, 38 | executable: bool, 39 | lamports: u64, 40 | rent_epoch: u64, 41 | data: Vec, 42 | } 43 | 44 | #[derive(Debug, Deserialize)] 45 | pub(crate) struct NotificationResult { 46 | #[allow(dead_code)] 47 | pub context: NotificationContext, 48 | pub value: NotificationValue, 49 | } 50 | 51 | #[derive(Debug, Deserialize)] 52 | pub(crate) struct NotificationParams { 53 | pub result: NotificationResult, 54 | pub subscription: u64, 55 | } 56 | 57 | #[derive(Debug, Deserialize)] 58 | #[serde(untagged)] 59 | pub(crate) enum SolanaMessage { 60 | Confirmation { 61 | #[allow(dead_code)] 62 | jsonrpc: String, 63 | result: u64, 64 | id: u64, 65 | }, 66 | Notification { 67 | #[allow(dead_code)] 68 | jsonrpc: String, 69 | method: String, 70 | params: NotificationParams, 71 | }, 72 | } 73 | 74 | impl TryFrom for Account { 75 | type Error = crate::Error; 76 | fn try_from(repr: AccountRepresentation) -> crate::Result { 77 | let data = match &repr.data[..] { 78 | [content, format] => match &format[..] { 79 | "base64" => base64::decode(&content)?, 80 | "base64+zstd" => { 81 | let zstd_data = base64::decode(content.as_bytes())?; 82 | 83 | let mut data = vec![]; 84 | let mut reader = 85 | zstd::stream::read::Decoder::new(zstd_data.as_slice())?; 86 | reader.read_to_end(&mut data)?; 87 | data 88 | } 89 | _ => vec![], 90 | }, 91 | _ => vec![], 92 | }; 93 | 94 | Ok(Account { 95 | lamports: repr.lamports, 96 | data, 97 | owner: Pubkey::from_str(&repr.owner)?, 98 | executable: repr.executable, 99 | rent_epoch: repr.rent_epoch, 100 | }) 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /src/network.rs: -------------------------------------------------------------------------------- 1 | use std::fmt::{Display, Formatter}; 2 | 3 | #[derive(Debug, PartialEq, Eq, Clone)] 4 | pub enum Network { 5 | Devnet, 6 | Testnet, 7 | Mainnet, 8 | Localhost, 9 | /// (rpc_url, wss_url) 10 | Custom(String, String), 11 | } 12 | 13 | impl Network { 14 | pub(crate) fn rpc_url(&self) -> String { 15 | match self { 16 | Network::Devnet => "https://api.devnet.solana.com", 17 | Network::Testnet => "https://api.testnet.solana.com", 18 | Network::Mainnet => "https://api.mainnet-beta.solana.com", 19 | Network::Localhost => "http://127.0.0.1:8899", 20 | Network::Custom(url, _) => url, 21 | } 22 | .to_owned() 23 | } 24 | pub(crate) fn wss_url(&self) -> String { 25 | match self { 26 | Network::Devnet | Network::Testnet | Network::Mainnet => self.rpc_url(), 27 | Network::Localhost => "http://127.0.0.1:8900".to_owned(), 28 | Network::Custom(_, url) => url.to_owned(), 29 | } 30 | } 31 | } 32 | 33 | impl Display for Network { 34 | fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { 35 | write!(f, "{}", self.rpc_url()) 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/rpc.rs: -------------------------------------------------------------------------------- 1 | use std::{thread, time::Duration}; 2 | use tokio::sync::oneshot::channel; 3 | 4 | use solana_client::rpc_client::RpcClient; 5 | use solana_sdk::{ 6 | account::Account, 7 | commitment_config::{CommitmentConfig, CommitmentLevel}, 8 | pubkey::Pubkey, 9 | }; 10 | 11 | use crate::error::Result; 12 | 13 | /// Helper to convert sync code to async using an oneshot channel 14 | /// uses a timeout to ensure continuation of executing in case 15 | /// sync_code would panic 16 | async fn build_async(timeout: Duration, sync_code: F) -> Result 17 | where 18 | F: FnOnce() -> T + Send + 'static, 19 | T: Send + std::fmt::Debug + 'static, 20 | { 21 | let (tx, rx) = channel::(); 22 | 23 | thread::spawn(move || { 24 | // TODO: if the sync_code panics this 25 | // call will wait forever 26 | let r = sync_code(); 27 | if let Err(e) = tx.send(r) { 28 | tracing::warn!(?e, "build_async: could not oneshot") 29 | } 30 | }); 31 | 32 | Ok(tokio::time::timeout(timeout, rx).await??) 33 | } 34 | 35 | #[derive(Clone)] 36 | pub struct ClientBuilder { 37 | rpc_url: String, 38 | rpc_timeout: Duration, 39 | commitment: CommitmentLevel, 40 | } 41 | 42 | impl ClientBuilder { 43 | pub fn new( 44 | rpc_url: String, 45 | rpc_timeout: Duration, 46 | commitment: CommitmentLevel, 47 | ) -> Self { 48 | Self { 49 | rpc_url, 50 | rpc_timeout, 51 | commitment, 52 | } 53 | } 54 | 55 | fn build(&self) -> RpcClient { 56 | RpcClient::new_with_commitment( 57 | self.rpc_url.clone(), 58 | CommitmentConfig { 59 | commitment: self.commitment, 60 | }, 61 | ) 62 | } 63 | } 64 | 65 | #[allow(unused)] 66 | pub async fn get_multiple_accounts( 67 | client: ClientBuilder, 68 | accounts: &[Pubkey], 69 | ) -> Result> { 70 | let unvalidated = { 71 | let accounts = accounts.to_vec(); 72 | let rpc_client = client.build(); 73 | build_async(client.rpc_timeout, move || { 74 | rpc_client.get_multiple_accounts(&accounts) 75 | }) 76 | .await?? 77 | }; 78 | 79 | let valid_accounts: Vec<_> = unvalidated 80 | .into_iter() 81 | .zip(accounts.iter()) 82 | .filter(|(o, _)| o.is_some()) 83 | .map(|(acc, key)| (*key, acc.unwrap())) 84 | .collect(); 85 | 86 | if valid_accounts.len() < accounts.len() { 87 | tracing::warn!( 88 | missing = %(valid_accounts.len() - accounts.len()), 89 | "non-existing accounts detected" 90 | ) 91 | } 92 | 93 | Ok(valid_accounts) 94 | } 95 | 96 | pub async fn get_account( 97 | client: ClientBuilder, 98 | account: Pubkey, 99 | ) -> Result<(Pubkey, Account)> { 100 | let rpc_client = client.build(); 101 | Ok(( 102 | account, 103 | build_async(client.rpc_timeout, move || rpc_client.get_account(&account)) 104 | .await??, 105 | )) 106 | } 107 | 108 | pub async fn get_program_accounts( 109 | client: ClientBuilder, 110 | program_id: &Pubkey, 111 | ) -> Result> { 112 | let accounts = { 113 | let program_id = *program_id; 114 | let rpc_client = client.build(); 115 | build_async(client.rpc_timeout, move || { 116 | rpc_client.get_program_accounts(&program_id) 117 | }) 118 | .await?? 119 | }; 120 | 121 | Ok(accounts) 122 | } 123 | -------------------------------------------------------------------------------- /src/stubborn.rs: -------------------------------------------------------------------------------- 1 | use std::time::Duration; 2 | 3 | use tokio::net::TcpStream; 4 | use tokio_tungstenite::{ 5 | tungstenite::{self, client::IntoClientRequest, handshake::client::Response}, 6 | MaybeTlsStream, WebSocketStream, 7 | }; 8 | use tracing_futures::Instrument; 9 | use tungstenite::error::Error; 10 | 11 | #[tracing::instrument] 12 | pub async fn connect_async( 13 | timeout: Duration, 14 | request: R, 15 | ) -> Result<(WebSocketStream>, Response), Error> 16 | where 17 | R: std::fmt::Debug + IntoClientRequest + Unpin + Clone, 18 | { 19 | let mut delay = 500; 20 | loop { 21 | match tokio::time::timeout( 22 | timeout, 23 | tokio_tungstenite::connect_async(request.clone()) 24 | .instrument(tracing::info_span!("tungstenite_connect_async")), 25 | ) 26 | .await 27 | { 28 | Err(e) => { 29 | tracing::warn!( 30 | error=?e, "timeout while trying to connect to websocket, retrying in {}", delay 31 | ); 32 | } 33 | Ok(result) => match result { 34 | Ok(r) => { 35 | return Ok(r); 36 | } 37 | Err(e) => { 38 | tracing::warn!(error=?e, "websocket connection error, retrying in {}", delay); 39 | } 40 | }, 41 | } 42 | tokio::time::sleep(Duration::from_millis(delay)).await; 43 | delay = std::cmp::min(8000, delay * 2); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/sync.rs: -------------------------------------------------------------------------------- 1 | use crate::{ 2 | blockchain::SubRequestCall, 3 | message::{NotificationParams, NotificationValue, SolanaMessage}, 4 | Error, Result, SyncOptions, 5 | }; 6 | use dashmap::DashMap; 7 | use futures::{ 8 | stream::{SplitSink, SplitStream}, 9 | SinkExt, StreamExt, 10 | }; 11 | use serde_json::json; 12 | use solana_client::client_error::reqwest::Url; 13 | use solana_sdk::{account::Account, pubkey::Pubkey}; 14 | use std::{ 15 | convert::TryInto, 16 | sync::{ 17 | atomic::{AtomicU64, Ordering}, 18 | Arc, 19 | }, 20 | }; 21 | use tokio::{ 22 | net::TcpStream, 23 | sync::{oneshot, RwLock}, 24 | }; 25 | use tokio_tungstenite::{ 26 | tungstenite::{self, Message}, 27 | MaybeTlsStream, WebSocketStream, 28 | }; 29 | use tracing::{debug, info, warn}; 30 | 31 | use crate::blockchain::AccountsMap; 32 | use crate::rpc; 33 | use crate::stubborn; 34 | 35 | type WsStream = WebSocketStream>; 36 | type WsReader = SplitStream; 37 | type WsWriter = SplitSink; 38 | 39 | #[derive(Debug)] 40 | pub(crate) struct AccountUpdate { 41 | pub pubkey: Pubkey, 42 | pub account: Account, 43 | } 44 | 45 | #[derive(Debug, Clone, Copy)] 46 | pub(crate) enum SubRequest { 47 | Account(Pubkey), 48 | Program(Pubkey), 49 | Ping, 50 | ReconnectAll, 51 | } 52 | 53 | pub(crate) struct SolanaChangeListener { 54 | url: Url, 55 | reader: Option, 56 | writer: Option, 57 | reqid: AtomicU64, 58 | pending: DashMap, 59 | subscriptions: DashMap, 60 | subs_history: RwLock>, 61 | sync_options: SyncOptions, 62 | client: rpc::ClientBuilder, 63 | accounts: Arc, 64 | } 65 | 66 | impl SolanaChangeListener { 67 | pub async fn new( 68 | client: rpc::ClientBuilder, 69 | accounts: Arc, 70 | sync_options: SyncOptions, 71 | ) -> Result { 72 | let mut url: Url = sync_options 73 | .network 74 | .wss_url() 75 | .parse() 76 | .map_err(|_| Error::InvalidArguemt)?; 77 | 78 | match url.scheme() { 79 | "http" => url.set_scheme("ws").unwrap(), 80 | "https" => url.set_scheme("wss").unwrap(), 81 | "ws" | "wss" => (), 82 | _ => panic!("unsupported cluster url scheme"), 83 | }; 84 | 85 | let (ws_stream, _) = 86 | stubborn::connect_async(sync_options.ws_connect_timeout, url.clone()) 87 | .await?; 88 | let (writer, reader) = ws_stream.split(); 89 | Ok(Self { 90 | url, 91 | reader: Some(reader), 92 | writer: Some(writer), 93 | reqid: AtomicU64::new(1), 94 | pending: DashMap::new(), 95 | subscriptions: DashMap::new(), 96 | subs_history: RwLock::new(Vec::new()), 97 | sync_options, 98 | client, 99 | accounts, 100 | }) 101 | } 102 | 103 | /// Send an account subscription request to Solana Cluster. 104 | /// 105 | /// When this method returns, it does not mean that a subscription 106 | /// has been successfully created, but only the the request was 107 | /// sent successfully. 108 | pub async fn subscribe_account( 109 | &mut self, 110 | account: Pubkey, 111 | oneshot: Option>>, 112 | ) -> Result<()> { 113 | self 114 | .subscribe_account_internal(account, oneshot, true) 115 | .await 116 | } 117 | 118 | #[tracing::instrument(skip(self, oneshot))] 119 | async fn subscribe_account_internal( 120 | &mut self, 121 | account: Pubkey, 122 | oneshot: Option>>, 123 | record: bool, 124 | ) -> Result<()> { 125 | let reqid = self.reqid.fetch_add(1, Ordering::SeqCst); 126 | let request = json!({ 127 | "jsonrpc": "2.0", 128 | "id": reqid, 129 | "method": "accountSubscribe", 130 | "params": [account.to_string(), { 131 | "encoding": "base64+zstd", 132 | "commitment": self.sync_options.commitment.to_string(), 133 | }] 134 | }); 135 | 136 | let sub_request = SubRequest::Account(account); 137 | 138 | if record { 139 | // keep a copy of this request in the subscriptions 140 | // history log, so that when a reconnect event occurs 141 | // all those subscription messages are going to be replayed 142 | let mut history = self.subs_history.write().await; 143 | history.push(sub_request); 144 | } 145 | 146 | // map jsonrpc request id to pubkey, later on when 147 | // the websocket responds with a subscription id, 148 | // the id will be correlated with a public key. 149 | // account change notifications don't mention 150 | // the account public key, instead they use the 151 | // solana-generated subscription id to identify 152 | // an account. 153 | self.pending.insert(reqid, (sub_request, oneshot)); 154 | loop { 155 | if let Some(ref mut writer) = self.writer { 156 | debug!(request=%request, "accountSubscribe send over websocket"); 157 | writer.send(Message::Text(request.to_string())).await?; 158 | break; 159 | } else { 160 | debug!("skipping sending no writer available"); 161 | tokio::task::yield_now().await; 162 | } 163 | } 164 | Ok(()) 165 | } 166 | 167 | /// Send a program subscription request to Solana Cluster. 168 | /// 169 | /// When this method returns, it does not mean that a subscription 170 | /// has been successfully created, but only the the request was 171 | /// sent successfully. 172 | pub async fn subscribe_program( 173 | &mut self, 174 | account: Pubkey, 175 | oneshot: Option>>, 176 | ) -> Result<()> { 177 | self 178 | .subscribe_program_internal(account, oneshot, true) 179 | .await 180 | } 181 | 182 | #[tracing::instrument(skip(self, oneshot))] 183 | async fn subscribe_program_internal( 184 | &mut self, 185 | account: Pubkey, 186 | oneshot: Option>>, 187 | record: bool, 188 | ) -> Result<()> { 189 | let reqid = self.reqid.fetch_add(1, Ordering::SeqCst); 190 | let request = json!({ 191 | "jsonrpc": "2.0", 192 | "id": reqid, 193 | "method": "programSubscribe", 194 | "params": [account.to_string(), { 195 | "encoding": "base64+zstd", 196 | "commitment": self.sync_options.commitment.to_string() 197 | }] 198 | }); 199 | 200 | let sub_request = SubRequest::Program(account); 201 | 202 | if record { 203 | // keep a copy of this request in the subscriptions 204 | // history log, so that when a reconnect event occurs 205 | // all those subscription messages are going to be replayed 206 | let mut history = self.subs_history.write().await; 207 | history.push(sub_request); 208 | } 209 | 210 | // map jsonrpc request id to pubkey, later on when 211 | // the websocket responds with a subscription id, 212 | // the id will be correlated with a public key. 213 | // account change notifications don't mention 214 | // the account public key, instead they use the 215 | // solana-generated subscription id to identify 216 | // an account. 217 | self.pending.insert(reqid, (sub_request, oneshot)); 218 | loop { 219 | if let Some(ref mut writer) = self.writer { 220 | debug!(request=%request, "programSubscribe send over websocket"); 221 | writer.send(Message::Text(request.to_string())).await?; 222 | break; 223 | } else { 224 | debug!("skipping sending no writer available"); 225 | tokio::task::yield_now().await; 226 | } 227 | } 228 | Ok(()) 229 | } 230 | 231 | pub async fn ping(&mut self) -> Result<()> { 232 | loop { 233 | if let Some(ref mut writer) = self.writer { 234 | debug!("ping send over websocket"); 235 | writer.send(Message::Ping(vec![])).await?; 236 | break; 237 | } else { 238 | debug!("skipping sending no writer available"); 239 | tokio::task::yield_now().await; 240 | } 241 | } 242 | Ok(()) 243 | } 244 | 245 | #[tracing::instrument(skip(self), level = "debug")] 246 | pub async fn recv(&mut self) -> Result { 247 | loop { 248 | if let Some(ref mut reader) = self.reader { 249 | if let Some(msg) = reader.next().await { 250 | match msg { 251 | Ok(msg) => return Ok(msg), 252 | Err(e) => { 253 | warn!("received ws error from solana: {:?}", &e); 254 | return Err(Error::WebSocketError(e)); 255 | } 256 | } 257 | } 258 | } else { 259 | tokio::task::yield_now().await; 260 | } 261 | } 262 | } 263 | 264 | pub async fn process_message( 265 | &mut self, 266 | msg: Message, 267 | ) -> Result> { 268 | let message = match msg { 269 | Message::Text(text) => serde_json::from_str(&text)?, 270 | Message::Pong(_) => { 271 | tracing::trace!("received Pong"); 272 | return Ok(None); 273 | } 274 | Message::Ping(_) => { 275 | if let Some(ref mut writer) = self.writer { 276 | writer.send(Message::Pong(vec![])).await?; 277 | tracing::trace!("received Ping"); 278 | } else { 279 | warn!("No writer available, cannot reply ping") 280 | } 281 | return Ok(None); 282 | } 283 | _ => Err(Error::UnsupportedRpcFormat)?, 284 | }; 285 | 286 | tracing::trace!(?message, "received from wss"); 287 | 288 | // This message is a JSON-RPC response to a subscription request. 289 | // Here we are mapping the request id with the subscription id, 290 | // and creating a map of subscription id => pubkey. 291 | // This type of message is not relevant to the external callers 292 | // of this method, so we keep looping and listening for interesting 293 | // notifications. 294 | use dashmap::mapref::entry::Entry::{Occupied, Vacant}; 295 | if let SolanaMessage::Confirmation { id, result, .. } = message { 296 | if let Some((_, (sub_request, oneshot))) = self.pending.remove(&id) { 297 | self.subscriptions.insert(result, sub_request); 298 | match sub_request { 299 | SubRequest::Account(account) => { 300 | let (key, acc) = 301 | rpc::get_account(self.client.clone(), account).await?; 302 | 303 | // only insert entry if we have not received an update 304 | // from our subscription 305 | match self.accounts.entry(key) { 306 | Occupied(mut e) => { 307 | let (_, updated) = e.get(); 308 | 309 | tracing::debug!(?account, ?updated, "occupied branch"); 310 | if !updated { 311 | // we update with our RPC values 312 | e.insert((acc.clone(), true)); 313 | } 314 | } 315 | Vacant(e) => { 316 | tracing::debug!(?account, "vacant branch"); 317 | e.insert((acc, true)); 318 | } 319 | } 320 | 321 | if let Some(oneshot) = oneshot { 322 | let account = self.accounts.get(&key).unwrap().0.clone(); 323 | 324 | tracing::debug!(?account, "oneshot send"); 325 | if oneshot.send(vec![account]).is_err() { 326 | tracing::warn!("receiver dropped") 327 | } 328 | } 329 | } 330 | SubRequest::Program(program_id) => { 331 | // we have a successful program subscription, so now lets get 332 | // all program accounts and add them to our accounts 333 | let accounts: Vec<_> = 334 | rpc::get_program_accounts(self.client.clone(), &program_id) 335 | .await?; 336 | 337 | // only insert entry if we have not received an update 338 | // from our subscription 339 | let mut result: Vec = vec![]; 340 | for (key, acc) in accounts { 341 | match self.accounts.entry(key) { 342 | Occupied(mut e) => { 343 | let (account, updated) = e.get().clone(); 344 | 345 | tracing::debug!( 346 | ?program_id, 347 | ?account, 348 | ?updated, 349 | "occupied branch" 350 | ); 351 | if !updated { 352 | // we update with our RPC values 353 | e.insert((acc.clone(), true)); 354 | result.push(acc); 355 | } else { 356 | // value updated over ws subscription 357 | // no need to update, and we push 358 | // the ws value to results to return the 359 | // latest 360 | result.push(account); 361 | } 362 | } 363 | Vacant(e) => { 364 | tracing::debug!(?program_id, ?acc, "vacant branch"); 365 | e.insert((acc, true)); 366 | } 367 | } 368 | } 369 | 370 | // return value all the way back to the caller 371 | if let Some(oneshot) = oneshot { 372 | tracing::debug!(?program_id, accounts_len=?result.len(), "oneshot send"); 373 | if oneshot.send(result).is_err() { 374 | tracing::warn!("receiver dropped") 375 | } 376 | } 377 | } 378 | SubRequest::ReconnectAll | SubRequest::Ping => { 379 | // note: we safely ignore these 380 | } 381 | }; 382 | 383 | debug!("created subscripton {} for {:?}", &result, &sub_request); 384 | } else { 385 | warn!("Unrecognized subscription id: ({}, {})", id, result); 386 | } 387 | } 388 | 389 | // This is a notification call from Solana telling us to either an 390 | // account or a program has changed. 391 | if let SolanaMessage::Notification { method, params, .. } = message { 392 | match &method[..] { 393 | "accountNotification" | "programNotification" => { 394 | return Ok(Some(self.account_notification_to_change(params)?)); 395 | } 396 | _ => { 397 | warn!("unrecognized notification type: {}", &method); 398 | return Ok(None); 399 | } 400 | } 401 | } 402 | return Ok(None); 403 | } 404 | 405 | #[tracing::instrument(skip(self))] 406 | pub async fn reconnect_all(&mut self) -> Result<()> { 407 | let old_reader = self.reader.take(); 408 | let old_writer = self.writer.take(); 409 | 410 | let (ws_stream, _) = stubborn::connect_async( 411 | self.sync_options.ws_connect_timeout, 412 | self.url.clone(), 413 | ) 414 | .await?; 415 | 416 | let (writer, reader) = ws_stream.split(); 417 | 418 | self.reader = Some(reader); 419 | self.writer = Some(writer); 420 | self.pending = DashMap::new(); 421 | self.subscriptions = DashMap::new(); 422 | 423 | // mark all values in account as cleared 424 | self.accounts.alter_all(|_, (account, _)| (account, false)); 425 | 426 | let mut stream = old_writer 427 | .unwrap() 428 | .reunite(old_reader.unwrap()) 429 | .map_err(|_| Error::InternalError)?; 430 | stream.close(None).await.unwrap_or_else(|e| { 431 | if let tungstenite::Error::AlreadyClosed = e { 432 | // this is expected. 433 | debug!("Connection to solana closed"); 434 | } else { 435 | // leave a trace in the log for easier debugging 436 | warn!("failed closing connection to Solana: {:?}", e); 437 | } 438 | }); 439 | 440 | let history = { 441 | let shared_history = self.subs_history.read().await; 442 | shared_history.clone() 443 | }; 444 | 445 | for sub in history.iter() { 446 | match sub { 447 | SubRequest::Account(acc) => { 448 | info!("recreating account subscription for {}", &acc); 449 | self.subscribe_account_internal(*acc, None, false).await? 450 | } 451 | SubRequest::Program(acc) => { 452 | info!("recreating program subscription for {}", &acc); 453 | self.subscribe_program_internal(*acc, None, false).await? 454 | } 455 | _ => panic!("invalid history value"), 456 | } 457 | } 458 | 459 | Ok(()) 460 | } 461 | 462 | fn account_notification_to_change( 463 | &self, 464 | params: NotificationParams, 465 | ) -> Result { 466 | match params.result.value { 467 | NotificationValue::Account(acc) => { 468 | if let Some(SubRequest::Account(pubkey)) = 469 | self.subscriptions.get(¶ms.subscription).as_deref() 470 | { 471 | Ok(AccountUpdate { 472 | pubkey: *pubkey, 473 | account: acc.try_into()?, 474 | }) 475 | } else { 476 | warn!("Unknown subscription: {}", ¶ms.subscription); 477 | Err(Error::UnknownSubscription) 478 | } 479 | } 480 | NotificationValue::Program(progacc) => Ok(AccountUpdate { 481 | pubkey: progacc.pubkey.parse()?, 482 | account: progacc.account.try_into()?, 483 | }), 484 | } 485 | } 486 | } 487 | --------------------------------------------------------------------------------