├── .github └── workflows │ └── ci.yml ├── .gitignore ├── README.md ├── content-discovery ├── .gitignore ├── Cargo.toml ├── README.md ├── iroh-mainline-content-discovery-cli │ ├── Cargo.toml │ └── src │ │ ├── args.rs │ │ └── main.rs ├── iroh-mainline-content-discovery │ ├── Cargo.toml │ ├── README.md │ └── src │ │ ├── client.rs │ │ ├── lib.rs │ │ ├── protocol.rs │ │ └── tls_utils │ │ ├── certificate.rs │ │ ├── mod.rs │ │ └── verifier.rs └── iroh-mainline-tracker │ ├── Cargo.toml │ └── src │ ├── args.rs │ ├── io.rs │ ├── iroh_blobs_util.rs │ ├── lib.rs │ ├── main.rs │ ├── options.rs │ ├── task_map.rs │ ├── tracker.rs │ └── tracker │ ├── tables.rs │ └── util.rs ├── h3-iroh ├── Cargo.toml ├── examples │ ├── client.rs │ ├── server-axum.rs │ └── server.rs └── src │ ├── axum.rs │ └── lib.rs ├── iroh-dag-sync ├── .gitignore ├── Cargo.toml ├── README.md └── src │ ├── args.rs │ ├── main.rs │ ├── protocol.rs │ ├── sync.rs │ ├── tables.rs │ ├── traversal.rs │ └── util.rs ├── iroh-pkarr-naming-system ├── Cargo.toml ├── README.md ├── examples │ └── cli.rs └── src │ └── lib.rs └── iroh-s3-bao-store ├── .gitignore ├── Cargo.toml ├── LICENSE-APACHE ├── LICENSE-MIT ├── README.md └── src ├── lib.rs └── main.rs /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | pull_request: 5 | merge_group: 6 | 7 | concurrency: 8 | group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} 9 | cancel-in-progress: true 10 | 11 | env: 12 | RUST_BACKTRACE: 1 13 | RUSTFLAGS: -Dwarnings 14 | RUSTDOCFLAGS: -Dwarnings 15 | MSRV: "1.75" 16 | RS_EXAMPLES_LIST: "content-discovery,iroh-pkarr-naming-system,iroh-s3-bao-store,iroh-dag-sync,h3-iroh" 17 | IROH_FORCE_STAGING_RELAYS: "1" 18 | 19 | jobs: 20 | build_and_test_nix: 21 | timeout-minutes: 30 22 | name: Build and test (Nix) 23 | runs-on: ${{ matrix.runner }} 24 | strategy: 25 | fail-fast: false 26 | matrix: 27 | name: [ubuntu-latest] 28 | rust: [stable] 29 | include: 30 | - name: ubuntu-latest 31 | os: ubuntu-latest 32 | release-os: linux 33 | release-arch: amd64 34 | runner: [self-hosted, linux, X64] 35 | env: 36 | SCCACHE_GHA_ENABLED: "true" 37 | RUSTC_WRAPPER: "sccache" 38 | steps: 39 | - name: Checkout 40 | uses: actions/checkout@v4 41 | with: 42 | submodules: recursive 43 | 44 | - name: Install ${{ matrix.rust }} 45 | uses: dtolnay/rust-toolchain@master 46 | with: 47 | toolchain: ${{ matrix.rust }} 48 | components: clippy,rustfmt 49 | 50 | - name: Run sccache-cache 51 | uses: mozilla-actions/sccache-action@v0.0.9 52 | 53 | - name: check 54 | run: | 55 | for i in ${RS_EXAMPLES_LIST//,/ } 56 | do 57 | echo "Checking $i" 58 | cargo check --manifest-path $i/Cargo.toml --all-features 59 | done 60 | env: 61 | RUST_LOG: ${{ runner.debug && 'DEBUG' || 'INFO'}} 62 | 63 | - name: fmt 64 | run: | 65 | for i in ${RS_EXAMPLES_LIST//,/ } 66 | do 67 | echo "Checking $i" 68 | cargo fmt --all --manifest-path $i/Cargo.toml -- --check 69 | done 70 | env: 71 | RUST_LOG: ${{ runner.debug && 'DEBUG' || 'INFO'}} 72 | 73 | - name: clippy 74 | run: | 75 | for i in ${RS_EXAMPLES_LIST//,/ } 76 | do 77 | echo "Checking $i" 78 | cargo clippy --manifest-path $i/Cargo.toml 79 | done 80 | env: 81 | RUST_LOG: ${{ runner.debug && 'DEBUG' || 'INFO'}} 82 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | *iroh-data 3 | */.sendme* 4 | 5 | # Added by cargo 6 | 7 | /target 8 | Cargo.lock 9 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Iroh Experiments 2 | 3 | This is for experiments with iroh by the n0 team. Things in here can be very 4 | low level and unpolished. 5 | 6 | Some of the things in this repo might make it to [iroh-examples] or even into 7 | iroh itself, most will not. 8 | 9 | ## Iroh-dag-sync 10 | 11 | An experiment how we could deal with DAGs in iroh, as well as how to safely 12 | handle non-blake3 hash functions. 13 | 14 | ## Content-discovery 15 | 16 | A complete content disccovery system for iroh, including a tracker, a client 17 | crate to use the tracker, and [pkarr] integration for finding trackers. 18 | 19 | ## Iroh-pkarr-naming-system 20 | 21 | Experiment how to do something similar to [ipns] using [pkarr] and the 22 | bittorrent [mainline] DHT. 23 | 24 | ## Iroh-s3-bao-store 25 | 26 | An iroh-blobs store implementation that keeps the data on s3. Useful to provide 27 | content-addressing to existing public resources. 28 | 29 | [iroh-examples]: https://github.com/n0-computer/iroh-examples 30 | [pkarr]: https://pkarr.org/ 31 | [ipns]: https://docs.ipfs.tech/concepts/ipns/ 32 | [mainline]: https://en.wikipedia.org/wiki/Mainline_DHT 33 | -------------------------------------------------------------------------------- /content-discovery/.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | iroh.config.toml 3 | -------------------------------------------------------------------------------- /content-discovery/Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | members = [ 3 | "iroh-mainline-content-discovery", 4 | "iroh-mainline-content-discovery-cli", 5 | "iroh-mainline-tracker", 6 | ] 7 | resolver = "2" 8 | 9 | [profile.release] 10 | debug = true 11 | 12 | [profile.optimized-release] 13 | inherits = 'release' 14 | debug = false 15 | lto = true 16 | debug-assertions = false 17 | opt-level = 3 18 | panic = 'abort' 19 | incremental = false 20 | 21 | 22 | [workspace.lints.rust] 23 | missing_debug_implementations = "warn" 24 | 25 | [workspace.lints.clippy] 26 | unused-async = "warn" 27 | 28 | [workspace.dependencies] 29 | iroh = { version ="0.35", features = ["discovery-pkarr-dht"] } 30 | iroh-base = "0.35" 31 | iroh-blobs = { version = "0.35", features = ["rpc"] } 32 | # explicitly specified until iroh minimal crates issues are solved, see https://github.com/n0-computer/iroh/pull/3255 33 | tokio = { version = "1.44.1" } 34 | tokio-stream = { version = "0.1.17" } 35 | mainline = { version = "5.4.0", default-features = false } 36 | pkarr = { version = "3.7.0", default-features = false } 37 | postcard = { version = "1", default-features = false } 38 | quinn = { package = "iroh-quinn", version = "0.13", default-features = false } 39 | anyhow = { version = "1", default-features = false } 40 | futures = { version = "0.3.25" } 41 | rcgen = { version = "0.13.1" } 42 | rustls = { version = "0.23", default-features = false, features = ["ring"] } 43 | genawaiter = { version = "0.99.1", features = ["futures03"] } 44 | -------------------------------------------------------------------------------- /content-discovery/README.md: -------------------------------------------------------------------------------- 1 | # Iroh content discovery 2 | 3 | This rust workspace provides global content discovery for iroh. 4 | 5 | *iroh-mainline-content-discovery* is a library that provides a discovery protocol, 6 | a client implementation, and a client command line utility. 7 | 8 | *iroh-mainline-tracker* is a server implementation for the content discovery 9 | protocol. 10 | 11 | ## Building from source 12 | 13 | Make sure you have an up to date version of [rust](https://www.rust-lang.org/) installed. Use the 14 | [rustup](https://rustup.rs/) tool to get the rust compiler `rustc` and build tool 15 | `cargo` for your platform. 16 | 17 | Then run `cargo build --release` from the root directory. The resulting binary 18 | will be in `target/release/iroh-mainline-tracker` 19 | 20 | ## Running the tracker 21 | 22 | ```sh 23 | iroh-mainline-tracker 24 | ``` 25 | 26 | Will run the server with a persistent node id and announce information. 27 | 28 | ## Announcing content 29 | 30 | When announcing content, you can give either iroh tickets or content hashes. 31 | 32 | ```sh 33 | iroh-mainline-content-discovery announce \ 34 | --tracker t3od3nblvk6csozc3oe7rjum7oebnnwwfkebolbxf2o66clzdyha \ 35 | blob:ealcoyhcjxyklzee4manl3b5see3k3nwekf6npw5oollcsflrsduiaicaiafetezhwjouayaycuadbes5ibqaq7qasiyqmqo74ijal7k7ec4pni5htntx4tpoawgvmbhaa3txa4uaa 36 | ``` 37 | 38 | ## Querying content 39 | 40 | When querying content, you can use tickets, hashes, or hash and format. 41 | 42 | When using tickets, the address part of the ticket will be ignored. 43 | 44 | ```sh 45 | iroh-mainline-content-discovery query \ 46 | --tracker t3od3nblvk6csozc3oe7rjum7oebnnwwfkebolbxf2o66clzdyha \ 47 | blob:ealcoyhcjxyklzee4manl3b5see3k3nwekf6npw5oollcsflrsduiaicaiafetezhwjouayaycuadbes5ibqaq7qasiyqmqo74ijal7k7ec4pni5htntx4tpoawgvmbhaa3txa4uaa 48 | ``` 49 | 50 | ## Verification 51 | 52 | Verification works in different ways depending if the content is partial or 53 | complete. 54 | 55 | For partial content, the tracker will just ask for the unverified content size. 56 | That's the only thing you can do for a node that possibly has just started 57 | downloading the content itself. 58 | 59 | For full content and blobs, the tracker will choose a random blake3 chunk of the 60 | data and download it. This is relatively cheap in terms of traffic (2 KiB), and 61 | since the chunk is random a host that has only partial content will be found 62 | eventually. 63 | 64 | For full content and hash sequences such as collections, the tracker will choose 65 | a random chunk of a random child. 66 | -------------------------------------------------------------------------------- /content-discovery/iroh-mainline-content-discovery-cli/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "iroh-mainline-content-discovery-cli" 3 | version = "0.5.0" 4 | edition = "2021" 5 | description = "Content discovery for iroh, using the bittorrent mainline DHT" 6 | license = "MIT OR Apache-2.0" 7 | 8 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 9 | 10 | [dependencies] 11 | iroh = { workspace = true } 12 | iroh-blobs = { workspace = true } 13 | iroh-mainline-content-discovery = { path = "../iroh-mainline-content-discovery" } 14 | mainline = { workspace = true } 15 | anyhow = { workspace = true, features = ["backtrace"] } 16 | futures = { version = "0.3.25" } 17 | clap = { version = "4", features = ["derive"] } 18 | tempfile = { version = "3.4" } 19 | derive_more = { version = "1.0.0-beta.1", features = ["debug", "display", "from", "try_into"] } 20 | tracing-subscriber = { version = "0.3", features = ["env-filter"] } 21 | tokio = { version = "1", features = ["io-util", "rt"] } 22 | -------------------------------------------------------------------------------- /content-discovery/iroh-mainline-content-discovery-cli/src/args.rs: -------------------------------------------------------------------------------- 1 | //! Command line arguments. 2 | use std::{ 3 | fmt::Display, 4 | net::{SocketAddr, SocketAddrV4}, 5 | str::FromStr, 6 | }; 7 | 8 | use clap::{Parser, Subcommand}; 9 | use iroh::NodeId; 10 | use iroh_blobs::{ticket::BlobTicket, Hash, HashAndFormat}; 11 | 12 | #[derive(Parser, Debug)] 13 | pub struct Args { 14 | #[clap(subcommand)] 15 | pub command: Commands, 16 | } 17 | 18 | #[derive(Subcommand, Debug)] 19 | pub enum Commands { 20 | Announce(AnnounceArgs), 21 | Query(QueryArgs), 22 | QueryDht(QueryDhtArgs), 23 | } 24 | 25 | /// Various ways to specify content. 26 | #[derive(Debug, Clone, derive_more::From)] 27 | pub enum ContentArg { 28 | Hash(Hash), 29 | HashAndFormat(HashAndFormat), 30 | Ticket(BlobTicket), 31 | } 32 | 33 | impl ContentArg { 34 | /// Get the hash and format of the content. 35 | pub fn hash_and_format(&self) -> HashAndFormat { 36 | match self { 37 | ContentArg::Hash(hash) => HashAndFormat::raw(*hash), 38 | ContentArg::HashAndFormat(haf) => *haf, 39 | ContentArg::Ticket(ticket) => HashAndFormat { 40 | hash: ticket.hash(), 41 | format: ticket.format(), 42 | }, 43 | } 44 | } 45 | } 46 | 47 | impl Display for ContentArg { 48 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 49 | match self { 50 | ContentArg::Hash(hash) => Display::fmt(hash, f), 51 | ContentArg::HashAndFormat(haf) => Display::fmt(haf, f), 52 | ContentArg::Ticket(ticket) => Display::fmt(ticket, f), 53 | } 54 | } 55 | } 56 | 57 | impl FromStr for ContentArg { 58 | type Err = anyhow::Error; 59 | 60 | fn from_str(s: &str) -> Result { 61 | if let Ok(hash) = Hash::from_str(s) { 62 | Ok(hash.into()) 63 | } else if let Ok(haf) = HashAndFormat::from_str(s) { 64 | Ok(haf.into()) 65 | } else if let Ok(ticket) = BlobTicket::from_str(s) { 66 | Ok(ticket.into()) 67 | } else { 68 | anyhow::bail!("invalid hash and format") 69 | } 70 | } 71 | } 72 | 73 | #[derive(Parser, Debug)] 74 | pub struct AnnounceArgs { 75 | /// trackers to announce to via udp 76 | #[clap(long)] 77 | pub udp_tracker: Vec, 78 | 79 | /// trackers to announce to via quic 80 | #[clap(long)] 81 | pub quic_tracker: Vec, 82 | 83 | /// trackers to announce to via magicsock 84 | #[clap(long)] 85 | pub magicsock_tracker: Vec, 86 | 87 | /// The content to announce. 88 | /// 89 | /// Content can be specified as a hash, a hash and format, or a ticket. 90 | /// If a hash is specified, the format is assumed to be raw. 91 | /// Unless a ticket is specified, the host must be specified. 92 | pub content: ContentArg, 93 | 94 | /// Announce that the peer has only partial data. 95 | #[clap(long)] 96 | pub partial: bool, 97 | 98 | /// the port to use for announcing via udp 99 | #[clap(long)] 100 | pub udp_port: Option, 101 | 102 | /// the ipv4 to use for announcing via iroh 103 | #[clap(long)] 104 | pub iroh_ipv4_addr: Option, 105 | 106 | /// the port to use for announcing via quic 107 | #[clap(long)] 108 | pub quic_port: Option, 109 | } 110 | 111 | #[derive(Parser, Debug)] 112 | pub struct QueryArgs { 113 | /// the tracker to query 114 | #[clap(long)] 115 | pub tracker: Vec, 116 | 117 | /// The content to find hosts for. 118 | pub content: ContentArg, 119 | 120 | /// Ask for hosts that were announced as having just partial data 121 | #[clap(long)] 122 | pub partial: bool, 123 | 124 | /// Ask for hosts that were recently checked and found to have some data 125 | #[clap(long)] 126 | pub verified: bool, 127 | 128 | /// the port to use for querying 129 | #[clap(long)] 130 | pub udp_port: Option, 131 | } 132 | 133 | #[derive(Parser, Debug)] 134 | pub struct QueryDhtArgs { 135 | /// The content to find hosts for. 136 | pub content: ContentArg, 137 | 138 | /// Ask for hosts that were announced as having just partial data 139 | #[clap(long)] 140 | pub partial: bool, 141 | 142 | /// Ask for hosts that were recently checked and found to have some data 143 | #[clap(long)] 144 | pub verified: bool, 145 | 146 | /// Parallelism for querying the dht 147 | #[clap(long)] 148 | pub query_parallelism: Option, 149 | 150 | /// the port to use for querying 151 | #[clap(long)] 152 | pub udp_port: Option, 153 | } 154 | -------------------------------------------------------------------------------- /content-discovery/iroh-mainline-content-discovery-cli/src/main.rs: -------------------------------------------------------------------------------- 1 | pub mod args; 2 | 3 | use std::{ 4 | net::{Ipv4Addr, SocketAddr, SocketAddrV4}, 5 | str::FromStr, 6 | }; 7 | 8 | use args::QueryDhtArgs; 9 | use clap::Parser; 10 | use futures::StreamExt; 11 | use iroh::endpoint; 12 | use iroh_mainline_content_discovery::{ 13 | create_quinn_client, 14 | protocol::{AbsoluteTime, Announce, AnnounceKind, Query, QueryFlags, SignedAnnounce}, 15 | to_infohash, UdpDiscovery, 16 | }; 17 | use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter}; 18 | 19 | use crate::args::{AnnounceArgs, Args, Commands, QueryArgs}; 20 | 21 | async fn announce(args: AnnounceArgs) -> anyhow::Result<()> { 22 | // todo: uncomment once the connection problems are fixed 23 | let Ok(key) = std::env::var("ANNOUNCE_SECRET") else { 24 | eprintln!("ANNOUNCE_SECRET environment variable must be set to a valid secret key"); 25 | anyhow::bail!("ANNOUNCE_SECRET env var not set"); 26 | }; 27 | let Ok(key) = iroh::SecretKey::from_str(&key) else { 28 | anyhow::bail!("ANNOUNCE_SECRET env var is not a valid secret key"); 29 | }; 30 | let content = args.content.hash_and_format(); 31 | let bind_addr = SocketAddr::V4(SocketAddrV4::new( 32 | Ipv4Addr::UNSPECIFIED, 33 | args.udp_port.unwrap_or_default(), 34 | )); 35 | let kind = if args.partial { 36 | AnnounceKind::Partial 37 | } else { 38 | AnnounceKind::Complete 39 | }; 40 | let timestamp = AbsoluteTime::now(); 41 | let announce = Announce { 42 | host: key.public(), 43 | kind, 44 | content, 45 | timestamp, 46 | }; 47 | let signed_announce = SignedAnnounce::new(announce, &key)?; 48 | if !args.udp_tracker.is_empty() { 49 | let discovery = UdpDiscovery::new(bind_addr).await?; 50 | for tracker in args.udp_tracker { 51 | println!("announcing via udp to {:?}: {}", tracker, content); 52 | discovery.add_tracker(tracker).await?; 53 | } 54 | discovery.announce_once(signed_announce).await?; 55 | } 56 | if !args.magicsock_tracker.is_empty() { 57 | let addr = args 58 | .iroh_ipv4_addr 59 | .unwrap_or_else(|| "0.0.0.0:0".parse().unwrap()); 60 | let iroh_endpoint = endpoint::Endpoint::builder() 61 | .bind_addr_v4(addr) 62 | .bind() 63 | .await?; 64 | for tracker in args.magicsock_tracker { 65 | println!("announcing via magicsock to {:?}: {}", tracker, content); 66 | let connection = iroh_endpoint 67 | .connect(tracker, iroh_mainline_content_discovery::protocol::ALPN) 68 | .await?; 69 | iroh_mainline_content_discovery::announce_iroh(connection, signed_announce).await?; 70 | } 71 | } 72 | if !args.quic_tracker.is_empty() { 73 | let bind_addr = SocketAddr::V4(SocketAddrV4::new( 74 | Ipv4Addr::UNSPECIFIED, 75 | args.quic_port.unwrap_or_default(), 76 | )); 77 | let quinn_endpoint = create_quinn_client( 78 | bind_addr, 79 | vec![iroh_mainline_content_discovery::protocol::ALPN.to_vec()], 80 | false, 81 | )?; 82 | for tracker in args.quic_tracker { 83 | println!("announcing via quic to {:?}: {}", tracker, content); 84 | let connection = quinn_endpoint.connect(tracker, "localhost")?.await?; 85 | iroh_mainline_content_discovery::announce_quinn(connection, signed_announce).await?; 86 | } 87 | } 88 | 89 | println!("done"); 90 | Ok(()) 91 | } 92 | 93 | async fn query(args: QueryArgs) -> anyhow::Result<()> { 94 | let bind_addr = SocketAddr::V4(SocketAddrV4::new( 95 | Ipv4Addr::UNSPECIFIED, 96 | args.udp_port.unwrap_or_default(), 97 | )); 98 | let discovery = iroh_mainline_content_discovery::UdpDiscovery::new(bind_addr).await?; 99 | for tracker in args.tracker { 100 | discovery.add_tracker(tracker).await?; 101 | } 102 | let q = Query { 103 | content: args.content.hash_and_format(), 104 | flags: QueryFlags { 105 | complete: !args.partial, 106 | verified: args.verified, 107 | }, 108 | }; 109 | let mut res = discovery.query(q).await?; 110 | while let Some(sa) = res.recv().await { 111 | if sa.verify().is_ok() { 112 | println!("{}: {:?}", sa.announce.host, sa.announce.kind); 113 | } else { 114 | println!("invalid announce"); 115 | } 116 | } 117 | Ok(()) 118 | } 119 | 120 | async fn query_dht(args: QueryDhtArgs) -> anyhow::Result<()> { 121 | let bind_addr = SocketAddr::V4(SocketAddrV4::new( 122 | Ipv4Addr::UNSPECIFIED, 123 | args.udp_port.unwrap_or_default(), 124 | )); 125 | let discovery = UdpDiscovery::new(bind_addr).await?; 126 | let dht = mainline::Dht::client()?; 127 | let q = Query { 128 | content: args.content.hash_and_format(), 129 | flags: QueryFlags { 130 | complete: !args.partial, 131 | verified: args.verified, 132 | }, 133 | }; 134 | println!("content corresponds to infohash {}", to_infohash(q.content)); 135 | 136 | let mut stream = discovery.query_dht(dht, q).await?; 137 | while let Some(announce) = stream.next().await { 138 | if announce.verify().is_ok() { 139 | println!("found verified provider {}", announce.host); 140 | } else { 141 | println!("got wrong signed announce!"); 142 | } 143 | } 144 | Ok(()) 145 | } 146 | 147 | // set the RUST_LOG env var to one of {debug,info,warn} to see logging info 148 | pub fn setup_logging() { 149 | tracing_subscriber::registry() 150 | .with(tracing_subscriber::fmt::layer().with_writer(std::io::stderr)) 151 | .with(EnvFilter::from_default_env()) 152 | .try_init() 153 | .ok(); 154 | } 155 | 156 | #[tokio::main(flavor = "multi_thread")] 157 | async fn main() -> anyhow::Result<()> { 158 | setup_logging(); 159 | let args = Args::parse(); 160 | match args.command { 161 | Commands::Announce(args) => announce(args).await, 162 | Commands::Query(args) => query(args).await, 163 | Commands::QueryDht(args) => query_dht(args).await, 164 | } 165 | } 166 | -------------------------------------------------------------------------------- /content-discovery/iroh-mainline-content-discovery/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "iroh-mainline-content-discovery" 3 | version = "0.6.0" 4 | edition = "2021" 5 | description = "Content discovery for iroh, using the bittorrent mainline DHT" 6 | license = "MIT OR Apache-2.0" 7 | 8 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 9 | 10 | [dependencies] 11 | # Required features for the protocol types. 12 | # 13 | # The protocol is using postcard, but we don't need a postcard dependency for just the type definitions 14 | iroh = { workspace = true } 15 | iroh-base = { workspace = true } 16 | iroh-blobs = { workspace = true } 17 | rand = "0.8.5" 18 | serde = { version = "1", features = ["derive"] } 19 | derive_more = { version = "1.0.0-beta.1", features = ["debug", "display", "from", "try_into"] } 20 | serde-big-array = "0.5.1" 21 | hex = "0.4.3" 22 | 23 | # Optional features for the client functionality 24 | tracing = { version = "0.1", optional = true } 25 | quinn = { workspace = true, optional = true } 26 | mainline = { workspace = true, optional = true, features = ["async"] } 27 | anyhow = { workspace = true, features = ["backtrace"], optional = true } 28 | postcard = { workspace = true, features = ["alloc", "use-std"], optional = true } 29 | futures = { workspace = true, optional = true } 30 | rcgen = { workspace = true, optional = true } 31 | rustls = { workspace = true, features = ["ring"], optional = true } 32 | genawaiter = { version = "0.99.1", features = ["futures03"], optional = true } 33 | tokio = { workspace = true, optional = true } 34 | tokio-stream = { workspace = true } 35 | 36 | # dependencies for the tls utils 37 | der = { version = "0.7", features = ["alloc", "derive"], optional = true } 38 | webpki = { package = "rustls-webpki", version = "0.102", optional = true } 39 | x509-parser = { version = "0.16", optional = true } 40 | thiserror = { version = "2", optional = true } 41 | ring = { version = "0.17", optional = true } 42 | 43 | [features] 44 | client = [ 45 | "dep:mainline", 46 | "dep:quinn", 47 | "dep:tracing", 48 | "dep:anyhow", 49 | "dep:rcgen", 50 | "dep:genawaiter", 51 | "dep:rustls", 52 | "dep:futures", 53 | "dep:postcard", 54 | "dep:tokio", 55 | "tls-utils", 56 | ] 57 | tls-utils = [ 58 | "dep:der", 59 | "dep:webpki", 60 | "dep:x509-parser", 61 | "dep:thiserror", 62 | "dep:ring", 63 | ] 64 | default = ["client"] 65 | -------------------------------------------------------------------------------- /content-discovery/iroh-mainline-content-discovery/README.md: -------------------------------------------------------------------------------- 1 | # Protocol and client for iroh mainline content discovery 2 | 3 | This provides a very minimal protocol for content discovery as well as a 4 | client library for the protocol. 5 | 6 | ## Features 7 | 8 | - client: the client that allows querying content discovery 9 | - tls-utils: utilities to set of quinn connections, used by client 10 | -------------------------------------------------------------------------------- /content-discovery/iroh-mainline-content-discovery/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! A library for discovering content using the mainline DHT. 2 | //! 3 | //! This library contains the protocol for announcing and querying content, as 4 | //! well as the client side implementation and a few helpers for p2p quinn 5 | //! connections. 6 | #[cfg(feature = "client")] 7 | mod client; 8 | pub mod protocol; 9 | #[cfg(feature = "client")] 10 | pub use client::*; 11 | #[cfg(feature = "tls-utils")] 12 | pub mod tls_utils; 13 | -------------------------------------------------------------------------------- /content-discovery/iroh-mainline-content-discovery/src/protocol.rs: -------------------------------------------------------------------------------- 1 | //! The protocol for communicating with the tracker. 2 | use std::{ 3 | ops::{Deref, Sub}, 4 | time::{Duration, SystemTime}, 5 | }; 6 | 7 | use iroh::NodeId; 8 | use iroh_blobs::HashAndFormat; 9 | use serde::{Deserialize, Serialize}; 10 | use serde_big_array::BigArray; 11 | 12 | /// The ALPN string for this protocol 13 | pub const ALPN: &[u8] = b"n0/tracker/1"; 14 | /// Maximum size of a request 15 | pub const REQUEST_SIZE_LIMIT: usize = 1024 * 16; 16 | 17 | /// Announce kind 18 | #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] 19 | pub enum AnnounceKind { 20 | /// The peer supposedly has some of the data. 21 | Partial = 0, 22 | /// The peer supposedly has the complete data. 23 | Complete, 24 | } 25 | 26 | impl AnnounceKind { 27 | pub fn from_complete(complete: bool) -> Self { 28 | if complete { 29 | Self::Complete 30 | } else { 31 | Self::Partial 32 | } 33 | } 34 | } 35 | 36 | #[derive(Debug, Clone, Copy, Serialize, Deserialize, Default, PartialEq, Eq, PartialOrd, Ord)] 37 | pub struct AbsoluteTime(u64); 38 | 39 | impl AbsoluteTime { 40 | pub fn now() -> Self { 41 | Self::try_from(SystemTime::now()).unwrap() 42 | } 43 | 44 | pub fn from_micros(micros: u64) -> Self { 45 | Self(micros) 46 | } 47 | 48 | pub fn as_micros(&self) -> u64 { 49 | self.0 50 | } 51 | } 52 | 53 | impl Sub for AbsoluteTime { 54 | type Output = Duration; 55 | 56 | fn sub(self, rhs: Self) -> Self::Output { 57 | Duration::from_micros(self.0 - rhs.0) 58 | } 59 | } 60 | 61 | impl TryFrom for AbsoluteTime { 62 | type Error = anyhow::Error; 63 | 64 | fn try_from(value: SystemTime) -> Result { 65 | Ok(Self( 66 | value 67 | .duration_since(std::time::UNIX_EPOCH) 68 | .expect("Time went backwards") 69 | .as_micros() 70 | .try_into() 71 | .expect("time too large"), 72 | )) 73 | } 74 | } 75 | 76 | impl From for SystemTime { 77 | fn from(value: AbsoluteTime) -> Self { 78 | std::time::UNIX_EPOCH + Duration::from_micros(value.0) 79 | } 80 | } 81 | 82 | /// Announce that a peer claims to have some blobs or set of blobs. 83 | /// 84 | /// A peer can announce having some data, but it should also be able to announce 85 | /// that another peer has the data. This is why the peer is included. 86 | #[derive(Debug, Clone, Copy, Serialize, Deserialize)] 87 | pub struct Announce { 88 | /// The peer that supposedly has the data. 89 | pub host: NodeId, 90 | /// The content that the peer claims to have. 91 | pub content: HashAndFormat, 92 | /// The kind of the announcement. 93 | pub kind: AnnounceKind, 94 | /// The timestamp of the announce. 95 | pub timestamp: AbsoluteTime, 96 | } 97 | 98 | /// A signed announce. 99 | #[derive(derive_more::Debug, Clone, Copy, Serialize, Deserialize)] 100 | pub struct SignedAnnounce { 101 | /// Announce. 102 | pub announce: Announce, 103 | /// Signature of the announce, signed by the host of the announce. 104 | /// 105 | /// The signature is over the announce, serialized with postcard. 106 | #[serde(with = "BigArray")] 107 | #[debug("{}", hex::encode(self.signature))] 108 | pub signature: [u8; 64], 109 | } 110 | 111 | impl Deref for SignedAnnounce { 112 | type Target = Announce; 113 | 114 | fn deref(&self) -> &Self::Target { 115 | &self.announce 116 | } 117 | } 118 | 119 | impl SignedAnnounce { 120 | /// Create a new signed announce. 121 | pub fn new(announce: Announce, secret_key: &iroh::SecretKey) -> anyhow::Result { 122 | let announce_bytes = postcard::to_allocvec(&announce)?; 123 | let signature = secret_key.sign(&announce_bytes).to_bytes(); 124 | Ok(Self { 125 | announce, 126 | signature, 127 | }) 128 | } 129 | 130 | /// Verify the announce, and return the announce if it's valid. 131 | pub fn verify(&self) -> anyhow::Result<()> { 132 | let announce_bytes = postcard::to_allocvec(&self.announce)?; 133 | let signature = iroh_base::Signature::from_bytes(&self.signature); 134 | self.announce.host.verify(&announce_bytes, &signature)?; 135 | Ok(()) 136 | } 137 | } 138 | 139 | /// Flags for a query. 140 | #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] 141 | pub struct QueryFlags { 142 | /// Only return peers that supposedly have the complete data. 143 | /// 144 | /// If this is false, the response might contain peers that only have some of the data. 145 | pub complete: bool, 146 | 147 | /// Only return hosts that have been verified. 148 | /// 149 | /// In case of a partial query, verification just means a check that the host exists 150 | /// and returns the size for the data. 151 | /// 152 | /// In case of a complete query, verification means that the host has been randomly 153 | /// probed for the data. 154 | pub verified: bool, 155 | } 156 | 157 | /// Query a peer for a blob or set of blobs. 158 | #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] 159 | pub struct Query { 160 | /// The content we want to find. 161 | /// 162 | /// It's a difference if a peer has a blob or a hash seq and all of its children. 163 | pub content: HashAndFormat, 164 | /// The mode of the query. 165 | pub flags: QueryFlags, 166 | } 167 | 168 | /// A response to a query. 169 | #[derive(Debug, Clone, Serialize, Deserialize)] 170 | pub struct QueryResponse { 171 | /// The hosts that supposedly have the content. 172 | /// 173 | /// If there are any addrs, they are as seen from the tracker, 174 | /// so they might or might not be useful. 175 | pub hosts: Vec, 176 | } 177 | 178 | /// A request to the tracker. 179 | #[derive(Debug, Clone, Serialize, Deserialize)] 180 | pub enum Request { 181 | /// Announce info 182 | Announce(SignedAnnounce), 183 | /// Query info 184 | Query(Query), 185 | } 186 | 187 | /// A response from the tracker. 188 | #[derive(Debug, Clone, Serialize, Deserialize)] 189 | pub enum Response { 190 | /// Response to a query 191 | QueryResponse(QueryResponse), 192 | } 193 | 194 | #[cfg(test)] 195 | mod tests {} 196 | -------------------------------------------------------------------------------- /content-discovery/iroh-mainline-content-discovery/src/tls_utils/certificate.rs: -------------------------------------------------------------------------------- 1 | //! X.509 certificate handling. 2 | //! 3 | //! This module handles generation, signing, and verification of certificates. 4 | //! 5 | //! Based on rust-libp2p/transports/tls/src/certificate.rs originally licensed under MIT by Parity 6 | //! Technologies (UK) Ltd. 7 | 8 | use std::sync::Arc; 9 | 10 | use der::{asn1::OctetStringRef, Decode, Encode, Sequence}; 11 | use iroh_base::{PublicKey, SecretKey, Signature}; 12 | use x509_parser::prelude::*; 13 | 14 | /// The libp2p Public Key Extension is a X.509 extension 15 | /// with the Object Identifier 1.3.6.1.4.1.53594.1.1, 16 | /// allocated by IANA to the libp2p project at Protocol Labs. 17 | const P2P_EXT_OID: [u64; 9] = [1, 3, 6, 1, 4, 1, 53594, 1, 1]; 18 | 19 | /// The peer signs the concatenation of the string `libp2p-tls-handshake:` 20 | /// and the public key that it used to generate the certificate carrying 21 | /// the libp2p Public Key Extension, using its private host key. 22 | /// This signature provides cryptographic proof that the peer was 23 | /// in possession of the private host key at the time the certificate was signed. 24 | const P2P_SIGNING_PREFIX: [u8; 21] = *b"libp2p-tls-handshake:"; 25 | 26 | // Certificates MUST use the NamedCurve encoding for elliptic curve parameters. 27 | // Similarly, hash functions with an output length less than 256 bits MUST NOT be used. 28 | static P2P_SIGNATURE_ALGORITHM: &rcgen::SignatureAlgorithm = &rcgen::PKCS_ECDSA_P256_SHA256; 29 | 30 | #[derive(Debug)] 31 | pub(crate) struct AlwaysResolvesCert(Arc); 32 | 33 | impl AlwaysResolvesCert { 34 | pub(crate) fn new( 35 | cert: rustls::pki_types::CertificateDer<'static>, 36 | key: &rustls::pki_types::PrivateKeyDer<'_>, 37 | ) -> Result { 38 | let certified_key = rustls::sign::CertifiedKey::new( 39 | vec![cert], 40 | rustls::crypto::ring::sign::any_ecdsa_type(key)?, 41 | ); 42 | Ok(Self(Arc::new(certified_key))) 43 | } 44 | } 45 | 46 | impl rustls::client::ResolvesClientCert for AlwaysResolvesCert { 47 | fn resolve( 48 | &self, 49 | _root_hint_subjects: &[&[u8]], 50 | _sigschemes: &[rustls::SignatureScheme], 51 | ) -> Option> { 52 | Some(Arc::clone(&self.0)) 53 | } 54 | 55 | fn has_certs(&self) -> bool { 56 | true 57 | } 58 | } 59 | 60 | impl rustls::server::ResolvesServerCert for AlwaysResolvesCert { 61 | fn resolve( 62 | &self, 63 | _client_hello: rustls::server::ClientHello<'_>, 64 | ) -> Option> { 65 | Some(Arc::clone(&self.0)) 66 | } 67 | } 68 | 69 | /// The public host key and the signature are ANS.1-encoded 70 | /// into the SignedKey data structure, which is carried in the libp2p Public Key Extension. 71 | #[derive(Clone, Debug, Eq, PartialEq, Sequence)] 72 | struct SignedKey<'a> { 73 | public_key: OctetStringRef<'a>, 74 | signature: OctetStringRef<'a>, 75 | } 76 | 77 | /// Generates a self-signed TLS certificate that includes a libp2p-specific 78 | /// certificate extension containing the public key of the given secret key. 79 | pub fn generate( 80 | identity_secret_key: &SecretKey, 81 | ) -> Result< 82 | ( 83 | rustls::pki_types::CertificateDer<'static>, 84 | rustls::pki_types::PrivateKeyDer<'static>, 85 | ), 86 | GenError, 87 | > { 88 | // SecretKey used to sign the certificate. 89 | // SHOULD NOT be related to the host's key. 90 | // Endpoints MAY generate a new key and certificate 91 | // for every connection attempt, or they MAY reuse the same key 92 | // and certificate for multiple connections. 93 | let certificate_keypair = rcgen::KeyPair::generate_for(P2P_SIGNATURE_ALGORITHM)?; 94 | let rustls_key = 95 | rustls::pki_types::PrivateKeyDer::try_from(certificate_keypair.serialize_der()) 96 | .expect("checked"); 97 | let certificate = { 98 | let mut params = rcgen::CertificateParams::default(); 99 | params.distinguished_name = rcgen::DistinguishedName::new(); 100 | params.custom_extensions.push(make_libp2p_extension( 101 | identity_secret_key, 102 | &certificate_keypair, 103 | )?); 104 | params 105 | .self_signed(&certificate_keypair) 106 | .expect("self signed certificate to be generated") 107 | }; 108 | 109 | Ok((certificate.der().clone(), rustls_key)) 110 | } 111 | 112 | /// Attempts to parse the provided bytes as a [`P2pCertificate`]. 113 | /// 114 | /// For this to succeed, the certificate must contain the specified extension and the signature must 115 | /// match the embedded public key. 116 | pub fn parse<'a>( 117 | certificate: &'a rustls::pki_types::CertificateDer<'_>, 118 | ) -> Result, ParseError> { 119 | let certificate = parse_unverified(certificate.as_ref())?; 120 | 121 | certificate.verify()?; 122 | 123 | Ok(certificate) 124 | } 125 | 126 | /// An X.509 certificate with a libp2p-specific extension 127 | /// is used to secure libp2p connections. 128 | #[derive(Debug)] 129 | pub struct P2pCertificate<'a> { 130 | certificate: X509Certificate<'a>, 131 | /// This is a specific libp2p Public Key Extension with two values: 132 | /// * the public host key 133 | /// * a signature performed using the private host key 134 | extension: P2pExtension, 135 | } 136 | 137 | /// The contents of the specific libp2p extension, containing the public host key 138 | /// and a signature performed using the private host key. 139 | #[derive(Debug)] 140 | pub struct P2pExtension { 141 | public_key: PublicKey, 142 | /// This signature provides cryptographic proof that the peer was 143 | /// in possession of the private host key at the time the certificate was signed. 144 | signature: Signature, 145 | } 146 | 147 | /// An error that occurs during certificate generation. 148 | #[derive(Debug, thiserror::Error)] 149 | #[error(transparent)] 150 | pub struct GenError(#[from] rcgen::Error); 151 | 152 | /// An error that occurs during certificate parsing. 153 | #[derive(Debug, thiserror::Error)] 154 | #[error(transparent)] 155 | pub struct ParseError(#[from] pub(crate) webpki::Error); 156 | 157 | /// An error that occurs during signature verification. 158 | #[derive(Debug, thiserror::Error)] 159 | #[error(transparent)] 160 | pub struct VerificationError(#[from] pub(crate) webpki::Error); 161 | 162 | /// Internal function that only parses but does not verify the certificate. 163 | /// 164 | /// Useful for testing but unsuitable for production. 165 | fn parse_unverified(der_input: &[u8]) -> Result { 166 | let x509 = X509Certificate::from_der(der_input) 167 | .map(|(_rest_input, x509)| x509) 168 | .map_err(|_| webpki::Error::BadDer)?; 169 | 170 | let p2p_ext_oid = der_parser::oid::Oid::from(&P2P_EXT_OID) 171 | .expect("This is a valid OID of p2p extension; qed"); 172 | 173 | let mut libp2p_extension = None; 174 | 175 | for ext in x509.extensions() { 176 | let oid = &ext.oid; 177 | if oid == &p2p_ext_oid && libp2p_extension.is_some() { 178 | // The extension was already parsed 179 | return Err(webpki::Error::BadDer); 180 | } 181 | 182 | if oid == &p2p_ext_oid { 183 | let signed_key = 184 | SignedKey::from_der(ext.value).map_err(|_| webpki::Error::ExtensionValueInvalid)?; 185 | let public_key_raw = signed_key.public_key.as_bytes(); 186 | let public_key = 187 | PublicKey::try_from(public_key_raw).map_err(|_| webpki::Error::UnknownIssuer)?; 188 | 189 | let signature = Signature::from_slice(signed_key.signature.as_bytes()) 190 | .map_err(|_| webpki::Error::UnknownIssuer)?; 191 | let ext = P2pExtension { 192 | public_key, 193 | signature, 194 | }; 195 | libp2p_extension = Some(ext); 196 | continue; 197 | } 198 | 199 | if ext.critical { 200 | // Endpoints MUST abort the connection attempt if the certificate 201 | // contains critical extensions that the endpoint does not understand. 202 | return Err(webpki::Error::UnsupportedCriticalExtension); 203 | } 204 | 205 | // Implementations MUST ignore non-critical extensions with unknown OIDs. 206 | } 207 | 208 | // The certificate MUST contain the libp2p Public Key Extension. 209 | // If this extension is missing, endpoints MUST abort the connection attempt. 210 | let extension = libp2p_extension.ok_or(webpki::Error::BadDer)?; 211 | 212 | let certificate = P2pCertificate { 213 | certificate: x509, 214 | extension, 215 | }; 216 | 217 | Ok(certificate) 218 | } 219 | 220 | fn make_libp2p_extension( 221 | identity_secret_key: &SecretKey, 222 | certificate_keypair: &rcgen::KeyPair, 223 | ) -> Result { 224 | // The peer signs the concatenation of the string `libp2p-tls-handshake:` 225 | // and the public key that it used to generate the certificate carrying 226 | // the libp2p Public Key Extension, using its private host key. 227 | let signature = { 228 | let mut msg = vec![]; 229 | msg.extend(P2P_SIGNING_PREFIX); 230 | msg.extend(certificate_keypair.public_key_der()); 231 | 232 | identity_secret_key.sign(&msg) 233 | }; 234 | 235 | let public_key = identity_secret_key.public(); 236 | let public_key_ref = OctetStringRef::new(&public_key.as_bytes()[..]) 237 | .map_err(|_| rcgen::Error::CouldNotParseKeyPair)?; 238 | let signature = signature.to_bytes(); 239 | let signature_ref = 240 | OctetStringRef::new(&signature).map_err(|_| rcgen::Error::CouldNotParseCertificate)?; 241 | let key = SignedKey { 242 | public_key: public_key_ref, 243 | signature: signature_ref, 244 | }; 245 | 246 | let mut extension_content = Vec::new(); 247 | key.encode_to_vec(&mut extension_content).expect("vec"); 248 | 249 | // This extension MAY be marked critical. 250 | let mut ext = rcgen::CustomExtension::from_oid_content(&P2P_EXT_OID, extension_content); 251 | ext.set_criticality(true); 252 | 253 | Ok(ext) 254 | } 255 | 256 | impl P2pCertificate<'_> { 257 | /// The [`PublicKey`] of the remote peer. 258 | pub fn peer_id(&self) -> PublicKey { 259 | self.extension.public_key 260 | } 261 | 262 | /// Verify the `signature` of the `message` signed by the secret key corresponding to the public key stored 263 | /// in the certificate. 264 | pub fn verify_signature( 265 | &self, 266 | signature_scheme: rustls::SignatureScheme, 267 | message: &[u8], 268 | signature: &[u8], 269 | ) -> Result<(), VerificationError> { 270 | let pk = self.public_key(signature_scheme)?; 271 | pk.verify(message, signature) 272 | .map_err(|_| webpki::Error::InvalidSignatureForPublicKey)?; 273 | 274 | Ok(()) 275 | } 276 | 277 | /// Get a [`ring::signature::UnparsedPublicKey`] for this `signature_scheme`. 278 | /// Return `Error` if the `signature_scheme` does not match the public key signature 279 | /// and hashing algorithm or if the `signature_scheme` is not supported. 280 | fn public_key( 281 | &self, 282 | signature_scheme: rustls::SignatureScheme, 283 | ) -> Result, webpki::Error> { 284 | use ring::signature; 285 | use rustls::SignatureScheme::*; 286 | 287 | let current_signature_scheme = self.signature_scheme()?; 288 | if signature_scheme != current_signature_scheme { 289 | // This certificate was signed with a different signature scheme 290 | return Err(webpki::Error::UnsupportedSignatureAlgorithmForPublicKey); 291 | } 292 | 293 | let verification_algorithm: &dyn signature::VerificationAlgorithm = match signature_scheme { 294 | ECDSA_NISTP256_SHA256 => &signature::ECDSA_P256_SHA256_ASN1, 295 | ECDSA_NISTP384_SHA384 => &signature::ECDSA_P384_SHA384_ASN1, 296 | ECDSA_NISTP521_SHA512 => { 297 | // See https://github.com/briansmith/ring/issues/824 298 | return Err(webpki::Error::UnsupportedSignatureAlgorithm); 299 | } 300 | ED25519 => &signature::ED25519, 301 | ED448 => { 302 | // See https://github.com/briansmith/ring/issues/463 303 | return Err(webpki::Error::UnsupportedSignatureAlgorithm); 304 | } 305 | // No support for RSA 306 | RSA_PKCS1_SHA256 | RSA_PKCS1_SHA384 | RSA_PKCS1_SHA512 | RSA_PSS_SHA256 307 | | RSA_PSS_SHA384 | RSA_PSS_SHA512 => { 308 | return Err(webpki::Error::UnsupportedSignatureAlgorithm) 309 | } 310 | // Similarly, hash functions with an output length less than 256 bits 311 | // MUST NOT be used, due to the possibility of collision attacks. 312 | // In particular, MD5 and SHA1 MUST NOT be used. 313 | RSA_PKCS1_SHA1 => return Err(webpki::Error::UnsupportedSignatureAlgorithm), 314 | ECDSA_SHA1_Legacy => return Err(webpki::Error::UnsupportedSignatureAlgorithm), 315 | Unknown(_) => return Err(webpki::Error::UnsupportedSignatureAlgorithm), 316 | _ => return Err(webpki::Error::UnsupportedSignatureAlgorithm), 317 | }; 318 | let spki = &self.certificate.tbs_certificate.subject_pki; 319 | let key = signature::UnparsedPublicKey::new( 320 | verification_algorithm, 321 | spki.subject_public_key.as_ref(), 322 | ); 323 | 324 | Ok(key) 325 | } 326 | 327 | /// This method validates the certificate according to libp2p TLS 1.3 specs. 328 | /// The certificate MUST: 329 | /// 1. be valid at the time it is received by the peer; 330 | /// 2. use the NamedCurve encoding; 331 | /// 3. use hash functions with an output length not less than 256 bits; 332 | /// 4. be self signed; 333 | /// 5. contain a valid signature in the specific libp2p extension. 334 | fn verify(&self) -> Result<(), webpki::Error> { 335 | use webpki::Error; 336 | 337 | // The certificate MUST have NotBefore and NotAfter fields set 338 | // such that the certificate is valid at the time it is received by the peer. 339 | if !self.certificate.validity().is_valid() { 340 | return Err(Error::InvalidCertValidity); 341 | } 342 | 343 | // Certificates MUST use the NamedCurve encoding for elliptic curve parameters. 344 | // Similarly, hash functions with an output length less than 256 bits 345 | // MUST NOT be used, due to the possibility of collision attacks. 346 | // In particular, MD5 and SHA1 MUST NOT be used. 347 | // Endpoints MUST abort the connection attempt if it is not used. 348 | let signature_scheme = self.signature_scheme()?; 349 | // Endpoints MUST abort the connection attempt if the certificate’s 350 | // self-signature is not valid. 351 | let raw_certificate = self.certificate.tbs_certificate.as_ref(); 352 | let signature = self.certificate.signature_value.as_ref(); 353 | // check if self signed 354 | self.verify_signature(signature_scheme, raw_certificate, signature) 355 | .map_err(|_| Error::SignatureAlgorithmMismatch)?; 356 | 357 | let subject_pki = self.certificate.public_key().raw; 358 | 359 | // The peer signs the concatenation of the string `libp2p-tls-handshake:` 360 | // and the public key that it used to generate the certificate carrying 361 | // the libp2p Public Key Extension, using its private host key. 362 | let mut msg = vec![]; 363 | msg.extend(P2P_SIGNING_PREFIX); 364 | msg.extend(subject_pki); 365 | 366 | // This signature provides cryptographic proof that the peer was in possession 367 | // of the private host key at the time the certificate was signed. 368 | // Peers MUST verify the signature, and abort the connection attempt 369 | // if signature verification fails. 370 | let user_owns_sk = self 371 | .extension 372 | .public_key 373 | .verify(&msg, &self.extension.signature) 374 | .is_ok(); 375 | if !user_owns_sk { 376 | return Err(Error::UnknownIssuer); 377 | } 378 | 379 | Ok(()) 380 | } 381 | 382 | /// Return the signature scheme corresponding to [`AlgorithmIdentifier`]s 383 | /// of `subject_pki` and `signature_algorithm` 384 | /// according to . 385 | fn signature_scheme(&self) -> Result { 386 | // Certificates MUST use the NamedCurve encoding for elliptic curve parameters. 387 | // Endpoints MUST abort the connection attempt if it is not used. 388 | use oid_registry::*; 389 | use rustls::SignatureScheme::*; 390 | 391 | let signature_algorithm = &self.certificate.signature_algorithm; 392 | let pki_algorithm = &self.certificate.tbs_certificate.subject_pki.algorithm; 393 | 394 | if pki_algorithm.algorithm == OID_KEY_TYPE_EC_PUBLIC_KEY { 395 | let signature_param = pki_algorithm 396 | .parameters 397 | .as_ref() 398 | .ok_or(webpki::Error::BadDer)? 399 | .as_oid() 400 | .map_err(|_| webpki::Error::BadDer)?; 401 | if signature_param == OID_EC_P256 402 | && signature_algorithm.algorithm == OID_SIG_ECDSA_WITH_SHA256 403 | { 404 | return Ok(ECDSA_NISTP256_SHA256); 405 | } 406 | if signature_param == OID_NIST_EC_P384 407 | && signature_algorithm.algorithm == OID_SIG_ECDSA_WITH_SHA384 408 | { 409 | return Ok(ECDSA_NISTP384_SHA384); 410 | } 411 | if signature_param == OID_NIST_EC_P521 412 | && signature_algorithm.algorithm == OID_SIG_ECDSA_WITH_SHA512 413 | { 414 | return Ok(ECDSA_NISTP521_SHA512); 415 | } 416 | return Err(webpki::Error::UnsupportedSignatureAlgorithm); 417 | } 418 | 419 | if signature_algorithm.algorithm == OID_SIG_ED25519 { 420 | return Ok(ED25519); 421 | } 422 | if signature_algorithm.algorithm == OID_SIG_ED448 { 423 | return Ok(ED448); 424 | } 425 | 426 | Err(webpki::Error::UnsupportedSignatureAlgorithm) 427 | } 428 | } 429 | 430 | #[cfg(test)] 431 | mod tests { 432 | use super::*; 433 | 434 | #[test] 435 | fn sanity_check() { 436 | let secret_key = SecretKey::generate(rand::thread_rng()); 437 | 438 | let (cert, _) = generate(&secret_key).unwrap(); 439 | let parsed_cert = parse(&cert).unwrap(); 440 | 441 | assert!(parsed_cert.verify().is_ok()); 442 | assert_eq!(secret_key.public(), parsed_cert.extension.public_key); 443 | } 444 | } 445 | -------------------------------------------------------------------------------- /content-discovery/iroh-mainline-content-discovery/src/tls_utils/mod.rs: -------------------------------------------------------------------------------- 1 | //! TLS configuration based on libp2p TLS specs. 2 | //! 3 | //! See . 4 | //! Based on rust-libp2p/transports/tls 5 | 6 | use std::sync::Arc; 7 | 8 | use iroh_base::{PublicKey, SecretKey}; 9 | use quinn::crypto::rustls::{NoInitialCipherSuite, QuicClientConfig, QuicServerConfig}; 10 | use tracing::warn; 11 | 12 | use self::certificate::AlwaysResolvesCert; 13 | 14 | pub mod certificate; 15 | mod verifier; 16 | 17 | /// Error for generating iroh p2p TLS configs. 18 | #[derive(Debug, thiserror::Error)] 19 | pub enum CreateConfigError { 20 | /// Error generating the certificate. 21 | #[error("Error generating the certificate")] 22 | CertError(#[from] certificate::GenError), 23 | /// Error creating QUIC config. 24 | #[error("Error creating QUIC config")] 25 | ConfigError(#[from] NoInitialCipherSuite), 26 | } 27 | 28 | /// Create a TLS client configuration. 29 | /// 30 | /// If *keylog* is `true` this will enable logging of the pre-master key to the file in the 31 | /// `SSLKEYLOGFILE` environment variable. This can be used to inspect the traffic for 32 | /// debugging purposes. 33 | pub fn make_client_config( 34 | secret_key: &SecretKey, 35 | remote_peer_id: Option, 36 | alpn_protocols: Vec>, 37 | keylog: bool, 38 | ) -> Result { 39 | let (certificate, secret_key) = certificate::generate(secret_key)?; 40 | 41 | let cert_resolver = Arc::new( 42 | AlwaysResolvesCert::new(certificate, &secret_key) 43 | .expect("Client cert key DER is valid; qed"), 44 | ); 45 | 46 | let mut crypto = rustls::ClientConfig::builder_with_provider(Arc::new( 47 | rustls::crypto::ring::default_provider(), 48 | )) 49 | .with_protocol_versions(verifier::PROTOCOL_VERSIONS) 50 | .expect("version supported by ring") 51 | .dangerous() 52 | .with_custom_certificate_verifier(Arc::new( 53 | verifier::Libp2pCertificateVerifier::with_remote_peer_id(remote_peer_id), 54 | )) 55 | .with_client_cert_resolver(cert_resolver); 56 | crypto.alpn_protocols = alpn_protocols; 57 | if keylog { 58 | warn!("enabling SSLKEYLOGFILE for TLS pre-master keys"); 59 | crypto.key_log = Arc::new(rustls::KeyLogFile::new()); 60 | } 61 | let config = crypto.try_into()?; 62 | Ok(config) 63 | } 64 | 65 | /// Create a TLS server configuration. 66 | /// 67 | /// If *keylog* is `true` this will enable logging of the pre-master key to the file in the 68 | /// `SSLKEYLOGFILE` environment variable. This can be used to inspect the traffic for 69 | /// debugging purposes. 70 | pub fn make_server_config( 71 | secret_key: &SecretKey, 72 | alpn_protocols: Vec>, 73 | keylog: bool, 74 | ) -> Result { 75 | let (certificate, secret_key) = certificate::generate(secret_key)?; 76 | 77 | let cert_resolver = Arc::new( 78 | AlwaysResolvesCert::new(certificate, &secret_key) 79 | .expect("Server cert key DER is valid; qed"), 80 | ); 81 | 82 | let mut crypto = rustls::ServerConfig::builder_with_provider(Arc::new( 83 | rustls::crypto::ring::default_provider(), 84 | )) 85 | .with_protocol_versions(verifier::PROTOCOL_VERSIONS) 86 | .expect("fixed config") 87 | .with_client_cert_verifier(Arc::new(verifier::Libp2pCertificateVerifier::new())) 88 | .with_cert_resolver(cert_resolver); 89 | crypto.alpn_protocols = alpn_protocols; 90 | if keylog { 91 | warn!("enabling SSLKEYLOGFILE for TLS pre-master keys"); 92 | crypto.key_log = Arc::new(rustls::KeyLogFile::new()); 93 | } 94 | let config = crypto.try_into()?; 95 | Ok(config) 96 | } 97 | -------------------------------------------------------------------------------- /content-discovery/iroh-mainline-content-discovery/src/tls_utils/verifier.rs: -------------------------------------------------------------------------------- 1 | //! TLS 1.3 certificates and handshakes handling for libp2p 2 | //! 3 | //! This module handles a verification of a client/server certificate chain 4 | //! and signatures allegedly by the given certificates. 5 | //! 6 | //! Based on rust-libp2p/transports/tls/src/verifier.rs originally licensed under MIT by Parity 7 | //! Technologies (UK) Ltd. 8 | use std::sync::Arc; 9 | 10 | use iroh_base::PublicKey; 11 | use rustls::{ 12 | client::danger::{HandshakeSignatureValid, ServerCertVerified, ServerCertVerifier}, 13 | pki_types::CertificateDer as Certificate, 14 | server::danger::{ClientCertVerified, ClientCertVerifier}, 15 | CertificateError, DigitallySignedStruct, DistinguishedName, OtherError, PeerMisbehaved, 16 | SignatureScheme, SupportedProtocolVersion, 17 | }; 18 | 19 | use super::certificate; 20 | 21 | /// The protocol versions supported by this verifier. 22 | /// 23 | /// The spec says: 24 | /// 25 | /// > The libp2p handshake uses TLS 1.3 (and higher). 26 | /// > Endpoints MUST NOT negotiate lower TLS versions. 27 | pub static PROTOCOL_VERSIONS: &[&SupportedProtocolVersion] = &[&rustls::version::TLS13]; 28 | 29 | /// Implementation of the `rustls` certificate verification traits for libp2p. 30 | /// 31 | /// Only TLS 1.3 is supported. TLS 1.2 should be disabled in the configuration of `rustls`. 32 | #[derive(Debug)] 33 | pub struct Libp2pCertificateVerifier { 34 | /// The peer ID we intend to connect to 35 | remote_peer_id: Option, 36 | } 37 | 38 | /// libp2p requires the following of X.509 server certificate chains: 39 | /// 40 | /// - Exactly one certificate must be presented. 41 | /// - The certificate must be self-signed. 42 | /// - The certificate must have a valid libp2p extension that includes a 43 | /// signature of its public key. 44 | impl Libp2pCertificateVerifier { 45 | pub fn new() -> Self { 46 | Self { 47 | remote_peer_id: None, 48 | } 49 | } 50 | pub fn with_remote_peer_id(remote_peer_id: Option) -> Self { 51 | Self { remote_peer_id } 52 | } 53 | 54 | /// Return the list of SignatureSchemes that this verifier will handle, 55 | /// in `verify_tls12_signature` and `verify_tls13_signature` calls. 56 | /// 57 | /// This should be in priority order, with the most preferred first. 58 | fn verification_schemes() -> Vec { 59 | vec![ 60 | // TODO SignatureScheme::ECDSA_NISTP521_SHA512 is not supported by `ring` yet 61 | SignatureScheme::ECDSA_NISTP384_SHA384, 62 | SignatureScheme::ECDSA_NISTP256_SHA256, 63 | // TODO SignatureScheme::ED448 is not supported by `ring` yet 64 | SignatureScheme::ED25519, 65 | // In particular, RSA SHOULD NOT be used. 66 | ] 67 | } 68 | } 69 | 70 | impl ServerCertVerifier for Libp2pCertificateVerifier { 71 | fn verify_server_cert( 72 | &self, 73 | end_entity: &Certificate, 74 | intermediates: &[Certificate], 75 | _server_name: &rustls::pki_types::ServerName, 76 | _ocsp_response: &[u8], 77 | _now: rustls::pki_types::UnixTime, 78 | ) -> Result { 79 | let peer_id = verify_presented_certs(end_entity, intermediates)?; 80 | 81 | if let Some(ref remote_peer_id) = self.remote_peer_id { 82 | // The public host key allows the peer to calculate the peer ID of the peer 83 | // it is connecting to. Clients MUST verify that the peer ID derived from 84 | // the certificate matches the peer ID they intended to connect to, 85 | // and MUST abort the connection if there is a mismatch. 86 | if remote_peer_id != &peer_id { 87 | return Err(rustls::Error::PeerMisbehaved( 88 | PeerMisbehaved::BadCertChainExtensions, 89 | )); 90 | } 91 | } 92 | 93 | Ok(ServerCertVerified::assertion()) 94 | } 95 | 96 | fn verify_tls12_signature( 97 | &self, 98 | _message: &[u8], 99 | _cert: &Certificate, 100 | _dss: &DigitallySignedStruct, 101 | ) -> Result { 102 | unreachable!("`PROTOCOL_VERSIONS` only allows TLS 1.3") 103 | } 104 | 105 | fn verify_tls13_signature( 106 | &self, 107 | message: &[u8], 108 | cert: &Certificate, 109 | dss: &DigitallySignedStruct, 110 | ) -> Result { 111 | verify_tls13_signature(cert, dss.scheme, message, dss.signature()) 112 | } 113 | 114 | fn supported_verify_schemes(&self) -> Vec { 115 | Self::verification_schemes() 116 | } 117 | } 118 | 119 | /// libp2p requires the following of X.509 client certificate chains: 120 | /// 121 | /// - Exactly one certificate must be presented. In particular, client 122 | /// authentication is mandatory in libp2p. 123 | /// - The certificate must be self-signed. 124 | /// - The certificate must have a valid libp2p extension that includes a 125 | /// signature of its public key. 126 | impl ClientCertVerifier for Libp2pCertificateVerifier { 127 | fn offer_client_auth(&self) -> bool { 128 | true 129 | } 130 | 131 | fn verify_client_cert( 132 | &self, 133 | end_entity: &Certificate, 134 | intermediates: &[Certificate], 135 | _now: rustls::pki_types::UnixTime, 136 | ) -> Result { 137 | verify_presented_certs(end_entity, intermediates)?; 138 | 139 | Ok(ClientCertVerified::assertion()) 140 | } 141 | 142 | fn verify_tls12_signature( 143 | &self, 144 | _message: &[u8], 145 | _cert: &Certificate, 146 | _dss: &DigitallySignedStruct, 147 | ) -> Result { 148 | unreachable!("`PROTOCOL_VERSIONS` only allows TLS 1.3") 149 | } 150 | 151 | fn verify_tls13_signature( 152 | &self, 153 | message: &[u8], 154 | cert: &Certificate, 155 | dss: &DigitallySignedStruct, 156 | ) -> Result { 157 | verify_tls13_signature(cert, dss.scheme, message, dss.signature()) 158 | } 159 | 160 | fn supported_verify_schemes(&self) -> Vec { 161 | Self::verification_schemes() 162 | } 163 | 164 | fn root_hint_subjects(&self) -> &[DistinguishedName] { 165 | &[][..] 166 | } 167 | } 168 | 169 | /// When receiving the certificate chain, an endpoint 170 | /// MUST check these conditions and abort the connection attempt if 171 | /// (a) the presented certificate is not yet valid, OR 172 | /// (b) if it is expired. 173 | /// Endpoints MUST abort the connection attempt if more than one certificate is received, 174 | /// or if the certificate’s self-signature is not valid. 175 | fn verify_presented_certs( 176 | end_entity: &Certificate, 177 | intermediates: &[Certificate], 178 | ) -> Result { 179 | if !intermediates.is_empty() { 180 | return Err(rustls::Error::General( 181 | "libp2p-tls requires exactly one certificate".into(), 182 | )); 183 | } 184 | 185 | let cert = certificate::parse(end_entity)?; 186 | 187 | Ok(cert.peer_id()) 188 | } 189 | 190 | fn verify_tls13_signature( 191 | cert: &Certificate, 192 | signature_scheme: SignatureScheme, 193 | message: &[u8], 194 | signature: &[u8], 195 | ) -> Result { 196 | certificate::parse(cert)?.verify_signature(signature_scheme, message, signature)?; 197 | 198 | Ok(HandshakeSignatureValid::assertion()) 199 | } 200 | 201 | impl From for rustls::Error { 202 | fn from(certificate::ParseError(e): certificate::ParseError) -> Self { 203 | use webpki::Error::*; 204 | match e { 205 | BadDer => rustls::Error::InvalidCertificate(CertificateError::BadEncoding), 206 | e => { 207 | rustls::Error::InvalidCertificate(CertificateError::Other(OtherError(Arc::new(e)))) 208 | } 209 | } 210 | } 211 | } 212 | impl From for rustls::Error { 213 | fn from(certificate::VerificationError(e): certificate::VerificationError) -> Self { 214 | use webpki::Error::*; 215 | match e { 216 | InvalidSignatureForPublicKey => { 217 | rustls::Error::InvalidCertificate(CertificateError::BadSignature) 218 | } 219 | UnsupportedSignatureAlgorithm | UnsupportedSignatureAlgorithmForPublicKey => { 220 | rustls::Error::InvalidCertificate(CertificateError::BadSignature) 221 | } 222 | e => { 223 | rustls::Error::InvalidCertificate(CertificateError::Other(OtherError(Arc::new(e)))) 224 | } 225 | } 226 | } 227 | } 228 | -------------------------------------------------------------------------------- /content-discovery/iroh-mainline-tracker/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "iroh-mainline-tracker" 3 | version = "0.3.0" 4 | edition = "2021" 5 | description = "Content tracker for iroh, using the bittorrent mainline DHT" 6 | license = "MIT OR Apache-2.0" 7 | 8 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 9 | 10 | [dependencies] 11 | anyhow = { workspace = true, features = ["backtrace"] } 12 | # needs to keep updated with the dep of iroh-blobs 13 | bao-tree = { version = "0.15.1", features = ["tokio_fsm"], default-features = false } 14 | bytes = "1" 15 | derive_more = { version = "1.0.0-beta.1", features = ["debug", "display", "from", "try_into"] } 16 | dirs-next = "2" 17 | ed25519-dalek = "2.1.0" 18 | futures = "0.3.25" 19 | hex = "0.4.3" 20 | humantime = "2.1.0" 21 | iroh = { workspace = true } 22 | iroh-blobs = { workspace = true } 23 | mainline = { workspace = true, features = ["async"] } 24 | pkarr = { workspace = true } 25 | postcard = { workspace = true, features = ["alloc", "use-std"] } 26 | rand = "0.8" 27 | rcgen = "0.12.0" 28 | redb = "1.5.0" 29 | rustls = "0.21" 30 | rustls-pki-types = "1.11" 31 | serde = { version = "1", features = ["derive"] } 32 | serde_json = "1.0.107" 33 | tempfile = "3.4" 34 | tokio = { version = "1", features = ["io-util", "rt"] } 35 | tokio-util = { version = "0.7", features = ["io-util", "io", "rt"] } 36 | toml = "0.7.3" 37 | tracing = "0.1" 38 | tracing-subscriber = { version = "0.3", features = ["env-filter"] } 39 | ttl_cache = "0.5.1" 40 | url = "2.5.0" 41 | genawaiter = { version = "0.99.1", features = ["futures03"] } 42 | iroh-mainline-content-discovery = { path = "../iroh-mainline-content-discovery", features = ["client"] } 43 | quinn = { workspace = true } 44 | 45 | clap = { version = "4", features = ["derive"], optional = true } 46 | serde-big-array = "0.5.1" 47 | 48 | [features] 49 | cli = ["clap"] 50 | default = ["cli"] 51 | 52 | [[bin]] 53 | name = "iroh-mainline-tracker" 54 | required-features = ["cli"] 55 | -------------------------------------------------------------------------------- /content-discovery/iroh-mainline-tracker/src/args.rs: -------------------------------------------------------------------------------- 1 | //! Command line arguments. 2 | use std::net::SocketAddrV4; 3 | 4 | use clap::Parser; 5 | 6 | #[derive(Parser, Debug)] 7 | pub struct Args { 8 | /// The port to listen on. 9 | #[clap(long)] 10 | pub iroh_ipv4_addr: Option, 11 | 12 | /// The quinn port to listen on. 13 | /// 14 | /// The server must be reachable under this port from the internet, via 15 | /// UDP for QUIC connections. 16 | #[clap(long)] 17 | pub quinn_port: Option, 18 | 19 | /// The raw udp port to listen on. 20 | #[clap(long)] 21 | pub udp_port: Option, 22 | 23 | #[clap(long)] 24 | pub quiet: bool, 25 | } 26 | -------------------------------------------------------------------------------- /content-discovery/iroh-mainline-tracker/src/io.rs: -------------------------------------------------------------------------------- 1 | //! Anything related to local IO, including logging, file formats, and file locations. 2 | use std::{ 3 | collections::BTreeMap, 4 | env, 5 | io::Write, 6 | path::{Path, PathBuf}, 7 | time::{Instant, SystemTime, UNIX_EPOCH}, 8 | }; 9 | 10 | use anyhow::Context; 11 | use iroh::NodeId; 12 | use iroh_blobs::{get::Stats, HashAndFormat}; 13 | use iroh_mainline_content_discovery::protocol::{AnnounceKind, SignedAnnounce}; 14 | use serde::{de::DeserializeOwned, Deserialize, Serialize}; 15 | use tracing_subscriber::{prelude::*, EnvFilter}; 16 | 17 | use crate::tracker::ProbeKind; 18 | 19 | pub const CONFIG_DEFAULTS_FILE: &str = "config.defaults.toml"; 20 | pub const CONFIG_DEBUG_FILE: &str = "config.debug.toml"; 21 | pub const CONFIG_FILE: &str = "config.toml"; 22 | pub const SERVER_KEY_FILE: &str = "server.key"; 23 | pub const CLIENT_KEY_FILE: &str = "client.key"; 24 | pub const TRACKER_HOME_ENV_VAR: &str = "IROH_TRACKER_HOME"; 25 | 26 | /// Data format of the announce data file. 27 | /// 28 | /// This should be easy to edit manually when serialized as json or toml. 29 | #[derive(Debug, Clone, Default, Serialize, Deserialize)] 30 | pub struct AnnounceData( 31 | pub BTreeMap>>, 32 | ); 33 | 34 | pub fn save_to_file(data: impl Serialize, path: &Path) -> anyhow::Result<()> { 35 | let data_dir = path.parent().context("non absolute data file")?; 36 | let ext = path 37 | .extension() 38 | .context("no extension")? 39 | .to_str() 40 | .context("not utf8")? 41 | .to_ascii_lowercase(); 42 | let mut temp = tempfile::NamedTempFile::new_in(data_dir)?; 43 | match ext.as_str() { 44 | "toml" => { 45 | let data = toml::to_string_pretty(&data)?; 46 | temp.write_all(data.as_bytes())?; 47 | } 48 | "json" => { 49 | let data = serde_json::to_string_pretty(&data)?; 50 | temp.write_all(data.as_bytes())?; 51 | } 52 | "postcard" => { 53 | let data = postcard::to_stdvec(&data)?; 54 | temp.write_all(&data)?; 55 | } 56 | _ => anyhow::bail!("unsupported extension"), 57 | } 58 | std::fs::rename(temp.into_temp_path(), path)?; 59 | Ok(()) 60 | } 61 | 62 | pub fn load_from_file(path: &Path) -> anyhow::Result { 63 | anyhow::ensure!(path.is_absolute(), "non absolute data file"); 64 | let ext = path 65 | .extension() 66 | .context("no extension")? 67 | .to_str() 68 | .context("not utf8")? 69 | .to_ascii_lowercase(); 70 | if !path.exists() { 71 | return Ok(T::default()); 72 | } 73 | match ext.as_str() { 74 | "toml" => { 75 | let data = std::fs::read_to_string(path)?; 76 | Ok(toml::from_str(&data)?) 77 | } 78 | "json" => { 79 | let data = std::fs::read_to_string(path)?; 80 | Ok(serde_json::from_str(&data)?) 81 | } 82 | "postcard" => { 83 | let data = std::fs::read(path)?; 84 | Ok(postcard::from_bytes(&data)?) 85 | } 86 | _ => anyhow::bail!("unsupported extension"), 87 | } 88 | } 89 | 90 | pub fn log_connection_attempt( 91 | path: &Option, 92 | host: &NodeId, 93 | t0: Instant, 94 | outcome: &anyhow::Result, 95 | ) -> anyhow::Result<()> { 96 | if let Some(path) = path { 97 | let now = SystemTime::now() 98 | .duration_since(UNIX_EPOCH) 99 | .unwrap() 100 | .as_secs_f64(); 101 | let outcome = match outcome { 102 | Ok(_) => "ok", 103 | Err(_) => "err", 104 | }; 105 | let line = format!( 106 | "{:.6},{},{:.6},{}\n", 107 | now, 108 | host, 109 | t0.elapsed().as_secs_f64(), 110 | outcome 111 | ); 112 | let mut file = std::fs::OpenOptions::new() 113 | .append(true) 114 | .create(true) 115 | .open(path) 116 | .unwrap(); 117 | file.write_all(line.as_bytes())?; 118 | } 119 | Ok(()) 120 | } 121 | 122 | pub fn log_probe_attempt( 123 | path: &Option, 124 | host: &NodeId, 125 | content: &HashAndFormat, 126 | kind: ProbeKind, 127 | t0: Instant, 128 | outcome: &anyhow::Result, 129 | ) -> anyhow::Result<()> { 130 | if let Some(path) = path { 131 | let now = SystemTime::now() 132 | .duration_since(UNIX_EPOCH) 133 | .unwrap() 134 | .as_secs_f64(); 135 | let outcome = match outcome { 136 | Ok(_) => "ok", 137 | Err(_) => "err", 138 | }; 139 | let line = format!( 140 | "{:.6},{},{},{:?},{:.6},{}\n", 141 | now, 142 | host, 143 | content, 144 | kind, 145 | t0.elapsed().as_secs_f64(), 146 | outcome 147 | ); 148 | let mut file = std::fs::OpenOptions::new() 149 | .append(true) 150 | .create(true) 151 | .open(path) 152 | .unwrap(); 153 | file.write_all(line.as_bytes())?; 154 | } 155 | Ok(()) 156 | } 157 | 158 | // set the RUST_LOG env var to one of {debug,info,warn} to see logging info 159 | pub fn setup_logging() { 160 | tracing_subscriber::registry() 161 | .with(tracing_subscriber::fmt::layer().with_writer(std::io::stderr)) 162 | .with(EnvFilter::from_default_env()) 163 | .try_init() 164 | .ok(); 165 | } 166 | 167 | pub fn tracker_home() -> anyhow::Result { 168 | Ok(if let Some(val) = env::var_os(TRACKER_HOME_ENV_VAR) { 169 | PathBuf::from(val) 170 | } else { 171 | dirs_next::data_dir() 172 | .ok_or_else(|| { 173 | anyhow::anyhow!("operating environment provides no directory for application data") 174 | })? 175 | .join("iroh_tracker") 176 | }) 177 | } 178 | 179 | pub fn tracker_path(file_name: impl AsRef) -> anyhow::Result { 180 | Ok(tracker_home()?.join(file_name)) 181 | } 182 | -------------------------------------------------------------------------------- /content-discovery/iroh-mainline-tracker/src/iroh_blobs_util.rs: -------------------------------------------------------------------------------- 1 | //! Utilities for advanced use of iroh::blobs. 2 | use std::sync::Arc; 3 | 4 | use bao_tree::{ChunkNum, ChunkRanges}; 5 | use bytes::Bytes; 6 | use iroh_blobs::{ 7 | get::{ 8 | fsm::{BlobContentNext, EndBlobNext}, 9 | Stats, 10 | }, 11 | hashseq::HashSeq, 12 | protocol::{GetRequest, RangeSpecSeq}, 13 | Hash, HashAndFormat, 14 | }; 15 | use rand::Rng; 16 | 17 | /// Get the claimed size of a blob from a peer. 18 | /// 19 | /// This is just reading the size header and then immediately closing the connection. 20 | /// It can be used to check if a peer has any data at all. 21 | pub async fn unverified_size( 22 | connection: &iroh::endpoint::Connection, 23 | hash: &Hash, 24 | ) -> anyhow::Result<(u64, Stats)> { 25 | let request = iroh_blobs::protocol::GetRequest::new( 26 | *hash, 27 | RangeSpecSeq::from_ranges(vec![ChunkRanges::from(ChunkNum(u64::MAX)..)]), 28 | ); 29 | let request = iroh_blobs::get::fsm::start(connection.clone(), request); 30 | let connected = request.next().await?; 31 | let iroh_blobs::get::fsm::ConnectedNext::StartRoot(start) = connected.next().await? else { 32 | unreachable!("expected start root"); 33 | }; 34 | let at_blob_header = start.next(); 35 | let (curr, size) = at_blob_header.next().await?; 36 | let stats = curr.finish().next().await?; 37 | Ok((size, stats)) 38 | } 39 | 40 | /// Get the verified size of a blob from a peer. 41 | /// 42 | /// This asks for the last chunk of the blob and validates the response. 43 | /// Note that this does not validate that the peer has all the data. 44 | pub async fn verified_size( 45 | connection: &iroh::endpoint::Connection, 46 | hash: &Hash, 47 | ) -> anyhow::Result<(u64, Stats)> { 48 | tracing::debug!("Getting verified size of {}", hash.to_hex()); 49 | let request = iroh_blobs::protocol::GetRequest::new( 50 | *hash, 51 | RangeSpecSeq::from_ranges(vec![ChunkRanges::from(ChunkNum(u64::MAX)..)]), 52 | ); 53 | let request = iroh_blobs::get::fsm::start(connection.clone(), request); 54 | let connected = request.next().await?; 55 | let iroh_blobs::get::fsm::ConnectedNext::StartRoot(start) = connected.next().await? else { 56 | unreachable!("expected start root"); 57 | }; 58 | let header = start.next(); 59 | let (mut curr, size) = header.next().await?; 60 | let end = loop { 61 | match curr.next().await { 62 | BlobContentNext::More((next, res)) => { 63 | let _ = res?; 64 | curr = next; 65 | } 66 | BlobContentNext::Done(end) => { 67 | break end; 68 | } 69 | } 70 | }; 71 | let EndBlobNext::Closing(closing) = end.next() else { 72 | unreachable!("expected closing"); 73 | }; 74 | let stats = closing.next().await?; 75 | tracing::debug!( 76 | "Got verified size of {}, {:.6}s", 77 | hash.to_hex(), 78 | stats.elapsed.as_secs_f64() 79 | ); 80 | Ok((size, stats)) 81 | } 82 | 83 | pub async fn get_hash_seq_and_sizes( 84 | connection: &iroh::endpoint::Connection, 85 | hash: &Hash, 86 | max_size: u64, 87 | ) -> anyhow::Result<(HashSeq, Arc<[u64]>)> { 88 | let content = HashAndFormat::hash_seq(*hash); 89 | tracing::debug!("Getting hash seq and children sizes of {}", content); 90 | let request = iroh_blobs::protocol::GetRequest::new( 91 | *hash, 92 | RangeSpecSeq::from_ranges_infinite([ 93 | ChunkRanges::all(), 94 | ChunkRanges::from(ChunkNum(u64::MAX)..), 95 | ]), 96 | ); 97 | let at_start = iroh_blobs::get::fsm::start(connection.clone(), request); 98 | let at_connected = at_start.next().await?; 99 | let iroh_blobs::get::fsm::ConnectedNext::StartRoot(start) = at_connected.next().await? else { 100 | unreachable!("query includes root"); 101 | }; 102 | let at_start_root = start.next(); 103 | let (at_blob_content, size) = at_start_root.next().await?; 104 | // check the size to avoid parsing a maliciously large hash seq 105 | if size > max_size { 106 | anyhow::bail!("size too large"); 107 | } 108 | let (mut curr, hash_seq) = at_blob_content.concatenate_into_vec().await?; 109 | let hash_seq = HashSeq::try_from(Bytes::from(hash_seq))?; 110 | let mut sizes = Vec::with_capacity(hash_seq.len()); 111 | let closing = loop { 112 | match curr.next() { 113 | EndBlobNext::MoreChildren(more) => { 114 | let hash = match hash_seq.get(sizes.len()) { 115 | Some(hash) => hash, 116 | None => break more.finish(), 117 | }; 118 | let at_header = more.next(hash); 119 | let (at_content, size) = at_header.next().await?; 120 | let next = at_content.drain().await?; 121 | sizes.push(size); 122 | curr = next; 123 | } 124 | EndBlobNext::Closing(closing) => break closing, 125 | } 126 | }; 127 | let _stats = closing.next().await?; 128 | tracing::debug!( 129 | "Got hash seq and children sizes of {}: {:?}", 130 | content, 131 | sizes 132 | ); 133 | Ok((hash_seq, sizes.into())) 134 | } 135 | 136 | /// Probe for a single chunk of a blob. 137 | pub async fn chunk_probe( 138 | connection: &iroh::endpoint::Connection, 139 | hash: &Hash, 140 | chunk: ChunkNum, 141 | ) -> anyhow::Result { 142 | let ranges = ChunkRanges::from(chunk..chunk + 1); 143 | let ranges = RangeSpecSeq::from_ranges([ranges]); 144 | let request = GetRequest::new(*hash, ranges); 145 | let request = iroh_blobs::get::fsm::start(connection.clone(), request); 146 | let connected = request.next().await?; 147 | let iroh_blobs::get::fsm::ConnectedNext::StartRoot(start) = connected.next().await? else { 148 | unreachable!("query includes root"); 149 | }; 150 | let header = start.next(); 151 | let (mut curr, _size) = header.next().await?; 152 | let end = loop { 153 | match curr.next().await { 154 | BlobContentNext::More((next, res)) => { 155 | res?; 156 | curr = next; 157 | } 158 | BlobContentNext::Done(end) => { 159 | break end; 160 | } 161 | } 162 | }; 163 | let EndBlobNext::Closing(closing) = end.next() else { 164 | unreachable!("query contains only one blob"); 165 | }; 166 | let stats = closing.next().await?; 167 | Ok(stats) 168 | } 169 | 170 | /// Given a sequence of sizes of children, generate a range spec that selects a 171 | /// random chunk of a random child. 172 | /// 173 | /// The random chunk is chosen uniformly from the chunks of the children, so 174 | /// larger children are more likely to be selected. 175 | pub fn random_hash_seq_ranges(sizes: &[u64], mut rng: impl Rng) -> RangeSpecSeq { 176 | let total_chunks = sizes 177 | .iter() 178 | .map(|size| ChunkNum::full_chunks(*size).0) 179 | .sum::(); 180 | let random_chunk = rng.gen_range(0..total_chunks); 181 | let mut remaining = random_chunk; 182 | let mut ranges = vec![]; 183 | ranges.push(ChunkRanges::empty()); 184 | for size in sizes.iter() { 185 | let chunks = ChunkNum::full_chunks(*size).0; 186 | if remaining < chunks { 187 | ranges.push(ChunkRanges::from( 188 | ChunkNum(remaining)..ChunkNum(remaining + 1), 189 | )); 190 | break; 191 | } else { 192 | remaining -= chunks; 193 | ranges.push(ChunkRanges::empty()); 194 | } 195 | } 196 | RangeSpecSeq::from_ranges(ranges) 197 | } 198 | -------------------------------------------------------------------------------- /content-discovery/iroh-mainline-tracker/src/lib.rs: -------------------------------------------------------------------------------- 1 | pub mod io; 2 | pub mod iroh_blobs_util; 3 | pub mod options; 4 | mod task_map; 5 | pub mod tracker; 6 | -------------------------------------------------------------------------------- /content-discovery/iroh-mainline-tracker/src/main.rs: -------------------------------------------------------------------------------- 1 | pub mod args; 2 | 3 | use std::{ 4 | net::SocketAddrV4, 5 | sync::{ 6 | atomic::{AtomicBool, Ordering}, 7 | Arc, 8 | }, 9 | time::{Duration, Instant}, 10 | }; 11 | 12 | use clap::Parser; 13 | use iroh::{Endpoint, NodeId}; 14 | use iroh_blobs::util::fs::load_secret_key; 15 | use iroh_mainline_content_discovery::{protocol::ALPN, tls_utils}; 16 | use iroh_mainline_tracker::{ 17 | io::{ 18 | self, load_from_file, setup_logging, tracker_home, tracker_path, CONFIG_DEBUG_FILE, 19 | CONFIG_DEFAULTS_FILE, CONFIG_FILE, SERVER_KEY_FILE, 20 | }, 21 | options::Options, 22 | tracker::Tracker, 23 | }; 24 | 25 | use crate::args::Args; 26 | 27 | static VERBOSE: AtomicBool = AtomicBool::new(false); 28 | 29 | fn set_verbose(verbose: bool) { 30 | VERBOSE.store(verbose, Ordering::Relaxed); 31 | } 32 | 33 | pub fn verbose() -> bool { 34 | VERBOSE.load(Ordering::Relaxed) 35 | } 36 | 37 | #[macro_export] 38 | macro_rules! log { 39 | ($($arg:tt)*) => { 40 | if $crate::verbose() { 41 | println!($($arg)*); 42 | } else { 43 | tracing::info!($($arg)*); 44 | } 45 | }; 46 | } 47 | 48 | /// Wait until the endpoint has figured out it's own DERP region. 49 | async fn await_relay_region(endpoint: &Endpoint) -> anyhow::Result<()> { 50 | let t0 = Instant::now(); 51 | loop { 52 | let addr = endpoint.node_addr().await?; 53 | if addr.relay_url().is_some() { 54 | break; 55 | } 56 | if t0.elapsed() > Duration::from_secs(10) { 57 | anyhow::bail!("timeout waiting for DERP region"); 58 | } 59 | tokio::time::sleep(Duration::from_millis(50)).await; 60 | } 61 | Ok(()) 62 | } 63 | 64 | async fn create_endpoint( 65 | key: iroh::SecretKey, 66 | ipv4_addr: SocketAddrV4, 67 | ) -> anyhow::Result { 68 | iroh::Endpoint::builder() 69 | .secret_key(key) 70 | .discovery_dht() 71 | .discovery_n0() 72 | .alpns(vec![ALPN.to_vec()]) 73 | .bind_addr_v4(ipv4_addr) 74 | .bind() 75 | .await 76 | } 77 | 78 | /// Accept an incoming connection and extract the client-provided [`NodeId`] and ALPN protocol. 79 | pub async fn accept_conn( 80 | mut conn: iroh::endpoint::Connecting, 81 | ) -> anyhow::Result<(NodeId, String, iroh::endpoint::Connection)> { 82 | let alpn = String::from_utf8(conn.alpn().await?)?; 83 | let conn = conn.await?; 84 | let peer_id = conn.remote_node_id()?; 85 | Ok((peer_id, alpn, conn)) 86 | } 87 | 88 | /// Write default options to a sample config file. 89 | fn write_debug() -> anyhow::Result<()> { 90 | let default_path = tracker_path(CONFIG_DEBUG_FILE)?; 91 | io::save_to_file(Options::debug(), &default_path)?; 92 | Ok(()) 93 | } 94 | 95 | /// Write default options to a sample config file. 96 | fn write_defaults() -> anyhow::Result<()> { 97 | let default_path = tracker_path(CONFIG_DEFAULTS_FILE)?; 98 | io::save_to_file(Options::default(), &default_path)?; 99 | Ok(()) 100 | } 101 | 102 | async fn server(args: Args) -> anyhow::Result<()> { 103 | set_verbose(!args.quiet); 104 | let home = tracker_home()?; 105 | tokio::fs::create_dir_all(&home).await?; 106 | let config_path = tracker_path(CONFIG_FILE)?; 107 | write_defaults()?; 108 | write_debug()?; 109 | let mut options = load_from_file::(&config_path)?; 110 | options.make_paths_relative(&home); 111 | tracing::info!("using options: {:#?}", options); 112 | // override options with args 113 | if let Some(quinn_port) = args.quinn_port { 114 | options.quinn_port = quinn_port; 115 | } 116 | if let Some(addr) = args.iroh_ipv4_addr { 117 | options.iroh_ipv4_addr = addr; 118 | } 119 | if let Some(udp_port) = args.udp_port { 120 | options.udp_port = udp_port; 121 | } 122 | log!("tracker starting using {}", home.display()); 123 | let key_path = tracker_path(SERVER_KEY_FILE)?; 124 | let key = load_secret_key(key_path).await?; 125 | // let server_config = configure_server(&key)?; 126 | // let udp_bind_addr = SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::UNSPECIFIED, options.udp_port)); 127 | // let udp_socket = tokio::net::UdpSocket::bind(udp_bind_addr).await?; 128 | // let quinn_bind_addr = 129 | // SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::UNSPECIFIED, options.quinn_port)); 130 | // let quinn_endpoint = quinn::Endpoint::server(server_config, quinn_bind_addr)?; 131 | // set the quinn port to the actual port we bound to so the DHT will announce it correctly 132 | // options.quinn_port = quinn_endpoint.local_addr()?.port(); 133 | let iroh_endpoint = create_endpoint(key.clone(), options.iroh_ipv4_addr).await?; 134 | let db = Tracker::new(options, iroh_endpoint.clone())?; 135 | db.dump().await?; 136 | await_relay_region(&iroh_endpoint).await?; 137 | let addr = iroh_endpoint.node_addr().await?; 138 | tracing::info!("listening on {:?}", addr); 139 | tracing::info!("tracker addr: {}\n", addr.node_id); 140 | // let db2 = db.clone(); 141 | let db3 = db.clone(); 142 | // let db4 = db.clone(); 143 | let iroh_accept_task = tokio::spawn(db.iroh_accept_loop(iroh_endpoint)); 144 | // let quinn_accept_task = tokio::spawn(db2.quinn_accept_loop(quinn_endpoint)); 145 | // let udp_accept_task = tokio::spawn(db4.udp_accept_loop(udp_socket)); 146 | let gc_task = tokio::spawn(db3.gc_loop()); 147 | tokio::select! { 148 | _ = tokio::signal::ctrl_c() => { 149 | tracing::info!("shutting down"); 150 | } 151 | res = iroh_accept_task => { 152 | tracing::error!("iroh accept task exited"); 153 | res??; 154 | } 155 | // res = quinn_accept_task => { 156 | // tracing::error!("quinn accept task exited"); 157 | // res??; 158 | // } 159 | // res = udp_accept_task => { 160 | // tracing::error!("udp accept task exited"); 161 | // res??; 162 | // } 163 | res = gc_task => { 164 | tracing::error!("gc task exited"); 165 | res??; 166 | } 167 | } 168 | Ok(()) 169 | } 170 | 171 | #[tokio::main(flavor = "multi_thread")] 172 | async fn main() -> anyhow::Result<()> { 173 | setup_logging(); 174 | let args = Args::parse(); 175 | server(args).await 176 | } 177 | 178 | // /// Returns default server configuration along with its certificate. 179 | // #[allow(clippy::field_reassign_with_default)] // https://github.com/rust-lang/rust-clippy/issues/6527 180 | // fn configure_server(secret_key: &iroh::SecretKey) -> anyhow::Result { 181 | // make_server_config(secret_key, 8, 1024, vec![ALPN.to_vec()]) 182 | // } 183 | 184 | /// Create a [`quinn::ServerConfig`] with the given secret key and limits. 185 | pub fn make_server_config( 186 | secret_key: &iroh::SecretKey, 187 | max_streams: u64, 188 | max_connections: u32, 189 | alpn_protocols: Vec>, 190 | ) -> anyhow::Result { 191 | let tls_server_config = tls_utils::make_server_config(secret_key, alpn_protocols, false)?; 192 | let mut server_config = quinn::ServerConfig::with_crypto(Arc::new(tls_server_config)); 193 | let mut transport_config = quinn::TransportConfig::default(); 194 | transport_config 195 | .max_concurrent_bidi_streams(max_streams.try_into()?) 196 | .max_concurrent_uni_streams(0u32.into()); 197 | 198 | server_config 199 | .transport_config(Arc::new(transport_config)) 200 | .max_incoming(max_connections as usize); 201 | Ok(server_config) 202 | } 203 | -------------------------------------------------------------------------------- /content-discovery/iroh-mainline-tracker/src/options.rs: -------------------------------------------------------------------------------- 1 | //! Options for the tracker 2 | use std::{ 3 | net::SocketAddrV4, 4 | path::{Path, PathBuf}, 5 | time::Duration, 6 | }; 7 | 8 | use serde::{Deserialize, Serialize}; 9 | 10 | #[derive(Debug, Clone, Serialize, Deserialize)] 11 | pub struct Options { 12 | // time after which an announce is considered stale 13 | #[serde(with = "serde_duration")] 14 | pub announce_timeout: Duration, 15 | 16 | // time after which an announce is considered so old that it is no longer worth storing 17 | // 18 | // we hold on to stale announces in case the node comes back and can be probed 19 | #[serde(with = "serde_duration")] 20 | pub announce_expiry: Duration, 21 | 22 | // time after which a probe is considered stale 23 | #[serde(with = "serde_duration")] 24 | pub probe_timeout: Duration, 25 | 26 | // time after which probed content is considered so old that it is no longer worth storing 27 | #[serde(with = "serde_duration")] 28 | pub probe_expiry: Duration, 29 | 30 | // interval between garbage collection runs 31 | #[serde(with = "serde_duration")] 32 | pub gc_interval: Duration, 33 | 34 | // interval between probing peers 35 | #[serde(with = "serde_duration")] 36 | pub probe_interval: Duration, 37 | 38 | /// Interval between DHT announces. 39 | /// 40 | /// The tracker will announce itself to the DHT for each hash it knows about 41 | /// every `dht_announce_interval` seconds. Setting this to a very low value 42 | /// risks getting throttled by the DHT. 43 | #[serde(with = "serde_duration")] 44 | pub dht_announce_interval: Duration, 45 | 46 | // max hash seq size in bytes 47 | pub max_hash_seq_size: u64, 48 | 49 | // log file for dial attempts 50 | pub dial_log: Option, 51 | 52 | // log file for probe attempts 53 | pub probe_log: Option, 54 | 55 | // binary database for announce data 56 | pub announce_data_path: PathBuf, 57 | 58 | /// The quinn port to listen on. This is also the port that will be announced 59 | /// to the DHT. Set to 0 to listen on a random port. 60 | pub quinn_port: u16, 61 | 62 | /// The iroh adr to listen on. Set port to 0 to listen on a random port. 63 | pub iroh_ipv4_addr: SocketAddrV4, 64 | 65 | /// The UDP port to listen on. Set to 0 to listen on a random port. 66 | pub udp_port: u16, 67 | } 68 | 69 | impl Default for Options { 70 | fn default() -> Self { 71 | Self { 72 | announce_timeout: Duration::from_secs(60 * 60 * 12), 73 | announce_expiry: Duration::from_secs(60 * 60 * 12 * 7), 74 | probe_timeout: Duration::from_secs(60 * 60 * 12), 75 | probe_expiry: Duration::from_secs(60 * 60 * 12 * 7), 76 | probe_interval: Duration::from_secs(60 * 60), 77 | gc_interval: Duration::from_secs(60 * 5), 78 | dht_announce_interval: Duration::from_secs(60 * 60 * 6), 79 | // max hash seq size is 16 * 1024 hashes of 32 bytes each 80 | max_hash_seq_size: 1024 * 16 * 32, 81 | dial_log: None, 82 | probe_log: None, 83 | announce_data_path: "announce.redb".into(), 84 | quinn_port: 0, 85 | iroh_ipv4_addr: "0.0.0.0:0".parse().unwrap(), 86 | udp_port: 0, 87 | } 88 | } 89 | } 90 | 91 | impl Options { 92 | /// Debug options for testing. These will spam the DHT and all peers, 93 | /// so use with care. 94 | pub fn debug() -> Self { 95 | Self { 96 | announce_timeout: Duration::from_secs(60), 97 | announce_expiry: Duration::from_secs(60 * 2), 98 | probe_timeout: Duration::from_secs(60), 99 | probe_expiry: Duration::from_secs(60 * 3), 100 | probe_interval: Duration::from_secs(20), 101 | gc_interval: Duration::from_secs(20), 102 | dht_announce_interval: Duration::from_secs(20), 103 | // max hash seq size is 16 * 1024 hashes of 32 bytes each 104 | max_hash_seq_size: 1024 * 16 * 32, 105 | dial_log: Some("dial.log".into()), 106 | probe_log: Some("probe.log".into()), 107 | announce_data_path: "announce.redb".into(), 108 | quinn_port: 0, 109 | iroh_ipv4_addr: "0.0.0.0:0".parse().unwrap(), 110 | udp_port: 0, 111 | } 112 | } 113 | 114 | /// Make the paths in the options relative to the given base path. 115 | pub fn make_paths_relative(&mut self, base: &Path) { 116 | #[allow(clippy::needless_borrows_for_generic_args)] 117 | if let Some(path) = &mut self.dial_log { 118 | *path = base.join(&path); 119 | } 120 | #[allow(clippy::needless_borrows_for_generic_args)] 121 | if let Some(path) = &mut self.probe_log { 122 | *path = base.join(&path); 123 | } 124 | self.announce_data_path = base.join(&self.announce_data_path); 125 | } 126 | } 127 | mod serde_duration { 128 | use serde::{de::Deserializer, ser::Serializer}; 129 | 130 | use super::*; 131 | 132 | pub fn serialize(duration: &Duration, serializer: S) -> Result { 133 | if serializer.is_human_readable() { 134 | serializer.serialize_str(humantime::Duration::from(*duration).to_string().as_str()) 135 | } else { 136 | duration.serialize(serializer) 137 | } 138 | } 139 | 140 | pub fn deserialize<'de, D: Deserializer<'de>>(deserializer: D) -> Result { 141 | if deserializer.is_human_readable() { 142 | let s = String::deserialize(deserializer)?; 143 | humantime::parse_duration(&s).map_err(serde::de::Error::custom) 144 | } else { 145 | Duration::deserialize(deserializer) 146 | } 147 | } 148 | } 149 | -------------------------------------------------------------------------------- /content-discovery/iroh-mainline-tracker/src/task_map.rs: -------------------------------------------------------------------------------- 1 | use std::{ 2 | collections::BTreeMap, 3 | sync::{Arc, Mutex}, 4 | }; 5 | 6 | use derive_more::Debug; 7 | use tokio::task::JoinHandle; 8 | use tokio_util::task::AbortOnDropHandle; 9 | 10 | /// A map of long lived or infinite tasks. 11 | #[derive(Clone, Debug)] 12 | pub struct TaskMap(Arc>); 13 | 14 | impl TaskMap { 15 | /// Create a new task map. 16 | pub fn publish(&self, key: T, task: JoinHandle<()>) { 17 | let mut tasks = self.0.tasks.lock().unwrap(); 18 | tasks.insert(key, AbortOnDropHandle::new(task)); 19 | } 20 | 21 | pub fn retain(&self, f: impl Fn(&T) -> bool) { 22 | let mut tasks = self.0.tasks.lock().unwrap(); 23 | tasks.retain(|k, _| f(k)); 24 | } 25 | } 26 | 27 | impl Default for TaskMap { 28 | fn default() -> Self { 29 | Self(Default::default()) 30 | } 31 | } 32 | 33 | #[derive(Debug)] 34 | struct Inner { 35 | tasks: Mutex>>, 36 | } 37 | 38 | impl Default for Inner { 39 | fn default() -> Self { 40 | Self { 41 | tasks: Default::default(), 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /content-discovery/iroh-mainline-tracker/src/tracker/tables.rs: -------------------------------------------------------------------------------- 1 | //! Table definitions and accessors for the redb database. 2 | use redb::{ReadableTable, TableDefinition, TableError}; 3 | 4 | use super::{AnnouncePath, AnnounceValue, ProbeValue}; 5 | 6 | pub(super) const ANNOUNCES_TABLE: TableDefinition = 7 | TableDefinition::new("announces-0"); 8 | pub(super) const PROBES_TABLE: TableDefinition = 9 | TableDefinition::new("probes-0"); 10 | 11 | pub(super) trait ReadableTables { 12 | fn announces(&self) -> &impl ReadableTable; 13 | fn probes(&self) -> &impl ReadableTable; 14 | } 15 | 16 | pub(super) struct Tables<'a, 'b> { 17 | pub announces: redb::Table<'a, 'b, AnnouncePath, AnnounceValue>, 18 | pub probes: redb::Table<'a, 'b, AnnouncePath, ProbeValue>, 19 | } 20 | 21 | impl<'db, 'txn> Tables<'db, 'txn> { 22 | pub fn new(tx: &'txn redb::WriteTransaction<'db>) -> std::result::Result { 23 | Ok(Self { 24 | announces: tx.open_table(ANNOUNCES_TABLE)?, 25 | probes: tx.open_table(PROBES_TABLE)?, 26 | }) 27 | } 28 | } 29 | 30 | impl ReadableTables for Tables<'_, '_> { 31 | fn announces(&self) -> &impl ReadableTable { 32 | &self.announces 33 | } 34 | fn probes(&self) -> &impl ReadableTable { 35 | &self.probes 36 | } 37 | } 38 | 39 | /// A struct similar to [`redb::ReadOnlyTable`] but for all tables that make up 40 | /// the blob store. 41 | pub(super) struct ReadOnlyTables<'txn> { 42 | pub announces: redb::ReadOnlyTable<'txn, AnnouncePath, AnnounceValue>, 43 | pub probes: redb::ReadOnlyTable<'txn, AnnouncePath, ProbeValue>, 44 | } 45 | 46 | impl<'txn> ReadOnlyTables<'txn> { 47 | pub fn new(tx: &'txn redb::ReadTransaction<'txn>) -> std::result::Result { 48 | Ok(Self { 49 | announces: tx.open_table(ANNOUNCES_TABLE)?, 50 | probes: tx.open_table(PROBES_TABLE)?, 51 | }) 52 | } 53 | } 54 | 55 | impl ReadableTables for ReadOnlyTables<'_> { 56 | fn announces(&self) -> &impl ReadableTable { 57 | &self.announces 58 | } 59 | fn probes(&self) -> &impl ReadableTable { 60 | &self.probes 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /content-discovery/iroh-mainline-tracker/src/tracker/util.rs: -------------------------------------------------------------------------------- 1 | use tokio::sync::mpsc; 2 | 3 | /// A wrapper for a flume receiver that allows peeking at the next message. 4 | #[derive(Debug)] 5 | pub(super) struct PeekableReceiver { 6 | msg: Option, 7 | recv: mpsc::Receiver, 8 | } 9 | 10 | impl PeekableReceiver { 11 | pub fn new(recv: mpsc::Receiver) -> Self { 12 | Self { msg: None, recv } 13 | } 14 | 15 | /// Peek at the next message. 16 | /// 17 | /// Will block if there are no messages. 18 | /// Returns None only if there are no more messages (sender is dropped). 19 | pub async fn peek(&mut self) -> Option<&T> { 20 | if self.msg.is_none() { 21 | self.msg = self.recv.recv().await; 22 | } 23 | self.msg.as_ref() 24 | } 25 | 26 | /// Receive the next message. 27 | /// 28 | /// Will block if there are no messages. 29 | /// Returns None only if there are no more messages (sender is dropped). 30 | pub async fn recv(&mut self) -> Option { 31 | if let Some(msg) = self.msg.take() { 32 | return Some(msg); 33 | } 34 | self.recv.recv().await 35 | } 36 | 37 | /// Push back a message. This will only work if there is room for it. 38 | /// Otherwise, it will fail and return the message. 39 | pub fn push_back(&mut self, msg: T) -> std::result::Result<(), T> { 40 | if self.msg.is_none() { 41 | self.msg = Some(msg); 42 | Ok(()) 43 | } else { 44 | Err(msg) 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /h3-iroh/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "h3-iroh" 3 | version = "0.1.0" 4 | edition = "2021" 5 | license = "MIT" 6 | 7 | [dependencies] 8 | anyhow = { version = "1", optional = true } 9 | axum = { version = "0.7", optional = true } 10 | bytes = "1" 11 | futures = "0.3" 12 | h3 = { version = "0.0.6", features = ["tracing"] } 13 | http = { version = "1.1", optional = true } 14 | http-body = { version = "1", optional = true } 15 | http-body-util = { version = "0.1", optional = true } 16 | hyper = { version = "1.5", optional = true } 17 | hyper-util = { version = "0.1", optional = true } 18 | iroh = "0.35" 19 | iroh-base = { version = "0.35", features = ["ticket"] } 20 | tokio = { version = "1", features = ["io-util"], default-features = false} 21 | tokio-util = "0.7" 22 | tower = { version = "0.5", optional = true } 23 | tracing = "0.1" 24 | 25 | [features] 26 | axum = [ 27 | "dep:anyhow", 28 | "dep:axum", 29 | "dep:http", 30 | "dep:http-body", 31 | "dep:http-body-util", 32 | "dep:hyper", 33 | "dep:hyper-util", 34 | "dep:tower", 35 | ] 36 | 37 | [dev-dependencies] 38 | anyhow = "1" 39 | clap = { version = "4", features = ["derive"] } 40 | http = "1" 41 | tracing-subscriber = "0.3" 42 | 43 | [[example]] 44 | name = "server-axum" 45 | required-features = ["axum"] 46 | -------------------------------------------------------------------------------- /h3-iroh/examples/client.rs: -------------------------------------------------------------------------------- 1 | use std::future; 2 | use std::str::FromStr; 3 | 4 | use anyhow::{bail, Context, Result}; 5 | use clap::Parser; 6 | use iroh::NodeAddr; 7 | use iroh_base::ticket::NodeTicket; 8 | use tokio::io::AsyncWriteExt; 9 | use tracing::info; 10 | 11 | #[derive(Parser, Debug)] 12 | #[command()] 13 | struct Args { 14 | #[arg(long, help = "Use SSLKEYLOGFILE environment variable to log TLS keys")] 15 | keylogfile: bool, 16 | #[arg()] 17 | uri: String, 18 | } 19 | 20 | #[tokio::main] 21 | async fn main() -> Result<()> { 22 | tracing_subscriber::fmt().init(); 23 | 24 | let args = Args::parse(); 25 | 26 | let uri: http::Uri = args.uri.parse()?; 27 | if uri.scheme_str() != Some("iroh+h3") { 28 | bail!("URI scheme must be iroh+h3"); 29 | } 30 | let ticket = uri.host().context("missing hostname in URI")?; 31 | let ticket = NodeTicket::from_str(ticket)?; 32 | let addr: NodeAddr = ticket.into(); 33 | 34 | let ep = iroh::Endpoint::builder() 35 | .keylog(args.keylogfile) 36 | .bind() 37 | .await?; 38 | 39 | let conn = ep.connect(addr, b"iroh+h3").await?; 40 | let conn = h3_iroh::Connection::new(conn); 41 | 42 | let (mut driver, mut send_request) = h3::client::new(conn).await?; 43 | 44 | let drive_fut = async move { 45 | future::poll_fn(|cx| driver.poll_close(cx)).await?; 46 | Ok::<(), anyhow::Error>(()) 47 | }; 48 | 49 | let req_fut = async move { 50 | info!("sending request"); 51 | let req = http::Request::builder().uri(uri).body(())?; 52 | let mut stream = send_request.send_request(req).await?; 53 | stream.finish().await?; 54 | 55 | info!("receiving response"); 56 | let resp = stream.recv_response().await?; 57 | info!( 58 | version = ?resp.version(), 59 | status = ?resp.status(), 60 | headers = ?resp.headers(), 61 | "response", 62 | ); 63 | while let Some(mut chunk) = stream.recv_data().await? { 64 | info!("chunk!"); 65 | let mut out = tokio::io::stdout(); 66 | out.write_all_buf(&mut chunk).await?; 67 | out.flush().await?; 68 | } 69 | 70 | Ok::<(), anyhow::Error>(()) 71 | }; 72 | 73 | let (req_res, drive_res) = tokio::join!(req_fut, drive_fut); 74 | req_res?; 75 | drive_res?; 76 | 77 | info!("closing ep"); 78 | ep.close().await; 79 | info!("ep closed"); 80 | 81 | Ok(()) 82 | } 83 | -------------------------------------------------------------------------------- /h3-iroh/examples/server-axum.rs: -------------------------------------------------------------------------------- 1 | //! Demonstration of an axum server serving h3 over iroh 2 | //! 3 | //! run using `cargo run --features axum --example server-axum` 4 | 5 | use anyhow::Result; 6 | use axum::response::Html; 7 | use axum::routing::get; 8 | use axum::Router; 9 | use iroh_base::ticket::NodeTicket; 10 | use tracing::info; 11 | 12 | #[tokio::main] 13 | async fn main() -> Result<()> { 14 | tracing_subscriber::fmt().init(); 15 | 16 | let app = Router::new().route("/", get(handler)); 17 | 18 | let ep = iroh::Endpoint::builder() 19 | .alpns(vec![b"iroh+h3".to_vec()]) 20 | .bind() 21 | .await?; 22 | info!("accepting connections on node: {}", ep.node_id()); 23 | 24 | // Wait for direct addresses and a RelayUrl before printing a NodeTicket. 25 | ep.direct_addresses().initialized().await?; 26 | ep.home_relay().initialized().await?; 27 | let ticket = NodeTicket::new(ep.node_addr().await?); 28 | info!("node ticket: {ticket}"); 29 | info!("run: cargo run --example client -- iroh+h3://{ticket}/"); 30 | 31 | h3_iroh::axum::serve(ep, app).await?; 32 | 33 | Ok(()) 34 | } 35 | 36 | async fn handler() -> Html<&'static str> { 37 | Html("

Hello, World!

") 38 | } 39 | -------------------------------------------------------------------------------- /h3-iroh/examples/server.rs: -------------------------------------------------------------------------------- 1 | //! Demonstration of using h3-iroh as a server without framework. 2 | //! 3 | //! run using `cargo run --example server -- --root .` 4 | 5 | use std::path::PathBuf; 6 | use std::sync::Arc; 7 | 8 | use anyhow::{bail, Result}; 9 | use bytes::{Bytes, BytesMut}; 10 | use clap::Parser; 11 | use h3::error::ErrorLevel; 12 | use h3::quic::BidiStream; 13 | use h3::server::RequestStream; 14 | use http::{Request, StatusCode}; 15 | use iroh::endpoint::Incoming; 16 | use iroh_base::ticket::NodeTicket; 17 | use tokio::fs::File; 18 | use tokio::io::AsyncReadExt; 19 | use tracing::{debug, error, field, info, info_span, Instrument, Span}; 20 | 21 | #[derive(Parser, Debug)] 22 | #[command()] 23 | struct Args { 24 | #[arg( 25 | short, 26 | long, 27 | name = "DIR", 28 | help = "Root directory to server files from, if omitted server will only respond OK" 29 | )] 30 | root: Option, 31 | } 32 | 33 | #[tokio::main] 34 | async fn main() -> Result<()> { 35 | tracing_subscriber::fmt().init(); 36 | 37 | let args = Args::parse(); 38 | 39 | let root = if let Some(root) = args.root { 40 | if !root.is_dir() { 41 | bail!("{}: is not a readable directory", root.display()); 42 | } else { 43 | info!("serving {}", root.display()); 44 | Arc::new(Some(root)) 45 | } 46 | } else { 47 | Arc::new(None) 48 | }; 49 | 50 | let ep = iroh::Endpoint::builder() 51 | .alpns(vec![b"iroh+h3".to_vec()]) 52 | .bind() 53 | .await?; 54 | info!("accepting connections on node: {}", ep.node_id()); 55 | 56 | // Wait for direct addresses and a RelayUrl before printing a NodeTicket. 57 | ep.direct_addresses().initialized().await?; 58 | ep.home_relay().initialized().await?; 59 | let ticket = NodeTicket::new(ep.node_addr().await?); 60 | info!("node ticket: {ticket}"); 61 | info!("run e.g.: cargo run --example client -- iroh+h3://{ticket}/Cargo.toml"); 62 | 63 | // Handle incoming connections 64 | while let Some(incoming) = ep.accept().await { 65 | tokio::spawn({ 66 | let root = root.clone(); 67 | async move { 68 | if let Err(err) = handle_connection(incoming, root).await { 69 | error!("failed connection: {err:#}"); 70 | } 71 | } 72 | .instrument(info_span!("conn", remote_node_id = field::Empty)) 73 | }); 74 | } 75 | ep.close().await; 76 | 77 | Ok(()) 78 | } 79 | 80 | async fn handle_connection(incoming: Incoming, root: Arc>) -> Result<()> { 81 | let conn = incoming.accept()?.await?; 82 | let remote_node_id = conn.remote_node_id()?; 83 | let span = Span::current(); 84 | span.record("remote_node_id", remote_node_id.fmt_short()); 85 | info!("new connection"); 86 | 87 | let mut h3_conn = h3::server::Connection::new(h3_iroh::Connection::new(conn)).await?; 88 | loop { 89 | match h3_conn.accept().await { 90 | Ok(Some((req, stream))) => { 91 | info!(?req, "new request"); 92 | tokio::spawn({ 93 | let root = root.clone(); 94 | async move { 95 | if let Err(err) = handle_request(req, stream, root).await { 96 | error!("request failed: {err:#}"); 97 | } 98 | } 99 | .instrument(info_span!("req")) 100 | }); 101 | } 102 | Ok(None) => { 103 | break; 104 | } 105 | Err(err) => { 106 | error!("accept error: {err:#}"); 107 | match err.get_error_level() { 108 | ErrorLevel::ConnectionError => break, 109 | ErrorLevel::StreamError => continue, 110 | } 111 | } 112 | } 113 | } 114 | 115 | Ok(()) 116 | } 117 | 118 | async fn handle_request( 119 | req: Request<()>, 120 | mut stream: RequestStream, 121 | serve_root: Arc>, 122 | ) -> Result<()> 123 | where 124 | T: BidiStream, 125 | { 126 | let (status, file) = match serve_root.as_deref() { 127 | None => (StatusCode::OK, None), 128 | Some(_) if req.uri().path().contains("..") => (StatusCode::NOT_FOUND, None), 129 | Some(root) => { 130 | let path = root.join(req.uri().path().strip_prefix('/').unwrap_or("")); 131 | debug!(path = %path.display(), "Opening file"); 132 | match File::open(&path).await { 133 | Ok(file) => (StatusCode::OK, Some(file)), 134 | Err(err) => { 135 | error!(path = %path.to_string_lossy(), "failed to open file: {err:#}"); 136 | (StatusCode::NOT_FOUND, None) 137 | } 138 | } 139 | } 140 | }; 141 | 142 | let resp = http::Response::builder().status(status).body(())?; 143 | match stream.send_response(resp).await { 144 | Ok(_) => info!("success"), 145 | Err(err) => error!("unable to send response: {err:#}"), 146 | } 147 | if let Some(mut file) = file { 148 | loop { 149 | let mut buf = BytesMut::with_capacity(4096 * 10); 150 | if file.read_buf(&mut buf).await? == 0 { 151 | break; 152 | } 153 | stream.send_data(buf.freeze()).await?; 154 | } 155 | } 156 | stream.finish().await?; 157 | Ok(()) 158 | } 159 | -------------------------------------------------------------------------------- /h3-iroh/src/axum.rs: -------------------------------------------------------------------------------- 1 | //! Support for an axum server. 2 | 3 | use anyhow::Result; 4 | use axum::Router; 5 | use bytes::{Buf, Bytes}; 6 | use http::{Request, Response, Version}; 7 | use iroh::Endpoint; 8 | use tracing::{debug, error, info_span, trace, warn, Instrument}; 9 | 10 | use h3::{error::ErrorLevel, quic::BidiStream, server::RequestStream}; 11 | use http_body_util::BodyExt; 12 | use tower::Service; 13 | 14 | static ALPN: &[u8] = b"iroh+h3"; 15 | 16 | /// Serves an axum router over iroh using the h3 protocol. 17 | /// 18 | /// This implementation is not production ready. E.g. it copies the entire requests and 19 | /// responses in memory. It serves more as an example. 20 | pub async fn serve(endpoint: Endpoint, router: axum::Router) -> Result<()> { 21 | endpoint.set_alpns(vec![ALPN.to_vec()]); 22 | while let Some(incoming) = endpoint.accept().await { 23 | trace!("accepting connection"); 24 | let router = router.clone(); 25 | tokio::spawn( 26 | async move { 27 | if let Err(err) = handle_connection(incoming, router).await { 28 | warn!("error accepting connection: {err:#}"); 29 | } 30 | } 31 | .instrument(info_span!("h3-connection")), 32 | ); 33 | } 34 | endpoint.close().await; 35 | Ok(()) 36 | } 37 | 38 | async fn handle_connection(incoming: iroh::endpoint::Incoming, router: Router) -> Result<()> { 39 | debug!("new connection established"); 40 | let conn = incoming.await?; 41 | let conn = crate::Connection::new(conn); 42 | let mut conn = h3::server::Connection::new(conn).await?; 43 | loop { 44 | match conn.accept().await { 45 | Ok(Some((req, stream))) => { 46 | let router = router.clone(); 47 | tokio::spawn( 48 | async move { 49 | if let Err(err) = handle_request(req, stream, router.clone()).await { 50 | warn!("handling request failed: {err}"); 51 | } 52 | } 53 | .instrument(info_span!("h3-request")), 54 | ); 55 | } 56 | Ok(None) => { 57 | break; // No more streams to be recieved 58 | } 59 | 60 | Err(err) => { 61 | error!("accept error: {err}"); 62 | match err.get_error_level() { 63 | ErrorLevel::ConnectionError => break, 64 | ErrorLevel::StreamError => continue, 65 | } 66 | } 67 | } 68 | } 69 | Ok(()) 70 | } 71 | 72 | /// Handles a single request, buffering the entire request. 73 | async fn handle_request( 74 | req: Request<()>, 75 | mut stream: RequestStream, 76 | mut tower_service: Router, 77 | ) -> Result<()> 78 | where 79 | T: BidiStream, 80 | { 81 | debug!("new request: {:#?}", req); 82 | // TODO: All this copying is no good. 83 | let mut body = vec![]; 84 | while let Ok(Some(data)) = stream.recv_data().await { 85 | body.extend_from_slice(data.chunk()); 86 | } 87 | let body = http_body_util::Full::new(Bytes::from(body)); 88 | 89 | let mut builder = axum::extract::Request::builder() 90 | .version(req.version()) 91 | .uri(req.uri()) 92 | .method(req.method()); 93 | *builder.headers_mut().expect("builder invariant") = req.headers().clone(); 94 | let req = builder.body(body)?; 95 | 96 | // Call Service 97 | let res = tower_service.call(req).await?; 98 | 99 | let mut builder = Response::builder() 100 | .status(res.status()) 101 | .version(Version::HTTP_3); 102 | *builder.headers_mut().expect("builder invariant") = res.headers().clone(); 103 | let response = builder.body(())?; 104 | 105 | stream.send_response(response).await.unwrap(); 106 | 107 | // send response body and trailers 108 | let mut buf = res.into_body().into_data_stream(); 109 | while let Some(chunk) = buf.frame().await { 110 | match chunk { 111 | Ok(frame) => { 112 | if frame.is_data() { 113 | let data = frame.into_data().unwrap(); 114 | stream.send_data(data).await.unwrap(); 115 | } else if frame.is_trailers() { 116 | let trailers = frame.into_trailers().unwrap(); 117 | stream.send_trailers(trailers).await.unwrap(); 118 | } 119 | } 120 | Err(err) => { 121 | warn!("Failed to read frame from response body stream: {err:#}"); 122 | } 123 | } 124 | } 125 | trace!("done"); 126 | Ok(stream.finish().await?) 127 | } 128 | -------------------------------------------------------------------------------- /h3-iroh/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! QUIC Transport implementation with Quinn 2 | //! 3 | //! This module implements QUIC traits with Quinn. 4 | #![deny(missing_docs)] 5 | 6 | use std::convert::TryInto; 7 | use std::fmt::{self, Display}; 8 | use std::future::Future; 9 | use std::pin::Pin; 10 | use std::sync::Arc; 11 | use std::task::{self, Poll}; 12 | 13 | use bytes::{Buf, Bytes, BytesMut}; 14 | use futures::{ready, stream, Stream, StreamExt}; 15 | use h3::ext::Datagram; 16 | use h3::quic::{self, Error, StreamId, WriteBuf}; 17 | use iroh::endpoint::{self, ApplicationClose, ClosedStream, ReadDatagram}; 18 | use tokio_util::sync::ReusableBoxFuture; 19 | use tracing::instrument; 20 | 21 | pub use iroh::endpoint::{AcceptBi, AcceptUni, Endpoint, OpenBi, OpenUni, VarInt, WriteError}; 22 | 23 | #[cfg(feature = "axum")] 24 | pub mod axum; 25 | 26 | /// BoxStream with Sync trait 27 | type BoxStreamSync<'a, T> = Pin + Sync + Send + 'a>>; 28 | 29 | /// A QUIC connection backed by Quinn 30 | /// 31 | /// Implements a [`quic::Connection`] backed by a [`endpoint::Connection`]. 32 | pub struct Connection { 33 | conn: iroh::endpoint::Connection, 34 | incoming_bi: BoxStreamSync<'static, as Future>::Output>, 35 | opening_bi: Option as Future>::Output>>, 36 | incoming_uni: BoxStreamSync<'static, as Future>::Output>, 37 | opening_uni: Option as Future>::Output>>, 38 | datagrams: BoxStreamSync<'static, as Future>::Output>, 39 | } 40 | 41 | impl Connection { 42 | /// Create a [`Connection`] from a [`endpoint::Connection`] 43 | pub fn new(conn: iroh::endpoint::Connection) -> Self { 44 | Self { 45 | conn: conn.clone(), 46 | incoming_bi: Box::pin(stream::unfold(conn.clone(), |conn| async { 47 | Some((conn.accept_bi().await, conn)) 48 | })), 49 | opening_bi: None, 50 | incoming_uni: Box::pin(stream::unfold(conn.clone(), |conn| async { 51 | Some((conn.accept_uni().await, conn)) 52 | })), 53 | opening_uni: None, 54 | datagrams: Box::pin(stream::unfold(conn, |conn| async { 55 | Some((conn.read_datagram().await, conn)) 56 | })), 57 | } 58 | } 59 | } 60 | 61 | /// The error type for [`Connection`] 62 | /// 63 | /// Wraps reasons a Quinn connection might be lost. 64 | #[derive(Debug)] 65 | pub struct ConnectionError(endpoint::ConnectionError); 66 | 67 | impl std::error::Error for ConnectionError {} 68 | 69 | impl fmt::Display for ConnectionError { 70 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 71 | self.0.fmt(f) 72 | } 73 | } 74 | 75 | impl Error for ConnectionError { 76 | fn is_timeout(&self) -> bool { 77 | matches!(self.0, endpoint::ConnectionError::TimedOut) 78 | } 79 | 80 | fn err_code(&self) -> Option { 81 | match self.0 { 82 | endpoint::ConnectionError::ApplicationClosed(ApplicationClose { 83 | error_code, .. 84 | }) => Some(error_code.into_inner()), 85 | _ => None, 86 | } 87 | } 88 | } 89 | 90 | impl From for ConnectionError { 91 | fn from(e: endpoint::ConnectionError) -> Self { 92 | Self(e) 93 | } 94 | } 95 | 96 | /// Types of errors when sending a datagram. 97 | #[derive(Debug)] 98 | pub enum SendDatagramError { 99 | /// Datagrams are not supported by the peer 100 | UnsupportedByPeer, 101 | /// Datagrams are locally disabled 102 | Disabled, 103 | /// The datagram was too large to be sent. 104 | TooLarge, 105 | /// Network error 106 | ConnectionLost(Box), 107 | } 108 | 109 | impl fmt::Display for SendDatagramError { 110 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 111 | match self { 112 | SendDatagramError::UnsupportedByPeer => write!(f, "datagrams not supported by peer"), 113 | SendDatagramError::Disabled => write!(f, "datagram support disabled"), 114 | SendDatagramError::TooLarge => write!(f, "datagram too large"), 115 | SendDatagramError::ConnectionLost(_) => write!(f, "connection lost"), 116 | } 117 | } 118 | } 119 | 120 | impl std::error::Error for SendDatagramError {} 121 | 122 | impl Error for SendDatagramError { 123 | fn is_timeout(&self) -> bool { 124 | false 125 | } 126 | 127 | fn err_code(&self) -> Option { 128 | match self { 129 | Self::ConnectionLost(err) => err.err_code(), 130 | _ => None, 131 | } 132 | } 133 | } 134 | 135 | impl From for SendDatagramError { 136 | fn from(value: endpoint::SendDatagramError) -> Self { 137 | match value { 138 | endpoint::SendDatagramError::UnsupportedByPeer => Self::UnsupportedByPeer, 139 | endpoint::SendDatagramError::Disabled => Self::Disabled, 140 | endpoint::SendDatagramError::TooLarge => Self::TooLarge, 141 | endpoint::SendDatagramError::ConnectionLost(err) => { 142 | Self::ConnectionLost(ConnectionError::from(err).into()) 143 | } 144 | } 145 | } 146 | } 147 | 148 | impl quic::Connection for Connection 149 | where 150 | B: Buf, 151 | { 152 | type RecvStream = RecvStream; 153 | type OpenStreams = OpenStreams; 154 | type AcceptError = ConnectionError; 155 | 156 | #[instrument(skip_all, level = "trace")] 157 | fn poll_accept_bidi( 158 | &mut self, 159 | cx: &mut task::Context<'_>, 160 | ) -> Poll, Self::AcceptError>> { 161 | let (send, recv) = match ready!(self.incoming_bi.poll_next_unpin(cx)) { 162 | Some(x) => x?, 163 | None => return Poll::Ready(Ok(None)), 164 | }; 165 | Poll::Ready(Ok(Some(Self::BidiStream { 166 | send: Self::SendStream::new(send), 167 | recv: Self::RecvStream::new(recv), 168 | }))) 169 | } 170 | 171 | #[instrument(skip_all, level = "trace")] 172 | fn poll_accept_recv( 173 | &mut self, 174 | cx: &mut task::Context<'_>, 175 | ) -> Poll, Self::AcceptError>> { 176 | let recv = match ready!(self.incoming_uni.poll_next_unpin(cx)) { 177 | Some(x) => x?, 178 | None => return Poll::Ready(Ok(None)), 179 | }; 180 | Poll::Ready(Ok(Some(Self::RecvStream::new(recv)))) 181 | } 182 | 183 | fn opener(&self) -> Self::OpenStreams { 184 | OpenStreams { 185 | conn: self.conn.clone(), 186 | opening_bi: None, 187 | opening_uni: None, 188 | } 189 | } 190 | } 191 | 192 | impl quic::OpenStreams for Connection 193 | where 194 | B: Buf, 195 | { 196 | type SendStream = SendStream; 197 | type BidiStream = BidiStream; 198 | type OpenError = ConnectionError; 199 | 200 | #[instrument(skip_all, level = "trace")] 201 | fn poll_open_bidi( 202 | &mut self, 203 | cx: &mut task::Context<'_>, 204 | ) -> Poll> { 205 | if self.opening_bi.is_none() { 206 | self.opening_bi = Some(Box::pin(stream::unfold(self.conn.clone(), |conn| async { 207 | Some((conn.clone().open_bi().await, conn)) 208 | }))); 209 | } 210 | 211 | let (send, recv) = 212 | ready!(self.opening_bi.as_mut().unwrap().poll_next_unpin(cx)).unwrap()?; 213 | Poll::Ready(Ok(Self::BidiStream { 214 | send: Self::SendStream::new(send), 215 | recv: RecvStream::new(recv), 216 | })) 217 | } 218 | 219 | #[instrument(skip_all, level = "trace")] 220 | fn poll_open_send( 221 | &mut self, 222 | cx: &mut task::Context<'_>, 223 | ) -> Poll> { 224 | if self.opening_uni.is_none() { 225 | self.opening_uni = Some(Box::pin(stream::unfold(self.conn.clone(), |conn| async { 226 | Some((conn.open_uni().await, conn)) 227 | }))); 228 | } 229 | 230 | let send = ready!(self.opening_uni.as_mut().unwrap().poll_next_unpin(cx)).unwrap()?; 231 | Poll::Ready(Ok(Self::SendStream::new(send))) 232 | } 233 | 234 | #[instrument(skip_all, level = "trace")] 235 | fn close(&mut self, code: h3::error::Code, reason: &[u8]) { 236 | self.conn.close( 237 | VarInt::from_u64(code.value()).expect("error code VarInt"), 238 | reason, 239 | ); 240 | } 241 | } 242 | 243 | impl quic::SendDatagramExt for Connection 244 | where 245 | B: Buf, 246 | { 247 | type Error = SendDatagramError; 248 | 249 | #[instrument(skip_all, level = "trace")] 250 | fn send_datagram(&mut self, data: Datagram) -> Result<(), SendDatagramError> { 251 | // TODO investigate static buffer from known max datagram size 252 | let mut buf = BytesMut::new(); 253 | data.encode(&mut buf); 254 | self.conn.send_datagram(buf.freeze())?; 255 | 256 | Ok(()) 257 | } 258 | } 259 | 260 | impl quic::RecvDatagramExt for Connection { 261 | type Buf = Bytes; 262 | 263 | type Error = ConnectionError; 264 | 265 | #[inline] 266 | #[instrument(skip_all, level = "trace")] 267 | fn poll_accept_datagram( 268 | &mut self, 269 | cx: &mut task::Context<'_>, 270 | ) -> Poll, Self::Error>> { 271 | match ready!(self.datagrams.poll_next_unpin(cx)) { 272 | Some(Ok(x)) => Poll::Ready(Ok(Some(x))), 273 | Some(Err(e)) => Poll::Ready(Err(e.into())), 274 | None => Poll::Ready(Ok(None)), 275 | } 276 | } 277 | } 278 | 279 | /// Stream opener backed by a Quinn connection 280 | /// 281 | /// Implements [`quic::OpenStreams`] using [`endpoint::Connection`], 282 | /// [`endpoint::OpenBi`], [`endpoint::OpenUni`]. 283 | pub struct OpenStreams { 284 | conn: endpoint::Connection, 285 | opening_bi: Option as Future>::Output>>, 286 | opening_uni: Option as Future>::Output>>, 287 | } 288 | 289 | impl quic::OpenStreams for OpenStreams 290 | where 291 | B: Buf, 292 | { 293 | type SendStream = SendStream; 294 | type BidiStream = BidiStream; 295 | type OpenError = ConnectionError; 296 | 297 | #[instrument(skip_all, level = "trace")] 298 | fn poll_open_bidi( 299 | &mut self, 300 | cx: &mut task::Context<'_>, 301 | ) -> Poll> { 302 | if self.opening_bi.is_none() { 303 | self.opening_bi = Some(Box::pin(stream::unfold(self.conn.clone(), |conn| async { 304 | Some((conn.open_bi().await, conn)) 305 | }))); 306 | } 307 | 308 | let (send, recv) = 309 | ready!(self.opening_bi.as_mut().unwrap().poll_next_unpin(cx)).unwrap()?; 310 | Poll::Ready(Ok(Self::BidiStream { 311 | send: Self::SendStream::new(send), 312 | recv: RecvStream::new(recv), 313 | })) 314 | } 315 | 316 | #[instrument(skip_all, level = "trace")] 317 | fn poll_open_send( 318 | &mut self, 319 | cx: &mut task::Context<'_>, 320 | ) -> Poll> { 321 | if self.opening_uni.is_none() { 322 | self.opening_uni = Some(Box::pin(stream::unfold(self.conn.clone(), |conn| async { 323 | Some((conn.open_uni().await, conn)) 324 | }))); 325 | } 326 | 327 | let send = ready!(self.opening_uni.as_mut().unwrap().poll_next_unpin(cx)).unwrap()?; 328 | Poll::Ready(Ok(Self::SendStream::new(send))) 329 | } 330 | 331 | #[instrument(skip_all, level = "trace")] 332 | fn close(&mut self, code: h3::error::Code, reason: &[u8]) { 333 | self.conn.close( 334 | VarInt::from_u64(code.value()).expect("error code VarInt"), 335 | reason, 336 | ); 337 | } 338 | } 339 | 340 | impl Clone for OpenStreams { 341 | fn clone(&self) -> Self { 342 | Self { 343 | conn: self.conn.clone(), 344 | opening_bi: None, 345 | opening_uni: None, 346 | } 347 | } 348 | } 349 | 350 | /// Quinn-backed bidirectional stream 351 | /// 352 | /// Implements [`quic::BidiStream`] which allows the stream to be split 353 | /// into two structs each implementing one direction. 354 | pub struct BidiStream 355 | where 356 | B: Buf, 357 | { 358 | send: SendStream, 359 | recv: RecvStream, 360 | } 361 | 362 | impl quic::BidiStream for BidiStream 363 | where 364 | B: Buf, 365 | { 366 | type SendStream = SendStream; 367 | type RecvStream = RecvStream; 368 | 369 | fn split(self) -> (Self::SendStream, Self::RecvStream) { 370 | (self.send, self.recv) 371 | } 372 | } 373 | 374 | impl quic::RecvStream for BidiStream { 375 | type Buf = Bytes; 376 | type Error = ReadError; 377 | 378 | fn poll_data( 379 | &mut self, 380 | cx: &mut task::Context<'_>, 381 | ) -> Poll, Self::Error>> { 382 | self.recv.poll_data(cx) 383 | } 384 | 385 | fn stop_sending(&mut self, error_code: u64) { 386 | self.recv.stop_sending(error_code) 387 | } 388 | 389 | fn recv_id(&self) -> StreamId { 390 | self.recv.recv_id() 391 | } 392 | } 393 | 394 | impl quic::SendStream for BidiStream 395 | where 396 | B: Buf, 397 | { 398 | type Error = SendStreamError; 399 | 400 | fn poll_ready(&mut self, cx: &mut task::Context<'_>) -> Poll> { 401 | self.send.poll_ready(cx) 402 | } 403 | 404 | fn poll_finish(&mut self, cx: &mut task::Context<'_>) -> Poll> { 405 | self.send.poll_finish(cx) 406 | } 407 | 408 | fn reset(&mut self, reset_code: u64) { 409 | self.send.reset(reset_code) 410 | } 411 | 412 | fn send_data>>(&mut self, data: D) -> Result<(), Self::Error> { 413 | self.send.send_data(data) 414 | } 415 | 416 | fn send_id(&self) -> StreamId { 417 | self.send.send_id() 418 | } 419 | } 420 | impl quic::SendStreamUnframed for BidiStream 421 | where 422 | B: Buf, 423 | { 424 | fn poll_send( 425 | &mut self, 426 | cx: &mut task::Context<'_>, 427 | buf: &mut D, 428 | ) -> Poll> { 429 | self.send.poll_send(cx, buf) 430 | } 431 | } 432 | 433 | /// Quinn-backed receive stream 434 | /// 435 | /// Implements a [`quic::RecvStream`] backed by a [`endpoint::RecvStream`]. 436 | pub struct RecvStream { 437 | stream: Option, 438 | read_chunk_fut: ReadChunkFuture, 439 | } 440 | 441 | type ReadChunkFuture = ReusableBoxFuture< 442 | 'static, 443 | ( 444 | endpoint::RecvStream, 445 | Result, endpoint::ReadError>, 446 | ), 447 | >; 448 | 449 | impl RecvStream { 450 | fn new(stream: endpoint::RecvStream) -> Self { 451 | Self { 452 | stream: Some(stream), 453 | // Should only allocate once the first time it's used 454 | read_chunk_fut: ReusableBoxFuture::new(async { unreachable!() }), 455 | } 456 | } 457 | } 458 | 459 | impl quic::RecvStream for RecvStream { 460 | type Buf = Bytes; 461 | type Error = ReadError; 462 | 463 | #[instrument(skip_all, level = "trace")] 464 | fn poll_data( 465 | &mut self, 466 | cx: &mut task::Context<'_>, 467 | ) -> Poll, Self::Error>> { 468 | if let Some(mut stream) = self.stream.take() { 469 | self.read_chunk_fut.set(async move { 470 | let chunk = stream.read_chunk(usize::MAX, true).await; 471 | (stream, chunk) 472 | }) 473 | }; 474 | 475 | let (stream, chunk) = ready!(self.read_chunk_fut.poll(cx)); 476 | self.stream = Some(stream); 477 | Poll::Ready(Ok(chunk?.map(|c| c.bytes))) 478 | } 479 | 480 | #[instrument(skip_all, level = "trace")] 481 | fn stop_sending(&mut self, error_code: u64) { 482 | self.stream 483 | .as_mut() 484 | .unwrap() 485 | .stop(VarInt::from_u64(error_code).expect("invalid error_code")) 486 | .ok(); 487 | } 488 | 489 | #[instrument(skip_all, level = "trace")] 490 | fn recv_id(&self) -> StreamId { 491 | self.stream 492 | .as_ref() 493 | .unwrap() 494 | .id() 495 | .index() 496 | .try_into() 497 | .expect("invalid stream id") 498 | } 499 | } 500 | 501 | /// The error type for [`RecvStream`] 502 | /// 503 | /// Wraps errors that occur when reading from a receive stream. 504 | #[derive(Debug)] 505 | pub struct ReadError(endpoint::ReadError); 506 | 507 | impl From for std::io::Error { 508 | fn from(value: ReadError) -> Self { 509 | value.0.into() 510 | } 511 | } 512 | 513 | impl std::error::Error for ReadError { 514 | fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { 515 | self.0.source() 516 | } 517 | } 518 | 519 | impl fmt::Display for ReadError { 520 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 521 | self.0.fmt(f) 522 | } 523 | } 524 | 525 | impl From for Arc { 526 | fn from(e: ReadError) -> Self { 527 | Arc::new(e) 528 | } 529 | } 530 | 531 | impl From for ReadError { 532 | fn from(e: endpoint::ReadError) -> Self { 533 | Self(e) 534 | } 535 | } 536 | 537 | impl Error for ReadError { 538 | fn is_timeout(&self) -> bool { 539 | matches!( 540 | self.0, 541 | endpoint::ReadError::ConnectionLost(endpoint::ConnectionError::TimedOut) 542 | ) 543 | } 544 | 545 | fn err_code(&self) -> Option { 546 | match self.0 { 547 | endpoint::ReadError::ConnectionLost(endpoint::ConnectionError::ApplicationClosed( 548 | ApplicationClose { error_code, .. }, 549 | )) => Some(error_code.into_inner()), 550 | endpoint::ReadError::Reset(error_code) => Some(error_code.into_inner()), 551 | _ => None, 552 | } 553 | } 554 | } 555 | 556 | /// Quinn-backed send stream 557 | /// 558 | /// Implements a [`quic::SendStream`] backed by a [`endpoint::SendStream`]. 559 | pub struct SendStream { 560 | stream: Option, 561 | writing: Option>, 562 | write_fut: WriteFuture, 563 | } 564 | 565 | type WriteFuture = 566 | ReusableBoxFuture<'static, (endpoint::SendStream, Result)>; 567 | 568 | impl SendStream 569 | where 570 | B: Buf, 571 | { 572 | fn new(stream: endpoint::SendStream) -> SendStream { 573 | Self { 574 | stream: Some(stream), 575 | writing: None, 576 | write_fut: ReusableBoxFuture::new(async { unreachable!() }), 577 | } 578 | } 579 | } 580 | 581 | impl quic::SendStream for SendStream 582 | where 583 | B: Buf, 584 | { 585 | type Error = SendStreamError; 586 | 587 | #[instrument(skip_all, level = "trace")] 588 | fn poll_ready(&mut self, cx: &mut task::Context<'_>) -> Poll> { 589 | if let Some(ref mut data) = self.writing { 590 | while data.has_remaining() { 591 | if let Some(mut stream) = self.stream.take() { 592 | let chunk = data.chunk().to_owned(); // FIXME - avoid copy 593 | self.write_fut.set(async move { 594 | let ret = stream.write(&chunk).await; 595 | (stream, ret) 596 | }); 597 | } 598 | 599 | let (stream, res) = ready!(self.write_fut.poll(cx)); 600 | self.stream = Some(stream); 601 | match res { 602 | Ok(cnt) => data.advance(cnt), 603 | Err(err) => { 604 | return Poll::Ready(Err(SendStreamError::Write(err))); 605 | } 606 | } 607 | } 608 | } 609 | self.writing = None; 610 | Poll::Ready(Ok(())) 611 | } 612 | 613 | #[instrument(skip_all, level = "trace")] 614 | fn poll_finish(&mut self, _cx: &mut task::Context<'_>) -> Poll> { 615 | Poll::Ready(self.stream.as_mut().unwrap().finish().map_err(|e| e.into())) 616 | } 617 | 618 | #[instrument(skip_all, level = "trace")] 619 | fn reset(&mut self, reset_code: u64) { 620 | let _ = self 621 | .stream 622 | .as_mut() 623 | .unwrap() 624 | .reset(VarInt::from_u64(reset_code).unwrap_or(VarInt::MAX)); 625 | } 626 | 627 | #[instrument(skip_all, level = "trace")] 628 | fn send_data>>(&mut self, data: D) -> Result<(), Self::Error> { 629 | if self.writing.is_some() { 630 | return Err(Self::Error::NotReady); 631 | } 632 | self.writing = Some(data.into()); 633 | Ok(()) 634 | } 635 | 636 | #[instrument(skip_all, level = "trace")] 637 | fn send_id(&self) -> StreamId { 638 | self.stream 639 | .as_ref() 640 | .unwrap() 641 | .id() 642 | .index() 643 | .try_into() 644 | .expect("invalid stream id") 645 | } 646 | } 647 | 648 | impl quic::SendStreamUnframed for SendStream 649 | where 650 | B: Buf, 651 | { 652 | #[instrument(skip_all, level = "trace")] 653 | fn poll_send( 654 | &mut self, 655 | cx: &mut task::Context<'_>, 656 | buf: &mut D, 657 | ) -> Poll> { 658 | if self.writing.is_some() { 659 | // This signifies a bug in implementation 660 | panic!("poll_send called while send stream is not ready") 661 | } 662 | 663 | let s = Pin::new(self.stream.as_mut().unwrap()); 664 | 665 | let res = ready!(s.poll_write(cx, buf.chunk())); 666 | match res { 667 | Ok(written) => { 668 | buf.advance(written); 669 | Poll::Ready(Ok(written)) 670 | } 671 | Err(err) => Poll::Ready(Err(SendStreamError::Write(err))), 672 | } 673 | } 674 | } 675 | 676 | /// The error type for [`SendStream`] 677 | /// 678 | /// Wraps errors that can happen writing to or polling a send stream. 679 | #[derive(Debug)] 680 | pub enum SendStreamError { 681 | /// Errors when writing, wrapping a [`endpoint::WriteError`] 682 | Write(WriteError), 683 | /// Error when the stream is not ready, because it is still sending 684 | /// data from a previous call 685 | NotReady, 686 | /// Error when the stream is closed 687 | StreamClosed(ClosedStream), 688 | } 689 | 690 | impl From for std::io::Error { 691 | fn from(value: SendStreamError) -> Self { 692 | match value { 693 | SendStreamError::Write(err) => err.into(), 694 | SendStreamError::NotReady => { 695 | std::io::Error::new(std::io::ErrorKind::Other, "send stream is not ready") 696 | } 697 | SendStreamError::StreamClosed(err) => err.into(), 698 | } 699 | } 700 | } 701 | 702 | impl std::error::Error for SendStreamError {} 703 | 704 | impl Display for SendStreamError { 705 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 706 | write!(f, "{:?}", self) 707 | } 708 | } 709 | 710 | impl From for SendStreamError { 711 | fn from(e: WriteError) -> Self { 712 | Self::Write(e) 713 | } 714 | } 715 | 716 | impl From for SendStreamError { 717 | fn from(value: ClosedStream) -> Self { 718 | Self::StreamClosed(value) 719 | } 720 | } 721 | 722 | impl Error for SendStreamError { 723 | fn is_timeout(&self) -> bool { 724 | matches!( 725 | self, 726 | Self::Write(endpoint::WriteError::ConnectionLost( 727 | endpoint::ConnectionError::TimedOut 728 | )) 729 | ) 730 | } 731 | 732 | fn err_code(&self) -> Option { 733 | match self { 734 | Self::Write(endpoint::WriteError::Stopped(error_code)) => Some(error_code.into_inner()), 735 | Self::Write(endpoint::WriteError::ConnectionLost( 736 | endpoint::ConnectionError::ApplicationClosed(ApplicationClose { 737 | error_code, .. 738 | }), 739 | )) => Some(error_code.into_inner()), 740 | _ => None, 741 | } 742 | } 743 | } 744 | 745 | impl From for Arc { 746 | fn from(e: SendStreamError) -> Self { 747 | Arc::new(e) 748 | } 749 | } 750 | -------------------------------------------------------------------------------- /iroh-dag-sync/.gitignore: -------------------------------------------------------------------------------- 1 | **/*.db 2 | **/*.car 3 | **/secret.key 4 | -------------------------------------------------------------------------------- /iroh-dag-sync/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "iroh-dag-sync" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | [dependencies] 7 | iroh-blobs = "0.35" 8 | iroh-gossip = "0.35" 9 | iroh = "0.35" 10 | iroh-base = { version ="0.35", features = ["ticket"] } 11 | iroh-car = "0.5.0" 12 | redb = "2.1.1" 13 | clap = { version = "4.5.7", features = ["derive"] } 14 | tracing-subscriber = "0.3.18" 15 | anyhow = "1.0.86" 16 | tokio = { version = "1.38.0", features = ["full"] } 17 | postcard = "1.0.8" 18 | serde = { version = "1.0.203", features = ["derive"] } 19 | futures-lite = "2.3.0" 20 | iroh-io = "0.6" 21 | tokio-util = { version = "0.7.11", features = ["rt"] } 22 | bao-tree = "0.15.1" 23 | genawaiter = "0.99.1" 24 | bytes = "1.6.0" 25 | hex = "0.4.3" 26 | ron = "0.8.1" 27 | rand = "0.8.5" 28 | tracing = "0.1.40" 29 | serde_bytes = "0.11.14" 30 | ipld-core = "0.4.1" 31 | multihash = "0.19.1" 32 | cid = { version = "0.11.1", features = ["serde"] } 33 | multihash-codetable = "0.1.3" 34 | serde_ipld_dagcbor = "0.6.1" 35 | -------------------------------------------------------------------------------- /iroh-dag-sync/README.md: -------------------------------------------------------------------------------- 1 | # Iroh dag sync 2 | 3 | Example how to use iroh protocols such as gossip and iroh-bytes, as well as 4 | iroh components such as the blob store, to sync possibly very deep DAGs like you 5 | would have when working with IPFS data, unixfs directories etc. 6 | 7 | As an added complexity, we will support non-BLAKE3 hash functions. 8 | 9 | # Getting started 10 | 11 | - First, generate some data. 12 | 13 | We need a car file. You can just import some directory into ipfs and then export 14 | it as a car file. Make sure to use --raw-leaves to have a more interesting dag. 15 | 16 | Let's use the linux kernel sources as an example. 17 | 18 | ``` 19 | > ipfs add ../linux --raw-leaves 20 | > ipfs dag export QmWyLtd4WEJe45UBqCZG94gYY9B8qF3k4DKFX3o2bodHmV > linux.car 21 | ``` 22 | 23 | - Import the data: 24 | 25 | ``` 26 | > cargo run --release import linux.car 27 | ... 28 | root: QmWyLtd4WEJe45UBqCZG94gYY9B8qF3k4DKFX3o2bodHmV 29 | ``` 30 | 31 | This will create two databases in the current directory. dag.db contains 32 | information about the structure of the dag, blobs.db (a directory) contains 33 | the raw data. 34 | 35 | - Start a node that makes the data available 36 | 37 | ``` 38 | > cargo run --release node 39 | I am irgkesdtbih664hq2fjgd6zf7g6mazqkr7deqzplavmwl3vdbboa 40 | ``` 41 | 42 | - Now try to sync 43 | 44 | In a *different directory*, start the sync process: 45 | 46 | ``` 47 | > mkdir tmp 48 | > cd tmp 49 | > cargo run --release sync --from irgkesdtbih664hq2fjgd6zf7g6mazqkr7deqzplavmwl3vdbboa QmWyLtd4WEJe45UBqCZG94gYY9B8qF3k4DKFX3o2bodHmV 50 | ``` 51 | 52 | This will traverse the entire DAG in depth-first, pre-order, left-to-right 53 | traversal order. Which may take a while. But - it is just a single request/ 54 | response pair, so we will saturate the wire. 55 | 56 | - Export the synced data 57 | 58 | ``` 59 | > cargo run --release export QmWyLtd4WEJe45UBqCZG94gYY9B8qF3k4DKFX3o2bodHmV --target output.car 60 | ``` 61 | 62 | Export without specifying a target just dumps the cids to stdout. 63 | 64 | # Advanced use 65 | 66 | ## Specifying traversal options 67 | 68 | When traversing DAGs, you can specify not just the root of a dag, but a more 69 | complex traversal config in [ron] notation. 70 | 71 | E.g. the command line below will fully traverse the DAG, but omit all cids with 72 | a codec of Raw (0x55). 73 | 74 | This is the "stem" of the dag, all non-leaf nodes, or to be precise all nodes 75 | that could potentially contain links. 76 | 77 | ``` 78 | > cargo run --release export --traversal 'Full(root:"QmWyLtd4WEJe45UBqCZG94gYY9B8qF3k4DKFX3o2bodHmV",filter:NoRaw)' 79 | ``` 80 | 81 | ## Specifying inline options 82 | 83 | For each blob of a traversal, the sender runs a predicate that decides whether 84 | the data should be inlined or not. E.g. we might want to sync small dag nodes 85 | immediately, but leave syncing large leafs to a different protocol. 86 | 87 | We can specify the predicate using the inline argument. 88 | 89 | For inlined blobs, we will receive the blake3 hash and the data for each blob. 90 | For non-inlined blobs we will just receive the blake3 hash for the blob. We can 91 | then get the data by another means if we need it. 92 | 93 | E.g. the example below syncs the linux kernel, but only the non-leaf blocks. 94 | We get the mapping from cid to blake3 hash for all leaf blocks and can get them 95 | by another means if we need them. 96 | 97 | ``` 98 | > cargo run --release export --traversal 'Full(root:"QmWyLtd4WEJe45UBqCZG94gYY9B8qF3k4DKFX3o2bodHmV")' --inline NoRaw 99 | ``` 100 | 101 | **Security note**: we must not rely on the mapping from cid to blake3 102 | hash, but most validate the mapping once we get the data by another means. 103 | 104 | Otherwise somebody might force us to download completely unrelated data. For 105 | inlined data this check against the hash contained in the cid is already taken 106 | care of. 107 | 108 | # Implementation 109 | 110 | ## Local store 111 | 112 | ### Non-blake3 hashes 113 | 114 | We reuse the iroh-blobs store, but have an additional table that maps 115 | from a non-blake3 hash to a blake3 hash. This table is only populated 116 | with local, validated data and is therefore assumed to be correct. 117 | 118 | ### IPLD formats and links 119 | 120 | We have an additional table that contains a mapping from a blake3 hash 121 | and an ipld codec/format to a sequence of links. Links in this case are cids. 122 | 123 | ## Sync algorithm 124 | 125 | The sync algorithm is centered around deterministic traversals of DAGs. 126 | 127 | ### Deterministic traversals 128 | 129 | A deterministic traversal of a complete DAG is simple. *Any* traversal is 130 | deterministic as long as it does not intentionally introduce randomness using 131 | e.g. random number generators or use of randomized hash based data structures. 132 | 133 | A deterministic traversal for an incomplete DAG is simply a traversal that 134 | stops as soon as some block that might contain links can not be found. 135 | 136 | ### Sync algorithm details 137 | 138 | To sync between two nodes, alice (sender) and bob (receiver), bob configures a 139 | deterministic traversal, for example a root cid and a traversal order. He 140 | communicates this information to alice. 141 | 142 | Alice now executes the traversal and loads the data for each cid, then 143 | sends it back to bob as a bao4 encoded message. The message starts with the 144 | blake3 hash, the size as a le encoded u64, and then the bao encoded chunks of 145 | the data. We use a chunk group size of 4, so chunk groups are 2^4*1024 = 16 KiB. 146 | 147 | Bob executes *the same* deterministic traversal while receiving data from alice. 148 | Every item from alice corresponds to a cid in the deterministic traversal. 149 | 150 | As an item is received, bob validates the item by computing the non-blake3 hash, 151 | then adds the data to the iroh blob store, and extracts links from the blob 152 | according to the format contained in the cid. 153 | 154 | Only the reception of this additional data might allow the traversal on bob's 155 | side to continue. So it is important that the traversal is as lazy as possible 156 | in traversing blobs. 157 | 158 | ### Possible traversals 159 | 160 | The above approach will work for any traversal provided that it is 161 | deterministic and that the same traversal is executed on both sides. 162 | 163 | - A traversal that has a continuous path from the root to each DAG node is 164 | guaranteed to complete even if bob has incomplete data or no data at all, since 165 | the DAG is built from the root. E.g. a traversal of a DAG that *omits* leaf 166 | nodes. 167 | 168 | - A traversal that does _not_ have a continuous path from the root to each DAG 169 | node relies on data already being present. E.g. a traversal that only produces 170 | leaf nodes. It will stop if some of the data needed for the traversal is not 171 | already present on the receiver node. 172 | 173 | - Traversals can be composed. E.g. you could have a traversal that only produces 174 | leafs of a dag, then a second stage that lets though all cids where the hash 175 | ends with an odd number, or filters based on a bitmap. 176 | 177 | - The simplest possible traversal is to just return the root cid. Using this, 178 | this protocol can be used to retrieve individual blobs or sequences of unrelated 179 | blobs. 180 | 181 | ### Possible strategy to sync deep DAGs with lots of data. 182 | 183 | Assuming you are connected to several nodes that each have a chain-like DAG 184 | with some big data blobs hanging off each chain node. A possible strategy 185 | to quickly sync could be the following: 186 | 187 | - Sync the stem of the chain from multiple or even all neighbouring nodes. 188 | - Once this is done, or slightly staggered, sync the leafs of the chain 189 | from multiple nodes in such a way that the download work is divided. 190 | E.g. odd hashes from node A, even hashes from node B. 191 | 192 | Alternatively the second step could be done as multiple single-cid sync requests 193 | to neighbours in a round robin way. 194 | 195 | First step: just get the branch nodes 196 | ``` 197 | cargo run --release sync --from bsmlrj4sodhaivs2r7tssw4zeasqqr42lk6xt4e42ikzazkp4huq --traversal 'Full(root:"QmWyLtd4WEJe45UBqCZG94gYY9B8qF3k4DKFX3o2bodHmV",filter:NoRaw)' 198 | ``` 199 | 200 | Second step: get the leaf nodes. Note that this query requires the first query for the 201 | traversal on the receiver side to be even possible. 202 | ``` 203 | cargo run --release sync --from bsmlrj4sodhaivs2r7tssw4zeasqqr42lk6xt4e42ikzazkp4huq --traversal 'Full(root:"QmWyLtd4WEJe45UBqCZG94gYY9B8qF3k4DKFX3o2bodHmV",filter:JustRaw)' 204 | ``` 205 | 206 | ## Network protocol 207 | 208 | The network protocol is a basic request/response protocol, using a QUIC stream. 209 | 210 | 211 | ### Request 212 | 213 | A request consists of traversal options and options to configure the inline 214 | predicate. In this example we are using a simple postcard-encoded rust enum 215 | for this, and using [ron] for parsing the enum. 216 | 217 | What traversals there should be is probably highly project dependent, so 218 | typically you would just define a custom ALPN and then define a number of 219 | possible traversals that are appropriate for your project. 220 | 221 | These traversals are written in rust and can be complex and highly optimized, 222 | unlike if you had a generic language to define traversals. 223 | 224 | ### Response 225 | 226 | The response is a sequence of frames. Each frame contains a discriminator byte, 227 | a blake3 hash, and *optionally* the bao4 encoded raw data corresponding to the 228 | hash. 229 | 230 | The response can be incrementally verified every 16 KiB to match the blake3 231 | hash. Once the response is complete, it can be validated to match the non-blake3 232 | hash function in the cid corresponding to the response. 233 | 234 | [ron]: https://docs.rs/ron/0.8.1/ron/#rusty-object-notation -------------------------------------------------------------------------------- /iroh-dag-sync/src/args.rs: -------------------------------------------------------------------------------- 1 | use std::{ 2 | net::{SocketAddrV4, SocketAddrV6}, 3 | path::PathBuf, 4 | }; 5 | 6 | use clap::Parser; 7 | use iroh::NodeId; 8 | 9 | use crate::protocol::Cid; 10 | 11 | #[derive(Debug, Parser)] 12 | pub struct Args { 13 | #[clap(subcommand)] 14 | pub cmd: SubCommand, 15 | } 16 | 17 | #[derive(Debug, Parser)] 18 | pub enum SubCommand { 19 | Import(ImportArgs), 20 | Export(ExportArgs), 21 | Node(NodeArgs), 22 | Sync(SyncArgs), 23 | } 24 | 25 | #[derive(Debug, Parser)] 26 | pub struct ImportArgs { 27 | #[clap(help = "The path to the CAR file to import")] 28 | pub path: PathBuf, 29 | } 30 | 31 | #[derive(Debug, Parser)] 32 | pub struct ExportArgs { 33 | #[clap(help = "The root cid to traverse")] 34 | pub cid: Option, 35 | 36 | #[clap(long, help = "Traversal method to use, full if omitted")] 37 | pub traversal: Option, 38 | 39 | #[clap(long, help = "The path to the CAR file to export to")] 40 | pub target: Option, 41 | } 42 | 43 | #[derive(Debug, Parser)] 44 | pub struct NodeArgs { 45 | #[clap(flatten)] 46 | pub net: NetArgs, 47 | } 48 | 49 | #[derive(Debug, Parser)] 50 | pub struct NetArgs { 51 | /// The IPv4 addr to listen on. 52 | #[clap(long)] 53 | pub iroh_ipv4_addr: Option, 54 | /// The IPv6 addr to listen on. 55 | #[clap(long)] 56 | pub iroh_ipv6_addr: Option, 57 | } 58 | 59 | #[derive(Debug, Parser)] 60 | pub struct SyncArgs { 61 | #[clap(flatten)] 62 | pub net: NetArgs, 63 | #[clap(help = "The root cid to sync")] 64 | pub root: Option, 65 | #[clap(long, help = "Traversal method to use, full if omitted")] 66 | pub traversal: Option, 67 | #[clap(long, help = "Which data to send inline")] 68 | pub inline: Option, 69 | #[clap(long, help = "The node to sync from")] 70 | pub from: NodeId, 71 | } 72 | -------------------------------------------------------------------------------- /iroh-dag-sync/src/main.rs: -------------------------------------------------------------------------------- 1 | use std::net::{SocketAddrV4, SocketAddrV6}; 2 | use std::sync::Arc; 3 | 4 | use anyhow::Context; 5 | use clap::Parser; 6 | use futures_lite::StreamExt; 7 | use ipld_core::codec::Links; 8 | use iroh::discovery::{dns::DnsDiscovery, pkarr::PkarrPublisher, ConcurrentDiscovery}; 9 | use iroh::NodeAddr; 10 | use iroh_base::ticket::NodeTicket; 11 | use iroh_blobs::store::{Map, MapEntry}; 12 | use iroh_blobs::{store::Store, BlobFormat}; 13 | use iroh_car::CarReader; 14 | use iroh_io::AsyncSliceReaderExt; 15 | use protocol::{ron_parser, Cid, Request}; 16 | use serde::{Deserialize, Serialize}; 17 | use sync::{handle_request, handle_sync_response}; 18 | use tables::{ReadOnlyTables, ReadableTables, Tables}; 19 | use tokio::io::AsyncWriteExt; 20 | use tokio_util::task::LocalPoolHandle; 21 | use traversal::{get_traversal, Traversal}; 22 | 23 | mod args; 24 | mod protocol; 25 | mod sync; 26 | mod tables; 27 | mod traversal; 28 | mod util; 29 | 30 | use args::Args; 31 | 32 | const SYNC_ALPN: &[u8] = b"DAG_SYNC/1"; 33 | 34 | async fn create_endpoint( 35 | ipv4_addr: Option, 36 | ipv6_addr: Option, 37 | ) -> anyhow::Result { 38 | let secret_key = util::get_or_create_secret()?; 39 | let discovery = Box::new(ConcurrentDiscovery::from_services(vec![ 40 | Box::new(DnsDiscovery::n0_dns()), 41 | Box::new(PkarrPublisher::n0_dns(secret_key.clone())), 42 | ])); 43 | 44 | let mut builder = iroh::Endpoint::builder() 45 | .secret_key(secret_key) 46 | .alpns(vec![SYNC_ALPN.to_vec()]) 47 | .discovery(discovery); 48 | if let Some(addr) = ipv4_addr { 49 | builder = builder.bind_addr_v4(addr); 50 | } 51 | if let Some(addr) = ipv6_addr { 52 | builder = builder.bind_addr_v6(addr); 53 | } 54 | 55 | let endpoint = builder.bind().await?; 56 | 57 | Ok(endpoint) 58 | } 59 | 60 | fn init_mapping_db(db: &redb::Database) -> anyhow::Result<()> { 61 | let tx = db.begin_write()?; 62 | tables::Tables::new(&tx)?; 63 | tx.commit()?; 64 | Ok(()) 65 | } 66 | 67 | #[tokio::main] 68 | async fn main() -> anyhow::Result<()> { 69 | tracing_subscriber::fmt::init(); 70 | let args = Args::parse(); 71 | let store = iroh_blobs::store::fs::Store::load("blobs.db").await?; 72 | let mapping_store = redb::Database::create("dag.db")?; 73 | init_mapping_db(&mapping_store)?; 74 | let rt = LocalPoolHandle::new(1); 75 | 76 | match args.cmd { 77 | args::SubCommand::Import(import_args) => { 78 | let tx = mapping_store.begin_write()?; 79 | let mut tables = tables::Tables::new(&tx)?; 80 | let file = tokio::fs::File::open(import_args.path).await?; 81 | let reader = CarReader::new(file).await?; 82 | let stream = reader.stream().enumerate(); 83 | tokio::pin!(stream); 84 | let mut first = None; 85 | while let Some((i, block)) = stream.next().await { 86 | let (cid, data) = block?; 87 | if first.is_none() { 88 | first = Some(cid); 89 | } 90 | let links: Vec<_> = 91 | serde_ipld_dagcbor::codec::DagCborCodec::links(&data)?.collect(); 92 | let tag = store.import_bytes(data.into(), BlobFormat::Raw).await?; 93 | let hash = tag.hash(); 94 | if !links.is_empty() { 95 | println!("{} {} {}", i, cid, links.len()); 96 | let links = serde_ipld_dagcbor::to_vec(&links)?; 97 | tables.data_to_links.insert((cid.codec(), *hash), links)?; 98 | } else { 99 | println!("{} {}", i, cid); 100 | } 101 | tables 102 | .hash_to_blake3 103 | .insert((cid.hash().code(), cid.hash().digest()), hash)?; 104 | } 105 | drop(tables); 106 | tx.commit()?; 107 | store.sync().await?; 108 | if let Some(first) = first { 109 | println!("root: {}", first); 110 | } 111 | } 112 | args::SubCommand::Export(args) => { 113 | let tx = mapping_store.begin_read()?; 114 | let tables = tables::ReadOnlyTables::new(&tx)?; 115 | let opts = protocol::TraversalOpts::from_args(&args.cid, &args.traversal)?; 116 | println!("using traversal: '{}'", ron_parser().to_string(&opts)?); 117 | let traversal = get_traversal(opts, &tables)?; 118 | match args.target { 119 | Some(target) => { 120 | let file = tokio::fs::File::create(target).await?; 121 | export_traversal(traversal, &store, file).await? 122 | } 123 | None => print_traversal(traversal, &store).await?, 124 | } 125 | } 126 | args::SubCommand::Node(args) => { 127 | let endpoint = 128 | create_endpoint(args.net.iroh_ipv4_addr, args.net.iroh_ipv6_addr).await?; 129 | endpoint.home_relay().initialized().await?; 130 | let addr = endpoint.node_addr().await?; 131 | println!("Node id:\n{}", addr.node_id); 132 | println!( 133 | "Listening on {:#?}, {:#?}", 134 | addr.relay_url, addr.direct_addresses 135 | ); 136 | println!("ticket:\n{}", NodeTicket::new(addr.clone())); 137 | while let Some(incoming) = endpoint.accept().await { 138 | let mut connecting = incoming.accept()?; 139 | let alpn = connecting.alpn().await?; 140 | match alpn.as_ref() { 141 | SYNC_ALPN => { 142 | let tx = mapping_store.begin_read()?; 143 | let tables = ReadOnlyTables::new(&tx)?; 144 | let store = store.clone(); 145 | rt.spawn_pinned(move || async move { 146 | handle_request(connecting, &tables, &store).await?; 147 | anyhow::Ok(()) 148 | }); 149 | } 150 | _ => { 151 | eprintln!("Unknown ALPN: {:?}", alpn); 152 | } 153 | } 154 | } 155 | } 156 | args::SubCommand::Sync(args) => { 157 | let endpoint = 158 | create_endpoint(args.net.iroh_ipv4_addr, args.net.iroh_ipv6_addr).await?; 159 | let traversal = protocol::TraversalOpts::from_args(&args.root, &args.traversal)?; 160 | println!("using traversal: '{}'", ron_parser().to_string(&traversal)?); 161 | let inline = protocol::InlineOpts::from_args(&args.inline)?; 162 | println!("using inline: '{}'", ron_parser().to_string(&inline)?); 163 | let tx = mapping_store.begin_write()?; 164 | let mut tables = Tables::new(&tx)?; 165 | let store = store.clone(); 166 | let endpoint = Arc::new(endpoint); 167 | let node = NodeAddr::from(args.from); 168 | let connection = endpoint.connect(node, SYNC_ALPN).await?; 169 | let request = protocol::Request::Sync(protocol::SyncRequest { 170 | traversal: traversal.clone(), 171 | inline, 172 | }); 173 | tracing::info!("sending request: {:?}", request); 174 | let request = postcard::to_allocvec(&request)?; 175 | tracing::info!("sending request: {} bytes", request.len()); 176 | tracing::info!("sending request: {}", hex::encode(&request)); 177 | let roundtrip: Request = postcard::from_bytes(&request).unwrap(); 178 | tracing::info!("roundtrip: {:?}", roundtrip); 179 | let (mut send, recv) = connection.open_bi().await?; 180 | send.write_all(&request).await?; 181 | send.finish()?; 182 | handle_sync_response(recv, &mut tables, &store, traversal).await?; 183 | drop(tables); 184 | tx.commit()?; 185 | } 186 | } 187 | 188 | Ok(()) 189 | } 190 | 191 | async fn print_traversal( 192 | traversal: T, 193 | store: &iroh_blobs::store::fs::Store, 194 | ) -> anyhow::Result<()> 195 | where 196 | T: Traversal, 197 | T::Db: ReadableTables, 198 | { 199 | let mut traversal = traversal; 200 | let mut first = None; 201 | let mut n = 0; 202 | while let Some(cid) = traversal.next().await? { 203 | if first.is_none() { 204 | first = Some(cid); 205 | } 206 | let blake3_hash = traversal 207 | .db_mut() 208 | .blake3_hash(cid.hash())? 209 | .context("blake3 hash not found")?; 210 | let data = store.get(&blake3_hash).await?.context("data not found")?; 211 | println!("{} {:x} {} {}", cid, cid.codec(), data.size().value(), n); 212 | n += 1; 213 | } 214 | 215 | if let Some(first) = first { 216 | println!("root: {}", first); 217 | } 218 | Ok(()) 219 | } 220 | 221 | async fn export_traversal( 222 | traversal: T, 223 | store: &iroh_blobs::store::fs::Store, 224 | mut file: tokio::fs::File, 225 | ) -> anyhow::Result<()> 226 | where 227 | T: Traversal, 228 | T::Db: ReadableTables, 229 | { 230 | #[derive(Serialize, Deserialize)] 231 | struct CarFileHeader { 232 | version: u64, 233 | roots: Vec, 234 | } 235 | 236 | let header = CarFileHeader { 237 | version: 1, 238 | roots: traversal.roots(), 239 | }; 240 | let header_bytes = serde_ipld_dagcbor::to_vec(&header)?; 241 | file.write_all(&postcard::to_allocvec(&(header_bytes.len() as u64))?) 242 | .await?; 243 | file.write_all(&header_bytes).await?; 244 | let mut traversal = traversal; 245 | let mut buffer = [0u8; 9]; 246 | while let Some(cid) = traversal.next().await? { 247 | let blake3_hash = traversal 248 | .db_mut() 249 | .blake3_hash(cid.hash())? 250 | .context("blake3 hash not found")?; 251 | let handle = store.get(&blake3_hash).await?.context("data not found")?; 252 | let data = handle.data_reader().read_to_end().await?; 253 | let mut block_bytes = cid.to_bytes(); // postcard::to_extend(&RawCidHeader::from_cid(&cid), Vec::new())?; 254 | // block_bytes.extend_from_slice(&cid.hash().digest()); // hash 255 | block_bytes.extend_from_slice(&data); 256 | let size: u64 = block_bytes.len() as u64; 257 | file.write_all(postcard::to_slice(&size, &mut buffer)?) 258 | .await?; 259 | file.write_all(&block_bytes).await?; 260 | } 261 | file.sync_all().await?; 262 | Ok(()) 263 | } 264 | -------------------------------------------------------------------------------- /iroh-dag-sync/src/protocol.rs: -------------------------------------------------------------------------------- 1 | use std::{collections::BTreeSet, fmt::Display, ops::Deref, str::FromStr}; 2 | 3 | use iroh_blobs::Hash; 4 | use serde::{Deserialize, Serialize}; 5 | use tokio::io::AsyncRead; 6 | 7 | #[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Clone)] 8 | pub enum Request { 9 | Sync(SyncRequest), 10 | } 11 | 12 | /// A wrapper around a Cid that can be serialized and deserialized. 13 | /// 14 | /// Yes, I know cid has a feature for serde, but I had some issues with DagCbor 15 | /// derive when using it, probably because libipld does not have the latest cid. 16 | #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Clone, Copy)] 17 | pub struct Cid(cid::Cid); 18 | 19 | impl Serialize for Cid { 20 | fn serialize(&self, serializer: S) -> Result { 21 | if serializer.is_human_readable() { 22 | serializer.serialize_str(&self.0.to_string()) 23 | } else { 24 | serializer.serialize_bytes(&self.0.to_bytes()) 25 | } 26 | } 27 | } 28 | 29 | impl<'de> Deserialize<'de> for Cid { 30 | fn deserialize>(deserializer: D) -> Result { 31 | if deserializer.is_human_readable() { 32 | let s = String::deserialize(deserializer)?; 33 | Ok(Cid(cid::Cid::try_from(s).map_err(serde::de::Error::custom)?)) 34 | } else { 35 | let bytes = serde_bytes::ByteBuf::deserialize(deserializer)?; 36 | Ok(Cid( 37 | cid::Cid::try_from(bytes.into_vec()).map_err(serde::de::Error::custom)? 38 | )) 39 | } 40 | } 41 | } 42 | 43 | impl From for Cid { 44 | fn from(cid: cid::Cid) -> Self { 45 | Self(cid) 46 | } 47 | } 48 | 49 | impl From for cid::Cid { 50 | fn from(scid: Cid) -> Self { 51 | scid.0 52 | } 53 | } 54 | 55 | impl Deref for Cid { 56 | type Target = cid::Cid; 57 | 58 | fn deref(&self) -> &Self::Target { 59 | &self.0 60 | } 61 | } 62 | 63 | impl std::fmt::Display for Cid { 64 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 65 | self.0.fmt(f) 66 | } 67 | } 68 | 69 | impl std::str::FromStr for Cid { 70 | type Err = cid::Error; 71 | 72 | fn from_str(s: &str) -> Result { 73 | Ok(Cid(cid::Cid::try_from(s)?)) 74 | } 75 | } 76 | 77 | /// A sync request is a request to sync a DAG from a remote node. 78 | /// 79 | /// The request contains basically a ipfs cid and a dag traversal method. 80 | /// 81 | /// The response is a sequence of blocks consisting of a sync response header 82 | /// and bao encoded block data. 83 | #[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Clone)] 84 | pub struct SyncRequest { 85 | /// walk method. must be one of the registered ones 86 | pub traversal: TraversalOpts, 87 | /// which data to send inline 88 | pub inline: InlineOpts, 89 | } 90 | 91 | #[derive(Debug, Serialize, Deserialize)] 92 | pub enum SyncResponseHeader { 93 | Hash(Hash), 94 | Data(Hash), 95 | } 96 | 97 | impl SyncResponseHeader { 98 | pub fn as_bytes(&self) -> [u8; 33] { 99 | let mut slice = [0u8; 33]; 100 | let res = postcard::to_slice(self, &mut slice).unwrap(); 101 | assert!(res.len() == slice.len()); 102 | slice 103 | } 104 | 105 | pub async fn from_stream(mut stream: impl AsyncRead + Unpin) -> anyhow::Result> { 106 | use tokio::io::AsyncReadExt; 107 | let mut buf = [0u8; 33]; 108 | match stream.read_exact(&mut buf).await { 109 | Ok(_) => Ok(Some(postcard::from_bytes(&buf)?)), 110 | Err(e) if e.kind() == std::io::ErrorKind::UnexpectedEof => Ok(None), 111 | Err(e) => Err(e.into()), 112 | } 113 | } 114 | } 115 | 116 | #[derive(Debug, Serialize, Deserialize)] 117 | #[repr(u8)] 118 | pub enum DataMode { 119 | /// no data is included 120 | None = 0, 121 | /// data is included 122 | Inline = 1, 123 | } 124 | 125 | /// Options to configure a traversal 126 | /// 127 | /// The exact traversal options will probably be project specific, since we don't 128 | /// want to come up with a generic language to specify graph traversals. 129 | /// 130 | /// Having a small number of highly optimized traversal methods under a custom 131 | /// protocol/APLN is probably best for specialized use cases. 132 | /// 133 | /// This is just an example of possible traversal options. 134 | #[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Clone)] 135 | pub enum TraversalOpts { 136 | /// A sequence of individual cids. 137 | /// 138 | /// This can be used to sync a single block or a sequence of unrelated blocks. 139 | /// Note that since we are getting individual blocks, the codec part of the cids 140 | /// is not relevant. 141 | Sequence(SequenceTraversalOpts), 142 | /// A full traversal of a DAG, with a set of already visited cids. 143 | Full(FullTraversalOpts), 144 | } 145 | 146 | #[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Clone)] 147 | pub struct SequenceTraversalOpts( 148 | /// The sequence of cids to traverse, in order. 149 | pub Vec, 150 | ); 151 | 152 | #[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Clone)] 153 | pub struct FullTraversalOpts { 154 | /// The root of the traversal. 155 | /// 156 | /// The codec part of the cid is relevant. E.g. for a cid with codec raw, 157 | /// 0x55, the traversal would always be just the root. 158 | pub root: Cid, 159 | /// The set of already visited cids. This can be used to abort a traversal 160 | /// once data that is already known is encountered. 161 | /// 162 | /// E.g. in case of a linked list shaped dag, you would insert here 163 | /// the cid of the last element that you have locally. 164 | pub visited: Option>, 165 | /// The order in which to traverse the DAG. 166 | /// 167 | /// Since a traversal will abort once a cid is encountered that is not 168 | /// present, this can influence how much data is fetched. 169 | #[serde(default)] 170 | pub order: Option, 171 | /// Filter to apply to the traversal. 172 | #[serde(default)] 173 | pub filter: Option, 174 | } 175 | 176 | #[derive(Debug, Serialize, Deserialize, Default, PartialEq, Eq, PartialOrd, Ord, Clone)] 177 | pub enum TraversalOrder { 178 | #[default] 179 | DepthFirstPreOrderLeftToRight, 180 | } 181 | 182 | #[derive(Debug, Serialize, Deserialize, Default, PartialEq, Eq, PartialOrd, Ord, Clone)] 183 | pub enum TraversalFilter { 184 | /// Include all cids. 185 | #[default] 186 | All, 187 | /// Exclude raw cids. 188 | NoRaw, 189 | /// Just raw cids. 190 | JustRaw, 191 | /// Exclude cids with a specific codec. 192 | Excude(BTreeSet), 193 | } 194 | 195 | #[derive(Debug, Serialize, Deserialize, Default, PartialEq, Eq, PartialOrd, Ord, Clone)] 196 | pub enum InlineOpts { 197 | /// Include data for all cids. 198 | #[default] 199 | All, 200 | /// Exclude raw cids. 201 | NoRaw, 202 | /// Exclude cids with a specific codec. 203 | Excude(BTreeSet), 204 | /// Never inline data. 205 | None, 206 | } 207 | 208 | impl InlineOpts { 209 | pub fn from_args(arg: &Option) -> anyhow::Result { 210 | match arg.as_ref() { 211 | Some(arg) => Ok(ron_parser().from_str(arg)?), 212 | None => Ok(InlineOpts::All), 213 | } 214 | } 215 | } 216 | 217 | pub fn ron_parser() -> ::ron::Options { 218 | ron::Options::default() 219 | .with_default_extension(ron::extensions::Extensions::IMPLICIT_SOME) 220 | .with_default_extension(ron::extensions::Extensions::UNWRAP_NEWTYPES) 221 | .with_default_extension(ron::extensions::Extensions::UNWRAP_VARIANT_NEWTYPES) 222 | } 223 | 224 | impl Display for TraversalOpts { 225 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 226 | let res = ron_parser().to_string(self).unwrap(); 227 | write!(f, "{}", res) 228 | } 229 | } 230 | 231 | impl FromStr for TraversalOpts { 232 | type Err = ron::de::SpannedError; 233 | 234 | fn from_str(s: &str) -> Result { 235 | ron_parser().from_str(s) 236 | } 237 | } 238 | 239 | impl TraversalOpts { 240 | pub fn from_args(root: &Option, traversal: &Option) -> anyhow::Result { 241 | match (root, traversal) { 242 | (Some(root), None) => Ok(TraversalOpts::Full(FullTraversalOpts { 243 | root: *root, 244 | visited: Default::default(), 245 | order: Default::default(), 246 | filter: Default::default(), 247 | })), 248 | (None, Some(traversal)) => Ok(TraversalOpts::from_str(traversal)?), 249 | (Some(_), Some(_)) => { 250 | anyhow::bail!("Either root or traversal must be specified, not both") 251 | } 252 | (None, None) => anyhow::bail!("Either root or traversal must be specified"), 253 | } 254 | } 255 | } 256 | 257 | #[cfg(test)] 258 | mod tests { 259 | use std::str::FromStr; 260 | 261 | use crate::protocol::{ 262 | ron_parser, Cid, FullTraversalOpts, InlineOpts, Request, SyncRequest, TraversalOpts, 263 | }; 264 | 265 | #[test] 266 | fn cid_json_roundtrip() { 267 | let ron = ron_parser(); 268 | let cid = Cid::from_str("QmWyLtd4WEJe45UBqCZG94gYY9B8qF3k4DKFX3o2bodHmV").unwrap(); 269 | let json = ron.to_string(&cid).unwrap(); 270 | let cid2 = ron.from_str(&json).unwrap(); 271 | assert_eq!(cid, cid2); 272 | } 273 | 274 | #[test] 275 | fn cid_postcard_roundtrip() { 276 | let cid = Cid::from_str("QmWyLtd4WEJe45UBqCZG94gYY9B8qF3k4DKFX3o2bodHmV").unwrap(); 277 | let bytes = postcard::to_allocvec(&cid).unwrap(); 278 | let cid2 = postcard::from_bytes(&bytes).unwrap(); 279 | assert_eq!(cid, cid2); 280 | } 281 | 282 | #[test] 283 | fn opts_postcard_roundtrip() { 284 | let cid = Cid::from_str("QmWyLtd4WEJe45UBqCZG94gYY9B8qF3k4DKFX3o2bodHmV").unwrap(); 285 | let opts = TraversalOpts::Full(FullTraversalOpts { 286 | root: cid, 287 | visited: Default::default(), 288 | order: Default::default(), 289 | filter: Default::default(), 290 | }); 291 | let bytes = postcard::to_allocvec(&opts).unwrap(); 292 | let opts2: TraversalOpts = postcard::from_bytes(&bytes).unwrap(); 293 | assert_eq!(opts, opts2); 294 | } 295 | 296 | #[test] 297 | fn request_postcard_roundtrip() { 298 | let cid = Cid::from_str("QmWyLtd4WEJe45UBqCZG94gYY9B8qF3k4DKFX3o2bodHmV").unwrap(); 299 | let opts = TraversalOpts::Full(FullTraversalOpts { 300 | root: cid, 301 | visited: Default::default(), 302 | order: Default::default(), 303 | filter: Default::default(), 304 | }); 305 | let request = Request::Sync(SyncRequest { 306 | traversal: opts, 307 | inline: InlineOpts::All, 308 | }); 309 | let bytes = postcard::to_allocvec(&request).unwrap(); 310 | let request2: Request = postcard::from_bytes(&bytes).unwrap(); 311 | assert_eq!(request, request2); 312 | } 313 | } 314 | -------------------------------------------------------------------------------- /iroh-dag-sync/src/sync.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Context; 2 | use bao_tree::{io::outboard::EmptyOutboard, BaoTree, ChunkRanges}; 3 | use iroh::endpoint::{Connecting, RecvStream, SendStream}; 4 | use iroh_blobs::{ 5 | protocol::RangeSpec, 6 | provider::{send_blob, EventSender}, 7 | store::{fs::Store, Store as _}, 8 | BlobFormat, IROH_BLOCK_SIZE, 9 | }; 10 | use iroh_io::{TokioStreamReader, TokioStreamWriter}; 11 | use multihash_codetable::MultihashDigest; 12 | use tokio::io::AsyncReadExt; 13 | 14 | use crate::{ 15 | protocol::{Cid, Request, SyncRequest, SyncResponseHeader, TraversalOpts}, 16 | tables::{ReadableTables, Tables}, 17 | traversal::{get_inline, get_traversal, Traversal}, 18 | }; 19 | 20 | const MAX_REQUEST_SIZE: usize = 1024 * 1024 * 16; 21 | 22 | pub async fn handle_request( 23 | mut connecting: Connecting, 24 | tables: &impl ReadableTables, 25 | blobs: &Store, 26 | ) -> anyhow::Result<()> { 27 | tracing::info!( 28 | "got connecting, {:?}", 29 | std::str::from_utf8(&connecting.alpn().await?) 30 | ); 31 | let connection = connecting.await?; 32 | tracing::info!("got connection, waiting for request"); 33 | let (send, mut recv) = connection.accept_bi().await?; 34 | tracing::info!("got request stream"); 35 | let request = recv.read_to_end(MAX_REQUEST_SIZE).await?; 36 | tracing::info!("got request message: {} bytes", request.len()); 37 | let request = postcard::from_bytes::(&request)?; 38 | tracing::info!("got request: {:?}", request); 39 | match request { 40 | Request::Sync(args) => { 41 | handle_sync_request(send, args, tables, blobs).await?; 42 | } 43 | } 44 | Ok(()) 45 | } 46 | 47 | pub async fn handle_sync_request( 48 | send: SendStream, 49 | request: SyncRequest, 50 | tables: &impl ReadableTables, 51 | blobs: &Store, 52 | ) -> anyhow::Result<()> { 53 | let traversal = get_traversal(request.traversal, tables)?; 54 | let inline = get_inline(request.inline)?; 55 | write_sync_response(send, traversal, blobs, inline).await?; 56 | Ok(()) 57 | } 58 | 59 | async fn write_sync_response( 60 | send: SendStream, 61 | traversal: T, 62 | blobs: &Store, 63 | inline: impl Fn(&Cid) -> bool, 64 | ) -> anyhow::Result<()> 65 | where 66 | T::Db: ReadableTables, 67 | { 68 | let mut traversal = traversal; 69 | // wrap the send stream in a TokioStreamWriter so we can use it from send_blob 70 | let mut send = TokioStreamWriter(send); 71 | while let Some(cid) = traversal.next().await? { 72 | let hash = traversal 73 | .db_mut() 74 | .blake3_hash(cid.hash())? 75 | .context("blake3 hash not found")?; 76 | if inline(&cid) { 77 | send.0 78 | .write_all(&SyncResponseHeader::Data(hash).as_bytes()) 79 | .await?; 80 | 81 | // TODO(ramfox): not exactly sure what this should be 82 | // Would be nice to have this be optional, or to have an empty Event 83 | let mk_progress = |end_offset| iroh_blobs::provider::Event::TransferProgress { 84 | connection_id: 0, 85 | request_id: 0, 86 | hash, 87 | end_offset, 88 | }; 89 | send_blob::( 90 | blobs, 91 | hash, 92 | &RangeSpec::all(), 93 | &mut send, 94 | EventSender::new(None), 95 | mk_progress, 96 | ) 97 | .await?; 98 | } else { 99 | send.0 100 | .write_all(&SyncResponseHeader::Hash(hash).as_bytes()) 101 | .await?; 102 | } 103 | } 104 | send.0.finish()?; 105 | Ok(()) 106 | } 107 | 108 | pub async fn handle_sync_response( 109 | recv: RecvStream, 110 | tables: &mut Tables<'_>, 111 | store: &Store, 112 | traversal: TraversalOpts, 113 | ) -> anyhow::Result<()> { 114 | let mut reader = TokioStreamReader(recv); 115 | let mut traversal = get_traversal(traversal, tables)?; 116 | loop { 117 | let Some(cid) = traversal.next().await? else { 118 | break; 119 | }; 120 | let Some(header) = SyncResponseHeader::from_stream(&mut reader.0).await? else { 121 | break; 122 | }; 123 | println!("{} {:?}", cid, header); 124 | let blake3_hash = match header { 125 | SyncResponseHeader::Hash(blake3_hash) => { 126 | // todo: get the data via another request 127 | println!("just got hash mapping {} {}", cid, blake3_hash); 128 | continue; 129 | } 130 | SyncResponseHeader::Data(hash) => hash, 131 | }; 132 | let size = reader.0.read_u64_le().await?; 133 | let outboard = EmptyOutboard { 134 | tree: BaoTree::new(size, IROH_BLOCK_SIZE), 135 | root: blake3_hash.into(), 136 | }; 137 | let mut buffer = Vec::new(); 138 | bao_tree::io::fsm::decode_ranges(&mut reader, ChunkRanges::all(), &mut buffer, outboard) 139 | .await?; 140 | let hasher = multihash_codetable::Code::try_from(cid.hash().code())?; 141 | let actual = hasher.digest(&buffer); 142 | if &actual != cid.hash() { 143 | anyhow::bail!("user hash mismatch"); 144 | } 145 | let data = bytes::Bytes::from(buffer); 146 | let tag = store.import_bytes(data.clone(), BlobFormat::Raw).await?; 147 | if tag.hash() != &blake3_hash { 148 | anyhow::bail!("blake3 hash mismatch"); 149 | } 150 | traversal.db_mut().insert_links(&cid, blake3_hash, &data)?; 151 | } 152 | Ok(()) 153 | } 154 | -------------------------------------------------------------------------------- /iroh-dag-sync/src/tables.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Context; 2 | use ipld_core::codec::Links; 3 | use iroh_blobs::Hash; 4 | use multihash::Multihash; 5 | use redb::{ReadableTable, TableDefinition}; 6 | 7 | /// Table mapping ipld hash to blake3 hash. 8 | const HASH_TO_BLAKE3: TableDefinition<(u64, &[u8]), Hash> = TableDefinition::new("hash_to_blake3"); 9 | /// Table mapping ipld format and blake3 hash to contained links 10 | /// 11 | /// For blobs containing no links, there should not be an entry in this table. 12 | const DATA_TO_LINKS: TableDefinition<(u64, Hash), Vec> = TableDefinition::new("data_to_links"); 13 | 14 | pub trait ReadableTables { 15 | fn hash_to_blake3(&self) -> &impl redb::ReadableTable<(u64, &'static [u8]), Hash>; 16 | fn data_to_links(&self) -> &impl redb::ReadableTable<(u64, Hash), Vec>; 17 | 18 | #[allow(dead_code)] 19 | fn has_links(&self, cid: &cid::Cid) -> anyhow::Result { 20 | let hash = self 21 | .hash_to_blake3() 22 | .get((cid.hash().code(), cid.hash().digest()))? 23 | .context("blake3 hash not found")?; 24 | Ok(self 25 | .data_to_links() 26 | .get((cid.codec(), hash.value()))? 27 | .is_some()) 28 | } 29 | 30 | /// Get the stored links for a given ipld hash. 31 | fn links(&self, cid: &cid::Cid) -> anyhow::Result>> { 32 | let hash = self 33 | .hash_to_blake3() 34 | .get((cid.hash().code(), cid.hash().digest()))? 35 | .context("blake3 hash not found")?; 36 | let Some(links) = self.data_to_links().get((cid.codec(), hash.value()))? else { 37 | return Ok(None); 38 | }; 39 | Ok(Some(serde_ipld_dagcbor::from_slice(&links.value())?)) 40 | } 41 | 42 | /// Get the blake3 hash for a given ipld hash. 43 | fn blake3_hash(&self, hash: &Multihash) -> anyhow::Result> { 44 | Ok(self 45 | .hash_to_blake3() 46 | .get((hash.code(), hash.digest()))? 47 | .map(|x| x.value())) 48 | } 49 | } 50 | 51 | impl ReadableTables for &T { 52 | fn hash_to_blake3(&self) -> &impl redb::ReadableTable<(u64, &'static [u8]), Hash> { 53 | (*self).hash_to_blake3() 54 | } 55 | 56 | fn data_to_links(&self) -> &impl redb::ReadableTable<(u64, Hash), Vec> { 57 | (*self).data_to_links() 58 | } 59 | } 60 | 61 | impl ReadableTables for &mut T { 62 | fn hash_to_blake3(&self) -> &impl redb::ReadableTable<(u64, &'static [u8]), Hash> { 63 | ReadableTables::hash_to_blake3(*self) 64 | } 65 | 66 | fn data_to_links(&self) -> &impl redb::ReadableTable<(u64, Hash), Vec> { 67 | ReadableTables::data_to_links(*self) 68 | } 69 | } 70 | 71 | pub struct Tables<'tx> { 72 | pub hash_to_blake3: redb::Table<'tx, (u64, &'static [u8]), Hash>, 73 | pub data_to_links: redb::Table<'tx, (u64, Hash), Vec>, 74 | } 75 | 76 | impl<'tx> Tables<'tx> { 77 | pub fn new(tx: &'tx redb::WriteTransaction) -> std::result::Result { 78 | Ok(Self { 79 | hash_to_blake3: tx.open_table(HASH_TO_BLAKE3)?, 80 | data_to_links: tx.open_table(DATA_TO_LINKS)?, 81 | }) 82 | } 83 | 84 | pub fn insert_links(&mut self, cid: &cid::Cid, hash: Hash, data: &[u8]) -> anyhow::Result<()> { 85 | let links: Vec<_> = serde_ipld_dagcbor::codec::DagCborCodec::links(data)?.collect(); 86 | self.hash_to_blake3 87 | .insert((cid.hash().code(), cid.hash().digest()), hash)?; 88 | if !links.is_empty() { 89 | let links = serde_ipld_dagcbor::to_vec(&links)?; 90 | self.data_to_links.insert((cid.codec(), hash), links)?; 91 | } 92 | Ok(()) 93 | } 94 | } 95 | 96 | impl ReadableTables for Tables<'_> { 97 | fn hash_to_blake3(&self) -> &impl redb::ReadableTable<(u64, &'static [u8]), Hash> { 98 | &self.hash_to_blake3 99 | } 100 | 101 | fn data_to_links(&self) -> &impl redb::ReadableTable<(u64, Hash), Vec> { 102 | &self.data_to_links 103 | } 104 | } 105 | 106 | pub struct ReadOnlyTables { 107 | pub hash_to_blake3: redb::ReadOnlyTable<(u64, &'static [u8]), Hash>, 108 | pub data_to_links: redb::ReadOnlyTable<(u64, Hash), Vec>, 109 | } 110 | 111 | impl ReadOnlyTables { 112 | pub fn new(tx: &redb::ReadTransaction) -> std::result::Result { 113 | Ok(Self { 114 | hash_to_blake3: tx.open_table(HASH_TO_BLAKE3)?, 115 | data_to_links: tx.open_table(DATA_TO_LINKS)?, 116 | }) 117 | } 118 | } 119 | 120 | impl ReadableTables for ReadOnlyTables { 121 | fn hash_to_blake3(&self) -> &impl redb::ReadableTable<(u64, &'static [u8]), Hash> { 122 | &self.hash_to_blake3 123 | } 124 | 125 | fn data_to_links(&self) -> &impl redb::ReadableTable<(u64, Hash), Vec> { 126 | &self.data_to_links 127 | } 128 | } 129 | -------------------------------------------------------------------------------- /iroh-dag-sync/src/traversal.rs: -------------------------------------------------------------------------------- 1 | use std::{collections::HashSet, pin::Pin}; 2 | 3 | use futures_lite::Future; 4 | 5 | use crate::{ 6 | protocol::{ 7 | Cid, FullTraversalOpts, InlineOpts, SequenceTraversalOpts, TraversalFilter, TraversalOpts, 8 | }, 9 | tables::ReadableTables, 10 | }; 11 | 12 | /// A DAG traversal over a database. 13 | /// 14 | /// This is very similar to a stream of Cids, but gives mutable access to the 15 | /// database between calls to `next` to perform changes. 16 | pub trait Traversal { 17 | type Db; 18 | 19 | fn roots(&self) -> Vec; 20 | 21 | fn next(&mut self) -> impl Future>>; 22 | 23 | fn db_mut(&mut self) -> &mut Self::Db; 24 | 25 | fn filter bool>(self, f: F) -> Filtered 26 | where 27 | Self: Sized, 28 | { 29 | Filtered { 30 | inner: self, 31 | filter: f, 32 | } 33 | } 34 | 35 | fn boxed<'a>(self) -> BoxedTraversal<'a, Self::Db> 36 | where 37 | Self: Sized + Unpin + 'a, 38 | { 39 | BoxedTraversal(Box::pin(BoxableTraversalImpl { inner: self })) 40 | } 41 | } 42 | 43 | pub struct BoxedTraversal<'a, D>(Pin + Unpin + 'a>>); 44 | 45 | impl Traversal for BoxedTraversal<'_, D> { 46 | type Db = D; 47 | 48 | fn roots(&self) -> Vec { 49 | self.0.roots() 50 | } 51 | 52 | async fn next(&mut self) -> anyhow::Result> { 53 | self.0.next().await 54 | } 55 | 56 | fn db_mut(&mut self) -> &mut D { 57 | self.0.db_mut() 58 | } 59 | } 60 | 61 | struct BoxableTraversalImpl> { 62 | inner: T, 63 | } 64 | 65 | trait BoxableTraversal { 66 | fn next(&mut self) -> Pin>> + '_>>; 67 | fn db_mut(&mut self) -> &mut D; 68 | fn roots(&self) -> Vec; 69 | } 70 | 71 | impl> BoxableTraversal for BoxableTraversalImpl { 72 | fn next(&mut self) -> Pin>> + '_>> { 73 | Box::pin(self.inner.next()) 74 | } 75 | 76 | fn db_mut(&mut self) -> &mut D { 77 | self.inner.db_mut() 78 | } 79 | 80 | fn roots(&self) -> Vec { 81 | self.inner.roots() 82 | } 83 | } 84 | 85 | pub struct Filtered { 86 | inner: T, 87 | filter: F, 88 | } 89 | 90 | impl bool> Traversal for Filtered { 91 | type Db = T::Db; 92 | 93 | async fn next(&mut self) -> anyhow::Result> { 94 | while let Some(item) = self.inner.next().await? { 95 | if (self.filter)(&item) { 96 | return Ok(Some(item)); 97 | } 98 | } 99 | Ok(None) 100 | } 101 | 102 | fn db_mut(&mut self) -> &mut Self::Db { 103 | self.inner.db_mut() 104 | } 105 | 106 | fn roots(&self) -> Vec { 107 | self.inner.roots() 108 | } 109 | } 110 | 111 | pub struct SequenceTraversal { 112 | cids: std::vec::IntoIter, 113 | db: T, 114 | } 115 | 116 | impl SequenceTraversal { 117 | pub fn new(db: D, cids: Vec) -> Self { 118 | Self { 119 | cids: cids.into_iter(), 120 | db, 121 | } 122 | } 123 | } 124 | 125 | impl Traversal for SequenceTraversal { 126 | type Db = D; 127 | 128 | async fn next(&mut self) -> anyhow::Result> { 129 | Ok(self.cids.next()) 130 | } 131 | 132 | fn db_mut(&mut self) -> &mut D { 133 | &mut self.db 134 | } 135 | 136 | fn roots(&self) -> Vec { 137 | self.cids.clone().collect() 138 | } 139 | } 140 | 141 | pub struct FullTraversal { 142 | prev: Option, 143 | stack: Vec, 144 | visited: HashSet, 145 | db: D, 146 | } 147 | 148 | impl FullTraversal { 149 | pub fn new(db: D, root: Cid, visited: HashSet) -> Self { 150 | let stack = vec![root]; 151 | Self { 152 | stack, 153 | visited, 154 | db, 155 | prev: None, 156 | } 157 | } 158 | } 159 | 160 | impl Traversal for FullTraversal { 161 | type Db = D; 162 | 163 | fn roots(&self) -> Vec { 164 | self.stack.clone() 165 | } 166 | 167 | async fn next(&mut self) -> anyhow::Result> { 168 | loop { 169 | // perform deferred work 170 | if let Some(cid) = self.prev.take() { 171 | if cid.codec() == 0x55 { 172 | // no need to traverse raw nodes 173 | continue; 174 | } 175 | if let Some(links) = self.db.links(&cid)? { 176 | for link in links.into_iter().rev() { 177 | self.stack.push(link.into()); 178 | } 179 | } 180 | } 181 | let Some(cid) = self.stack.pop() else { 182 | break; 183 | }; 184 | if self.visited.contains(&cid) { 185 | continue; 186 | } 187 | self.visited.insert(cid); 188 | // defer the reading of the data etc, since we might not have the data yet 189 | self.prev = Some(cid); 190 | return Ok(Some(cid)); 191 | } 192 | Ok(None) 193 | } 194 | 195 | fn db_mut(&mut self) -> &mut D { 196 | &mut self.db 197 | } 198 | } 199 | 200 | pub fn get_traversal<'a, D: ReadableTables + Unpin + 'a>( 201 | opts: TraversalOpts, 202 | db: D, 203 | ) -> anyhow::Result> { 204 | Ok(match opts { 205 | TraversalOpts::Sequence(SequenceTraversalOpts(cids)) => { 206 | SequenceTraversal::new(db, cids).boxed() 207 | } 208 | TraversalOpts::Full(FullTraversalOpts { 209 | root, 210 | visited, 211 | filter, 212 | .. 213 | }) => { 214 | let visited = visited.unwrap_or_default(); 215 | let filter = filter.unwrap_or_default(); 216 | let traversal = FullTraversal::new(db, root, visited.into_iter().collect()); 217 | match filter { 218 | TraversalFilter::All => traversal.boxed(), 219 | TraversalFilter::NoRaw => traversal.filter(|cid| cid.codec() != 0x55).boxed(), 220 | TraversalFilter::JustRaw => traversal.filter(|cid| cid.codec() == 0x55).boxed(), 221 | TraversalFilter::Excude(codecs) => { 222 | let codecs: HashSet = codecs.into_iter().collect(); 223 | traversal 224 | .filter(move |cid| !codecs.contains(&cid.codec())) 225 | .boxed() 226 | } 227 | } 228 | } 229 | }) 230 | } 231 | 232 | pub type InlineCb = Box bool>; 233 | 234 | pub fn get_inline(inline: InlineOpts) -> anyhow::Result { 235 | Ok(match inline { 236 | InlineOpts::All => Box::new(|_| true), 237 | InlineOpts::NoRaw => Box::new(|cid| cid.codec() != 0x55), 238 | InlineOpts::Excude(codecs) => { 239 | let codecs: HashSet = codecs.into_iter().collect(); 240 | Box::new(move |cid| !codecs.contains(&cid.codec())) 241 | } 242 | InlineOpts::None => Box::new(|_| false), 243 | }) 244 | } 245 | -------------------------------------------------------------------------------- /iroh-dag-sync/src/util.rs: -------------------------------------------------------------------------------- 1 | use anyhow::anyhow; 2 | use iroh::SecretKey; 3 | 4 | /// Get the secret key from a file or generate a new one. 5 | pub fn get_or_create_secret() -> anyhow::Result { 6 | const FILENAME: &str = "secret.key"; 7 | match std::fs::read(FILENAME) { 8 | Ok(data) => { 9 | let secret = 10 | SecretKey::from_bytes(&data.try_into().map_err(|_| anyhow!("invalid secret key"))?); 11 | Ok(secret) 12 | } 13 | Err(_) => { 14 | let secret = SecretKey::generate(rand::thread_rng()); 15 | std::fs::write(FILENAME, secret.to_bytes())?; 16 | Ok(secret) 17 | } 18 | } 19 | } 20 | 21 | // /// Serialize a serializable object with a varint length prefix. 22 | // pub fn to_framed(s: &S) -> anyhow::Result> { 23 | // let mut data = postcard::to_allocvec(s)?; 24 | // let mut header = [0u8; 9]; 25 | // let header = postcard::to_slice(&(data.len() as u64), &mut header)?; 26 | // data.splice(0..0, header.into_iter().map(|x| *x)); 27 | // Ok(data) 28 | // } 29 | 30 | // /// Deserialize a serializable object with a varint length prefix. 31 | // /// 32 | // /// Note that this relies on that the minimum size of the content is more than 8 bytes. 33 | // pub async fn read_framed( 34 | // mut reader: impl AsyncRead + Unpin, 35 | // ) -> anyhow::Result> { 36 | // let mut header = [0u8; 9]; 37 | // match reader.read_exact(&mut header).await { 38 | // Ok(_) => {} 39 | // Err(e) if e.kind() == std::io::ErrorKind::UnexpectedEof => return Ok(None), 40 | // Err(e) => return Err(e.into()), 41 | // } 42 | // let (len, rest) = postcard::take_from_bytes::(&header)?; 43 | // let mut data = vec![0u8; len as usize]; 44 | // data[0..rest.len()].copy_from_slice(rest); 45 | // reader.read_exact(&mut data[rest.len()..]).await?; 46 | // let data = postcard::from_bytes(&data)?; 47 | // Ok(Some(data)) 48 | // } 49 | -------------------------------------------------------------------------------- /iroh-pkarr-naming-system/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "iroh-pkarr-naming-system" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | anyhow = "1.0.79" 10 | derive_more = "0.99.17" 11 | iroh = "0.34" 12 | iroh-blobs = "0.34" 13 | pkarr = { version = "2.3.1", features = ["async", "dht"] } 14 | tokio = "1.35.1" 15 | tokio-util = "0.7.12" 16 | tracing = "0.1.40" 17 | tracing-subscriber = { version = "0.3", features = ["env-filter"] } 18 | -------------------------------------------------------------------------------- /iroh-pkarr-naming-system/README.md: -------------------------------------------------------------------------------- 1 | # IPNS, the iroh pkarr naming system 2 | 3 | This is a minimalist reimplementation of the idea of [ipfs_ipns] using the 4 | bittorrent mainline DHT via [pkarr]. 5 | 6 | It can be used to publish an iroh blake3 content hash under an ed25519 public 7 | key, and to retrieve the latest content hash. 8 | 9 | [pkarr]: https://pkarr.org 10 | [ipfs_ipns]: https://docs.ipfs.tech/concepts/ipns/ 11 | -------------------------------------------------------------------------------- /iroh-pkarr-naming-system/examples/cli.rs: -------------------------------------------------------------------------------- 1 | use iroh_blobs::{ticket::BlobTicket, Hash, HashAndFormat}; 2 | use iroh_pkarr_naming_system::{Record, IPNS}; 3 | use std::{fmt::Display, process, str::FromStr}; 4 | 5 | /// Various ways to specify content. 6 | #[derive(Debug, Clone, derive_more::From)] 7 | pub enum ContentArg { 8 | Hash(Hash), 9 | HashAndFormat(HashAndFormat), 10 | Ticket(BlobTicket), 11 | } 12 | 13 | impl ContentArg { 14 | /// Get the hash and format of the content. 15 | pub fn hash_and_format(&self) -> HashAndFormat { 16 | match self { 17 | ContentArg::Hash(hash) => HashAndFormat::raw(*hash), 18 | ContentArg::HashAndFormat(haf) => *haf, 19 | ContentArg::Ticket(ticket) => HashAndFormat { 20 | hash: ticket.hash(), 21 | format: ticket.format(), 22 | }, 23 | } 24 | } 25 | } 26 | 27 | impl Display for ContentArg { 28 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 29 | match self { 30 | ContentArg::Hash(hash) => Display::fmt(hash, f), 31 | ContentArg::HashAndFormat(haf) => Display::fmt(haf, f), 32 | ContentArg::Ticket(ticket) => Display::fmt(ticket, f), 33 | } 34 | } 35 | } 36 | 37 | impl FromStr for ContentArg { 38 | type Err = anyhow::Error; 39 | 40 | fn from_str(s: &str) -> Result { 41 | if let Ok(hash) = Hash::from_str(s) { 42 | Ok(hash.into()) 43 | } else if let Ok(haf) = HashAndFormat::from_str(s) { 44 | Ok(haf.into()) 45 | } else if let Ok(ticket) = BlobTicket::from_str(s) { 46 | Ok(ticket.into()) 47 | } else { 48 | anyhow::bail!("invalid hash and format") 49 | } 50 | } 51 | } 52 | 53 | #[tokio::main] 54 | async fn main() -> anyhow::Result<()> { 55 | tracing_subscriber::fmt::init(); 56 | let args = std::env::args(); 57 | let args = args.into_iter().skip(1).collect::>(); 58 | match args.len() { 59 | // resolve a record 60 | 1 => { 61 | let public_key = iroh::PublicKey::from_str(&args[0])?; 62 | let ipns = IPNS::new()?; 63 | let record = ipns.resolve(public_key).await?; 64 | if let Some(Record::Content { content }) = record { 65 | println!("Found content {}", content); 66 | } else { 67 | println!("No record found"); 68 | } 69 | } 70 | // publish a record 71 | 2 => { 72 | let secret_key = iroh::SecretKey::from_str(&args[0])?; 73 | let public_key = secret_key.public(); 74 | let zid = pkarr::PublicKey::try_from(public_key.as_bytes())?.to_z32(); 75 | let content = ContentArg::from_str(&args[1])?.hash_and_format(); 76 | let record = Record::Content { content }; 77 | let ipns = IPNS::new()?; 78 | println!("Publishing record to: {}", public_key); 79 | println!("pkarr z32: {}", zid); 80 | println!("see https://app.pkarr.org/?pk={}", zid); 81 | ipns.publish(secret_key, Some(record)).await?; 82 | tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; 83 | } 84 | _ => { 85 | println!("Usage: cli # publish a record"); 86 | println!("Usage: cli # resolve a record"); 87 | process::exit(1); 88 | } 89 | } 90 | Ok(()) 91 | } 92 | -------------------------------------------------------------------------------- /iroh-pkarr-naming-system/src/lib.rs: -------------------------------------------------------------------------------- 1 | use std::{ 2 | collections::BTreeMap, 3 | str::FromStr, 4 | sync::{Arc, Mutex}, 5 | time::Duration, 6 | }; 7 | 8 | use anyhow::Context; 9 | use iroh::SecretKey; 10 | use iroh_blobs::HashAndFormat; 11 | use pkarr::{ 12 | dns::{ 13 | rdata::{RData, TXT}, 14 | Name, Packet, ResourceRecord, CLASS, 15 | }, 16 | SignedPacket, 17 | }; 18 | use tokio_util::task::AbortOnDropHandle; 19 | 20 | /// The key for the content of an IPNS record. 21 | const CONTENT_KEY: &str = "_content.iroh."; 22 | /// Republish delay for the DHT. This is only for when the info does not change. 23 | /// If the info changes, it will be published immediately. 24 | const REPUBLISH_DELAY: Duration = Duration::from_secs(60 * 60); 25 | /// Initial publish delay. This is to avoid spamming the DHT when there are 26 | /// frequent network changes at startup. 27 | const INITIAL_PUBLISH_DELAY: Duration = Duration::from_millis(500); 28 | 29 | /// An IPNS record. 30 | /// 31 | /// This is a record that can be published to the iroh pkarr naming system. 32 | #[derive(Clone, Debug)] 33 | pub enum Record { 34 | /// Content only. 35 | Content { content: HashAndFormat }, 36 | } 37 | 38 | impl Record { 39 | fn content(&self) -> Option<&HashAndFormat> { 40 | match self { 41 | Record::Content { content } => Some(content), 42 | } 43 | } 44 | } 45 | 46 | /// An iroh pkarr naming system publisher constantly republishes any number of records. 47 | #[derive(Clone, Debug)] 48 | pub struct IPNS(Arc); 49 | 50 | #[derive(Debug)] 51 | struct Inner { 52 | pkarr: Arc, 53 | packets: Mutex)>>, 54 | } 55 | 56 | impl IPNS { 57 | /// Create a new default IPNS publisher. 58 | pub fn new() -> anyhow::Result { 59 | let inner = Inner { 60 | pkarr: Arc::new(pkarr::PkarrClientBuilder::default().build()?), 61 | packets: Mutex::new(BTreeMap::default()), 62 | }; 63 | Ok(Self(Arc::new(inner))) 64 | } 65 | 66 | /// Publish a record for a keypair, or stop publishing if `record` is `None`. 67 | pub async fn publish( 68 | &self, 69 | secret_key: SecretKey, 70 | record: Option, 71 | ) -> anyhow::Result<()> { 72 | let key = secret_key.public(); 73 | if let Some(record) = record { 74 | let pkarr = self.0.pkarr.clone(); 75 | let signed_packet: SignedPacket = Self::to_signed_packet(&secret_key, &record, 0)?; 76 | let publish_task = tokio::spawn(async move { 77 | tokio::time::sleep(INITIAL_PUBLISH_DELAY).await; 78 | loop { 79 | let res = pkarr.publish(&signed_packet); 80 | match res { 81 | Ok(()) => { 82 | tracing::info!("Published record"); 83 | } 84 | Err(e) => { 85 | tracing::warn!("Failed to publish record: {}", e); 86 | } 87 | } 88 | tokio::time::sleep(REPUBLISH_DELAY).await; 89 | } 90 | }); 91 | let mut packets = self.0.packets.lock().unwrap(); 92 | packets.insert(key, (record, AbortOnDropHandle::new(publish_task))); 93 | } else { 94 | let mut packets = self.0.packets.lock().unwrap(); 95 | packets.remove(&key); 96 | }; 97 | Ok(()) 98 | } 99 | 100 | /// Resolve a record for a public key. 101 | pub async fn resolve(&self, public_key: iroh::PublicKey) -> anyhow::Result> { 102 | let public_key = 103 | pkarr::PublicKey::try_from(public_key.as_bytes()).context("invalid public key")?; 104 | let packet = self.0.pkarr.resolve(&public_key)?; 105 | packet.map(Self::to_record).transpose() 106 | } 107 | 108 | /// Produce a signed packet for a record. 109 | fn to_signed_packet( 110 | secret_key: &SecretKey, 111 | record: &Record, 112 | ttl: u32, 113 | ) -> anyhow::Result { 114 | let keypair = pkarr::Keypair::from_secret_key(&secret_key.to_bytes()); 115 | let mut packet = Packet::new_reply(0); 116 | if let Some(content) = record.content() { 117 | packet.answers.push(ResourceRecord::new( 118 | Name::new(CONTENT_KEY).unwrap(), 119 | CLASS::IN, 120 | ttl, 121 | RData::TXT(TXT::try_from(content.to_string().as_str())?.into_owned()), 122 | )); 123 | } 124 | Ok(SignedPacket::from_packet(&keypair, &packet)?) 125 | } 126 | 127 | fn to_record(packet: SignedPacket) -> anyhow::Result { 128 | // first DERP URL, if any 129 | let content = packet 130 | .resource_records(CONTENT_KEY) 131 | .filter_map(filter_txt) 132 | .map(|url| anyhow::Ok(HashAndFormat::from_str(&url)?)) 133 | .next() 134 | .transpose()? 135 | .context("no content found in IPNS record")?; 136 | 137 | Ok(Record::Content { content }) 138 | } 139 | } 140 | 141 | fn filter_txt(rr: &ResourceRecord) -> Option { 142 | if rr.class != CLASS::IN { 143 | return None; 144 | } 145 | if let RData::TXT(txt) = &rr.rdata { 146 | String::try_from(txt.clone()).ok() 147 | } else { 148 | None 149 | } 150 | } 151 | -------------------------------------------------------------------------------- /iroh-s3-bao-store/.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | tmp/* 3 | .sendme-* 4 | -------------------------------------------------------------------------------- /iroh-s3-bao-store/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "iroh-s3-bao-store" 3 | version = "0.1.0" 4 | edition = "2021" 5 | authors = ["Rüdiger Klaehn ", "n0 team"] 6 | keywords = ["scp", "sftp", "network", "p2p", "holepunching"] 7 | categories = ["network-programming"] 8 | license = "Apache-2.0/MIT" 9 | repository = "https://github.com/n0-computer/dumb-pipe" 10 | description = "A cli tool to send directories over the network, with NAT hole punching" 11 | rust-version = "1.75" 12 | 13 | [dependencies] 14 | anyhow = "1.0.75" 15 | bao-tree = "0.15.1" 16 | base32 = "0.4.0" 17 | bytes = "1.5.0" 18 | clap = { version = "4.4.10", features = ["derive"] } 19 | console = "0.15.7" 20 | flume = "0.11.0" 21 | futures-lite = "2.3" 22 | hex = "0.4.3" 23 | indicatif = "0.17.7" 24 | iroh = "0.35" 25 | iroh-blobs = "0.35" 26 | iroh-io = { version = "0.6", features = ["x-http"] } 27 | num_cpus = "1.16.0" 28 | rand = "0.8.5" 29 | redb = "1.5.0" 30 | serde = { version = "1.0.195", features = ["derive"] } 31 | serde-xml-rs = "0.6.0" 32 | tokio = { version = "1.34.0", features = ["full"] } 33 | tracing = "0.1.40" 34 | tracing-subscriber = { version = "0.3.18", features = ["env-filter"] } 35 | url = "2.5.0" 36 | walkdir = "2.4.0" 37 | 38 | [dev-dependencies] 39 | duct = "0.13.6" 40 | nix = { version = "0.27", features = ["signal", "process"] } 41 | rand = "0.8.5" 42 | serde_json = "1.0.108" 43 | tempfile = "3.8.1" 44 | -------------------------------------------------------------------------------- /iroh-s3-bao-store/LICENSE-APACHE: -------------------------------------------------------------------------------- 1 | Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at 2 | 3 | http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. -------------------------------------------------------------------------------- /iroh-s3-bao-store/LICENSE-MIT: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. -------------------------------------------------------------------------------- /iroh-s3-bao-store/README.md: -------------------------------------------------------------------------------- 1 | # S3-store 2 | 3 | This is an example how to use iroh-blobs to serve content from the web, 4 | e.g. from an s3 bucket. 5 | 6 | This works by downloading the content and computing an outboard in memory. 7 | The data itself remains remote. The upside is that the data is now content-addressed, 8 | so any change will be immediately detected and will lead to a failure to serve 9 | the content, just as if the changed data was not there at all. 10 | 11 | # serve-urls 12 | 13 | This just takes a list of urls and serves them all as a collection. 14 | 15 | # serve-s3 16 | 17 | This scans the index xml of a s3 bucket and creates a collection from it. 18 | To use this, you must configure or find a public s3 bucket with a index enabled. 19 | 20 | Below an example bucket policy: 21 | 22 | ```json 23 | { 24 | "Version": "2012-10-17", 25 | "Statement": [ 26 | { 27 | "Sid": "AddPerm", 28 | "Effect": "Allow", 29 | "Principal": "*", 30 | "Action": "s3:GetObject", 31 | "Resource": "arn:aws:s3:::__my-bucket-name__/*" 32 | }, 33 | { 34 | "Effect": "Allow", 35 | "Principal": "*", 36 | "Action": "s3:ListBucket", 37 | "Resource": "arn:aws:s3:::__my-bucket-name__" 38 | } 39 | ] 40 | } 41 | ``` 42 | -------------------------------------------------------------------------------- /iroh-s3-bao-store/src/lib.rs: -------------------------------------------------------------------------------- 1 | use std::collections::HashMap; 2 | use std::io; 3 | use std::sync::{Arc, Mutex}; 4 | 5 | use bao_tree::io::fsm::Outboard; 6 | use bao_tree::io::outboard::{PostOrderMemOutboard, PreOrderMemOutboard}; 7 | use bao_tree::BaoTree; 8 | use bytes::Bytes; 9 | use iroh_blobs::store::bao_tree::blake3; 10 | use iroh_blobs::store::{BaoBlobSize, MapEntry}; 11 | use iroh_blobs::Hash; 12 | use iroh_blobs::IROH_BLOCK_SIZE; 13 | use iroh_io::{AsyncSliceReader, AsyncSliceReaderExt, HttpAdapter}; 14 | use url::Url; 15 | 16 | #[derive(Debug, Clone, Default)] 17 | pub struct S3Store(Arc); 18 | 19 | #[derive(Debug, Default)] 20 | struct Inner { 21 | entries: Mutex>, 22 | } 23 | 24 | impl S3Store { 25 | pub async fn import_mem(&self, data: Bytes) -> anyhow::Result { 26 | let size = data.as_ref().len() as u64; 27 | let (outboard, hash) = { 28 | let outboard = PostOrderMemOutboard::create(&data, IROH_BLOCK_SIZE).flip(); 29 | let hash = outboard.root; 30 | (outboard.data, hash) 31 | }; 32 | let tree = BaoTree::new(size, IROH_BLOCK_SIZE); 33 | let outboard = PreOrderMemOutboard { 34 | root: hash, 35 | tree, 36 | data: outboard.into(), 37 | }; 38 | let mut state = self.0.entries.lock().unwrap(); 39 | state.insert( 40 | hash, 41 | Entry::new(hash.into(), size, DataDescriptor::Inline(data), outboard), 42 | ); 43 | Ok(hash.into()) 44 | } 45 | 46 | pub async fn import_url(&self, url: Url) -> anyhow::Result { 47 | let mut http_adapter = HttpAdapter::new(url.clone()); 48 | let data = http_adapter.read_to_end().await?; 49 | let size = data.len() as u64; 50 | let (outboard, hash) = { 51 | let outboard = PostOrderMemOutboard::create(data, IROH_BLOCK_SIZE).flip(); 52 | let hash = outboard.root; 53 | (outboard.data, hash) 54 | }; 55 | let tree = BaoTree::new(size, IROH_BLOCK_SIZE); 56 | let outboard = PreOrderMemOutboard { 57 | root: hash, 58 | tree, 59 | data: outboard.into(), 60 | }; 61 | let mut state = self.0.entries.lock().unwrap(); 62 | state.insert( 63 | hash, 64 | Entry::new( 65 | hash.into(), 66 | size, 67 | DataDescriptor::Url(Arc::new(url)), 68 | outboard, 69 | ), 70 | ); 71 | Ok(hash.into()) 72 | } 73 | } 74 | 75 | #[derive(Debug, Clone)] 76 | pub enum DataDescriptor { 77 | Url(Arc), 78 | Inline(Bytes), 79 | } 80 | 81 | #[derive(Debug, Clone)] 82 | pub struct Entry { 83 | hash: Hash, 84 | size: u64, 85 | data: DataDescriptor, 86 | outboard: PreOrderMemOutboard, 87 | } 88 | 89 | impl Entry { 90 | pub fn new( 91 | hash: Hash, 92 | size: u64, 93 | data: DataDescriptor, 94 | outboard: PreOrderMemOutboard, 95 | ) -> Self { 96 | Self { 97 | hash, 98 | size, 99 | outboard, 100 | data, 101 | } 102 | } 103 | } 104 | 105 | impl MapEntry for Entry { 106 | fn size(&self) -> BaoBlobSize { 107 | BaoBlobSize::Verified(self.size) 108 | } 109 | 110 | fn hash(&self) -> Hash { 111 | self.hash 112 | } 113 | 114 | fn is_complete(&self) -> bool { 115 | true 116 | } 117 | 118 | async fn outboard(&self) -> io::Result { 119 | Ok(self.outboard.clone()) 120 | } 121 | 122 | async fn data_reader(&self) -> io::Result { 123 | Ok(match self.data { 124 | DataDescriptor::Url(ref url) => { 125 | let http_adapter = HttpAdapter::new(url.as_ref().clone()); 126 | self::File::S3(http_adapter) 127 | } 128 | DataDescriptor::Inline(ref bytes) => self::File::Inline(bytes.clone()), 129 | }) 130 | } 131 | } 132 | 133 | /// DataReader can only be on s3 134 | #[derive(Debug)] 135 | pub enum File { 136 | S3(iroh_io::HttpAdapter), 137 | Inline(Bytes), 138 | } 139 | 140 | impl AsyncSliceReader for File { 141 | async fn read_at(&mut self, offset: u64, len: usize) -> io::Result { 142 | match self { 143 | Self::S3(s3) => s3.read_at(offset, len).await, 144 | Self::Inline(ref mut bytes) => bytes.read_at(offset, len).await, 145 | } 146 | } 147 | 148 | async fn size(&mut self) -> io::Result { 149 | match self { 150 | Self::S3(s3) => s3.size().await, 151 | Self::Inline(bytes) => bytes.size().await, 152 | } 153 | } 154 | } 155 | 156 | impl iroh_blobs::store::Map for S3Store { 157 | type Entry = Entry; 158 | 159 | async fn get(&self, hash: &iroh_blobs::Hash) -> io::Result> { 160 | let key: blake3::Hash = (*hash).into(); 161 | Ok(self.0.entries.lock().unwrap().get(&key).cloned()) 162 | } 163 | } 164 | -------------------------------------------------------------------------------- /iroh-s3-bao-store/src/main.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Context; 2 | use clap::{Parser, Subcommand}; 3 | use indicatif::{ 4 | HumanBytes, HumanDuration, MultiProgress, ProgressBar, ProgressDrawTarget, ProgressStyle, 5 | }; 6 | use iroh::{Endpoint, NodeAddr, SecretKey}; 7 | use iroh_blobs::{ 8 | provider::{self, handle_connection, CustomEventSender, EventSender}, 9 | ticket::BlobTicket, 10 | util::local_pool::LocalPool, 11 | BlobFormat, 12 | }; 13 | use iroh_io::{AsyncSliceReaderExt, HttpAdapter}; 14 | use iroh_s3_bao_store::S3Store; 15 | use serde::Deserialize; 16 | use std::net::{SocketAddrV4, SocketAddrV6}; 17 | use std::{ 18 | fmt::{Display, Formatter}, 19 | str::FromStr, 20 | sync::Arc, 21 | time::Duration, 22 | }; 23 | use url::Url; 24 | 25 | /// Send a file or directory between two machines, using blake3 verified streaming. 26 | /// 27 | /// For all subcommands, you can specify a secret key using the IROH_SECRET 28 | /// environment variable. If you don't, a random one will be generated. 29 | /// 30 | /// You can also specify the address for the iroh socket. If you don't, a random one 31 | /// will be chosen. 32 | #[derive(Parser, Debug)] 33 | pub struct Args { 34 | #[clap(subcommand)] 35 | pub command: Commands, 36 | } 37 | 38 | #[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] 39 | pub enum Format { 40 | #[default] 41 | Hex, 42 | Cid, 43 | } 44 | 45 | impl FromStr for Format { 46 | type Err = anyhow::Error; 47 | 48 | fn from_str(s: &str) -> Result { 49 | match s.to_ascii_lowercase().as_str() { 50 | "hex" => Ok(Format::Hex), 51 | "cid" => Ok(Format::Cid), 52 | _ => Err(anyhow::anyhow!("invalid format")), 53 | } 54 | } 55 | } 56 | 57 | impl Display for Format { 58 | fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { 59 | match self { 60 | Format::Hex => write!(f, "hex"), 61 | Format::Cid => write!(f, "cid"), 62 | } 63 | } 64 | } 65 | 66 | fn print_hash(hash: &iroh_blobs::Hash, format: Format) -> String { 67 | match format { 68 | Format::Hex => hash.to_hex().to_string(), 69 | Format::Cid => hash.to_string(), 70 | } 71 | } 72 | 73 | #[derive(Subcommand, Debug)] 74 | pub enum Commands { 75 | /// Send a file or directory. 76 | ServeS3(ServeS3Args), 77 | 78 | /// Receive a file or directory. 79 | ServeUrls(ImportS3Args), 80 | } 81 | 82 | #[derive(Parser, Debug)] 83 | pub struct CommonArgs { 84 | /// The IPv4 addr for the iroh socket to listen on. 85 | /// 86 | /// Defauls to a random free port, but it can be useful to specify a fixed 87 | /// port, e.g. to configure a firewall rule. 88 | #[clap(long)] 89 | pub iroh_ipv4_addr: Option, 90 | 91 | /// The IPv6 addr for the iroh socket to listen on. 92 | #[clap(long)] 93 | pub iroh_ipv6_addr: Option, 94 | 95 | #[clap(long, default_value_t = Format::Hex)] 96 | pub format: Format, 97 | 98 | #[clap(short = 'v', long, action = clap::ArgAction::Count)] 99 | pub verbose: u8, 100 | } 101 | 102 | #[derive(Parser, Debug)] 103 | pub struct ServeS3Args { 104 | /// Url to the s3 bucket root. 105 | pub url: Url, 106 | 107 | /// Top level directory name. 108 | #[clap(long)] 109 | pub name: Option, 110 | 111 | #[clap(flatten)] 112 | pub common: CommonArgs, 113 | } 114 | 115 | #[derive(Parser, Debug)] 116 | pub struct ImportS3Args { 117 | /// Url to the s3 bucket root. 118 | pub url: Vec, 119 | 120 | #[clap(flatten)] 121 | pub common: CommonArgs, 122 | } 123 | 124 | /// Get the secret key or generate a new one. 125 | /// 126 | /// Print the secret key to stderr if it was generated, so the user can save it. 127 | fn get_or_create_secret(print: bool) -> anyhow::Result { 128 | match std::env::var("IROH_SECRET") { 129 | Ok(secret) => SecretKey::from_str(&secret).context("invalid secret"), 130 | Err(_) => { 131 | let key = SecretKey::generate(rand::thread_rng()); 132 | if print { 133 | eprintln!("using secret key {}", key); 134 | } 135 | Ok(key) 136 | } 137 | } 138 | } 139 | 140 | #[derive(Debug, Clone)] 141 | struct SendStatus { 142 | /// the multiprogress bar 143 | mp: MultiProgress, 144 | } 145 | 146 | impl SendStatus { 147 | fn new() -> Self { 148 | let mp = MultiProgress::new(); 149 | mp.set_draw_target(ProgressDrawTarget::stderr()); 150 | Self { mp } 151 | } 152 | 153 | fn new_client(&self) -> ClientStatus { 154 | let current = self.mp.add(ProgressBar::hidden()); 155 | current.set_style( 156 | ProgressStyle::default_spinner() 157 | .template("{spinner:.green} [{elapsed_precise}] {msg}") 158 | .unwrap(), 159 | ); 160 | current.enable_steady_tick(Duration::from_millis(100)); 161 | current.set_message("waiting for requests"); 162 | ClientStatus { 163 | current: current.into(), 164 | } 165 | } 166 | } 167 | 168 | #[derive(Debug, Clone)] 169 | struct ClientStatus { 170 | current: Arc, 171 | } 172 | 173 | impl Drop for ClientStatus { 174 | fn drop(&mut self) { 175 | if Arc::strong_count(&self.current) == 1 { 176 | self.current.finish_and_clear(); 177 | } 178 | } 179 | } 180 | 181 | impl CustomEventSender for ClientStatus { 182 | fn send(&self, event: iroh_blobs::provider::Event) -> futures_lite::future::Boxed<()> { 183 | self.try_send(event); 184 | Box::pin(std::future::ready(())) 185 | } 186 | 187 | fn try_send(&self, event: iroh_blobs::provider::Event) { 188 | tracing::info!("{:?}", event); 189 | let msg = match event { 190 | provider::Event::ClientConnected { connection_id } => { 191 | Some(format!("{} got connection", connection_id)) 192 | } 193 | provider::Event::TransferBlobCompleted { 194 | connection_id, 195 | hash, 196 | index, 197 | size, 198 | .. 199 | } => Some(format!( 200 | "{} transfer blob completed {} {} {}", 201 | connection_id, 202 | hash, 203 | index, 204 | HumanBytes(size) 205 | )), 206 | provider::Event::TransferCompleted { 207 | connection_id, 208 | stats, 209 | .. 210 | } => Some(format!( 211 | "{} transfer completed {} {}", 212 | connection_id, 213 | stats.send.write_bytes.size, 214 | HumanDuration(stats.send.write_bytes.stats.duration) 215 | )), 216 | provider::Event::TransferAborted { connection_id, .. } => { 217 | Some(format!("{} transfer completed", connection_id)) 218 | } 219 | _ => None, 220 | }; 221 | if let Some(msg) = msg { 222 | self.current.set_message(msg); 223 | } 224 | } 225 | } 226 | 227 | async fn serve_db( 228 | db: S3Store, 229 | iroh_ipv4_addr: Option, 230 | iroh_ipv6_addr: Option, 231 | on_addr: impl FnOnce(NodeAddr) -> anyhow::Result<()>, 232 | ) -> anyhow::Result<()> { 233 | let secret_key = get_or_create_secret(true)?; 234 | // create an iroh endpoint 235 | let mut builder = Endpoint::builder() 236 | .alpns(vec![iroh_blobs::protocol::ALPN.to_vec()]) 237 | .secret_key(secret_key); 238 | 239 | if let Some(addr) = iroh_ipv4_addr { 240 | builder = builder.bind_addr_v4(addr); 241 | } 242 | if let Some(addr) = iroh_ipv6_addr { 243 | builder = builder.bind_addr_v6(addr); 244 | } 245 | // wait for the endpoint to be ready 246 | let endpoint = builder.bind().await?; 247 | // wait for the endpoint to figure out its address before making a ticket 248 | endpoint.home_relay().initialized().await?; 249 | // make a ticket 250 | let addr = endpoint.node_addr().await?; 251 | on_addr(addr)?; 252 | let lp = LocalPool::single(); 253 | let ps = SendStatus::new(); 254 | let sc = Arc::new(ps.new_client()); 255 | loop { 256 | let Some(connecting) = endpoint.accept().await else { 257 | tracing::info!("no more incoming connections, exiting"); 258 | break; 259 | }; 260 | let db = db.clone(); 261 | let lph = lp.handle().clone(); 262 | let sc = sc.clone(); 263 | let conn = connecting.await?; 264 | tokio::spawn(handle_connection(conn, db, EventSender::new(Some(sc)), lph)); 265 | } 266 | Ok(()) 267 | } 268 | 269 | async fn serve_s3(args: ServeS3Args) -> anyhow::Result<()> { 270 | let root = args.url; 271 | let xml = HttpAdapter::new(root.clone()).read_to_end().await?; 272 | let xml = String::from_utf8_lossy(&xml); 273 | tracing::debug!("{}", xml); 274 | let bucket: ListBucketResult = serde_xml_rs::from_str(&xml)?; 275 | let db = S3Store::default(); 276 | let mut hashes = Vec::new(); 277 | let safe_bucket_name = || root.to_string().replace('/', "_"); 278 | let prefix = args.name.unwrap_or_else(safe_bucket_name); 279 | for path in bucket.contents.iter().map(|c| c.key.clone()) { 280 | let url = root.join(&path)?; 281 | let hash = db.import_url(url).await?; 282 | let name = format!("{prefix}/{path}"); 283 | hashes.push((name, hash)); 284 | } 285 | let collection = hashes 286 | .iter() 287 | .cloned() 288 | .collect::(); 289 | let blobs = collection.to_blobs(); 290 | let mut last_hash = None; 291 | for blob in blobs { 292 | last_hash = Some(db.import_mem(blob).await?); 293 | } 294 | 295 | serve_db( 296 | db, 297 | args.common.iroh_ipv4_addr, 298 | args.common.iroh_ipv6_addr, 299 | |addr| { 300 | if let Some(hash) = last_hash { 301 | let ticket = BlobTicket::new(addr.clone(), hash, BlobFormat::HashSeq)?; 302 | println!("collection: {}", ticket); 303 | } 304 | Ok(()) 305 | }, 306 | ) 307 | .await?; 308 | Ok(()) 309 | } 310 | 311 | async fn serve_urls(args: ImportS3Args) -> anyhow::Result<()> { 312 | let db = S3Store::default(); 313 | let mut hashes = Vec::new(); 314 | for url in args.url { 315 | let hash = db.import_url(url.clone()).await?; 316 | println!("added {}, {}", url, print_hash(&hash, args.common.format)); 317 | let name = url.to_string().replace('/', "_"); 318 | hashes.push((name, hash)); 319 | } 320 | let collection = hashes 321 | .iter() 322 | .cloned() 323 | .collect::(); 324 | let blobs = collection.to_blobs(); 325 | let mut last_hash = None; 326 | for blob in blobs { 327 | last_hash = Some(db.import_mem(blob).await?); 328 | } 329 | 330 | serve_db( 331 | db, 332 | args.common.iroh_ipv4_addr, 333 | args.common.iroh_ipv6_addr, 334 | |addr| { 335 | for (name, hash) in &hashes { 336 | let ticket = BlobTicket::new(addr.clone(), *hash, BlobFormat::Raw)?; 337 | println!("{} {}", name, ticket); 338 | } 339 | if let Some(hash) = last_hash { 340 | let ticket = BlobTicket::new(addr.clone(), hash, BlobFormat::HashSeq)?; 341 | println!("collection: {}", ticket); 342 | } 343 | Ok(()) 344 | }, 345 | ) 346 | .await?; 347 | Ok(()) 348 | } 349 | 350 | /// The ListBucketResult xml structure returned by s3. 351 | #[derive(Debug, Deserialize)] 352 | struct ListBucketResult { 353 | #[serde(rename = "Contents", default)] 354 | contents: Vec, 355 | } 356 | 357 | /// The Contents xml structure returned by s3. 358 | #[derive(Debug, Deserialize)] 359 | struct Contents { 360 | #[serde(rename = "Key", default)] 361 | key: String, 362 | } 363 | 364 | #[tokio::main] 365 | async fn main() -> anyhow::Result<()> { 366 | tracing_subscriber::fmt::init(); 367 | let args = Args::parse(); 368 | match args.command { 369 | Commands::ServeS3(args) => serve_s3(args).await, 370 | Commands::ServeUrls(args) => serve_urls(args).await, 371 | } 372 | } 373 | --------------------------------------------------------------------------------