├── .github └── workflows │ └── rust.yaml ├── .gitignore ├── CHANGELOG.md ├── Cargo.toml ├── LICENSE ├── README.md ├── docs ├── censorship-resistance.md ├── dht_size_estimate.md ├── plot.png ├── simulation │ ├── Cargo.toml │ └── src │ │ └── main.rs └── standard-deviation-vs-lookups.png ├── examples ├── README.md ├── announce_peer.rs ├── bootstrap.rs ├── cache_bootstrap.rs ├── count_ips_close_to_key.rs ├── get_immutable.rs ├── get_mutable.rs ├── get_peers.rs ├── logging.rs ├── mark_recapture_dht.rs ├── measure_dht.rs ├── put_immutable.rs ├── put_mutable.rs └── request_filter.rs └── src ├── async_dht.rs ├── common.rs ├── common ├── id.rs ├── immutable.rs ├── messages.rs ├── messages │ └── internal.rs ├── mutable.rs ├── node.rs └── routing_table.rs ├── dht.rs ├── lib.rs ├── rpc.rs └── rpc ├── closest_nodes.rs ├── config.rs ├── info.rs ├── iterative_query.rs ├── put_query.rs ├── server.rs ├── server ├── peers.rs └── tokens.rs └── socket.rs /.github/workflows/rust.yaml: -------------------------------------------------------------------------------- 1 | name: Rust CI 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | pull_request: 7 | branches: [ main ] 8 | 9 | jobs: 10 | build-and-test: 11 | runs-on: ubuntu-latest 12 | 13 | steps: 14 | - uses: actions/checkout@v2 15 | 16 | - name: Set up Rust 17 | uses: actions-rs/toolchain@v1 18 | with: 19 | profile: minimal 20 | toolchain: stable 21 | components: rustfmt, clippy 22 | override: true 23 | 24 | - name: Check formatting 25 | run: cargo fmt -- --check 26 | 27 | - name: Lint with Clippy 28 | run: cargo clippy --workspace --all-features --bins --tests 29 | 30 | - name: Build 31 | run: cargo build --release --workspace --all-features --verbose 32 | 33 | 34 | - name: Run tests 35 | run: cargo test --all-features --workspace --verbose 36 | 37 | - name: Run docs 38 | run: cargo doc --workspace --all-features --no-deps --document-private-items --verbose 39 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | Cargo.lock 3 | reference 4 | /docs/simulation/target 5 | examples/bootstrapping_nodes 6 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | All notable changes to mainline dht will be documented in this file. 4 | 5 | ## [5.0.0](https://github.com/pubky/mainline/compare/v4.2.1...v5.0.0) - 2025-02-11 6 | 7 | ### Added 8 | 9 | - Add `Id::from_ipv4()`. 10 | - Add `Id::is_valid_for_ipv4`. 11 | - Add `RoutingTable::nodes()` iterator. 12 | - Add `DhtBuilder::server_mode` to force server mode. 13 | - Add `DhtBuilder::public_ip` for manually setting the node's public ip to generate secure node `Id` from. 14 | - Add [adaptive mode](https://github.com/pubky/mainline?tab=readme-ov-file#adaptive-mode). 15 | - Add `DhtBuilder::extra_bootstrap()` to add more bootstrapping nodes from previous sessions. 16 | - Add `Dht::bootstrapped()` and `AsyncDht::bootstrapped()` to wait for the routing table to be bootstrapped. 17 | - Add `RoutingTable::to_bootstrap()`, `Dht::to_bootstrap()`, and `AsyncDht::to_bootstrap()` to export the addresses nodes in the routing table. 18 | - Add `Info::public_address()` which returns the best estimate for this node's public address. 19 | - Add `Info::firewalled()` which returns whether or not this node is firewalled, or publicly accessible. 20 | - Add `Info::server_mode()` which returns whether or not this node is running in server mode. 21 | - Add `DhtBuilder::info()` to export a thread safe and lightweight summary of the node's information and statistics. 22 | - Add `cache_bootstrap.rs` example to show how you can store your routing table to disk and use it for subsequent bootstrapping. 23 | - Add `Dht::get_mutable_most_recent()` and `AsyncDht::get_mutable_most_recent()` to get the most recent mutable item from the network. 24 | - Add `PutQueryError::Timeout` in case put query is terminated unsuccessfully, but no error responses. 25 | - Add `PutMutableError::Concurrrency(ConcurrrencyError)` for all cases where a `Lost Update Problem` may occur (read `Dht::put_mutable` documentation for more details). 26 | - Add `Dht::get_closest_nodes()` and `AsyncDht::get_closest_nodes()` to return the closest nodes (that support BEP_0044) with valid tokens. 27 | - Add `Dht::put()` and `AsyncDht::put()` to put a request to the closest nodes, and optionally to extra arbitrary nodes with valid tokens. 28 | - Add `Testnet::leak()` to keep the dht network running as a `'static` 29 | - Add `MutableError `. 30 | - Add `DecodeIdError` 31 | - Export `Dhtbuilder`. 32 | - Export `RoutingTable`. 33 | - Support `BEP_0042 DHT Security extension` when running in server mode. 34 | 35 | ### Removed 36 | 37 | - Remove `bytes` dependency. 38 | - Remove `ipv6` optionality and commit to `ipv4`. 39 | - Remove `Id::to_vec()`. 40 | - Exported `ClosestNodes`, you have to use it from `mainline::rpc`. 41 | - Removed `Node::unique()`, `Node::with_id()`, `Node::with_address()`, and `Node::with_token()`. 42 | - Removed `RoutingTable::default()`. 43 | - Removed exporting `rpc` module, and `Rpc` struct. 44 | - Removed `Dht::shutdown()` and `AsyncDht::shutdown()`. 45 | - Removed `DhtWasShutdown` 46 | - Removed `DefaultServer` export. 47 | 48 | ### Changed 49 | 50 | - Rename `Settings` to `ClientBuilder`. 51 | - `Dht`, and `AsyncDht` is now behind a feature flag `node`, so you can include the `Rpc` only and build your own node. 52 | - All methods that were returning `Result` now return `T`. 53 | - Enable calling `Dht::announce_peer()` and `Dht::put_immutable()` multiple times concurrently. 54 | - Return `PutMutableError::Concurrrency(ConcurrrencyError)` from `Dht::put_mutable()`. 55 | - `Info::local_addr()` is infallible. 56 | - `MutableItem::seq()` returns `i64` instead of a reference. 57 | - `Dht::put_immutable()` and `AsyncDh::put_immutable()` take `&[u8]` instead of `bytes::Bytes`. 58 | - `Dht::get_immutable()` and `AsyncDh::get_immutable()` return boxed slice `Box<[u8]>` instead of `bytes::Bytes`. 59 | - `Dht::put_immutable()` and `AsyncDh::put_immutable()` return `PutImmutableError`. 60 | - `Dht::announce_peer()` and `AsyncDh::announce_peer()` return `AnnouncePeerError`. 61 | - `Dht::put_mutable()` and `AsyncDh::put_mutable()` return `PutMutableError`. 62 | - All tracing logs are either `TRACE` (for krpcsocket), `DEBUG`, or `INFO` only for rare and singular events, 63 | like starting the node, updating the node Id, or switching to server mode (from adaptive mode). 64 | - Change `PutError` to contain transparent elements for generic `PutQueryError`, and more specialized `ConcurrrencyError`. 65 | - Remove `MutableItem::cas` field, and add optional `CAS` parameter to `Dht::put_mutable` and `AsyncDht::put_mutable`. 66 | - `Dht::find_node()` and `AsyncDht::find_node()` return `Box<[Node]>` instead of `Vec`. 67 | - `Node` is `Send` and `Sync`, and cheap to clone using an internal `Arc`. 68 | - `Node::new()` take `Id` and `SocketAddrV4`. 69 | - `RoutingTable::new()` takes an `Id`. 70 | - Return `GetIterator` and `GetStream` from `get_` methods from `Dht` and `AsyncDht` instead of exposing `flume`. 71 | - Remove `Server` trait and replace it with `RequestFilter` trait. 72 | - `DhtBuilder` is not consuming, thanks to `Config` being `Clone`. 73 | 74 | ## [4.2.0](https://github.com/pubky/mainline/compare/v4.1.0...v4.2.0) - 2024-12-13 75 | 76 | ### Added 77 | 78 | - Make MutableItem de/serializable (mikedilger) 79 | 80 | ## [4.1.0](https://github.com/pubky/mainline/compare/v3.0.0...v4.1.0) - 2024-11-29 81 | 82 | ### Added 83 | 84 | - Export `errors` module containing `PutError` as a part of the response of `Rpc::put`. 85 | - `Dht::find_node()` and `AsyncDht::find_node()` to find the closest nodes to a certain target. 86 | - `Dht::info()` and `AsyncDht::info()` some internal information about the node from one method. 87 | - `Info::dht_size_estimate` to get the ongoing dht size estimate resulting from watching results of all queries. 88 | - `Info::id` to get the Id of the node. 89 | - `measure_dht` example to estimate the DHT size. 90 | 91 | ### Changed 92 | 93 | - Removed all internal panic `#![deny(clippy::unwrap_used)]`. 94 | - `Testnet::new(size)` returns a `Result`. 95 | - `Dht::local_addr()` and `AsyncDht::local_addr()` replaced with `::info()`. 96 | - `Dht::shutdown()` and `AsyncDht::shutdown()` are now idempotent, and returns `()`. 97 | - `Rpc::drop` uses `tracing::debug!()` to log dropping the Rpc. 98 | - `Id::as_bytes()` instead of exposing internal `bytes` property. 99 | - Replace crate `Error` with more granular errors. 100 | - Replace Flume's `RecvError` with `expect()` message, since the sender should never be dropped to soon. 101 | - `DhtWasShutdown` error is a standalone error. 102 | - `InvalidIdSize` error is a standalone error. 103 | - Rename `DhtSettings` to `Settings` 104 | - Rename `DhtServer` to `DefaultServer` 105 | - `Dht::get_immutable()` and `AsyncDht::get_immutable()` return `Result, DhtWasShutdown>` 106 | - `Node` fields are now all private, with `id()` and `address()` getters. 107 | - Changed `Settings` to be a the Builder, and make fields private. 108 | - Replaced `Rpc::new()` with `Settings::build_rpc()`. 109 | - Update the client version from `RS01` to `RS04` 110 | 111 | ### Removed 112 | 113 | - Removed `mainline::error::Error` and `mainline::error::Result`. 114 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "mainline" 3 | version = "5.4.0" 4 | authors = ["nuh.dev"] 5 | edition = "2021" 6 | description = "Simple, robust, BitTorrent's Mainline DHT implementation" 7 | homepage = "https://github.com/pubky/mainline" 8 | license = "MIT" 9 | keywords = ["bittorrent", "torrent", "dht", "kademlia", "mainline"] 10 | categories = ["network-programming"] 11 | repository = "https://github.com/pubky/mainline" 12 | exclude = ["/docs/*", "/examples/*"] 13 | 14 | [dependencies] 15 | getrandom = "0.2" 16 | serde_bencode = "^0.2.4" 17 | serde = { version = "1.0.217", features = ["derive"] } 18 | serde_bytes = "0.11.15" 19 | thiserror = "2.0.11" 20 | crc = "3.2.1" 21 | sha1_smol = "1.0.1" 22 | ed25519-dalek = "2.1.1" 23 | tracing = "0.1" 24 | lru = { version = "0.13.0", default-features = false } 25 | dyn-clone = "1.0.18" 26 | 27 | document-features = "0.2.10" 28 | 29 | # `node` dependencies 30 | flume = { version = "0.11.1", features = [], default-features = false, optional = true } 31 | 32 | # `async` dependencies 33 | futures-lite = { version = "2.6.0", default-features = false, optional = true } 34 | 35 | [dev-dependencies] 36 | clap = { version = "4.5.29", features = ["derive"] } 37 | futures = "0.3.31" 38 | tracing-subscriber = "0.3" 39 | ctrlc = "3.4.5" 40 | histo = "1.0.0" 41 | rayon = "1.10" 42 | dashmap = "6.1" 43 | flume = "0.11.1" 44 | colored = "3.0.0" 45 | chrono = "0.4" 46 | 47 | [features] 48 | ## Include [Dht] node. 49 | node = ["dep:flume"] 50 | ## Enable [Dht::as_async()] to use [async_dht::AsyncDht]. 51 | async = ["node", "flume/async", "dep:futures-lite"] 52 | 53 | full = ["async"] 54 | 55 | default = ["full"] 56 | 57 | [package.metadata.docs.rs] 58 | all-features = true 59 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 raptorswing 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Mainline 2 | 3 | Simple, robust, BitTorrent's [Mainline](https://en.wikipedia.org/wiki/Mainline_DHT) DHT implementation. 4 | 5 | This library is focused on being the best and simplest Rust client for Mainline, especially focused on reliable and fast time-to-first-response. 6 | 7 | It should work as a routing / storing node (server mode) as well, and has been running in production for many months without an issue. 8 | However if you are concerned about spam or DoS, you should consider implementing [rate limiting](#rate-limiting). 9 | 10 | **[API Docs](https://docs.rs/mainline/latest/mainline/)** 11 | 12 | ## Getting started 13 | 14 | Check the [Examples](https://github.com/Pubky/mainline/tree/main/examples). 15 | 16 | ## Features 17 | 18 | ### Client 19 | 20 | Running as a client, means you can store and query for values on the DHT, but not accept any incoming requests. 21 | 22 | ```rust 23 | use mainline::Dht; 24 | 25 | let dht = Dht::client().unwrap(); 26 | ``` 27 | 28 | Supported BEPs: 29 | - [x] [BEP_0005 DHT Protocol](https://www.bittorrent.org/beps/bep_0005.html) 30 | - [x] [BEP_0042 DHT Security extension](https://www.bittorrent.org/beps/bep_0042.html) 31 | - [x] [BEP_0043 Read-only DHT Nodes](https://www.bittorrent.org/beps/bep_0043.html) 32 | - [x] [BEP_0044 Storing arbitrary data in the DHT](https://www.bittorrent.org/beps/bep_0044.html) 33 | 34 | This implementation also includes [measures against Vertical Sybil Attacks](./docs/sybil-resistance.md). 35 | 36 | ### Server 37 | 38 | Running as a server is the same as a client, but you also respond to incoming requests and serve as a routing and storing node, supporting the general routing of the DHT, and contributing to the storage capacity of the DHT. 39 | 40 | ```rust 41 | use mainline::Dht; 42 | 43 | let dht = Dht::server().unwrap(); // or `Dht::builder::server_mode().build();` 44 | ``` 45 | 46 | Supported BEPs: 47 | - [x] [BEP_0005 DHT Protocol](https://www.bittorrent.org/beps/bep_0005.html) 48 | - [x] [BEP_0042 DHT Security extension](https://www.bittorrent.org/beps/bep_0042.html) 49 | - [x] [BEP_0043 Read-only DHT Nodes](https://www.bittorrent.org/beps/bep_0043.html) 50 | - [x] [BEP_0044 Storing arbitrary data in the DHT](https://www.bittorrent.org/beps/bep_0044.html) 51 | 52 | #### Rate limiting 53 | 54 | The server implementation has no rate-limiting, you can run your own [request filter](./examples/request_filter.rs) and apply your custom rate-limiting. 55 | However, that limit/block will only apply _after_ parsing incoming messages, and it won't affect handling incoming responses. 56 | 57 | ### Adaptive mode 58 | 59 | The default Adaptive mode will start the node in client mode, and after 15 minutes of running with a publicly accessible address, 60 | it will switch to server mode. This way nodes that can serve as routing nodes (accessible and less likely to churn), serve as such. 61 | 62 | If you want to explicitly start in Server mode, because you know you are not running behind firewall, 63 | you can call `Dht::builder().server_mode().build()`, and you can optionally add your known public ip so the node doesn't have to depend on, 64 | votes from responding nodes: `Dht::builder().server_mode().public_ip().build()`. 65 | 66 | ## Acknowledgment 67 | 68 | This implementation was possible thanks to [Webtorrent's Bittorrent-dht](https://github.com/webtorrent/bittorrent-dht) as a reference, 69 | and [Rustydht-lib](https://github.com/raptorswing/rustydht-lib) that saved me a lot of time, especially at the serialization and deserialization of Bencode messages. 70 | -------------------------------------------------------------------------------- /docs/censorship-resistance.md: -------------------------------------------------------------------------------- 1 | # Censorship Resistance 2 | 3 | ## Overview 4 | 5 | One of the main criticism against distributed hash tables are their susceptibility to Sybil attacks, 6 | and by extension censorship. This document is an overview over the problem and how this implementation minimizes this risk. 7 | 8 | [Real-World Sybil Attacks in BitTorrent Mainline DHT](https://www.cl.cam.ac.uk/~lw525/publications/security.pdf) paper divides Sybil attacks 9 | into “horizontal”, and “vertical”, the former tries to flood the entire network with Sybil nodes, while the later tries to target specific regions of 10 | the ID space, to censor specific info-hashes. 11 | 12 | Our strategy in this document is to first: explain how can we transform all vertical attacks to horizontal attacks by necessity, and second: explore the 13 | cost of such horizontal attacks and the cost of resisting such attacks, and we consider the system resistant to censorship, if the cost of resistance to 14 | horizontal Sybil attacks are much lower than the cost of sustaining such attacks for extended periods of time. 15 | 16 | ### Non Goals 17 | 18 | For the sake of this document we will NOT discuss extreme forms of censorship like filtering out UDP packets that look like Bittorrent messages at the ISP level. 19 | Or filtering out packets that includes specific info hashes. This form of censorship apply to more than just DHTs, including DNS queries and more. And are better 20 | handled using VPNs and other firewall circumvention solutions. Including HTTPs relays that are hard to filter out or predict their purpose. 21 | 22 | We will focus on how to keep DHTs resistant to vulnerabilities that are inherint to their nature as open networks without a central reputation auhtority. 23 | 24 | Similarly, we will not discuss the effect of Sybil attacks on privacy, if one wants to keep their queries private, they are advised to use a VPN or a trusted HTTPs server to relay their queries. 25 | 26 | ## Vertical Sybil Attacks 27 | 28 | ### Challenge 29 | 30 | In a DHT, nodes store a piece of information with a redundancy factor `k` (usually 20), meaning that a node tries to find the 31 | `k` closest nodes to the info hash using XOR metric defined in [BEP_0005](https://www.bittorrent.org/beps/bep_0005.html) before 32 | storing the data in these nodes. 33 | 34 | This static redundancy factor, opens the room for Vertical Sybil attacks is where a malicious actor runs enough nodes close to an info hash 35 | that a writer only writes to the attacker Sybil nodes, making it easy for that attacker to censors that information from the rest of the network. 36 | 37 | Consider the following example, with a Dht of size `8` and `k=2`, drawing nodes at their distances to a given target, should look like this: 38 | 39 | ```md 40 | (1) (2) (3) (4) (5) (6) (7) (8) 41 | |------|------|------|------|------|------|------|------|------|------|------|------|------|------|------| 42 | 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 43 | ``` 44 | 45 | So, if an attacker injected two (even closer) nodes, that don't match the distribution of the rest of network (Vertical Sybil as opposed to Horizontal Sybil), 46 | then you would expect the example above to look like this instead: 47 | 48 | ```md 49 | (s1) (s2) (1) (2) (3) (4) (5) (6) (7) (8) 50 | |------|------|------|------|------|------|------|------|------|------|------|------|------|------|------| 51 | 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 52 | ``` 53 | 54 | As you can see, if we only store data at the closest `k=2` nodes, the data would be only stored within attacker nodes, thus successefully censored. 55 | 56 | ### Uniform Distribution 57 | 58 | The example above and the solution explained next, both assume a uniform distribution of nodes over the ID space, 59 | besides the fact that such distribution can empirically observed, it is also enforced with security extension [BEP_0042](https://www.bittorrent.org/beps/bep_0042.html) 60 | that limits the number of nodes to 8 for each IP, and uniformly disrtibute these 8 nodes over the entire ID space. 61 | 62 | ### Solution 63 | 64 | To circumvent vertical sybil attack, we make sure to store data to as many of the closest nodes -that responded to our GET query- as necessary 65 | to satisfy both the following requirements: 66 | 67 | #### One or more nodes are further from the target than the `expected distance to k (edk)`. 68 | 69 | To understand what that means, consider that we have a rough estimation of the DHT size (which we obtain as explained in the 70 | documentation of the [Dht Size Estimate](./dht_size_estimate.md)), then we can _expect_ that the closest `k` nodes, are going to be 71 | within a range `edk`. For example, continuing the example from above, in a Dht of `8` nodes in a `16` ID space, we can expect 72 | the closest `2` nodes, within distance `4`. 73 | 74 | ```md 75 | (s1) (s2) (1) (2) [edk] (3) (4) (5) (6) (7) (8) 76 | |------|------|------|------|------|------|------|------|------|------|------|------|------|------|------| 77 | 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 78 | ``` 79 | 80 | This is similar but a bit more accurate than the average distance of the `k`th nodes from previous queries. 81 | 82 | If we store data in all nodes until `edk` (the expected distance of the first 2 nodes in this example), we would store the data at at least 2 honest nodes. 83 | 84 | Because the nature of the DHT queries, we should expect to get a response from at least one of these honest nodes as we query closer and closer nodes to the target info hash. 85 | 86 | #### Minimum number of unique subnets with 6 bits prefix. 87 | 88 | An extreme, and unlikely, but possible way to defeat our `edk` approach to detect vertical sybil attacks, is to DDoS all the honest nodes 89 | and replace them with enough nodes owned by the attacker. 90 | 91 | To find enough nodes to replace the nodes until `edk` the attacker needs ~4 `/8` blocks, or a single `/6` block. 92 | 93 | However, we can make this much more expensive, by keeping track of the number of unique `6 bit prefixes` in each GET query response, 94 | and store data to enough nodes that have enough unique prefixes to match the average from previous queries. 95 | 96 | At the time of writing, this usually means the attacker needs to control up to 12 `/6` blocks. 97 | 98 | ## Extreme Vertical Sybil Attacks 99 | 100 | While we are satisfied that this static metrics to circumvent Sybil attacks make them prohibitively expensive, let's consider what 101 | happens in the very unlikely event that an attacker has enough resources and motivation to brute force them both. 102 | 103 | In this case, an attacker acquires a so many Ips in so many subnets that they can both DDoS all the nodes until the expected distance to the 20th node, 104 | and inject at least 20 nodes with as many unique `6 bit` prefix in their IPs as the average of the rest of the network. 105 | 106 | Eventually, the writer will notice that reads after writes (GET after PUT, resolving after publishing) doesn't work for them, which can only be explained 107 | by an extreme targeted censorship attack. 108 | 109 | Once the writer notices this, they can manuaully start publishing to more and more nodes around the target, for example, instead of publishing to closest 20 nodes, 110 | start publishing to closest 200, or 2000, where readers, without any manual intervention will be likely to find the data as they are approaching the target. 111 | 112 | The writer can do that, by making GET queries to random targets that share enough prefix bits with their target, to find more and more nodes around their target, 113 | then store their data to these responding nodes. 114 | 115 | It is unfortunate that the writer needs to react manuaully at all, but given how extreme this attack is, we are satisfied with 116 | the defese being way cheaper than the attack, making the attack not only unsustainable, but also unlikely to happen, given that the attacker knows 117 | it won't be sustainable. 118 | 119 | ## Horizontal Sybil Attacks 120 | 121 | If an attacker can't perform a vertical Sybil attack, it has to run > 20 times the number of current honest nodes to have a good chance of taking over an info hash, 122 | i.e being in control of all 20 closest nodes to a target. 123 | 124 | Firstly, because we have a good way to estimate the dht size, we can all see the DHT size suddenly increasing 20x, which at least gives us all a chance to react to such extreme attack. 125 | 126 | Secondly, because of BEP_0042, an IPv4 can't have any more than 8 nodes, so an attacker needs to at least have control of millions of IP addresses. 127 | 128 | Thirdly, the current DHT size estimate seems to be near the limits enforced by BEP_0042 (~10 million nodes), which means an attacker will 129 | need to create more than 9 million nodes and try to replace already running nodes with their Sybil nodes, except that [BEP_0005](https://www.bittorrent.org/beps/bep_0005.html) favors older nodes 130 | than newer ones. 131 | 132 | To summarize, an attacker needs to have control over millions of IP addresses, actually run millions of nodes, hope that existing nodes churn enough to give them a chance to replace them in nodes routing tables, 133 | and hope that no one notices or reacts to such attack, and even then they need to sustain that attack, because as soon as they give up, the network resumes its normal operation. 134 | 135 | It is safe to say that much simpler modes of censorship are much more likely to be employed instead. 136 | 137 | ## Conclusion 138 | 139 | While theoritically DHTs are not immune to Sybil nodes, and while it is impossible to stop attempts to inject nodes all over the DHT to snoop on traffic, it is not at all easy or practical to 140 | disrupt the operation of a large DHT network. 141 | 142 | The security of a DHT thus boils down to the number of honest nodes, as long as we don't see a massive decline of the size of the DHT, Mainline will remain as unstopable as a network based on 143 | the Internet can be. 144 | -------------------------------------------------------------------------------- /docs/dht_size_estimate.md: -------------------------------------------------------------------------------- 1 | # Dht Size Estimattion 2 | 3 | This is a documentation for the Dht size estimation used in this Mainline Dht implementation, 4 | within the context of [Sybil Resistance](./sybil-resistance.md). 5 | 6 | If you want to see a live estimation of the Dht size, you can run (in the root directory): 7 | 8 | ``` 9 | cd ./simulation 10 | cargo run 11 | ``` 12 | 13 | ## How does it work? 14 | 15 | In order to get an accurate calculation of the Dht size, you should take 16 | as many lookups (at uniformly disrtibuted target) as you can, 17 | and calculate the average of the estimations based on their responding nodes. 18 | 19 | Consider a Dht with a 4 bit key space. 20 | Then we can map nodes in that keyspace by their distance to a given target of a lookup. 21 | 22 | Assuming a random but uniform distribution of nodes (which can be measured independently), 23 | you should see nodes distributed somewhat like this: 24 | 25 | ```md 26 | (1) (2) (3) (4) (5) (6) (7) (8) 27 | |------|------|------|------|------|------|------|------|------|------|------|------|------|------|------| 28 | 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 29 | ``` 30 | 31 | So if you make a lookup and optained this partial view of the network: 32 | ```md 33 | (1) (2) (3) (4) (5) 34 | |------|------|------|------|------|------|------|------|------|------|------|------|------|------|------| 35 | 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 36 | ``` 37 | 38 | Note: you see exponentially less further nodes than closer ones, which is what you should expect from how 39 | the routing table works. 40 | 41 | Seeing one node at distance (d1=2), suggests that the routing table might contain 8 nodes, 42 | since its full length is 8 times (d1). 43 | 44 | Similarily, seeing two nodes at (d2=3), suggests that the routing table might contain ~11 45 | nodes, since the key space is more than (d2). 46 | 47 | If we repeat this estimation for as many nodes as the routing table's `k` bucket size, 48 | and take their average, we get a more accurate estimation of the dht. 49 | 50 | ## Formula 51 | 52 | The estimated number of Dht size, at each distance `di`, is `en_i = i * d_max / di` where `i` is the 53 | count of nodes discovered until this distance and `d_max` is the size of the key space. 54 | 55 | The final Dht size estimation is the least-squares fit of `en_1 + en_2 + .. + en_n` 56 | 57 | ## Simulation 58 | 59 | Running this [simulation](../examples/dht_size_estimate.rs) for 2 million nodes and a after 16 lookups, we observe: 60 | 61 | - Mean estimate: 2,123,314 nodes 62 | - Standard deviation: 7% 63 | - 95% Confidence Interval: +-14% 64 | 65 | Meaning that after 12 lookups, you can be confident you are not overestimating the Dht size by more than 10%, 66 | in fact you are most likely underestimating it slightly due to the limitation of real networks. 67 | 68 | ![distribution of estimated dht size after 4 lookups](./plot.png) 69 | 70 | Finally the standard deviation seems to follow a power law `stddev = 0.281 * lookups^-0.529`. Meaning after only 4 lookups, you can get an estimate with 95% confidence interval of +-28%. 71 | 72 | ![Standard deviation relationship with number of lookups](./standard-deviation-vs-lookups.png) 73 | 74 | ## Mapping simulation to real networks 75 | 76 | While the Mean estimate in the simulation slightly over estimate the real size in the simulation, the opposite is what should be expected in real networks. 77 | 78 | Unlike the simulation above, real networks are not perfect, meaning there is an error factor that can't be hard coded, 79 | as it depends on the response rate of nodes you query, the more requests timeout before you get a response, the more nodes 80 | you will miss, and the smaller you will think the Dht is. 81 | 82 | This is an error on the side of conservatism. And I can't think of anything in the real world that could distort the results 83 | expected from this simulation to the direction of overestimating the Dht size. 84 | 85 | ### See it yourself 86 | 87 | You can measure the Dht size yourself by running: 88 | 89 | ``` 90 | cargo run --example measure_dht 91 | ``` 92 | 93 | Note that the estimate will be understated if you are using a hostile network to UDP packets or behind a VPN making your requests look like they are coming from the 94 | same IP as many other users (causing nodes to rate limit your requests more often). 95 | 96 | ## Acknowledgment 97 | 98 | This size estimation was based on [A New Method for Estimating P2P Network Size](https://eli.sohl.com/2020/06/05/dht-size-estimation.html#fnref:query-count) 99 | -------------------------------------------------------------------------------- /docs/plot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pubky/mainline/184dfb7b3cc55570ddb53d8f8e0212bf5d166542/docs/plot.png -------------------------------------------------------------------------------- /docs/simulation/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "sim" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | [dependencies] 7 | clap = { version = "4.5.20", features = ["derive"] } 8 | ctrlc = "3.4.5" 9 | mainline = { path = "../.." } 10 | num_cpus = "1.16.0" 11 | plotters = "0.3.7" 12 | statrs = "0.17.1" 13 | -------------------------------------------------------------------------------- /docs/simulation/src/main.rs: -------------------------------------------------------------------------------- 1 | use std::collections::BTreeMap; 2 | use std::collections::HashMap; 3 | use std::io::Write; 4 | use std::sync::atomic::{AtomicBool, Ordering}; 5 | use std::sync::{Arc, Mutex}; 6 | use std::thread; 7 | 8 | use std::net::SocketAddrV4; 9 | 10 | use clap::Parser; 11 | use full_palette::GREY; 12 | use mainline::{ClosestNodes, Id, Node}; 13 | use plotters::prelude::*; 14 | use statrs::statistics::*; 15 | 16 | const DEFAULT_DHT_SIZE: usize = 2_000_000; 17 | 18 | const DEFAULT_LOOKUPS: usize = 16; 19 | 20 | #[derive(Parser)] 21 | #[command(author, version, about, long_about = None)] 22 | struct Cli { 23 | /// Size of the Dht to sample 24 | dht_size: Option, 25 | /// Number of lookups in each simulation before estimating the dht size 26 | lookups: Option, 27 | } 28 | 29 | fn main() { 30 | let cli = Cli::parse(); 31 | 32 | let dht_size = cli.dht_size.unwrap_or(DEFAULT_DHT_SIZE); 33 | let lookups = cli.lookups.unwrap_or(DEFAULT_LOOKUPS); 34 | 35 | let cpus = num_cpus::get() - 1; 36 | let stop_event = Arc::new(AtomicBool::new(false)); 37 | 38 | let estimates = Arc::new(Mutex::new(vec![])); 39 | 40 | println!("Building a DHT with {} nodes...", dht_size); 41 | let dht = build_dht(dht_size); 42 | 43 | let mut handles = Vec::new(); 44 | for _ in 0..cpus { 45 | let stop_event = stop_event.clone(); 46 | let estimates = estimates.clone(); 47 | let dht = dht.clone(); 48 | 49 | let handle = thread::spawn(move || { 50 | while !stop_event.load(Ordering::Relaxed) { 51 | let estimate = simulate(&dht, lookups); 52 | 53 | let mut estimates = estimates.lock().unwrap(); 54 | estimates.push(estimate as f64); 55 | 56 | print!("\rsimulations {}", estimates.len()); 57 | std::io::stdout().flush().unwrap(); 58 | } 59 | }); 60 | handles.push(handle); 61 | } 62 | 63 | println!("\nEstimating Dht size after {lookups} lookups"); 64 | 65 | // Handle Ctrl+C to gracefully stop threads 66 | ctrlc::set_handler(move || { 67 | stop_event.store(true, Ordering::Relaxed); 68 | }) 69 | .expect("Error setting Ctrl+C handler"); 70 | 71 | for handle in handles { 72 | handle.join().expect("Thread panicked"); 73 | } 74 | 75 | let estimates = estimates.lock().unwrap(); 76 | println!("\nDone.\n"); 77 | 78 | let manifest_dir = std::env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR is not set"); 79 | let file_path = std::path::Path::new(&manifest_dir).join("../plot.png"); 80 | 81 | draw_plot( 82 | dht_size as f64, 83 | lookups, 84 | &estimates, 85 | file_path.to_str().unwrap(), 86 | ); 87 | } 88 | 89 | /// Build a Dht with `size` number of nodes, uniformly distributed across the key space. 90 | fn build_dht(size: usize) -> Arc> { 91 | let mut dht = BTreeMap::new(); 92 | for i in 0..size { 93 | let node = Node::new(Id::random(), SocketAddrV4::new((i as u32).into(), i as u16)); 94 | dht.insert(*node.id(), node); 95 | } 96 | 97 | Arc::new(dht) 98 | } 99 | 100 | /// Simulate the Dht size estimation of a node running and performed 101 | /// `lookups` number of queries to random Ids. 102 | /// 103 | /// Does the same as `ClosestNodes` in `mainline`, averaged over number of lookups. 104 | fn simulate(dht: &BTreeMap, lookups: usize) -> usize { 105 | (0..lookups) 106 | .map(|_| { 107 | let target = Id::random(); 108 | 109 | let mut closest_nodes = ClosestNodes::new(target); 110 | 111 | for (_, node) in dht.range(target..).take(200) { 112 | closest_nodes.add(node.clone().into()) 113 | } 114 | for (_, node) in dht.range(..target).rev().take(200) { 115 | closest_nodes.add(node.clone().into()) 116 | } 117 | 118 | let estimate = closest_nodes.dht_size_estimate(); 119 | 120 | estimate as usize 121 | }) 122 | .sum::() 123 | / lookups 124 | } 125 | 126 | fn draw_plot(expected: f64, lookups: usize, data: &Vec, filename: &str) { 127 | let data = Data::new(data.to_vec()); 128 | 129 | let mean = data.mean().unwrap(); 130 | let std_dev = data.std_dev().unwrap(); 131 | 132 | let margin = 3.0 * std_dev; 133 | 134 | let x_min = mean - margin; 135 | let x_max = mean + margin; 136 | 137 | let mean = data.mean().unwrap(); 138 | 139 | println!("Statistics:"); 140 | println!("\tDht size: {:.0}", expected); 141 | println!("\tMean: {:.0}", mean); 142 | // println!("\tError factor: {:.3}", (mean - expected) / expected); 143 | println!("\tStandard Deviation: {:.0}%", (std_dev / expected) * 100.0); 144 | println!( 145 | "\t95% Confidence Interval: +-{:.0}%", 146 | ((std_dev * 2.0) / expected) * 100.0 147 | ); 148 | 149 | let mut bands = HashMap::new(); 150 | let band_width = ((x_max - x_min) as u32 / 200).max(1); 151 | 152 | let mut y_max = 0; 153 | 154 | for estimate in data.iter() { 155 | // round to nearest 1/1000th 156 | let band = (*estimate as u32 / band_width) * band_width; 157 | 158 | let count = bands.get(&band).unwrap_or(&(0 as u32)) + 1; 159 | bands.insert(band, count); 160 | 161 | y_max = y_max.max(count); 162 | } 163 | 164 | // Set up the drawing area (800x600 pixels) 165 | let root = BitMapBackend::new(filename, (800, 600)).into_drawing_area(); 166 | root.fill(&WHITE).unwrap(); 167 | 168 | // Create a chart with labels for both axes 169 | let mut chart = ChartBuilder::on(&root) 170 | .caption( 171 | format!("{} nodes, {} lookups", expected, lookups), 172 | ("sans-serif", 40), 173 | ) 174 | .margin(10) 175 | .x_label_area_size(35) 176 | .y_label_area_size(40) 177 | .build_cartesian_2d(x_min as u32..x_max as u32, 0u32..y_max) 178 | .unwrap(); 179 | 180 | chart 181 | .configure_mesh() 182 | .disable_y_mesh() 183 | .light_line_style(WHITE.mix(0.3)) 184 | .bold_line_style(GREY.stroke_width(1)) 185 | .y_desc("Count") 186 | .x_desc("Nodes") 187 | .axis_desc_style(("sans-serif", 20)) 188 | .draw() 189 | .unwrap(); 190 | 191 | chart 192 | .draw_series( 193 | Histogram::vertical(&chart) 194 | .style(RGBAColor(62, 106, 163, 1.0).filled()) 195 | .data(bands.iter().map(|(x, y)| (*x, *y))), 196 | ) 197 | .unwrap(); 198 | 199 | // Save the result to file 200 | root.present().unwrap(); 201 | } 202 | -------------------------------------------------------------------------------- /docs/standard-deviation-vs-lookups.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pubky/mainline/184dfb7b3cc55570ddb53d8f8e0212bf5d166542/docs/standard-deviation-vs-lookups.png -------------------------------------------------------------------------------- /examples/README.md: -------------------------------------------------------------------------------- 1 | # Examples 2 | 3 | ## Core API Examples 4 | These examples demonstrate the main functionality of the Mainline DHT library: 5 | 6 | ### Setup 7 | ```sh 8 | # Bootstrap a DHT node 9 | cargo run --example bootstrap 10 | 11 | # Implement a custom request filter 12 | cargo run --example request_filter 13 | 14 | # Cache and reuse bootstrap nodes 15 | cargo run --example cache_bootstrap 16 | 17 | # Advanced logging configuration 18 | cargo run --example logging 19 | ``` 20 | 21 | ### Anounce/GET Peers 22 | ```sh 23 | # Announce as a peer 24 | cargo run --example announce_peer <40 bytes hex info_hash> 25 | 26 | # Find peers 27 | cargo run --example get_peers <40 bytes hex info_hash> 28 | ``` 29 | 30 | ### PUT/GET Arbitrary Immutable values. 31 | ```sh 32 | # Store immutable data 33 | cargo run --example put_immutable 34 | 35 | # Retrieve immutable data 36 | cargo run --example get_immutable <40 bytes hex target from put_immutable> 37 | ``` 38 | 39 | ### PUT/GET Arbitrary Mutable items. 40 | ```sh 41 | 42 | # Store mutable data 43 | cargo run --example put_mutable <64 bytes hex secret_key> 44 | 45 | # Retrieve mutable data 46 | cargo run --example get_mutable <40 bytes hex target from put_mutable> 47 | ``` 48 | 49 | --- 50 | 51 | ## Analysis & Research Tools 52 | These examples are for DHT network analysis and research purposes: 53 | 54 | > Note: These tools are not part of the main API and are provided for curiosity/research only. 55 | 56 | ```sh 57 | # Analyze DHT node distribution 58 | cargo run --example count_ips_close_to_key 59 | 60 | # Estimate DHT size (Mark-Recapture method) 61 | cargo run --example mark_recapture_dht 62 | 63 | # Measure DHT network size 64 | cargo run --example measure_dht 65 | ``` 66 | -------------------------------------------------------------------------------- /examples/announce_peer.rs: -------------------------------------------------------------------------------- 1 | use std::{str::FromStr, time::Instant}; 2 | 3 | use mainline::{Dht, Id}; 4 | 5 | use clap::Parser; 6 | 7 | use tracing::Level; 8 | use tracing_subscriber; 9 | 10 | #[derive(Parser)] 11 | #[command(author, version, about, long_about = None)] 12 | struct Cli { 13 | /// info_hash to annouce a peer on 14 | infohash: String, 15 | } 16 | 17 | fn main() { 18 | tracing_subscriber::fmt().with_max_level(Level::INFO).init(); 19 | 20 | let cli = Cli::parse(); 21 | 22 | let info_hash = Id::from_str(cli.infohash.as_str()).expect("invalid infohash"); 23 | 24 | let dht = Dht::client().unwrap(); 25 | 26 | println!("\nAnnouncing peer on an infohash: {} ...\n", cli.infohash); 27 | 28 | println!("\n=== COLD QUERY ==="); 29 | announce(&dht, info_hash); 30 | 31 | println!("\n=== SUBSEQUENT QUERY ==="); 32 | announce(&dht, info_hash); 33 | } 34 | 35 | fn announce(dht: &Dht, info_hash: Id) { 36 | let start = Instant::now(); 37 | 38 | dht.announce_peer(info_hash, Some(6991)) 39 | .expect("announce_peer failed"); 40 | 41 | println!( 42 | "Announced peer in {:?} seconds", 43 | start.elapsed().as_secs_f32() 44 | ); 45 | } 46 | -------------------------------------------------------------------------------- /examples/bootstrap.rs: -------------------------------------------------------------------------------- 1 | use mainline::Dht; 2 | 3 | use tracing::Level; 4 | use tracing_subscriber; 5 | 6 | fn main() { 7 | tracing_subscriber::fmt() 8 | .with_max_level(Level::DEBUG) 9 | .init(); 10 | 11 | let client = Dht::client().unwrap(); 12 | 13 | client.bootstrapped(); 14 | 15 | let info = client.info(); 16 | 17 | println!("{:?}", info); 18 | } 19 | -------------------------------------------------------------------------------- /examples/cache_bootstrap.rs: -------------------------------------------------------------------------------- 1 | //! Demonstrates caching and reusing bootstrapping nodes from the running 2 | //! node's routing table. 3 | //! 4 | //! Saves the bootstrapping nodes in `examples/bootstrapping_nodes.toml` relative to 5 | //! the script's directory, regardless of where the script is run from. 6 | 7 | use std::fs; 8 | use std::io::{Read, Write}; 9 | use std::path::PathBuf; 10 | 11 | use mainline::Dht; 12 | 13 | use tracing::Level; 14 | use tracing_subscriber; 15 | 16 | fn main() { 17 | tracing_subscriber::fmt() 18 | .with_max_level(Level::DEBUG) 19 | .init(); 20 | 21 | let mut builder = Dht::builder(); 22 | 23 | let examples_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("examples"); 24 | let nodes_file = examples_dir.join("bootstrapping_nodes"); 25 | if nodes_file.exists() { 26 | let mut file = 27 | fs::File::open(&nodes_file).expect("Failed to open bootstrapping nodes file"); 28 | let mut content = String::new(); 29 | file.read_to_string(&mut content) 30 | .expect("Failed to read bootstrapping nodes file"); 31 | 32 | let cached_nodes = content 33 | .lines() 34 | .map(|line| line.to_string()) 35 | .collect::>(); 36 | 37 | // To confirm that these old nodes are still viable, 38 | // try `builder.bootstrap(&cached_nodes)` instead, 39 | // this way you don't rely on default bootstrap nodes. 40 | builder.extra_bootstrap(&cached_nodes); 41 | }; 42 | 43 | let client = builder.build().unwrap(); 44 | 45 | client.bootstrapped(); 46 | 47 | let bootstrap = client.to_bootstrap(); 48 | 49 | let bootstrap_content = bootstrap.join("\n"); 50 | let mut file = fs::File::create(&nodes_file).expect("Failed to save bootstrapping nodes"); 51 | file.write(bootstrap_content.as_bytes()) 52 | .expect("Failed to write bootstrapping nodes"); 53 | } 54 | -------------------------------------------------------------------------------- /examples/count_ips_close_to_key.rs: -------------------------------------------------------------------------------- 1 | /** 2 | * Counts all IP addresses around a random target ID and counts the number of hits, each IP gets. 3 | * Does this by initializing a new DHT node for each lookups to reach the target from different directions. 4 | * 5 | * The result shows how sloppy the lookup algorithms are. 6 | * 7 | Prints a histogram with the collected nodes 8 | First column are the buckets indicating the hit rate. 3 .. 12 summerizes the nodes that get hit with a probability of 3 to 12% in each lookup. 9 | Second column indicates the number of nodes that this bucket contains. [19] means 19 nodes got hit with a probability of 3 to 12%. 10 | Third column is a visualization of the number of nodes [19]. 11 | 12 | Example1: 13 | 3 .. 12 [ 19 ]: ∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎ 14 | Within one lookup, 19 nodes got hit in 3 to 12% of the cases. These are rarely found therefore. 15 | 16 | Example2: 17 | 84 .. 93 [ 15 ]: ∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎ 18 | Within one lookup, 15 nodes got hit in 84 to 93% of the cases. These nodes are therefore found in almost all lookups. 19 | 20 | Full example: 21 | 3 .. 12 [ 19 ]: ∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎ 22 | 12 .. 21 [ 2 ]: ∎∎ 23 | 21 .. 30 [ 3 ]: ∎∎∎ 24 | 30 .. 39 [ 2 ]: ∎∎ 25 | 39 .. 48 [ 3 ]: ∎∎∎ 26 | 48 .. 57 [ 0 ]: 27 | 57 .. 66 [ 0 ]: 28 | 66 .. 75 [ 0 ]: 29 | 75 .. 84 [ 1 ]: ∎ 30 | 84 .. 93 [ 15 ]: ∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎ 31 | */ 32 | use histo::Histogram; 33 | use mainline::{Dht, Id, Node}; 34 | use std::{ 35 | collections::{HashMap, HashSet}, 36 | net::Ipv4Addr, 37 | sync::mpsc::channel, 38 | }; 39 | use tracing::Level; 40 | 41 | const K: usize = 20; // Not really k but we take the k closest nodes into account. 42 | const MAX_DISTANCE: u8 = 150; // Health check to not include outrageously distant nodes. 43 | const USE_RANDOM_BOOTSTRAP_NODES: bool = false; 44 | 45 | fn main() { 46 | tracing_subscriber::fmt().with_max_level(Level::WARN).init(); 47 | 48 | let target = Id::random(); 49 | let mut ip_hits: HashMap = HashMap::new(); 50 | let (tx_interrupted, rx_interrupted) = channel(); 51 | 52 | println!("Count all IP addresses around a random target_key={target} k={K} max_distance={MAX_DISTANCE} random_boostrap={USE_RANDOM_BOOTSTRAP_NODES}."); 53 | println!("Press CTRL+C to show the histogram"); 54 | println!(); 55 | 56 | ctrlc::set_handler(move || { 57 | println!(); 58 | println!("Received Ctrl+C! Finishing current lookup. Hold on..."); 59 | tx_interrupted.send(()).unwrap(); 60 | }) 61 | .expect("Error setting Ctrl-C handler"); 62 | 63 | let mut last_nodes: HashSet = HashSet::new(); 64 | let mut lookup_count = 0; 65 | while rx_interrupted.try_recv().is_err() { 66 | lookup_count += 1; 67 | let dht = init_dht(USE_RANDOM_BOOTSTRAP_NODES); 68 | let nodes = dht.find_node(target); 69 | let nodes: Box<[Node]> = nodes 70 | .iter() 71 | .filter(|node| target.distance(node.id()) < MAX_DISTANCE) 72 | .cloned() 73 | .collect(); 74 | let closest_nodes = nodes.iter().take(K).cloned().collect::>(); 75 | let sockets: HashSet = closest_nodes 76 | .iter() 77 | .map(|node| *node.address().ip()) 78 | .collect(); 79 | for socket in sockets.iter() { 80 | let previous = ip_hits.get(socket); 81 | match previous { 82 | Some(val) => { 83 | ip_hits.insert(socket.clone(), val + 1); 84 | } 85 | None => { 86 | ip_hits.insert(socket.clone(), 1); 87 | } 88 | }; 89 | } 90 | 91 | if closest_nodes.is_empty() { 92 | continue; 93 | } 94 | let closest_node = closest_nodes.first().unwrap(); 95 | let closest_distance = target.distance(closest_node.id()); 96 | let furthest_node = closest_nodes.last().unwrap(); 97 | let furthest_distance = target.distance(furthest_node.id()); 98 | 99 | let overlap_with_last_lookup: HashSet = 100 | sockets.intersection(&last_nodes).map(|ip| *ip).collect(); 101 | 102 | let overlap = overlap_with_last_lookup.len() as f64 / K as f64; 103 | last_nodes = sockets; 104 | println!( 105 | "lookup={:02} Ips found {}. Closest node distance: {}, furthest node distance: {}, overlap with previous lookup {}%", 106 | lookup_count, 107 | ip_hits.len(), 108 | closest_distance, 109 | furthest_distance, 110 | (overlap*100 as f64) as usize 111 | ); 112 | } 113 | 114 | println!(); 115 | println!("Histogram"); 116 | print_histogram(ip_hits, lookup_count); 117 | } 118 | 119 | fn print_histogram(hits: HashMap, lookup_count: usize) { 120 | /* 121 | 122 | */ 123 | let mut histogram = Histogram::with_buckets(10); 124 | let percents: HashMap = hits 125 | .into_iter() 126 | .map(|(ip, hits)| { 127 | let percent = (hits as f32 / lookup_count as f32) * 100 as f32; 128 | (ip, percent as u64) 129 | }) 130 | .collect(); 131 | 132 | for (_, percent) in percents.iter() { 133 | histogram.add(percent.clone()); 134 | } 135 | 136 | println!("{}", histogram); 137 | } 138 | 139 | fn get_random_boostrap_nodes2() -> Vec { 140 | let dht = Dht::client().unwrap(); 141 | let nodes = dht.find_node(Id::random()); 142 | let addrs = nodes 143 | .iter() 144 | .map(|node| node.address().to_string()) 145 | .collect::>(); 146 | let slice: Vec = addrs[..8].into_iter().map(|va| va.clone()).collect(); 147 | slice 148 | } 149 | 150 | fn init_dht(use_random_boostrap_nodes: bool) -> Dht { 151 | if use_random_boostrap_nodes { 152 | let bootstrap = get_random_boostrap_nodes2(); 153 | return Dht::builder().bootstrap(&bootstrap).build().unwrap(); 154 | } else { 155 | Dht::client().unwrap() 156 | } 157 | } 158 | -------------------------------------------------------------------------------- /examples/get_immutable.rs: -------------------------------------------------------------------------------- 1 | use std::{str::FromStr, time::Instant}; 2 | 3 | use mainline::{Dht, Id}; 4 | 5 | use clap::Parser; 6 | 7 | use tracing::Level; 8 | use tracing_subscriber; 9 | 10 | #[derive(Parser)] 11 | #[command(author, version, about, long_about = None)] 12 | struct Cli { 13 | /// Immutable data sha1 hash to lookup. 14 | target: String, 15 | } 16 | 17 | fn main() { 18 | tracing_subscriber::fmt().with_max_level(Level::INFO).init(); 19 | 20 | let cli = Cli::parse(); 21 | 22 | let info_hash = Id::from_str(cli.target.as_str()).expect("Invalid info_hash"); 23 | 24 | let dht = Dht::client().unwrap(); 25 | 26 | println!("\nLooking up immutable data: {} ...\n", cli.target); 27 | 28 | println!("\n=== COLD QUERY ==="); 29 | get_immutable(&dht, info_hash); 30 | 31 | println!("\n=== SUBSEQUENT QUERY ==="); 32 | get_immutable(&dht, info_hash); 33 | } 34 | 35 | fn get_immutable(dht: &Dht, info_hash: Id) { 36 | let start = Instant::now(); 37 | 38 | // No need to stream responses, just print the first result, since 39 | // all immutable data items are guaranteed to be the same. 40 | let value = dht 41 | .get_immutable(info_hash) 42 | .expect("Failed to find the immutable value for the provided info_hash"); 43 | 44 | let string = String::from_utf8(value.to_vec()) 45 | .expect("expected immutable data to be valid utf-8 for this demo"); 46 | 47 | println!( 48 | "Got result in {:?} milliseconds\n", 49 | start.elapsed().as_millis() 50 | ); 51 | 52 | println!("Got immutable data: {:?}", string); 53 | 54 | println!( 55 | "\nQuery exhausted in {:?} milliseconds", 56 | start.elapsed().as_millis(), 57 | ); 58 | } 59 | -------------------------------------------------------------------------------- /examples/get_mutable.rs: -------------------------------------------------------------------------------- 1 | use ed25519_dalek::VerifyingKey; 2 | use std::convert::TryFrom; 3 | use tracing::Level; 4 | use tracing_subscriber; 5 | 6 | use std::time::Instant; 7 | 8 | use mainline::{Dht, MutableItem}; 9 | 10 | use clap::Parser; 11 | 12 | #[derive(Parser)] 13 | #[command(author, version, about, long_about = None)] 14 | struct Cli { 15 | /// Mutable data public key. 16 | public_key: String, 17 | } 18 | 19 | fn main() { 20 | tracing_subscriber::fmt() 21 | // Switch to DEBUG to see incoming values and the IP of the responding nodes 22 | .with_max_level(Level::INFO) 23 | .init(); 24 | 25 | let cli = Cli::parse(); 26 | 27 | let public_key = from_hex(cli.public_key.clone()).to_bytes(); 28 | let dht = Dht::client().unwrap(); 29 | 30 | println!("Looking up mutable item: {} ...", cli.public_key); 31 | 32 | println!("\n=== COLD LOOKUP ==="); 33 | get_first(&dht, &public_key); 34 | 35 | println!("\n=== SUBSEQUENT LOOKUP ==="); 36 | get_first(&dht, &public_key); 37 | 38 | println!("\n=== GET MOST RECENT ==="); 39 | let start = Instant::now(); 40 | 41 | println!("\nLooking up the most recent value.."); 42 | let item = dht.get_mutable_most_recent(&public_key, None); 43 | 44 | if let Some(item) = item { 45 | println!("Found the most recent value:"); 46 | print_value(&item); 47 | } else { 48 | println!("Not found"); 49 | } 50 | 51 | println!( 52 | "\nQuery exhausted in {:?} seconds.", 53 | start.elapsed().as_secs_f32(), 54 | ); 55 | } 56 | 57 | fn get_first(dht: &Dht, public_key: &[u8; 32]) { 58 | let start = Instant::now(); 59 | if let Some(item) = dht.get_mutable(public_key, None, None).next() { 60 | println!( 61 | "\nGot first result in {:?} milliseconds:", 62 | start.elapsed().as_millis() 63 | ); 64 | print_value(&item); 65 | } else { 66 | println!("Not Found") 67 | } 68 | } 69 | 70 | fn print_value(item: &MutableItem) { 71 | match String::from_utf8(item.value().to_vec()) { 72 | Ok(string) => { 73 | println!(" mutable item: {:?}, seq: {:?}", string, item.seq()); 74 | } 75 | Err(_) => { 76 | println!(" mutable item: {:?}, seq: {:?}", item.value(), item.seq(),); 77 | } 78 | }; 79 | } 80 | 81 | fn from_hex(s: String) -> VerifyingKey { 82 | if s.len() % 2 != 0 { 83 | panic!("Number of Hex characters should be even"); 84 | } 85 | 86 | let mut bytes = Vec::with_capacity(s.len() / 2); 87 | 88 | for i in 0..s.len() / 2 { 89 | let byte_str = &s[i * 2..(i * 2) + 2]; 90 | let byte = u8::from_str_radix(byte_str, 16).expect("Invalid hex character"); 91 | bytes.push(byte); 92 | } 93 | 94 | VerifyingKey::try_from(bytes.as_slice()).expect("Invalid mutable key") 95 | } 96 | -------------------------------------------------------------------------------- /examples/get_peers.rs: -------------------------------------------------------------------------------- 1 | use std::{str::FromStr, time::Instant}; 2 | 3 | use mainline::{Dht, Id}; 4 | 5 | use clap::Parser; 6 | 7 | use tracing::Level; 8 | use tracing_subscriber; 9 | 10 | #[derive(Parser)] 11 | #[command(author, version, about, long_about = None)] 12 | struct Cli { 13 | /// info_hash to lookup peers for 14 | infohash: String, 15 | } 16 | 17 | fn main() { 18 | tracing_subscriber::fmt() 19 | // Switch to DEBUG to see incoming values and the IP of the responding nodes 20 | .with_max_level(Level::INFO) 21 | .init(); 22 | 23 | let cli = Cli::parse(); 24 | 25 | let info_hash = Id::from_str(cli.infohash.as_str()).expect("Expected info_hash"); 26 | 27 | let dht = Dht::client().unwrap(); 28 | 29 | println!("Looking up peers for info_hash: {} ...", info_hash); 30 | println!("\n=== COLD QUERY ==="); 31 | get_peers(&dht, &info_hash); 32 | 33 | println!("\n=== SUBSEQUENT QUERY ==="); 34 | println!("Looking up peers for info_hash: {} ...", info_hash); 35 | get_peers(&dht, &info_hash); 36 | } 37 | 38 | fn get_peers(dht: &Dht, info_hash: &Id) { 39 | let start = Instant::now(); 40 | let mut first = false; 41 | 42 | let mut count = 0; 43 | 44 | for peer in dht.get_peers(*info_hash) { 45 | if !first { 46 | first = true; 47 | println!( 48 | "Got first result in {:?} milliseconds:", 49 | start.elapsed().as_millis() 50 | ); 51 | 52 | println!("peer {:?}", peer,); 53 | } 54 | 55 | count += 1; 56 | } 57 | 58 | println!( 59 | "\nQuery exhausted in {:?} milliseconds, got {:?} peers.", 60 | start.elapsed().as_millis(), 61 | count 62 | ); 63 | } 64 | -------------------------------------------------------------------------------- /examples/logging.rs: -------------------------------------------------------------------------------- 1 | use colored::*; 2 | use mainline::Dht; 3 | use std::{ 4 | thread, 5 | time::{Duration, SystemTime, UNIX_EPOCH}, 6 | }; 7 | use tracing::{debug, enabled, info, Level, Subscriber}; 8 | use tracing_subscriber::{ 9 | fmt::{format::Writer, FmtContext, FormatEvent, FormatFields}, 10 | registry::LookupSpan, 11 | }; 12 | 13 | // Constants for common strings 14 | const SOCKET_OUT: &str = "SOCKET OUT"; 15 | const SOCKET_IN: &str = "SOCKET IN"; 16 | const SOCKET_ERROR: &str = "SOCKET ERROR"; 17 | 18 | /// Custom formatter for DHT logs 19 | #[derive(Default)] 20 | struct DhtFormatter; 21 | 22 | impl FormatEvent for DhtFormatter 23 | where 24 | S: Subscriber + for<'a> LookupSpan<'a>, 25 | N: for<'a> FormatFields<'a> + 'static, 26 | { 27 | fn format_event( 28 | &self, 29 | _ctx: &FmtContext<'_, S, N>, 30 | mut writer: Writer<'_>, 31 | event: &tracing::Event<'_>, 32 | ) -> std::fmt::Result { 33 | let metadata = event.metadata(); 34 | let level = metadata.level(); 35 | 36 | let mut fields = String::with_capacity(256); // Pre-allocate string buffer 37 | let mut visitor = StringVisitor::new(&mut fields); 38 | event.record(&mut visitor); 39 | 40 | // Format based on target and content 41 | let formatted = if metadata.target() == "dht_socket" { 42 | format_socket_message(&fields) 43 | } else { 44 | format_dht_message(&fields) 45 | }; 46 | 47 | // Add level color 48 | let level_color = match *level { 49 | Level::TRACE => "TRACE".bright_black(), 50 | Level::DEBUG => "DEBUG".blue(), 51 | Level::INFO => "INFO".green(), 52 | Level::WARN => "WARN".yellow(), 53 | Level::ERROR => "ERROR".red(), 54 | }; 55 | 56 | // Exibe o timestamp como segundos desde a época Unix 57 | let timestamp = SystemTime::now() 58 | .duration_since(UNIX_EPOCH) 59 | .unwrap_or_default() 60 | .as_secs(); 61 | 62 | writeln!(writer, "{} {} {}", level_color, timestamp, formatted) 63 | } 64 | } 65 | 66 | // Helper to extract message string from event 67 | struct StringVisitor<'a>(&'a mut String); 68 | 69 | impl<'a> StringVisitor<'a> { 70 | fn new(s: &'a mut String) -> Self { 71 | StringVisitor(s) 72 | } 73 | } 74 | 75 | impl<'a> tracing::field::Visit for StringVisitor<'a> { 76 | fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn std::fmt::Debug) { 77 | self.0.push_str(field.name()); 78 | self.0.push('='); 79 | self.0.push_str(&format!("{:?} ", value)); 80 | } 81 | 82 | fn record_str(&mut self, field: &tracing::field::Field, value: &str) { 83 | self.0.push_str(field.name()); 84 | self.0.push('='); 85 | self.0.push_str(value); 86 | self.0.push(' '); 87 | } 88 | 89 | fn record_i64(&mut self, field: &tracing::field::Field, value: i64) { 90 | self.0.push_str(field.name()); 91 | self.0.push('='); 92 | self.0.push_str(&value.to_string()); 93 | self.0.push(' '); 94 | } 95 | 96 | fn record_u64(&mut self, field: &tracing::field::Field, value: u64) { 97 | self.0.push_str(field.name()); 98 | self.0.push('='); 99 | self.0.push_str(&value.to_string()); 100 | self.0.push(' '); 101 | } 102 | 103 | fn record_bool(&mut self, field: &tracing::field::Field, value: bool) { 104 | self.0.push_str(field.name()); 105 | self.0.push('='); 106 | self.0.push_str(&value.to_string()); 107 | self.0.push(' '); 108 | } 109 | } 110 | 111 | /// Custom logger that formats DHT messages 112 | #[derive(Debug)] 113 | struct DhtLogger; 114 | 115 | impl DhtLogger { 116 | #[inline] 117 | fn info(msg: &str) { 118 | info!("{}", format_dht_message(msg)); 119 | } 120 | 121 | #[inline] 122 | fn debug(msg: &str) { 123 | debug!("{}", format_dht_message(msg)); 124 | } 125 | } 126 | 127 | /// Format a DHT message for better readability 128 | fn format_dht_message(msg: &str) -> String { 129 | if msg.contains("socket_message_sending") || msg.contains("socket_message_receiving") { 130 | format_socket_message(msg) 131 | } else if msg.contains("Sending") { 132 | format_outgoing_message(msg) 133 | } else if msg.contains("Received") { 134 | format_incoming_message(msg) 135 | } else if msg.contains("socket_error") { 136 | format_socket_error(msg) 137 | } else if msg.contains("socket_validation") { 138 | format_socket_validation(msg) 139 | } else { 140 | msg.to_string() 141 | } 142 | } 143 | 144 | /// Format socket error messages 145 | fn format_socket_error(msg: &str) -> String { 146 | let mut details = String::with_capacity(512); 147 | 148 | // Extract error details using the new extractor para capturar erros com espaços completos 149 | if let Some(error) = extract_error_field(msg, "error=") { 150 | details.push_str(&format!(" Error: {}\n", error.red())); 151 | } 152 | 153 | // Extract from address 154 | if let Some(from) = extract_field(msg, "from=") { 155 | details.push_str(&format!(" From: {}\n", from.bright_blue())); 156 | } 157 | 158 | // Extract transaction ID 159 | if let Some(tid) = extract_field(msg, "transaction_id:") { 160 | details.push_str(&format!(" Transaction ID: {}\n", tid.bright_yellow())); 161 | } 162 | 163 | // Extract message content 164 | if let Some(message) = extract_field(msg, "message=") { 165 | details.push_str("\n Raw Message (Hex):\n"); 166 | let hex_dump = message 167 | .as_bytes() 168 | .iter() 169 | .map(|b| format!("{:02x}", b)) 170 | .collect::>() 171 | .chunks(16) 172 | .map(|chunk| chunk.join(" ")) 173 | .collect::>() 174 | .join("\n"); 175 | details.push_str(&format!(" {}\n", hex_dump)); 176 | } 177 | 178 | format!("[{}]\n{}", SOCKET_ERROR.red().bold(), details) 179 | } 180 | 181 | /// Format socket messages with detailed information 182 | fn format_socket_message(msg: &str) -> String { 183 | // Determine direction and color 184 | let (direction, color) = if msg.contains("socket_message_sending") { 185 | (SOCKET_OUT, "bright_blue") 186 | } else if msg.contains("socket_message_receiving") { 187 | (SOCKET_IN, "bright_green") 188 | } else if msg.contains("socket_error") { 189 | (SOCKET_ERROR, "red") 190 | } else { 191 | (SOCKET_IN, "bright_green") 192 | }; 193 | 194 | let direction = match color { 195 | "bright_blue" => direction.bright_blue(), 196 | "bright_green" => direction.bright_green(), 197 | "red" => direction.red().bold(), 198 | _ => direction.normal(), 199 | }; 200 | 201 | let mut details = String::with_capacity(512); 202 | 203 | // Special handling for error messages 204 | if msg.contains("context=socket_error") { 205 | let mut details = String::new(); 206 | 207 | if let Some(error_msg) = extract_field(msg, "message=") { 208 | // Extract error message parts 209 | let parts: Vec<&str> = error_msg.split('\n').collect(); 210 | 211 | if parts.len() >= 3 { 212 | // First line contains main error 213 | let error = parts[0].replace("Failed to parse packet bytes: ", ""); 214 | details.push_str(&format!(" Error Type: {}\n", "Parse Error")); 215 | details.push_str(&format!(" Details: {}\n", error)); 216 | 217 | // Second line contains address 218 | if let Some(addr) = parts[1].strip_prefix("From: ") { 219 | details.push_str(&format!(" Remote Addr: {}\n", addr)); 220 | } 221 | 222 | // Extract transaction ID if present 223 | if let Some(tid) = extract_field(msg, "transaction_id:") { 224 | details.push_str(&format!(" Transaction ID: {}\n", tid.bright_yellow())); 225 | } 226 | 227 | // Third line contains raw message 228 | if let Some(raw) = parts[2].strip_prefix("Raw message: ") { 229 | details.push_str("\n Raw Message (Hex):\n"); 230 | let hex_dump = raw 231 | .as_bytes() 232 | .iter() 233 | .map(|b| format!("{:02x}", b)) 234 | .collect::>() 235 | .chunks(16) 236 | .map(|chunk| chunk.join(" ")) 237 | .collect::>() 238 | .join("\n"); 239 | details.push_str(&format!(" {}\n", hex_dump)); 240 | } 241 | } 242 | } 243 | return format!("[{}]\n{}", SOCKET_ERROR.red().bold(), details); 244 | } 245 | 246 | // Improved message type parsing 247 | if let Some(msg_type) = msg 248 | .split("message_type: ") 249 | .nth(1) 250 | .or_else(|| msg.split("message_type=").nth(1)) 251 | { 252 | let type_str = if msg_type.contains("Response(FindNode") 253 | || msg_type.contains("Response { FindNode") 254 | { 255 | "FindNode Response".green() 256 | } else if msg_type.contains("Response(GetPeers") || msg_type.contains("Response { GetPeers") 257 | { 258 | if msg_type.contains("nodes: Some") || msg_type.contains("nodes: [") { 259 | "GetPeers Response (With Nodes)".green() 260 | } else { 261 | "GetPeers Response (No Nodes)".green() 262 | } 263 | } else if msg_type.contains("Response(NoValues") || msg_type.contains("Response { NoValues") 264 | { 265 | "GetPeers Response (With Nodes)".green() 266 | } else if msg_type.contains("Response(Ping") || msg_type.contains("Response { Ping") { 267 | "Ping Response".green() 268 | } else if msg_type.contains("Request(RequestSpecific") || msg_type.contains("Request {") { 269 | if msg_type.contains("FindNode") { 270 | "FindNode Request".yellow() 271 | } else if msg_type.contains("GetPeers") { 272 | "GetPeers Request".yellow() 273 | } else if msg_type.contains("Ping") { 274 | "Ping Request".yellow() 275 | } else if msg_type.contains("AnnouncePeer") { 276 | "AnnouncePeer Request".yellow() 277 | } else { 278 | "Unknown Request Type".red() 279 | } 280 | } else if msg_type.contains("Error") { 281 | "Error Response".red() 282 | } else { 283 | "Unknown Message Type".red() 284 | }; 285 | details.push_str(&format!(" Message Type: {}\n", type_str)); 286 | 287 | // Extract IDs (both requester and responder) 288 | if let Some(id) = extract_field(msg_type, "requester_id: Id(") 289 | .or_else(|| extract_field(msg_type, "requester_id: ")) 290 | { 291 | details.push_str(&format!(" Requester ID: {}\n", &id[..40].bright_yellow())); 292 | } 293 | if let Some(id) = extract_field(msg_type, "responder_id: Id(") 294 | .or_else(|| extract_field(msg_type, "responder_id: ")) 295 | { 296 | details.push_str(&format!(" Responder ID: {}\n", &id[..40].bright_yellow())); 297 | } 298 | 299 | // Extract version if present (format it nicely) 300 | if let Some(version) = msg 301 | .split("version: Some([") 302 | .nth(1) 303 | .or_else(|| msg.split("version: [").nth(1)) 304 | // Try both formats 305 | { 306 | if let Some(version_end) = version.split("])").next() { 307 | let version_nums: Vec<&str> = version_end 308 | .split(',') 309 | .map(|s| s.trim()) // Remove whitespace 310 | .collect(); 311 | 312 | if version_nums.len() >= 4 { 313 | let formatted_version = format!( 314 | "{}.{}.{}.{}", 315 | version_nums[0], 316 | version_nums[1], 317 | version_nums[2], 318 | version_nums[3].trim_end_matches(']') // Remove extra bracket if present 319 | ); 320 | details.push_str(&format!(" Version: {}\n", formatted_version.bright_cyan())); 321 | } 322 | } 323 | } 324 | 325 | // Extract read_only flag 326 | if let Some(read_only) = extract_field(msg, "read_only: ") { 327 | details.push_str(&format!(" Read Only: {}\n", read_only.yellow())); 328 | } 329 | 330 | // Extract token if present 331 | if let Some(token) = extract_field(msg_type, "token: ") { 332 | let clean_token = token 333 | .trim_start_matches('[') 334 | .trim_end_matches(']') 335 | .trim_end_matches(',') 336 | .trim(); 337 | details.push_str(&format!(" Token: {}\n", clean_token.bright_yellow())); 338 | } 339 | 340 | // Extract and format nodes if present 341 | if msg_type.contains("nodes: [") || msg_type.contains("nodes: Some([") { 342 | let nodes = format_nodes(msg_type); 343 | if !nodes.is_empty() { 344 | details.push_str(" Nodes:\n"); 345 | details.push_str(&nodes); 346 | details.push('\n'); 347 | } 348 | } 349 | } 350 | 351 | // Transaction ID 352 | if let Some(tid) = extract_field(msg, "transaction_id:") { 353 | details.push_str(&format!( 354 | " Transaction ID: {}\n", 355 | tid.trim_end_matches(',').bright_yellow() 356 | )); 357 | } 358 | 359 | // Context 360 | if let Some(context) = extract_field(msg, "context=") { 361 | let context_str = match context { 362 | "socket_message_sending" => "Sending DHT message", 363 | "socket_message_receiving" => "Receiving DHT message", 364 | "socket_error" => "Socket error occurred", 365 | "socket_validation" => "Socket validation", 366 | _ => context, 367 | }; 368 | details.push_str(&format!(" Context: {}\n", context_str.bright_black())); 369 | } 370 | 371 | // Raw message at the end for debugging 372 | details.push_str("\n Raw Log:\n"); 373 | details.push_str(&format!(" {}\n", msg.bright_black())); 374 | 375 | format!("[{}]\n{}", direction, details) 376 | } 377 | 378 | /// Format outgoing messages 379 | #[inline] 380 | fn format_outgoing_message(msg: &str) -> String { 381 | format_message(msg, SOCKET_OUT.bright_blue()) 382 | } 383 | 384 | /// Format incoming messages 385 | #[inline] 386 | fn format_incoming_message(msg: &str) -> String { 387 | format_message(msg, SOCKET_IN.bright_green()) 388 | } 389 | 390 | /// Common message formatting logic 391 | fn format_message(msg: &str, direction: ColoredString) -> String { 392 | let msg_type = match () { 393 | _ if msg.contains("find_node") => "FIND_NODE".yellow(), 394 | _ if msg.contains("get_peers") => "GET_PEERS".purple(), 395 | _ if msg.contains("announce_peer") => "ANNOUNCE".cyan(), 396 | _ if msg.contains("ping") => "PING".bright_black(), 397 | _ => "OTHER".white(), 398 | }; 399 | 400 | format!( 401 | "[{}] {} | {}\n{}", 402 | direction, 403 | msg_type, 404 | "DHT Message".bright_black(), 405 | format_message_details(msg) 406 | ) 407 | } 408 | 409 | /// Format message details in a structured way 410 | fn format_message_details(msg: &str) -> String { 411 | let mut details = String::with_capacity(256); 412 | 413 | // Extract and format common fields 414 | let fields = [ 415 | ("id=", "ID", "bright_yellow"), 416 | ("address=", "Address", "bright_blue"), 417 | ("nodes=", "Nodes", ""), 418 | ("values=", "Values", "bright_magenta"), 419 | ("token=", "Token", "bright_yellow"), 420 | ("transaction_id=", "Transaction ID", "bright_yellow"), 421 | ]; 422 | 423 | for (field, label, color) in fields.iter() { 424 | if let Some(value) = extract_field(msg, field) { 425 | if *field == "nodes=" { 426 | details.push_str(&format!(" {}:\n{}\n", label, format_nodes(value))); 427 | } else { 428 | let colored_value = match *color { 429 | "bright_yellow" => value.bright_yellow(), 430 | "bright_blue" => value.bright_blue(), 431 | "bright_magenta" => value.bright_magenta(), 432 | _ => value.normal(), 433 | }; 434 | details.push_str(&format!(" {}: {}\n", label, colored_value)); 435 | } 436 | } 437 | } 438 | 439 | details 440 | } 441 | 442 | /// Extract a field from the message 443 | #[inline] 444 | fn extract_field<'a>(msg: &'a str, field: &str) -> Option<&'a str> { 445 | msg.split(field).nth(1).and_then(|s| { 446 | let s = s.trim_start(); 447 | if s.starts_with('"') { 448 | s.trim_start_matches('"').split('"').next() 449 | } else { 450 | s.split_whitespace().next() 451 | } 452 | }) 453 | } 454 | 455 | /// Extract an error field from the message 456 | #[inline] 457 | fn extract_error_field<'a>(msg: &'a str, field: &str) -> Option<&'a str> { 458 | msg.split(field).nth(1).map(|s| { 459 | let s = s.trim_start(); 460 | // Define delimitadores que indicam o final do valor do campo. 461 | let delimiters = [" from=", " transaction_id:", " message=", "\n"]; 462 | let mut end = s.len(); 463 | for delim in delimiters.iter() { 464 | if let Some(idx) = s.find(delim) { 465 | end = end.min(idx); 466 | } 467 | } 468 | s[..end].trim() 469 | }) 470 | } 471 | 472 | /// Format nodes list for better readability 473 | fn format_nodes(msg: &str) -> String { 474 | msg.split("Node {") 475 | .skip(1) 476 | .filter_map(|node_str| { 477 | let id = extract_field(node_str, "Id(")?; 478 | let addr = node_str 479 | .split("address: ") 480 | .nth(1)? 481 | .split(',') 482 | .next()? 483 | .trim(); 484 | 485 | // Extract last_seen if present and clean up any trailing characters 486 | let last_seen = if let Some(last_seen_str) = node_str.split("last_seen: ").nth(1) { 487 | if let Some(value) = last_seen_str 488 | .split([',', '}', ']', ')']) // Split on any of these characters 489 | .next() 490 | .map(|s| s.trim()) 491 | { 492 | format!(", {}", format!("last_seen: {}", value).bright_magenta()) 493 | } else { 494 | String::new() 495 | } 496 | } else { 497 | String::new() 498 | }; 499 | 500 | Some(format!( 501 | " {} @ {}{}", 502 | id[..40].bright_yellow(), 503 | addr.bright_blue(), 504 | last_seen 505 | )) 506 | }) 507 | .collect::>() 508 | .join("\n") 509 | } 510 | 511 | fn format_socket_validation(msg: &str) -> String { 512 | format!("[{}] {}", "SOCKET VALIDATION".magenta().bold(), msg) 513 | } 514 | 515 | fn main() { 516 | // Configure logging with our custom formatter 517 | tracing_subscriber::fmt() 518 | .with_max_level(Level::TRACE) 519 | .with_thread_ids(true) 520 | .with_target(true) 521 | .with_file(true) 522 | .with_ansi(true) 523 | .with_line_number(true) 524 | .event_format(DhtFormatter::default()) 525 | .init(); 526 | 527 | // Configure and start the DHT node in server mode 528 | let dht = Dht::builder() 529 | .server_mode() 530 | .build() 531 | .expect("Failed to create DHT server"); 532 | 533 | DhtLogger::info("DHT server node is running! Press Ctrl+C to stop."); 534 | // Wait for bootstrap to complete 535 | DhtLogger::info("Waiting for bootstrap..."); 536 | dht.bootstrapped(); 537 | DhtLogger::info("Bootstrap complete!"); 538 | 539 | // Keep the program running and show periodic information 540 | loop { 541 | thread::sleep(Duration::from_secs(30)); 542 | let info = dht.info(); 543 | 544 | // Header 545 | DhtLogger::info(""); // Blank line to separate 546 | DhtLogger::info(&format!( 547 | " {}", 548 | "=== DHT Node Status ===".bright_cyan().bold() 549 | )); 550 | 551 | // Basic node information 552 | DhtLogger::info(&format!( 553 | " Node ID : {}", 554 | info.id().to_string().bright_yellow() 555 | )); 556 | 557 | DhtLogger::info(&format!( 558 | " Local Addr : {}", 559 | info.local_addr().to_string().bright_blue() 560 | )); 561 | 562 | if let Some(addr) = info.public_address() { 563 | DhtLogger::info(&format!( 564 | " Public Addr: {}", 565 | addr.to_string().bright_green() 566 | )); 567 | } 568 | 569 | DhtLogger::info(&format!( 570 | " Firewalled : {}", 571 | if info.firewalled() { 572 | "Yes".red().bold() 573 | } else { 574 | "No".green().bold() 575 | } 576 | )); 577 | 578 | DhtLogger::info(&format!( 579 | " Server Mode: {}", 580 | if info.server_mode() { 581 | "Active".green().bold() 582 | } else { 583 | "Inactive".yellow().bold() 584 | } 585 | )); 586 | 587 | // Statistics separator 588 | DhtLogger::info(""); 589 | DhtLogger::info(&format!( 590 | " {}", 591 | "=== Network Statistics ===".bright_cyan().bold() 592 | )); 593 | 594 | // Network statistics 595 | let (size_estimate, std_dev) = info.dht_size_estimate(); 596 | DhtLogger::info(&format!( 597 | " Estimated nodes in network: {} (±{}%)", 598 | size_estimate.to_string().cyan().bold(), 599 | format!("{:.1}", std_dev * 100.0).yellow() 600 | )); 601 | 602 | DhtLogger::info(""); // Blank line to separate updates 603 | 604 | // Debug info in compact format 605 | if enabled!(Level::DEBUG) { 606 | DhtLogger::debug(&format!(" Raw DHT Info: {:#?}", info)); 607 | } 608 | 609 | DhtLogger::info(""); // Blank line to separate updates 610 | } 611 | } 612 | -------------------------------------------------------------------------------- /examples/mark_recapture_dht.rs: -------------------------------------------------------------------------------- 1 | //! # DHT Size Estimation using Mark-Recapture Method 2 | //! 3 | //! This example demonstrates how to estimate the size of a Distributed Hash Table (DHT) 4 | //! using the Mark-Recapture method, specifically the Chapman estimator. 5 | //! 6 | //! The program continuously samples nodes from the DHT, marking and recapturing nodes 7 | //! to estimate the total population size (i.e., the total number of nodes in the DHT). 8 | //! It stops sampling when a minimum overlap between the marked and recaptured samples is achieved 9 | //! or when a maximum number of random node ID lookups is reached. 10 | //! 11 | //! ## Why Run This Example? 12 | //! 13 | //! - **Educational Purpose**: To understand how the methods of the `mainline` crate can be applied 14 | //! to estimate the size of a DHT or other distributed systems without any implicit assumption of 15 | //! node id distributions. 16 | //! 17 | //! - **Alternative Estimation Method**: While there are more time and data-efficient methods 18 | //! to measure mainline DHT size (see `measure_dht.rs`), this example provides an alternative 19 | //! approach that might be useful in certain scenarios or for comparison purposes. 20 | //! 21 | //! - **Practical Application**: For developers and researchers working with DHTs who are 22 | //! interested in estimating the network size without requiring global knowledge of the network. 23 | //! 24 | //! ## Notes 25 | //! 26 | //! - The default parameters are set to take about ~2 hours to complete. 27 | //! - Adjust the constants `MIN_OVERLAP`, `MAX_RANDOM_NODE_IDS`, and `BATCH_SIZE` 28 | //! as needed to balance between accuracy and computation time. 29 | //! 30 | //! ## How It Works 31 | //! 32 | //! 1. **Marking Phase**: Random node IDs are generated, and the nodes closest to these IDs 33 | //! are collected as the "marked" sample. 34 | //! 35 | //! 2. **Recapture Phase**: Another set of random node IDs are generated, and the nodes 36 | //! closest to these IDs are collected as the "recapture" sample. 37 | //! 38 | //! 3. **Estimation**: The overlap between the marked and recaptured samples is used to 39 | //! estimate the total population size using the Chapman estimator. 40 | //! 41 | //! ## Limitations 42 | //! 43 | //! - The estimation accuracy depends on the overlap between the samples. 44 | //! - The method assumes that the DHT is stable during the sampling period. 45 | //! - More efficient methods exist for measuring DHT size, and this method may not be suitable 46 | //! for large-scale or time-sensitive applications. 47 | //! 48 | 49 | use dashmap::DashSet; 50 | use mainline::{Dht, Id}; 51 | use rayon::{prelude::*, ThreadPool, ThreadPoolBuilder}; 52 | use tracing::{debug, info, Level}; 53 | 54 | /// Adjust as needed. Default will take about ~2 hours 55 | // Minimum number of overlapping nodes to stop sampling. 56 | const MIN_OVERLAP: usize = 10_000; 57 | // Maximum number of sampling rounds before stopping. Avoid infinite loops. 58 | const MAX_RANDOM_NODE_IDS: usize = 100_000; 59 | // Number of parallel lookups. Ideally not bigger than number of threads available. Display progress every N. 60 | const BATCH_SIZE: usize = 16; 61 | const Z_SCORE: f64 = 1.96; 62 | 63 | /// Represents the DHT size estimation result. 64 | struct EstimateResult { 65 | estimate: f64, 66 | standard_error: f64, 67 | lower_bound: f64, 68 | upper_bound: f64, 69 | } 70 | 71 | impl EstimateResult { 72 | fn display(&self) { 73 | println!("\nFinal Estimate:"); 74 | println!( 75 | "Estimated DHT Size: {} nodes", 76 | format_number(self.estimate as usize) 77 | ); 78 | println!( 79 | "95% Confidence Interval: {} - {} nodes", 80 | format_number(self.lower_bound as usize), 81 | format_number(self.upper_bound as usize) 82 | ); 83 | println!( 84 | "Standard Error: {} nodes", 85 | format_number(self.standard_error as usize) 86 | ); 87 | } 88 | } 89 | 90 | fn main() { 91 | // Initialize the logger. 92 | tracing_subscriber::fmt().with_max_level(Level::INFO).init(); 93 | 94 | println!("Estimating DHT size using Mark-Recapture with continuous sampling..."); 95 | println!( 96 | "Stopping at {} overlapped nodes or {} max random node id lookup iterations.", 97 | format_number(MIN_OVERLAP), 98 | format_number(MAX_RANDOM_NODE_IDS) 99 | ); 100 | 101 | // Configure the Rayon thread pool with a same num of threads as batch size 102 | let pool = ThreadPoolBuilder::new() 103 | .num_threads(BATCH_SIZE) 104 | .build() 105 | .expect("Failed to build Rayon thread pool"); 106 | 107 | // Initialize the DHT client. 108 | let dht = Dht::client().expect("Failed to create DHT client"); 109 | 110 | // Collect samples from the DHT. 111 | let (marked_sample, recapture_sample) = 112 | collect_samples(&dht, pool, MIN_OVERLAP, MAX_RANDOM_NODE_IDS); 113 | 114 | // Display the final statistics. 115 | if let Some(estimate) = compute_estimate(&marked_sample, &recapture_sample) { 116 | estimate.display(); 117 | } else { 118 | println!("Unable to calculate the DHT size estimate due to insufficient overlap."); 119 | } 120 | } 121 | 122 | fn collect_samples( 123 | dht: &Dht, 124 | pool: ThreadPool, 125 | min_overlap: usize, 126 | max_unique_random_node_ids: usize, 127 | ) -> (DashSet, DashSet) { 128 | let marked_sample = DashSet::new(); 129 | let recapture_sample = DashSet::new(); 130 | let mut total_iterations = 0; 131 | 132 | let mut size = 0.0; 133 | let mut confidence = 0.0; 134 | 135 | loop { 136 | if total_iterations >= max_unique_random_node_ids { 137 | println!("Reached maximum number of random node ID lookups."); 138 | break; 139 | } 140 | 141 | let mark_random_ids: Vec<_> = (0..BATCH_SIZE).map(|_| Id::random()).collect(); 142 | let recapture_random_ids: Vec<_> = (0..BATCH_SIZE).map(|_| Id::random()).collect(); 143 | 144 | // Perform sampling in the thread pool 145 | pool.install(|| { 146 | // Sample for marked_sample in parallel. 147 | mark_random_ids.par_iter().for_each(|random_id| { 148 | for node in dht.find_node(*random_id) { 149 | marked_sample.insert(*node.id()); 150 | } 151 | }); 152 | 153 | // Sample for recapture_sample in parallel. 154 | recapture_random_ids.par_iter().for_each(|random_id| { 155 | for node in dht.find_node(*random_id) { 156 | recapture_sample.insert(*node.id()); 157 | } 158 | }); 159 | }); 160 | 161 | total_iterations += BATCH_SIZE; 162 | 163 | // Compute overlap. 164 | let overlap = marked_sample 165 | .iter() 166 | .filter(|id| recapture_sample.contains(id)) 167 | .count(); 168 | 169 | if let Some(estimate) = compute_estimate(&marked_sample, &recapture_sample) { 170 | size = estimate.estimate; 171 | confidence = estimate.standard_error * Z_SCORE; 172 | } 173 | 174 | info!( 175 | "Sampled {}/{} random IDs. Found {}/{} overlapping nodes. Estimate is {}±{}", 176 | format_number(total_iterations), 177 | format_number(max_unique_random_node_ids), 178 | format_number(overlap), 179 | format_number(min_overlap), 180 | format_number(size as usize), 181 | format_number(confidence as usize) 182 | ); 183 | 184 | if overlap >= min_overlap { 185 | println!("Sufficient overlap achieved."); 186 | break; 187 | } 188 | } 189 | 190 | (marked_sample, recapture_sample) 191 | } 192 | 193 | /// Computes the DHT size estimate using the Chapman estimator. 194 | /// 195 | /// # Arguments 196 | /// 197 | /// * `marked_sample` - The marked sample as a DashSet. 198 | /// * `recapture_sample` - The recapture sample as a DashSet. 199 | /// 200 | /// # Returns 201 | /// 202 | /// An `Option` containing the estimate and statistical data. 203 | fn compute_estimate( 204 | marked_sample: &DashSet, 205 | recapture_sample: &DashSet, 206 | ) -> Option { 207 | let n1 = marked_sample.len() as f64; 208 | let n2 = recapture_sample.len() as f64; 209 | 210 | // Compute overlap (m). 211 | let m = marked_sample 212 | .iter() 213 | .filter(|id| recapture_sample.contains(id)) 214 | .count() as f64; 215 | 216 | debug!("\nComputing estimate with:"); 217 | debug!("Marked sample size (n1): {}", n1); 218 | debug!("Recapture sample size (n2): {}", n2); 219 | debug!("Overlap size (m): {}", m); 220 | 221 | if m > 0.0 { 222 | // Chapman estimator formula. 223 | let estimate = ((n1 + 1.0) * (n2 + 1.0) / (m + 1.0)) - 1.0; 224 | 225 | // Calculate variance and standard error. 226 | let variance = 227 | ((n1 + 1.0) * (n2 + 1.0) * (n1 - m) * (n2 - m)) / ((m + 1.0).powi(2) * (m + 2.0)); 228 | let standard_error = variance.sqrt(); 229 | 230 | // 95% confidence interval. 231 | let margin_of_error = Z_SCORE * standard_error; 232 | let lower_bound = (estimate - margin_of_error).max(0.0); 233 | let upper_bound = estimate + margin_of_error; 234 | 235 | Some(EstimateResult { 236 | estimate, 237 | standard_error, 238 | lower_bound, 239 | upper_bound, 240 | }) 241 | } else { 242 | None 243 | } 244 | } 245 | 246 | fn format_number(num: usize) -> String { 247 | if num >= 1_000_000_000 { 248 | format!("{:.1}B", num as f64 / 1_000_000_000.0) 249 | } else if num >= 1_000_000 { 250 | format!("{:.1}M", num as f64 / 1_000_000.0) 251 | } else if num >= 1_000 { 252 | format!("{:.1}K", num as f64 / 1_000.0) 253 | } else { 254 | num.to_string() 255 | } 256 | } 257 | -------------------------------------------------------------------------------- /examples/measure_dht.rs: -------------------------------------------------------------------------------- 1 | use mainline::{Dht, Id}; 2 | use tracing::Level; 3 | 4 | fn main() { 5 | tracing_subscriber::fmt().with_max_level(Level::INFO).init(); 6 | 7 | let dht = Dht::client().unwrap(); 8 | 9 | println!("Calculating Dht size by sampling random lookup queries..",); 10 | 11 | for lookups in 1.. { 12 | let _ = dht.find_node(Id::random()); 13 | 14 | let info = dht.info(); 15 | let (estimate, std_dev) = info.dht_size_estimate(); 16 | 17 | println!( 18 | "Dht size estimate after {} lookups: {} +-{:.0}% nodes", 19 | lookups, 20 | format_number(estimate), 21 | (std_dev * 2.0) * 100.0 22 | ); 23 | 24 | std::thread::sleep(std::time::Duration::from_millis(500)); 25 | } 26 | } 27 | 28 | fn format_number(num: usize) -> String { 29 | // Handle large numbers and format with suffixes 30 | if num >= 1_000_000_000 { 31 | return format!("{:.1}B", num as f64 / 1_000_000_000.0); 32 | } else if num >= 1_000_000 { 33 | return format!("{:.1}M", num as f64 / 1_000_000.0); 34 | } else if num >= 1_000 { 35 | return format!("{:.1}K", num as f64 / 1_000.0); 36 | } 37 | 38 | // Format with commas for thousands 39 | let num_str = num.to_string(); 40 | let mut result = String::new(); 41 | let len = num_str.len(); 42 | 43 | for (i, c) in num_str.chars().enumerate() { 44 | // Add a comma before every three digits, except for the first part 45 | if i > 0 && (len - i) % 3 == 0 { 46 | result.push(','); 47 | } 48 | result.push(c); 49 | } 50 | 51 | result 52 | } 53 | -------------------------------------------------------------------------------- /examples/put_immutable.rs: -------------------------------------------------------------------------------- 1 | use std::time::Instant; 2 | 3 | use mainline::Dht; 4 | 5 | use clap::Parser; 6 | 7 | use tracing::Level; 8 | use tracing_subscriber; 9 | 10 | #[derive(Parser)] 11 | #[command(author, version, about, long_about = None)] 12 | struct Cli { 13 | /// Value to store on the DHT 14 | value: String, 15 | } 16 | 17 | fn main() { 18 | tracing_subscriber::fmt().with_max_level(Level::INFO).init(); 19 | 20 | let cli = Cli::parse(); 21 | 22 | let dht = Dht::client().unwrap(); 23 | let value = cli.value.as_bytes(); 24 | 25 | println!("\nStoring immutable data: {} ...\n", cli.value); 26 | println!("\n=== COLD QUERY ==="); 27 | put_immutable(&dht, &value); 28 | 29 | println!("\n=== SUBSEQUENT QUERY ==="); 30 | put_immutable(&dht, &value); 31 | } 32 | 33 | fn put_immutable(dht: &Dht, value: &[u8]) { 34 | let start = Instant::now(); 35 | 36 | let info_hash = dht.put_immutable(value).expect("put immutable failed"); 37 | 38 | println!( 39 | "Stored immutable data as {:?} in {:?} milliseconds", 40 | info_hash, 41 | start.elapsed().as_millis() 42 | ); 43 | } 44 | -------------------------------------------------------------------------------- /examples/put_mutable.rs: -------------------------------------------------------------------------------- 1 | use std::{convert::TryFrom, time::Instant}; 2 | 3 | use mainline::{Dht, MutableItem, SigningKey}; 4 | 5 | use clap::Parser; 6 | 7 | use tracing::Level; 8 | use tracing_subscriber; 9 | 10 | #[derive(Parser)] 11 | #[command(author, version, about, long_about = None)] 12 | struct Cli { 13 | /// Mutable data public key. 14 | secret_key: String, 15 | /// Value to store on the DHT 16 | value: String, 17 | } 18 | 19 | fn main() { 20 | tracing_subscriber::fmt().with_max_level(Level::INFO).init(); 21 | 22 | let cli = Cli::parse(); 23 | 24 | let dht = Dht::client().unwrap(); 25 | 26 | let signer = from_hex(cli.secret_key); 27 | 28 | println!( 29 | "\nStoring mutable data: \"{}\" for public_key: {} ...", 30 | cli.value, 31 | to_hex(signer.verifying_key().as_bytes()) 32 | ); 33 | 34 | println!("\n=== COLD QUERY ==="); 35 | put(&dht, &signer, cli.value.as_bytes(), None); 36 | 37 | println!("\n=== SUBSEQUENT QUERY ==="); 38 | // You can now republish to the same closest nodes 39 | // skipping the the lookup step. 40 | put(&dht, &signer, cli.value.as_bytes(), None); 41 | } 42 | 43 | fn put(dht: &Dht, signer: &SigningKey, value: &[u8], salt: Option<&[u8]>) { 44 | let start = Instant::now(); 45 | 46 | let (item, cas) = if let Some(most_recent) = 47 | dht.get_mutable_most_recent(signer.verifying_key().as_bytes(), salt) 48 | { 49 | // 1. Optionally Create a new value to take the most recent's value in consideration. 50 | let mut new_value = most_recent.value().to_vec(); 51 | 52 | println!( 53 | "Found older value {:?}, appending new value to the old...", 54 | new_value 55 | ); 56 | 57 | new_value.extend_from_slice(value); 58 | 59 | // 2. Increment the sequence number to be higher than the most recent's. 60 | let most_recent_seq = most_recent.seq(); 61 | let new_seq = most_recent_seq + 1; 62 | 63 | println!("Found older seq {most_recent_seq} incremnting sequence to {new_seq}...",); 64 | 65 | ( 66 | MutableItem::new(signer.clone(), &new_value, new_seq, salt), 67 | // 3. Use the most recent [MutableItem::seq] as a `CAS`. 68 | Some(most_recent_seq), 69 | ) 70 | } else { 71 | (MutableItem::new(signer.clone(), value, 1, salt), None) 72 | }; 73 | 74 | dht.put_mutable(item, cas).unwrap(); 75 | 76 | println!( 77 | "Stored mutable data as {:?} in {:?} milliseconds", 78 | to_hex(signer.verifying_key().as_bytes()), 79 | start.elapsed().as_millis() 80 | ); 81 | } 82 | 83 | fn from_hex(s: String) -> SigningKey { 84 | if s.len() % 2 != 0 { 85 | panic!("Number of Hex characters should be even"); 86 | } 87 | 88 | let mut bytes = Vec::with_capacity(s.len() / 2); 89 | 90 | for i in 0..s.len() / 2 { 91 | let byte_str = &s[i * 2..(i * 2) + 2]; 92 | let byte = u8::from_str_radix(byte_str, 16).expect("Invalid hex character"); 93 | bytes.push(byte); 94 | } 95 | 96 | SigningKey::try_from(bytes.as_slice()).expect("Invalid signing key") 97 | } 98 | 99 | fn to_hex(bytes: &[u8]) -> String { 100 | let hex_chars: String = bytes.iter().map(|byte| format!("{:02x}", byte)).collect(); 101 | 102 | hex_chars 103 | } 104 | -------------------------------------------------------------------------------- /examples/request_filter.rs: -------------------------------------------------------------------------------- 1 | use std::net::SocketAddrV4; 2 | 3 | use mainline::{Dht, RequestFilter, RequestSpecific, ServerSettings}; 4 | use tracing::{info, Level}; 5 | 6 | #[derive(Debug, Default, Clone)] 7 | struct Filter; 8 | 9 | impl RequestFilter for Filter { 10 | fn allow_request(&self, request: &RequestSpecific, from: SocketAddrV4) -> bool { 11 | info!(?request, ?from, "Got Request"); 12 | 13 | true 14 | } 15 | } 16 | 17 | fn main() { 18 | tracing_subscriber::fmt().with_max_level(Level::INFO).init(); 19 | 20 | let client = Dht::builder() 21 | .server_mode() 22 | .server_settings(ServerSettings { 23 | filter: Box::new(Filter), 24 | ..Default::default() 25 | }) 26 | .build() 27 | .unwrap(); 28 | 29 | client.bootstrapped(); 30 | 31 | let info = client.info(); 32 | 33 | println!("{:?}", info); 34 | 35 | loop {} 36 | } 37 | -------------------------------------------------------------------------------- /src/common.rs: -------------------------------------------------------------------------------- 1 | //! Miscellaneous common structs used throughout the library. 2 | 3 | mod id; 4 | mod immutable; 5 | pub mod messages; 6 | mod mutable; 7 | mod node; 8 | mod routing_table; 9 | 10 | pub use id::*; 11 | pub use immutable::*; 12 | pub use messages::*; 13 | pub use mutable::*; 14 | pub use node::*; 15 | pub use routing_table::*; 16 | -------------------------------------------------------------------------------- /src/common/id.rs: -------------------------------------------------------------------------------- 1 | //! Kademlia node Id or a lookup target 2 | use crc::{Crc, CRC_32_ISCSI}; 3 | use getrandom::getrandom; 4 | use serde::{Deserialize, Serialize}; 5 | use std::convert::TryInto; 6 | use std::{ 7 | fmt::{self, Debug, Display, Formatter}, 8 | net::{IpAddr, Ipv4Addr, SocketAddr}, 9 | str::FromStr, 10 | }; 11 | 12 | /// The size of node IDs in bits. 13 | pub const ID_SIZE: usize = 20; 14 | pub const MAX_DISTANCE: u8 = ID_SIZE as u8 * 8; 15 | 16 | const IPV4_MASK: u32 = 0x030f3fff; 17 | const CASTAGNOLI: Crc = Crc::::new(&CRC_32_ISCSI); 18 | 19 | #[derive(Clone, Copy, PartialEq, Ord, PartialOrd, Eq, Hash, Serialize, Deserialize)] 20 | /// Kademlia node Id or a lookup target 21 | pub struct Id([u8; ID_SIZE]); 22 | 23 | impl Id { 24 | /// Generate a random Id 25 | pub fn random() -> Id { 26 | let mut bytes: [u8; 20] = [0; 20]; 27 | getrandom::getrandom(&mut bytes).expect("getrandom"); 28 | 29 | Id(bytes) 30 | } 31 | 32 | /// Create a new Id from some bytes. Returns Err if the input is not 20 bytes long. 33 | pub fn from_bytes>(bytes: T) -> Result { 34 | let bytes = bytes.as_ref(); 35 | if bytes.len() != ID_SIZE { 36 | return Err(InvalidIdSize(bytes.len())); 37 | } 38 | 39 | let mut tmp: [u8; ID_SIZE] = [0; ID_SIZE]; 40 | tmp[..ID_SIZE].clone_from_slice(&bytes[..ID_SIZE]); 41 | 42 | Ok(Id(tmp)) 43 | } 44 | 45 | /// Simplified XOR distance between this Id and a target Id. 46 | /// 47 | /// The distance is the number of trailing non zero bits in the XOR result. 48 | /// 49 | /// Distance to self is 0 50 | /// Distance to the furthest Id is 160 51 | /// Distance to an Id with 5 leading matching bits is 155 52 | pub fn distance(&self, other: &Id) -> u8 { 53 | MAX_DISTANCE - self.xor(other).leading_zeros() 54 | } 55 | 56 | /// Returns the number of leading zeros in the binary representation of `self`. 57 | pub fn leading_zeros(&self) -> u8 { 58 | for (i, byte) in self.0.iter().enumerate() { 59 | if *byte != 0 { 60 | // leading zeros so far + laedinge zeros of this byte 61 | return (i as u32 * 8 + byte.leading_zeros()) as u8; 62 | } 63 | } 64 | 65 | 160 66 | } 67 | 68 | /// Performs bitwise XOR between two Ids 69 | pub fn xor(&self, other: &Id) -> Id { 70 | let mut result = [0_u8; 20]; 71 | 72 | for (i, (a, b)) in self.0.iter().zip(other.0).enumerate() { 73 | result[i] = a ^ b; 74 | } 75 | 76 | result.into() 77 | } 78 | 79 | /// Returns a byte slice of this Id. 80 | pub fn as_bytes(&self) -> &[u8; 20] { 81 | &self.0 82 | } 83 | 84 | /// Create a new Id according to [BEP_0042](http://bittorrent.org/beps/bep_0042.html). 85 | pub fn from_addr(addr: &SocketAddr) -> Id { 86 | let ip = addr.ip(); 87 | 88 | Id::from_ip(ip) 89 | } 90 | 91 | /// Create a new Id from an Ipv4 address according to [BEP_0042](http://bittorrent.org/beps/bep_0042.html). 92 | pub fn from_ip(ip: IpAddr) -> Id { 93 | match ip { 94 | IpAddr::V4(addr) => Id::from_ipv4(addr), 95 | IpAddr::V6(_addr) => unimplemented!("Ipv6 is not supported"), 96 | } 97 | } 98 | 99 | /// Create a new Id from an Ipv4 address according to [BEP_0042](http://bittorrent.org/beps/bep_0042.html). 100 | pub fn from_ipv4(ipv4: Ipv4Addr) -> Id { 101 | let mut bytes = [0_u8; 21]; 102 | getrandom(&mut bytes).expect("getrandom"); 103 | 104 | from_ipv4_and_r(bytes[1..].try_into().expect("infallible"), ipv4, bytes[0]) 105 | } 106 | 107 | /// Validate that this Id is valid with respect to [BEP_0042](http://bittorrent.org/beps/bep_0042.html). 108 | pub fn is_valid_for_ip(&self, ipv4: Ipv4Addr) -> bool { 109 | if ipv4.is_private() || ipv4.is_link_local() || ipv4.is_loopback() { 110 | return true; 111 | } 112 | 113 | let expected = first_21_bits(&id_prefix_ipv4(ipv4, self.0[ID_SIZE - 1])); 114 | 115 | self.first_21_bits() == expected 116 | } 117 | 118 | pub(crate) fn first_21_bits(&self) -> [u8; 3] { 119 | first_21_bits(&self.0) 120 | } 121 | } 122 | 123 | fn first_21_bits(bytes: &[u8]) -> [u8; 3] { 124 | [bytes[0], bytes[1], bytes[2] & 0xf8] 125 | } 126 | 127 | fn from_ipv4_and_r(bytes: [u8; 20], ip: Ipv4Addr, r: u8) -> Id { 128 | let mut bytes = bytes; 129 | let prefix = id_prefix_ipv4(ip, r); 130 | 131 | // Set first 21 bits to the prefix 132 | bytes[0] = prefix[0]; 133 | bytes[1] = prefix[1]; 134 | // set the first 5 bits of the 3rd byte to the remaining 5 bits of the prefix 135 | bytes[2] = (prefix[2] & 0xf8) | (bytes[2] & 0x7); 136 | 137 | // Set the last byte to the random r 138 | bytes[ID_SIZE - 1] = r; 139 | 140 | Id(bytes) 141 | } 142 | 143 | fn id_prefix_ipv4(ip: Ipv4Addr, r: u8) -> [u8; 3] { 144 | let r32: u32 = r.into(); 145 | let ip_int: u32 = u32::from_be_bytes(ip.octets()); 146 | let masked_ip: u32 = (ip_int & IPV4_MASK) | (r32 << 29); 147 | 148 | let mut digest = CASTAGNOLI.digest(); 149 | digest.update(&masked_ip.to_be_bytes()); 150 | 151 | let crc = digest.finalize(); 152 | 153 | crc.to_be_bytes()[..3] 154 | .try_into() 155 | .expect("Failed to convert bytes 0-2 of the crc into a 3-byte array") 156 | } 157 | 158 | impl Display for Id { 159 | fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { 160 | #[allow(clippy::format_collect)] 161 | let hex_chars: String = self.0.iter().map(|byte| format!("{:02x}", byte)).collect(); 162 | 163 | write!(f, "{}", hex_chars) 164 | } 165 | } 166 | 167 | impl From<[u8; ID_SIZE]> for Id { 168 | fn from(bytes: [u8; ID_SIZE]) -> Id { 169 | Id(bytes) 170 | } 171 | } 172 | 173 | impl From<&[u8; ID_SIZE]> for Id { 174 | fn from(bytes: &[u8; ID_SIZE]) -> Id { 175 | Id(*bytes) 176 | } 177 | } 178 | 179 | impl From for [u8; ID_SIZE] { 180 | fn from(value: Id) -> Self { 181 | value.0 182 | } 183 | } 184 | 185 | impl FromStr for Id { 186 | type Err = DecodeIdError; 187 | 188 | fn from_str(s: &str) -> Result { 189 | if s.len() % 2 != 0 { 190 | return Err(DecodeIdError::OddNumberOfCharacters); 191 | } 192 | 193 | let mut bytes = Vec::with_capacity(s.len() / 2); 194 | 195 | for i in 0..s.len() / 2 { 196 | let byte_str = &s[i * 2..(i * 2) + 2]; 197 | if let Ok(byte) = u8::from_str_radix(byte_str, 16) { 198 | bytes.push(byte); 199 | } else { 200 | return Err(DecodeIdError::InvalidHexCharacter(byte_str.into())); 201 | } 202 | } 203 | 204 | Ok(Id::from_bytes(bytes)?) 205 | } 206 | } 207 | 208 | impl Debug for Id { 209 | fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { 210 | write!(f, "Id({})", self) 211 | } 212 | } 213 | 214 | #[derive(Debug)] 215 | pub struct InvalidIdSize(usize); 216 | 217 | impl std::error::Error for InvalidIdSize {} 218 | 219 | impl std::fmt::Display for InvalidIdSize { 220 | fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { 221 | write!(f, "Invalid Id size, expected 20, got {0}", self.0) 222 | } 223 | } 224 | 225 | #[derive(thiserror::Error, Debug)] 226 | /// Mainline crate error enum. 227 | pub enum DecodeIdError { 228 | /// Id is expected to by 20 bytes. 229 | #[error(transparent)] 230 | InvalidIdSize(#[from] InvalidIdSize), 231 | 232 | #[error("Hex encoding should contain an even number of hex characters")] 233 | /// Hex encoding should contain an even number of hex characters 234 | OddNumberOfCharacters, 235 | 236 | /// Invalid hex character 237 | #[error("Invalid Id encoding: {0}")] 238 | InvalidHexCharacter(String), 239 | } 240 | 241 | #[cfg(test)] 242 | mod test { 243 | use super::*; 244 | 245 | #[test] 246 | fn distance_to_self() { 247 | let id = Id::random(); 248 | let distance = id.distance(&id); 249 | assert_eq!(distance, 0) 250 | } 251 | 252 | #[test] 253 | fn distance_to_id() { 254 | let id = Id::from_str("0639A1E24FBB8AB277DF033476AB0DE10FAB3BDC").unwrap(); 255 | 256 | let target = Id::from_str("035b1aeb9737ade1a80933594f405d3f772aa08e").unwrap(); 257 | 258 | let distance = id.distance(&target); 259 | 260 | assert_eq!(distance, 155) 261 | } 262 | 263 | #[test] 264 | fn distance_to_random_id() { 265 | let id = Id::random(); 266 | let target = Id::random(); 267 | 268 | let distance = id.distance(&target); 269 | 270 | assert_ne!(distance, 0) 271 | } 272 | 273 | #[test] 274 | fn distance_to_furthest() { 275 | let id = Id::random(); 276 | 277 | let mut opposite = [0_u8; 20]; 278 | for (i, &value) in id.as_bytes().iter().enumerate() { 279 | opposite[i] = value ^ 0xff; 280 | } 281 | let target = Id::from_bytes(opposite).unwrap(); 282 | 283 | let distance = id.distance(&target); 284 | 285 | assert_eq!(distance, MAX_DISTANCE) 286 | } 287 | 288 | #[test] 289 | fn from_u8_20() { 290 | let bytes = [8; 20]; 291 | 292 | let id: Id = bytes.into(); 293 | 294 | assert_eq!(*id.as_bytes(), bytes); 295 | } 296 | 297 | #[test] 298 | fn from_ipv4() { 299 | let vectors = vec![ 300 | (Ipv4Addr::new(124, 31, 75, 21), 1, [0x5f, 0xbf, 0xbf]), 301 | (Ipv4Addr::new(21, 75, 31, 124), 86, [0x5a, 0x3c, 0xe9]), 302 | (Ipv4Addr::new(65, 23, 51, 170), 22, [0xa5, 0xd4, 0x32]), 303 | (Ipv4Addr::new(84, 124, 73, 14), 65, [0x1b, 0x03, 0x21]), 304 | (Ipv4Addr::new(43, 213, 53, 83), 90, [0xe5, 0x6f, 0x6c]), 305 | ]; 306 | 307 | for vector in vectors { 308 | test(vector.0, vector.1, vector.2); 309 | } 310 | 311 | fn test(ip: Ipv4Addr, r: u8, expected_prefix: [u8; 3]) { 312 | let id = Id::random(); 313 | let result = from_ipv4_and_r(*id.as_bytes(), ip, r); 314 | let prefix = first_21_bits(result.as_bytes()); 315 | 316 | assert_eq!(prefix, first_21_bits(&expected_prefix)); 317 | assert_eq!(result.as_bytes()[ID_SIZE - 1], r); 318 | } 319 | } 320 | 321 | #[test] 322 | fn is_valid_for_ipv4() { 323 | let valid_vectors = vec![ 324 | ( 325 | Ipv4Addr::new(124, 31, 75, 21), 326 | "5fbfbff10c5d6a4ec8a88e4c6ab4c28b95eee401", 327 | ), 328 | ( 329 | Ipv4Addr::new(21, 75, 31, 124), 330 | "5a3ce9c14e7a08645677bbd1cfe7d8f956d53256", 331 | ), 332 | ( 333 | Ipv4Addr::new(65, 23, 51, 170), 334 | "a5d43220bc8f112a3d426c84764f8c2a1150e616", 335 | ), 336 | ( 337 | Ipv4Addr::new(84, 124, 73, 14), 338 | "1b0321dd1bb1fe518101ceef99462b947a01ff41", 339 | ), 340 | ( 341 | Ipv4Addr::new(43, 213, 53, 83), 342 | "e56f6cbf5b7c4be0237986d5243b87aa6d51305a", 343 | ), 344 | ]; 345 | 346 | for vector in valid_vectors { 347 | test(vector.0, vector.1); 348 | } 349 | 350 | fn test(ip: Ipv4Addr, hex: &str) { 351 | let id = Id::from_str(hex).unwrap(); 352 | 353 | assert!(id.is_valid_for_ip(ip)); 354 | } 355 | } 356 | } 357 | -------------------------------------------------------------------------------- /src/common/immutable.rs: -------------------------------------------------------------------------------- 1 | //! Helper functions for immutable items. 2 | 3 | use sha1_smol::Sha1; 4 | 5 | use super::ID_SIZE; 6 | use crate::Id; 7 | 8 | pub fn validate_immutable(v: &[u8], target: Id) -> bool { 9 | hash_immutable(v) == *target.as_bytes() 10 | } 11 | 12 | pub fn hash_immutable(v: &[u8]) -> [u8; ID_SIZE] { 13 | let mut encoded = Vec::with_capacity(v.len() + 3); 14 | encoded.extend(format!("{}:", v.len()).bytes()); 15 | encoded.extend_from_slice(v); 16 | 17 | let mut hasher = Sha1::new(); 18 | hasher.update(&encoded); 19 | 20 | hasher.digest().bytes() 21 | } 22 | 23 | #[cfg(test)] 24 | mod test { 25 | use super::*; 26 | use std::str::FromStr; 27 | 28 | #[test] 29 | fn test_validate_immutable() { 30 | let v = vec![ 31 | 171, 118, 111, 111, 174, 109, 195, 32, 138, 140, 113, 176, 76, 135, 116, 132, 156, 126, 32 | 75, 173, 33 | ]; 34 | 35 | let target = Id::from_bytes([ 36 | 2, 23, 113, 43, 67, 11, 185, 26, 26, 30, 204, 238, 204, 1, 13, 84, 52, 40, 86, 231, 37 | ]) 38 | .unwrap(); 39 | 40 | assert!(validate_immutable(&v, target)); 41 | assert!(!validate_immutable(&v[1..], target)); 42 | } 43 | 44 | #[test] 45 | 46 | fn test_hash_immutable() { 47 | let v = b"From the river to the sea, Palestine will be free"; 48 | let target = Id::from_str("4238af8aff56cf6e0007d9d2003bf23d33eea7c3").unwrap(); 49 | 50 | assert_eq!(hash_immutable(v), *target.as_bytes()); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/common/messages/internal.rs: -------------------------------------------------------------------------------- 1 | use serde::{Deserialize, Serialize}; 2 | use serde_bytes::ByteBuf; 3 | 4 | #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] 5 | pub struct DHTMessage { 6 | #[serde(rename = "t", with = "serde_bytes")] 7 | // Only few messages received seems to not use exactly 2 bytes, 8 | // and they don't seem to have a version. 9 | pub transaction_id: [u8; 2], 10 | 11 | #[serde(default)] 12 | #[serde(rename = "v", with = "serde_bytes")] 13 | pub version: Option<[u8; 4]>, 14 | 15 | #[serde(flatten)] 16 | pub variant: DHTMessageVariant, 17 | 18 | #[serde(default)] 19 | #[serde(with = "serde_bytes")] 20 | // Ipv6 is not supported anyways. 21 | pub ip: Option<[u8; 6]>, 22 | 23 | #[serde(default)] 24 | #[serde(rename = "ro")] 25 | pub read_only: Option, 26 | } 27 | 28 | impl DHTMessage { 29 | pub fn from_bytes(bytes: &[u8]) -> Result { 30 | let obj = serde_bencode::from_bytes(bytes)?; 31 | Ok(obj) 32 | } 33 | 34 | pub fn to_bytes(&self) -> Result, serde_bencode::Error> { 35 | serde_bencode::to_bytes(self) 36 | } 37 | } 38 | 39 | #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] 40 | #[serde(tag = "y")] 41 | pub enum DHTMessageVariant { 42 | #[serde(rename = "q")] 43 | Request(DHTRequestSpecific), 44 | 45 | #[serde(rename = "r")] 46 | Response(DHTResponseSpecific), 47 | 48 | #[serde(rename = "e")] 49 | Error(DHTErrorSpecific), 50 | } 51 | 52 | #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] 53 | #[serde(tag = "q")] 54 | pub enum DHTRequestSpecific { 55 | #[serde(rename = "ping")] 56 | Ping { 57 | #[serde(rename = "a")] 58 | arguments: DHTPingRequestArguments, 59 | }, 60 | 61 | #[serde(rename = "find_node")] 62 | FindNode { 63 | #[serde(rename = "a")] 64 | arguments: DHTFindNodeRequestArguments, 65 | }, 66 | 67 | #[serde(rename = "get_peers")] 68 | GetPeers { 69 | #[serde(rename = "a")] 70 | arguments: DHTGetPeersRequestArguments, 71 | }, 72 | 73 | #[serde(rename = "announce_peer")] 74 | AnnouncePeer { 75 | #[serde(rename = "a")] 76 | arguments: DHTAnnouncePeerRequestArguments, 77 | }, 78 | 79 | #[serde(rename = "get")] 80 | GetValue { 81 | #[serde(rename = "a")] 82 | arguments: DHTGetValueRequestArguments, 83 | }, 84 | 85 | #[serde(rename = "put")] 86 | PutValue { 87 | #[serde(rename = "a")] 88 | arguments: DHTPutValueRequestArguments, 89 | }, 90 | } 91 | 92 | #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] 93 | #[serde(untagged)] // This means order matters! Order these from most to least detailed 94 | pub enum DHTResponseSpecific { 95 | GetMutable { 96 | #[serde(rename = "r")] 97 | arguments: DHTGetMutableResponseArguments, 98 | }, 99 | 100 | NoMoreRecentValue { 101 | #[serde(rename = "r")] 102 | arguments: DHTNoMoreRecentValueResponseArguments, 103 | }, 104 | 105 | GetImmutable { 106 | #[serde(rename = "r")] 107 | arguments: DHTGetImmutableResponseArguments, 108 | }, 109 | 110 | GetPeers { 111 | #[serde(rename = "r")] 112 | arguments: DHTGetPeersResponseArguments, 113 | }, 114 | 115 | NoValues { 116 | #[serde(rename = "r")] 117 | arguments: DHTNoValuesResponseArguments, 118 | }, 119 | 120 | FindNode { 121 | #[serde(rename = "r")] 122 | arguments: DHTFindNodeResponseArguments, 123 | }, 124 | 125 | Ping { 126 | #[serde(rename = "r")] 127 | arguments: DHTPingResponseArguments, 128 | }, 129 | } 130 | 131 | #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] 132 | pub struct DHTErrorSpecific { 133 | #[serde(rename = "e")] 134 | pub error_info: (i32, String), 135 | } 136 | 137 | #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] 138 | pub enum DHTErrorValue { 139 | #[serde(rename = "")] 140 | ErrorCode(i32), 141 | ErrorDescription(String), 142 | } 143 | 144 | // === PING === 145 | 146 | #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] 147 | pub struct DHTPingRequestArguments { 148 | #[serde(with = "serde_bytes")] 149 | pub id: [u8; 20], 150 | } 151 | 152 | #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] 153 | pub struct DHTPingResponseArguments { 154 | #[serde(with = "serde_bytes")] 155 | pub id: [u8; 20], 156 | } 157 | 158 | // === FIND NODE === 159 | 160 | #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] 161 | pub struct DHTFindNodeRequestArguments { 162 | #[serde(with = "serde_bytes")] 163 | pub id: [u8; 20], 164 | 165 | #[serde(with = "serde_bytes")] 166 | pub target: [u8; 20], 167 | } 168 | 169 | #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] 170 | pub struct DHTFindNodeResponseArguments { 171 | #[serde(with = "serde_bytes")] 172 | pub id: [u8; 20], 173 | 174 | #[serde(with = "serde_bytes")] 175 | pub nodes: Box<[u8]>, 176 | } 177 | 178 | #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] 179 | pub struct DHTNoValuesResponseArguments { 180 | #[serde(with = "serde_bytes")] 181 | pub id: [u8; 20], 182 | 183 | #[serde(with = "serde_bytes")] 184 | pub token: Box<[u8]>, 185 | 186 | #[serde(with = "serde_bytes")] 187 | #[serde(default)] 188 | pub nodes: Option>, 189 | } 190 | 191 | // === Get Peers === 192 | 193 | #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] 194 | pub struct DHTGetPeersRequestArguments { 195 | #[serde(with = "serde_bytes")] 196 | pub id: [u8; 20], 197 | 198 | #[serde(with = "serde_bytes")] 199 | pub info_hash: [u8; 20], 200 | } 201 | 202 | #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] 203 | pub struct DHTGetPeersResponseArguments { 204 | #[serde(with = "serde_bytes")] 205 | pub id: [u8; 20], 206 | 207 | #[serde(with = "serde_bytes")] 208 | pub token: Box<[u8]>, 209 | 210 | #[serde(with = "serde_bytes")] 211 | #[serde(default)] 212 | pub nodes: Option>, 213 | 214 | // values are not optional, because if they are missing this missing 215 | // we can just treat this as DHTNoValuesResponseArguments 216 | pub values: Vec, 217 | } 218 | 219 | // === Announce Peer === 220 | 221 | #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] 222 | pub struct DHTAnnouncePeerRequestArguments { 223 | #[serde(with = "serde_bytes")] 224 | pub id: [u8; 20], 225 | 226 | #[serde(with = "serde_bytes")] 227 | pub info_hash: [u8; 20], 228 | 229 | pub port: u16, 230 | 231 | #[serde(with = "serde_bytes")] 232 | pub token: Box<[u8]>, 233 | 234 | #[serde(default)] 235 | pub implied_port: Option, 236 | } 237 | 238 | // === Get Value === 239 | 240 | #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] 241 | pub struct DHTGetValueRequestArguments { 242 | #[serde(with = "serde_bytes")] 243 | pub id: [u8; 20], 244 | 245 | #[serde(with = "serde_bytes")] 246 | pub target: [u8; 20], 247 | 248 | #[serde(default)] 249 | pub seq: Option, 250 | } 251 | 252 | #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] 253 | pub struct DHTGetImmutableResponseArguments { 254 | #[serde(with = "serde_bytes")] 255 | pub id: [u8; 20], 256 | 257 | #[serde(with = "serde_bytes")] 258 | pub token: Box<[u8]>, 259 | 260 | #[serde(with = "serde_bytes")] 261 | #[serde(default)] 262 | pub nodes: Option>, 263 | 264 | #[serde(with = "serde_bytes")] 265 | pub v: Box<[u8]>, 266 | } 267 | 268 | #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] 269 | pub struct DHTNoMoreRecentValueResponseArguments { 270 | #[serde(with = "serde_bytes")] 271 | pub id: [u8; 20], 272 | 273 | #[serde(with = "serde_bytes")] 274 | pub token: Box<[u8]>, 275 | 276 | #[serde(with = "serde_bytes")] 277 | #[serde(default)] 278 | pub nodes: Option>, 279 | 280 | pub seq: i64, 281 | } 282 | 283 | #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] 284 | pub struct DHTGetMutableResponseArguments { 285 | #[serde(with = "serde_bytes")] 286 | pub id: [u8; 20], 287 | 288 | #[serde(with = "serde_bytes")] 289 | pub token: Box<[u8]>, 290 | 291 | #[serde(with = "serde_bytes")] 292 | #[serde(default)] 293 | pub nodes: Option>, 294 | 295 | #[serde(with = "serde_bytes")] 296 | pub v: Box<[u8]>, 297 | 298 | #[serde(with = "serde_bytes")] 299 | pub k: [u8; 32], 300 | 301 | #[serde(with = "serde_bytes")] 302 | pub sig: [u8; 64], 303 | 304 | pub seq: i64, 305 | } 306 | 307 | // === Put Value === 308 | 309 | #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] 310 | pub struct DHTPutValueRequestArguments { 311 | #[serde(with = "serde_bytes")] 312 | pub id: [u8; 20], 313 | 314 | #[serde(with = "serde_bytes")] 315 | pub target: [u8; 20], 316 | 317 | #[serde(with = "serde_bytes")] 318 | pub token: Box<[u8]>, 319 | 320 | #[serde(with = "serde_bytes")] 321 | pub v: Box<[u8]>, 322 | 323 | #[serde(with = "serde_bytes")] 324 | #[serde(default)] 325 | pub k: Option<[u8; 32]>, 326 | 327 | #[serde(with = "serde_bytes")] 328 | #[serde(default)] 329 | pub sig: Option<[u8; 64]>, 330 | 331 | #[serde(default)] 332 | pub seq: Option, 333 | 334 | #[serde(default)] 335 | pub cas: Option, 336 | 337 | #[serde(with = "serde_bytes")] 338 | #[serde(default)] 339 | pub salt: Option>, 340 | } 341 | -------------------------------------------------------------------------------- /src/common/mutable.rs: -------------------------------------------------------------------------------- 1 | //! Helper functions and structs for mutable items. 2 | 3 | use ed25519_dalek::{Signature, Signer, SigningKey, Verifier, VerifyingKey}; 4 | use serde::{Deserialize, Serialize}; 5 | use sha1_smol::Sha1; 6 | use std::convert::TryFrom; 7 | 8 | use crate::Id; 9 | 10 | use super::PutMutableRequestArguments; 11 | 12 | #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] 13 | /// [BEP_0044](https://www.bittorrent.org/beps/bep_0044.html)'s Mutable item. 14 | pub struct MutableItem { 15 | /// hash of the key and optional salt 16 | target: Id, 17 | /// ed25519 public key 18 | key: [u8; 32], 19 | /// sequence number 20 | pub(crate) seq: i64, 21 | /// mutable value 22 | pub(crate) value: Box<[u8]>, 23 | /// ed25519 signature 24 | #[serde(with = "serde_bytes")] 25 | signature: [u8; 64], 26 | /// Optional salt 27 | salt: Option>, 28 | } 29 | 30 | impl MutableItem { 31 | /// Create a new mutable item from a signing key, value, sequence number and optional salt. 32 | pub fn new(signer: SigningKey, value: &[u8], seq: i64, salt: Option<&[u8]>) -> Self { 33 | let signable = encode_signable(seq, value, salt); 34 | let signature = signer.sign(&signable); 35 | 36 | Self::new_signed_unchecked( 37 | signer.verifying_key().to_bytes(), 38 | signature.into(), 39 | value, 40 | seq, 41 | salt, 42 | ) 43 | } 44 | 45 | /// Return the target of a [MutableItem] by hashing its `public_key` and an optional `salt` 46 | pub fn target_from_key(public_key: &[u8; 32], salt: Option<&[u8]>) -> Id { 47 | let mut encoded = vec![]; 48 | 49 | encoded.extend(public_key); 50 | 51 | if let Some(salt) = salt { 52 | encoded.extend(salt); 53 | } 54 | 55 | let mut hasher = Sha1::new(); 56 | hasher.update(&encoded); 57 | let bytes = hasher.digest().bytes(); 58 | 59 | bytes.into() 60 | } 61 | 62 | /// Create a new mutable item from an already signed value. 63 | pub fn new_signed_unchecked( 64 | key: [u8; 32], 65 | signature: [u8; 64], 66 | value: &[u8], 67 | seq: i64, 68 | salt: Option<&[u8]>, 69 | ) -> Self { 70 | Self { 71 | target: MutableItem::target_from_key(&key, salt), 72 | key, 73 | value: value.into(), 74 | seq, 75 | signature, 76 | salt: salt.map(|s| s.into()), 77 | } 78 | } 79 | 80 | pub(crate) fn from_dht_message( 81 | target: Id, 82 | key: &[u8], 83 | v: Box<[u8]>, 84 | seq: i64, 85 | signature: &[u8], 86 | salt: Option>, 87 | ) -> Result { 88 | let key = VerifyingKey::try_from(key).map_err(|_| MutableError::InvalidMutablePublicKey)?; 89 | 90 | let signature = 91 | Signature::from_slice(signature).map_err(|_| MutableError::InvalidMutableSignature)?; 92 | 93 | key.verify(&encode_signable(seq, &v, salt.as_deref()), &signature) 94 | .map_err(|_| MutableError::InvalidMutableSignature)?; 95 | 96 | Ok(Self { 97 | target, 98 | key: key.to_bytes(), 99 | value: v, 100 | seq, 101 | signature: signature.to_bytes(), 102 | salt, 103 | }) 104 | } 105 | 106 | // === Getters === 107 | 108 | /// Returns the target (info hash) of this item. 109 | pub fn target(&self) -> &Id { 110 | &self.target 111 | } 112 | 113 | /// Returns a reference to the 32 bytes Ed25519 public key of this item. 114 | pub fn key(&self) -> &[u8; 32] { 115 | &self.key 116 | } 117 | 118 | /// Returns a byte slice of the value of this item. 119 | pub fn value(&self) -> &[u8] { 120 | &self.value 121 | } 122 | 123 | /// Returns the `seq` (sequence) number of this item. 124 | pub fn seq(&self) -> i64 { 125 | self.seq 126 | } 127 | 128 | /// Returns the signature over this item. 129 | pub fn signature(&self) -> &[u8; 64] { 130 | &self.signature 131 | } 132 | 133 | /// Returns the `Salt` value used for generating the 134 | /// [Self::target] if any. 135 | pub fn salt(&self) -> Option<&[u8]> { 136 | self.salt.as_deref() 137 | } 138 | } 139 | 140 | pub fn encode_signable(seq: i64, value: &[u8], salt: Option<&[u8]>) -> Box<[u8]> { 141 | let mut signable = vec![]; 142 | 143 | if let Some(salt) = salt { 144 | signable.extend(format!("4:salt{}:", salt.len()).into_bytes()); 145 | signable.extend(salt); 146 | } 147 | 148 | signable.extend(format!("3:seqi{}e1:v{}:", seq, value.len()).into_bytes()); 149 | signable.extend(value); 150 | 151 | signable.into() 152 | } 153 | 154 | #[derive(thiserror::Error, Debug)] 155 | /// Mainline crate error enum. 156 | pub enum MutableError { 157 | #[error("Invalid mutable item signature")] 158 | /// Invalid mutable item signature 159 | InvalidMutableSignature, 160 | 161 | #[error("Invalid mutable item public key")] 162 | /// Invalid mutable item public key 163 | InvalidMutablePublicKey, 164 | } 165 | 166 | impl PutMutableRequestArguments { 167 | /// Create a [PutMutableRequestArguments] from a [MutableItem], 168 | /// and an optional CAS condition, which is usually the [MutableItem::seq] 169 | /// of the most recent known [MutableItem] 170 | pub fn from(item: MutableItem, cas: Option) -> Self { 171 | Self { 172 | target: item.target, 173 | v: item.value, 174 | k: item.key, 175 | seq: item.seq, 176 | sig: item.signature, 177 | salt: item.salt, 178 | cas, 179 | } 180 | } 181 | } 182 | 183 | impl From for MutableItem { 184 | fn from(request: PutMutableRequestArguments) -> Self { 185 | Self { 186 | target: request.target, 187 | value: request.v, 188 | key: request.k, 189 | seq: request.seq, 190 | signature: request.sig, 191 | salt: request.salt, 192 | } 193 | } 194 | } 195 | 196 | #[cfg(test)] 197 | mod tests { 198 | use super::*; 199 | 200 | #[test] 201 | fn signable_without_salt() { 202 | let signable = encode_signable(4, b"Hello world!", None); 203 | 204 | assert_eq!(&*signable, b"3:seqi4e1:v12:Hello world!"); 205 | } 206 | #[test] 207 | fn signable_with_salt() { 208 | let signable = encode_signable(4, b"Hello world!", Some(b"foobar")); 209 | 210 | assert_eq!(&*signable, b"4:salt6:foobar3:seqi4e1:v12:Hello world!"); 211 | } 212 | } 213 | -------------------------------------------------------------------------------- /src/common/node.rs: -------------------------------------------------------------------------------- 1 | //! Struct and implementation of the Node entry in the Kademlia routing table 2 | use std::{ 3 | fmt::{self, Debug, Formatter}, 4 | net::SocketAddrV4, 5 | sync::Arc, 6 | time::{Duration, Instant}, 7 | }; 8 | 9 | use crate::common::Id; 10 | 11 | /// The age of a node's last_seen time before it is considered stale and removed from a full bucket 12 | /// on inserting a new node. 13 | pub const STALE_TIME: Duration = Duration::from_secs(15 * 60); 14 | const MIN_PING_BACKOFF_INTERVAL: Duration = Duration::from_secs(10); 15 | pub const TOKEN_ROTATE_INTERVAL: Duration = Duration::from_secs(60 * 5); 16 | 17 | #[derive(PartialEq)] 18 | pub(crate) struct NodeInner { 19 | pub(crate) id: Id, 20 | pub(crate) address: SocketAddrV4, 21 | pub(crate) token: Option>, 22 | pub(crate) last_seen: Instant, 23 | } 24 | 25 | impl NodeInner { 26 | pub fn random() -> Self { 27 | Self { 28 | id: Id::random(), 29 | address: SocketAddrV4::new(0.into(), 0), 30 | token: None, 31 | last_seen: Instant::now(), 32 | } 33 | } 34 | } 35 | 36 | #[derive(Clone, PartialEq)] 37 | /// Node entry in Kademlia routing table 38 | pub struct Node(pub(crate) Arc); 39 | 40 | impl Debug for Node { 41 | fn fmt(&self, fmt: &mut Formatter<'_>) -> fmt::Result { 42 | fmt.debug_struct("Node") 43 | .field("id", &self.0.id) 44 | .field("address", &self.0.address) 45 | .field("last_seen", &self.0.last_seen.elapsed().as_secs()) 46 | .finish() 47 | } 48 | } 49 | 50 | impl Node { 51 | /// Creates a new Node from an id and socket address. 52 | pub fn new(id: Id, address: SocketAddrV4) -> Node { 53 | Node(Arc::new(NodeInner { 54 | id, 55 | address, 56 | token: None, 57 | last_seen: Instant::now(), 58 | })) 59 | } 60 | 61 | pub(crate) fn new_with_token(id: Id, address: SocketAddrV4, token: Box<[u8]>) -> Self { 62 | Node(Arc::new(NodeInner { 63 | id, 64 | address, 65 | token: Some(token), 66 | last_seen: Instant::now(), 67 | })) 68 | } 69 | 70 | /// Creates a node with random Id for testing purposes. 71 | pub fn random() -> Node { 72 | Node(Arc::new(NodeInner::random())) 73 | } 74 | 75 | /// Create a node that is unique per `i` as it has a random Id and sets IP and port to `i` 76 | #[cfg(test)] 77 | pub fn unique(i: usize) -> Node { 78 | Node::new(Id::random(), SocketAddrV4::new((i as u32).into(), i as u16)) 79 | } 80 | 81 | // === Getters === 82 | 83 | /// Returns the id of this node 84 | pub fn id(&self) -> &Id { 85 | &self.0.id 86 | } 87 | 88 | /// Returns the address of this node 89 | pub fn address(&self) -> SocketAddrV4 { 90 | self.0.address 91 | } 92 | 93 | /// Returns the token we received from this node if any. 94 | pub fn token(&self) -> Option> { 95 | self.0.token.clone() 96 | } 97 | 98 | /// Node is last seen more than a threshold ago. 99 | pub fn is_stale(&self) -> bool { 100 | self.0.last_seen.elapsed() > STALE_TIME 101 | } 102 | 103 | /// Node's token was received 5 minutes ago or less 104 | pub fn valid_token(&self) -> bool { 105 | self.0.last_seen.elapsed() <= TOKEN_ROTATE_INTERVAL 106 | } 107 | 108 | pub(crate) fn should_ping(&self) -> bool { 109 | self.0.last_seen.elapsed() > MIN_PING_BACKOFF_INTERVAL 110 | } 111 | 112 | /// Returns true if both nodes have the same ip and port 113 | pub fn same_address(&self, other: &Self) -> bool { 114 | self.0.address == other.0.address 115 | } 116 | 117 | /// Returns true if both nodes have the same ip 118 | pub fn same_ip(&self, other: &Self) -> bool { 119 | self.0.address.ip() == other.0.address.ip() 120 | } 121 | 122 | /// Node [Id] is valid for its IP address. 123 | /// 124 | /// Check [BEP_0042](https://www.bittorrent.org/beps/bep_0042.html). 125 | pub fn is_secure(&self) -> bool { 126 | self.0.id.is_valid_for_ip(*self.0.address.ip()) 127 | } 128 | 129 | /// Returns true if Any of the existing nodes: 130 | /// - Have the same IP as this node, And: 131 | /// = The existing nodes is Not secure. 132 | /// = The existing nodes is secure And shares the same first 21 bits. 133 | /// 134 | /// Effectively, allows only One non-secure node or Eight secure nodes from the same IP, in the routing table or ClosestNodes. 135 | pub(crate) fn already_exists(&self, nodes: &[Self]) -> bool { 136 | nodes.iter().any(|existing| { 137 | self.same_ip(existing) 138 | && (!existing.is_secure() 139 | || self.id().first_21_bits() == existing.id().first_21_bits()) 140 | }) 141 | } 142 | } 143 | -------------------------------------------------------------------------------- /src/common/routing_table.rs: -------------------------------------------------------------------------------- 1 | //! Simplified Kademlia routing table 2 | 3 | use std::collections::BTreeMap; 4 | use std::slice::Iter; 5 | 6 | use crate::common::{Id, Node}; 7 | use crate::rpc::ClosestNodes; 8 | 9 | /// K = the default maximum size of a k-bucket. 10 | pub const MAX_BUCKET_SIZE_K: usize = 20; 11 | 12 | #[derive(Debug, Clone)] 13 | /// Simplified Kademlia routing table 14 | pub struct RoutingTable { 15 | id: Id, 16 | buckets: BTreeMap, 17 | } 18 | 19 | impl RoutingTable { 20 | /// Create a new [RoutingTable] with a given id. 21 | pub fn new(id: Id) -> Self { 22 | let buckets = BTreeMap::new(); 23 | 24 | RoutingTable { id, buckets } 25 | } 26 | 27 | /// Returns the [Id] of this node, where the distance is measured from. 28 | pub fn id(&self) -> &Id { 29 | &self.id 30 | } 31 | 32 | /// Returns the map of distances and their [KBucket] 33 | pub(crate) fn buckets(&self) -> &BTreeMap { 34 | &self.buckets 35 | } 36 | 37 | // === Public Methods === 38 | 39 | /// Attempts to add a node to this routing table, and return `true` if it did. 40 | pub fn add(&mut self, node: Node) -> bool { 41 | let distance = self.id.distance(node.id()); 42 | 43 | if distance == 0 { 44 | // Do not add self to the routing_table 45 | return false; 46 | } 47 | 48 | if self 49 | .buckets() 50 | .values() 51 | .any(|bucket| node.already_exists(&bucket.nodes)) 52 | { 53 | return false; 54 | }; 55 | 56 | let bucket = self.buckets.entry(distance).or_default(); 57 | 58 | bucket.add(node) 59 | } 60 | 61 | /// Remove a node from this routing table. 62 | pub fn remove(&mut self, node_id: &Id) { 63 | let distance = self.id.distance(node_id); 64 | 65 | if let Some(bucket) = self.buckets.get_mut(&distance) { 66 | bucket.remove(node_id) 67 | } 68 | } 69 | 70 | /// Return the closest nodes to the target while prioritizing secure nodes, 71 | /// as defined in [BEP_0042](https://www.bittorrent.org/beps/bep_0042.html) 72 | pub fn closest(&self, target: Id) -> Box<[Node]> { 73 | let mut closest = ClosestNodes::new(target); 74 | 75 | for bucket in self.buckets.values() { 76 | for node in &bucket.nodes { 77 | closest.add(node.clone()); 78 | } 79 | } 80 | 81 | closest.nodes()[..MAX_BUCKET_SIZE_K.min(closest.len())].into() 82 | } 83 | 84 | /// Secure version of [Self::closest] that tries to circumvent sybil attacks. 85 | pub fn closest_secure( 86 | &self, 87 | target: Id, 88 | dht_size_estimate: usize, 89 | subnets: usize, 90 | ) -> Vec { 91 | let mut closest = ClosestNodes::new(target); 92 | 93 | for node in self.nodes() { 94 | closest.add(node); 95 | } 96 | 97 | closest 98 | .take_until_secure(dht_size_estimate, subnets) 99 | .to_vec() 100 | } 101 | 102 | /// Returns `true` if this routing table is empty. 103 | pub fn is_empty(&self) -> bool { 104 | self.buckets.values().all(|bucket| bucket.is_empty()) 105 | } 106 | 107 | /// Return the number of nodes in this routing table. 108 | pub fn size(&self) -> usize { 109 | self.buckets 110 | .values() 111 | .fold(0, |acc, bucket| acc + bucket.nodes.len()) 112 | } 113 | 114 | /// Returns an iterator over the nodes in this routing table. 115 | pub fn nodes(&self) -> RoutingTableIterator { 116 | RoutingTableIterator { 117 | bucket_index: 1, 118 | node_index: 0, 119 | table: self, 120 | } 121 | } 122 | 123 | /// Export an owned vector of nodes from this routing table. 124 | pub fn to_owned_nodes(&self) -> Vec { 125 | self.nodes().collect() 126 | } 127 | 128 | /// Turn this routing table to a list of bootstrapping nodes. 129 | pub fn to_bootstrap(&self) -> Vec { 130 | self.nodes() 131 | .filter(|n| !n.is_stale()) 132 | .map(|n| n.address().to_string()) 133 | .collect() 134 | } 135 | 136 | // === Private Methods === 137 | 138 | #[cfg(test)] 139 | fn contains(&self, node_id: &Id) -> bool { 140 | let distance = self.id.distance(node_id); 141 | 142 | if let Some(bucket) = self.buckets.get(&distance) { 143 | if bucket.contains(node_id) { 144 | return true; 145 | } 146 | } 147 | false 148 | } 149 | } 150 | 151 | pub struct RoutingTableIterator<'a> { 152 | bucket_index: u8, 153 | node_index: usize, 154 | table: &'a RoutingTable, 155 | } 156 | 157 | impl Iterator for RoutingTableIterator<'_> { 158 | type Item = Node; 159 | 160 | fn next(&mut self) -> Option { 161 | while self.bucket_index <= 160 { 162 | if let Some(current_bucket) = self.table.buckets.get(&self.bucket_index) { 163 | if let Some(current_node) = current_bucket.nodes.get(self.node_index) { 164 | self.node_index += 1; 165 | 166 | if self.node_index == current_bucket.nodes.len() { 167 | self.node_index = 0; 168 | self.bucket_index += 1; 169 | } 170 | 171 | return Some(current_node.clone()); 172 | } 173 | }; 174 | 175 | self.bucket_index += 1; 176 | } 177 | 178 | None 179 | } 180 | } 181 | 182 | /// Kbuckets are similar to LRU caches that checks and evicts unresponsive nodes, 183 | /// without dropping any responsive nodes in the process. 184 | #[derive(Debug, Clone)] 185 | pub struct KBucket { 186 | /// Nodes in the k-bucket, sorted by the least recently seen. 187 | nodes: Vec, 188 | } 189 | 190 | impl KBucket { 191 | pub fn new() -> Self { 192 | KBucket { 193 | nodes: Vec::with_capacity(MAX_BUCKET_SIZE_K), 194 | } 195 | } 196 | 197 | // === Getters === 198 | 199 | // === Public Methods === 200 | 201 | pub fn add(&mut self, incoming: Node) -> bool { 202 | if let Some(index) = self.iter().position(|n| n.id() == incoming.id()) { 203 | let existing = self.nodes[index].clone(); 204 | 205 | // If the incoming node is secure, then we trust its IP address for this Id, 206 | // and even if it changed its port number, we should accept it. 207 | // 208 | // If neither nodes are secure for this Id, but the incoming is the same IP, 209 | // then add the incoming one, effectively updating the node's 210 | // `last_seen` and moving it to the end of the bucket. 211 | // Possibly also updating the port, which is a good thing, instead of waiting 212 | // for the old port to timeout (not responding to Pings). 213 | // 214 | // Using same ip instead of same address, allow 215 | if incoming.is_secure() || (!existing.is_secure() && existing.same_ip(&incoming)) { 216 | self.nodes.remove(index); 217 | self.nodes.push(incoming); 218 | 219 | true 220 | } else { 221 | false 222 | } 223 | } else if self.nodes.len() < MAX_BUCKET_SIZE_K { 224 | self.nodes.push(incoming); 225 | true 226 | } else if self.nodes[0].is_stale() { 227 | // Remove the least recently seen node and add the new one 228 | self.nodes.remove(0); 229 | self.nodes.push(incoming); 230 | 231 | true 232 | } else { 233 | false 234 | } 235 | } 236 | 237 | pub fn remove(&mut self, node_id: &Id) { 238 | self.nodes.retain(|node| node.id() != node_id); 239 | } 240 | 241 | pub fn is_empty(&self) -> bool { 242 | self.nodes.is_empty() 243 | } 244 | 245 | pub fn iter(&self) -> Iter<'_, Node> { 246 | self.nodes.iter() 247 | } 248 | 249 | #[cfg(test)] 250 | fn contains(&self, id: &Id) -> bool { 251 | self.iter().any(|node| node.id() == id) 252 | } 253 | } 254 | 255 | impl Default for KBucket { 256 | fn default() -> Self { 257 | Self::new() 258 | } 259 | } 260 | 261 | #[cfg(test)] 262 | mod test { 263 | use std::net::SocketAddrV4; 264 | use std::str::FromStr; 265 | use std::sync::Arc; 266 | use std::time::Instant; 267 | 268 | use crate::common::{Id, KBucket, Node, NodeInner, RoutingTable, MAX_BUCKET_SIZE_K}; 269 | 270 | #[test] 271 | fn table_is_empty() { 272 | let mut table = RoutingTable::new(Id::random()); 273 | assert!(table.is_empty()); 274 | 275 | table.add(Node::random()); 276 | assert!(!table.is_empty()); 277 | } 278 | 279 | #[test] 280 | fn to_vec() { 281 | let mut table = RoutingTable::new(Id::random()); 282 | 283 | let mut expected_nodes: Vec = vec![]; 284 | 285 | for i in 0..MAX_BUCKET_SIZE_K { 286 | expected_nodes.push(Node::unique(i)); 287 | } 288 | 289 | for node in &expected_nodes { 290 | table.add(node.clone()); 291 | } 292 | 293 | let mut sorted_table = table.nodes().collect::>(); 294 | sorted_table.sort_by(|a, b| a.id().cmp(b.id())); 295 | 296 | let mut sorted_expected = expected_nodes.to_vec(); 297 | sorted_expected.sort_by(|a, b| a.id().cmp(b.id())); 298 | 299 | assert_eq!(sorted_table, sorted_expected); 300 | } 301 | 302 | #[test] 303 | fn contains() { 304 | let mut table = RoutingTable::new(Id::random()); 305 | 306 | let node = Node::random(); 307 | 308 | assert!(!table.contains(&node.id())); 309 | 310 | table.add(node.clone()); 311 | assert!(table.contains(&node.id())); 312 | } 313 | 314 | #[test] 315 | fn remove() { 316 | let mut table = RoutingTable::new(Id::random()); 317 | 318 | let node = Node::random(); 319 | 320 | table.add(node.clone()); 321 | assert!(table.contains(&node.id())); 322 | 323 | table.remove(node.id()); 324 | assert!(!table.contains(&node.id())); 325 | } 326 | 327 | #[test] 328 | fn buckets_are_sets() { 329 | let mut table = RoutingTable::new(Id::random()); 330 | 331 | let node1 = Node::random(); 332 | let node2 = Node::new(*node1.id(), node1.address()); 333 | 334 | table.add(node1); 335 | table.add(node2); 336 | 337 | assert_eq!(table.size(), 1); 338 | } 339 | 340 | #[test] 341 | fn should_not_add_self() { 342 | let mut table = RoutingTable::new(Id::random()); 343 | let node = Node::new(*table.id(), SocketAddrV4::new(0.into(), 0)); 344 | 345 | table.add(node.clone()); 346 | 347 | assert!(!table.add(node)); 348 | assert!(table.is_empty()) 349 | } 350 | 351 | #[test] 352 | fn should_not_add_more_than_k() { 353 | let mut bucket = KBucket::new(); 354 | 355 | for i in 0..MAX_BUCKET_SIZE_K { 356 | let node = Node::random(); 357 | assert!(bucket.add(node), "Failed to add node {}", i); 358 | } 359 | 360 | let node = Node::random(); 361 | 362 | assert!(!bucket.add(node)); 363 | } 364 | 365 | #[test] 366 | fn should_update_existing_node() { 367 | // Same address 368 | { 369 | let mut bucket = KBucket::new(); 370 | 371 | let node1 = Node::random(); 372 | let node2 = Node::new(*node1.id(), node1.address()); 373 | 374 | bucket.add(node1.clone()); 375 | bucket.add(Node::random()); 376 | 377 | assert_ne!(bucket.nodes[1].id(), node1.id()); 378 | 379 | bucket.add(node2); 380 | 381 | assert_eq!(bucket.nodes.len(), 2); 382 | assert_eq!(bucket.nodes[1].id(), node1.id()); 383 | } 384 | 385 | // Different port 386 | { 387 | let mut bucket = KBucket::new(); 388 | 389 | let node1 = Node::random(); 390 | let node2 = Node::new(*node1.id(), SocketAddrV4::new(*node1.address().ip(), 1)); 391 | 392 | bucket.add(node1.clone()); 393 | bucket.add(Node::random()); 394 | 395 | assert_ne!(bucket.nodes[1].id(), node1.id()); 396 | 397 | bucket.add(node2.clone()); 398 | 399 | assert_eq!(bucket.nodes.len(), 2); 400 | assert_eq!(bucket.nodes[1].id(), node1.id()); 401 | } 402 | 403 | { 404 | let mut bucket = KBucket::new(); 405 | 406 | let secure = Node(Arc::new(NodeInner { 407 | id: Id::from_str("5a3ce9c14e7a08645677bbd1cfe7d8f956d53256").unwrap(), 408 | address: SocketAddrV4::new([21, 75, 31, 124].into(), 0), 409 | token: None, 410 | last_seen: Instant::now(), 411 | })); 412 | 413 | let unsecure = Node::new(*secure.id(), SocketAddrV4::new([0, 0, 0, 0].into(), 1)); 414 | 415 | { 416 | bucket.add(unsecure.clone()); 417 | bucket.add(secure.clone()); 418 | 419 | assert_eq!(bucket.nodes[0].address(), secure.address()) 420 | } 421 | 422 | { 423 | bucket.add(secure.clone()); 424 | bucket.add(unsecure.clone()); 425 | 426 | assert_eq!(bucket.nodes[0].address(), secure.address()) 427 | } 428 | } 429 | 430 | // Different ip 431 | { 432 | let mut bucket = KBucket::new(); 433 | 434 | let node1 = Node::random(); 435 | let node2 = Node::new(*node1.id(), SocketAddrV4::new([0, 0, 0, 1].into(), 1)); 436 | 437 | bucket.add(node1.clone()); 438 | bucket.add(Node::random()); 439 | 440 | assert_ne!(bucket.nodes[1].id(), node1.id()); 441 | 442 | bucket.add(node2.clone()); 443 | 444 | assert_eq!(bucket.nodes.len(), 2); 445 | assert_ne!(bucket.nodes[1].id(), node1.id()); 446 | assert_ne!(bucket.nodes[1].address(), node2.address()); 447 | } 448 | } 449 | 450 | #[test] 451 | fn closest() { 452 | let ids = [ 453 | "fb449c17f6c34fadea26a5a83e1952e815e001ea", 454 | "e63b72f95aacee40ad087f83afb475645739f669", 455 | "58c65677e3833cb0f15733a6363cc4cb1352f90a", 456 | "fd042ff1404b495720ad8345404ff5f25acd02a8", 457 | "dbed34a2c8db568fe59c10adcca9e81825b3dcfd", 458 | "079d40b746b5721f59972ebde423429739844914", 459 | "094f1d2fb4b95ba2c3250b014a9f06d13cd9eb9a", 460 | "98805a55523458c56d59339266bdcecc82370ecd", 461 | "0a1d6cce47c60f2c7357e9fec2910192de6eb336", 462 | "fb689ce0e18c2c22f316976d3ae524aed4137773", 463 | "0d01c32b4cf386b0b784b718b999d0e9dac07876", 464 | "9465e80d80f707b222c4ae6ee81c02b62f607629", 465 | "6cdc012328cc7a3a9a5b967e93387686e19c9f75", 466 | "99719dfc220b145e2aac71d6b3e276731d85be1c", 467 | "94d2037bbc534a5f1d672ce3e3350576c2b78ed1", 468 | "b48d0aeb94cd3766f23d2ac098bbccf01485dc20", 469 | "3b6e1c05f199edd7dee87d3cc8422c8f0ed02358", 470 | "d9b50c6ca730c89f8fc9f518136cef6139dd2252", 471 | "15827c92e6efbc4f56e507e548409c4bc04360bf", 472 | "3c8ff1e484c21132f8e6b8112a2feab984536f57", 473 | "c9a8163fa3e85065d46567bfac39b5452cfb3ae8", 474 | "ef79f77e9eed9ad51094ce2747e2c4fdc3a81326", 475 | "81f038cabb8a845f39da0d40716bf0707da55187", 476 | "907fdf0aa137200b395bc210763ed947b03dfc2e", 477 | "b0bce9873042aee29cbc7ec395647f6cc7a482f8", 478 | "e6b8d5567bc05d9b68f23d562645bc030729abc9", 479 | "74667cb7c629fb7e63749134b16e27446984c517", 480 | "cdc7f4d5825dc316de20d998bc0f1c5e91e36a5e", 481 | "701e7b5af5fabcf0bc3de97cb05a7c00da3e53c6", 482 | "36eb09b1db4af2b11312742faa2bb42621fce753", 483 | "9e4923966754c02b036698e95f95cec8fc40a9d2", 484 | "0e43d66e9da1bfc7e2581155dfd1b8f4be57d3f1", 485 | "647679a0d8816d2f62200e7b6ef6171297756dd5", 486 | "c03d9008add37f8414cb41549448bb2dcb5c6c9b", 487 | "dff82b028a6ec033e00b387df8e386417b92a47c", 488 | "42e8b38494b0ee11003592da11b5cbe43332190e", 489 | "03161976385301ac9b965202e8f3922cef840790", 490 | "7d598e5726fb58501d8cc65faf6b676bab7cb4bc", 491 | "54ddde105d3f2c6ea7a5e7641ff24522eea2e784", 492 | "3a75532b5916c772c1b7a18627bf170cf915aeb3", 493 | "fa2b38321419e63cb890f8a8b5c53a1c4728a10a", 494 | "a3ba598bee9da287092f4f2f3864322af38e1824", 495 | "a94df01f21d870a006748b6ab3c04d31428c959d", 496 | "396aabc66c603617f376409053d1e2cec3813101", 497 | "a7b4becc2304da63792eb6c33f95677b2e7c9f8c", 498 | "58b1623af15a9828ccf41b8cee47d123c5cfe8b6", 499 | "3cb7eeac7be3a0195a9243537d452f790ccf1ca9", 500 | "e0296cfc4726d91a1f7f041e24638a1276a08bed", 501 | "aeb03edad3edc7c54a3c5f7916ecba981e65ce91", 502 | "4a81a4596b7c4b8706fd8b5c88ddfde18ca72293", 503 | "0d4e9ae7c486e5a0361bd4e3b918b6bdca89cfcb", 504 | "81d394b44403315f9845c3da6f018b8daedd89ef", 505 | "345630675ff0f319c8f2bb355edf59f9bd93072f", 506 | "b61fbd992a13af05feba939f597b5f6ee61188e3", 507 | "5ea45447e2e79a5f3b3d8c2f68aebdabf71c42f9", 508 | "84325dd6fbd9a93f4ab61d091a9562a6c6111df4", 509 | "e7c796aeecd47cfd01a2d62fd3fb1d41aafa2464", 510 | "897457b33c4eb1ffcab08331877108cbf3fac6de", 511 | "833843b1f33e720c17bccfb75647a49040861b4c", 512 | "06b49c253d3fc9800cfd75605d26426f8ccb89af", 513 | "5024212c42bed9f45e48c450147fecb3e934fc4e", 514 | "5a9de8041b045a7a4f85b71a6dc6a794a7fcd4ea", 515 | "70cad33774ddacb51ed1918adedeb67ff13a3b1e", 516 | "840d201e3c213c01b4ab85983efaac44f0671552", 517 | "aa7ffc7999a1b1bb79ce19b61c37f70331f492d6", 518 | "e2ec0c07e15411564292b5fa75246e4c385f4411", 519 | "38c1a0d14f548d4d81655920ec564b08e9fcf5e6", 520 | "1d128b8343569c7e9a8985879fafd325d458d31c", 521 | "4fcd30cbe02b74cece57babac93aded26ecdc893", 522 | "57d8a6d782ee1df62ceebd5d10884805ed382336", 523 | "54443ed3476d1d542f37bf069973bbd2b64c1b27", 524 | "0e7ba6c5e4c29cf4fff25733892b63cf2a6efdfc", 525 | "18824378226a6d33bcdbe39dd3bc9ee656ce20a2", 526 | "93b0cb01befc90b65a0026acf85bea2fefec7d44", 527 | "d65e378a1ec70cc79ae5b4469ae7f0e8939033fe", 528 | "9230a2f8ac81e73f16c63dd60adb030328fbc983", 529 | "302de797c9d73275ea184d7f6a8bf77364a8fd52", 530 | "cea92f6e6612ef408d8c22ad5c1ed602bb2aedbf", 531 | "353f2ff278f4ee038e7b217276a82d6ed0617130", 532 | "e962e3a1946afa0d3ee97f3a0418cb3489a5f84c", 533 | "a4e42b6cf98e957684aa4e7006940d31bcb76b1f", 534 | "57af8f960b2450ffa0dc5bc7314fece53996d4d0", 535 | "28e73f73084bc8e91fe9ec0a5581b583ef468d8c", 536 | "9481589ddec9a6d9ad2cee7f73e8319aab3f1e95", 537 | "edec09cc7476cd019560874def4af852bfeaffe3", 538 | "6c3ae2cf5f9452d5176788e15635c5958581c931", 539 | "f547b9717e84036c3d5eefec6d6ee3bfa5af89cb", 540 | "87b51f4bf1ccd41cda3aa85c71da5de56aeeda33", 541 | "e743092a576b92c8c05e04d5d2b23f2838825fd1", 542 | "e713b84894b761e2a4e20fd0e5a81ae48a6b6f9d", 543 | "6b1abca34099d2436bac8ab25aa17a57cbfe1564", 544 | "93cb2977e536a680c043b158345254c14b946d52", 545 | "8c2754fa9e93cbf1cccfd9241ebe0cc141199cfe", 546 | "13e4abf95a8a9e6525419b4db7b1704ed0a2789d", 547 | "8d53d453d7cfbb9bc386e128fa68aca388a5ddc6", 548 | "caebf39e9c9b48d87277f2a13faa5931a24819a4", 549 | "5025ca6cda98f31bc3ef321dd9a015b7f06b8bfa", 550 | "531fbf18fdf3e513091614f20d65e920a505ca41", 551 | "2f81e6159f7de0bc90c8a1db661b33bffbee85fd", 552 | "85d4d9954f3a28228a2786b320ad58a46a13f37b", 553 | ]; 554 | 555 | let nodes: Vec = ids 556 | .iter() 557 | .enumerate() 558 | .map(|(i, str)| { 559 | let id = Id::from_str(str).unwrap(); 560 | Node(Arc::new(NodeInner { 561 | id, 562 | address: SocketAddrV4::new((i as u32).into(), i as u16), 563 | token: None, 564 | last_seen: Instant::now(), 565 | })) 566 | }) 567 | .collect(); 568 | 569 | let local_id = Id::from_str("ba3042eb2d373b19e7c411ce6826e31b37be0b2e").unwrap(); 570 | 571 | let mut table = RoutingTable::new(local_id); 572 | 573 | for node in nodes { 574 | table.add(node); 575 | } 576 | 577 | { 578 | let expected_closest_ids: Vec<_> = [ 579 | "897457b33c4eb1ffcab08331877108cbf3fac6de", 580 | "907fdf0aa137200b395bc210763ed947b03dfc2e", 581 | "9230a2f8ac81e73f16c63dd60adb030328fbc983", 582 | "93b0cb01befc90b65a0026acf85bea2fefec7d44", 583 | "93cb2977e536a680c043b158345254c14b946d52", 584 | "9465e80d80f707b222c4ae6ee81c02b62f607629", 585 | "9481589ddec9a6d9ad2cee7f73e8319aab3f1e95", 586 | "94d2037bbc534a5f1d672ce3e3350576c2b78ed1", 587 | "98805a55523458c56d59339266bdcecc82370ecd", 588 | "99719dfc220b145e2aac71d6b3e276731d85be1c", 589 | "9e4923966754c02b036698e95f95cec8fc40a9d2", 590 | "a3ba598bee9da287092f4f2f3864322af38e1824", 591 | "a4e42b6cf98e957684aa4e7006940d31bcb76b1f", 592 | "a7b4becc2304da63792eb6c33f95677b2e7c9f8c", 593 | "a94df01f21d870a006748b6ab3c04d31428c959d", 594 | "aa7ffc7999a1b1bb79ce19b61c37f70331f492d6", 595 | "aeb03edad3edc7c54a3c5f7916ecba981e65ce91", 596 | "b0bce9873042aee29cbc7ec395647f6cc7a482f8", 597 | "b48d0aeb94cd3766f23d2ac098bbccf01485dc20", 598 | "b61fbd992a13af05feba939f597b5f6ee61188e3", 599 | ] 600 | .iter() 601 | .map(|id| Id::from_str(id).unwrap()) 602 | .collect(); 603 | 604 | let target = local_id; 605 | let closest = table.closest(target); 606 | 607 | let mut closest_ids: Vec = closest.iter().map(|n| *n.id()).collect(); 608 | closest_ids.sort(); 609 | 610 | assert_eq!(closest_ids, expected_closest_ids); 611 | } 612 | 613 | { 614 | let expected_closest_ids: Vec<_> = [ 615 | "c03d9008add37f8414cb41549448bb2dcb5c6c9b", 616 | "c9a8163fa3e85065d46567bfac39b5452cfb3ae8", 617 | "cdc7f4d5825dc316de20d998bc0f1c5e91e36a5e", 618 | "cea92f6e6612ef408d8c22ad5c1ed602bb2aedbf", 619 | "d65e378a1ec70cc79ae5b4469ae7f0e8939033fe", 620 | "d9b50c6ca730c89f8fc9f518136cef6139dd2252", 621 | "dbed34a2c8db568fe59c10adcca9e81825b3dcfd", 622 | "dff82b028a6ec033e00b387df8e386417b92a47c", 623 | "e0296cfc4726d91a1f7f041e24638a1276a08bed", 624 | "e2ec0c07e15411564292b5fa75246e4c385f4411", 625 | "e63b72f95aacee40ad087f83afb475645739f669", 626 | "e6b8d5567bc05d9b68f23d562645bc030729abc9", 627 | "e7c796aeecd47cfd01a2d62fd3fb1d41aafa2464", 628 | "e962e3a1946afa0d3ee97f3a0418cb3489a5f84c", 629 | "edec09cc7476cd019560874def4af852bfeaffe3", 630 | "ef79f77e9eed9ad51094ce2747e2c4fdc3a81326", 631 | "fa2b38321419e63cb890f8a8b5c53a1c4728a10a", 632 | "fb449c17f6c34fadea26a5a83e1952e815e001ea", 633 | "fb689ce0e18c2c22f316976d3ae524aed4137773", 634 | "fd042ff1404b495720ad8345404ff5f25acd02a8", 635 | ] 636 | .iter() 637 | .map(|str| Id::from_str(str).unwrap()) 638 | .collect(); 639 | 640 | let target = Id::from_str("d1406a3d3a8354d566f21dba8bd06c537cde2a20").unwrap(); 641 | let closest = table.closest(target); 642 | 643 | let mut closest_ids: Vec = closest.iter().map(|n| *n.id()).collect(); 644 | closest_ids.sort(); 645 | 646 | assert_eq!(closest_ids, expected_closest_ids); 647 | } 648 | } 649 | } 650 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | #![doc = include_str!("../README.md")] 2 | //! ## Feature flags 3 | #![doc = document_features::document_features!()] 4 | //! 5 | 6 | #![deny(missing_docs)] 7 | #![deny(rustdoc::broken_intra_doc_links)] 8 | #![cfg_attr(not(test), deny(clippy::unwrap_used))] 9 | 10 | mod common; 11 | #[cfg(feature = "node")] 12 | mod dht; 13 | mod rpc; 14 | 15 | // Public modules 16 | #[cfg(feature = "async")] 17 | pub mod async_dht; 18 | 19 | pub use common::{Id, MutableItem, Node, RoutingTable}; 20 | 21 | #[cfg(feature = "node")] 22 | pub use dht::{Dht, DhtBuilder, Testnet}; 23 | #[cfg(feature = "node")] 24 | pub use rpc::{ 25 | messages::{MessageType, PutRequestSpecific, RequestSpecific}, 26 | server::{RequestFilter, ServerSettings, MAX_INFO_HASHES, MAX_PEERS, MAX_VALUES}, 27 | ClosestNodes, DEFAULT_REQUEST_TIMEOUT, 28 | }; 29 | 30 | pub use ed25519_dalek::SigningKey; 31 | 32 | pub mod errors { 33 | //! Exported errors 34 | #[cfg(feature = "node")] 35 | pub use super::common::ErrorSpecific; 36 | #[cfg(feature = "node")] 37 | pub use super::dht::PutMutableError; 38 | #[cfg(feature = "node")] 39 | pub use super::rpc::{ConcurrencyError, PutError, PutQueryError}; 40 | 41 | pub use super::common::DecodeIdError; 42 | pub use super::common::MutableError; 43 | } 44 | -------------------------------------------------------------------------------- /src/rpc/closest_nodes.rs: -------------------------------------------------------------------------------- 1 | use std::{collections::HashSet, convert::TryInto}; 2 | 3 | use crate::{common::MAX_BUCKET_SIZE_K, Id, Node}; 4 | 5 | #[derive(Debug, Clone)] 6 | /// Manage closest nodes found in a query. 7 | /// 8 | /// Useful to estimate the Dht size. 9 | pub struct ClosestNodes { 10 | target: Id, 11 | nodes: Vec, 12 | } 13 | 14 | impl ClosestNodes { 15 | /// Create a new instance of [ClosestNodes]. 16 | pub fn new(target: Id) -> Self { 17 | Self { 18 | target, 19 | nodes: Vec::with_capacity(200), 20 | } 21 | } 22 | 23 | // === Getters === 24 | 25 | /// Returns the target of the query for these closest nodes. 26 | pub fn target(&self) -> Id { 27 | self.target 28 | } 29 | 30 | /// Returns a slice of the nodes array. 31 | pub fn nodes(&self) -> &[Node] { 32 | &self.nodes 33 | } 34 | 35 | /// Returns the number of nodes. 36 | pub fn len(&self) -> usize { 37 | self.nodes.len() 38 | } 39 | 40 | /// Returns true if there are no nodes. 41 | pub fn is_empty(&self) -> bool { 42 | self.nodes.is_empty() 43 | } 44 | 45 | // === Public Methods === 46 | 47 | /// Add a node. 48 | pub fn add(&mut self, node: Node) { 49 | let seek = node.id().xor(&self.target); 50 | 51 | if node.already_exists(&self.nodes) { 52 | return; 53 | } 54 | 55 | if let Err(pos) = self.nodes.binary_search_by(|prope| { 56 | if prope.is_secure() && !node.is_secure() { 57 | std::cmp::Ordering::Less 58 | } else if !prope.is_secure() && node.is_secure() { 59 | std::cmp::Ordering::Greater 60 | } else if prope.id() == node.id() { 61 | std::cmp::Ordering::Equal 62 | } else { 63 | prope.id().xor(&self.target).cmp(&seek) 64 | } 65 | }) { 66 | self.nodes.insert(pos, node) 67 | } 68 | } 69 | 70 | /// Take enough nodes closest to the target, until the following are satisfied: 71 | /// 1. At least the closest `k` nodes (20). 72 | /// 2. The last node should be at a distance `edk` which is the expected distance of the 20th 73 | /// node given previous estimations of the DHT size. 74 | /// 3. The number of subnets with unique 6 bits prefix in nodes ipv4 addresses match or exceeds 75 | /// the average from previous queries. 76 | /// 77 | /// If one or more of these conditions are not met, then we just take all responding nodes 78 | /// and store data at them. 79 | pub fn take_until_secure( 80 | &self, 81 | previous_dht_size_estimate: usize, 82 | average_subnets: usize, 83 | ) -> &[Node] { 84 | let mut until_secure = 0; 85 | 86 | // 20 / dht_size_estimate == expected_dk / ID space 87 | // so expected_dk = 20 * ID space / dht_size_estimate 88 | let expected_dk = 89 | (20.0 * u128::MAX as f64 / (previous_dht_size_estimate as f64 + 1.0)) as u128; 90 | 91 | let mut subnets = HashSet::new(); 92 | 93 | for node in &self.nodes { 94 | let distance = distance(&self.target, node); 95 | 96 | subnets.insert(subnet(node)); 97 | 98 | if distance >= expected_dk && subnets.len() >= average_subnets { 99 | break; 100 | } 101 | 102 | until_secure += 1; 103 | } 104 | 105 | &self.nodes[0..until_secure.max(MAX_BUCKET_SIZE_K).min(self.nodes().len())] 106 | } 107 | 108 | /// Count the number of subnets with unique 6 bits prefix in ipv4 109 | pub fn subnets_count(&self) -> u8 { 110 | if self.nodes.is_empty() { 111 | return 20; 112 | } 113 | 114 | let mut subnets = HashSet::new(); 115 | 116 | for node in self.nodes.iter().take(MAX_BUCKET_SIZE_K) { 117 | subnets.insert(subnet(node)); 118 | } 119 | 120 | subnets.len() as u8 121 | } 122 | 123 | /// An estimation of the Dht from the distribution of closest nodes 124 | /// responding to a query. 125 | /// 126 | /// [Read more](https://github.com/pubky/mainline/blob/main/docs/dht_size_estimate.md) 127 | pub fn dht_size_estimate(&self) -> f64 { 128 | dht_size_estimate( 129 | self.nodes 130 | .iter() 131 | .take(MAX_BUCKET_SIZE_K) 132 | .map(|node| distance(&self.target, node)), 133 | ) 134 | } 135 | } 136 | 137 | fn subnet(node: &Node) -> u8 { 138 | ((node.address().ip().to_bits() >> 26) & 0b0011_1111) as u8 139 | } 140 | 141 | fn distance(target: &Id, node: &Node) -> u128 { 142 | let xor = node.id().xor(target); 143 | 144 | // Round up the lower 4 bytes to get a u128 from u160. 145 | u128::from_be_bytes(xor.as_bytes()[0..16].try_into().expect("infallible")) 146 | } 147 | 148 | fn dht_size_estimate(distances: I) -> f64 149 | where 150 | I: IntoIterator, 151 | { 152 | let mut sum = 0.0; 153 | let mut count = 0; 154 | 155 | // Ignoring the first node, as that gives the best result in simulations. 156 | for distance in distances { 157 | count += 1; 158 | 159 | sum += count as f64 * distance as f64; 160 | } 161 | 162 | if count == 0 { 163 | return 0.0; 164 | } 165 | 166 | let lsq_constant = (count * (count + 1) * (2 * count + 1) / 6) as f64; 167 | 168 | lsq_constant * u128::MAX as f64 / sum 169 | } 170 | 171 | #[cfg(test)] 172 | mod tests { 173 | use std::{collections::BTreeMap, net::SocketAddrV4, str::FromStr, sync::Arc, time::Instant}; 174 | 175 | use crate::common::NodeInner; 176 | 177 | use super::*; 178 | 179 | #[test] 180 | fn add_sorted_by_id() { 181 | let target = Id::random(); 182 | 183 | let mut closest_nodes = ClosestNodes::new(target); 184 | 185 | for i in 0..100 { 186 | let node = Node::unique(i); 187 | closest_nodes.add(node.clone()); 188 | closest_nodes.add(node); 189 | } 190 | 191 | assert_eq!(closest_nodes.nodes().len(), 100); 192 | 193 | let distances = closest_nodes 194 | .nodes() 195 | .iter() 196 | .map(|n| n.id().distance(&target)) 197 | .collect::>(); 198 | 199 | let mut sorted = distances.clone(); 200 | sorted.sort(); 201 | 202 | assert_eq!(sorted, distances); 203 | } 204 | 205 | #[test] 206 | fn order_by_secure_id() { 207 | let unsecure = Node::random(); 208 | let secure = Node(Arc::new(NodeInner { 209 | id: Id::from_str("5a3ce9c14e7a08645677bbd1cfe7d8f956d53256").unwrap(), 210 | address: SocketAddrV4::new([21, 75, 31, 124].into(), 0), 211 | token: None, 212 | last_seen: Instant::now(), 213 | })); 214 | 215 | let mut closest_nodes = ClosestNodes::new(*unsecure.id()); 216 | 217 | closest_nodes.add(unsecure.clone()); 218 | closest_nodes.add(secure.clone()); 219 | 220 | assert_eq!(closest_nodes.nodes(), vec![secure, unsecure]) 221 | } 222 | 223 | #[test] 224 | fn take_until_expected_distance_to_20th_node() { 225 | let target = Id::random(); 226 | let dht_size_estimate = 200; 227 | 228 | let mut closest_nodes = ClosestNodes::new(target); 229 | 230 | let target_bytes = target.as_bytes(); 231 | 232 | for i in 0..dht_size_estimate { 233 | let node = Node::unique(i); 234 | closest_nodes.add(node); 235 | } 236 | 237 | let mut sybil = ClosestNodes::new(target); 238 | 239 | for _ in 0..20 { 240 | let mut bytes = target_bytes.to_vec(); 241 | bytes[18..].copy_from_slice(&Id::random().as_bytes()[18..]); 242 | let node = Node::new(Id::random(), SocketAddrV4::new(0.into(), 0)); 243 | 244 | sybil.add(node.clone()); 245 | closest_nodes.add(node); 246 | } 247 | 248 | let closest = closest_nodes.take_until_secure(dht_size_estimate, 0); 249 | 250 | assert!((closest.len() - sybil.nodes().len()) > 10); 251 | } 252 | 253 | #[test] 254 | fn simulation() { 255 | let lookups = 4; 256 | let acceptable_margin = 0.2; 257 | let sims = 10; 258 | let dht_size = 2500_f64; 259 | 260 | let mean = (0..sims) 261 | .map(|_| simulate(dht_size as usize, lookups) as f64) 262 | .sum::() 263 | / (sims as f64); 264 | 265 | let margin = (mean - dht_size).abs() / dht_size; 266 | 267 | assert!(margin <= acceptable_margin); 268 | } 269 | 270 | fn simulate(dht_size: usize, lookups: usize) -> usize { 271 | let mut nodes = BTreeMap::new(); 272 | for i in 0..dht_size { 273 | let node = Node::unique(i); 274 | nodes.insert(*node.id(), node); 275 | } 276 | 277 | (0..lookups) 278 | .map(|_| { 279 | let target = Id::random(); 280 | 281 | let mut closest_nodes = ClosestNodes::new(target); 282 | 283 | for (_, node) in nodes.range(target..).take(100) { 284 | closest_nodes.add(node.clone().into()) 285 | } 286 | for (_, node) in nodes.range(..target).rev().take(100) { 287 | closest_nodes.add(node.clone().into()) 288 | } 289 | 290 | let estimate = closest_nodes.dht_size_estimate(); 291 | 292 | estimate as usize 293 | }) 294 | .sum::() 295 | / lookups 296 | } 297 | } 298 | -------------------------------------------------------------------------------- /src/rpc/config.rs: -------------------------------------------------------------------------------- 1 | use std::{ 2 | net::{Ipv4Addr, SocketAddrV4}, 3 | time::Duration, 4 | }; 5 | 6 | use super::{ServerSettings, DEFAULT_REQUEST_TIMEOUT}; 7 | 8 | #[derive(Debug, Clone)] 9 | /// Dht Configurations 10 | pub struct Config { 11 | /// Bootstrap nodes 12 | /// 13 | /// Defaults to [super::DEFAULT_BOOTSTRAP_NODES] 14 | pub bootstrap: Option>, 15 | /// Explicit port to listen on. 16 | /// 17 | /// Defaults to None 18 | pub port: Option, 19 | /// UDP socket request timeout duration. 20 | /// 21 | /// The longer this duration is, the longer queries take until they are deemeed "done". 22 | /// The shortet this duration is, the more responses from busy nodes we miss out on, 23 | /// which affects the accuracy of queries trying to find closest nodes to a target. 24 | /// 25 | /// Defaults to [DEFAULT_REQUEST_TIMEOUT] 26 | pub request_timeout: Duration, 27 | /// Server to respond to incoming Requests 28 | pub server_settings: ServerSettings, 29 | /// Whether or not to start in server mode from the get go. 30 | /// 31 | /// Defaults to false where it will run in [Adaptive mode](https://github.com/pubky/mainline?tab=readme-ov-file#adaptive-mode). 32 | pub server_mode: bool, 33 | /// A known public IPv4 address for this node to generate 34 | /// a secure node Id from according to [BEP_0042](https://www.bittorrent.org/beps/bep_0042.html) 35 | /// 36 | /// Defaults to None, where we depend on suggestions from responding nodes. 37 | pub public_ip: Option, 38 | } 39 | 40 | impl Default for Config { 41 | fn default() -> Self { 42 | Self { 43 | bootstrap: None, 44 | port: None, 45 | request_timeout: DEFAULT_REQUEST_TIMEOUT, 46 | server_settings: Default::default(), 47 | server_mode: false, 48 | public_ip: None, 49 | } 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/rpc/info.rs: -------------------------------------------------------------------------------- 1 | use std::net::SocketAddrV4; 2 | 3 | use crate::Id; 4 | 5 | use super::Rpc; 6 | 7 | /// Information and statistics about this mainline node. 8 | #[derive(Debug, Clone)] 9 | pub struct Info { 10 | id: Id, 11 | local_addr: SocketAddrV4, 12 | public_address: Option, 13 | firewalled: bool, 14 | dht_size_estimate: (usize, f64), 15 | server_mode: bool, 16 | } 17 | 18 | impl Info { 19 | /// This Node's [Id] 20 | pub fn id(&self) -> &Id { 21 | &self.id 22 | } 23 | /// Local UDP Ipv4 socket address that this node is listening on. 24 | pub fn local_addr(&self) -> SocketAddrV4 { 25 | self.local_addr 26 | } 27 | /// Returns the best guess for this node's Public address. 28 | /// 29 | /// If [crate::DhtBuilder::public_ip] was set, this is what will be returned 30 | /// (plus the local port), otherwise it will rely on consensus from 31 | /// responding nodes voting on our public IP and port. 32 | pub fn public_address(&self) -> Option { 33 | self.public_address 34 | } 35 | /// Returns `true` if we can't confirm that [Self::public_address] is publicly addressable. 36 | /// 37 | /// If this node is firewalled, it won't switch to server mode if it is in adaptive mode, 38 | /// but if [crate::DhtBuilder::server_mode] was set to true, then whether or not this node is firewalled 39 | /// won't matter. 40 | pub fn firewalled(&self) -> bool { 41 | self.firewalled 42 | } 43 | 44 | /// Returns whether or not this node is running in server mode. 45 | pub fn server_mode(&self) -> bool { 46 | self.server_mode 47 | } 48 | 49 | /// Returns: 50 | /// 1. Normal Dht size estimate based on all closer `nodes` in query responses. 51 | /// 2. Standard deviaiton as a function of the number of samples used in this estimate. 52 | /// 53 | /// [Read more](https://github.com/pubky/mainline/blob/main/docs/dht_size_estimate.md) 54 | pub fn dht_size_estimate(&self) -> (usize, f64) { 55 | self.dht_size_estimate 56 | } 57 | } 58 | 59 | impl From<&Rpc> for Info { 60 | fn from(rpc: &Rpc) -> Self { 61 | Self { 62 | id: *rpc.id(), 63 | local_addr: rpc.local_addr(), 64 | dht_size_estimate: rpc.dht_size_estimate(), 65 | public_address: rpc.public_address(), 66 | firewalled: rpc.firewalled(), 67 | server_mode: rpc.server_mode(), 68 | } 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /src/rpc/iterative_query.rs: -------------------------------------------------------------------------------- 1 | //! Manage iterative queries and their corresponding request/response. 2 | 3 | use std::collections::HashMap; 4 | use std::collections::HashSet; 5 | use std::net::SocketAddrV4; 6 | 7 | use tracing::{debug, trace}; 8 | 9 | use super::{socket::KrpcSocket, ClosestNodes}; 10 | use crate::common::{FindNodeRequestArguments, GetPeersRequestArguments, GetValueRequestArguments}; 11 | use crate::{ 12 | common::{Id, Node, RequestSpecific, RequestTypeSpecific, MAX_BUCKET_SIZE_K}, 13 | rpc::Response, 14 | }; 15 | 16 | /// An iterative process of concurrently sending a request to the closest known nodes to 17 | /// the target, updating the routing table with closer nodes discovered in the responses, and 18 | /// repeating this process until no closer nodes (that aren't already queried) are found. 19 | #[derive(Debug)] 20 | pub(crate) struct IterativeQuery { 21 | pub request: RequestSpecific, 22 | closest: ClosestNodes, 23 | responders: ClosestNodes, 24 | inflight_requests: Vec, 25 | visited: HashSet, 26 | responses: Vec, 27 | public_address_votes: HashMap, 28 | } 29 | 30 | #[derive(Debug)] 31 | pub enum GetRequestSpecific { 32 | FindNode(FindNodeRequestArguments), 33 | GetPeers(GetPeersRequestArguments), 34 | GetValue(GetValueRequestArguments), 35 | } 36 | 37 | impl GetRequestSpecific { 38 | pub fn target(&self) -> &Id { 39 | match self { 40 | GetRequestSpecific::FindNode(args) => &args.target, 41 | GetRequestSpecific::GetPeers(args) => &args.info_hash, 42 | GetRequestSpecific::GetValue(args) => &args.target, 43 | } 44 | } 45 | } 46 | 47 | impl IterativeQuery { 48 | pub fn new(requester_id: Id, target: Id, request: GetRequestSpecific) -> Self { 49 | let request_type = match request { 50 | GetRequestSpecific::FindNode(s) => RequestTypeSpecific::FindNode(s), 51 | GetRequestSpecific::GetPeers(s) => RequestTypeSpecific::GetPeers(s), 52 | GetRequestSpecific::GetValue(s) => RequestTypeSpecific::GetValue(s), 53 | }; 54 | 55 | trace!(?target, ?request_type, "New Query"); 56 | 57 | Self { 58 | request: RequestSpecific { 59 | requester_id, 60 | request_type, 61 | }, 62 | 63 | closest: ClosestNodes::new(target), 64 | responders: ClosestNodes::new(target), 65 | 66 | inflight_requests: Vec::new(), 67 | visited: HashSet::new(), 68 | 69 | responses: Vec::new(), 70 | 71 | public_address_votes: HashMap::new(), 72 | } 73 | } 74 | 75 | // === Getters === 76 | 77 | pub fn target(&self) -> Id { 78 | self.responders.target() 79 | } 80 | 81 | /// Closest nodes according to other nodes. 82 | pub fn closest(&self) -> &ClosestNodes { 83 | &self.closest 84 | } 85 | 86 | /// Return the closest responding nodes after the query is done. 87 | pub fn responders(&self) -> &ClosestNodes { 88 | &self.responders 89 | } 90 | 91 | pub fn responses(&self) -> &[Response] { 92 | &self.responses 93 | } 94 | 95 | pub fn best_address(&self) -> Option { 96 | let mut max = 0_u16; 97 | let mut best_addr = None; 98 | 99 | for (addr, count) in self.public_address_votes.iter() { 100 | if *count > max { 101 | max = *count; 102 | best_addr = Some(*addr); 103 | }; 104 | } 105 | 106 | best_addr 107 | } 108 | 109 | // === Public Methods === 110 | 111 | /// Force start query traversal by visiting closest nodes. 112 | pub fn start(&mut self, socket: &mut KrpcSocket) { 113 | self.visit_closest(socket); 114 | } 115 | 116 | /// Add a candidate node to query on next tick if it is among the closest nodes. 117 | pub fn add_candidate(&mut self, node: Node) { 118 | // ready for a ipv6 routing table? 119 | self.closest.add(node); 120 | } 121 | 122 | /// Add a vote for this node's address. 123 | pub fn add_address_vote(&mut self, address: SocketAddrV4) { 124 | self.public_address_votes 125 | .entry(address) 126 | .and_modify(|counter| *counter += 1) 127 | .or_insert(1); 128 | } 129 | 130 | /// Visit explicitly given addresses, and add them to the visited set. 131 | /// only used from the Rpc when calling bootstrapping nodes. 132 | pub fn visit(&mut self, socket: &mut KrpcSocket, address: SocketAddrV4) { 133 | let tid = socket.request(address, self.request.clone()); 134 | self.inflight_requests.push(tid); 135 | 136 | let tid = socket.request( 137 | address, 138 | RequestSpecific { 139 | requester_id: Id::random(), 140 | request_type: RequestTypeSpecific::Ping, 141 | }, 142 | ); 143 | self.inflight_requests.push(tid); 144 | 145 | self.visited.insert(address); 146 | } 147 | 148 | /// Return true if a response (by transaction_id) is expected by this query. 149 | pub fn inflight(&self, tid: u16) -> bool { 150 | self.inflight_requests.contains(&tid) 151 | } 152 | 153 | /// Add a node that responded with a token as a probable storage node. 154 | pub fn add_responding_node(&mut self, node: Node) { 155 | self.responders.add(node) 156 | } 157 | 158 | /// Store received response. 159 | pub fn response(&mut self, from: SocketAddrV4, response: Response) { 160 | let target = self.target(); 161 | 162 | debug!(?target, ?response, ?from, "Query got response"); 163 | 164 | self.responses.push(response.to_owned()); 165 | } 166 | 167 | /// Query closest nodes for this query's target and message. 168 | /// 169 | /// Returns true if it is done. 170 | pub fn tick(&mut self, socket: &mut KrpcSocket) -> bool { 171 | // Visit closest nodes 172 | self.visit_closest(socket); 173 | 174 | // If no more inflight_requests are inflight in the socket (not timed out), 175 | // then the query is done. 176 | let done = !self 177 | .inflight_requests 178 | .iter() 179 | .any(|&tid| socket.inflight(&tid)); 180 | 181 | if done { 182 | debug!(id=?self.target(), closest = ?self.closest.len(), visited = ?self.visited.len(), responders = ?self.responders.len(), "Done query"); 183 | }; 184 | 185 | done 186 | } 187 | 188 | // === Private Methods === 189 | 190 | /// Visit the closest candidates and remove them as candidates 191 | fn visit_closest(&mut self, socket: &mut KrpcSocket) { 192 | let to_visit = self 193 | .closest 194 | .nodes() 195 | .iter() 196 | .take(MAX_BUCKET_SIZE_K) 197 | .filter(|node| !self.visited.contains(&node.address())) 198 | .map(|node| node.address()) 199 | .collect::>(); 200 | 201 | for address in to_visit { 202 | self.visit(socket, address); 203 | } 204 | } 205 | } 206 | -------------------------------------------------------------------------------- /src/rpc/put_query.rs: -------------------------------------------------------------------------------- 1 | use tracing::{debug, trace}; 2 | 3 | use crate::{ 4 | common::{ 5 | ErrorSpecific, Id, PutRequest, PutRequestSpecific, RequestSpecific, RequestTypeSpecific, 6 | }, 7 | Node, 8 | }; 9 | 10 | use super::socket::KrpcSocket; 11 | 12 | #[derive(Debug)] 13 | /// Once an [super::IterativeQuery] is done, or if a previous cached one was a vailable, 14 | /// we can store data at the closest nodes using this PutQuery, that keeps track of 15 | /// acknowledging nodes, and or errors. 16 | pub struct PutQuery { 17 | pub target: Id, 18 | /// Nodes that confirmed success 19 | stored_at: u8, 20 | inflight_requests: Vec, 21 | pub request: PutRequestSpecific, 22 | errors: Vec<(u8, ErrorSpecific)>, 23 | extra_nodes: Box<[Node]>, 24 | } 25 | 26 | impl PutQuery { 27 | pub fn new(target: Id, request: PutRequestSpecific, extra_nodes: Option>) -> Self { 28 | Self { 29 | target, 30 | stored_at: 0, 31 | inflight_requests: Vec::new(), 32 | request, 33 | errors: Vec::new(), 34 | extra_nodes: extra_nodes.unwrap_or(Box::new([])), 35 | } 36 | } 37 | 38 | pub fn start( 39 | &mut self, 40 | socket: &mut KrpcSocket, 41 | closest_nodes: &[Node], 42 | ) -> Result<(), PutError> { 43 | if self.started() { 44 | panic!("should not call PutQuery::start() twice"); 45 | }; 46 | 47 | let target = self.target; 48 | trace!(?target, "PutQuery start"); 49 | 50 | if closest_nodes.is_empty() { 51 | Err(PutQueryError::NoClosestNodes)?; 52 | } 53 | 54 | if closest_nodes.len() > u8::MAX as usize { 55 | panic!("should not send PUT query to more than 256 nodes") 56 | } 57 | 58 | for node in closest_nodes.iter().chain(self.extra_nodes.iter()) { 59 | // Set correct values to the request placeholders 60 | if let Some(token) = node.token() { 61 | let tid = socket.request( 62 | node.address(), 63 | RequestSpecific { 64 | requester_id: Id::random(), 65 | request_type: RequestTypeSpecific::Put(PutRequest { 66 | token, 67 | put_request_type: self.request.clone(), 68 | }), 69 | }, 70 | ); 71 | 72 | self.inflight_requests.push(tid); 73 | } 74 | } 75 | 76 | Ok(()) 77 | } 78 | 79 | pub fn started(&self) -> bool { 80 | !self.inflight_requests.is_empty() 81 | } 82 | 83 | pub fn inflight(&self, tid: u16) -> bool { 84 | self.inflight_requests.contains(&tid) 85 | } 86 | 87 | pub fn success(&mut self) { 88 | debug!(target = ?self.target, "PutQuery got success response"); 89 | self.stored_at += 1 90 | } 91 | 92 | pub fn error(&mut self, error: ErrorSpecific) { 93 | debug!(target = ?self.target, ?error, "PutQuery got error"); 94 | 95 | if let Some(pos) = self 96 | .errors 97 | .iter() 98 | .position(|(_, err)| error.code == err.code) 99 | { 100 | // Increment the count of the existing error 101 | self.errors[pos].0 += 1; 102 | 103 | // Move the updated element to maintain the order (highest count first) 104 | let mut i = pos; 105 | while i > 0 && self.errors[i].0 > self.errors[i - 1].0 { 106 | self.errors.swap(i, i - 1); 107 | i -= 1; 108 | } 109 | } else { 110 | // Add the new error with a count of 1 111 | self.errors.push((1, error)); 112 | } 113 | } 114 | 115 | /// Check if the query is done, and if so send the query target to the receiver if any. 116 | pub fn tick(&mut self, socket: &KrpcSocket) -> Result { 117 | // Didn't start yet. 118 | if self.inflight_requests.is_empty() { 119 | return Ok(false); 120 | } 121 | 122 | // And all queries got responses or timedout 123 | if self.is_done(socket) { 124 | let target = self.target; 125 | 126 | if self.stored_at == 0 { 127 | let most_common_error = self.most_common_error(); 128 | 129 | debug!( 130 | ?target, 131 | ?most_common_error, 132 | nodes_count = self.inflight_requests.len(), 133 | "Put Query: failed" 134 | ); 135 | 136 | return Err(most_common_error 137 | .map(|(_, error)| error) 138 | .unwrap_or(PutQueryError::Timeout.into())); 139 | } 140 | 141 | debug!(?target, stored_at = ?self.stored_at, "PutQuery Done successfully"); 142 | 143 | return Ok(true); 144 | } else if let Some(most_common_error) = self.majority_nodes_rejected_put_mutable() { 145 | let target = self.target; 146 | 147 | debug!( 148 | ?target, 149 | ?most_common_error, 150 | nodes_count = self.inflight_requests.len(), 151 | "PutQuery for MutableItem was rejected by most nodes with 3xx code." 152 | ); 153 | 154 | return Err(most_common_error)?; 155 | } 156 | 157 | Ok(false) 158 | } 159 | 160 | fn is_done(&self, socket: &KrpcSocket) -> bool { 161 | !self 162 | .inflight_requests 163 | .iter() 164 | .any(|&tid| socket.inflight(&tid)) 165 | } 166 | 167 | fn majority_nodes_rejected_put_mutable(&self) -> Option { 168 | let half = ((self.inflight_requests.len() / 2) + 1) as u8; 169 | 170 | if matches!(self.request, PutRequestSpecific::PutMutable(_)) { 171 | return self.most_common_error().and_then(|(count, error)| { 172 | if count >= half { 173 | if let PutError::Concurrency(err) = error { 174 | Some(err) 175 | } else { 176 | None 177 | } 178 | } else { 179 | None 180 | } 181 | }); 182 | }; 183 | 184 | None 185 | } 186 | 187 | fn most_common_error(&self) -> Option<(u8, PutError)> { 188 | self.errors 189 | .first() 190 | .and_then(|(count, error)| match error.code { 191 | 301 => Some((*count, PutError::from(ConcurrencyError::CasFailed))), 192 | 302 => Some((*count, PutError::from(ConcurrencyError::NotMostRecent))), 193 | _ => None, 194 | }) 195 | } 196 | } 197 | 198 | #[derive(thiserror::Error, Debug, Clone)] 199 | /// PutQuery errors 200 | pub enum PutError { 201 | /// Common PutQuery errors 202 | #[error(transparent)] 203 | Query(#[from] PutQueryError), 204 | 205 | #[error(transparent)] 206 | /// PutQuery for [crate::MutableItem] errors 207 | Concurrency(#[from] ConcurrencyError), 208 | } 209 | 210 | #[derive(thiserror::Error, Debug, Clone)] 211 | /// Common PutQuery errors 212 | pub enum PutQueryError { 213 | /// Failed to find any nodes close, usually means dht node failed to bootstrap, 214 | /// so the routing table is empty. Check the machine's access to UDP socket, 215 | /// or find better bootstrapping nodes. 216 | #[error("Failed to find any nodes close to store value at")] 217 | NoClosestNodes, 218 | 219 | /// Either Put Query faild to store at any nodes, and most nodes responded 220 | /// with a non `301` nor `302` errors. 221 | /// 222 | /// Either way; contains the most common error response. 223 | #[error("Query Error Response")] 224 | ErrorResponse(ErrorSpecific), 225 | 226 | /// PutQuery timed out with no responses neither success or errors 227 | #[error("PutQuery timed out with no responses neither success or errors")] 228 | Timeout, 229 | } 230 | 231 | #[derive(thiserror::Error, Debug, Clone)] 232 | /// PutQuery for [crate::MutableItem] errors 233 | pub enum ConcurrencyError { 234 | /// Trying to PUT mutable items with the same `key`, and `salt` but different `seq`. 235 | /// 236 | /// Moreover, the more recent item does _NOT_ mention the the earlier 237 | /// item's `seq` in its `cas` field. 238 | /// 239 | /// This risks a [Lost Update Problem](https://en.wikipedia.org/wiki/Write-write_conflict). 240 | /// 241 | /// Try reading most recent mutable item before writing again, 242 | /// and make sure to set the `cas` field. 243 | #[error("Conflict risk, try reading most recent item before writing again.")] 244 | ConflictRisk, 245 | 246 | /// The [crate::MutableItem::seq] is less than or equal the sequence from another signed item. 247 | /// 248 | /// Try reading most recent mutable item before writing again. 249 | #[error("MutableItem::seq is not the most recent, try reading most recent item before writing again.")] 250 | NotMostRecent, 251 | 252 | /// The `CAS` condition does not match the `seq` of the most recent knonw signed item. 253 | #[error("CAS check failed, try reading most recent item before writing again.")] 254 | CasFailed, 255 | } 256 | -------------------------------------------------------------------------------- /src/rpc/server.rs: -------------------------------------------------------------------------------- 1 | //! Modules needed only for nodes running in server mode (not read-only). 2 | 3 | pub mod peers; 4 | pub mod tokens; 5 | 6 | use std::{fmt::Debug, net::SocketAddrV4, num::NonZeroUsize}; 7 | 8 | use dyn_clone::DynClone; 9 | use lru::LruCache; 10 | use tracing::debug; 11 | 12 | use crate::common::{ 13 | validate_immutable, AnnouncePeerRequestArguments, ErrorSpecific, FindNodeRequestArguments, 14 | FindNodeResponseArguments, GetImmutableResponseArguments, GetMutableResponseArguments, 15 | GetPeersRequestArguments, GetPeersResponseArguments, GetValueRequestArguments, Id, MutableItem, 16 | NoMoreRecentValueResponseArguments, NoValuesResponseArguments, PingResponseArguments, 17 | PutImmutableRequestArguments, PutMutableRequestArguments, PutRequest, PutRequestSpecific, 18 | RequestTypeSpecific, ResponseSpecific, RoutingTable, 19 | }; 20 | 21 | use peers::PeersStore; 22 | use tokens::Tokens; 23 | 24 | pub use crate::common::{MessageType, RequestSpecific}; 25 | 26 | /// Default maximum number of info_hashes for which to store peers. 27 | pub const MAX_INFO_HASHES: usize = 2000; 28 | /// Default maximum number of peers to store per info_hash. 29 | pub const MAX_PEERS: usize = 500; 30 | /// Default maximum number of Immutable and Mutable items to store. 31 | pub const MAX_VALUES: usize = 1000; 32 | 33 | /// A trait for filtering incoming requests to a DHT node and 34 | /// decide whether to allow handling it or rate limit or ban 35 | /// the requester, or prohibit specific requests' details. 36 | pub trait RequestFilter: Send + Sync + Debug + DynClone { 37 | /// Returns true if the request from this source is allowed. 38 | fn allow_request(&self, request: &RequestSpecific, from: SocketAddrV4) -> bool; 39 | } 40 | 41 | dyn_clone::clone_trait_object!(RequestFilter); 42 | 43 | #[derive(Debug, Clone)] 44 | struct DefaultFilter; 45 | 46 | impl RequestFilter for DefaultFilter { 47 | fn allow_request(&self, _request: &RequestSpecific, _from: SocketAddrV4) -> bool { 48 | true 49 | } 50 | } 51 | 52 | #[derive(Debug)] 53 | /// A server that handles incoming requests. 54 | /// 55 | /// Supports [BEP_005](https://www.bittorrent.org/beps/bep_0005.html) and [BEP_0044](https://www.bittorrent.org/beps/bep_0044.html). 56 | /// 57 | /// But it doesn't implement any rate-limiting or blocking. 58 | pub struct Server { 59 | /// Tokens generator 60 | tokens: Tokens, 61 | /// Peers store 62 | peers: PeersStore, 63 | /// Immutable values store 64 | immutable_values: LruCache>, 65 | /// Mutable values store 66 | mutable_values: LruCache, 67 | /// Filter requests before handling them. 68 | filter: Box, 69 | } 70 | 71 | impl Default for Server { 72 | fn default() -> Self { 73 | Self::new(ServerSettings::default()) 74 | } 75 | } 76 | 77 | #[derive(Debug, Clone)] 78 | /// Settings for the default dht server. 79 | pub struct ServerSettings { 80 | /// The maximum info_hashes for which to store peers. 81 | /// 82 | /// Defaults to [MAX_INFO_HASHES] 83 | pub max_info_hashes: usize, 84 | /// The maximum peers to store per info_hash. 85 | /// 86 | /// Defaults to [MAX_PEERS] 87 | pub max_peers_per_info_hash: usize, 88 | /// Maximum number of immutable values to store. 89 | /// 90 | /// Defaults to [MAX_VALUES] 91 | pub max_immutable_values: usize, 92 | /// Maximum number of mutable values to store. 93 | /// 94 | /// Defaults to [MAX_VALUES] 95 | pub max_mutable_values: usize, 96 | /// Filter requests before handling them. 97 | /// 98 | /// Defaults to a function that always returns true. 99 | pub filter: Box, 100 | } 101 | 102 | impl Default for ServerSettings { 103 | fn default() -> Self { 104 | Self { 105 | max_info_hashes: MAX_INFO_HASHES, 106 | max_peers_per_info_hash: MAX_PEERS, 107 | max_mutable_values: MAX_VALUES, 108 | max_immutable_values: MAX_VALUES, 109 | 110 | filter: Box::new(DefaultFilter), 111 | } 112 | } 113 | } 114 | 115 | impl Server { 116 | /// Creates a new [Server] 117 | pub fn new(settings: ServerSettings) -> Self { 118 | let tokens = Tokens::new(); 119 | 120 | Self { 121 | tokens, 122 | peers: PeersStore::new( 123 | NonZeroUsize::new(settings.max_info_hashes).unwrap_or( 124 | NonZeroUsize::new(MAX_INFO_HASHES).expect("MAX_PEERS is NonZeroUsize"), 125 | ), 126 | NonZeroUsize::new(settings.max_peers_per_info_hash) 127 | .unwrap_or(NonZeroUsize::new(MAX_PEERS).expect("MAX_PEERS is NonZeroUsize")), 128 | ), 129 | 130 | immutable_values: LruCache::new( 131 | NonZeroUsize::new(settings.max_immutable_values) 132 | .unwrap_or(NonZeroUsize::new(MAX_VALUES).expect("MAX_VALUES is NonZeroUsize")), 133 | ), 134 | mutable_values: LruCache::new( 135 | NonZeroUsize::new(settings.max_mutable_values) 136 | .unwrap_or(NonZeroUsize::new(MAX_VALUES).expect("MAX_VALUES is NonZeroUsize")), 137 | ), 138 | filter: settings.filter, 139 | } 140 | } 141 | 142 | /// Returns an optional response or an error for a request. 143 | /// 144 | /// Passed to the Rpc to send back to the requester. 145 | pub fn handle_request( 146 | &mut self, 147 | routing_table: &RoutingTable, 148 | from: SocketAddrV4, 149 | request: RequestSpecific, 150 | ) -> Option { 151 | if !self.filter.allow_request(&request, from) { 152 | return None; 153 | } 154 | 155 | // Lazily rotate secrets before handling a request 156 | if self.tokens.should_update() { 157 | self.tokens.rotate() 158 | } 159 | 160 | let requester_id = request.requester_id; 161 | 162 | Some(match request.request_type { 163 | RequestTypeSpecific::Ping => { 164 | MessageType::Response(ResponseSpecific::Ping(PingResponseArguments { 165 | responder_id: *routing_table.id(), 166 | })) 167 | } 168 | RequestTypeSpecific::FindNode(FindNodeRequestArguments { target, .. }) => { 169 | MessageType::Response(ResponseSpecific::FindNode(FindNodeResponseArguments { 170 | responder_id: *routing_table.id(), 171 | nodes: routing_table.closest(target), 172 | })) 173 | } 174 | RequestTypeSpecific::GetPeers(GetPeersRequestArguments { info_hash, .. }) => { 175 | MessageType::Response(match self.peers.get_random_peers(&info_hash) { 176 | Some(peers) => ResponseSpecific::GetPeers(GetPeersResponseArguments { 177 | responder_id: *routing_table.id(), 178 | token: self.tokens.generate_token(from).into(), 179 | nodes: Some(routing_table.closest(info_hash)), 180 | values: peers, 181 | }), 182 | None => ResponseSpecific::NoValues(NoValuesResponseArguments { 183 | responder_id: *routing_table.id(), 184 | token: self.tokens.generate_token(from).into(), 185 | nodes: Some(routing_table.closest(info_hash)), 186 | }), 187 | }) 188 | } 189 | RequestTypeSpecific::GetValue(GetValueRequestArguments { target, seq, .. }) => { 190 | if seq.is_some() { 191 | MessageType::Response(self.handle_get_mutable(routing_table, from, target, seq)) 192 | } else if let Some(v) = self.immutable_values.get(&target) { 193 | MessageType::Response(ResponseSpecific::GetImmutable( 194 | GetImmutableResponseArguments { 195 | responder_id: *routing_table.id(), 196 | token: self.tokens.generate_token(from).into(), 197 | nodes: Some(routing_table.closest(target)), 198 | v: v.clone(), 199 | }, 200 | )) 201 | } else { 202 | MessageType::Response(self.handle_get_mutable(routing_table, from, target, seq)) 203 | } 204 | } 205 | RequestTypeSpecific::Put(PutRequest { 206 | token, 207 | put_request_type, 208 | }) => match put_request_type { 209 | PutRequestSpecific::AnnouncePeer(AnnouncePeerRequestArguments { 210 | info_hash, 211 | port, 212 | implied_port, 213 | .. 214 | }) => { 215 | if !self.tokens.validate(from, &token) { 216 | debug!( 217 | ?info_hash, 218 | ?requester_id, 219 | ?from, 220 | request_type = "announce_peer", 221 | "Invalid token" 222 | ); 223 | 224 | return Some(MessageType::Error(ErrorSpecific { 225 | code: 203, 226 | description: "Bad token".to_string(), 227 | })); 228 | } 229 | 230 | let peer = match implied_port { 231 | Some(true) => from, 232 | _ => SocketAddrV4::new(*from.ip(), port), 233 | }; 234 | 235 | self.peers 236 | .add_peer(info_hash, (&request.requester_id, peer)); 237 | 238 | return Some(MessageType::Response(ResponseSpecific::Ping( 239 | PingResponseArguments { 240 | responder_id: *routing_table.id(), 241 | }, 242 | ))); 243 | } 244 | PutRequestSpecific::PutImmutable(PutImmutableRequestArguments { 245 | v, 246 | target, 247 | .. 248 | }) => { 249 | if !self.tokens.validate(from, &token) { 250 | debug!( 251 | ?target, 252 | ?requester_id, 253 | ?from, 254 | request_type = "put_immutable", 255 | "Invalid token" 256 | ); 257 | 258 | return Some(MessageType::Error(ErrorSpecific { 259 | code: 203, 260 | description: "Bad token".to_string(), 261 | })); 262 | } 263 | 264 | if v.len() > 1000 { 265 | debug!(?target, ?requester_id, ?from, size = ?v.len(), "Message (v field) too big."); 266 | 267 | return Some(MessageType::Error(ErrorSpecific { 268 | code: 205, 269 | description: "Message (v field) too big.".to_string(), 270 | })); 271 | } 272 | if !validate_immutable(&v, target) { 273 | debug!(?target, ?requester_id, ?from, v = ?v, "Target doesn't match the sha1 hash of v field."); 274 | 275 | return Some(MessageType::Error(ErrorSpecific { 276 | code: 203, 277 | description: "Target doesn't match the sha1 hash of v field" 278 | .to_string(), 279 | })); 280 | } 281 | 282 | self.immutable_values.put(target, v); 283 | 284 | return Some(MessageType::Response(ResponseSpecific::Ping( 285 | PingResponseArguments { 286 | responder_id: *routing_table.id(), 287 | }, 288 | ))); 289 | } 290 | PutRequestSpecific::PutMutable(PutMutableRequestArguments { 291 | target, 292 | v, 293 | k, 294 | seq, 295 | sig, 296 | salt, 297 | cas, 298 | .. 299 | }) => { 300 | if !self.tokens.validate(from, &token) { 301 | debug!( 302 | ?target, 303 | ?requester_id, 304 | ?from, 305 | request_type = "put_mutable", 306 | "Invalid token" 307 | ); 308 | return Some(MessageType::Error(ErrorSpecific { 309 | code: 203, 310 | description: "Bad token".to_string(), 311 | })); 312 | } 313 | if v.len() > 1000 { 314 | return Some(MessageType::Error(ErrorSpecific { 315 | code: 205, 316 | description: "Message (v field) too big.".to_string(), 317 | })); 318 | } 319 | if let Some(ref salt) = salt { 320 | if salt.len() > 64 { 321 | return Some(MessageType::Error(ErrorSpecific { 322 | code: 207, 323 | description: "salt (salt field) too big.".to_string(), 324 | })); 325 | } 326 | } 327 | if let Some(previous) = self.mutable_values.get(&target) { 328 | if let Some(cas) = cas { 329 | if previous.seq() != cas { 330 | debug!( 331 | ?target, 332 | ?requester_id, 333 | ?from, 334 | "CAS mismatched, re-read value and try again." 335 | ); 336 | 337 | return Some(MessageType::Error(ErrorSpecific { 338 | code: 301, 339 | description: "CAS mismatched, re-read value and try again." 340 | .to_string(), 341 | })); 342 | } 343 | }; 344 | 345 | if seq < previous.seq() { 346 | debug!( 347 | ?target, 348 | ?requester_id, 349 | ?from, 350 | "Sequence number less than current." 351 | ); 352 | 353 | return Some(MessageType::Error(ErrorSpecific { 354 | code: 302, 355 | description: "Sequence number less than current.".to_string(), 356 | })); 357 | } 358 | } 359 | 360 | match MutableItem::from_dht_message(target, &k, v, seq, &sig, salt) { 361 | Ok(item) => { 362 | self.mutable_values.put(target, item); 363 | 364 | MessageType::Response(ResponseSpecific::Ping(PingResponseArguments { 365 | responder_id: *routing_table.id(), 366 | })) 367 | } 368 | Err(error) => { 369 | debug!(?target, ?requester_id, ?from, ?error, "Invalid signature"); 370 | 371 | MessageType::Error(ErrorSpecific { 372 | code: 206, 373 | description: "Invalid signature".to_string(), 374 | }) 375 | } 376 | } 377 | } 378 | }, 379 | }) 380 | } 381 | 382 | /// Handle get mutable request 383 | fn handle_get_mutable( 384 | &mut self, 385 | routing_table: &RoutingTable, 386 | from: SocketAddrV4, 387 | target: Id, 388 | seq: Option, 389 | ) -> ResponseSpecific { 390 | match self.mutable_values.get(&target) { 391 | Some(item) => { 392 | let no_more_recent_values = seq.map(|request_seq| item.seq() <= request_seq); 393 | 394 | match no_more_recent_values { 395 | Some(true) => { 396 | ResponseSpecific::NoMoreRecentValue(NoMoreRecentValueResponseArguments { 397 | responder_id: *routing_table.id(), 398 | token: self.tokens.generate_token(from).into(), 399 | nodes: Some(routing_table.closest(target)), 400 | seq: item.seq(), 401 | }) 402 | } 403 | _ => ResponseSpecific::GetMutable(GetMutableResponseArguments { 404 | responder_id: *routing_table.id(), 405 | token: self.tokens.generate_token(from).into(), 406 | nodes: Some(routing_table.closest(target)), 407 | v: item.value().into(), 408 | k: *item.key(), 409 | seq: item.seq(), 410 | sig: *item.signature(), 411 | }), 412 | } 413 | } 414 | None => ResponseSpecific::NoValues(NoValuesResponseArguments { 415 | responder_id: *routing_table.id(), 416 | token: self.tokens.generate_token(from).into(), 417 | nodes: Some(routing_table.closest(target)), 418 | }), 419 | } 420 | } 421 | } 422 | -------------------------------------------------------------------------------- /src/rpc/server/peers.rs: -------------------------------------------------------------------------------- 1 | //! Manage announced peers for info_hashes 2 | 3 | use std::{net::SocketAddrV4, num::NonZeroUsize}; 4 | 5 | use crate::common::Id; 6 | 7 | use getrandom::getrandom; 8 | use lru::LruCache; 9 | 10 | const CHANCE_SCALE: f32 = 2.0 * (1u32 << 31) as f32; 11 | 12 | #[derive(Debug, Clone)] 13 | /// An LRU cache of "Peers" per info hashes. 14 | /// 15 | /// Read [BEP_0005](https://www.bittorrent.org/beps/bep_0005.html) for more information. 16 | pub struct PeersStore { 17 | info_hashes: LruCache>, 18 | max_peers: NonZeroUsize, 19 | } 20 | 21 | impl PeersStore { 22 | /// Create a new store of peers announced on info hashes. 23 | pub fn new(max_info_hashes: NonZeroUsize, max_peers: NonZeroUsize) -> Self { 24 | Self { 25 | info_hashes: LruCache::new(max_info_hashes), 26 | max_peers, 27 | } 28 | } 29 | 30 | /// Add a peer for an info hash. 31 | pub fn add_peer(&mut self, info_hash: Id, peer: (&Id, SocketAddrV4)) { 32 | if let Some(info_hash_lru) = self.info_hashes.get_mut(&info_hash) { 33 | info_hash_lru.put(*peer.0, peer.1); 34 | } else { 35 | let mut info_hash_lru = LruCache::new(self.max_peers); 36 | info_hash_lru.put(*peer.0, peer.1); 37 | self.info_hashes.put(info_hash, info_hash_lru); 38 | }; 39 | } 40 | 41 | /// Returns a random set of peers per an info hash. 42 | pub fn get_random_peers(&mut self, info_hash: &Id) -> Option> { 43 | if let Some(info_hash_lru) = self.info_hashes.get(info_hash) { 44 | let size = info_hash_lru.len(); 45 | let target_size = 20; 46 | 47 | if size == 0 { 48 | return None; 49 | } 50 | if size < target_size { 51 | return Some( 52 | info_hash_lru 53 | .iter() 54 | .map(|n| n.1.to_owned()) 55 | .collect::>(), 56 | ); 57 | } 58 | 59 | let mut results = Vec::with_capacity(20); 60 | 61 | let mut chunk = vec![0_u8; info_hash_lru.iter().len() * 4]; 62 | getrandom(chunk.as_mut_slice()).expect("getrandom"); 63 | 64 | for (index, (_, addr)) in info_hash_lru.iter().enumerate() { 65 | // Calculate the chance of adding the current item based on remaining items and slots 66 | let remaining_slots = target_size - results.len(); 67 | let remaining_items = info_hash_lru.len() - index; 68 | let current_chance = 69 | ((remaining_slots as f32 / remaining_items as f32) * CHANCE_SCALE) as u32; 70 | 71 | // Get random integer from the chunk 72 | let rand_int = 73 | u32::from_le_bytes(chunk[index..index + 4].try_into().expect("infallible")); 74 | 75 | // Randomly decide to add the item based on the current chance 76 | if rand_int < current_chance { 77 | results.push(*addr); 78 | if results.len() == target_size { 79 | break; 80 | } 81 | } 82 | } 83 | 84 | return Some(results); 85 | } 86 | 87 | None 88 | } 89 | } 90 | 91 | #[cfg(test)] 92 | mod test { 93 | use super::*; 94 | 95 | #[test] 96 | fn max_info_hashes() { 97 | let mut store = PeersStore::new( 98 | NonZeroUsize::new(1).unwrap(), 99 | NonZeroUsize::new(100).unwrap(), 100 | ); 101 | 102 | let info_hash_a = Id::random(); 103 | let info_hash_b = Id::random(); 104 | 105 | store.add_peer( 106 | info_hash_a, 107 | (&info_hash_a, SocketAddrV4::new([127, 0, 1, 1].into(), 0)), 108 | ); 109 | store.add_peer( 110 | info_hash_b, 111 | (&info_hash_b, SocketAddrV4::new([127, 0, 1, 1].into(), 0)), 112 | ); 113 | 114 | assert_eq!(store.info_hashes.len(), 1); 115 | assert_eq!( 116 | store.get_random_peers(&info_hash_b), 117 | Some([SocketAddrV4::new([127, 0, 1, 1].into(), 0)].into()) 118 | ); 119 | } 120 | 121 | #[test] 122 | fn all_peers() { 123 | let mut store = 124 | PeersStore::new(NonZeroUsize::new(1).unwrap(), NonZeroUsize::new(2).unwrap()); 125 | 126 | let info_hash_a = Id::random(); 127 | let info_hash_b = Id::random(); 128 | let info_hash_c = Id::random(); 129 | 130 | store.add_peer( 131 | info_hash_a, 132 | (&info_hash_a, SocketAddrV4::new([127, 0, 1, 1].into(), 0)), 133 | ); 134 | store.add_peer( 135 | info_hash_a, 136 | (&info_hash_b, SocketAddrV4::new([127, 0, 1, 2].into(), 0)), 137 | ); 138 | store.add_peer( 139 | info_hash_a, 140 | (&info_hash_c, SocketAddrV4::new([127, 0, 1, 3].into(), 0)), 141 | ); 142 | 143 | assert_eq!( 144 | store.get_random_peers(&info_hash_a), 145 | Some( 146 | [ 147 | SocketAddrV4::new([127, 0, 1, 3].into(), 0), 148 | SocketAddrV4::new([127, 0, 1, 2].into(), 0), 149 | ] 150 | .into() 151 | ) 152 | ); 153 | } 154 | 155 | #[test] 156 | fn random_peers_subset() { 157 | let mut store = PeersStore::new( 158 | NonZeroUsize::new(1).unwrap(), 159 | NonZeroUsize::new(200).unwrap(), 160 | ); 161 | 162 | let info_hash = Id::random(); 163 | 164 | for i in 0..200 { 165 | store.add_peer( 166 | info_hash, 167 | (&Id::random(), SocketAddrV4::new([127, 0, 1, i].into(), 0)), 168 | ) 169 | } 170 | 171 | assert_eq!(store.info_hashes.get(&info_hash).unwrap().len(), 200); 172 | 173 | let sample = store.get_random_peers(&info_hash).unwrap(); 174 | 175 | assert_eq!(sample.len(), 20); 176 | } 177 | } 178 | -------------------------------------------------------------------------------- /src/rpc/server/tokens.rs: -------------------------------------------------------------------------------- 1 | //! Manage tokens for remote client IPs. 2 | 3 | use crc::{Crc, CRC_32_ISCSI}; 4 | use getrandom::getrandom; 5 | use std::{ 6 | fmt::{self, Debug, Formatter}, 7 | net::SocketAddrV4, 8 | time::Instant, 9 | }; 10 | 11 | use tracing::trace; 12 | 13 | const SECRET_SIZE: usize = 20; 14 | const TOKEN_SIZE: usize = 4; 15 | const CASTAGNOLI: Crc = Crc::::new(&CRC_32_ISCSI); 16 | 17 | /// Tokens generator. 18 | /// 19 | /// Read [BEP_0005](https://www.bittorrent.org/beps/bep_0005.html) for more information. 20 | #[derive(Clone)] 21 | pub struct Tokens { 22 | prev_secret: [u8; SECRET_SIZE], 23 | curr_secret: [u8; SECRET_SIZE], 24 | last_updated: Instant, 25 | } 26 | 27 | impl Debug for Tokens { 28 | fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { 29 | write!(f, "Tokens (_)") 30 | } 31 | } 32 | 33 | impl Tokens { 34 | /// Create a Tokens generator. 35 | pub fn new() -> Self { 36 | Tokens { 37 | prev_secret: random(), 38 | curr_secret: random(), 39 | last_updated: Instant::now(), 40 | } 41 | } 42 | 43 | // === Public Methods === 44 | 45 | /// Returns `true` if the current secret needs to be updated after an interval. 46 | pub fn should_update(&self) -> bool { 47 | self.last_updated.elapsed() > crate::common::TOKEN_ROTATE_INTERVAL 48 | } 49 | 50 | /// Validate that the token was generated within the past 10 minutes 51 | pub fn validate(&mut self, address: SocketAddrV4, token: &[u8]) -> bool { 52 | let prev = self.internal_generate_token(address, self.prev_secret); 53 | let curr = self.internal_generate_token(address, self.curr_secret); 54 | 55 | token == curr || token == prev 56 | } 57 | 58 | /// Rotate the tokens secret. 59 | pub fn rotate(&mut self) { 60 | trace!("Rotating secrets"); 61 | 62 | self.prev_secret = self.curr_secret; 63 | self.curr_secret = random(); 64 | 65 | self.last_updated = Instant::now(); 66 | } 67 | 68 | /// Generates a new token for a remote peer. 69 | pub fn generate_token(&mut self, address: SocketAddrV4) -> [u8; 4] { 70 | self.internal_generate_token(address, self.curr_secret) 71 | } 72 | 73 | // === Private Methods === 74 | 75 | fn internal_generate_token( 76 | &mut self, 77 | address: SocketAddrV4, 78 | secret: [u8; SECRET_SIZE], 79 | ) -> [u8; TOKEN_SIZE] { 80 | let mut digest = CASTAGNOLI.digest(); 81 | 82 | let octets: Box<[u8]> = address.ip().octets().into(); 83 | 84 | digest.update(&octets); 85 | digest.update(&secret); 86 | 87 | let checksum = digest.finalize(); 88 | 89 | checksum.to_be_bytes() 90 | } 91 | } 92 | 93 | impl Default for Tokens { 94 | fn default() -> Self { 95 | Self::new() 96 | } 97 | } 98 | 99 | fn random() -> [u8; SECRET_SIZE] { 100 | let mut bytes = [0_u8; SECRET_SIZE]; 101 | getrandom(&mut bytes).expect("getrandom"); 102 | 103 | bytes 104 | } 105 | 106 | #[cfg(test)] 107 | mod test { 108 | 109 | use super::*; 110 | 111 | #[test] 112 | fn valid_tokens() { 113 | let mut tokens = Tokens::new(); 114 | 115 | let address = SocketAddrV4::new([127, 0, 0, 1].into(), 6881); 116 | let token = tokens.generate_token(address); 117 | 118 | assert!(tokens.validate(address, &token)) 119 | } 120 | } 121 | -------------------------------------------------------------------------------- /src/rpc/socket.rs: -------------------------------------------------------------------------------- 1 | //! UDP socket layer managing incoming/outgoing requests and responses. 2 | 3 | use std::cmp::Ordering; 4 | use std::net::{SocketAddr, SocketAddrV4, UdpSocket}; 5 | use std::time::{Duration, Instant}; 6 | use tracing::{debug, error, trace}; 7 | 8 | use crate::common::{ErrorSpecific, Message, MessageType, RequestSpecific, ResponseSpecific}; 9 | 10 | use super::config::Config; 11 | 12 | const VERSION: [u8; 4] = [82, 83, 0, 5]; // "RS" version 05 13 | const MTU: usize = 2048; 14 | 15 | pub const DEFAULT_PORT: u16 = 6881; 16 | /// Default request timeout before abandoning an inflight request to a non-responding node. 17 | pub const DEFAULT_REQUEST_TIMEOUT: Duration = Duration::from_millis(2000); // 2 seconds 18 | pub const READ_TIMEOUT: Duration = Duration::from_millis(10); 19 | 20 | /// A UdpSocket wrapper that formats and correlates DHT requests and responses. 21 | #[derive(Debug)] 22 | pub struct KrpcSocket { 23 | next_tid: u16, 24 | socket: UdpSocket, 25 | pub(crate) server_mode: bool, 26 | request_timeout: Duration, 27 | /// We don't need a HashMap, since we know the capacity is `65536` requests. 28 | /// Requests are also ordered by their transaction_id and thus sent_at, so lookup is fast. 29 | inflight_requests: Vec, 30 | 31 | local_addr: SocketAddrV4, 32 | } 33 | 34 | #[derive(Debug)] 35 | pub struct InflightRequest { 36 | tid: u16, 37 | to: SocketAddrV4, 38 | sent_at: Instant, 39 | } 40 | 41 | impl KrpcSocket { 42 | pub(crate) fn new(config: &Config) -> Result { 43 | let request_timeout = config.request_timeout; 44 | let port = config.port; 45 | 46 | let socket = if let Some(port) = port { 47 | UdpSocket::bind(SocketAddr::from(([0, 0, 0, 0], port)))? 48 | } else { 49 | match UdpSocket::bind(SocketAddr::from(([0, 0, 0, 0], DEFAULT_PORT))) { 50 | Ok(socket) => Ok(socket), 51 | Err(_) => UdpSocket::bind(SocketAddr::from(([0, 0, 0, 0], 0))), 52 | }? 53 | }; 54 | 55 | let local_addr = match socket.local_addr()? { 56 | SocketAddr::V4(addr) => addr, 57 | SocketAddr::V6(_) => unimplemented!("KrpcSocket does not support Ipv6"), 58 | }; 59 | 60 | socket.set_read_timeout(Some(READ_TIMEOUT))?; 61 | 62 | Ok(Self { 63 | socket, 64 | next_tid: 0, 65 | server_mode: config.server_mode, 66 | request_timeout, 67 | inflight_requests: Vec::with_capacity(u16::MAX as usize), 68 | 69 | local_addr, 70 | }) 71 | } 72 | 73 | #[cfg(test)] 74 | pub(crate) fn server() -> Result { 75 | Self::new(&Config { 76 | server_mode: true, 77 | ..Default::default() 78 | }) 79 | } 80 | 81 | #[cfg(test)] 82 | pub(crate) fn client() -> Result { 83 | Self::new(&Config::default()) 84 | } 85 | 86 | // === Getters === 87 | 88 | /// Returns the address the server is listening to. 89 | #[inline] 90 | pub fn local_addr(&self) -> SocketAddrV4 { 91 | self.local_addr 92 | } 93 | 94 | // === Public Methods === 95 | 96 | /// Returns true if this message's transaction_id is still inflight 97 | pub fn inflight(&self, transaction_id: &u16) -> bool { 98 | self.inflight_requests 99 | .binary_search_by(|request| request.tid.cmp(transaction_id)) 100 | .is_ok() 101 | } 102 | 103 | /// Send a request to the given address and return the transaction_id 104 | pub fn request(&mut self, address: SocketAddrV4, request: RequestSpecific) -> u16 { 105 | let message = self.request_message(request); 106 | trace!(context = "socket_message_sending", message = ?message); 107 | 108 | self.inflight_requests.push(InflightRequest { 109 | tid: message.transaction_id, 110 | to: address, 111 | sent_at: Instant::now(), 112 | }); 113 | 114 | let tid = message.transaction_id; 115 | let _ = self.send(address, message).map_err(|e| { 116 | debug!(?e, "Error sending request message"); 117 | }); 118 | 119 | tid 120 | } 121 | 122 | /// Send a response to the given address. 123 | pub fn response( 124 | &mut self, 125 | address: SocketAddrV4, 126 | transaction_id: u16, 127 | response: ResponseSpecific, 128 | ) { 129 | let message = 130 | self.response_message(MessageType::Response(response), address, transaction_id); 131 | trace!(context = "socket_message_sending", message = ?message); 132 | let _ = self.send(address, message).map_err(|e| { 133 | debug!(?e, "Error sending response message"); 134 | }); 135 | } 136 | 137 | /// Send an error to the given address. 138 | pub fn error(&mut self, address: SocketAddrV4, transaction_id: u16, error: ErrorSpecific) { 139 | let message = self.response_message(MessageType::Error(error), address, transaction_id); 140 | let _ = self.send(address, message).map_err(|e| { 141 | debug!(?e, "Error sending error message"); 142 | }); 143 | } 144 | 145 | /// Receives a single krpc message on the socket. 146 | /// On success, returns the dht message and the origin. 147 | pub fn recv_from(&mut self) -> Option<(Message, SocketAddrV4)> { 148 | let mut buf = [0u8; MTU]; 149 | 150 | // Cleanup timed-out transaction_ids. 151 | // Find the first timedout request, and delete all earlier requests. 152 | match self.inflight_requests.binary_search_by(|request| { 153 | if request.sent_at.elapsed() > self.request_timeout { 154 | Ordering::Less 155 | } else { 156 | Ordering::Greater 157 | } 158 | }) { 159 | Ok(index) => { 160 | self.inflight_requests.drain(..index); 161 | } 162 | Err(index) => { 163 | self.inflight_requests.drain(..index); 164 | } 165 | }; 166 | 167 | if let Ok((amt, SocketAddr::V4(from))) = self.socket.recv_from(&mut buf) { 168 | let bytes = &buf[..amt]; 169 | 170 | if from.port() == 0 { 171 | trace!( 172 | context = "socket_validation", 173 | message = "Response from port 0" 174 | ); 175 | return None; 176 | } 177 | 178 | match Message::from_bytes(bytes) { 179 | Ok(message) => { 180 | // Parsed correctly. 181 | let should_return = match message.message_type { 182 | MessageType::Request(_) => { 183 | trace!( 184 | context = "socket_message_receiving", 185 | ?message, 186 | ?from, 187 | "Received request message" 188 | ); 189 | 190 | true 191 | } 192 | MessageType::Response(_) => { 193 | trace!( 194 | context = "socket_message_receiving", 195 | ?message, 196 | ?from, 197 | "Received response message" 198 | ); 199 | 200 | self.is_expected_response(&message, &from) 201 | } 202 | MessageType::Error(_) => { 203 | trace!( 204 | context = "socket_message_receiving", 205 | ?message, 206 | ?from, 207 | "Received error message" 208 | ); 209 | 210 | self.is_expected_response(&message, &from) 211 | } 212 | }; 213 | 214 | if should_return { 215 | return Some((message, from)); 216 | } 217 | } 218 | Err(error) => { 219 | trace!(context = "socket_error", ?error, ?from, message = ?String::from_utf8_lossy(bytes), "Received invalid Bencode message."); 220 | } 221 | }; 222 | }; 223 | 224 | None 225 | } 226 | 227 | // === Private Methods === 228 | 229 | fn is_expected_response(&mut self, message: &Message, from: &SocketAddrV4) -> bool { 230 | // Positive or an error response or to an inflight request. 231 | match self 232 | .inflight_requests 233 | .binary_search_by(|request| request.tid.cmp(&message.transaction_id)) 234 | { 235 | Ok(index) => { 236 | let inflight_request = self 237 | .inflight_requests 238 | .get(index) 239 | .expect("should be infallible"); 240 | 241 | if compare_socket_addr(&inflight_request.to, from) { 242 | // Confirm that it is a response we actually sent. 243 | self.inflight_requests.remove(index); 244 | 245 | return true; 246 | } else { 247 | trace!( 248 | context = "socket_validation", 249 | message = "Response from wrong address" 250 | ); 251 | } 252 | } 253 | Err(_) => { 254 | trace!( 255 | context = "socket_validation", 256 | message = "Unexpected response id" 257 | ); 258 | } 259 | } 260 | 261 | false 262 | } 263 | 264 | /// Increments self.next_tid and returns the previous value. 265 | fn tid(&mut self) -> u16 { 266 | // We don't bother much with reusing freed transaction ids, 267 | // since the timeout is so short we are unlikely to run out 268 | // of 65535 ids in 2 seconds. 269 | let tid = self.next_tid; 270 | self.next_tid = self.next_tid.wrapping_add(1); 271 | tid 272 | } 273 | 274 | /// Set transactin_id, version and read_only 275 | fn request_message(&mut self, message: RequestSpecific) -> Message { 276 | let transaction_id = self.tid(); 277 | 278 | Message { 279 | transaction_id, 280 | message_type: MessageType::Request(message), 281 | version: Some(VERSION), 282 | read_only: !self.server_mode, 283 | requester_ip: None, 284 | } 285 | } 286 | 287 | /// Same as request_message but with request transaction_id and the requester_ip. 288 | fn response_message( 289 | &mut self, 290 | message: MessageType, 291 | requester_ip: SocketAddrV4, 292 | request_tid: u16, 293 | ) -> Message { 294 | Message { 295 | transaction_id: request_tid, 296 | message_type: message, 297 | version: Some(VERSION), 298 | read_only: !self.server_mode, 299 | // BEP_0042 Only relevant in responses. 300 | requester_ip: Some(requester_ip), 301 | } 302 | } 303 | 304 | /// Send a raw dht message 305 | fn send(&mut self, address: SocketAddrV4, message: Message) -> Result<(), SendMessageError> { 306 | self.socket.send_to(&message.to_bytes()?, address)?; 307 | trace!(context = "socket_message_sending", message = ?message); 308 | Ok(()) 309 | } 310 | } 311 | 312 | #[derive(thiserror::Error, Debug)] 313 | /// Mainline crate error enum. 314 | pub enum SendMessageError { 315 | /// Errors related to parsing DHT messages. 316 | #[error("Failed to parse packet bytes: {0}")] 317 | BencodeError(#[from] serde_bencode::Error), 318 | 319 | #[error(transparent)] 320 | /// Transparent [std::io::Error] 321 | IO(#[from] std::io::Error), 322 | } 323 | 324 | // Same as SocketAddr::eq but ignores the ip if it is unspecified for testing reasons. 325 | fn compare_socket_addr(a: &SocketAddrV4, b: &SocketAddrV4) -> bool { 326 | if a.port() != b.port() { 327 | return false; 328 | } 329 | 330 | if a.ip().is_unspecified() { 331 | return true; 332 | } 333 | 334 | a.ip() == b.ip() 335 | } 336 | 337 | #[cfg(test)] 338 | mod test { 339 | use std::thread; 340 | 341 | use crate::common::{Id, PingResponseArguments, RequestTypeSpecific}; 342 | 343 | use super::*; 344 | 345 | #[test] 346 | fn tid() { 347 | let mut socket = KrpcSocket::server().unwrap(); 348 | 349 | assert_eq!(socket.tid(), 0); 350 | assert_eq!(socket.tid(), 1); 351 | assert_eq!(socket.tid(), 2); 352 | 353 | socket.next_tid = u16::MAX; 354 | 355 | assert_eq!(socket.tid(), 65535); 356 | assert_eq!(socket.tid(), 0); 357 | } 358 | 359 | #[test] 360 | fn recv_request() { 361 | let mut server = KrpcSocket::server().unwrap(); 362 | let server_address = server.local_addr(); 363 | 364 | let mut client = KrpcSocket::client().unwrap(); 365 | client.next_tid = 120; 366 | 367 | let client_address = client.local_addr(); 368 | let request = RequestSpecific { 369 | requester_id: Id::random(), 370 | request_type: RequestTypeSpecific::Ping, 371 | }; 372 | 373 | let expected_request = request.clone(); 374 | 375 | let server_thread = thread::spawn(move || loop { 376 | if let Some((message, from)) = server.recv_from() { 377 | assert_eq!(from.port(), client_address.port()); 378 | assert_eq!(message.transaction_id, 120); 379 | assert!(message.read_only, "Read-only should be true"); 380 | assert_eq!(message.version, Some(VERSION), "Version should be 'RS'"); 381 | assert_eq!(message.message_type, MessageType::Request(expected_request)); 382 | break; 383 | } 384 | }); 385 | 386 | client.request(server_address, request); 387 | 388 | server_thread.join().unwrap(); 389 | } 390 | 391 | #[test] 392 | fn recv_response() { 393 | let (tx, rx) = flume::bounded(1); 394 | 395 | let mut client = KrpcSocket::client().unwrap(); 396 | let client_address = client.local_addr(); 397 | 398 | let responder_id = Id::random(); 399 | let response = ResponseSpecific::Ping(PingResponseArguments { responder_id }); 400 | 401 | let server_thread = thread::spawn(move || { 402 | let mut server = KrpcSocket::client().unwrap(); 403 | let server_address = server.local_addr(); 404 | tx.send(server_address).unwrap(); 405 | 406 | loop { 407 | server.inflight_requests.push(InflightRequest { 408 | tid: 8, 409 | to: client_address, 410 | sent_at: Instant::now(), 411 | }); 412 | 413 | if let Some((message, from)) = server.recv_from() { 414 | assert_eq!(from.port(), client_address.port()); 415 | assert_eq!(message.transaction_id, 8); 416 | assert!(message.read_only, "Read-only should be true"); 417 | assert_eq!(message.version, Some(VERSION), "Version should be 'RS'"); 418 | assert_eq!( 419 | message.message_type, 420 | MessageType::Response(ResponseSpecific::Ping(PingResponseArguments { 421 | responder_id, 422 | })) 423 | ); 424 | break; 425 | } 426 | } 427 | }); 428 | 429 | let server_address = rx.recv().unwrap(); 430 | 431 | client.response(server_address, 8, response); 432 | 433 | server_thread.join().unwrap(); 434 | } 435 | 436 | #[test] 437 | fn ignore_response_from_wrong_address() { 438 | let mut server = KrpcSocket::client().unwrap(); 439 | let server_address = server.local_addr(); 440 | 441 | let mut client = KrpcSocket::client().unwrap(); 442 | 443 | let client_address = client.local_addr(); 444 | 445 | server.inflight_requests.push(InflightRequest { 446 | tid: 8, 447 | to: SocketAddrV4::new([127, 0, 0, 1].into(), client_address.port() + 1), 448 | sent_at: Instant::now(), 449 | }); 450 | 451 | let response = ResponseSpecific::Ping(PingResponseArguments { 452 | responder_id: Id::random(), 453 | }); 454 | 455 | let _ = response.clone(); 456 | 457 | let server_thread = thread::spawn(move || { 458 | thread::sleep(Duration::from_millis(5)); 459 | assert!( 460 | server.recv_from().is_none(), 461 | "Should not receive a response from wrong address" 462 | ); 463 | }); 464 | 465 | client.response(server_address, 8, response); 466 | 467 | server_thread.join().unwrap(); 468 | } 469 | } 470 | --------------------------------------------------------------------------------