├── .envrc ├── .gitattributes ├── rustfmt.toml ├── assets ├── chunk.png ├── metadata.png ├── overview.png └── weight-scoring.png ├── migrations ├── scoresdb │ ├── 20250222003351_db.down.sql │ └── 20250222003351_db.up.sql └── metadatadb │ ├── 20250516054233_metadata_db.down.sql │ └── 20250516054233_metadata_db.up.sql ├── crates ├── storb_validator │ ├── src │ │ ├── metadata │ │ │ ├── mod.rs │ │ │ ├── sync.rs │ │ │ └── models.rs │ │ ├── utils.rs │ │ ├── signature.rs │ │ ├── constants.rs │ │ ├── middleware.rs │ │ ├── quic.rs │ │ ├── apikey.rs │ │ ├── scoring.rs │ │ ├── repair.rs │ │ └── lib.rs │ └── Cargo.toml ├── storb_miner │ ├── src │ │ ├── constants.rs │ │ ├── miner.rs │ │ ├── middleware.rs │ │ ├── store.rs │ │ ├── routes.rs │ │ └── lib.rs │ └── Cargo.toml ├── storb_cli │ ├── src │ │ ├── constants.rs │ │ ├── cli │ │ │ ├── mod.rs │ │ │ ├── miner.rs │ │ │ ├── validator.rs │ │ │ ├── apikey_manager.rs │ │ │ └── args.rs │ │ ├── log.rs │ │ ├── config.rs │ │ └── main.rs │ └── Cargo.toml └── storb_base │ ├── src │ ├── constants.rs │ ├── verification.rs │ ├── utils.rs │ ├── version.rs │ ├── piece_hash.rs │ ├── memory_db.rs │ ├── sync.rs │ └── lib.rs │ ├── Cargo.toml │ └── build.rs ├── rust-toolchain.toml ├── .dockerignore ├── .gitignore ├── nix └── shell.nix ├── .github └── workflows │ ├── ci.yml │ └── docker.yml ├── config └── validator.docker-compose.yaml ├── LICENSE ├── Dockerfile ├── docs ├── compute.md ├── miner.md ├── overview.md └── validator.md ├── settings.toml.example ├── flake.nix ├── flake.lock ├── README.md └── Cargo.toml /.envrc: -------------------------------------------------------------------------------- 1 | use flake 2 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | * text=auto eol=lf 2 | -------------------------------------------------------------------------------- /rustfmt.toml: -------------------------------------------------------------------------------- 1 | edition = "2021" 2 | group_imports = "StdExternalCrate" 3 | -------------------------------------------------------------------------------- /assets/chunk.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/threetau/storb/HEAD/assets/chunk.png -------------------------------------------------------------------------------- /assets/metadata.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/threetau/storb/HEAD/assets/metadata.png -------------------------------------------------------------------------------- /assets/overview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/threetau/storb/HEAD/assets/overview.png -------------------------------------------------------------------------------- /migrations/scoresdb/20250222003351_db.down.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS miner_stats; 2 | -------------------------------------------------------------------------------- /crates/storb_validator/src/metadata/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod db; 2 | pub mod models; 3 | pub mod sync; 4 | -------------------------------------------------------------------------------- /assets/weight-scoring.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/threetau/storb/HEAD/assets/weight-scoring.png -------------------------------------------------------------------------------- /crates/storb_miner/src/constants.rs: -------------------------------------------------------------------------------- 1 | use std::time::Duration; 2 | 3 | pub const INFO_API_RATE_LIMIT_DURATION: Duration = Duration::from_secs(60); 4 | pub const INFO_API_RATE_LIMIT_MAX_REQUESTS: usize = 60; 5 | -------------------------------------------------------------------------------- /rust-toolchain.toml: -------------------------------------------------------------------------------- 1 | [toolchain] 2 | channel = "nightly" 3 | components = [ 4 | "cargo", 5 | "clippy", 6 | "rust-analyzer", 7 | "rust-std", 8 | "rustc", 9 | "rustfmt", 10 | ] 11 | -------------------------------------------------------------------------------- /crates/storb_cli/src/constants.rs: -------------------------------------------------------------------------------- 1 | pub const NAME: &str = "storb"; 2 | pub const BIN_NAME: &str = "storb"; 3 | pub const VERSION: &str = env!("CARGO_PKG_VERSION"); 4 | pub const ABOUT: &str = env!("CARGO_PKG_DESCRIPTION"); 5 | -------------------------------------------------------------------------------- /crates/storb_cli/src/cli/mod.rs: -------------------------------------------------------------------------------- 1 | use clap::Command; 2 | 3 | pub mod apikey_manager; 4 | pub mod args; 5 | pub mod miner; 6 | pub mod validator; 7 | 8 | pub fn builtin() -> Vec { 9 | vec![miner::cli(), validator::cli(), apikey_manager::cli()] 10 | } 11 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | .config/ 3 | .direnv 4 | .sqlx 5 | .vscode 6 | result/ 7 | target/ 8 | 9 | # Cap'n Proto 10 | *_capnp.rs 11 | 12 | # Storb 13 | *.db 14 | *.storb-local 15 | *.storb-state 16 | logs/ 17 | object_store/ 18 | settings.toml 19 | !settings.toml.example 20 | storb_data/ 21 | 22 | # Testing 23 | test_dir_* 24 | 25 | # Certificates 26 | *.der 27 | *.pem 28 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | .config/ 3 | .direnv 4 | .sqlx 5 | .vscode 6 | result/ 7 | target/ 8 | 9 | # Cap'n Proto 10 | *_capnp.rs 11 | 12 | # Storb 13 | *.db 14 | *.storb-local 15 | *.storb-state 16 | logs/ 17 | object_store/ 18 | settings.toml 19 | !settings.toml.example 20 | storb_data/ 21 | 22 | # crsqlite 23 | crsqlite/ 24 | 25 | # Testing 26 | test_dir_* 27 | 28 | # Certificates 29 | *.der 30 | *.pem 31 | -------------------------------------------------------------------------------- /migrations/metadatadb/20250516054233_metadata_db.down.sql: -------------------------------------------------------------------------------- 1 | -- Add down migration script here 2 | 3 | DROP INDEX IF EXISTS idx_chunks_by_hash; 4 | DROP INDEX IF EXISTS idx_pieces_by_hash; 5 | DROP INDEX IF EXISTS idx_pieces_validator; 6 | 7 | DROP TABLE IF EXISTS chunk_challenge_history; 8 | DROP TABLE IF EXISTS piece_repair_history; 9 | DROP TABLE IF EXISTS chunk_pieces; 10 | DROP TABLE IF EXISTS tracker_chunks; 11 | DROP TABLE IF EXISTS pieces; 12 | DROP TABLE IF EXISTS chunks; 13 | DROP TABLE IF EXISTS trackers; 14 | DROP TABLE IF EXISTS miner_pieces; 15 | DROP TABLE IF EXISTS pieces_to_repair; 16 | DROP TABLE IF EXISTS account_nonces; 17 | -------------------------------------------------------------------------------- /migrations/scoresdb/20250222003351_db.up.sql: -------------------------------------------------------------------------------- 1 | -- Table for miner stats -- 2 | CREATE TABLE miner_stats ( 3 | miner_uid INTEGER PRIMARY KEY, 4 | alpha FLOAT DEFAULT 18.0, 5 | beta FLOAT DEFAULT 36.0, 6 | challenge_successes INTEGER DEFAULT 0, 7 | challenge_attempts INTEGER DEFAULT 0, 8 | retrieval_successes INTEGER DEFAULT 0, 9 | retrieval_attempts INTEGER DEFAULT 0, 10 | store_successes INTEGER DEFAULT 0, 11 | store_attempts INTEGER DEFAULT 0, 12 | total_successes INTEGER DEFAULT 0 13 | ); 14 | 15 | WITH RECURSIVE numbers AS ( 16 | SELECT 0 AS value 17 | UNION ALL 18 | SELECT value + 1 19 | FROM numbers 20 | WHERE value < 255 21 | ) 22 | 23 | INSERT INTO miner_stats (miner_uid) 24 | SELECT value FROM numbers; 25 | -------------------------------------------------------------------------------- /nix/shell.nix: -------------------------------------------------------------------------------- 1 | {pkgs ? import {}}: let 2 | inherit (pkgs) lib; 3 | in 4 | pkgs.mkShell rec { 5 | packages = 6 | [ 7 | # pkgs.capnproto 8 | pkgs.rust-analyzer 9 | ] 10 | ++ lib.optionals pkgs.stdenv.hostPlatform.isLinux [ 11 | pkgs.mold 12 | ]; 13 | 14 | buildInputs = [ 15 | pkgs.llvmPackages_18.clang 16 | pkgs.openssl 17 | pkgs.rustToolchain 18 | ]; 19 | 20 | LD_LIBRARY_PATH = lib.makeLibraryPath buildInputs; 21 | LIBCLANG_PATH = "${pkgs.llvmPackages_18.libclang.lib}/lib"; 22 | 23 | OPENSSL_DIR = "${pkgs.openssl.dev}"; 24 | OPENSSL_LIB_DIR = "${pkgs.lib.getLib pkgs.openssl}/lib"; 25 | OPENSSL_NO_VENDOR = 1; 26 | 27 | PKG_CONFIG_PATH = "${pkgs.openssl.dev}/lib/pkgconfig"; 28 | } 29 | -------------------------------------------------------------------------------- /crates/storb_miner/src/miner.rs: -------------------------------------------------------------------------------- 1 | use std::path::PathBuf; 2 | use std::sync::Arc; 3 | 4 | use base::{BaseNeuron, BaseNeuronConfig, NeuronError}; 5 | use tokio::sync::RwLock; 6 | 7 | #[derive(Clone)] 8 | pub struct MinerConfig { 9 | pub neuron_config: BaseNeuronConfig, 10 | pub store_dir: PathBuf, 11 | } 12 | 13 | /// The Storb miner. 14 | #[derive(Clone)] 15 | pub struct Miner { 16 | pub config: MinerConfig, 17 | pub neuron: Arc>, 18 | } 19 | 20 | impl Miner { 21 | pub async fn new(config: MinerConfig) -> Result { 22 | let neuron_config = config.neuron_config.clone(); 23 | let neuron = Arc::new(RwLock::new(BaseNeuron::new(neuron_config).await?)); 24 | let miner = Miner { config, neuron }; 25 | Ok(miner) 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /crates/storb_miner/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "storb_miner" 3 | version = "0.2.4" 4 | edition.workspace = true 5 | 6 | [dependencies] 7 | # Storb 8 | storb_base.workspace = true 9 | 10 | # Third party 11 | anyhow.workspace = true 12 | axum.workspace = true 13 | bincode.workspace = true 14 | blake3.workspace = true 15 | bytes.workspace = true 16 | crabtensor.workspace = true 17 | dashmap.workspace = true 18 | hex.workspace = true 19 | futures.workspace = true 20 | libp2p.workspace = true 21 | quinn.workspace = true 22 | rand.workspace = true 23 | rcgen.workspace = true 24 | serde.workspace = true 25 | tokio.workspace = true 26 | tokio-serde.workspace = true 27 | tracing.workspace = true 28 | tracing-appender.workspace = true 29 | tracing-subscriber.workspace = true 30 | utoipa.workspace = true 31 | uuid.workspace = true 32 | 33 | [lints] 34 | workspace = true 35 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | on: 3 | pull_request: 4 | branches: 5 | - "**" 6 | push: 7 | branches: 8 | - "main" 9 | 10 | jobs: 11 | checks: 12 | name: Checks for linting and formatting 13 | runs-on: ubuntu-latest 14 | steps: 15 | - name: Checkout code 16 | uses: actions/checkout@v4 17 | 18 | - name: Set up Rust 19 | uses: dtolnay/rust-toolchain@nightly 20 | with: 21 | components: clippy, rustfmt 22 | 23 | - name: Set up Rust cache 24 | uses: Swatinem/rust-cache@v2 25 | 26 | - name: Linting check 27 | run: | 28 | cargo clippy --workspace --all-targets --all-features -- --deny warnings 29 | cargo clippy --workspace --all-targets --no-default-features -- --deny warnings 30 | 31 | - name: Formatting check 32 | run: cargo fmt --check --all 33 | -------------------------------------------------------------------------------- /crates/storb_base/src/constants.rs: -------------------------------------------------------------------------------- 1 | use std::time::Duration; 2 | 3 | pub const NEURON_SYNC_TIMEOUT: Duration = Duration::from_secs(60); 4 | 5 | pub const PIECE_LENGTH_FUNC_MIN_SIZE: u64 = 16 * 1024; // 16 KiB 6 | pub const PIECE_LENGTH_FUNC_MAX_SIZE: u64 = 256 * 1024 * 1024; // 256 MiB 7 | pub const PIECE_LENGTH_SCALING: f64 = 0.5; 8 | pub const PIECE_LENGTH_OFFSET: f64 = 8.39; 9 | 10 | /// parameters for erasure coding 11 | /// TODO(k_and_m): we might change how we determined these in the future - related issue: https://github.com/storb-tech/storb/issues/66 12 | pub const CHUNK_K: usize = 4; 13 | pub const CHUNK_M: usize = 8; 14 | 15 | /// Timeout for HTTP requests to /info endpoint. 16 | pub const INFO_REQ_TIMEOUT: Duration = Duration::from_secs(5); 17 | 18 | /// Timeout params for upload requests 19 | pub const MIN_BANDWIDTH: u64 = 100 * 1024; // 100 KiB/s 20 | 21 | pub const SYNC_BUFFER_SIZE: usize = 32; 22 | -------------------------------------------------------------------------------- /crates/storb_cli/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "storb_cli" 3 | version.workspace = true 4 | description.workspace = true 5 | edition.workspace = true 6 | 7 | [[bin]] 8 | name = "storb" 9 | path = "src/main.rs" 10 | 11 | [dependencies] 12 | # Storb 13 | storb_base.workspace = true 14 | storb_miner.workspace = true 15 | storb_validator.workspace = true 16 | 17 | # Third party 18 | anyhow.workspace = true 19 | chrono.workspace = true 20 | clap.workspace = true 21 | config.workspace = true 22 | crabtensor.workspace = true 23 | expanduser.workspace = true 24 | opentelemetry-otlp.workspace = true 25 | opentelemetry.workspace = true 26 | opentelemetry_sdk.workspace = true 27 | opentelemetry-appender-tracing.workspace = true 28 | libp2p.workspace = true 29 | rcgen.workspace = true 30 | serde.workspace = true 31 | tokio.workspace = true 32 | tokio-serde.workspace = true 33 | tracing.workspace = true 34 | tracing-appender.workspace = true 35 | tracing-subscriber.workspace = true 36 | zfec-rs.workspace = true 37 | 38 | [lints] 39 | workspace = true 40 | -------------------------------------------------------------------------------- /config/validator.docker-compose.yaml: -------------------------------------------------------------------------------- 1 | services: 2 | storb: 3 | image: ghcr.io/storb-tech/storb:latest 4 | environment: 5 | - NODE_TYPE=validator 6 | - API_PORT=${API_PORT:-6969} 7 | - QUIC_PORT=${QUIC_PORT:-6970} 8 | - DHT_PORT=${DHT_PORT:-6942} 9 | - RUST_LOG="${RUST_LOG:-info,libp2p=info}" 10 | ports: 11 | - "80:80" 12 | - "443:443" 13 | - "${API_PORT:-6969}:${API_PORT:-6969}" 14 | - "${QUIC_PORT:-6970}:${QUIC_PORT:-6970}/udp" 15 | - "${DHT_PORT:-6942}:${DHT_PORT:-6942}/tcp" 16 | - "${DHT_PORT:-6942}:${DHT_PORT:-6942}/udp" 17 | volumes: 18 | - ../:/app 19 | - ~/.bittensor/wallets:/root/.bittensor/wallets 20 | working_dir: /app 21 | restart: unless-stopped 22 | labels: 23 | - "com.centurylinklabs.watchtower.enable=true" 24 | watchtower: 25 | image: containrrr/watchtower 26 | volumes: 27 | - /var/run/docker.sock:/var/run/docker.sock 28 | command: --cleanup --include-restarting --interval 60 --label-enable 29 | restart: unless-stopped 30 | -------------------------------------------------------------------------------- /crates/storb_base/src/verification.rs: -------------------------------------------------------------------------------- 1 | use std::slice; 2 | 3 | use crabtensor::{sign::KeypairSignature, AccountId}; 4 | use serde::{Deserialize, Serialize}; 5 | 6 | use crate::NodeUID; 7 | 8 | #[derive(Debug, Deserialize, Serialize)] 9 | #[repr(C)] 10 | pub struct KeyRegistrationInfo { 11 | pub uid: NodeUID, 12 | pub account_id: AccountId, 13 | } 14 | 15 | #[derive(Debug, Deserialize, Serialize)] 16 | #[repr(C)] 17 | pub struct VerificationMessage { 18 | pub netuid: NodeUID, 19 | pub miner: KeyRegistrationInfo, 20 | pub validator: KeyRegistrationInfo, 21 | } 22 | 23 | impl AsRef<[u8]> for VerificationMessage { 24 | fn as_ref(&self) -> &[u8] { 25 | // NOTE: This is safe as this is aligned with u8, and is repr(C) 26 | unsafe { slice::from_raw_parts(self as *const _ as *const u8, size_of::()) } 27 | } 28 | } 29 | 30 | /// The payload containing the message and its signature that is sent to the miner 31 | #[derive(Debug, Deserialize, Serialize)] 32 | #[repr(C)] 33 | pub struct HandshakePayload { 34 | pub signature: KeypairSignature, 35 | pub message: VerificationMessage, 36 | } 37 | -------------------------------------------------------------------------------- /crates/storb_base/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "storb_base" 3 | version.workspace = true 4 | edition.workspace = true 5 | 6 | [lib] 7 | crate-type = ["lib"] 8 | path = "src/lib.rs" 9 | name = "base" 10 | 11 | [dependencies] 12 | anyhow.workspace = true 13 | axum.workspace = true 14 | async-trait.workspace = true 15 | bimap.workspace = true 16 | bincode.workspace = true 17 | blake3.workspace = true 18 | chrono.workspace = true 19 | crabtensor.workspace = true 20 | dashmap.workspace = true 21 | dirs.workspace = true 22 | ed25519-dalek.workspace = true 23 | futures.workspace = true 24 | hex.workspace = true 25 | libp2p.workspace = true 26 | lru.workspace = true 27 | rand.workspace = true 28 | reqwest.workspace = true 29 | rustls.workspace = true 30 | rusqlite.workspace = true 31 | subxt.workspace = true 32 | serde.workspace = true 33 | tempfile.workspace = true 34 | thiserror.workspace = true 35 | tokio.workspace = true 36 | tracing.workspace = true 37 | tracing-subscriber.workspace = true 38 | zfec-rs.workspace = true 39 | 40 | [build-dependencies] 41 | ureq.workspace = true 42 | zip.workspace = true 43 | 44 | [lints] 45 | workspace = true 46 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024-present Storb Technologies Ltd 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:24.04 AS builder 2 | 3 | RUN apt-get update && apt-get install -y --no-install-recommends \ 4 | ca-certificates clang curl libclang-dev libssl-dev libudev-dev pkg-config && \ 5 | rm -rf /var/lib/apt/lists/* 6 | 7 | RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y && \ 8 | . $HOME/.cargo/env && \ 9 | rustup install nightly && \ 10 | rustup default nightly 11 | 12 | WORKDIR /app 13 | 14 | COPY . . 15 | 16 | RUN . $HOME/.cargo/env && \ 17 | cargo build --release 18 | 19 | FROM ubuntu:24.04 AS runtime 20 | 21 | # Install necessary runtime dependencies 22 | RUN apt-get update && apt-get install -y --no-install-recommends \ 23 | ca-certificates libssl-dev libudev-dev pkg-config 24 | 25 | COPY --from=builder /app/target/release/storb /usr/local/bin/storb 26 | COPY --from=builder /app/crsqlite /app/crsqlite 27 | 28 | ENV RUST_LOG="info,libp2p=info,opentelemetry-http=info,opentelemetry-otlp=info,hyper_util=info" 29 | 30 | ENV NODE_TYPE="" 31 | 32 | VOLUME ["/app", "/root/.bittensor/wallets"] 33 | 34 | WORKDIR /app 35 | 36 | CMD ["sh", "-c", "/usr/local/bin/storb ${NODE_TYPE}", "--validator.crsqlite_file", "/app/crsqlite/crsqlite.so"] 37 | -------------------------------------------------------------------------------- /docs/compute.md: -------------------------------------------------------------------------------- 1 | # Compute Requirements & Recommendations 2 | 3 | As an operator of a miner or validator, you should be monitoring your resource utilisation and scale accordingly. 4 | If you believe that the requirements or recommendations could be updated to better reflect real operating conditions, please contact the Storb developers. 5 | 6 | ## Miner 7 | 8 | | Component | Minimum requirement | Recommended | 9 | | --------- | ------------------- | ----------------- | 10 | | CPU | 8 cores, 2.5 GHz | 16 cores, 3.5 GHz | 11 | | RAM | DDR4 16 GiB | DDR5 32 GiB | 12 | | Swap | 16 GiB | 32 GiB | 13 | | Storage | 4 TiB | 20 TiB | 14 | | Network | 1 Gb/s | 10+ Gb/s | 15 | 16 | ## Validator 17 | 18 | | Component | Minimum requirement | Recommended | 19 | | --------- | ------------------- | ----------------- | 20 | | CPU | 8 cores, 2.5 GHz | 16 cores, 3.5 GHz | 21 | | RAM | DDR4 32 GiB | DDR5 64 GiB | 22 | | Swap | 32 GiB | 64 GiB | 23 | | Storage | 1 TiB | 2 TiB | 24 | | Network | 1 Gb/s | 10+ Gb/s | 25 | -------------------------------------------------------------------------------- /crates/storb_cli/src/cli/miner.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | use base::BaseNeuronConfig; 3 | use clap::{Arg, ArgAction, ArgMatches, Command}; 4 | use expanduser::expanduser; 5 | use storb_miner; 6 | use storb_miner::miner::MinerConfig; 7 | 8 | use super::args::get_neuron_config; 9 | use crate::config::Settings; 10 | use crate::get_config_value; 11 | 12 | pub fn cli() -> Command { 13 | Command::new("miner") 14 | .about("Run a Storb miner") 15 | .args([Arg::new("store_dir") 16 | .long("store-dir") 17 | .value_name("directory") 18 | .help("Directory for the object store") 19 | .action(ArgAction::Set)]) 20 | } 21 | 22 | pub fn exec(args: &ArgMatches, settings: &Settings) -> Result<()> { 23 | let store_dir = expanduser(get_config_value!( 24 | args, 25 | "store_dir", 26 | String, 27 | settings.miner.store_dir 28 | ))?; 29 | 30 | // Get validator config with CLI overrides 31 | let neuron_config: BaseNeuronConfig = get_neuron_config(args, settings)?; 32 | let miner_config = MinerConfig { 33 | neuron_config, 34 | store_dir, 35 | }; 36 | 37 | storb_miner::run(miner_config); 38 | Ok(()) 39 | } 40 | -------------------------------------------------------------------------------- /settings.toml.example: -------------------------------------------------------------------------------- 1 | version = "0.2.4" 2 | log_level = "INFO" # Must be one of TRACE, DEBUG, INFO, WARN, ERROR 3 | 4 | netuid = 26 # Testnet is 269 5 | external_ip = "0.0.0.0" 6 | api_port = 6969 7 | post_ip = false 8 | 9 | wallet_path = "~/.bittensor/wallets" 10 | wallet_name = "default" 11 | hotkey_name = "default" 12 | 13 | otel_api_key = "" 14 | otel_endpoint = "" 15 | otel_service_name = "" 16 | 17 | mock = false 18 | 19 | load_old_nodes = true 20 | min_stake_threshold = 1000 21 | 22 | db_file = "storb_data/database.db" 23 | metadatadb_file = "storb_data/metadata.db" 24 | neurons_dir = "storb_data/neurons.state" 25 | 26 | [subtensor] 27 | network = "finney" 28 | address = "wss://entrypoint-finney.opentensor.ai:443" 29 | insecure = true 30 | 31 | [neuron] 32 | sync_frequency = 300 33 | 34 | [miner] 35 | store_dir = "object_store" 36 | 37 | [validator] 38 | scores_state_file = "storb_data/scores.storb-state" 39 | api_keys_db = "storb_data/api_keys.db" 40 | crsqlite_file = "crsqlite/crsqlite.so" 41 | sync_stake_threshold = 50000 42 | 43 | [validator.neuron] 44 | num_concurrent_forwards = 1 45 | disable_set_weights = false 46 | 47 | [validator.query] 48 | batch_size = 20 49 | num_uids = 10 50 | timeout = 5 51 | -------------------------------------------------------------------------------- /crates/storb_validator/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "storb_validator" 3 | version = "0.2.4" 4 | edition.workspace = true 5 | 6 | [dependencies] 7 | # Storb 8 | storb_base.workspace = true 9 | storb_miner.workspace = true 10 | 11 | # Third party 12 | anyhow.workspace = true 13 | axum.workspace = true 14 | bincode.workspace = true 15 | blake3.workspace = true 16 | chrono.workspace = true 17 | crabtensor.workspace = true 18 | dashmap.workspace = true 19 | futures.workspace = true 20 | hex.workspace = true 21 | libp2p.workspace = true 22 | ndarray.workspace = true 23 | opentelemetry.workspace = true 24 | opentelemetry_sdk.workspace = true 25 | opentelemetry-otlp.workspace = true 26 | quinn.workspace = true 27 | r2d2.workspace = true 28 | r2d2_sqlite.workspace = true 29 | rand.workspace = true 30 | rcgen.workspace = true 31 | reqwest.workspace = true 32 | rustls.workspace = true 33 | rusqlite.workspace = true 34 | serde.workspace = true 35 | serde_json.workspace = true 36 | sp-core.workspace = true 37 | subxt.workspace = true 38 | tokio.workspace = true 39 | tokio-serde.workspace = true 40 | tokio-stream.workspace = true 41 | tempfile.workspace = true 42 | thiserror.workspace = true 43 | tracing.workspace = true 44 | tracing-subscriber.workspace = true 45 | uuid.workspace = true 46 | utoipa.workspace = true 47 | 48 | [lints] 49 | workspace = true 50 | -------------------------------------------------------------------------------- /docs/miner.md: -------------------------------------------------------------------------------- 1 | # Storb Miners 2 | 3 | Miners are the backbone of Storb. They are responsible for storing and serving pieces of files to validators, and by proxy, to the end users. 4 | 5 | ## Setup 6 | 7 | ### Configuration 8 | 9 | Have a look over the `settings.toml` file. There are various parameters there that can be modified. Alternatively, one can just set these parameters through the CLI as shown in the following step. 10 | 11 | ### Running miner 12 | 13 | - Mainnet 14 | 15 | ```bash 16 | ./target/release/storb miner \ 17 | --netuid 26 --external-ip EXTERNAL_IP \ 18 | --api-port API_PORT \ 19 | --quic-port QUIC_PORT \ 20 | --wallet-name WALLET_NAME \ 21 | --hotkey-name HOTKEY_NAME \ 22 | --subtensor.address wss://entrypoint-finney.opentensor.ai:443 \ 23 | --neuron.sync-frequency 120 \ 24 | --min-stake-threshold 10000 \ 25 | --post-ip 26 | ``` 27 | 28 | - Testnet 29 | 30 | ```bash 31 | ./target/release/storb miner \ 32 | --netuid 269 --external-ip EXTERNAL_IP \ 33 | --api-port API_PORT \ 34 | --quic-port QUIC_PORT \ 35 | --wallet-name WALLET_NAME \ 36 | --hotkey-name HOTKEY_NAME \ 37 | --subtensor.address wss://test.finney.opentensor.ai:443 \ 38 | --neuron.sync-frequency 120 \ 39 | --min-stake-threshold 0 \ 40 | --post-ip 41 | ``` 42 | -------------------------------------------------------------------------------- /crates/storb_base/src/utils.rs: -------------------------------------------------------------------------------- 1 | use std::net::{IpAddr, SocketAddr}; 2 | 3 | use libp2p::{multiaddr::Protocol, Multiaddr}; 4 | 5 | /// Convert Multiaddr to SocketAddr. 6 | /// The multiaddr format might be something like: `/ip4/127.0.0.1/udp/9000`. 7 | pub fn multiaddr_to_socketaddr(addr: &Multiaddr) -> Option { 8 | let mut iter = addr.iter(); 9 | 10 | // Look for IP protocol component 11 | let ip = match iter.next()? { 12 | Protocol::Ip4(ip) => IpAddr::V4(ip), 13 | Protocol::Ip6(ip) => IpAddr::V6(ip), 14 | _ => return None, 15 | }; 16 | 17 | // Look for TCP protocol component with port 18 | let port = match iter.next()? { 19 | Protocol::Tcp(port) => port, 20 | Protocol::Udp(port) => port, 21 | _ => return None, 22 | }; 23 | 24 | // Construct and return the SocketAddr 25 | Some(SocketAddr::new(ip, port)) 26 | } 27 | 28 | pub fn is_valid_external_addr(addr: &Multiaddr) -> bool { 29 | let mut components = addr.iter(); 30 | match components.next() { 31 | Some(Protocol::Ip4(ip)) => !ip.is_loopback() && !ip.is_unspecified() && !ip.is_private(), 32 | Some(Protocol::Ip6(ip)) => !ip.is_loopback() && !ip.is_unspecified(), // Add more sophisticated IPv6 checks if needed (e.g., ULA) 33 | Some(Protocol::Dns(_)) 34 | | Some(Protocol::Dns4(_)) 35 | | Some(Protocol::Dns6(_)) 36 | | Some(Protocol::Dnsaddr(_)) => true, // Assume DNS is externally resolvable 37 | _ => false, // Not an IP or DNS address we can easily validate as external 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /flake.nix: -------------------------------------------------------------------------------- 1 | { 2 | description = "Storb: An object storage subnet on the Bittensor network"; 3 | 4 | inputs = { 5 | nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; 6 | 7 | rust-overlay = { 8 | url = "github:oxalica/rust-overlay"; 9 | inputs.nixpkgs.follows = "nixpkgs"; 10 | }; 11 | 12 | crane.url = "github:ipetkov/crane"; 13 | flake-compat.url = "github:edolstra/flake-compat"; 14 | }; 15 | 16 | outputs = { 17 | nixpkgs, 18 | rust-overlay, 19 | crane, 20 | ... 21 | }: let 22 | systems = [ 23 | "x86_64-linux" 24 | "x86_64-darwin" 25 | "aarch64-linux" 26 | "aarch64-darwin" 27 | ]; 28 | 29 | overlays = { 30 | rust-overlay = rust-overlay.overlays.default; 31 | rust-toolchain = final: prev: { 32 | rustToolchain = final.rust-bin.fromRustupToolchainFile ./rust-toolchain.toml; 33 | }; 34 | }; 35 | 36 | mkPkgs = system: 37 | import nixpkgs { 38 | inherit system; 39 | overlays = builtins.attrValues overlays; 40 | }; 41 | 42 | forAllSystems = f: nixpkgs.lib.genAttrs systems (system: f (mkPkgs system)); 43 | in { 44 | packages = forAllSystems (pkgs: { 45 | # Add packages as necessary 46 | }); 47 | 48 | devShells = forAllSystems (pkgs: { 49 | default = import ./nix/shell.nix {inherit pkgs;}; 50 | }); 51 | 52 | formatter = forAllSystems (pkgs: pkgs.alejandra); 53 | 54 | overlays = 55 | overlays 56 | // { 57 | default = nixpkgs.lib.composeManyExtensions (builtins.attrValues overlays); 58 | }; 59 | }; 60 | } 61 | -------------------------------------------------------------------------------- /crates/storb_miner/src/middleware.rs: -------------------------------------------------------------------------------- 1 | use std::collections::VecDeque; 2 | use std::net::IpAddr; 3 | use std::sync::Arc; 4 | use std::{net::SocketAddr, time::Instant}; 5 | 6 | use axum::{ 7 | body::Body, 8 | extract::ConnectInfo, 9 | http::{Request, StatusCode}, 10 | middleware::Next, 11 | response::Response, 12 | Extension, 13 | }; 14 | use dashmap::DashMap; 15 | use tracing::warn; 16 | 17 | use crate::constants::{INFO_API_RATE_LIMIT_DURATION, INFO_API_RATE_LIMIT_MAX_REQUESTS}; 18 | 19 | pub type InfoApiRateLimiter = Arc>>; 20 | 21 | pub async fn info_api_rate_limit_middleware( 22 | ConnectInfo(addr): ConnectInfo, 23 | Extension(limiter): Extension, 24 | request: Request, 25 | next: Next, 26 | ) -> Result { 27 | let ip = addr.ip(); 28 | 29 | let now = Instant::now(); 30 | let limit_start_time = now - INFO_API_RATE_LIMIT_DURATION; 31 | 32 | let mut entry = limiter.entry(ip).or_default(); 33 | let timestamps: &mut VecDeque = entry.value_mut(); 34 | 35 | while let Some(ts) = timestamps.front() { 36 | if *ts >= limit_start_time { 37 | break; 38 | } 39 | timestamps.pop_front(); 40 | } 41 | 42 | if timestamps.len() >= INFO_API_RATE_LIMIT_MAX_REQUESTS { 43 | drop(entry); 44 | warn!("Rate limit exceeded for IP: {}", ip); 45 | Err(StatusCode::TOO_MANY_REQUESTS) 46 | } else { 47 | timestamps.push_back(now); 48 | drop(entry); 49 | Ok(next.run(request).await) 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /crates/storb_base/src/version.rs: -------------------------------------------------------------------------------- 1 | use std::num::ParseIntError; 2 | use std::result::Result; 3 | use std::str::FromStr; 4 | 5 | use thiserror::Error; 6 | 7 | #[derive(Debug, Error)] 8 | pub enum VersionError { 9 | #[error(transparent)] 10 | ParseIntError(#[from] ParseIntError), 11 | #[error("The version string `{0}` has too many parts, needs to be in the form `x.y.z`")] 12 | TooManyParts(String), 13 | } 14 | 15 | /// The version of Storb. 16 | #[derive(Clone, Debug, serde::Deserialize, serde::Serialize)] 17 | pub struct Version { 18 | pub major: u16, 19 | pub minor: u16, 20 | pub patch: u16, 21 | } 22 | 23 | /// For getting the spec version (u64) from the Version. 24 | /// This will allow for a version `x.yyy.zz` to be converted to `xyyyzz`. 25 | impl From<&Version> for u64 { 26 | fn from(version: &Version) -> Self { 27 | (version.major as u64) * 100_000 + (version.minor as u64) * 100 + (version.patch as u64) 28 | } 29 | } 30 | 31 | /// For getting the Version from a &str. 32 | impl FromStr for Version { 33 | type Err = VersionError; 34 | 35 | fn from_str(s: &str) -> Result { 36 | let mut parts: Vec = vec![]; 37 | for part in s.split('.') { 38 | parts.push(part.parse()?); 39 | } 40 | 41 | let mut parts_iter = parts.into_iter(); 42 | 43 | let major = parts_iter.next().unwrap_or(0); 44 | let minor = parts_iter.next().unwrap_or(0); 45 | let patch = parts_iter.next().unwrap_or(0); 46 | 47 | if parts_iter.next().is_some() { 48 | return Err(VersionError::TooManyParts(s.to_string())); 49 | } 50 | 51 | Ok(Version { 52 | major, 53 | minor, 54 | patch, 55 | }) 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /crates/storb_validator/src/utils.rs: -------------------------------------------------------------------------------- 1 | use std::sync::Arc; 2 | 3 | use base::NodeUID; 4 | use libp2p::Multiaddr; 5 | use rand::{rngs::StdRng, RngCore, SeedableRng}; 6 | use subxt::ext::codec::Compact; 7 | 8 | use crate::validator::Validator; 9 | 10 | pub fn generate_synthetic_data(size: usize) -> Vec { 11 | let mut data = vec![0u8; size]; // Create a vector of "size" bytes initialized to zero 12 | let mut rng: StdRng = SeedableRng::from_entropy(); 13 | rng.fill_bytes(&mut data); 14 | data 15 | } 16 | 17 | pub async fn get_id_quic_uids( 18 | validator: Arc, 19 | ) -> (Compact, Vec, Vec) { 20 | let neuron_guard = validator.neuron.read().await; 21 | let address_book = neuron_guard.address_book.clone(); 22 | let validator_id = match neuron_guard.clone().local_node_info.uid { 23 | Some(id) => Compact(id), 24 | None => { 25 | return (Compact(0), Vec::new(), Vec::new()); 26 | } 27 | }; 28 | drop(neuron_guard); 29 | 30 | // Filter addresses and get associated UIDs 31 | let mut quic_addresses_with_uids = Vec::new(); 32 | for entry in address_book.iter() { 33 | let uid = entry.key(); 34 | let node_info = entry.value(); 35 | if let Some(quic_addr) = node_info.quic_address.clone() { 36 | quic_addresses_with_uids.push((quic_addr, *uid)); 37 | } 38 | } 39 | drop(address_book); 40 | let quic_addresses: Vec = quic_addresses_with_uids 41 | .iter() 42 | .map(|(addr, _)| addr.clone()) 43 | .collect(); 44 | let miner_uids: Vec = quic_addresses_with_uids 45 | .iter() 46 | .map(|(_, uid)| *uid) 47 | .collect(); 48 | 49 | (validator_id, quic_addresses, miner_uids) 50 | } 51 | -------------------------------------------------------------------------------- /crates/storb_base/src/piece_hash.rs: -------------------------------------------------------------------------------- 1 | use std::fmt::{self, Display, Formatter}; 2 | use std::io::Write; 3 | use std::ops::{Deref, Index}; 4 | use std::slice; 5 | 6 | use crate::piece::PieceHash; 7 | 8 | pub struct PieceHashStr(String); 9 | 10 | impl PieceHashStr { 11 | pub fn new(piece_hash: String) -> Result { 12 | if piece_hash.len() != 64 { 13 | return Err(format!( 14 | "The piece hash {piece_hash} is not 64 characters long" 15 | )); 16 | } 17 | 18 | Ok(Self(piece_hash)) 19 | } 20 | } 21 | 22 | impl Index for PieceHashStr 23 | where 24 | I: slice::SliceIndex, 25 | { 26 | type Output = I::Output; 27 | 28 | #[inline] 29 | fn index(&self, index: I) -> &I::Output { 30 | &self.0[index] 31 | } 32 | } 33 | 34 | impl Deref for PieceHashStr { 35 | type Target = String; 36 | 37 | #[inline] 38 | fn deref(&self) -> &Self::Target { 39 | &self.0 40 | } 41 | } 42 | 43 | impl From for String { 44 | #[inline] 45 | fn from(val: PieceHashStr) -> Self { 46 | val.0 47 | } 48 | } 49 | 50 | impl Display for PieceHashStr { 51 | #[inline] 52 | fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { 53 | Display::fmt(&**self, f) 54 | } 55 | } 56 | 57 | pub fn piecehash_str_to_bytes(piecehash: &PieceHashStr) -> Result { 58 | let final_piece_hash = hex::decode(piecehash.as_str()) 59 | .map_err(|err| format!("Failed to decode piecehash into hex: {err}"))?; 60 | let mut piece_hash_bytes = [0u8; 32]; 61 | let mut w: &mut [u8] = &mut piece_hash_bytes; 62 | w.write_all(&final_piece_hash) 63 | .map_err(|err| format!("Failed to write bytes: {err}"))?; 64 | 65 | Ok(PieceHash(piece_hash_bytes)) 66 | } 67 | -------------------------------------------------------------------------------- /crates/storb_validator/src/signature.rs: -------------------------------------------------------------------------------- 1 | use rustls::client::danger::ServerCertVerifier; 2 | 3 | /// Insecure certificate verifier that accepts all certificates. 4 | /// This should only be used for testing purposes. 5 | #[derive(Debug)] 6 | pub struct InsecureCertVerifier; 7 | 8 | impl ServerCertVerifier for InsecureCertVerifier { 9 | fn verify_server_cert( 10 | &self, 11 | _end_entity: &rustls::pki_types::CertificateDer<'_>, 12 | _intermediates: &[rustls::pki_types::CertificateDer<'_>], 13 | _server_name: &rustls::pki_types::ServerName<'_>, 14 | _ocsp_response: &[u8], 15 | _now: rustls::pki_types::UnixTime, 16 | ) -> Result { 17 | // Always trust the server certificate. 18 | Ok(rustls::client::danger::ServerCertVerified::assertion()) 19 | } 20 | 21 | fn verify_tls12_signature( 22 | &self, 23 | _message: &[u8], 24 | _cert: &rustls::pki_types::CertificateDer<'_>, 25 | _dss: &rustls::DigitallySignedStruct, 26 | ) -> Result { 27 | Ok(rustls::client::danger::HandshakeSignatureValid::assertion()) 28 | } 29 | 30 | fn verify_tls13_signature( 31 | &self, 32 | _message: &[u8], 33 | _cert: &rustls::pki_types::CertificateDer<'_>, 34 | _dss: &rustls::DigitallySignedStruct, 35 | ) -> Result { 36 | Ok(rustls::client::danger::HandshakeSignatureValid::assertion()) 37 | } 38 | 39 | fn supported_verify_schemes(&self) -> Vec { 40 | vec![ 41 | rustls::SignatureScheme::RSA_PKCS1_SHA256, 42 | rustls::SignatureScheme::ECDSA_NISTP256_SHA256, 43 | ] 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /.github/workflows/docker.yml: -------------------------------------------------------------------------------- 1 | name: Docker Build and Publish 2 | 3 | on: 4 | release: 5 | types: [published] 6 | workflow_dispatch: 7 | 8 | env: 9 | REGISTRY: ghcr.io 10 | IMAGE_NAME: storb-tech/storb 11 | 12 | jobs: 13 | build-and-push: 14 | runs-on: ubuntu-latest 15 | permissions: 16 | contents: read 17 | packages: write 18 | 19 | steps: 20 | - name: Checkout repository 21 | uses: actions/checkout@v4 22 | 23 | - name: Set up QEMU 24 | uses: docker/setup-qemu-action@v3 25 | 26 | - name: Set up Docker Buildx 27 | uses: docker/setup-buildx-action@v3 28 | 29 | - name: Log in to the Container registry 30 | uses: docker/login-action@v3 31 | with: 32 | registry: ${{ env.REGISTRY }} 33 | username: ${{ github.actor }} 34 | password: ${{ secrets.GITHUB_TOKEN }} 35 | 36 | - name: Extract metadata (tags, labels) for Docker 37 | id: meta 38 | uses: docker/metadata-action@v5 39 | with: 40 | images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} 41 | tags: | 42 | # Release tags (v1.0.0, 1.0.0) 43 | type=semver,pattern={{version}} 44 | type=semver,pattern={{raw}} 45 | # Latest tag on release or main branch 46 | type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v') }} 47 | # SHA for every build 48 | type=sha,prefix=sha-,format=short 49 | 50 | - name: Build and push Docker image 51 | uses: docker/build-push-action@v6 52 | with: 53 | context: . 54 | file: ./Dockerfile 55 | push: true 56 | platforms: linux/amd64 57 | tags: ${{ steps.meta.outputs.tags }} 58 | labels: ${{ steps.meta.outputs.labels }} 59 | cache-from: type=gha 60 | cache-to: type=gha,mode=max 61 | -------------------------------------------------------------------------------- /crates/storb_validator/src/constants.rs: -------------------------------------------------------------------------------- 1 | use std::time::Duration; 2 | 3 | // Owner UID number on mainnet 4 | pub const OWNER_UID: usize = 4; 5 | // Weight to be assigned to the owner UID 6 | pub const OWNER_WEIGHT: f64 = 0.9; 7 | 8 | /// Buffer size for the metadatadb's MPSC channel. 9 | pub const DB_MPSC_BUFFER_SIZE: usize = 100; 10 | 11 | pub const METADATADB_SYNC_FREQUENCY: u64 = 600; // 5 minutes 12 | 13 | pub const NONCE_CLEANUP_FREQUENCY: u64 = 240; // Every 2 minutes 14 | pub const NONCE_EXPIRATION_TIME: u64 = 3600; // 60 minutes 15 | 16 | pub const PIECE_REPAIR_FREQUENCY: u64 = 660; // Every 11 minutes 17 | 18 | // TODO: should we increase min required miners? 19 | // TODO: should we use it in consume_bytes to determine number of miners to distribute to? 20 | // NOTE: see: https://github.com/storb-tech/storb/issues/66 21 | pub const MIN_REQUIRED_MINERS: usize = 1; 22 | pub const SYNTHETIC_CHALLENGE_FREQUENCY: u64 = 300; 23 | 24 | pub const MAX_CHALLENGE_PIECE_NUM: i32 = 5; 25 | pub const SYNTH_CHALLENGE_TIMEOUT: f64 = 1.0; // TODO: modify this 26 | pub const SYNTH_CHALLENGE_WAIT_BEFORE_RETRIEVE: f64 = 3.0; 27 | pub const MIN_SYNTH_CHUNK_SIZE: usize = 512 * 1024; // minimum size of synthetic data in bytes 28 | pub const MAX_SYNTH_CHUNK_SIZE: usize = 8 * 1024 * 1024; // maximum size of synthetic data in bytes 29 | pub const MAX_SYNTH_CHALLENGE_MINER_NUM: usize = 50; // maximum number of miners to challenge 30 | 31 | // constants for MetadataDB 32 | pub const DB_MAX_LIFETIME: u64 = 3600; // Close connections after 1 hour 33 | pub const IDLE_TIMEOUT: u64 = 600; // Close connections after 10 minutes of inactivity 34 | pub const CONNECTION_TIMEOUT: u64 = 30; // Timeout for establishing a connection 35 | pub const TAO_IN_RAO: f64 = 1_000_000_000.0; // 1 TAO = 1,000,000,000 RAO 36 | 37 | pub const INFO_API_RATE_LIMIT_DURATION: Duration = Duration::from_secs(60); 38 | pub const INFO_API_RATE_LIMIT_MAX_REQUESTS: usize = 10; 39 | 40 | // Initial values for alpha and beta used in the scoring system 41 | // These were empirically derived to minimise reliable node churn 42 | pub const INITIAL_ALPHA: f64 = 18.0; 43 | pub const INITIAL_BETA: f64 = 36.0; 44 | pub const LAMBDA: f64 = 0.99; 45 | pub const AUDIT_WEIGHT: f64 = 1.0; 46 | -------------------------------------------------------------------------------- /crates/storb_cli/src/log.rs: -------------------------------------------------------------------------------- 1 | use std::io::stderr; 2 | 3 | use opentelemetry_appender_tracing::layer::OpenTelemetryTracingBridge; 4 | use opentelemetry_sdk::logs::{SdkLogger, SdkLoggerProvider}; 5 | use tracing_appender::non_blocking::WorkerGuard; 6 | use tracing_appender::rolling::{RollingFileAppender, Rotation}; 7 | use tracing_subscriber::prelude::*; 8 | use tracing_subscriber::registry::Registry; 9 | use tracing_subscriber::{fmt, EnvFilter}; 10 | 11 | /// Print to stderr and exit with a non-zero exit code 12 | #[macro_export] 13 | macro_rules! fatal { 14 | ($($arg:tt)*) => {{ 15 | eprintln!($($arg)*); 16 | std::process::exit(1); 17 | }}; 18 | } 19 | 20 | /// Initialise the global logger 21 | pub fn new( 22 | log_level: &str, 23 | otel_layer: OpenTelemetryTracingBridge, 24 | ) -> (WorkerGuard, WorkerGuard) { 25 | match log_level { 26 | "TRACE" | "DEBUG" | "INFO" | "WARN" | "ERROR" => {} 27 | _ => { 28 | fatal!("Invalid log level `{log_level}`. Valid levels are: TRACE, DEBUG, INFO, WARN, ERROR"); 29 | } 30 | }; 31 | 32 | let filter = EnvFilter::try_from_default_env() 33 | .or_else(|_| EnvFilter::try_new(log_level)) 34 | .expect("Failed to create log filter"); 35 | 36 | let appender = RollingFileAppender::builder() 37 | .rotation(Rotation::DAILY) 38 | .filename_suffix("log") 39 | .build("logs") 40 | .expect("Failed to initialise rolling file appender"); 41 | 42 | let (non_blocking_file, file_guard) = tracing_appender::non_blocking(appender); 43 | let (non_blocking_stdout, stdout_guard) = tracing_appender::non_blocking(stderr()); 44 | 45 | let logger = Registry::default() 46 | .with(filter) 47 | .with( 48 | fmt::Layer::default() 49 | .with_writer(non_blocking_stdout) 50 | .with_line_number(true), 51 | ) 52 | .with( 53 | fmt::Layer::default() 54 | .with_writer(non_blocking_file) 55 | .with_line_number(true) 56 | .with_ansi(false), 57 | ) 58 | .with(otel_layer); 59 | 60 | tracing::subscriber::set_global_default(logger).expect("Failed to initialise logger"); 61 | 62 | (file_guard, stdout_guard) 63 | } 64 | -------------------------------------------------------------------------------- /flake.lock: -------------------------------------------------------------------------------- 1 | { 2 | "nodes": { 3 | "crane": { 4 | "locked": { 5 | "lastModified": 1750266157, 6 | "narHash": "sha256-tL42YoNg9y30u7zAqtoGDNdTyXTi8EALDeCB13FtbQA=", 7 | "owner": "ipetkov", 8 | "repo": "crane", 9 | "rev": "e37c943371b73ed87faf33f7583860f81f1d5a48", 10 | "type": "github" 11 | }, 12 | "original": { 13 | "owner": "ipetkov", 14 | "repo": "crane", 15 | "type": "github" 16 | } 17 | }, 18 | "flake-compat": { 19 | "locked": { 20 | "lastModified": 1747046372, 21 | "narHash": "sha256-CIVLLkVgvHYbgI2UpXvIIBJ12HWgX+fjA8Xf8PUmqCY=", 22 | "owner": "edolstra", 23 | "repo": "flake-compat", 24 | "rev": "9100a0f413b0c601e0533d1d94ffd501ce2e7885", 25 | "type": "github" 26 | }, 27 | "original": { 28 | "owner": "edolstra", 29 | "repo": "flake-compat", 30 | "type": "github" 31 | } 32 | }, 33 | "nixpkgs": { 34 | "locked": { 35 | "lastModified": 1750994206, 36 | "narHash": "sha256-3u6rEbIX9CN/5A5/mc3u0wIO1geZ0EhjvPBXmRDHqWM=", 37 | "owner": "NixOS", 38 | "repo": "nixpkgs", 39 | "rev": "80d50fc87924c2a0d346372d242c27973cf8cdbf", 40 | "type": "github" 41 | }, 42 | "original": { 43 | "owner": "NixOS", 44 | "ref": "nixpkgs-unstable", 45 | "repo": "nixpkgs", 46 | "type": "github" 47 | } 48 | }, 49 | "root": { 50 | "inputs": { 51 | "crane": "crane", 52 | "flake-compat": "flake-compat", 53 | "nixpkgs": "nixpkgs", 54 | "rust-overlay": "rust-overlay" 55 | } 56 | }, 57 | "rust-overlay": { 58 | "inputs": { 59 | "nixpkgs": [ 60 | "nixpkgs" 61 | ] 62 | }, 63 | "locked": { 64 | "lastModified": 1751165203, 65 | "narHash": "sha256-3QhlpAk2yn+ExwvRLtaixWsVW1q3OX3KXXe0l8VMLl4=", 66 | "owner": "oxalica", 67 | "repo": "rust-overlay", 68 | "rev": "90f547b90e73d3c6025e66c5b742d6db51c418c3", 69 | "type": "github" 70 | }, 71 | "original": { 72 | "owner": "oxalica", 73 | "repo": "rust-overlay", 74 | "type": "github" 75 | } 76 | } 77 | }, 78 | "root": "root", 79 | "version": 7 80 | } 81 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Storb 2 | 3 | Storb is a decentralized object storage subnet built on the Bittensor network. It aims to be a distributed, fault-tolerant, and efficient digital storage solution. 4 | 5 | ## Features 6 | 7 | - **Decentralization**: Utilizes a network of nodes to store data redundantly. 8 | 9 | - **Erasure Coding**: Enhances data reliability and storage efficiency by fragmenting and distributing data across multiple nodes, allowing reconstruction even if some fragments are lost. 10 | 11 | - **Incentivized Storage**: Storb leverages the power of the Bittensor network. The subnet rewards miners for contributing reliable and responsive storage resources, with validators ensuring data integrity. Bittensor serves as the ideal incentive layer for this. 12 | 13 | For an overview of how the subnet works, [see here](docs/overview.md). 14 | 15 | ## Installation 16 | 17 | 1. **Clone the repository**: 18 | 19 | ```bash 20 | git clone https://github.com/storb-tech/storb.git 21 | cd storb 22 | ``` 23 | 24 | 2. **Set up environment and dependencies**: 25 | 26 | Install the Rust nightly toolchain. 27 | 28 | Ensure that the following dependencies are installed: 29 | 30 | ```bash 31 | sudo apt update 32 | sudo apt install build-essential clang libclang-dev libssl-dev pkg-config 33 | ``` 34 | 35 | **OR** 36 | 37 | If you use NixOS or the Nix package manager, you can use the provided flakes in this repository to get set up. It will install the necessary dependencies and Rust toolchains for you. 38 | 39 | 3. **Compile**: 40 | 41 | ```bash 42 | cargo build --release 43 | 44 | # Your executable: 45 | ./target/release/storb 46 | ``` 47 | 48 | 4. **Configure and run node**: 49 | 50 | Copy the `settings.toml.example` file and name it `settings.toml`, then update the configuration options as needed. 51 | 52 | Specific instructions for the miner and validator: 53 | 54 | - [**Miner**](docs/miner.md) 55 | - [**Validator**](docs/validator.md) 56 | 57 | ## Contributing 58 | 59 | We welcome contributions to enhance Storb. Please fork the repository and submit a pull request with your improvements. 60 | 61 | ## License 62 | 63 | This project is licensed under the MIT License. See the [LICENSE](./LICENSE) file for details. 64 | 65 | ## Contact 66 | 67 | For questions or support, please open an issue in this repository or contact the maintainers on the Storb or Bittensor Discord server. 68 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | resolver = "2" 3 | members = ["crates/*"] 4 | default-members = ["crates/storb_cli"] 5 | 6 | [workspace.package] 7 | version = "0.2.4" 8 | description = "An object storage subnet on the Bittensor network" 9 | edition = "2021" 10 | authors = ["Storb Technologies Ltd"] 11 | homepage = "https://storb.dev" 12 | license = "MIT" 13 | readme = "README.md" 14 | 15 | [workspace.dependencies] 16 | # Storb 17 | storb_base = { version = "*", path = "crates/storb_base" } 18 | storb_miner = { version = "*", path = "crates/storb_miner" } 19 | storb_validator = { version = "*", path = "crates/storb_validator" } 20 | 21 | # Crabtensor 22 | crabtensor = { git = "https://github.com/storb-tech/crabtensor.git", tag = "v0.6.0" } 23 | sp-core = "36.1.0" 24 | subxt = "0.41.0" 25 | 26 | # Third party 27 | anyhow = "1.0.95" 28 | axum = { version = "0.8.1", features = ["multipart", "macros"] } 29 | async-trait = "0.1" 30 | bimap = "0.6.3" 31 | bincode = "1.3.3" 32 | blake3 = "1.5.5" 33 | bytes = "1.9.0" 34 | chrono = { version = "0.4.39", features = ["serde"] } 35 | clap = { version = "4.5.26", features = ["derive"] } 36 | config = { version = "0.15.6", features = ["toml"] } 37 | dashmap = "6.1.0" 38 | dirs = "6.0.0" 39 | ed25519-dalek = { version = "2.1.1", features = ["serde", "pkcs8"] } 40 | expanduser = "1.2.2" 41 | futures = "0.3.31" 42 | hex = "0.4.3" 43 | libp2p = { version = "0.55.0", features = ["full", "serde"] } 44 | lru = "0.13.0" 45 | ndarray = { version = "0.16.1", features = ["serde"] } 46 | opentelemetry = { version = "0.29.1", default-features = false, features = [ 47 | "trace", 48 | ] } 49 | opentelemetry_sdk = { version = "0.29.0", default-features = false, features = [ 50 | "trace", 51 | ] } 52 | opentelemetry-otlp = "0.29.0" 53 | opentelemetry-appender-tracing = "0.29.1" 54 | quinn = "0.11.2" 55 | r2d2 = "0.8" 56 | r2d2_sqlite = "0.28" 57 | rand = "0.8.5" 58 | rcgen = { version = "0.13.2", features = ["crypto", "pem", "ring"] } 59 | reqwest = { version = "0.12.12", features = ["json"] } 60 | rusqlite = { version = "0.35.0", features = ["bundled", "backup", "load_extension"] } 61 | rustls = { version = "0.23.22" } 62 | serde = { version = "1.0.217", features = ["derive"] } 63 | serde_json = { version = "1.0.137" } 64 | tempfile = "3.2.0" 65 | thiserror = "2.0.11" 66 | tokio = { version = "1.44.2", features = ["full"] } 67 | tokio-serde = { version = "0.9.0", features = ["serde"] } 68 | tokio-stream = "0.1.17" 69 | tracing = "0.1.41" 70 | tracing-appender = "0.2.3" 71 | tracing-subscriber = { version = "0.3.19", features = [ 72 | "ansi", 73 | "env-filter", 74 | "fmt", 75 | "registry", 76 | "std", 77 | ] } 78 | ureq = "3.0.11" 79 | utoipa = "5" 80 | uuid = "1.12.1" 81 | zfec-rs = { git = "https://github.com/thornleywalker/zfec-rs.git", rev = "3f3a3720def2294dc62e65f614862f1a7ddd3187" } 82 | zip = "4.0.0" 83 | 84 | [workspace.lints.clippy] 85 | uninlined_format_args = "allow" 86 | 87 | [profile.release] 88 | codegen-units = 1 89 | opt-level = 2 90 | -------------------------------------------------------------------------------- /crates/storb_base/src/memory_db.rs: -------------------------------------------------------------------------------- 1 | use std::sync::Arc; 2 | 3 | use anyhow::Result; 4 | use rusqlite::{Connection, DatabaseName}; 5 | use tokio::sync::Mutex; 6 | use tracing::info; 7 | 8 | pub struct MemoryDb { 9 | pub conn: Arc>, 10 | pub db_path: String, 11 | } 12 | 13 | impl MemoryDb { 14 | pub async fn new(db_path: &str) -> Result { 15 | // Create in-memory connection 16 | let conn_arc = Arc::new(Mutex::new(Connection::open_in_memory()?)); 17 | 18 | let mut conn = conn_arc.lock().await; 19 | conn.restore(DatabaseName::Main, db_path, Some(|_p| ()))?; 20 | 21 | Ok(Self { 22 | conn: conn_arc.clone(), 23 | db_path: db_path.to_string(), 24 | }) 25 | } 26 | 27 | pub async fn run_backup(&self) { 28 | let db_path = self.db_path.clone(); 29 | let conn = self.conn.clone(); 30 | 31 | info!("Backing up database"); 32 | { 33 | let conn = conn.lock().await; 34 | conn.backup(DatabaseName::Main, &db_path, None).unwrap(); 35 | } 36 | info!("Database backup complete"); 37 | } 38 | } 39 | 40 | #[cfg(test)] 41 | mod tests { 42 | // use std::sync::Once; 43 | use tempfile::NamedTempFile; 44 | 45 | use super::*; 46 | // use tokio::time::sleep; 47 | 48 | // This runs before any tests 49 | // static INIT: Once = Once::new(); 50 | 51 | // fn setup_logging() { 52 | // INIT.call_once(|| { 53 | // tracing_subscriber::fmt() 54 | // .with_max_level(tracing::Level::DEBUG) 55 | // .with_test_writer() // This ensures output goes to the test console 56 | // .init(); 57 | // }); 58 | // } 59 | 60 | // helper function to initialize the disk database 61 | fn init_disk_db(db_path: &str) -> Result<()> { 62 | let conn = Connection::open(db_path)?; 63 | conn.execute("CREATE TABLE test (id INTEGER PRIMARY KEY, value TEXT)", [])?; 64 | conn.execute("INSERT INTO test (value) VALUES (?1)", ["test_value"])?; 65 | Ok(()) 66 | } 67 | 68 | #[tokio::test] 69 | async fn test_memory_db_creation() -> Result<()> { 70 | // Create a temporary file for testing 71 | let temp_file = NamedTempFile::new()?; 72 | let db_path = temp_file.path().to_str().unwrap(); 73 | 74 | // Create an initial SQLite database with some test data 75 | init_disk_db(db_path)?; 76 | 77 | // Create MemoryDb instance 78 | let memory_db = MemoryDb::new(db_path).await?; 79 | 80 | // Verify data was loaded correctly 81 | let conn = memory_db.conn.lock().await; 82 | let mut stmt = conn.prepare("SELECT value FROM test WHERE id = 1")?; 83 | let value: String = stmt.query_row([], |row| row.get(0))?; 84 | 85 | assert_eq!(value, "test_value"); 86 | Ok(()) 87 | } 88 | 89 | #[tokio::test] 90 | async fn test_invalid_db_path() { 91 | let result = MemoryDb::new("/nonexistent/path/db.sqlite").await; 92 | assert!(result.is_err()); 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /crates/storb_cli/src/config.rs: -------------------------------------------------------------------------------- 1 | //! Settings for Storb, defined in the settings.toml file. 2 | 3 | use config::{Config, ConfigError, File}; 4 | use serde::Deserialize; 5 | 6 | #[allow(unused)] 7 | #[derive(Debug, Deserialize)] 8 | pub struct Subtensor { 9 | pub network: String, 10 | pub address: String, 11 | #[serde(default)] 12 | pub insecure: bool, 13 | } 14 | 15 | #[allow(unused)] 16 | #[derive(Debug, Deserialize)] 17 | pub struct Neuron { 18 | pub sync_frequency: u64, 19 | } 20 | 21 | #[allow(unused)] 22 | #[derive(Debug, Deserialize)] 23 | pub struct Miner { 24 | pub store_dir: String, 25 | } 26 | 27 | #[allow(unused)] 28 | #[derive(Debug, Deserialize)] 29 | pub struct ValidatorNeuron { 30 | pub num_concurrent_forwards: u64, 31 | pub disable_set_weights: bool, 32 | } 33 | 34 | #[allow(unused)] 35 | #[derive(Debug, Deserialize)] 36 | pub struct ValidatorQuery { 37 | pub batch_size: u64, 38 | pub num_uids: u16, 39 | pub timeout: u64, 40 | } 41 | 42 | #[allow(unused)] 43 | #[derive(Debug, Deserialize)] 44 | pub struct Validator { 45 | pub scores_state_file: String, 46 | pub crsqlite_file: String, 47 | pub api_keys_db: String, 48 | pub sync_stake_threshold: u64, 49 | pub neuron: ValidatorNeuron, 50 | pub query: ValidatorQuery, 51 | } 52 | 53 | #[allow(unused)] 54 | #[derive(Debug, Deserialize)] 55 | pub struct Settings { 56 | pub version: String, 57 | pub log_level: String, // TODO: Add function to convert str -> enum 58 | 59 | pub netuid: u16, 60 | pub external_ip: String, 61 | pub api_port: u16, 62 | pub quic_port: u16, 63 | pub post_ip: bool, 64 | 65 | pub wallet_path: String, 66 | pub wallet_name: String, 67 | pub hotkey_name: String, 68 | 69 | pub otel_api_key: String, 70 | pub otel_endpoint: String, 71 | pub otel_service_name: String, 72 | 73 | pub mock: bool, 74 | 75 | pub load_old_nodes: bool, 76 | pub min_stake_threshold: u64, 77 | 78 | pub db_file: String, 79 | pub metadatadb_file: String, 80 | pub neurons_dir: String, 81 | 82 | pub subtensor: Subtensor, 83 | pub neuron: Neuron, 84 | 85 | pub miner: Miner, 86 | pub validator: Validator, 87 | } 88 | 89 | impl Settings { 90 | /// Load settings and create a new `Settings` instance. 91 | pub(crate) fn new(config_file: Option<&str>) -> Result { 92 | let file: &str = match config_file { 93 | Some(name) => name, 94 | None => "settings.toml", 95 | }; 96 | 97 | let s = Config::builder() 98 | .add_source(File::with_name(file)) 99 | .build()?; 100 | 101 | s.try_deserialize() 102 | } 103 | } 104 | 105 | /// Macro to get a value from CLI args if present, otherwise use the settings value. 106 | /// 107 | /// # Example 108 | /// 109 | /// ```rust 110 | /// get_config_value(args, "arg_name", String, settings.arg_name); 111 | /// ``` 112 | #[macro_export] 113 | macro_rules! get_config_value { 114 | ($args:expr, $arg_name:expr, $arg_type:ty, $settings:expr) => { 115 | match $args.try_get_one::<$arg_type>($arg_name) { 116 | Ok(Some(value)) => value, 117 | Ok(None) => &$settings, 118 | Err(err) => { 119 | tracing::warn!("Failed to load CLI config, loading default settings. Error: {err}"); 120 | &$settings 121 | } 122 | } 123 | }; 124 | } 125 | -------------------------------------------------------------------------------- /docs/overview.md: -------------------------------------------------------------------------------- 1 | # Storb - An Overview 2 | 3 | ## TLDR 4 | 5 | The subnet can be used as shown below: 6 | 7 | ![overview](../assets/overview.png) 8 | 9 | ### Uploading files 10 | 11 | Before performing in write operations, a user must first generate and sign a nonce with a sr25519 keypair and supply it in the relevant endpoints. 12 | In the future a client cli tool and/or SDK will be created to make this process much more seamless and automatic. 13 | 14 | #### Generate nonce 15 | 16 | - Generate a nonce for your account id (formatted as an ss58 address). For example: 17 | 18 | ```bash 19 | curl -X GET http://{ip}:{port}/nonce?account_id=5HeHkTeToUHmoZisZcoQDF1aJFR1Q8bZJY18FVqTV6fr8kvA" -H "X-API-Key: API_KEY" 20 | ``` 21 | 22 | - This will then return a nonce. For example: 23 | 24 | ``` 25 | fc52edb98e03d5e45381f7dd9e85353b84c5a4b419daf5efe0298ccbfd1d938c 26 | ``` 27 | 28 | - Using [`storb-sign`](https://github.com/Shr1ftyy/storb-sign) to sign this would output: 29 | 30 | ``` 31 | 0xe23e5bc68d39a6130d1d224d08472e687d6227ce5030fc53887829f5d278e2500dc7a81ba84bad424e42648ed76c640835e614225b39a5fe6fd57f266896ac82 32 | ``` 33 | 34 | - Which without the hex prefix "0x" would be: 35 | 36 | ``` 37 | e23e5bc68d39a6130d1d224d08472e687d6227ce5030fc53887829f5d278e2500dc7a81ba84bad424e42648ed76c640835e614225b39a5fe6fd57f266896ac82 38 | ``` 39 | 40 | #### Uploading file 41 | 42 | - Client hits a validator endpoint to upload a file. This can be done by sending a file to its http endpoint, and supplying it with the signature we previously obtained: 43 | 44 | ```bash 45 | curl -X POST http://{ip}:{port}/file?account_id=5HeHkTeToUHmoZisZcoQDF1aJFR1Q8bZJY18FVqTV6fr8kvA?signature=e23e5bc68d39a6130d1d224d08472e687d6227ce5030fc53887829f5d278e2500dc7a81ba84bad424e42648ed76c640835e614225b39a5fe6fd57f266896ac82 -F "file=@{path/to/file}" -H "X-API-Key: API_KEY" 46 | ``` 47 | 48 | - The validator splits up the file into erasure-coded pieces that are then distributed to miners. 49 | - Returns an infohash which can be used by user to download the file from the network: 50 | 51 | ``` 52 | 4e44d931392d68ec0318f09d48267d05c4a8d9fe852832adeeaefc47a892d23c 53 | ``` 54 | 55 | ### Retrieving files 56 | 57 | - Client requests for a file through a validator. Also done through its http endpoint: 58 | 59 | ```bash 60 | curl -X GET http://{ip}:{port}/file?infohash={infohash} -H "X-API-Key: API_KEY" 61 | ``` 62 | 63 | - The validator uses its metadata db to determine where the file pieces are stored then requests the pieces from the miners. 64 | - The validator reconstructs the file with the pieces and sends it back to the client. 65 | 66 | ### Deleting files 67 | 68 | - Client requests to delete a file through a validator: 69 | 70 | ```bash 71 | curl -X DELETE http://{ip}:{port}/file?infohash={infohash}?account_id={account_id}?signature={signature} -H "X-API-Key: API_KEY" 72 | ``` 73 | 74 | - The validator deletes the file pieces from the miners and removes the metadata from its database. 75 | 76 | ## Scoring Mechanism 77 | 78 | Our scoring mechanism uses a Bayesian approach to score the reliability of miners. For a succint overview of our scoring system please read our [litepaper](https://github.com/storb-tech/storb-research/blob/main/papers/Bayesian%20Scoring%20Litepaper.pdf) 79 | 80 | ## Chunking and Piecing 81 | 82 | Files are split into erasure-coded chunks, and subsequently split into pieces and stored across various miners for redundancy. 83 | 84 | ![chunk](../assets/chunk.png) 85 | 86 | ## Sqlite + cr-sqlite for File Metadata and Syncing 87 | 88 | File metadata — which is useful for querying miners for pieces, and, eventually, reconstructing files — is stored across validators in the subnet in the form of sqlite databases, all of which are synced with the help of [cr-sqlite](https://github.com/vlcn-io/cr-sqlite) 89 | 90 | ![metadata](../assets/metadata.png) 91 | -------------------------------------------------------------------------------- /crates/storb_base/build.rs: -------------------------------------------------------------------------------- 1 | use std::env; 2 | use std::fs; 3 | use std::io::{Cursor, Read}; 4 | use std::path::{Path, PathBuf}; 5 | 6 | use zip::ZipArchive; 7 | 8 | fn main() { 9 | println!("cargo::rerun-if-changed=build.rs"); 10 | 11 | // Use custom directory, otherwise use default 12 | let lib_dir = if let Ok(custom_dir) = env::var("CRSQLITE_LIB_DIR") { 13 | PathBuf::from(custom_dir) 14 | } else { 15 | // OUT_DIR format: ./target//build//out 16 | let out_dir = env::var("OUT_DIR").unwrap(); 17 | let lib_dir = PathBuf::from(out_dir) 18 | .ancestors() 19 | .nth(5) // Go up from OUT_DIR to project root 20 | .unwrap() 21 | .join("crsqlite"); 22 | 23 | // Create directory if it doesn't exist 24 | fs::create_dir_all(&lib_dir).unwrap(); 25 | lib_dir 26 | }; 27 | 28 | download_crsqlite(&lib_dir); 29 | } 30 | 31 | /// Downloads the cr-sqlite library based on the target OS and architecture. 32 | fn download_crsqlite(lib_dir: &Path) -> PathBuf { 33 | let target_os = env::var("CARGO_CFG_TARGET_OS").unwrap(); 34 | let target_arch = env::var("CARGO_CFG_TARGET_ARCH").unwrap(); 35 | 36 | let (lib_name, download_url) = match (target_os.as_str(), target_arch.as_str()) { 37 | ("linux", "x86_64") => ( 38 | "crsqlite.so", 39 | "https://github.com/vlcn-io/cr-sqlite/releases/latest/download/crsqlite-linux-x86_64.zip" 40 | ), 41 | ("linux", "aarch64") => ( 42 | "crsqlite.so", 43 | "https://github.com/vlcn-io/cr-sqlite/releases/latest/download/crsqlite-linux-aarch64.zip" 44 | ), 45 | ("macos", "x86_64") => ( 46 | "crsqlite.dylib", 47 | "https://github.com/vlcn-io/cr-sqlite/releases/latest/download/crsqlite-darwin-x86_64.zip" 48 | ), 49 | ("macos", "aarch64") => ( 50 | "crsqlite.dylib", 51 | "https://github.com/vlcn-io/cr-sqlite/releases/latest/download/crsqlite-darwin-aarch64.zip" 52 | ), 53 | _ => panic!("Unsupported platform: {}-{}", target_os, target_arch), 54 | }; 55 | 56 | let lib_path = lib_dir.join(lib_name); 57 | 58 | if lib_path.exists() { 59 | // println!( 60 | // "cargo::warning=cr-sqlite library already exists at {}", 61 | // lib_path.display() 62 | // ); 63 | println!("Hello world"); 64 | return lib_path; 65 | } 66 | 67 | // Download and extract the library 68 | let client = ureq::Agent::new_with_defaults(); 69 | let response = client 70 | .get(download_url) 71 | .call() 72 | .unwrap_or_else(|e| panic!("Failed to download cr-sqlite: {}", e)); 73 | 74 | let mut archive_data = Vec::new(); 75 | response 76 | .into_body() 77 | .into_reader() 78 | .read_to_end(&mut archive_data) 79 | .unwrap_or_else(|e| panic!("Failed to read download: {}", e)); 80 | 81 | // Extract based on file type 82 | if download_url.ends_with(".zip") { 83 | extract_zip(&archive_data, lib_dir, lib_name); 84 | } 85 | 86 | if !lib_path.exists() { 87 | panic!( 88 | "Failed to extract cr-sqlite library to {}", 89 | lib_path.display() 90 | ); 91 | } 92 | 93 | lib_path 94 | } 95 | 96 | fn extract_zip(data: &[u8], lib_dir: &Path, lib_name: &str) { 97 | let cursor = Cursor::new(data); 98 | let mut archive = ZipArchive::new(cursor).unwrap(); 99 | 100 | for i in 0..archive.len() { 101 | let mut file = archive.by_index(i).unwrap(); 102 | 103 | if file.name().ends_with(lib_name) { 104 | let dest_path = lib_dir.join(lib_name); 105 | let mut dest_file = fs::File::create(&dest_path).unwrap(); 106 | std::io::copy(&mut file, &mut dest_file).unwrap(); 107 | break; 108 | } 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /docs/validator.md: -------------------------------------------------------------------------------- 1 | # Storb Validators 2 | 3 | Validators play a crucial role in the Storb network by serving as gateways to the storage subnet. They handle the storage and retrieval of files, ensuring data integrity and availability. 4 | 5 | ## Setup 6 | 7 | ### Configuration 8 | 9 | Have a look over the `settings.toml` file. There are various parameters there that can be modified. Alternatively, one can just set these parameters through the CLI as shown in the following steps. 10 | 11 | ### Setting up databases 12 | 13 | You'll also need to set up the local databases using SQLx. 14 | 15 | #### Install SQLx CLI 16 | 17 | ```bash 18 | cargo install sqlx-cli 19 | ``` 20 | 21 | #### Score database 22 | 23 | ```bash 24 | sqlx database create --database-url "sqlite://storb_data/database.db" 25 | sqlx migrate run --source migrations/scoresdb/ --database-url "sqlite://storb_data/database.db" 26 | ``` 27 | 28 | #### Metadata database 29 | 30 | ```bash 31 | sqlx database create --database-url "sqlite://storb_data/metadata.db" 32 | sqlx migrate run --source migrations/metadatadb/ --database-url "sqlite://storb_data/metadata.db" 33 | ``` 34 | 35 | #### Installing cr-sqlite 36 | 37 | You will also need to install the cr-sqlite extension for sqlite. 38 | The cr-sqlite extension for SQLite is also required. We automatically download the correct library for the target system during the build step. By default, it is downloaded to a `crsqlite` folder in the project root, but you can specify a directory to use with the `CRSQLITE_LIB_DIR` environment variable. 39 | 40 | If you used a custom install directory (or are not using Linux), update the `crsqlite_file` parameter in `settings.toml` to point to the location of the `crsqlite.so` file if it isn't already. For example: 41 | 42 | ```toml 43 | [validator] 44 | crsqlite_file = "/path/to/storb/repo/crsqlite/crsqlite.so" 45 | ``` 46 | 47 | ### Running validator 48 | 49 | #### Mainnet 50 | 51 | ```bash 52 | ./target/release/storb validator \ 53 | --netuid 26 \ 54 | --external-ip EXTERNAL_IP \ 55 | --api-port API_PORT \ 56 | --wallet-name WALLET_NAME \ 57 | --hotkey-name HOTKEY_NAME \ 58 | --subtensor.address wss://entrypoint-finney.opentensor.ai:443 \ 59 | --post-ip 60 | ``` 61 | 62 | #### Testnet 63 | 64 | ```bash 65 | ./target/release/storb validator \ 66 | --netuid 269 \ 67 | --external-ip EXTERNAL_IP \ 68 | --api-port API_PORT \ 69 | --wallet-name WALLET_NAME \ 70 | --hotkey-name HOTKEY_NAME \ 71 | --subtensor.address wss://test.finney.opentensor.ai:443 \ 72 | --post-ip 73 | ``` 74 | 75 | #### Using Docker and Watchtower 76 | 77 | Make sure that you have first cloned this repository and filled out the `settings.toml` file with the necessary parameters. You will also need to specify environment variables for `API_PORT`, `QUIC_PORT`, and `DHT_PORT` if they differ from the defaults for port forwarding in the Docker container. 78 | 79 | Then, run the following: 80 | 81 | ```bash 82 | docker compose up -f config/validator.docker-compose.yaml -d 83 | ``` 84 | 85 | ### API Access 86 | 87 | Validators can serve as gateways to the subnet, thereby letting users upload and download files to and from miners. We provide a cli tool to help manage api access. 88 | 89 | #### Generating an API Key 90 | 91 | The following example generates an api key that has a capped download and upload quota, as well as a rate limit of 60 requests/min. 92 | 93 | ```bash 94 | $ ./target/release/storb apikey create --name "capped" --rate-limit 60 --upload-limit 10485983 --download-limit 10485983 95 | 96 | ✨ Created API key: storb_bac03afd-cc44-4362-8d11-d604e10aebe7 97 | Name: capped 98 | Rate limit: 100 requests/minute 99 | Upload limit: 10485983 bytes 100 | Download limit: 10485983 bytes 101 | ``` 102 | 103 | #### Deleting an API Key 104 | 105 | ```bash 106 | $ ./target/release/storb apikey delete storb_bac03afd-cc44-4362-8d11-d604e10aebe7 107 | ✅ API key deleted successfully 108 | ``` 109 | 110 | For more information on how to use the cli tool run 111 | 112 | ```bash 113 | ./target/release/storb apikey --help 114 | ``` 115 | -------------------------------------------------------------------------------- /crates/storb_validator/src/middleware.rs: -------------------------------------------------------------------------------- 1 | use std::collections::VecDeque; 2 | use std::net::IpAddr; 3 | use std::sync::Arc; 4 | use std::{net::SocketAddr, time::Instant}; 5 | 6 | use axum::{ 7 | body::Body, 8 | extract::ConnectInfo, 9 | http::{Request, StatusCode}, 10 | middleware::Next, 11 | response::Response, 12 | Extension, 13 | }; 14 | use dashmap::DashMap; 15 | use tokio::sync::RwLock; 16 | use tracing::warn; 17 | 18 | use crate::constants::INFO_API_RATE_LIMIT_MAX_REQUESTS; 19 | use crate::{apikey::ApiKeyManager, constants::INFO_API_RATE_LIMIT_DURATION}; 20 | 21 | pub type InfoApiRateLimiter = Arc>>; 22 | 23 | pub async fn require_api_key( 24 | req: Request, 25 | next: Next, 26 | ) -> Result { 27 | tracing::debug!("In require_api_key middleware"); 28 | 29 | let api_key = req 30 | .headers() 31 | .get("X-API-Key") 32 | .and_then(|v| v.to_str().ok()) 33 | .ok_or_else(|| { 34 | tracing::error!("No API key found in request headers"); 35 | (StatusCode::UNAUTHORIZED, "Missing API key".to_string()) 36 | })?; 37 | 38 | let api_key_manager = req 39 | .extensions() 40 | .get::>>() 41 | .ok_or_else(|| { 42 | tracing::error!("ApiKeyManager not found in request extensions"); 43 | ( 44 | StatusCode::INTERNAL_SERVER_ERROR, 45 | "Internal server error".to_string(), 46 | ) 47 | })?; 48 | 49 | let key_info = api_key_manager 50 | .read() 51 | .await 52 | .validate_key(api_key) 53 | .await 54 | .map_err(|e| { 55 | tracing::error!("Error validating API key: {}", e); 56 | ( 57 | StatusCode::INTERNAL_SERVER_ERROR, 58 | "Internal server error".to_string(), 59 | ) 60 | })? 61 | .ok_or_else(|| { 62 | tracing::error!("Invalid API key: {}", api_key); 63 | (StatusCode::UNAUTHORIZED, "Invalid API key".to_string()) 64 | })?; 65 | 66 | // Check rate limit only 67 | if let Some(rate_limit) = key_info.rate_limit { 68 | let within_limit = api_key_manager 69 | .read() 70 | .await 71 | .check_rate_limit(api_key, rate_limit) 72 | .await 73 | .map_err(|e| { 74 | tracing::error!("Failed to check rate limit: {}", e); 75 | ( 76 | StatusCode::INTERNAL_SERVER_ERROR, 77 | "Internal server error".to_string(), 78 | ) 79 | })?; 80 | 81 | if !within_limit { 82 | tracing::error!("Rate limit exceeded for API key: {}", api_key); 83 | return Err(( 84 | StatusCode::TOO_MANY_REQUESTS, 85 | "Rate limit exceeded".to_string(), 86 | )); 87 | } 88 | } 89 | 90 | Ok(next.run(req).await) 91 | } 92 | 93 | pub async fn info_api_rate_limit_middleware( 94 | ConnectInfo(addr): ConnectInfo, 95 | Extension(limiter): Extension, 96 | request: Request, 97 | next: Next, 98 | ) -> Result { 99 | let ip = addr.ip(); 100 | 101 | let now = Instant::now(); 102 | let limit_start_time = now - INFO_API_RATE_LIMIT_DURATION; 103 | 104 | let mut entry = limiter.entry(ip).or_default(); 105 | let timestamps: &mut VecDeque = entry.value_mut(); 106 | 107 | while let Some(ts) = timestamps.front() { 108 | if *ts >= limit_start_time { 109 | break; 110 | } 111 | timestamps.pop_front(); 112 | } 113 | 114 | if timestamps.len() >= INFO_API_RATE_LIMIT_MAX_REQUESTS { 115 | drop(entry); 116 | warn!("Rate limit exceeded for IP: {}", ip); 117 | Err(StatusCode::TOO_MANY_REQUESTS) 118 | } else { 119 | timestamps.push_back(now); 120 | drop(entry); 121 | Ok(next.run(request).await) 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /crates/storb_cli/src/cli/validator.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | use base::BaseNeuronConfig; 3 | use clap::{Arg, ArgAction, ArgMatches, Command}; 4 | use expanduser::expanduser; 5 | use storb_validator; 6 | use storb_validator::validator::ValidatorConfig; 7 | 8 | use super::args::get_neuron_config; 9 | use crate::config::Settings; 10 | use crate::get_config_value; 11 | 12 | pub fn cli() -> Command { 13 | Command::new("validator") 14 | .about("Run a Storb validator") 15 | .args([ 16 | // Validator settings 17 | Arg::new("scores_state_file") 18 | .long("scores-state-file") 19 | .value_name("path") 20 | .help("The path to the scores state file") 21 | .action(ArgAction::Set), 22 | Arg::new("crsqlite_file") 23 | .long("crsqlite-file") 24 | .value_name("path") 25 | .help("The path to the cr-sqlite lib") 26 | .action(ArgAction::Set), 27 | Arg::new("sync_stake_threshold") 28 | .long("sync-stake-threshold") 29 | .value_name("amount") 30 | .help("The minimum stake amount for a neuron to be synced") 31 | .action(ArgAction::Set), 32 | Arg::new("api_keys_db") 33 | .long("api-keys-db") 34 | .value_name("path") 35 | .help("The path to the API keys database") 36 | .action(ArgAction::Set), 37 | // Neuron settings 38 | Arg::new("neuron.num_concurrent_forwards") 39 | .long("neuron.num-concurrent-forwards") 40 | .value_name("value") 41 | .help("The number of concurrent forwards running at any time") 42 | .action(ArgAction::Set), 43 | Arg::new("neuron.disable_set_weights") 44 | .long("neuron.disable-set-weights") 45 | .help("Disable weight setting") 46 | .action(ArgAction::SetTrue), 47 | // Query settings 48 | Arg::new("query.batch_size") 49 | .long("query.batch-size") 50 | .value_name("size") 51 | .help("Max store query batch size") 52 | .action(ArgAction::Set), 53 | Arg::new("query.num_uids") 54 | .long("query.num-uids") 55 | .value_name("num_uids") 56 | .help("Max number of uids to query per store request") 57 | .action(ArgAction::Set), 58 | Arg::new("query.timeout") 59 | .long("query.timeout") 60 | .value_name("timeout") 61 | .help("Query timeout in seconds") 62 | .action(ArgAction::Set), 63 | ]) 64 | } 65 | 66 | pub fn exec(args: &ArgMatches, settings: &Settings) -> Result<()> { 67 | let scores_state_file = expanduser(get_config_value!( 68 | args, 69 | "scores_state_file", 70 | String, 71 | settings.validator.scores_state_file 72 | ))?; 73 | let crsqlite_file = expanduser(get_config_value!( 74 | args, 75 | "crsqlite_file", 76 | String, 77 | settings.validator.crsqlite_file 78 | ))?; 79 | let sync_stake_threshold = *get_config_value!( 80 | args, 81 | "sync_stake_threshold", 82 | u64, 83 | settings.validator.sync_stake_threshold 84 | ); 85 | let api_keys_db = expanduser(get_config_value!( 86 | args, 87 | "api_keys_db", 88 | String, 89 | settings.validator.api_keys_db 90 | ))?; 91 | 92 | // Get validator config with CLI overrides 93 | let neuron_config: BaseNeuronConfig = get_neuron_config(args, settings)?; 94 | let validator_config = ValidatorConfig { 95 | scores_state_file, 96 | crsqlite_file, 97 | sync_stake_threshold, 98 | neuron_config, 99 | api_keys_db, 100 | otel_api_key: get_config_value!(args, "otel_api_key", String, &settings.otel_api_key) 101 | .to_string(), 102 | otel_endpoint: get_config_value!(args, "otel_endpoint", String, &settings.otel_endpoint) 103 | .to_string(), 104 | otel_service_name: get_config_value!( 105 | args, 106 | "otel_service_name", 107 | String, 108 | &settings.otel_service_name 109 | ) 110 | .to_string(), 111 | }; 112 | 113 | storb_validator::run(validator_config); 114 | Ok(()) 115 | } 116 | -------------------------------------------------------------------------------- /crates/storb_validator/src/quic.rs: -------------------------------------------------------------------------------- 1 | use std::error::Error; 2 | use std::net::{IpAddr, Ipv4Addr, SocketAddr}; 3 | use std::sync::Arc; 4 | use std::time::Duration; 5 | 6 | use anyhow::{anyhow, Context, Result}; 7 | use base::utils::multiaddr_to_socketaddr; 8 | use libp2p::Multiaddr; 9 | use quinn::crypto::rustls::QuicClientConfig; 10 | use quinn::{ClientConfig, Connection, Endpoint}; 11 | use rustls::ClientConfig as RustlsClientConfig; 12 | use tracing::{error, info}; 13 | 14 | use crate::signature::InsecureCertVerifier; 15 | 16 | // TODO: Make this configurable? 17 | pub const QUIC_CONNECTION_TIMEOUT: Duration = Duration::from_secs(3); 18 | // TODO: turn this into a setting? 19 | const MIN_REQUIRED_MINERS: usize = 1; // Minimum number of miners needed for operation 20 | const MAX_ATTEMPTS: i32 = 5; 21 | 22 | /// Establishes QUIC connections to miners from a list of QUIC addresses. 23 | pub async fn establish_miner_connections( 24 | quic_addresses: Vec, 25 | ) -> Result> { 26 | let mut connection_futures = Vec::new(); 27 | 28 | for quic_addr in quic_addresses { 29 | let client = make_client_endpoint(SocketAddr::new(IpAddr::V4(Ipv4Addr::UNSPECIFIED), 0)) 30 | .map_err(|e| anyhow!(e.to_string()))?; 31 | let socket_addr = multiaddr_to_socketaddr(&quic_addr) 32 | .context("Could not get SocketAddr from Multiaddr. Ensure that the Multiaddr is not missing any components")?; 33 | 34 | connection_futures.push(async move { 35 | match create_quic_client(&client, socket_addr).await { 36 | Ok(connection) => Some((socket_addr, connection)), 37 | Err(e) => { 38 | error!("Failed to establish connection with {}: {}", socket_addr, e); 39 | None 40 | } 41 | } 42 | }); 43 | } 44 | 45 | let connections: Vec<_> = futures::future::join_all(connection_futures) 46 | .await 47 | .into_iter() 48 | .flatten() 49 | .collect(); 50 | 51 | if connections.len() < MIN_REQUIRED_MINERS { 52 | return Err(anyhow!( 53 | "Failed to establish minimum required connections. Got {} out of minimum {}", 54 | connections.len(), 55 | MIN_REQUIRED_MINERS 56 | )); 57 | } 58 | 59 | Ok(connections) 60 | } 61 | 62 | /// Configures a QUIC client with insecure certificate verification 63 | pub fn configure_client() -> Result> { 64 | let mut tls_config = RustlsClientConfig::builder() 65 | .with_root_certificates(rustls::RootCertStore::empty()) 66 | .with_no_client_auth(); 67 | 68 | tls_config 69 | .dangerous() 70 | .set_certificate_verifier(Arc::new(InsecureCertVerifier)); 71 | 72 | let quic_config = QuicClientConfig::try_from(tls_config).unwrap_or_else(|e| { 73 | panic!("Failed to create QUIC client config: {:?}", e); 74 | }); 75 | 76 | let client_config = ClientConfig::new(Arc::new(quic_config)); 77 | Ok(client_config) 78 | } 79 | 80 | /// Creates a QUIC client endpoint bound to specified address with automatic retries 81 | pub fn make_client_endpoint( 82 | bind_addr: SocketAddr, 83 | ) -> Result> { 84 | let client_cfg = configure_client()?; 85 | 86 | let mut attempts = 0; 87 | while attempts < MAX_ATTEMPTS { 88 | match Endpoint::client(bind_addr) { 89 | Ok(mut endpoint) => { 90 | endpoint.set_default_client_config(client_cfg.clone()); 91 | return Ok(endpoint); 92 | } 93 | Err(_) if attempts < MAX_ATTEMPTS - 1 => { 94 | attempts += 1; 95 | std::thread::sleep(std::time::Duration::from_millis(100)); 96 | continue; 97 | } 98 | Err(e) => return Err(e.into()), 99 | } 100 | } 101 | 102 | Err("Failed to create endpoint after maximum attempts".into()) 103 | } 104 | 105 | /// Creates QUIC client connection with provided endpoint and address 106 | pub async fn create_quic_client(client: &Endpoint, addr: SocketAddr) -> Result { 107 | info!("Creating QUIC client connection to {}", addr); 108 | 109 | let connection = tokio::time::timeout(QUIC_CONNECTION_TIMEOUT, async { 110 | let conn = client.connect(addr, "0.0.0.0")?; 111 | match conn.await { 112 | Ok(connection) => Ok(connection), 113 | Err(e) => { 114 | error!("Failed to establish QUIC connection: {}", e); 115 | Err(anyhow!("QUIC onnection failed: {}", e)) 116 | } 117 | } 118 | }) 119 | .await 120 | .map_err(|_| anyhow!("QUIC connection timed out"))??; 121 | 122 | info!("QUIC connection established successfully"); 123 | Ok(connection) 124 | } 125 | -------------------------------------------------------------------------------- /migrations/metadatadb/20250516054233_metadata_db.up.sql: -------------------------------------------------------------------------------- 1 | -- Add up migration script here 2 | 3 | -- infohashes 4 | CREATE TABLE IF NOT EXISTS infohashes ( 5 | infohash BLOB PRIMARY KEY NOT NULL DEFAULT '', 6 | name VARCHAR(4096) NOT NULL DEFAULT 'default', 7 | length INTEGER NOT NULL DEFAULT 0, 8 | chunk_size INTEGER NOT NULL DEFAULT 0, 9 | chunk_count INTEGER NOT NULL DEFAULT 0, 10 | owner_account_id BLOB NOT NULL DEFAULT '', 11 | creation_timestamp DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, 12 | signature TEXT NOT NULL DEFAULT '' 13 | ); 14 | 15 | -- Chunks 16 | CREATE TABLE IF NOT EXISTS chunks ( 17 | chunk_hash BLOB PRIMARY KEY NOT NULL DEFAULT '', 18 | k INTEGER NOT NULL DEFAULT 0, 19 | m INTEGER NOT NULL DEFAULT 0, 20 | chunk_size INTEGER NOT NULL DEFAULT 0, 21 | padlen INTEGER NOT NULL DEFAULT 0, 22 | original_chunk_size INTEGER NOT NULL DEFAULT 0, 23 | ref_count INTEGER NOT NULL DEFAULT 1 24 | ); 25 | 26 | -- Chunk‑tracker mapping 27 | CREATE TABLE IF NOT EXISTS tracker_chunks ( 28 | infohash BLOB NOT NULL DEFAULT '', 29 | chunk_idx INTEGER NOT NULL DEFAULT 0, 30 | chunk_hash BLOB NOT NULL DEFAULT '', 31 | PRIMARY KEY (infohash, chunk_idx) 32 | ); 33 | 34 | -- Pieces 35 | CREATE TABLE IF NOT EXISTS pieces ( 36 | piece_hash BLOB PRIMARY KEY NOT NULL DEFAULT '', 37 | piece_size INTEGER NOT NULL DEFAULT 0, 38 | piece_type INTEGER NOT NULL DEFAULT 0, -- 0: Data, 1: Parity 39 | miners TEXT NOT NULL DEFAULT '', -- JSON array of miner IDs 40 | ref_count INTEGER NOT NULL DEFAULT 1 41 | ); 42 | 43 | -- A mapping of miner uids to pieces they store 44 | CREATE TABLE IF NOT EXISTS miner_pieces ( 45 | miner_uid INTEGER NOT NULL, 46 | piece_hash BLOB NOT NULL, 47 | UNIQUE (miner_uid, piece_hash) 48 | ); 49 | 50 | -- Piece‑chunk mapping 51 | CREATE TABLE IF NOT EXISTS chunk_pieces ( 52 | chunk_hash BLOB NOT NULL DEFAULT '', 53 | piece_idx INTEGER NOT NULL DEFAULT 0, 54 | piece_hash BLOB NOT NULL DEFAULT '', 55 | PRIMARY KEY (chunk_hash, piece_idx) 56 | ); 57 | 58 | -- Table of pieces to repair, and the miner uids which failed to serve them -- 59 | CREATE TABLE pieces_to_repair ( 60 | piece_hash BLOB, -- piece hash 61 | miners TEXT NOT NULL DEFAULT '[]', -- JSON array of miner IDs who failed to serve the piece 62 | UNIQUE (piece_hash, miners) 63 | ); 64 | 65 | -- Piece‑repair history 66 | CREATE TABLE IF NOT EXISTS piece_repair_history ( 67 | piece_repair_hash BLOB PRIMARY KEY NOT NULL DEFAULT '', 68 | piece_hash BLOB NOT NULL DEFAULT '', 69 | chunk_hash BLOB NOT NULL DEFAULT '', 70 | validator_id INTEGER NOT NULL DEFAULT 0, 71 | timestamp DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, 72 | signature TEXT NOT NULL DEFAULT '' 73 | ); 74 | 75 | -- Chunk‑challenge history 76 | CREATE TABLE IF NOT EXISTS chunk_challenge_history ( 77 | challenge_hash BLOB PRIMARY KEY NOT NULL DEFAULT '', 78 | chunk_hash BLOB NOT NULL DEFAULT '', 79 | validator_id INTEGER NOT NULL DEFAULT 0, 80 | miners_challenged TEXT NOT NULL DEFAULT '', -- JSON array of miner IDs 81 | miners_successful TEXT NOT NULL DEFAULT '', -- JSON array of miner IDs 82 | piece_repair_hash BLOB, 83 | timestamp DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, 84 | signature TEXT NOT NULL DEFAULT '' 85 | ); 86 | 87 | -- Nonce tracking to prevent replay attacks 88 | CREATE TABLE IF NOT EXISTS account_nonces ( 89 | account_id BLOB NOT NULL, 90 | nonce BLOB NOT NULL, 91 | timestamp DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, 92 | PRIMARY KEY (account_id, nonce) 93 | ); 94 | 95 | -- Indexes 96 | CREATE INDEX IF NOT EXISTS idx_chunks_by_hash ON chunks(chunk_hash); 97 | CREATE INDEX IF NOT EXISTS idx_pieces_by_hash ON pieces(piece_hash); 98 | CREATE INDEX IF NOT EXISTS idx_chunks_ref_count ON chunks(ref_count); 99 | CREATE INDEX IF NOT EXISTS idx_pieces_ref_count ON pieces(ref_count); 100 | CREATE INDEX IF NOT EXISTS idx_chunk_pieces_by_chunk ON chunk_pieces(chunk_hash); 101 | CREATE INDEX IF NOT EXISTS idx_tracker_chunks_by_infohash ON tracker_chunks(infohash); 102 | CREATE INDEX IF NOT EXISTS idx_infohashes_by_owner ON infohashes(owner_account_id); 103 | CREATE INDEX IF NOT EXISTS idx_piece_repair_by_piece ON piece_repair_history(piece_hash); 104 | CREATE INDEX IF NOT EXISTS idx_chunk_challenge_by_chunk ON chunk_challenge_history(chunk_hash); 105 | CREATE INDEX IF NOT EXISTS idx_account_nonces_timestamp ON account_nonces(timestamp); 106 | CREATE INDEX IF NOT EXISTS idx_miner_pieces_by_miner ON miner_pieces(miner_uid); 107 | CREATE INDEX IF NOT EXISTS idx_miner_pieces_by_piece ON miner_pieces(piece_hash); 108 | CREATE INDEX IF NOT EXISTS idx_pieces_to_repair ON pieces_to_repair(piece_hash); 109 | -------------------------------------------------------------------------------- /crates/storb_cli/src/main.rs: -------------------------------------------------------------------------------- 1 | #![allow(clippy::uninlined_format_args)] 2 | use std::collections::HashMap; 3 | 4 | use anyhow::Result; 5 | use clap::Command; 6 | use constants::{ABOUT, BIN_NAME, NAME, VERSION}; 7 | use expanduser::expanduser; 8 | use opentelemetry_appender_tracing::layer::OpenTelemetryTracingBridge; 9 | use opentelemetry_otlp::{LogExporter, Protocol, WithExportConfig, WithHttpConfig}; 10 | use opentelemetry_sdk::{logs::SdkLoggerProvider, Resource}; 11 | use tracing::info; 12 | 13 | mod cli; 14 | mod config; 15 | mod constants; 16 | mod log; 17 | 18 | pub fn main() -> Result<()> { 19 | let about_text = format!("{} {}\n{}", NAME, VERSION, ABOUT); 20 | let usage_text = format!("{} [options] []", BIN_NAME); 21 | let after_help_text = format!( 22 | "See '{} help ' for more information on a command", 23 | BIN_NAME 24 | ); 25 | 26 | let storb = Command::new("storb") 27 | .bin_name(BIN_NAME) 28 | .name(NAME) 29 | .version(VERSION) 30 | .about(about_text) 31 | .override_usage(usage_text) 32 | .after_help(after_help_text) 33 | .args(cli::args::common_args()) 34 | .arg_required_else_help(true) 35 | .subcommands(cli::builtin()) 36 | .subcommand_required(true); 37 | 38 | let matches = storb.get_matches(); 39 | 40 | // Gets the config file as str 41 | let config_file_raw: Option<&str> = match matches.try_get_one::("config") { 42 | Ok(config_path) => config_path.map(|s| s.as_str()), 43 | Err(error) => { 44 | fatal!("Error while parsing config file flag: {error}") 45 | } 46 | }; 47 | 48 | let expanded_path = expanduser(config_file_raw.unwrap_or("settings.toml")) 49 | .map_err(|e| fatal!("Error while expanding config file path: {e}")) 50 | .unwrap_or_else(|_| fatal!("Failed to expand config file path")); 51 | 52 | let config_file = match expanded_path.to_str() { 53 | Some(s) => Some(s.to_owned()), 54 | None => { 55 | fatal!( 56 | "Config path is not valid UTF-8: {}", 57 | expanded_path.display() 58 | ); 59 | } 60 | }; 61 | 62 | // CLI values take precedence over settings.toml 63 | let settings = match config::Settings::new(config_file.as_deref()) { 64 | Ok(s) => s, 65 | Err(error) => fatal!("Failed to parse settings file: {error:?}"), 66 | }; 67 | 68 | // Initialise logger and set the logging level 69 | let log_level_arg = match matches.try_get_one::("log_level") { 70 | Ok(level) => level, 71 | Err(error) => { 72 | fatal!("Error while parsing log level flag: {error}"); 73 | } 74 | }; 75 | let log_level = match log_level_arg { 76 | Some(level) => level, 77 | None => &settings.log_level, 78 | }; 79 | 80 | let otel_api_key_args = match matches.try_get_one::("otel_api_key") { 81 | Ok(key) => key, 82 | Err(error) => { 83 | fatal!("Error while parsing otel_api_key flag: {error}"); 84 | } 85 | }; 86 | let otel_api_key = match otel_api_key_args { 87 | Some(key) => key, 88 | None => &settings.otel_api_key, 89 | }; 90 | 91 | let otel_endpoint_args = match matches.try_get_one::("otel_endpoint") { 92 | Ok(endpoint) => endpoint, 93 | Err(error) => { 94 | fatal!("Error while parsing oltp_endpoint flag: {error}"); 95 | } 96 | }; 97 | let otel_endpoint = match otel_endpoint_args { 98 | Some(endpoint) => endpoint, 99 | None => &settings.otel_endpoint, 100 | }; 101 | 102 | let otel_service_name_args = match matches.try_get_one::("otel_service_name") { 103 | Ok(name) => name, 104 | Err(error) => { 105 | fatal!("Error while parsing otel_service_name flag: {error}"); 106 | } 107 | }; 108 | let otel_service_name = match otel_service_name_args { 109 | Some(name) => name.clone(), 110 | None => settings.otel_service_name.clone(), 111 | }; 112 | 113 | let otel_layer = if otel_api_key.trim().is_empty() { 114 | info!("No OTEL API key provided; skipping telemetry"); 115 | // Build a no-export provider so nothing is sent 116 | let provider = SdkLoggerProvider::builder().build(); 117 | OpenTelemetryTracingBridge::new(&provider) 118 | } else { 119 | info!("OTEL API key provided; enabling telemetry"); 120 | let mut otel_headers: HashMap = HashMap::new(); 121 | otel_headers.insert("X-Api-Key".to_string(), otel_api_key.to_string()); 122 | let url: String = otel_endpoint.to_owned() + "logs"; 123 | 124 | let identifier_resource = Resource::builder() 125 | .with_attribute(opentelemetry::KeyValue::new( 126 | "service.name", 127 | otel_service_name + " - v" + VERSION, 128 | )) 129 | .build(); 130 | 131 | let otel_exporters = LogExporter::builder() 132 | .with_http() 133 | .with_endpoint(url) 134 | .with_protocol(Protocol::HttpBinary) 135 | .with_headers(otel_headers) 136 | .build() 137 | .expect("Failed to create OTEL log expoter"); 138 | 139 | let otel_provider = SdkLoggerProvider::builder() 140 | .with_batch_exporter(otel_exporters) 141 | .with_resource(identifier_resource) 142 | .build(); 143 | 144 | OpenTelemetryTracingBridge::new(&otel_provider) 145 | }; 146 | 147 | let _guards = log::new(log_level.as_str(), otel_layer); 148 | info!("Initialised logger with log level {log_level}"); 149 | 150 | match matches.subcommand() { 151 | Some(("miner", cmd)) => cli::miner::exec(cmd, &settings)?, 152 | Some(("validator", cmd)) => cli::validator::exec(cmd, &settings)?, 153 | Some(("apikey", cmd)) => cli::apikey_manager::handle_command(cmd)?, 154 | _ => unreachable!(), 155 | } 156 | 157 | Ok(()) 158 | } 159 | -------------------------------------------------------------------------------- /crates/storb_cli/src/cli/apikey_manager.rs: -------------------------------------------------------------------------------- 1 | use std::path::PathBuf; 2 | 3 | use anyhow::Result; 4 | use chrono::{Duration, Utc}; 5 | use clap::{arg, Command}; 6 | use storb_validator::apikey::{ApiKeyConfig, ApiKeyManager}; 7 | use tokio::runtime::Runtime; 8 | 9 | use crate::config::Settings; 10 | 11 | pub fn cli() -> Command { 12 | Command::new("apikey") 13 | .about("Manage API keys for validator access") 14 | .arg( 15 | arg!(--"api-keys-db" "Path to the API keys database") 16 | .required(false) 17 | .global(true) 18 | .id("api-keys-db"), // Add this line to set the argument ID 19 | ) 20 | .subcommand( 21 | Command::new("create") 22 | .about("Create a new API key") 23 | .arg(arg!(-n --name "Name for the API key")) 24 | .arg( 25 | arg!(-e --expires "Expiration time in days") 26 | .required(false) 27 | .value_parser(clap::value_parser!(u64)), 28 | ) 29 | .arg( 30 | arg!(-r --"rate-limit" "Rate limit in requests per minute") 31 | .required(false) 32 | .value_parser(clap::value_parser!(u32)), 33 | ) 34 | .arg( 35 | arg!(-u --"upload-limit" "Maximum upload bytes") 36 | .required(false) 37 | .value_parser(clap::value_parser!(u64)), 38 | ) 39 | .arg( 40 | arg!(-d --"download-limit" "Maximum download bytes") 41 | .required(false) 42 | .value_parser(clap::value_parser!(u64)), 43 | ), 44 | ) 45 | .subcommand(Command::new("list").about("List all API keys")) 46 | .subcommand( 47 | Command::new("delete") 48 | .about("Delete an API key") 49 | .arg(arg!( "API key to delete")), 50 | ) 51 | } 52 | 53 | pub fn handle_command(matches: &clap::ArgMatches) -> Result<()> { 54 | // Create a new tokio runtime 55 | let rt = Runtime::new()?; 56 | 57 | // create new API key manager with the database path from args 58 | let db_path = matches 59 | .get_one::("api-keys-db") 60 | .map(PathBuf::from) 61 | .unwrap_or_else(|| { 62 | let settings = Settings::new(None).expect("Failed to load settings"); 63 | PathBuf::from(&settings.validator.api_keys_db) 64 | }); 65 | let api_key_manager = ApiKeyManager::new(db_path).expect("Failed to create API key manager"); 66 | 67 | match matches.subcommand() { 68 | Some(("create", create_matches)) => { 69 | let name = create_matches.get_one::("name").unwrap(); 70 | let expires_days = create_matches.get_one::("expires"); 71 | let rate_limit = create_matches.get_one::("rate-limit").copied(); 72 | let upload_limit = create_matches.get_one::("upload-limit").copied(); 73 | let download_limit = create_matches.get_one::("download-limit").copied(); 74 | 75 | let expires_at = expires_days.map(|days| Utc::now() + Duration::days(*days as i64)); 76 | 77 | let key = rt.block_on(api_key_manager.create_key(ApiKeyConfig { 78 | name: name.clone(), 79 | expires_at, 80 | rate_limit, 81 | upload_limit, 82 | download_limit, 83 | }))?; 84 | 85 | println!("✨ Created API key: {}", key.key); 86 | println!("Name: {}", key.name); 87 | if let Some(expires) = key.expires_at { 88 | println!("Expires: {}", expires.format("%Y-%m-%d %H:%M:%S UTC")); 89 | } 90 | if let Some(rate) = key.rate_limit { 91 | println!("Rate limit: {} requests/minute", rate); 92 | } 93 | if let Some(limit) = key.upload_limit { 94 | println!("Upload limit: {} bytes", limit); 95 | } 96 | if let Some(limit) = key.download_limit { 97 | println!("Download limit: {} bytes", limit); 98 | } 99 | } 100 | Some(("list", _)) => { 101 | let keys = rt.block_on(api_key_manager.list_keys())?; 102 | if keys.is_empty() { 103 | println!("No API keys found"); 104 | return Ok(()); 105 | } 106 | 107 | println!("Found {} API keys:", keys.len()); 108 | for key in keys { 109 | println!("\n🔑 Key: {}", key.key); 110 | println!(" Name: {}", key.name); 111 | println!( 112 | " Created: {}", 113 | key.created_at.format("%Y-%m-%d %H:%M:%S UTC") 114 | ); 115 | if let Some(expires) = key.expires_at { 116 | println!(" Expires: {}", expires.format("%Y-%m-%d %H:%M:%S UTC")); 117 | } 118 | if let Some(rate) = key.rate_limit { 119 | println!(" Rate limit: {} requests/minute", rate); 120 | } 121 | println!(" Upload used: {} bytes", key.upload_used); 122 | if let Some(limit) = key.upload_limit { 123 | println!(" Upload limit: {} bytes", limit); 124 | } 125 | println!(" Download used: {} bytes", key.download_used); 126 | if let Some(limit) = key.download_limit { 127 | println!(" Download limit: {} bytes", limit); 128 | } 129 | } 130 | } 131 | Some(("delete", delete_matches)) => { 132 | let key = delete_matches.get_one::("KEY").unwrap(); 133 | if rt.block_on(api_key_manager.delete_key(key))? { 134 | println!("✅ API key deleted successfully"); 135 | } else { 136 | println!("❌ API key not found"); 137 | } 138 | } 139 | _ => unreachable!(), 140 | } 141 | 142 | Ok(()) 143 | } 144 | -------------------------------------------------------------------------------- /crates/storb_miner/src/store.rs: -------------------------------------------------------------------------------- 1 | use std::fs; 2 | use std::io::Error; 3 | use std::path::{Path, PathBuf}; 4 | 5 | use base::piece_hash::PieceHashStr; 6 | use tokio::fs::File; 7 | use tokio::io::{AsyncReadExt, AsyncWriteExt}; 8 | use tracing::info; 9 | 10 | /// Object store to store the piece data 11 | #[derive(Clone)] 12 | pub struct ObjectStore { 13 | path: PathBuf, 14 | } 15 | 16 | impl ObjectStore { 17 | /// Create a new object store 18 | pub fn new>(store_dir: P) -> Result { 19 | let mut path = PathBuf::new(); 20 | path = path.join(store_dir); 21 | 22 | let object_store = ObjectStore { path: path.clone() }; 23 | 24 | if fs::exists(&path).expect("Could not check if file exists or not") { 25 | return Ok(object_store); 26 | } 27 | 28 | fs::create_dir(&path)?; 29 | for i in 0..=0xFF { 30 | fs::create_dir(path.join(format!("{i:02x}")))?; 31 | } 32 | 33 | Ok(object_store) 34 | } 35 | 36 | /// Read piece data in bytes from the store 37 | pub async fn read(&self, piece_hash: &PieceHashStr) -> Result, Error> { 38 | let path = self.path.join(&piece_hash[0..2]).join(&piece_hash[2..]); 39 | 40 | let mut buffer = vec![]; 41 | let mut f = File::open(path).await?; 42 | f.read_to_end(&mut buffer).await?; 43 | 44 | Ok(buffer) 45 | } 46 | 47 | /// Write piece data to the store 48 | pub async fn write(&self, piece_hash: &PieceHashStr, data: &Vec) -> Result { 49 | info!("Writing piece {piece_hash} to store"); 50 | 51 | let folder = self.path.join(&piece_hash[0..2]); 52 | if !fs::exists(&folder).expect("Could not check if file exists or not") { 53 | fs::create_dir(&folder)?; 54 | } 55 | 56 | let path = folder.join(&piece_hash[2..]); 57 | 58 | let mut file = File::create(&path).await?; 59 | file.write_all(data.as_slice()).await?; 60 | 61 | Ok(path) 62 | } 63 | 64 | /// Get a reference to the path of the object store 65 | pub fn path(&self) -> &PathBuf { 66 | &self.path 67 | } 68 | } 69 | 70 | #[cfg(test)] 71 | mod tests { 72 | use std::fs; 73 | use std::io::Error; 74 | 75 | use base::piece_hash::PieceHashStr; 76 | use rand::Rng; 77 | 78 | use crate::store::ObjectStore; 79 | 80 | fn generate_rand_string(len: usize) -> String { 81 | let charset = b"0123456789abcdef"; 82 | let mut rng = rand::thread_rng(); 83 | let random_string: String = (0..len) 84 | .map(|_| charset[rng.gen_range(0..charset.len())] as char) 85 | .collect(); 86 | random_string 87 | } 88 | 89 | fn generate_rand_bytes(len: usize) -> Vec { 90 | let mut rng = rand::thread_rng(); 91 | let random_bytes: Vec = (0..len).map(|_| rng.gen()).collect(); 92 | random_bytes 93 | } 94 | 95 | fn generate_rand_hash() -> String { 96 | blake3::hash(generate_rand_string(64).as_bytes()) 97 | .to_hex() 98 | .to_string() 99 | } 100 | 101 | fn create_test_dir_str() -> String { 102 | format!("test_dir_{}", generate_rand_string(10)) 103 | } 104 | 105 | fn cleanup_test_dir(path: &str) -> Result<(), Error> { 106 | fs::remove_dir_all(path) 107 | } 108 | 109 | #[test] 110 | fn test_creating_object_store() { 111 | let test_dir = create_test_dir_str(); 112 | let store = ObjectStore::new(&test_dir).unwrap(); 113 | 114 | assert!(fs::exists(&store.path).unwrap()); 115 | cleanup_test_dir(&test_dir).unwrap(); 116 | } 117 | 118 | #[test] 119 | fn test_creating_object_store_existing_dir() { 120 | let test_dir = create_test_dir_str(); 121 | let _ = ObjectStore::new(&test_dir).unwrap(); 122 | // Ensure creating an object store where directory already exists works 123 | let store = ObjectStore::new(&test_dir).unwrap(); 124 | 125 | assert!(fs::exists(&store.path).unwrap()); 126 | cleanup_test_dir(&test_dir).unwrap(); 127 | } 128 | 129 | #[tokio::test] 130 | async fn test_create_file() { 131 | let test_dir = create_test_dir_str(); 132 | let store = ObjectStore::new(&test_dir).unwrap(); 133 | let piece_hash = PieceHashStr::new(generate_rand_hash()).unwrap(); 134 | let data = generate_rand_bytes(1024); 135 | 136 | let piece_path = store.write(&piece_hash, &data).await.unwrap(); 137 | 138 | assert!(fs::exists(store.path()).unwrap()); 139 | assert!(fs::exists(&piece_path).unwrap()); 140 | cleanup_test_dir(&test_dir).unwrap(); 141 | } 142 | 143 | #[tokio::test] 144 | async fn test_read_file() { 145 | let test_dir = create_test_dir_str(); 146 | let store = ObjectStore::new(&test_dir).unwrap(); 147 | let piece_hash = PieceHashStr::new(generate_rand_hash()).unwrap(); 148 | let data = generate_rand_bytes(1024); 149 | 150 | let piece_path = store.write(&piece_hash, &data).await.unwrap(); 151 | 152 | assert!(fs::exists(store.path()).unwrap()); 153 | assert!(fs::exists(&piece_path).unwrap()); 154 | 155 | let read_data = store.read(&piece_hash).await.unwrap(); 156 | assert!(data == read_data); 157 | 158 | cleanup_test_dir(&test_dir).unwrap(); 159 | } 160 | 161 | #[tokio::test] 162 | async fn test_overwrite_existing_file() { 163 | let test_dir = create_test_dir_str(); 164 | let store = ObjectStore::new(&test_dir).unwrap(); 165 | let piece_hash = PieceHashStr::new(generate_rand_hash()).unwrap(); 166 | let data = generate_rand_bytes(1024); 167 | let data2 = generate_rand_bytes(1024); 168 | 169 | let _ = store.write(&piece_hash, &data).await.unwrap(); 170 | // Call write once again to overwrite the file 171 | let piece_path = store.write(&piece_hash, &data2).await.unwrap(); 172 | 173 | assert!(fs::exists(store.path()).unwrap()); 174 | assert!(fs::exists(&piece_path).unwrap()); 175 | 176 | let read_data = store.read(&piece_hash).await.unwrap(); 177 | assert!(data != read_data); 178 | assert!(data2 == read_data); 179 | 180 | cleanup_test_dir(&test_dir).unwrap(); 181 | } 182 | } 183 | -------------------------------------------------------------------------------- /crates/storb_validator/src/metadata/sync.rs: -------------------------------------------------------------------------------- 1 | use std::sync::Arc; 2 | 3 | use base::utils::multiaddr_to_socketaddr; 4 | use libp2p::Multiaddr; 5 | use thiserror::Error; 6 | use tracing::{debug, info, warn}; 7 | 8 | use crate::{ 9 | constants::TAO_IN_RAO, 10 | metadata::{db::MetadataDB, models::CrSqliteChanges}, 11 | validator::Validator, 12 | }; 13 | 14 | #[derive(Debug, Error)] 15 | pub enum SyncError { 16 | #[error("Failed to get db changes from validator: {0}")] 17 | GetDBChangesError(String), 18 | } 19 | 20 | async fn get_crsql_changes( 21 | address: Multiaddr, 22 | site_id: Vec, 23 | min_db_version: u64, 24 | ) -> Result, SyncError> { 25 | // // We need to specify the site_id to exclude our own changes 26 | // get ipv4 address and port from the multiaddr 27 | let socket_addr = multiaddr_to_socketaddr(&address) 28 | .ok_or_else(|| SyncError::GetDBChangesError("Invalid multiaddr".to_string()))?; 29 | let ip = socket_addr.ip(); 30 | let port = socket_addr.port(); 31 | let addr = format!("{}:{}", ip, port); 32 | 33 | let endpoint = format!( 34 | "http://{}/db_changes?site_id_exclude={}&min_db_version={}", 35 | addr, 36 | hex::encode(&site_id), 37 | min_db_version 38 | ); 39 | let url = reqwest::Url::parse(&endpoint).map_err(|e| { 40 | warn!("Failed to parse URL: {:?}", e); 41 | SyncError::GetDBChangesError(format!("Invalid URL: {:?}", e)) 42 | })?; 43 | let response = reqwest::get(url).await.map_err(|e| { 44 | warn!("Failed to get changes from {}: {:?}", addr, e); 45 | SyncError::GetDBChangesError(format!("Request failed: {:?}", e)) 46 | })?; 47 | if !response.status().is_success() { 48 | warn!( 49 | "Failed to get changes from {}: {:?}", 50 | endpoint, 51 | response.status() 52 | ); 53 | return Err(SyncError::GetDBChangesError(format!( 54 | "Request failed with status: {:?}", 55 | response.status() 56 | ))); 57 | } 58 | 59 | // deserialize the response into Vec 60 | let changes: Vec = 61 | bincode::deserialize(&response.bytes().await.map_err(|e| { 62 | warn!("Failed to read response bytes: {:?}", e); 63 | SyncError::GetDBChangesError(format!("Could not read response bytes: {:?}", e)) 64 | })?) 65 | .map_err(|e| { 66 | warn!("Failed to deserialize CrSqliteChanges: {:?}", e); 67 | SyncError::GetDBChangesError(format!("Failed to deserialize response: {:?}", e)) 68 | })?; 69 | info!( 70 | "Successfully retrieved {} changes from {}", 71 | changes.len(), 72 | addr 73 | ); 74 | Ok(changes) 75 | } 76 | 77 | pub async fn sync_metadata_db(validator: Arc) -> Result<(), SyncError> { 78 | info!("Syncing MetadataDB..."); 79 | let neuron = validator.neuron.read().await; 80 | 81 | let validator_uid = neuron 82 | .local_node_info 83 | .uid 84 | .ok_or_else(|| SyncError::GetDBChangesError("Local node UID is not set".to_string()))?; 85 | let neurons = neuron.neurons.read().await; 86 | let address_book = neuron.address_book.clone(); 87 | let sync_stake_threshold = validator.config.sync_stake_threshold; 88 | // get the site id of our validator from the metadatadb 89 | let site_id = match MetadataDB::get_site_id(&validator.metadatadb_sender).await { 90 | Ok(id) => id, 91 | Err(e) => { 92 | warn!("Failed to get site ID: {:?}", e); 93 | return Err(SyncError::GetDBChangesError(format!( 94 | "Failed to get site ID: {:?}", 95 | e 96 | ))); 97 | } 98 | }; 99 | 100 | // get the db version of our validator 101 | let min_db_version = match MetadataDB::get_db_version(&validator.metadatadb_sender).await { 102 | Ok(version) => version, 103 | Err(e) => { 104 | warn!("Failed to get DB version: {:?}", e); 105 | return Err(SyncError::GetDBChangesError(format!( 106 | "Failed to get DB version: {:?}", 107 | e 108 | ))); 109 | } 110 | }; 111 | 112 | // Iterate through the neurons 113 | for neuron_info in neurons.iter() { 114 | // Skip self 115 | if neuron_info.uid == validator_uid { 116 | continue; 117 | } 118 | 119 | // Check to see if the neuron stake is above the stake streshold 120 | let stake = neuron_info 121 | .stake 122 | .iter() 123 | .map(|(_, stake)| stake.0) 124 | .sum::() as f64 125 | / TAO_IN_RAO; 126 | debug!( 127 | "Neuron {} has stake {}, threshold is {}", 128 | neuron_info.uid, stake, sync_stake_threshold 129 | ); 130 | if stake < sync_stake_threshold as f64 { 131 | debug!("skipping neuron {}", neuron_info.uid); 132 | continue; 133 | } 134 | // If it does have enough stake, we can sync the metadata 135 | // We first get the http address for the neuron's api 136 | let node_info = match address_book.get(&neuron_info.uid) { 137 | Some(addr) => addr, 138 | None => { 139 | warn!("No node info found for neuron {}", neuron_info.uid); 140 | continue; 141 | } 142 | }; 143 | 144 | let address = match &node_info.http_address { 145 | Some(addr) => addr.clone(), 146 | None => { 147 | warn!("No HTTP address found for neuron {}", neuron_info.uid); 148 | continue; 149 | } 150 | }; 151 | 152 | // Use the address to ask the neuron for changes in its db from its /db_changes endpoint 153 | let db_changes = match get_crsql_changes(address, site_id.clone(), min_db_version).await { 154 | Ok(changes) => changes, 155 | Err(e) => { 156 | warn!( 157 | "Failed to get changes for neuron {}: {:?}", 158 | neuron_info.uid, e 159 | ); 160 | continue; 161 | } 162 | }; 163 | 164 | // Apply the changes to the local metadata database with MetadataDB::insert_crsqlite_changes 165 | if let Err(e) = 166 | MetadataDB::insert_crsqlite_changes(&validator.metadatadb_sender, db_changes).await 167 | { 168 | warn!( 169 | "Failed to insert changes for neuron {}: {:?}", 170 | neuron_info.uid, e 171 | ); 172 | } else { 173 | info!( 174 | "Successfully synced metadata for neuron {}", 175 | neuron_info.uid 176 | ); 177 | } 178 | } 179 | Ok(()) 180 | } 181 | -------------------------------------------------------------------------------- /crates/storb_validator/src/metadata/models.rs: -------------------------------------------------------------------------------- 1 | use base::{ 2 | piece::{ChunkHash, InfoHash, PieceHash, PieceType}, 3 | NodeUID, 4 | }; 5 | use chrono::{DateTime, Utc}; 6 | use crabtensor::{sign::KeypairSignature, AccountId}; 7 | use rusqlite::types::ValueRef; 8 | use rusqlite::{ 9 | types::{FromSql, FromSqlResult, ToSqlOutput}, 10 | ToSql, 11 | }; 12 | use serde::{Deserialize, Serialize}; 13 | use subxt::ext::codec::Compact; 14 | use tracing::debug; 15 | 16 | // Newtype wrapper for DateTime 17 | #[derive(Debug, Clone, PartialEq, PartialOrd, Serialize, Deserialize)] 18 | pub struct SqlDateTime(pub DateTime); 19 | 20 | impl FromSql for SqlDateTime { 21 | // from datetime to sql datetime, to the nanosecond 22 | fn column_result(value: rusqlite::types::ValueRef<'_>) -> FromSqlResult { 23 | let timestamp = i64::column_result(value)?; 24 | Option::from(DateTime::::from_timestamp_nanos(timestamp)) 25 | .map(SqlDateTime) 26 | .ok_or(rusqlite::types::FromSqlError::InvalidType) 27 | } 28 | } 29 | 30 | impl ToSql for SqlDateTime { 31 | fn to_sql(&self) -> rusqlite::Result> { 32 | let timestamp = 33 | self.0 34 | .timestamp_nanos_opt() 35 | .ok_or(rusqlite::Error::ToSqlConversionFailure( 36 | "Failed to convert DateTime to i64".into(), 37 | ))?; 38 | Ok(ToSqlOutput::from(timestamp)) 39 | } 40 | } 41 | 42 | /// Represents a chunk entry 43 | /// 44 | /// Contains metadata for a chunk including its hash, associated piece hashes, 45 | #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] 46 | pub struct ChunkValue { 47 | pub chunk_hash: ChunkHash, 48 | pub k: u64, 49 | pub m: u64, 50 | pub chunk_size: u64, 51 | pub padlen: u64, 52 | pub original_chunk_size: u64, 53 | } 54 | 55 | /// Represents a tracker entry with identity and nonce for replay protection 56 | #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] 57 | pub struct InfohashValue { 58 | pub infohash: InfoHash, 59 | pub name: String, 60 | pub length: u64, 61 | pub chunk_size: u64, 62 | pub chunk_count: u64, 63 | pub owner_account_id: SqlAccountId, 64 | pub creation_timestamp: DateTime, 65 | pub signature: KeypairSignature, 66 | } 67 | 68 | impl InfohashValue { 69 | /// Verify that the signature is valid for this infohash value 70 | pub fn verify_signature(&self, nonce: &[u8; 32]) -> bool { 71 | let message = self.get_signature_message(nonce); 72 | // Log info 73 | debug!( 74 | "Verifying signature for infohash {} with nonce {} with signature {} with account id {}", 75 | hex::encode(self.infohash.0), 76 | hex::encode(nonce), 77 | hex::encode(self.signature.0), 78 | self.owner_account_id.0, 79 | ); 80 | 81 | crabtensor::sign::verify_signature(&self.owner_account_id.0, &self.signature, &message) 82 | } 83 | 84 | /// Get the message that should be signed for this infohash 85 | pub fn get_signature_message(&self, nonce: &[u8; 32]) -> Vec { 86 | let mut message = Vec::new(); 87 | message.extend_from_slice(nonce); // Include nonce in signature 88 | message 89 | } 90 | } 91 | 92 | /// Represents a piece entry 93 | /// 94 | /// Contains the piece hash, indices indicating its position, its type, 95 | #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] 96 | pub struct PieceValue { 97 | pub piece_hash: PieceHash, 98 | pub piece_size: u64, 99 | pub piece_type: PieceType, 100 | // TODO: shouldn't this be a set instead of a vector? 101 | pub miners: Vec>, 102 | } 103 | 104 | /// Represents a piece challenge history entry 105 | #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] 106 | pub struct PieceChallengeHistory { 107 | pub piece_repair_hash: [u8; 32], 108 | pub piece_hash: PieceHash, 109 | pub chunk_hash: ChunkHash, 110 | pub validator_id: Compact, 111 | pub timestamp: DateTime, 112 | pub signature: KeypairSignature, 113 | } 114 | 115 | /// Represents a chunk challenge history entry 116 | #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] 117 | pub struct ChunkChallengeHistory { 118 | pub challenge_hash: [u8; 32], 119 | pub chunk_hash: ChunkHash, 120 | pub validator_id: Compact, 121 | pub miners_challenged: Vec>, 122 | pub miners_successful: Vec>, 123 | // reference to the piece repair hash if any 124 | pub piece_repair_hash: [u8; 32], 125 | pub timestamp: DateTime, 126 | pub signature: KeypairSignature, 127 | } 128 | 129 | #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] 130 | pub enum CrSqliteValue { 131 | Null, 132 | Integer(i64), 133 | Real(f64), 134 | Text(String), 135 | Blob(Vec), 136 | } 137 | 138 | impl From> for CrSqliteValue { 139 | fn from(value_ref: ValueRef<'_>) -> Self { 140 | match value_ref { 141 | ValueRef::Null => CrSqliteValue::Null, 142 | ValueRef::Integer(i) => CrSqliteValue::Integer(i), 143 | ValueRef::Real(f) => CrSqliteValue::Real(f), 144 | ValueRef::Text(t) => CrSqliteValue::Text(String::from_utf8_lossy(t).to_string()), 145 | ValueRef::Blob(b) => CrSqliteValue::Blob(b.to_vec()), 146 | } 147 | } 148 | } 149 | 150 | // write impl for converting CrSqliteValue to a sqlite value 151 | impl ToSql for CrSqliteValue { 152 | fn to_sql(&self) -> rusqlite::Result> { 153 | match self { 154 | CrSqliteValue::Null => Ok(ToSqlOutput::from(rusqlite::types::Null)), 155 | CrSqliteValue::Integer(i) => Ok(ToSqlOutput::from(*i)), 156 | CrSqliteValue::Real(f) => Ok(ToSqlOutput::from(*f)), 157 | CrSqliteValue::Text(t) => Ok(ToSqlOutput::from(t.clone())), 158 | CrSqliteValue::Blob(b) => Ok(ToSqlOutput::from(b.clone())), 159 | } 160 | } 161 | } 162 | 163 | /// Represents the changes in the SQLite database 164 | #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] 165 | pub struct CrSqliteChanges { 166 | pub table: String, 167 | pub pk: CrSqliteValue, 168 | pub cid: String, 169 | pub val: CrSqliteValue, 170 | pub col_version: u64, 171 | pub db_version: u64, 172 | pub site_id: Vec, 173 | pub cl: u64, 174 | pub seq: u64, 175 | } 176 | 177 | // Newtype wrapper for AccountId 178 | #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] 179 | pub struct SqlAccountId(pub AccountId); 180 | 181 | impl From<&AccountId> for SqlAccountId { 182 | fn from(id: &AccountId) -> Self { 183 | SqlAccountId(id.clone()) 184 | } 185 | } 186 | 187 | impl AsRef for SqlAccountId { 188 | fn as_ref(&self) -> &AccountId { 189 | &self.0 190 | } 191 | } 192 | 193 | impl ToSql for SqlAccountId { 194 | fn to_sql(&self) -> rusqlite::Result> { 195 | Ok(ToSqlOutput::from(&self.0 .0[..])) 196 | } 197 | } 198 | 199 | impl FromSql for SqlAccountId { 200 | fn column_result(value: rusqlite::types::ValueRef<'_>) -> FromSqlResult { 201 | match value { 202 | ValueRef::Blob(b) => { 203 | if b.len() == 32 { 204 | let mut array = [0u8; 32]; 205 | array.copy_from_slice(b); 206 | Ok(SqlAccountId(AccountId::from(array))) 207 | } else { 208 | Err(rusqlite::types::FromSqlError::InvalidBlobSize { 209 | expected_size: 32, 210 | blob_size: b.len(), 211 | }) 212 | } 213 | } 214 | _ => Err(rusqlite::types::FromSqlError::InvalidType), 215 | } 216 | } 217 | } 218 | 219 | // Add nonce management models 220 | #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] 221 | pub struct NonceRequest { 222 | pub account_id: AccountId, 223 | } 224 | 225 | #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] 226 | pub struct NonceResponse { 227 | pub nonce: [u8; 32], 228 | pub timestamp: DateTime, 229 | } 230 | -------------------------------------------------------------------------------- /crates/storb_miner/src/routes.rs: -------------------------------------------------------------------------------- 1 | use std::collections::HashMap; 2 | 3 | use axum::body::Bytes; 4 | use axum::extract::{self, Query}; 5 | use axum::http::StatusCode; 6 | use axum::response::IntoResponse; 7 | use base::piece_hash::{piecehash_str_to_bytes, PieceHashStr}; 8 | use base::verification::HandshakePayload; 9 | use crabtensor::sign::verify_signature; 10 | use tracing::{debug, error, info, trace}; 11 | 12 | use crate::MinerState; 13 | 14 | /// Router function to get information on a given node 15 | #[utoipa::path( 16 | get, 17 | path = "/info", 18 | responses( 19 | (status = 200, description = "Successfully got node info", body = String), 20 | (status = 500, description = "Internal server error", body = String) 21 | ), 22 | tag = "Info" 23 | )] 24 | pub async fn node_info( 25 | state: axum::extract::State, 26 | ) -> Result)> { 27 | trace!("Got node info req"); 28 | let local_node_info = state.local_node_info.clone(); 29 | let serialized_local_node_info = bincode::serialize(&local_node_info).map_err(|e| { 30 | error!("Error while deserialising local node info: {e}"); 31 | ( 32 | StatusCode::INTERNAL_SERVER_ERROR, 33 | bincode::serialize("An internal server error occurred").unwrap_or_default(), 34 | ) 35 | })?; 36 | 37 | Ok((StatusCode::OK, serialized_local_node_info)) 38 | } 39 | 40 | /// Handshake verification between a miner and a validator 41 | #[utoipa::path( 42 | post, 43 | path = "/handshake", 44 | responses( 45 | (status = 200, description = "Successfully shaken hands", body = String), 46 | (status = 401, description = "Unauthorized", body = String), 47 | (status = 500, description = "Internal server error", body = String) 48 | ), 49 | tag = "Handshake" 50 | )] 51 | pub async fn handshake( 52 | state: extract::State, 53 | bytes: Bytes, 54 | ) -> Result { 55 | info!("Got handshake request"); 56 | 57 | let payload = bincode::deserialize::(&bytes).map_err(|err| { 58 | error!("Error while deserializing bytes: {err}"); 59 | StatusCode::INTERNAL_SERVER_ERROR 60 | })?; 61 | let verified = verify_signature( 62 | &payload.message.validator.account_id, 63 | &payload.signature, 64 | &payload.message, 65 | ); 66 | 67 | let address_book = state.clone().miner.neuron.read().await.address_book.clone(); 68 | 69 | let validator_info = address_book 70 | .get(&payload.message.validator.uid.clone()) 71 | .ok_or_else(|| { 72 | error!("Error while getting validator info"); 73 | StatusCode::INTERNAL_SERVER_ERROR 74 | })? 75 | .clone(); 76 | let validator_hotkey = validator_info.neuron_info.hotkey.clone(); 77 | 78 | if !verified || validator_hotkey != payload.message.validator.account_id { 79 | return Ok(StatusCode::UNAUTHORIZED); 80 | } 81 | 82 | Ok(StatusCode::OK) 83 | } 84 | 85 | /// Get a file piece. 86 | #[utoipa::path( 87 | get, 88 | path = "/piece", 89 | responses( 90 | (status = 200, description = "Piece fetched successfully", body = String), 91 | (status = 400, description = "Missing piecehash parameter", body = String), 92 | (status = 401, description = "Unauthorized", body = String), 93 | (status = 500, description = "Internal server error during piece retrieval", body = String) 94 | ), 95 | params( 96 | ("piecehash" = String, Path, description = "The piecehash of the piece to retrieve."), 97 | ("handshake" = String, Path, description = "The handshake payload containing the signature to verify the validator."), 98 | ), 99 | tag = "Piece Download" 100 | )] 101 | pub async fn get_piece( 102 | state: extract::State, 103 | Query(params): Query>, 104 | ) -> Result)> { 105 | info!("Get piece request"); 106 | 107 | // Verify that the signature is valid. If not, the miner can reject the request from the validator 108 | let handshake = params.get("handshake").ok_or_else(|| { 109 | ( 110 | StatusCode::BAD_REQUEST, 111 | bincode::serialize("Missing handshake parameter").unwrap_or_default(), 112 | ) 113 | })?; 114 | let handshake_data = hex::decode(handshake).map_err(|err| { 115 | error!("Error while deserializing hex string: {err}"); 116 | ( 117 | StatusCode::INTERNAL_SERVER_ERROR, 118 | bincode::serialize("An internal server error occurred").unwrap_or_default(), 119 | ) 120 | })?; 121 | let payload = bincode::deserialize::(&handshake_data).map_err(|err| { 122 | error!("Error while deserializing bytes: {err}"); 123 | ( 124 | StatusCode::INTERNAL_SERVER_ERROR, 125 | bincode::serialize("An internal server error occurred").unwrap_or_default(), 126 | ) 127 | })?; 128 | let verified = verify_signature( 129 | &payload.message.validator.account_id, 130 | &payload.signature, 131 | &payload.message, 132 | ); 133 | 134 | let address_book = state.clone().miner.neuron.read().await.address_book.clone(); 135 | 136 | let validator_info = address_book 137 | .get(&payload.message.validator.uid.clone()) 138 | .ok_or_else(|| { 139 | error!("Error while getting validator info"); 140 | ( 141 | StatusCode::INTERNAL_SERVER_ERROR, 142 | bincode::serialize("An internal server error occurred").unwrap_or_default(), 143 | ) 144 | })? 145 | .clone(); 146 | let validator_hotkey = validator_info.neuron_info.hotkey.clone(); 147 | 148 | if !verified || validator_hotkey != payload.message.validator.account_id { 149 | return Ok(( 150 | StatusCode::UNAUTHORIZED, 151 | bincode::serialize("Signature verification failed. Unauthorized access") 152 | .unwrap_or_default(), 153 | )); 154 | } 155 | 156 | debug!("Signature verification successful:"); 157 | debug!("{:?}", payload); 158 | 159 | // Continue if the signature is valid: 160 | 161 | let piece_hash_query = params.get("piecehash").ok_or_else(|| { 162 | ( 163 | StatusCode::BAD_REQUEST, 164 | bincode::serialize("Missing piecehash parameter").unwrap_or_default(), 165 | ) 166 | })?; 167 | 168 | info!("Piece hash: {}", piece_hash_query); 169 | let store = state.clone().object_store.clone(); 170 | let object_store = store.lock().await; 171 | // If an error occurs during piece hash decode, it is bound to be a user error. 172 | let piece_hash = PieceHashStr::new(piece_hash_query.clone()).map_err(|_| { 173 | ( 174 | StatusCode::BAD_REQUEST, 175 | bincode::serialize("The piecehash is invalid. Was it the correct length?") 176 | .unwrap_or_default(), 177 | ) 178 | })?; 179 | 180 | let data = object_store.read(&piece_hash).await.map_err(|err| { 181 | error!("Failed to retrieve data for piecehash {piece_hash}: {err}"); 182 | ( 183 | StatusCode::INTERNAL_SERVER_ERROR, 184 | bincode::serialize("An internal server error occurred").unwrap_or_default(), 185 | ) 186 | })?; 187 | 188 | let serialised_data = base::piece::serialise_piece_response(&base::piece::PieceResponse { 189 | piece_hash: piecehash_str_to_bytes(&piece_hash).map_err(|err| { 190 | error!("Failed to convert piece hash to raw bytes: {err}"); 191 | ( 192 | StatusCode::INTERNAL_SERVER_ERROR, 193 | bincode::serialize("An internal server error occurred").unwrap_or_default(), 194 | ) 195 | })?, 196 | piece_data: data, 197 | }) 198 | .map_err(|err| { 199 | error!("Failed to serialise piece data (piecehash={piece_hash}): {err}"); 200 | ( 201 | StatusCode::INTERNAL_SERVER_ERROR, 202 | bincode::serialize("An internal server error occurred").unwrap_or_default(), 203 | ) 204 | })?; 205 | 206 | Ok((StatusCode::OK, serialised_data)) 207 | } 208 | -------------------------------------------------------------------------------- /crates/storb_cli/src/cli/args.rs: -------------------------------------------------------------------------------- 1 | //! Contains arguments that are used by multiple subcommands, e.g. 2 | //! `miner` and `validator`. 3 | 4 | use std::str::FromStr; 5 | 6 | use anyhow::Result; 7 | use base::version::Version; 8 | use base::{BaseNeuronConfig, NeuronConfig, SubtensorConfig}; 9 | use clap::{value_parser, Arg, ArgAction, ArgMatches}; 10 | use expanduser::expanduser; 11 | 12 | use crate::{config::Settings, get_config_value}; 13 | 14 | pub fn common_args() -> Vec { 15 | vec![ 16 | Arg::new("config") 17 | .long("config-file") 18 | .value_name("file") 19 | .help("Use a custom config file") 20 | .action(ArgAction::Set) 21 | .global(true), 22 | Arg::new("netuid") 23 | .long("netuid") 24 | .value_name("id") 25 | .value_parser(value_parser!(u16)) 26 | .help("Subnet netuid") 27 | .action(ArgAction::Set) 28 | .global(true), 29 | Arg::new("external_ip") 30 | .long("external-ip") 31 | .value_name("ip") 32 | .help("External IP") 33 | .action(ArgAction::Set) 34 | .global(true), 35 | Arg::new("api_port") 36 | .long("api-port") 37 | .value_name("port") 38 | .value_parser(value_parser!(u16)) 39 | .help("API port for the node") 40 | .action(ArgAction::Set) 41 | .global(true), 42 | Arg::new("quic_port") 43 | .long("quic-port") 44 | .value_name("port") 45 | .value_parser(value_parser!(u16)) 46 | .help("QUIC port for the node") 47 | .action(ArgAction::Set) 48 | .global(true), 49 | Arg::new("post_ip") 50 | .long("post-ip") 51 | .help("Whether to post the IP to the chain or not") 52 | .action(ArgAction::SetTrue) 53 | .global(true), 54 | Arg::new("wallet_path") 55 | .long("wallet-path") 56 | .value_name("wallet") 57 | .help("Path of wallets") 58 | .action(ArgAction::Set) 59 | .global(true), 60 | Arg::new("wallet_name") 61 | .long("wallet-name") 62 | .value_name("wallet") 63 | .help("Name of the Bittensor wallet") 64 | .action(ArgAction::Set) 65 | .global(true), 66 | Arg::new("hotkey_name") 67 | .long("hotkey-name") 68 | .value_name("hotkey") 69 | .help("Hotkey associated with the wallet") 70 | .action(ArgAction::Set) 71 | .global(true), 72 | Arg::new("otel_api_key") 73 | .long("otel-api-key") 74 | .value_name("key") 75 | .help("API key for OpenTelemetry") 76 | .action(ArgAction::Set) 77 | .global(true), 78 | Arg::new("otel_endpoint") 79 | .long("otel-endpoint") 80 | .value_name("endpoint") 81 | .help("Endpoint for OpenTelemetry") 82 | .action(ArgAction::Set) 83 | .global(true), 84 | Arg::new("otel_service_name") 85 | .long("otel-service-name") 86 | .value_name("name") 87 | .help("Service name for OpenTelemetry") 88 | .action(ArgAction::Set) 89 | .global(true), 90 | Arg::new("mock") 91 | .long("mock") 92 | .help("Mock for testing") 93 | .action(ArgAction::SetTrue) 94 | .global(true), 95 | Arg::new("load_old_nodes") 96 | .long("load-old-nodes") 97 | .help("Load old nodes") 98 | .action(ArgAction::SetTrue) 99 | .global(true), 100 | Arg::new("min_stake_threshold") 101 | .long("min-stake-threshold") 102 | .value_name("threshold") 103 | .value_parser(value_parser!(u64)) 104 | .help("Minimum stake threshold") 105 | .action(ArgAction::Set) 106 | .global(true), 107 | Arg::new("log_level") 108 | .long("log-level") 109 | .value_name("level") 110 | .help("Set the log level") 111 | .action(ArgAction::Set) 112 | .global(true), 113 | // Databases 114 | Arg::new("db_file") 115 | .long("db-file") 116 | .value_name("path") 117 | .help("Path to the score database file") 118 | .action(ArgAction::Set) 119 | .global(true), 120 | Arg::new("metadatadb_file") 121 | .long("metadatadb-file") 122 | .value_name("path") 123 | .help("Path to the metadata database file") 124 | .action(ArgAction::Set) 125 | .global(true), 126 | // Subtensor 127 | Arg::new("subtensor.network") 128 | .long("subtensor.network") 129 | .value_name("network") 130 | .help("Subtensor network name") 131 | .action(ArgAction::Set) 132 | .global(true), 133 | Arg::new("subtensor.address") 134 | .long("subtensor.address") 135 | .value_name("address") 136 | .help("Subtensor network address") 137 | .action(ArgAction::Set) 138 | .global(true), 139 | Arg::new("subtensor.insecure") 140 | .long("subtensor.insecure") 141 | .help("Enable insecure connection to subtensor") 142 | .action(ArgAction::SetTrue) 143 | .global(true), 144 | // Neuron 145 | Arg::new("neuron.sync_frequency") 146 | .long("neuron.sync-frequency") 147 | .value_name("frequency") 148 | .value_parser(value_parser!(u64)) 149 | .help("The default sync frequency for nodes") 150 | .action(ArgAction::Set) 151 | .global(true), 152 | ] 153 | } 154 | 155 | pub fn get_neuron_config(args: &ArgMatches, settings: &Settings) -> Result { 156 | let api_port = *get_config_value!(args, "api_port", u16, settings.api_port); 157 | let quic_port = *get_config_value!(args, "quic_port", u16, settings.quic_port); 158 | 159 | let subtensor_config = SubtensorConfig { 160 | network: get_config_value!( 161 | args, 162 | "subtensor.network", 163 | String, 164 | &settings.subtensor.network 165 | ) 166 | .to_string(), 167 | address: get_config_value!( 168 | args, 169 | "subtensor.address", 170 | String, 171 | &settings.subtensor.address 172 | ) 173 | .to_string(), 174 | insecure: args.get_flag("subtensor.insecure") || settings.subtensor.insecure, 175 | }; 176 | 177 | Ok(BaseNeuronConfig { 178 | version: Version::from_str(&settings.version)?, 179 | netuid: *get_config_value!(args, "netuid", u16, settings.netuid), 180 | wallet_path: expanduser(get_config_value!( 181 | args, 182 | "wallet_path", 183 | String, 184 | &settings.wallet_path 185 | ))?, 186 | wallet_name: get_config_value!(args, "wallet_name", String, &settings.wallet_name) 187 | .to_string(), 188 | hotkey_name: get_config_value!(args, "hotkey_name", String, &settings.hotkey_name) 189 | .to_string(), 190 | external_ip: get_config_value!(args, "external_ip", String, &settings.external_ip).clone(), 191 | api_port, 192 | quic_port: Some(quic_port), // TODO: surely there's a better way to do this 193 | post_ip: *get_config_value!(args, "post_ip", bool, &settings.post_ip), 194 | mock: *get_config_value!(args, "mock", bool, &settings.mock), 195 | load_old_nodes: *get_config_value!(args, "load_old_nodes", bool, &settings.load_old_nodes), 196 | min_stake_threshold: *get_config_value!( 197 | args, 198 | "min_stake_threshold", 199 | u64, 200 | &settings.min_stake_threshold 201 | ), 202 | db_file: expanduser(get_config_value!( 203 | args, 204 | "db_file", 205 | String, 206 | &settings.db_file 207 | ))?, 208 | metadatadb_file: expanduser(get_config_value!( 209 | args, 210 | "metadatadb_file", 211 | String, 212 | &settings.metadatadb_file 213 | ))?, 214 | neurons_dir: expanduser(get_config_value!( 215 | args, 216 | "neurons_dir", 217 | String, 218 | &settings.neurons_dir 219 | ))?, 220 | subtensor: subtensor_config, 221 | neuron: NeuronConfig { 222 | sync_frequency: *get_config_value!( 223 | args, 224 | "neuron.sync_frequency", 225 | u64, 226 | &settings.neuron.sync_frequency 227 | ), 228 | }, 229 | otel_api_key: get_config_value!(args, "otel_api_key", String, &settings.otel_api_key) 230 | .to_string(), 231 | otel_endpoint: get_config_value!(args, "otel_endpoint", String, &settings.otel_endpoint) 232 | .to_string(), 233 | otel_service_name: get_config_value!( 234 | args, 235 | "otel_service_name", 236 | String, 237 | &settings.otel_service_name 238 | ) 239 | .to_string(), 240 | }) 241 | } 242 | -------------------------------------------------------------------------------- /crates/storb_base/src/sync.rs: -------------------------------------------------------------------------------- 1 | use std::error::Error as StdError; 2 | use std::net::Ipv4Addr; 3 | 4 | use crabtensor::api::runtime_apis::neuron_info_runtime_api::NeuronInfoRuntimeApi; 5 | use crabtensor::api::runtime_types::pallet_subtensor::rpc_info::neuron_info::NeuronInfoLite; 6 | use crabtensor::AccountId; 7 | use futures::stream::{self, StreamExt}; 8 | use thiserror::Error; 9 | use tracing::{debug, error, info, warn}; 10 | 11 | use crate::constants::{INFO_REQ_TIMEOUT, SYNC_BUFFER_SIZE}; 12 | use crate::{BaseNeuron, LocalNodeInfo, NodeInfo, NodeUID}; 13 | 14 | #[derive(Debug, Error)] 15 | pub enum SyncError { 16 | #[error("Failed to get node info: {0}")] 17 | GetNodeInfoError(String), 18 | } 19 | pub trait Synchronizable { 20 | fn sync_metagraph( 21 | &mut self, 22 | ) -> impl std::future::Future, Box>> + Send; 23 | fn get_remote_node_info( 24 | addr: reqwest::Url, 25 | ) -> impl std::future::Future> + Send; 26 | } 27 | 28 | // Implementation for BaseNeuron 29 | impl Synchronizable for BaseNeuron { 30 | async fn get_remote_node_info(addr: reqwest::Url) -> Result { 31 | debug!("Requesting node info from: {}", addr.to_string()); 32 | let req_client = reqwest::Client::builder() 33 | .timeout(INFO_REQ_TIMEOUT) 34 | .build() 35 | .map_err(|e| { 36 | error!("Failed to build HTTP client: {:?}", e); 37 | SyncError::GetNodeInfoError(format!("Could not build HTTP client: {e}")) 38 | })?; 39 | 40 | let response = tokio::time::timeout(INFO_REQ_TIMEOUT, req_client.get(addr.clone()).send()) 41 | .await 42 | .map_err(|e| { 43 | error!("Request timed out for {}: {:?}", addr, e); 44 | SyncError::GetNodeInfoError(format!("Request timed out: {e}")) 45 | })? 46 | .map_err(|e| { 47 | error!("Failed to send request to {}: {:?}", addr, e); 48 | SyncError::GetNodeInfoError(format!("Could not send request: {:?}", e)) 49 | })?; 50 | 51 | // Read the response bytes. 52 | let data = response.bytes().await.map_err(|e| { 53 | error!("Failed to read response bytes: {:?}", e); 54 | SyncError::GetNodeInfoError(format!("Could not read response bytes: {:?}", e)) 55 | })?; 56 | 57 | debug!("get_remote_node_info - raw data: {:?}", data); 58 | 59 | // Deserialize the response using bincode. 60 | let local_node_info: LocalNodeInfo = 61 | bincode::deserialize::(&data).map_err(|e| { 62 | error!("Failed to deserialize LocalNodeInfo: {:?}", e); 63 | SyncError::GetNodeInfoError(format!("Failed to deserialize LocalNodeInfo: {:?}", e)) 64 | })?; 65 | 66 | Ok(local_node_info) 67 | } 68 | 69 | /// Synchronise the local metagraph state with chain. 70 | async fn sync_metagraph( 71 | &mut self, 72 | ) -> Result, Box> { 73 | info!("Starting sync_metagraph"); 74 | let start = std::time::Instant::now(); 75 | let subtensor = Self::get_subtensor_connection( 76 | self.config.subtensor.insecure, 77 | &self.config.subtensor.address, 78 | ) 79 | .await?; 80 | 81 | info!("Getting latest block..."); 82 | let current_block = subtensor 83 | .blocks() 84 | .at_latest() 85 | .await 86 | .map_err(|e| format!("Failed to get latest block: {:?}", e))?; 87 | info!("Got latest block in {:?}", start.elapsed()); 88 | 89 | info!("Getting runtime API..."); 90 | let runtime_api = subtensor.runtime_api().at(current_block.reference()); 91 | let mut local_node_info = self.local_node_info.clone(); 92 | // TODO: error out if cant access chain when calling above functions 93 | 94 | // TODO: is there a nicer way to pass self for NeuronInfoRuntimeApi? 95 | let neurons_payload = 96 | NeuronInfoRuntimeApi::get_neurons_lite(&NeuronInfoRuntimeApi {}, self.config.netuid); 97 | 98 | // TODO: error out if cant access chain when calling above function 99 | let neurons: Vec> = runtime_api.call(neurons_payload).await?; 100 | 101 | let original_neurons = self.neurons.read().await.clone(); 102 | 103 | info!("Got neurons from metagraph"); 104 | 105 | // Check if neurons have changed or not 106 | let mut changed_neurons: Vec = neurons.iter().map(|neuron| neuron.uid).collect(); 107 | 108 | if original_neurons.len() < neurons.len() { 109 | changed_neurons = neurons 110 | .iter() 111 | .zip(original_neurons.iter()) 112 | .filter(|(curr, og)| curr.hotkey != og.hotkey) 113 | .map(|(curr, _)| curr.uid) 114 | .collect(); 115 | 116 | let mut new_neurons: Vec = neurons[original_neurons.len()..] 117 | .iter() 118 | .map(|neuron| neuron.uid) 119 | .collect(); 120 | 121 | changed_neurons.append(&mut new_neurons); 122 | 123 | let mut neurons_guard = self.neurons.write().await; 124 | *neurons_guard = neurons.clone(); 125 | } else if !original_neurons.is_empty() { 126 | changed_neurons = neurons 127 | .iter() 128 | .zip(original_neurons.iter()) 129 | .filter(|(curr, og)| curr.hotkey != og.hotkey) 130 | .map(|(curr, _)| curr.uid) 131 | .collect(); 132 | 133 | let mut neurons_guard = self.neurons.write().await; 134 | *neurons_guard = neurons.clone(); 135 | } 136 | 137 | drop(original_neurons); 138 | 139 | info!("Updated local neurons state"); 140 | 141 | // After updating neurons state, save to disk 142 | info!("Saving neurons state to disk"); 143 | Self::save_neurons_state_to_disk(&self.config.neurons_dir, &self.neurons.read().await)?; 144 | info!("Saved neurons state to disk"); 145 | 146 | if local_node_info.uid.is_none() { 147 | // Find our local node uid or crash the program 148 | let my_neuron = Self::find_neuron_info(&neurons, self.signer.account_id()) 149 | .expect("Local node not found in neuron list"); 150 | local_node_info.uid = Some(my_neuron.uid); 151 | } 152 | 153 | self.local_node_info = local_node_info; 154 | 155 | let mut futures = Vec::new(); 156 | 157 | // Create futures for all node info requests 158 | for (neuron_uid, _) in neurons.iter().enumerate() { 159 | let neuron_info: NeuronInfoLite = neurons[neuron_uid].clone(); 160 | 161 | // Skip self to avoid deadlock 162 | if neuron_uid == self.local_node_info.uid.ok_or("Node UID did not exist")? as usize { 163 | continue; 164 | } 165 | 166 | let neuron_ip = neuron_info.axon_info.ip; 167 | let neuron_port = neuron_info.axon_info.port; 168 | 169 | // Skip invalid IPs/ports 170 | if neuron_ip == 0 { 171 | warn!( 172 | "Invalid IP for neuron {:?}. Node has never been started", 173 | neuron_info.uid 174 | ); 175 | continue; 176 | } 177 | 178 | if neuron_port == 0 { 179 | error!("Invalid port for neuron: {:?}", neuron_info.uid); 180 | continue; 181 | } 182 | 183 | let ip = Ipv4Addr::from((neuron_ip & 0xffff_ffff) as u32); 184 | let url_raw = format!("http://{}:{}/info", ip, neuron_port); 185 | let url = reqwest::Url::parse(&url_raw).map_err(|e| { 186 | error!("Failed to parse URL: {:?}", e); 187 | SyncError::GetNodeInfoError(format!("Failed to parse URL: {:?}", e)) 188 | })?; 189 | 190 | // Create future for this node's info request 191 | let future = async move { 192 | match Self::get_remote_node_info(url).await { 193 | Ok(remote_node_info) => Some((neuron_info, remote_node_info)), 194 | Err(err) => { 195 | warn!( 196 | "Failed to get remote node info (it may be offline): {:?}", 197 | err 198 | ); 199 | None 200 | } 201 | } 202 | }; 203 | 204 | futures.push(future); 205 | } 206 | 207 | // Execute all futures concurrently and collect results 208 | let stream = stream::iter(futures) 209 | .buffer_unordered(SYNC_BUFFER_SIZE) 210 | .collect::>(); 211 | let results = stream.await; 212 | 213 | // Process results and update address book 214 | for result in results.into_iter().flatten() { 215 | let (neuron_info, remote_node_info) = result; 216 | 217 | let node_info = NodeInfo { 218 | neuron_info: neuron_info.clone(), 219 | http_address: remote_node_info.http_address, 220 | quic_address: remote_node_info.quic_address, 221 | version: remote_node_info.version, 222 | }; 223 | 224 | self.address_book.insert(neuron_info.uid, node_info.clone()); 225 | } 226 | 227 | info!("Completed sync_metagraph in {:?}", start.elapsed()); 228 | info!("Done syncing metagraph"); 229 | 230 | Ok(changed_neurons) 231 | } 232 | } 233 | -------------------------------------------------------------------------------- /crates/storb_validator/src/apikey.rs: -------------------------------------------------------------------------------- 1 | use std::path::PathBuf; 2 | use std::sync::Arc; 3 | 4 | use anyhow::Result; 5 | use chrono::{DateTime, Utc}; 6 | use rusqlite::{params, Connection}; 7 | use serde::{Deserialize, Serialize}; 8 | use tokio::sync::Mutex; 9 | use uuid::Uuid; 10 | 11 | #[derive(Debug, Serialize, Deserialize)] 12 | pub struct ApiKey { 13 | pub key: String, 14 | pub name: String, 15 | pub created_at: DateTime, 16 | pub expires_at: Option>, 17 | pub rate_limit: Option, // Requests per minute 18 | pub upload_limit: Option, // Max total upload bytes 19 | pub download_limit: Option, // Max total download bytes 20 | pub upload_used: u64, 21 | pub download_used: u64, 22 | } 23 | 24 | #[derive(Debug, Serialize, Deserialize)] 25 | pub struct ApiKeyConfig { 26 | pub name: String, 27 | pub expires_at: Option>, 28 | pub rate_limit: Option, 29 | pub upload_limit: Option, 30 | pub download_limit: Option, 31 | } 32 | 33 | #[derive(Debug)] 34 | pub struct ApiUsageLog { 35 | pub key: String, 36 | pub endpoint: String, 37 | pub timestamp: DateTime, 38 | pub upload_bytes: u64, 39 | pub download_bytes: u64, 40 | } 41 | 42 | #[derive(Debug)] 43 | pub struct ApiKeyManager { 44 | conn: Arc>, 45 | } 46 | 47 | impl ApiKeyManager { 48 | /// Create a new API key manager 49 | pub fn new(db_path: PathBuf) -> Result { 50 | let conn = Connection::open(db_path)?; 51 | 52 | // Create API keys table 53 | conn.execute( 54 | "CREATE TABLE IF NOT EXISTS api_keys ( 55 | key TEXT PRIMARY KEY, 56 | name TEXT NOT NULL, 57 | created_at INTEGER NOT NULL, 58 | expires_at INTEGER, 59 | rate_limit INTEGER, 60 | upload_limit INTEGER, 61 | download_limit INTEGER, 62 | upload_used INTEGER NOT NULL DEFAULT 0, 63 | download_used INTEGER NOT NULL DEFAULT 0 64 | )", 65 | [], 66 | )?; 67 | 68 | // Create usage logs table 69 | conn.execute( 70 | "CREATE TABLE IF NOT EXISTS api_usage_logs ( 71 | id INTEGER PRIMARY KEY AUTOINCREMENT, 72 | key TEXT NOT NULL, 73 | endpoint TEXT NOT NULL, 74 | timestamp INTEGER NOT NULL, 75 | upload_bytes INTEGER NOT NULL DEFAULT 0, 76 | download_bytes INTEGER NOT NULL DEFAULT 0, 77 | FOREIGN KEY(key) REFERENCES api_keys(key) ON DELETE CASCADE 78 | )", 79 | [], 80 | )?; 81 | 82 | // Create index for efficient rate limit checking 83 | conn.execute( 84 | "CREATE INDEX IF NOT EXISTS idx_usage_logs_key_timestamp 85 | ON api_usage_logs(key, timestamp)", 86 | [], 87 | )?; 88 | 89 | Ok(Self { 90 | conn: Arc::new(Mutex::new(conn)), 91 | }) 92 | } 93 | 94 | /// Create a new API key 95 | pub async fn create_key(&self, config: ApiKeyConfig) -> Result { 96 | let key = format!("storb_{}", Uuid::new_v4()); 97 | let now = Utc::now(); 98 | 99 | let conn = self.conn.lock().await; 100 | conn.execute( 101 | "INSERT INTO api_keys ( 102 | key, name, created_at, expires_at, rate_limit, 103 | upload_limit, download_limit, upload_used, download_used 104 | ) VALUES (?, ?, ?, ?, ?, ?, ?, 0, 0)", 105 | params![ 106 | key, 107 | config.name, 108 | now.timestamp(), 109 | config.expires_at.map(|d| d.timestamp()), 110 | config.rate_limit, 111 | config.upload_limit, 112 | config.download_limit, 113 | ], 114 | )?; 115 | 116 | Ok(ApiKey { 117 | key, 118 | name: config.name, 119 | created_at: now, 120 | expires_at: config.expires_at, 121 | rate_limit: config.rate_limit, 122 | upload_limit: config.upload_limit, 123 | download_limit: config.download_limit, 124 | upload_used: 0, 125 | download_used: 0, 126 | }) 127 | } 128 | 129 | /// Validate the API key and check if it has expired 130 | pub async fn validate_key(&self, key: &str) -> Result> { 131 | let conn = self.conn.lock().await; 132 | let mut stmt = conn.prepare("SELECT * FROM api_keys WHERE key = ?")?; 133 | 134 | let key_row = stmt.query_row([key], |row| { 135 | Ok(ApiKey { 136 | key: row.get(0)?, 137 | name: row.get(1)?, 138 | created_at: DateTime::from_timestamp(row.get(2)?, 0).unwrap_or_default(), 139 | expires_at: row 140 | .get::<_, Option>(3)? 141 | .map(|ts| DateTime::from_timestamp(ts, 0).unwrap_or_default()), 142 | rate_limit: row.get(4)?, 143 | upload_limit: row.get(5)?, 144 | download_limit: row.get(6)?, 145 | upload_used: row.get(7)?, 146 | download_used: row.get(8)?, 147 | }) 148 | }); 149 | 150 | match key_row { 151 | Ok(key) => { 152 | // Check if key has expired 153 | if let Some(expires_at) = key.expires_at { 154 | if expires_at < Utc::now() { 155 | return Ok(None); 156 | } 157 | } 158 | Ok(Some(key)) 159 | } 160 | Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None), 161 | Err(e) => Err(e.into()), 162 | } 163 | } 164 | 165 | /// Update the usage stats for uploads and downloads 166 | pub async fn update_usage( 167 | &self, 168 | key: &str, 169 | upload_bytes: u64, 170 | download_bytes: u64, 171 | ) -> Result<()> { 172 | let conn = self.conn.lock().await; 173 | conn.execute( 174 | "UPDATE api_keys SET 175 | upload_used = upload_used + ?, 176 | download_used = download_used + ? 177 | WHERE key = ?", 178 | params![upload_bytes, download_bytes, key], 179 | )?; 180 | Ok(()) 181 | } 182 | 183 | /// List all API keys 184 | pub async fn list_keys(&self) -> Result> { 185 | let conn = self.conn.lock().await; 186 | let mut stmt = conn.prepare("SELECT * FROM api_keys")?; 187 | let keys = stmt.query_map([], |row| { 188 | Ok(ApiKey { 189 | key: row.get(0)?, 190 | name: row.get(1)?, 191 | created_at: DateTime::from_timestamp(row.get(2)?, 0).unwrap_or_default(), 192 | expires_at: row 193 | .get::<_, Option>(3)? 194 | .map(|ts| DateTime::from_timestamp(ts, 0).unwrap_or_default()), 195 | rate_limit: row.get(4)?, 196 | upload_limit: row.get(5)?, 197 | download_limit: row.get(6)?, 198 | upload_used: row.get(7)?, 199 | download_used: row.get(8)?, 200 | }) 201 | })?; 202 | 203 | let mut result = Vec::new(); 204 | for key in keys { 205 | result.push(key?); 206 | } 207 | Ok(result) 208 | } 209 | 210 | pub async fn delete_key(&self, key: &str) -> Result { 211 | let conn = self.conn.lock().await; 212 | let rows = conn.execute("DELETE FROM api_keys WHERE key = ?", [key])?; 213 | Ok(rows > 0) 214 | } 215 | 216 | /// Check the quota for upload and download 217 | pub async fn check_quota( 218 | &self, 219 | key: &str, 220 | upload_bytes: u64, 221 | download_bytes: u64, 222 | ) -> Result { 223 | let api_key = match self.validate_key(key).await? { 224 | Some(key) => key, 225 | None => return Ok(false), 226 | }; 227 | 228 | // Check upload quota 229 | if let Some(upload_limit) = api_key.upload_limit { 230 | if api_key.upload_used + upload_bytes > upload_limit { 231 | return Ok(false); 232 | } 233 | } 234 | 235 | // Check download quota 236 | if let Some(download_limit) = api_key.download_limit { 237 | if api_key.download_used + download_bytes > download_limit { 238 | return Ok(false); 239 | } 240 | } 241 | 242 | Ok(true) 243 | } 244 | 245 | /// Check the rate limit allowed by the API key 246 | pub async fn check_rate_limit(&self, key: &str, rate_limit: u32) -> Result { 247 | let conn = self.conn.lock().await; 248 | let one_minute_ago = (Utc::now() - chrono::Duration::minutes(1)).timestamp(); 249 | 250 | let count: i64 = conn.query_row( 251 | "SELECT COUNT(*) FROM api_usage_logs 252 | WHERE key = ? AND timestamp > ?", 253 | params![key, one_minute_ago], 254 | |row| row.get(0), 255 | )?; 256 | 257 | Ok(count + 1 < rate_limit as i64) // +1 for the current request 258 | } 259 | 260 | pub async fn log_api_usage( 261 | &self, 262 | key: &str, 263 | endpoint: &str, 264 | upload_bytes: u64, 265 | download_bytes: u64, 266 | ) -> Result<()> { 267 | let conn = self.conn.lock().await; 268 | let now = Utc::now(); 269 | 270 | conn.execute( 271 | "INSERT INTO api_usage_logs ( 272 | key, endpoint, timestamp, upload_bytes, download_bytes 273 | ) VALUES (?, ?, ?, ?, ?)", 274 | params![key, endpoint, now.timestamp(), upload_bytes, download_bytes], 275 | )?; 276 | 277 | Ok(()) 278 | } 279 | 280 | /// Clean up old logs 281 | pub async fn cleanup_old_logs(&self, days_to_keep: i64) -> Result<()> { 282 | let conn = self.conn.lock().await; 283 | let cutoff = (Utc::now() - chrono::Duration::days(days_to_keep)).timestamp(); 284 | 285 | conn.execute( 286 | "DELETE FROM api_usage_logs WHERE timestamp < ?", 287 | params![cutoff], 288 | )?; 289 | 290 | Ok(()) 291 | } 292 | } 293 | -------------------------------------------------------------------------------- /crates/storb_validator/src/scoring.rs: -------------------------------------------------------------------------------- 1 | use std::fs::{self, File}; 2 | use std::path::{Path, PathBuf}; 3 | use std::sync::Arc; 4 | 5 | use anyhow::{Context, Result}; 6 | use base::memory_db::MemoryDb; 7 | use ndarray::{array, s, Array, Array1}; 8 | use rusqlite::params; 9 | use serde::{Deserialize, Serialize}; 10 | use tracing::{debug, error, info, warn}; 11 | 12 | use crate::constants::{INITIAL_ALPHA, INITIAL_BETA, LAMBDA}; 13 | 14 | /// ScoreState stores the scores for each miner. 15 | /// 16 | /// The final EMA score consists of the weighted sum of the normalized response 17 | /// rate and challenge scores. 18 | /// 19 | /// The response rate /// and challenge statistics are stored in an SQLite database. 20 | /// Those stats are calculated then saved into the EMA score. 21 | #[derive(Clone, Debug, Deserialize, Serialize)] 22 | pub struct ScoreState { 23 | /// The current exponential moving average (EMA) score of the miners. 24 | pub ema_scores: Array1, 25 | /// Previous EMA scores before reset, used as baseline 26 | pub previous_scores: Array1, 27 | } 28 | 29 | pub struct ScoringSystem { 30 | /// Database connection pool for the given DB driver. 31 | pub db: Arc, 32 | /// Path to the state file. 33 | pub state_file: PathBuf, 34 | /// Score state for miners. 35 | pub state: ScoreState, 36 | pub initial_alpha: f64, 37 | pub initial_beta: f64, 38 | // forgetting factor for challenges 39 | pub lambda: f64, 40 | } 41 | 42 | // normalization function that produces an s curve 43 | #[inline] 44 | pub fn normalize_min_max(arr: &Array1) -> Array1 { 45 | let min = arr.iter().cloned().fold(f64::INFINITY, f64::min); 46 | let max = arr.iter().cloned().fold(f64::NEG_INFINITY, f64::max); 47 | 48 | if (max - min).abs() < 1e-9 { 49 | return arr.mapv(|_| 0.0); // Avoid division by zero 50 | } 51 | 52 | arr.mapv(|x| (x - min) / (max - min)) 53 | } 54 | 55 | fn get_new_alpha_beta( 56 | beta: f64, 57 | alpha: f64, 58 | lambda: f64, 59 | weight: f64, 60 | success: bool, 61 | ) -> (f64, f64) { 62 | let v = if success { 1.0 } else { 0.0 }; 63 | let new_alpha = lambda * alpha + weight * (1.0 + v) / 2.0; 64 | let new_beta = lambda * beta + weight * (1.0 - v) / 2.0; 65 | (new_beta, new_alpha) 66 | } 67 | 68 | impl ScoringSystem { 69 | pub async fn new(db_file: &PathBuf, scoring_state_file: &Path) -> Result { 70 | let db_path = PathBuf::new().join(db_file); 71 | if !fs::exists(&db_path)? { 72 | warn!( 73 | "Database file did not exist at location {:?}. Created a new file as a result", 74 | &db_path 75 | ); 76 | File::create(&db_path)?; 77 | } 78 | let db_path_str = db_path 79 | .to_str() 80 | .context("Could not convert path to string")?; 81 | 82 | if !fs::exists(scoring_state_file)? { 83 | warn!( 84 | "Score state file did not exist at location {:?}. Created a new file as a result", 85 | &scoring_state_file 86 | ); 87 | File::create(scoring_state_file)?; 88 | } 89 | 90 | // create new MemoryDb 91 | let db = Arc::new(MemoryDb::new(db_path_str).await?); 92 | 93 | let state = ScoreState { 94 | ema_scores: array![], 95 | previous_scores: array![], 96 | }; 97 | 98 | let mut scoring_system = Self { 99 | db, 100 | state, 101 | state_file: scoring_state_file.to_path_buf(), 102 | initial_alpha: INITIAL_ALPHA, 103 | initial_beta: INITIAL_BETA, 104 | lambda: LAMBDA, // forgetting factor for challenges 105 | }; 106 | 107 | match scoring_system.load_state() { 108 | Ok(_) => debug!("Loaded state successfully"), 109 | Err(e) => { 110 | error!("Could not load the state: {}", e); 111 | } 112 | } 113 | 114 | Ok(scoring_system) 115 | } 116 | 117 | /// Load scores state from teh state file. 118 | fn load_state(&mut self) -> Result<()> { 119 | let buf: Vec = fs::read(&self.state_file)?; 120 | self.state = bincode::deserialize::(&buf[..])?; 121 | 122 | Ok(()) 123 | } 124 | 125 | /// Save scores state to the state file. 126 | fn save_state(&mut self) -> Result<()> { 127 | let buf = bincode::serialize(&self.state)?; 128 | fs::write(&self.state_file, buf)?; 129 | Ok(()) 130 | } 131 | 132 | /// Reset all statistics for a given miner UID in the database 133 | async fn reset_miner_stats(&self, uid: usize) -> Result<(), rusqlite::Error> { 134 | let db = self.db.clone(); 135 | let conn = db.conn.lock().await; 136 | conn.execute( 137 | "UPDATE miner_stats SET 138 | alpha = ?1, 139 | beta = ?2, 140 | store_successes = 0, 141 | store_attempts = 0, 142 | retrieval_successes = 0, 143 | retrieval_attempts = 0, 144 | total_successes = 0 145 | WHERE miner_uid = ?3", 146 | params![self.initial_alpha, self.initial_beta, uid], 147 | )?; 148 | debug!("Reset stats for UID {} in database", uid); 149 | Ok(()) 150 | } 151 | 152 | pub async fn update_alpha_beta_db( 153 | &mut self, 154 | miner_uid: u16, 155 | weight: f64, 156 | success: bool, 157 | ) -> Result<(), rusqlite::Error> { 158 | let conn = self.db.conn.lock().await; 159 | // get current alpha and beta from the database 160 | let (alpha, beta): (f64, f64) = conn.query_row( 161 | "SELECT alpha, beta FROM miner_stats WHERE miner_uid = ?", 162 | [miner_uid], 163 | |row| Ok((row.get(0)?, row.get(1)?)), 164 | )?; 165 | 166 | // calculate new alpha and beta 167 | let (new_beta, new_alpha) = get_new_alpha_beta(beta, alpha, LAMBDA, weight, success); 168 | // update alpha and beta in the database 169 | conn.execute( 170 | "UPDATE miner_stats SET alpha = ?1, beta = ?2 WHERE miner_uid = ?3", 171 | params![new_alpha, new_beta, miner_uid], 172 | )?; 173 | debug!( 174 | "Updated alpha and beta for miner UID {}: alpha = {}, beta = {}", 175 | miner_uid, new_alpha, new_beta 176 | ); 177 | Ok(()) 178 | } 179 | 180 | /// Update scores for each of the miners. 181 | pub async fn update_scores(&mut self, neuron_count: usize, uids_to_update: Vec) { 182 | let extend_array = |old: &Array1, new_size: usize| -> Array1 { 183 | let mut new_array = Array::::zeros(new_size); 184 | new_array.slice_mut(s![0..old.len()]).assign(old); 185 | new_array 186 | }; 187 | 188 | info!("uids to update: {:?}", uids_to_update); 189 | 190 | // Only extend/initialize if current arrays are empty or new size is larger 191 | let current_size = self.state.ema_scores.len(); 192 | if current_size < neuron_count { 193 | let state = &self.state; 194 | 195 | let mut new_ema_scores = extend_array(&state.ema_scores, neuron_count); 196 | let mut new_previous_scores = extend_array(&state.previous_scores, neuron_count); 197 | 198 | // Initialize new UIDs and reset scores for updated UIDs 199 | for uid in uids_to_update.iter() { 200 | let uid = *uid as usize; 201 | new_ema_scores[uid] = 0.0; 202 | new_previous_scores[uid] = 0.0; 203 | 204 | // Reset database stats 205 | if let Err(e) = self.reset_miner_stats(uid).await { 206 | error!("Failed to reset stats for UID {} in database: {}", uid, e); 207 | } 208 | } 209 | 210 | self.state.ema_scores = new_ema_scores; 211 | self.state.previous_scores = new_previous_scores; 212 | } else { 213 | // If we don't need to extend arrays, just reset scores for updated UIDs 214 | for uid in uids_to_update.iter() { 215 | let uid = *uid as usize; 216 | self.state.ema_scores[uid] = 0.0; 217 | self.state.previous_scores[uid] = 0.0; 218 | 219 | // Reset database stats 220 | if let Err(e) = self.reset_miner_stats(uid).await { 221 | error!("Failed to reset stats for UID {} in database: {}", uid, e); 222 | } 223 | } 224 | } 225 | 226 | // Rest of the existing update_scores implementation... 227 | // connect to db and compute new scores 228 | let state = &mut self.state; 229 | 230 | let mut response_rate_scores = Array1::::zeros(state.ema_scores.len()); 231 | for (miner_uid, _) in state.ema_scores.iter().enumerate() { 232 | let db = self.db.clone(); 233 | let conn = db.conn.lock().await; 234 | 235 | // TODO: error handling 236 | let alpha: f64 = conn 237 | .query_row( 238 | "SELECT alpha FROM miner_stats WHERE miner_uid = ?", 239 | [miner_uid], 240 | |row| row.get(0), 241 | ) 242 | .unwrap_or(self.initial_alpha); 243 | let beta: f64 = conn 244 | .query_row( 245 | "SELECT beta FROM miner_stats WHERE miner_uid = ?", 246 | [miner_uid], 247 | |row| row.get(0), 248 | ) 249 | .unwrap_or(self.initial_beta); 250 | 251 | response_rate_scores[miner_uid] = alpha / (alpha + beta); 252 | } 253 | 254 | // zero out nans in response rate scores 255 | let nan_indices: Vec<_> = response_rate_scores 256 | .iter() 257 | .enumerate() 258 | .filter(|(_, &score)| score.is_nan()) 259 | .map(|(i, _)| i) 260 | .collect(); 261 | 262 | for i in nan_indices { 263 | response_rate_scores[i] = 0.0; 264 | } 265 | 266 | state.ema_scores = response_rate_scores.clone(); 267 | 268 | info!("response rate scores: {}", response_rate_scores); 269 | info!("new scores: {}", state.ema_scores); 270 | 271 | match self.save_state() { 272 | Ok(_) => info!("Saved state successfully"), 273 | Err(e) => { 274 | error!("Could not save the state: {}", e); 275 | // TODO: submit to telemetry 276 | } 277 | } 278 | } 279 | } 280 | -------------------------------------------------------------------------------- /crates/storb_validator/src/repair.rs: -------------------------------------------------------------------------------- 1 | use std::{collections::HashMap, sync::Arc}; 2 | 3 | use anyhow::Context; 4 | use base::{constants::MIN_BANDWIDTH, NodeUID}; 5 | use quinn::Connection; 6 | use tokio::sync::mpsc; 7 | use tracing::{error, info, warn}; 8 | 9 | use crate::{download, metadata, upload, utils::get_id_quic_uids, validator::Validator}; 10 | 11 | // Function which called to check for what pieces to repair from the MetadataDB 12 | // and distributes the pieces to different miners 13 | pub async fn repair_pieces(validator: Arc) -> Result<(), Box> { 14 | // query the database to get the pieces that need repair 15 | let metadatadb_sender = validator.clone().metadatadb_sender.clone(); 16 | let signer = Arc::new(validator.neuron.read().await.signer.clone()); 17 | let pieces_to_repair = 18 | match metadata::db::MetadataDB::get_pieces_for_repair(&metadatadb_sender.clone()).await { 19 | Ok(pieces) => pieces, 20 | Err(e) => { 21 | error!("Failed to get pieces for repair: {}", e); 22 | // convert the error into a Box 23 | // and return it 24 | return Err(Box::new(e)); 25 | } 26 | }; 27 | 28 | if pieces_to_repair.is_empty() { 29 | info!("No pieces to repair."); 30 | return Ok(()); 31 | } 32 | 33 | info!("Found {} pieces to repair.", pieces_to_repair.len()); 34 | 35 | let vali_clone = validator.clone(); 36 | let base_neuron_guard = vali_clone.neuron.read().await; 37 | let vali_uid = base_neuron_guard 38 | .local_node_info 39 | .uid 40 | .context("Failed to get UID for validator")?; 41 | drop(base_neuron_guard); 42 | 43 | // Download, and redistribute the pieces to miners 44 | for (piece_hash, miners) in pieces_to_repair { 45 | // get piece from metadatadb 46 | let piece = 47 | match metadata::db::MetadataDB::get_piece(&metadatadb_sender.clone(), piece_hash).await 48 | { 49 | Ok(piece) => piece, 50 | Err(e) => { 51 | error!( 52 | "Failed to get piece {} from MetadataDB: {}", 53 | hex::encode(piece_hash), 54 | e 55 | ); 56 | continue; 57 | } 58 | }; 59 | 60 | let piece_size: f64 = piece.piece_size as f64; 61 | let timeout_duration = 62 | std::time::Duration::from_secs_f64(piece_size / MIN_BANDWIDTH as f64); 63 | 64 | let req_client = match reqwest::Client::builder().timeout(timeout_duration).build() { 65 | Ok(client) => client, 66 | Err(e) => { 67 | error!("Failed to create HTTP client: {}", e); 68 | continue; 69 | } 70 | }; 71 | 72 | // Try to download the piece from the miners 73 | // Once one of the miners responds, and it's 74 | // hash is valid, we can stop trying to download 75 | let mut download_piece_futures: Vec>> = 76 | Vec::new(); 77 | let (download_task_tx, mut download_task_rx) = 78 | mpsc::channel::, anyhow::Error>>(miners.len()); 79 | 80 | let validator_for_download = validator.clone(); 81 | for miner in &miners { 82 | let miner_uid = *miner; 83 | // get node info for the miner 84 | let miner_node_info = match validator 85 | .clone() 86 | .neuron 87 | .read() 88 | .await 89 | .address_book 90 | .get(&miner_uid) 91 | { 92 | Some(node_info) => node_info.clone(), 93 | None => { 94 | warn!("Miner UID {} not found in address book.", miner_uid); 95 | continue; // Skip this miner if not found 96 | } 97 | }; 98 | 99 | let piece_hash_clone = piece_hash; 100 | let req_client_clone = req_client.clone(); 101 | let download_task_tx_clone = download_task_tx.clone(); 102 | let signer_clone = signer.clone(); 103 | let validator_clone = validator_for_download.clone(); 104 | 105 | let download_process_piece = tokio::spawn(async move { 106 | // download the piece with get_piece_from_miner and process if with process_piece_response 107 | // if the piece is valid, send it to the download_task_tx channel 108 | let scoring_system = validator_clone.scoring_system.clone(); 109 | let download_response = match download::get_piece_from_miner( 110 | req_client_clone, 111 | &miner_node_info, 112 | piece_hash_clone, 113 | signer_clone.clone(), 114 | vali_uid, 115 | scoring_system.clone(), 116 | ) 117 | .await 118 | { 119 | Ok(response) => response, 120 | Err(e) => { 121 | error!( 122 | "Failed to download piece {} from miner {}: {}", 123 | hex::encode(piece_hash_clone), 124 | miner_uid, 125 | e 126 | ); 127 | return Err(e); 128 | } 129 | }; 130 | 131 | match download::process_piece_response( 132 | download_response, 133 | piece_hash_clone, 134 | miner_uid, 135 | ) 136 | .await 137 | { 138 | Ok(valid_piece) => { 139 | // Send the valid piece to the channel 140 | if let Err(e) = download_task_tx_clone.send(Ok(valid_piece)).await { 141 | error!("Failed to send valid piece to channel: {}", e); 142 | } 143 | Ok(()) 144 | } 145 | Err(e) => { 146 | error!( 147 | "Failed to process piece response from miner {}: {}", 148 | miner_uid, e 149 | ); 150 | Err(e) 151 | } 152 | } 153 | }); 154 | download_piece_futures.push(download_process_piece); 155 | } 156 | 157 | // Process the results of the downloads, we monitor the download_task_rx channel 158 | // and wait for the first valid piece to be received 159 | // If we receive a valid piece, we can store it in a variable, 160 | // and abort() the futures in download_piece_futures 161 | let mut valid_piece: Option> = None; 162 | for _ in 0..miners.len() { 163 | match download_task_rx.recv().await { 164 | Some(Ok(piece_data)) => { 165 | valid_piece = Some(piece_data); 166 | break; // We found a valid piece, we can stop waiting 167 | } 168 | Some(Err(e)) => { 169 | error!("Error receiving piece data: {}", e); 170 | } 171 | None => { 172 | error!("Download task channel closed unexpectedly."); 173 | break; 174 | } 175 | } 176 | } 177 | 178 | // abort all download futures if we found a valid piece 179 | if valid_piece.is_some() { 180 | for download_future in download_piece_futures { 181 | download_future.abort(); 182 | } 183 | } 184 | 185 | let valid_piece = 186 | valid_piece.ok_or_else(|| anyhow::anyhow!("No valid piece found after downloading"))?; 187 | 188 | // TODO(repair): exclude the miners that got deregistered? 189 | // Distribute the pieces to the top miners 190 | 191 | // First, we get the scores of the miners from the scoring system 192 | let scoring_system = validator.clone().scoring_system.clone(); 193 | let miner_scores = scoring_system.read().await.state.ema_scores.clone(); 194 | 195 | // The indices of miner_scores are the uids of the miners 196 | // We want to sort them by their scores, and *try* distribute the each piece with upload_piece_data 197 | // to n_i of them, where n_i=2 * m_i where m_i 198 | // is the number of miners that lost piece i 199 | // once we have successfully uploaded m_i pieces to m_i unique miners, we can stop 200 | // distributing the piece 201 | let mut sorted_miners: Vec = (0..miner_scores.len()) 202 | .filter(|&i| miner_scores[i] > 0.0) // Filter out miners with zero scores 203 | .collect(); 204 | sorted_miners.sort_by(|&a, &b| { 205 | miner_scores[b] 206 | .partial_cmp(&miner_scores[a]) 207 | .unwrap_or(std::cmp::Ordering::Equal) 208 | }); 209 | 210 | let num_miners_to_distribute = 2 * miners.len(); 211 | let mut distributed_count = 0; 212 | 213 | // Get quic addresses and miner uids, and establish connections with the miners 214 | let (_, quic_addresses, miner_uids) = get_id_quic_uids(validator.clone()).await; 215 | 216 | upload::log_connection_attempt(&quic_addresses, &miner_uids); 217 | 218 | let miner_connections = upload::establish_and_validate_connections(quic_addresses) 219 | .await 220 | .context("Failed to establish connections with miners")?; 221 | 222 | // create hashmap of miner_uids->miner_connection. 223 | // each miner_uid coresponds to the miner connection in miner_connections 224 | let uid_to_connection: HashMap = miner_uids 225 | .iter() 226 | .zip(miner_connections.iter()) 227 | .map(|(&uid, conn)| (uid, conn.1.clone())) 228 | .collect(); 229 | 230 | upload::log_connection_success(&miner_connections); 231 | 232 | // Distribute to miners, if there are not enough miners, then we automatically loop back around the sorted_miners 233 | // TODP(repair): we should probably have a better way to handle this 234 | for &miner_uid in sorted_miners.iter().cycle().take(num_miners_to_distribute) { 235 | // Get the miner's node info 236 | let miner_node_info = match validator 237 | .clone() 238 | .neuron 239 | .read() 240 | .await 241 | .address_book 242 | .get(&(miner_uid as u16)) 243 | { 244 | Some(node_info) => node_info.clone(), 245 | None => { 246 | warn!("Miner UID {} not found in address book.", miner_uid); 247 | continue; // Skip this miner if not found 248 | } 249 | }; 250 | 251 | let conn = uid_to_connection[&(miner_uid as u16)].clone(); 252 | let valid_piece_clone = valid_piece.clone(); // TODO(repair): is there a better way to handle this instead of cloning the piece? 253 | 254 | // Upload the piece to the miner 255 | if let Err(e) = upload::upload_piece_data( 256 | validator.clone().neuron.clone(), 257 | miner_node_info, 258 | &conn, 259 | valid_piece_clone, 260 | ) 261 | .await 262 | { 263 | error!( 264 | "Failed to upload piece {} to miner {}: {}", 265 | hex::encode(piece_hash), 266 | miner_uid, 267 | e 268 | ); 269 | continue; // Skip this miner if upload fails 270 | } 271 | 272 | distributed_count += 1; 273 | if distributed_count >= miners.len() { 274 | break; // We have successfully distributed to enough miners 275 | } 276 | } 277 | } 278 | Ok(()) 279 | } 280 | -------------------------------------------------------------------------------- /crates/storb_miner/src/lib.rs: -------------------------------------------------------------------------------- 1 | use std::error::Error; 2 | use std::net::{SocketAddr, UdpSocket}; 3 | use std::sync::Arc; 4 | use std::time::Duration; 5 | 6 | use anyhow::{Context, Result}; 7 | use axum::middleware::from_fn; 8 | use axum::routing::{get, post}; 9 | use axum::Extension; 10 | use base::constants::NEURON_SYNC_TIMEOUT; 11 | use base::piece_hash::PieceHashStr; 12 | use base::sync::Synchronizable; 13 | use base::verification::HandshakePayload; 14 | use base::LocalNodeInfo; 15 | use crabtensor::sign::verify_signature; 16 | use dashmap::DashMap; 17 | use middleware::InfoApiRateLimiter; 18 | use miner::{Miner, MinerConfig}; 19 | use quinn::rustls::pki_types::PrivatePkcs8KeyDer; 20 | use quinn::{Endpoint, ServerConfig, VarInt}; 21 | use rcgen::generate_simple_self_signed; 22 | use store::ObjectStore; 23 | use tokio::sync::Mutex; 24 | use tokio::time; 25 | use tracing::{debug, error, info}; 26 | 27 | pub mod constants; 28 | mod middleware; 29 | pub mod miner; 30 | mod routes; 31 | pub mod store; 32 | 33 | #[derive(Clone)] 34 | pub struct MinerState { 35 | pub miner: Arc, 36 | pub object_store: Arc>, 37 | pub local_node_info: LocalNodeInfo, 38 | } 39 | 40 | /// Configures the QUIC server with a self-signed certificate for localhost 41 | /// 42 | /// # Returns 43 | /// - A ServerConfig containing the server configuration with the self-signed cert 44 | /// - An error if server configuration fails 45 | fn configure_server( 46 | external_ip: String, 47 | ) -> Result> { 48 | let cert = generate_simple_self_signed(vec!["localhost".into(), external_ip])?; 49 | let cert_der = cert.cert; 50 | let priv_key = PrivatePkcs8KeyDer::from(cert.key_pair.serialize_der()); 51 | 52 | let server_config = ServerConfig::with_single_cert(vec![cert_der.into()], priv_key.into()); 53 | 54 | Ok(server_config?) 55 | } 56 | 57 | async fn main(config: MinerConfig) -> Result<()> { 58 | info!("Configuring miner server..."); 59 | let server_config = configure_server(config.clone().neuron_config.external_ip).unwrap(); 60 | 61 | let miner = Arc::new(Miner::new(config.clone()).await.unwrap()); 62 | 63 | let sync_miner = miner.clone(); 64 | 65 | let object_store = Arc::new(Mutex::new( 66 | ObjectStore::new(&config.store_dir).expect("Failed to initialize object store"), 67 | )); 68 | 69 | // Spawn background sync task 70 | tokio::spawn(async move { 71 | let local_miner = sync_miner.clone(); 72 | let neuron = local_miner.neuron.clone(); 73 | let mut interval = time::interval(Duration::from_secs( 74 | config.neuron_config.neuron.sync_frequency, 75 | )); 76 | loop { 77 | interval.tick().await; 78 | info!("Syncing miner"); 79 | match tokio::time::timeout(NEURON_SYNC_TIMEOUT, async { 80 | let start = std::time::Instant::now(); 81 | let sync_result = neuron.write().await.sync_metagraph().await; 82 | (sync_result, start.elapsed()) 83 | }) 84 | .await 85 | { 86 | Ok((Ok(_), elapsed)) => { 87 | info!("Miner sync completed in {:?}", elapsed); 88 | } 89 | Ok((Err(e), elapsed)) => { 90 | error!("Miner sync failed: {}. Elapsed time: {:?}", e, elapsed); 91 | } 92 | Err(_) => { 93 | error!("Miner sync timed out after {:?}", NEURON_SYNC_TIMEOUT); 94 | } 95 | } 96 | } 97 | }); 98 | 99 | let state = MinerState { 100 | miner: miner.clone(), 101 | object_store, 102 | local_node_info: miner.neuron.read().await.local_node_info.clone(), 103 | }; 104 | 105 | let assigned_quic_port = config 106 | .neuron_config 107 | .quic_port 108 | .expect("Could not assign quic port"); 109 | 110 | let socket = UdpSocket::bind(format!("0.0.0.0:{}", assigned_quic_port)) 111 | .expect("Failed to bind UDP socket"); 112 | 113 | let endpoint = Endpoint::new( 114 | Default::default(), 115 | Some(server_config), 116 | socket, 117 | Arc::new(quinn::TokioRuntime), 118 | ) 119 | .expect("Failed to create QUIC endpoint"); 120 | 121 | let info_api_rate_limit_state: InfoApiRateLimiter = Arc::new(DashMap::new()); 122 | let app = axum::Router::new() 123 | .route( 124 | "/info", 125 | get(routes::node_info).route_layer(from_fn(middleware::info_api_rate_limit_middleware)), 126 | ) 127 | .layer(Extension(info_api_rate_limit_state)) 128 | .route("/handshake", post(routes::handshake)) 129 | .route("/piece", get(routes::get_piece)) 130 | .with_state(state.clone()); 131 | 132 | let addr = SocketAddr::from(([0, 0, 0, 0], config.neuron_config.api_port)); 133 | info!("Miner HTTP server listening on {}", addr); 134 | 135 | let http_server = tokio::spawn(async move { 136 | axum::serve( 137 | tokio::net::TcpListener::bind(addr) 138 | .await 139 | .context("Failed to bind HTTP server") 140 | .unwrap(), 141 | app.into_make_service_with_connect_info::(), 142 | ) 143 | .await 144 | .context("HTTP server failed") 145 | .unwrap(); 146 | }); 147 | 148 | let quic_server = tokio::spawn(async move { 149 | while let Some(incoming) = endpoint.accept().await { 150 | let object_store = state.object_store.clone(); 151 | let miner = state.miner.clone(); 152 | 153 | tokio::spawn(async move { 154 | match incoming.await { 155 | Ok(conn) => { 156 | debug!("New connection from {}", conn.remote_address()); 157 | 158 | while let Ok((mut send, mut recv)) = conn.accept_bi().await { 159 | debug!("New bidirectional stream opened"); 160 | 161 | // Read the payload size 162 | let mut payload_size_buf = [0u8; 8]; 163 | if let Err(e) = recv.read_exact(&mut payload_size_buf).await { 164 | error!("Failed to read piece size: {e}"); 165 | continue; 166 | } 167 | let payload_size = u64::from_be_bytes(payload_size_buf) as usize; 168 | 169 | // Read signature payload and verify 170 | // let mut signature_buf = [0u8; size_of::()]; 171 | let mut signature_buf = vec![0u8; payload_size]; 172 | if let Err(e) = recv.read_exact(&mut signature_buf).await { 173 | error!("Failed to read signature: {e}"); 174 | continue; 175 | } 176 | debug!("signature_buf: {:?}", signature_buf); 177 | debug!("payload size: {:?}", payload_size); 178 | let signature_payload = 179 | match bincode::deserialize::(&signature_buf) { 180 | Ok(data) => data, 181 | Err(err) => { 182 | error!( 183 | "Failed to deserialize payload from bytes: {:?}", 184 | err 185 | ); 186 | continue; 187 | } 188 | }; 189 | let verified = verify_signature( 190 | &signature_payload.message.validator.account_id, 191 | &signature_payload.signature, 192 | &signature_payload.message, 193 | ); 194 | 195 | let address_book = 196 | miner.clone().neuron.read().await.address_book.clone(); 197 | 198 | info!( 199 | "Received handshake from validator {} with uid {}", 200 | signature_payload.message.validator.account_id, 201 | signature_payload.message.validator.uid 202 | ); 203 | 204 | let validator_info = if let Some(vali_info) = 205 | address_book.get(&signature_payload.message.validator.uid) 206 | { 207 | vali_info.clone() 208 | } else { 209 | error!("Error while getting validator info"); 210 | continue; 211 | }; 212 | let validator_hotkey = validator_info.neuron_info.hotkey.clone(); 213 | 214 | if !verified 215 | || validator_hotkey 216 | != signature_payload.message.validator.account_id 217 | { 218 | error!( 219 | "Failed to verify signature from validator with uid {}", 220 | signature_payload.message.validator.uid 221 | ); 222 | conn.close( 223 | VarInt::from_u32(401), 224 | "Signature verification failed".as_bytes(), 225 | ); 226 | return; 227 | } 228 | debug!("Signature verification successful:"); 229 | debug!("{:?}", signature_payload); 230 | 231 | // Read the piece size 232 | let mut piece_size_buf = [0u8; 8]; 233 | if let Err(e) = recv.read_exact(&mut piece_size_buf).await { 234 | error!("Failed to read piece size: {e}"); 235 | continue; 236 | } 237 | let piece_size = u64::from_be_bytes(piece_size_buf) as usize; 238 | debug!("Received piece size: {piece_size} bytes"); 239 | 240 | // Process the piece 241 | let mut buffer = vec![0u8; piece_size]; 242 | let mut bytes_read = 0; 243 | let mut piece = Vec::new(); 244 | 245 | while bytes_read < piece_size { 246 | match recv.read(&mut buffer[..piece_size - bytes_read]).await { 247 | Ok(Some(n)) if n > 0 => { 248 | piece.extend_from_slice(&buffer[..n]); 249 | bytes_read += n; 250 | } 251 | Ok(_) => break, // End of stream 252 | Err(e) => { 253 | error!("Error reading piece: {e}"); 254 | break; 255 | } 256 | } 257 | } 258 | 259 | if bytes_read == 0 { 260 | continue; // No data received 261 | } 262 | 263 | info!("Received piece of exactly {} bytes", bytes_read); 264 | 265 | let hash_raw = blake3::hash(&piece); 266 | let hash: String = hash_raw.to_hex().to_string(); 267 | 268 | if let Err(e) = send.write_all(hash_raw.as_bytes()).await { 269 | error!("Failed to send hash: {}", e); 270 | continue; 271 | } 272 | // Send delimiter after hash 273 | if let Err(e) = send.write_all(b"\n").await { 274 | error!("Failed to send delimiter: {}", e); 275 | continue; 276 | } 277 | 278 | let piece_hash = 279 | PieceHashStr::new(hash).expect("Failed to create PieceHash"); // TODO: handle error 280 | object_store 281 | .lock() 282 | .await 283 | .write(&piece_hash, &piece) 284 | .await 285 | .expect("Failed to write piece to store"); 286 | info!("Finished storing piece of size: {}", bytes_read); 287 | } 288 | } 289 | Err(e) => { 290 | error!("Connection failed: {}", e); 291 | } 292 | } 293 | }); 294 | } 295 | }); 296 | 297 | let _ = tokio::try_join!(http_server, quic_server); 298 | 299 | Ok(()) 300 | } 301 | 302 | /// Run the miner 303 | pub fn run(config: MinerConfig) { 304 | let _ = tokio::runtime::Builder::new_multi_thread() 305 | .enable_all() 306 | .build() 307 | .unwrap() 308 | .block_on(main(config)); 309 | } 310 | -------------------------------------------------------------------------------- /crates/storb_base/src/lib.rs: -------------------------------------------------------------------------------- 1 | use std::fs; 2 | use std::net::Ipv4Addr; 3 | use std::path::PathBuf; 4 | use std::sync::Arc; 5 | 6 | use crabtensor::api::runtime_apis::neuron_info_runtime_api::NeuronInfoRuntimeApi; 7 | use crabtensor::api::runtime_types::pallet_subtensor::rpc_info::neuron_info::NeuronInfoLite; 8 | use crabtensor::axon::{serve_axon_payload, AxonProtocol}; 9 | use crabtensor::subtensor::Subtensor; 10 | use crabtensor::wallet::{hotkey_location, load_key_seed, signer_from_seed, Signer}; 11 | use crabtensor::AccountId; 12 | use dashmap::DashMap; 13 | use libp2p::{multiaddr::multiaddr, Multiaddr}; 14 | use serde::ser::StdError; 15 | use subxt::utils::H256; 16 | use tokio::sync::RwLock; 17 | use tracing::{error, info}; 18 | 19 | use crate::version::Version; 20 | 21 | pub mod constants; 22 | pub mod memory_db; 23 | pub mod piece; 24 | pub mod piece_hash; 25 | pub mod sync; 26 | pub mod utils; 27 | pub mod verification; 28 | pub mod version; 29 | 30 | // Node UID : NodeInfo 31 | pub type NodeUID = u16; 32 | pub type AddressBook = Arc>; 33 | 34 | const SUBTENSOR_SERVING_RATE_LIMIT_EXCEEDED: &str = "Custom error: 12"; 35 | 36 | #[derive(Debug)] 37 | pub enum NeuronError { 38 | SubtensorError(String), 39 | ConfigError(String), 40 | NodeInfoError(String), 41 | } 42 | 43 | impl std::fmt::Display for NeuronError { 44 | fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { 45 | match self { 46 | NeuronError::SubtensorError(e) => write!(f, "Subtensor error: {}", e), 47 | NeuronError::ConfigError(e) => write!(f, "Configuration error: {}", e), 48 | NeuronError::NodeInfoError(e) => write!(f, "Node info error: {}", e), 49 | } 50 | } 51 | } 52 | 53 | impl std::error::Error for NeuronError {} 54 | 55 | #[derive(Clone, Debug)] 56 | pub struct SubtensorConfig { 57 | pub network: String, 58 | pub address: String, 59 | pub insecure: bool, 60 | } 61 | 62 | #[derive(Clone, Debug)] 63 | pub struct NeuronConfig { 64 | pub sync_frequency: u64, 65 | } 66 | 67 | #[derive(Clone, Debug)] 68 | pub struct BaseNeuronConfig { 69 | pub version: Version, 70 | pub netuid: u16, 71 | pub external_ip: String, 72 | pub api_port: u16, 73 | pub quic_port: Option, 74 | pub post_ip: bool, 75 | 76 | pub wallet_path: PathBuf, 77 | pub wallet_name: String, 78 | pub hotkey_name: String, 79 | 80 | pub otel_api_key: String, 81 | pub otel_endpoint: String, 82 | pub otel_service_name: String, 83 | 84 | pub mock: bool, 85 | 86 | pub load_old_nodes: bool, 87 | pub min_stake_threshold: u64, 88 | 89 | pub db_file: PathBuf, 90 | pub metadatadb_file: PathBuf, 91 | pub neurons_dir: PathBuf, 92 | 93 | pub subtensor: SubtensorConfig, 94 | 95 | pub neuron: NeuronConfig, 96 | } 97 | 98 | #[derive(Debug, Clone)] 99 | pub struct NodeInfo { 100 | pub neuron_info: NeuronInfoLite, 101 | pub http_address: Option, 102 | pub quic_address: Option, 103 | pub version: Version, 104 | } 105 | 106 | #[derive(Clone, serde::Serialize, serde::Deserialize)] 107 | pub struct LocalNodeInfo { 108 | pub uid: Option, 109 | pub http_address: Option, 110 | pub quic_address: Option, 111 | pub version: Version, 112 | } 113 | 114 | // TODO: Break this up into separateto prevent locking constantly 115 | #[derive(Clone)] 116 | pub struct BaseNeuron { 117 | pub config: BaseNeuronConfig, 118 | pub neurons: Arc>>>, 119 | pub address_book: AddressBook, 120 | pub signer: Signer, 121 | pub local_node_info: LocalNodeInfo, 122 | } 123 | 124 | // TODO: add a function for loading neuron state? (see TODO(420)) 125 | impl BaseNeuron { 126 | pub async fn get_subtensor_connection( 127 | insecure: bool, 128 | url: &String, 129 | ) -> Result { 130 | if insecure { 131 | return Subtensor::from_insecure_url(url) 132 | .await 133 | .map_err(|e| NeuronError::SubtensorError(e.to_string())); 134 | } 135 | 136 | Subtensor::from_url(url) 137 | .await 138 | .map_err(|e| NeuronError::SubtensorError(e.to_string())) 139 | } 140 | 141 | pub async fn new(config: BaseNeuronConfig) -> Result { 142 | let wallet_path: PathBuf = config.wallet_path.clone(); 143 | 144 | let hotkey_location: PathBuf = hotkey_location( 145 | wallet_path, 146 | config.wallet_name.clone(), 147 | config.hotkey_name.clone(), 148 | ); 149 | 150 | info!("Loading hotkey: path = {:?}", hotkey_location); 151 | 152 | let seed = load_key_seed(&hotkey_location).unwrap(); 153 | 154 | let signer = signer_from_seed(&seed).unwrap(); 155 | 156 | // try and load the neurons from disk - load state 157 | let neurons_arc = 158 | if let Ok(loaded_neurons) = Self::load_neurons_state_from_disk(&config.neurons_dir) { 159 | info!("Loaded {} neurons from saved state", loaded_neurons.len()); 160 | Arc::new(RwLock::new(loaded_neurons)) 161 | } else { 162 | info!("No saved neuron state found, starting fresh"); 163 | Arc::new(RwLock::new(Vec::new())) 164 | }; 165 | 166 | let address_book = Arc::new(DashMap::new()); 167 | 168 | let external_ip: Ipv4Addr = config.external_ip.parse().expect("Invalid IP address"); 169 | 170 | let local_http_address = Some(multiaddr!(Ip4(external_ip), Tcp(config.api_port))); 171 | 172 | let local_quic_address = config 173 | .quic_port 174 | .map(|port| multiaddr!(Ip4(external_ip), Udp(port))); 175 | 176 | let local_node_info = LocalNodeInfo { 177 | uid: None, 178 | http_address: local_http_address, 179 | quic_address: local_quic_address, 180 | version: config.version.clone(), 181 | }; 182 | 183 | let neuron = BaseNeuron { 184 | config: config.clone(), 185 | neurons: neurons_arc, 186 | address_book: address_book.clone(), 187 | signer, 188 | local_node_info, 189 | }; 190 | 191 | // Check registration status 192 | neuron.check_registration().await?; 193 | 194 | // Post IP if specified 195 | info!("Should post ip?: {}", config.post_ip); 196 | if config.post_ip { 197 | let address = format!("{}:{}", config.external_ip, config.api_port) 198 | .parse() 199 | .expect("Failed to parse address when attempting to post IP. Is it correct?"); 200 | info!("Serving axon as: {}", address); 201 | 202 | let payload = serve_axon_payload(config.netuid, address, AxonProtocol::Udp); 203 | 204 | let subtensor = Self::get_subtensor_connection( 205 | config.subtensor.insecure, 206 | &config.subtensor.address, 207 | ) 208 | .await?; 209 | subtensor 210 | .tx() 211 | .sign_and_submit_default(&payload, &neuron.signer) 212 | .await 213 | .unwrap_or_else(|err| { 214 | error!("Failed to post IP to Subtensor"); 215 | if format!("{:?}", err).contains(SUBTENSOR_SERVING_RATE_LIMIT_EXCEEDED) { 216 | error!("Invalid Transaction: {err}"); 217 | error!("Axon info not updated due to rate limit"); 218 | H256::zero() 219 | } else { 220 | panic!("Unexpected error: {err}"); 221 | } 222 | }); 223 | 224 | info!("Successfully served axon!"); 225 | } 226 | 227 | Ok(neuron) 228 | } 229 | 230 | /// Get info of a neuron (node) from a list of neurons that matches the account ID. 231 | pub fn find_neuron_info<'a>( 232 | neurons: &'a [NeuronInfoLite], 233 | account_id: &AccountId, 234 | ) -> Option<&'a NeuronInfoLite> { 235 | neurons.iter().find(|neuron| &neuron.hotkey == account_id) 236 | } 237 | 238 | /// Check whether the neuron is registered in the subnet or not. 239 | pub async fn check_registration(&self) -> Result<(), NeuronError> { 240 | let subtensor = Self::get_subtensor_connection( 241 | self.config.subtensor.insecure, 242 | &self.config.subtensor.address, 243 | ) 244 | .await?; 245 | let current_block = subtensor.blocks().at_latest().await.unwrap(); 246 | let runtime_api = subtensor.runtime_api().at(current_block.reference()); 247 | 248 | // TODO: is there a nicer way to pass self to NeuronInfoRuntimeApi? 249 | let neurons_payload = 250 | NeuronInfoRuntimeApi::get_neurons_lite(&NeuronInfoRuntimeApi {}, self.config.netuid); 251 | 252 | // TODO: error out if cant access chain when calling above function 253 | 254 | let neurons: Vec> = 255 | runtime_api.call(neurons_payload).await.unwrap(); 256 | 257 | let neuron_info = Self::find_neuron_info(&neurons, self.signer.account_id()); 258 | match neuron_info { 259 | Some(_) => Ok(()), 260 | None => Err(NeuronError::ConfigError( 261 | "Neuron is not registered".to_string(), 262 | )), 263 | } 264 | } 265 | 266 | pub fn save_neurons_state_to_disk( 267 | neurons_dir: &PathBuf, 268 | neurons: &[NeuronInfoLite], 269 | ) -> Result<(), Box> { 270 | let buf = bincode::serialize(&neurons) 271 | .map_err(|e| format!("Failed to serialize neurons: {}", e))?; 272 | 273 | fs::write(neurons_dir, buf) 274 | .map_err(|e| format!("Failed to write neurons state: {}", e).into()) 275 | } 276 | 277 | pub fn load_neurons_state_from_disk( 278 | neurons_dir: &PathBuf, 279 | ) -> Result>, Box> { 280 | if !neurons_dir.exists() { 281 | return Ok(Vec::new()); 282 | } 283 | 284 | let buf = 285 | fs::read(neurons_dir).map_err(|e| format!("Failed to read neurons state: {}", e))?; 286 | 287 | bincode::deserialize(&buf) 288 | .map_err(|e| format!("Failed to deserialize neurons: {}", e).into()) 289 | } 290 | } 291 | 292 | #[cfg(test)] 293 | mod tests { 294 | use std::fs::{self, create_dir_all}; 295 | 296 | use super::*; 297 | 298 | fn setup_test_wallet() -> (PathBuf, String) { 299 | let temp_dir = std::env::temp_dir().join("storb_test_wallets"); 300 | create_dir_all(&temp_dir).unwrap(); 301 | 302 | let wallet_name = "test_wallet"; 303 | let hotkey_name = "test_hotkey"; 304 | 305 | let wallet_path = temp_dir.join(wallet_name).join("hotkeys"); 306 | create_dir_all(&wallet_path).unwrap(); 307 | 308 | let hotkey_file = wallet_path.join(hotkey_name); 309 | let random_seed = vec![0u8; 32]; 310 | fs::write(&hotkey_file, random_seed).unwrap(); 311 | 312 | (temp_dir, wallet_name.to_string()) 313 | } 314 | 315 | fn get_test_config() -> BaseNeuronConfig { 316 | let (wallet_path, wallet_name) = setup_test_wallet(); 317 | 318 | BaseNeuronConfig { 319 | version: Version { 320 | major: 1, 321 | minor: 0, 322 | patch: 0, 323 | }, 324 | netuid: 1, 325 | external_ip: "127.0.0.1".to_string(), 326 | api_port: 8080, 327 | quic_port: Some(8081), 328 | post_ip: false, 329 | wallet_path, 330 | wallet_name, 331 | hotkey_name: "test_hotkey".to_string(), 332 | mock: false, 333 | load_old_nodes: false, 334 | min_stake_threshold: 0, 335 | db_file: "./test.db".into(), 336 | metadatadb_file: "./test.db".into(), 337 | neurons_dir: "./test_neurons".into(), 338 | subtensor: SubtensorConfig { 339 | network: "finney".to_string(), 340 | address: "wss://test.finney.opentensor.ai:443".to_string(), 341 | insecure: false, 342 | }, 343 | neuron: NeuronConfig { 344 | sync_frequency: 100, 345 | }, 346 | otel_api_key: "".to_string(), 347 | otel_endpoint: "".to_string(), 348 | otel_service_name: "".to_string(), 349 | } 350 | } 351 | 352 | #[tokio::test] 353 | async fn test_get_subtensor_connection() { 354 | let config = get_test_config(); 355 | let result = BaseNeuron::get_subtensor_connection( 356 | config.subtensor.insecure, 357 | &config.subtensor.address, 358 | ) 359 | .await; 360 | assert!(result.is_ok()); 361 | } 362 | 363 | // TODO: update tests 364 | 365 | // #[tokio::test] 366 | // async fn test_find_neuron_info() { 367 | // let neurons = vec![]; 368 | // let account_id = AccountId::from([0; 32]); 369 | // info!("account_id: {account_id}"); 370 | // let result = BaseNeuron::find_neuron_info(&neurons, &account_id); 371 | // assert!(result.is_none()); 372 | // } 373 | 374 | // TODO: get wallet handling working properly for testing 375 | // #[tokio::test] 376 | // async fn test_check_registration() { 377 | // let config = get_test_config(); 378 | // let neuron = BaseNeuron::new(config).await.unwrap(); 379 | // let result = neuron.check_registration().await; 380 | // assert!(result.is_err()); // Should fail since test wallet not registered 381 | // } 382 | 383 | // #[tokio::test] 384 | // async fn test_sync_metagraph() { 385 | // let config = get_test_config(); 386 | // let mut neuron = BaseNeuron::new(config).await.unwrap(); 387 | // neuron.sync_metagraph().await; 388 | // let neurons = neuron.get_neurons(); 389 | // let neurons_read = neurons.read().unwrap(); 390 | // assert!(neurons_read.len() > 0); // Verify we can read neurons after sync 391 | // } 392 | 393 | // impl Drop for BaseNeuron { 394 | // fn drop(&mut self) { 395 | // if Path::new(&self.config.wallet_path).exists() { 396 | // let _ = fs::remove_dir_all(&self.config.wallet_path); 397 | // } 398 | // } 399 | // } 400 | } 401 | -------------------------------------------------------------------------------- /crates/storb_validator/src/lib.rs: -------------------------------------------------------------------------------- 1 | #![allow(clippy::uninlined_format_args)] 2 | use std::collections::HashMap; 3 | use std::net::SocketAddr; 4 | use std::sync::Arc; 5 | use std::time::Duration; 6 | 7 | use anyhow::{Context, Result}; 8 | use axum::extract::DefaultBodyLimit; 9 | use axum::middleware::from_fn; 10 | use axum::routing::{delete, get, post}; 11 | use axum::{Extension, Router}; 12 | use base::constants::NEURON_SYNC_TIMEOUT; 13 | use base::sync::Synchronizable; 14 | use base::{LocalNodeInfo, NeuronError}; 15 | use chrono::TimeDelta; 16 | use constants::{METADATADB_SYNC_FREQUENCY, SYNTHETIC_CHALLENGE_FREQUENCY}; 17 | use dashmap::DashMap; 18 | use middleware::{require_api_key, InfoApiRateLimiter}; 19 | use opentelemetry::global; 20 | use opentelemetry_otlp::{MetricExporter, WithExportConfig, WithHttpConfig}; 21 | use opentelemetry_sdk::metrics::SdkMeterProvider; 22 | use opentelemetry_sdk::Resource; 23 | use routes::{download_file, node_info, upload_file}; 24 | use tokio::time::interval; 25 | use tokio::{sync::RwLock, time}; 26 | use tracing::{debug, error, info, warn}; 27 | use validator::{Validator, ValidatorConfig}; 28 | 29 | use crate::constants::{NONCE_CLEANUP_FREQUENCY, NONCE_EXPIRATION_TIME}; 30 | use crate::metadata::sync::sync_metadata_db; 31 | use crate::routes::{delete_file, generate_nonce, get_crsqlite_changes}; 32 | 33 | pub mod apikey; 34 | mod constants; 35 | mod download; 36 | mod metadata; 37 | mod middleware; 38 | mod quic; 39 | mod repair; 40 | mod routes; 41 | mod scoring; 42 | mod signature; 43 | mod upload; 44 | mod utils; 45 | pub mod validator; 46 | 47 | const MAX_BODY_SIZE: usize = 10 * 1024 * 1024 * 1024; // 10 GiB 48 | 49 | /// State maintained by the validator service 50 | /// 51 | /// We derive Clone here to allow this state to be shared between request handlers, 52 | /// as Axum requires state types to be cloneable to share them across requests. 53 | #[derive(Clone)] 54 | struct ValidatorState { 55 | pub validator: Arc, 56 | pub local_node_info: LocalNodeInfo, 57 | } 58 | 59 | /// QUIC validator server that accepts file uploads, sends files to miner via QUIC, 60 | /// and returns hashes. This server serves as an intermediary between HTTP clients 61 | /// and the backing storage+processing miner. Below is an overview of how it works: 62 | /// 63 | /// 1. Producer produces bytes 64 | /// - a. Read collection of bytes from multipart form 65 | /// - b. Fill up a shared buffer with that collection 66 | /// - c. Signal that its done 67 | /// 2. Consumer consumes bytes 68 | /// - a. Reads a certain chunk of the collection bytes from shared buffer 69 | /// - b. FECs it into pieces. A background thread is spawned to: 70 | /// - distribute these pieces to a selected set of miners 71 | /// - verify pieces are being stored 72 | /// - update miner statistics 73 | /// 74 | /// On success, returns Ok(()). 75 | /// On failure, returns error with details of the failure. 76 | pub async fn run_validator(config: ValidatorConfig) -> Result<()> { 77 | let validator = Arc::new(Validator::new(config.clone()).await?); 78 | 79 | let neuron = validator.neuron.clone(); 80 | 81 | let validator_for_sync = validator.clone(); 82 | let validator_for_metadatadb = validator.clone(); 83 | let validator_for_repair = validator.clone(); 84 | let validator_for_challenges = validator.clone(); 85 | let validator_for_backup = validator.clone(); 86 | let validator_for_nonce_cleanup = validator.clone(); 87 | let sync_frequency = config.clone().neuron_config.neuron.sync_frequency; 88 | 89 | let sync_config = config.clone(); 90 | 91 | let identifier_resource = Resource::builder() 92 | .with_attribute(opentelemetry::KeyValue::new( 93 | "service.name", 94 | config.otel_service_name, 95 | )) 96 | .build(); 97 | let mut otel_headers: HashMap = HashMap::new(); 98 | otel_headers.insert("X-Api-Key".to_string(), config.otel_api_key.clone()); 99 | let url = config.otel_endpoint.clone() + "metrics"; 100 | let metrics_exporter = MetricExporter::builder() 101 | .with_http() 102 | .with_endpoint(url) 103 | .with_protocol(opentelemetry_otlp::Protocol::HttpBinary) 104 | .with_headers(otel_headers) 105 | .build() 106 | .map_err(|e| { 107 | NeuronError::ConfigError(format!("Failed to build OTEL MetricExporter: {:?}", e)) 108 | })?; 109 | 110 | let metrics_provider = SdkMeterProvider::builder() 111 | .with_periodic_exporter(metrics_exporter) 112 | .with_resource(identifier_resource) 113 | .build(); 114 | 115 | global::set_meter_provider(metrics_provider.clone()); 116 | 117 | let meter = global::meter("validator::metrics"); 118 | let weights_counter = meter 119 | .f64_counter("miner.weight") 120 | .with_description("Current weight per miner") 121 | .with_unit("score") 122 | .build(); 123 | 124 | info!("Set up OTEL metrics exporter"); 125 | 126 | tokio::spawn(async move { 127 | let local_validator = validator_for_sync; 128 | let scoring_system = local_validator.scoring_system.clone(); 129 | let neuron = neuron.clone(); 130 | 131 | info!( 132 | "Starting validator sync task with frequency: {} seconds", 133 | sync_frequency 134 | ); 135 | let mut interval = time::interval(Duration::from_secs(sync_frequency)); 136 | loop { 137 | interval.tick().await; 138 | info!("Syncing validator"); 139 | // Wrap the sync operation in a timeout 140 | match tokio::time::timeout(NEURON_SYNC_TIMEOUT, async { 141 | let start = std::time::Instant::now(); 142 | let sync_result = neuron.write().await.sync_metagraph().await; 143 | (sync_result, start.elapsed()) 144 | }) 145 | .await 146 | { 147 | Ok((result, duration)) => match result { 148 | Ok(uids) => { 149 | info!("Sync completed in {:?}", duration); 150 | let neuron_guard = neuron.read().await; 151 | let neuron_count = neuron_guard.neurons.read().await.len(); 152 | scoring_system 153 | .write() 154 | .await 155 | .update_scores(neuron_count, uids.clone()) 156 | .await; 157 | 158 | let ema_scores = 159 | scoring_system.clone().read().await.state.ema_scores.clone(); 160 | 161 | match Validator::set_weights( 162 | neuron_guard.clone(), 163 | ema_scores, 164 | sync_config.clone(), 165 | weights_counter.clone(), 166 | ) 167 | .await 168 | { 169 | Ok(_) => info!("Successfully set weights on chain"), 170 | Err(e) => error!("Failed to set weights on chain: {}", e), 171 | }; 172 | 173 | // Queue pieces for the deregistered uids for repair 174 | for uid in uids.iter() { 175 | match metadata::db::MetadataDB::queue_pieces_for_repair( 176 | &local_validator.metadatadb_sender, 177 | *uid, 178 | ) 179 | .await 180 | { 181 | Ok(_) => info!("Queued pieces for UID {} for repair", uid), 182 | Err(e) => error!("Failed to queue pieces for UID {}: {}", uid, e), 183 | }; 184 | } 185 | } 186 | Err(err) => { 187 | error!("Failed to sync metagraph: {}", err); 188 | } 189 | }, 190 | Err(_elapsed) => { 191 | error!( 192 | "Sync operation timed out after {} seconds", 193 | NEURON_SYNC_TIMEOUT.as_secs_f32() 194 | ); 195 | error!( 196 | "Current stack trace:\n{:?}", 197 | std::backtrace::Backtrace::capture() 198 | ); 199 | 200 | // Additional debugging info 201 | let guard = neuron.try_read(); 202 | match guard { 203 | Ok(_) => info!("Neuron read lock is available"), 204 | Err(_) => error!("Neuron read lock is held - possible deadlock"), 205 | }; 206 | 207 | let scoring_guard = scoring_system.try_write(); 208 | match scoring_guard { 209 | Ok(_) => info!("Scoring system write lock is available"), 210 | Err(_) => error!("Scoring system write lock is held - possible deadlock"), 211 | }; 212 | } 213 | } 214 | info!("Done syncing validator"); 215 | } 216 | }); 217 | 218 | // Spawn background synthetic challenge tasks 219 | tokio::spawn(async move { 220 | let mut interval = time::interval(Duration::from_secs(SYNTHETIC_CHALLENGE_FREQUENCY)); 221 | loop { 222 | // Interval tick MUST be called before the validator write, or else it will block 223 | // for in the initial loop-through 224 | interval.tick().await; 225 | info!("Running synthetic challenges"); 226 | match validator_for_challenges.run_synthetic_challenges().await { 227 | Ok(_) => debug!("Synthetic challenges ran successfully"), 228 | Err(err) => error!("Synthetic challenges failed to run: {err}"), 229 | }; 230 | } 231 | }); 232 | 233 | // Spawn background metadata database syncing task with metadata::sync::sync_metadata_db 234 | tokio::spawn(async move { 235 | let mut interval = interval(Duration::from_secs(METADATADB_SYNC_FREQUENCY)); 236 | loop { 237 | interval.tick().await; 238 | info!("Starting MetadataDB sync task"); 239 | match sync_metadata_db(validator_for_metadatadb.clone()).await { 240 | Ok(_) => info!("Metadata DB sync completed successfully"), 241 | Err(err) => error!("Metadata DB sync failed: {}", err), 242 | } 243 | } 244 | }); 245 | 246 | // Spawn background piece repair task 247 | tokio::spawn(async move { 248 | let mut interval: time::Interval = 249 | interval(Duration::from_secs(constants::PIECE_REPAIR_FREQUENCY)); 250 | loop { 251 | interval.tick().await; 252 | info!("Running piece repair task"); 253 | match repair::repair_pieces(validator_for_repair.clone()).await { 254 | Ok(_) => info!("Piece repair task completed successfully"), 255 | Err(err) => warn!("Piece repair task failed: {}", err), 256 | } 257 | } 258 | }); 259 | 260 | let vali_clone = validator_for_backup.clone(); 261 | // Spawn background backup task 262 | tokio::spawn(async move { 263 | let mut interval = interval(Duration::from_secs(sync_frequency)); 264 | loop { 265 | interval.tick().await; 266 | vali_clone 267 | .scoring_system 268 | .write() 269 | .await 270 | .db 271 | .run_backup() 272 | .await; // TODO: constant 273 | } 274 | }); 275 | 276 | // Add a periodic cleanup task for expired nonces 277 | tokio::spawn(async move { 278 | let mut interval = 279 | tokio::time::interval(std::time::Duration::from_secs(NONCE_CLEANUP_FREQUENCY)); 280 | 281 | loop { 282 | interval.tick().await; 283 | info!("Running nonce cleanup task"); 284 | // get the command sender from the validator 285 | let command_sender = validator_for_nonce_cleanup.metadatadb_sender.clone(); 286 | match metadata::db::MetadataDB::cleanup_expired_nonces( 287 | &command_sender, 288 | TimeDelta::try_seconds(NONCE_EXPIRATION_TIME as i64) 289 | .expect("Failed to create TimeDelta for nonce expiration"), 290 | ) 291 | .await 292 | { 293 | Ok(deleted_count) => { 294 | if deleted_count > 0 { 295 | info!("Cleaned up {} expired nonces", deleted_count); 296 | } 297 | } 298 | Err(e) => { 299 | error!("Failed to cleanup expired nonces: {:?}", e); 300 | } 301 | } 302 | } 303 | }); 304 | 305 | let db_path = config.api_keys_db.clone(); 306 | // Initialize API key manager 307 | let api_key_manager = Arc::new(RwLock::new(apikey::ApiKeyManager::new(db_path)?)); 308 | let info_api_rate_limit_state: InfoApiRateLimiter = Arc::new(DashMap::new()); 309 | 310 | // Create protected routes that require API key 311 | let protected_routes = Router::new() 312 | .route("/file", post(upload_file)) 313 | .route("/file", get(download_file)) 314 | .route("/file", delete(delete_file)) 315 | .route("/nonce", post(generate_nonce)) 316 | .route_layer(from_fn(require_api_key)); 317 | // Create main router with all routes 318 | let app = Router::new() 319 | .merge(protected_routes) // Add protected routes 320 | .route( 321 | "/info", 322 | get(node_info).route_layer(from_fn(middleware::info_api_rate_limit_middleware)), 323 | ) 324 | .layer(Extension(info_api_rate_limit_state)) 325 | .layer(Extension(api_key_manager)) 326 | .layer(DefaultBodyLimit::max(MAX_BODY_SIZE)) 327 | .route( 328 | "/db_changes", 329 | get(get_crsqlite_changes), // TODO(syncing): Add db change rate limiting middleware? 330 | ) 331 | .with_state(ValidatorState { 332 | validator: validator.clone(), 333 | local_node_info: validator.neuron.read().await.local_node_info.clone(), 334 | }); 335 | 336 | // Start server 337 | let addr = SocketAddr::from(([0, 0, 0, 0], config.neuron_config.api_port)); 338 | info!("Validator HTTP server listening on {}", addr); 339 | 340 | axum::serve( 341 | tokio::net::TcpListener::bind(addr) 342 | .await 343 | .context("Failed to bind HTTP server")?, 344 | app.into_make_service_with_connect_info::(), 345 | ) 346 | .await 347 | .context("HTTP server failed")?; 348 | 349 | Ok(()) 350 | } 351 | 352 | /// Main entry point for the validator service 353 | async fn main(config: ValidatorConfig) { 354 | if std::env::var("PANIC_ABORT") 355 | .map(|v| v == "1") 356 | .unwrap_or(false) 357 | { 358 | std::panic::set_hook(Box::new(|info| { 359 | eprintln!("One of the threads panicked {}", info); 360 | std::process::abort(); 361 | })); 362 | } 363 | rustls::crypto::ring::default_provider() 364 | .install_default() 365 | .expect("Failed to install rustls crypto provider"); 366 | run_validator(config) 367 | .await 368 | .expect("Failed to run the validator") 369 | } 370 | 371 | /// Runs the main async runtime 372 | pub fn run(config: ValidatorConfig) { 373 | tokio::runtime::Builder::new_multi_thread() 374 | .enable_all() 375 | .build() 376 | .unwrap() 377 | .block_on(main(config)) 378 | } 379 | --------------------------------------------------------------------------------