├── GFS.png ├── .gitignore ├── .env.docker ├── .env ├── Dockerfile ├── docker-compose.yml ├── src ├── config.rs ├── client.rs ├── auth.rs ├── main.rs ├── worker.rs └── master.rs ├── test-harness ├── deno.json ├── deno.lock └── test.js ├── Cargo.toml ├── README.md └── Cargo.lock /GFS.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/watthedoodle/rdfs/HEAD/GFS.png -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | test.txt 3 | sandbox/ 4 | snapshot 5 | prune 6 | *.new -------------------------------------------------------------------------------- /.env.docker: -------------------------------------------------------------------------------- 1 | RDFS_ENDPOINT=http://master:8888 2 | RDFS_TOKEN=695bfaf2-f381-470b-945c-6cb11fa7a73c -------------------------------------------------------------------------------- /.env: -------------------------------------------------------------------------------- 1 | export RDFS_ENDPOINT=http://localhost:8888 2 | export RDFS_TOKEN=695bfaf2-f381-470b-945c-6cb11fa7a73c -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM rust:1.81-alpine AS builder 2 | WORKDIR /usr/src/rdfs 3 | COPY . . 4 | RUN apk add openssl-dev musl-dev 5 | RUN cargo build --release 6 | 7 | FROM alpine 8 | COPY --from=builder /usr/src/rdfs/target/release/rdfs /usr/local/bin/rdfs 9 | ENTRYPOINT ["rdfs"] -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | master: 3 | image: rdfs 4 | build: 5 | context: . 6 | args: 7 | DOCKER_BUILDKIT: 1 8 | ports: 9 | - 8888:8888 10 | env_file: .env.docker 11 | command: mode master 12 | 13 | worker: 14 | image: rdfs 15 | env_file: .env.docker 16 | command: mode worker 17 | -------------------------------------------------------------------------------- /src/config.rs: -------------------------------------------------------------------------------- 1 | use std::env; 2 | 3 | #[derive(Clone)] 4 | pub struct Config { 5 | pub endpoint: String, 6 | pub token: String, 7 | } 8 | 9 | pub fn get() -> Option { 10 | if let (Ok(x), Ok(y)) = (env::var("RDFS_ENDPOINT"), env::var("RDFS_TOKEN")) { 11 | return Some(Config { 12 | endpoint: x, 13 | token: y, 14 | }); 15 | } 16 | None 17 | } 18 | -------------------------------------------------------------------------------- /test-harness/deno.json: -------------------------------------------------------------------------------- 1 | { 2 | "tasks": { 3 | "test-token": "deno test --allow-all --filter 'token'", 4 | "test-get": "deno test --allow-all --filter 'get-chunk-ok'", 5 | "test-store": "deno test --allow-all --filter 'store-chunk'", 6 | "test-delete": "deno test --allow-all --filter 'delete-chunk'", 7 | "test-send": "deno test --allow-all --filter 'send-chunk'", 8 | "test": "deno test --allow-all test.js" 9 | } 10 | } -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "rdfs" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | [dependencies] 7 | axum = { version = "0.7.7", features = ["macros"] } 8 | base64 = "0.22.1" 9 | chrono = "0.4.38" 10 | clap = { version = "^4.5.18", features = ["derive"] } 11 | lazy_static = "1.5.0" 12 | rand = "0.8.5" 13 | serde = { version = "1.0.210", features = ["derive"] } 14 | serde_json = "1.0.128" 15 | tokio = { version = "1.40.0", features = ["rt-multi-thread"] } 16 | tracing = "0.1.40" 17 | tracing-subscriber = "0.3.18" 18 | ureq = { version = "2.10.1", features = ["json"] } 19 | -------------------------------------------------------------------------------- /src/client.rs: -------------------------------------------------------------------------------- 1 | use tracing::info; 2 | 3 | pub fn list(path: &Option) { 4 | info!( 5 | "todo: list all remote objects based on optional path '{:?}'...", 6 | path 7 | ) 8 | } 9 | 10 | pub fn get(file: &String) { 11 | info!("todo: get remote objects based on file name '{}'...", file); 12 | } 13 | 14 | pub fn add(file: &String) { 15 | info!("todo: add remote objects based on file name '{}'...", file); 16 | } 17 | 18 | pub fn remove(file: &String) { 19 | info!( 20 | "todo: remove remote object based on file name '{}'...", 21 | file 22 | ); 23 | } 24 | -------------------------------------------------------------------------------- /test-harness/deno.lock: -------------------------------------------------------------------------------- 1 | { 2 | "version": "3", 3 | "packages": { 4 | "specifiers": { 5 | "jsr:@std/assert": "jsr:@std/assert@1.0.6", 6 | "jsr:@std/internal@^1.0.4": "jsr:@std/internal@1.0.4" 7 | }, 8 | "jsr": { 9 | "@std/assert@1.0.6": { 10 | "integrity": "1904c05806a25d94fe791d6d883b685c9e2dcd60e4f9fc30f4fc5cf010c72207", 11 | "dependencies": [ 12 | "jsr:@std/internal@^1.0.4" 13 | ] 14 | }, 15 | "@std/internal@1.0.4": { 16 | "integrity": "62e8e4911527e5e4f307741a795c0b0a9e6958d0b3790716ae71ce085f755422" 17 | } 18 | } 19 | }, 20 | "remote": {} 21 | } 22 | -------------------------------------------------------------------------------- /src/auth.rs: -------------------------------------------------------------------------------- 1 | use axum::{ 2 | extract::Request, 3 | http::{HeaderMap, StatusCode}, 4 | middleware::Next, 5 | response::Response, 6 | }; 7 | 8 | use crate::config; 9 | 10 | pub async fn authorise( 11 | headers: HeaderMap, 12 | request: Request, 13 | next: Next, 14 | ) -> Result { 15 | if let Some(config) = config::get() { 16 | if let Some(token) = headers.get("x-rdfs-token") { 17 | if *token == *config.token { 18 | let response = next.run(request).await; 19 | return Ok(response); 20 | } 21 | } 22 | } 23 | Err(StatusCode::UNAUTHORIZED) 24 | } 25 | -------------------------------------------------------------------------------- /src/main.rs: -------------------------------------------------------------------------------- 1 | use clap::{Parser, Subcommand}; 2 | use tracing::warn; 3 | #[macro_use] 4 | extern crate lazy_static; 5 | 6 | mod auth; 7 | mod client; 8 | mod config; 9 | mod master; 10 | mod worker; 11 | 12 | const LOGO: &'static str = r#" 13 | 14 | ██████ ██████ ███████ ███████ 15 | ██ ██ ██ ██ ██ ██ 16 | ██████ ██ ██ █████ ███████ 17 | ██ ██ ██ ██ ██ ██ 18 | ██ ██ ██████ ██ ███████ 19 | 20 | a toy distributed file system 21 | "#; 22 | 23 | #[derive(Parser, Default, Debug)] 24 | #[clap( 25 | author = "Wat The Doodle ", 26 | version, 27 | about=LOGO 28 | )] 29 | #[command(help_template( 30 | "\ 31 | {before-help}{name} {version} 32 | {author-with-newline}{about-with-newline} 33 | {usage-heading} {usage} 34 | 35 | {all-args}{after-help} 36 | " 37 | ))] 38 | struct Arguments { 39 | /// use commands: List, Get, Add, and Remove 40 | #[command(subcommand)] 41 | cmd: Option, 42 | } 43 | 44 | #[derive(Subcommand, Debug, Clone)] 45 | enum Commands { 46 | /// List all remote files e.g rdfs list 47 | List { path: Option }, 48 | /// Get a remote file e.g rdfs get foo.txt 49 | Get { file: String }, 50 | /// Add a remote file e.g rdfs add foo.txt 51 | Add { file: String }, 52 | /// Remove a remote file e.g rdfs remove foo.txt 53 | Remove { file: String }, 54 | /// Mode: run the binary in either as a "Master" or "Worker" node 55 | Mode { 56 | /// kind: allowed values are "master" or "worker" 57 | kind: String, 58 | /// port: a custom port. default is 8888 59 | port: Option, 60 | }, 61 | } 62 | 63 | #[tokio::main] 64 | async fn main() { 65 | tracing_subscriber::fmt::init(); 66 | let args = Arguments::parse(); 67 | 68 | match &args.cmd { 69 | Some(Commands::List { path }) => { 70 | client::list(path); 71 | } 72 | Some(Commands::Get { file }) => { 73 | client::get(&file); 74 | } 75 | Some(Commands::Add { file }) => { 76 | client::add(&file); 77 | } 78 | Some(Commands::Remove { file }) => client::remove(&file), 79 | Some(Commands::Mode { kind, port }) => match kind.as_ref() { 80 | "master" => { 81 | let default_port = match port { 82 | Some(p) => p, 83 | None => &8888, 84 | }; 85 | let _ = master::init(default_port).await; 86 | } 87 | "worker" => { 88 | let default_port = match port { 89 | Some(p) => p, 90 | None => &8888, 91 | }; 92 | let _ = worker::init(default_port).await; 93 | } 94 | _ => { 95 | warn!("illegal mode, please select option master or worker!"); 96 | } 97 | }, 98 | None => { 99 | warn!("Unknown command!"); 100 | } 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /test-harness/test.js: -------------------------------------------------------------------------------- 1 | import { assert, assertEquals } from "jsr:@std/assert"; 2 | 3 | /* ------------------------------------------------------------------------------------------------- 4 | WARNING: we are assuming that the worker node is already running, later on we may need 5 | some automations in order to spin up the worker node before running this test 6 | ------------------------------------------------------------------------------------------------- */ 7 | const Token = "695bfaf2-f381-470b-945c-6cb11fa7a73c"; 8 | 9 | Deno.test("x-rdfs-token", async () => { 10 | let _ = await fetch("http://localhost:8888/", { 11 | headers: { 12 | "x-rdfs-token": Token, 13 | }, 14 | }).then((x) => x.text().then((data) => ({ status: x.status, body: data }))) 15 | .then((data) => assertEquals(data.status, 200)); 16 | }); 17 | 18 | Deno.test("get-chunk-404", async () => { 19 | let _ = await fetch("http://localhost:8888/get-chunk", { 20 | method: "POST", 21 | headers: { 22 | "x-rdfs-token": Token, 23 | "Content-Type": "application/json", 24 | }, 25 | body: JSON.stringify({ "id": "this-file-does-not-exist" }), 26 | }).then((x) => x.text().then((data) => ({ status: x.status, body: data }))) 27 | .then((data) => { 28 | // console.log(data.body) 29 | assertEquals(data.status, 404); 30 | }); 31 | }); 32 | 33 | Deno.test("get-chunk-ok", async () => { 34 | let _ = await fetch("http://localhost:8888/get-chunk", { 35 | method: "POST", 36 | headers: { 37 | "x-rdfs-token": Token, 38 | "Content-Type": "application/json", 39 | }, 40 | body: JSON.stringify({ "id": "README.md" }), 41 | }).then((x) => x.text().then((data) => ({ status: x.status, body: data }))) 42 | .then((data) => { 43 | console.log(data.body); 44 | assertEquals(data.status, 200); 45 | }); 46 | }); 47 | 48 | Deno.test("store-chunk", async () => { 49 | let _ = await fetch("http://localhost:8888/store-chunk", { 50 | method: "POST", 51 | headers: { 52 | "x-rdfs-token": Token, 53 | "Content-Type": "application/json", 54 | }, 55 | body: JSON.stringify({ 56 | "id": "test.txt", 57 | "chunk": "dGhpcyBpcyBhIHRlc3QgZmlsZSE=", 58 | }), 59 | }).then((x) => x.text().then((data) => ({ status: x.status, body: data }))) 60 | .then((data) => { 61 | console.log(data.body); 62 | assertEquals(data.status, 200); 63 | }); 64 | }); 65 | 66 | Deno.test("delete-chunk", async () => { 67 | let _ = await fetch("http://localhost:8888/delete-chunk", { 68 | method: "POST", 69 | headers: { 70 | "x-rdfs-token": Token, 71 | "Content-Type": "application/json", 72 | }, 73 | body: JSON.stringify({ "id": "test.txt" }), 74 | }).then((x) => x.text().then((data) => ({ status: x.status, body: data }))) 75 | .then((data) => { 76 | // console.log(data.body); 77 | assertEquals(data.status, 200); 78 | }); 79 | }); 80 | 81 | Deno.test("send-chunk", async () => { 82 | let _ = await fetch("http://localhost:8888/send-chunk", { 83 | method: "POST", 84 | headers: { 85 | "x-rdfs-token": Token, 86 | "Content-Type": "application/json", 87 | }, 88 | body: JSON.stringify({ "id": "README.md", "target": "http://localhost:9999" }), 89 | }).then((x) => x.text().then((data) => ({ status: x.status, body: data }))) 90 | .then((data) => { 91 | console.log(data.body); 92 | assertEquals(data.status, 200); 93 | }); 94 | }); 95 | -------------------------------------------------------------------------------- /src/worker.rs: -------------------------------------------------------------------------------- 1 | use crate::config; 2 | use crate::config::Config; 3 | use axum::extract; 4 | use axum::extract::State; 5 | use axum::http::StatusCode; 6 | use axum::middleware; 7 | use axum::response::{IntoResponse, Json, Response}; 8 | use axum::routing::{get, post}; 9 | use axum::Router; 10 | use base64::prelude::*; 11 | use serde::{Deserialize, Serialize}; 12 | use std::fs; 13 | use std::fs::remove_file; 14 | use std::io::Write; 15 | use std::path::Path; 16 | use std::time::Duration; 17 | use tracing::{error, info}; 18 | 19 | use crate::auth; 20 | 21 | pub async fn init(port: &i16) { 22 | println!("{}", crate::LOGO); 23 | info!("launching node in [worker] mode on port {}...", port); 24 | 25 | if let Some(config) = config::get() { 26 | let app = Router::new() 27 | .route("/", get(hello)) 28 | .route("/get-chunk", post(get_chunk)) 29 | .route("/store-chunk", post(store_chunk)) 30 | .route("/delete-chunk", post(delete_chunk)) 31 | .route("/send-chunk", post(send_chunk)) 32 | .route_layer(middleware::from_fn(auth::authorise)) 33 | .with_state(config.clone()); 34 | 35 | let listener = tokio::net::TcpListener::bind(format!("0.0.0.0:{}", port)) 36 | .await 37 | .unwrap(); 38 | tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); 39 | 40 | let _ = tokio::task::spawn_blocking(move || background_heartbeat(config)).await; 41 | } else { 42 | error!("unable able to load the valid cluster configuration. Please make sure the ENV 'RDFS_ENDPOINT' and 'RDFS_TOKEN' are set"); 43 | } 44 | } 45 | 46 | #[derive(Deserialize, Serialize)] 47 | pub struct MetaChunk { 48 | pub id: String, 49 | } 50 | 51 | #[derive(Deserialize, Serialize)] 52 | struct Chunk { 53 | id: String, 54 | chunk: String, 55 | } 56 | 57 | #[derive(Deserialize, Serialize)] 58 | struct SendChunk { 59 | id: String, 60 | target: String, 61 | } 62 | 63 | async fn hello(State(state): State) -> String { 64 | let response = format!("configurtion token -> '{}'", state.token); 65 | response.to_string() 66 | } 67 | 68 | #[axum::debug_handler] 69 | async fn get_chunk(extract::Json(payload): extract::Json) -> Response { 70 | info!("get-chunk with ID [{}]", &payload.id); 71 | // todo: we can use regex to make sure that the payload ID is legal e.g - format 72 | if !Path::new(&payload.id).exists() { 73 | return StatusCode::NOT_FOUND.into_response(); 74 | } 75 | 76 | if let Ok(chunk) = fs::read(&payload.id) { 77 | return Json(Chunk { 78 | id: payload.id, 79 | chunk: BASE64_STANDARD.encode(chunk), 80 | }) 81 | .into_response(); 82 | } 83 | StatusCode::INTERNAL_SERVER_ERROR.into_response() 84 | } 85 | 86 | #[axum::debug_handler] 87 | async fn store_chunk(extract::Json(payload): extract::Json) -> Response { 88 | info!("store-chunk with ID [{}]", &payload.id); 89 | 90 | if let Ok(mut file) = fs::File::create(&payload.id) { 91 | if let Ok(chunk) = BASE64_STANDARD.decode(&payload.chunk) { 92 | if let Ok(_) = file.write(&chunk) { 93 | return Json(MetaChunk { 94 | id: payload.id.to_string(), 95 | }) 96 | .into_response(); 97 | } 98 | } 99 | } 100 | StatusCode::INTERNAL_SERVER_ERROR.into_response() 101 | } 102 | 103 | #[axum::debug_handler] 104 | async fn delete_chunk(extract::Json(payload): extract::Json) -> Response { 105 | info!("delete-chunk with ID [{}]", &payload.id); 106 | 107 | if let Ok(_) = remove_file(&payload.id) { 108 | return Json(MetaChunk { id: payload.id }).into_response(); 109 | } 110 | 111 | StatusCode::INTERNAL_SERVER_ERROR.into_response() 112 | } 113 | 114 | #[axum::debug_handler] 115 | async fn send_chunk( 116 | State(state): State, 117 | extract::Json(payload): extract::Json, 118 | ) -> Response { 119 | info!("send-chunk [{}] to -> {}", &payload.id, &payload.target); 120 | 121 | if !Path::new(&payload.id).exists() { 122 | return StatusCode::NOT_FOUND.into_response(); 123 | } 124 | 125 | if let Ok(chunk) = fs::read(&payload.id) { 126 | let data = Chunk { 127 | id: payload.id.clone(), 128 | chunk: BASE64_STANDARD.encode(chunk), 129 | }; 130 | // NOTE: we may need to move this i/o call into it's own thread via spawn_blocking 131 | match ureq::post(&format!("{}/store-chunk", payload.target)) 132 | .set("x-rdfs-token", &state.token) 133 | .send_json(data) 134 | { 135 | Ok(_) => { 136 | return Json(MetaChunk { id: payload.id }).into_response(); 137 | } 138 | _ => { 139 | return StatusCode::INTERNAL_SERVER_ERROR.into_response(); 140 | } 141 | } 142 | } 143 | StatusCode::INTERNAL_SERVER_ERROR.into_response() 144 | } 145 | 146 | fn background_heartbeat(config: Config) { 147 | info!("initiating the background heartbeat..."); 148 | loop { 149 | // TODO: later on we could sent worker node meta information e.g disk space 150 | // to the master node. 151 | let _ = ureq::post(&format!("{}/heartbeat", config.endpoint)) 152 | .set("x-rdfs-token", &config.token) 153 | .call(); 154 | std::thread::sleep(Duration::from_millis(4000)); 155 | } 156 | } 157 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # RDFS 2 | 3 | 🔧 Rust Distributed File System (RDFS) a toy implementation of the Google File System (GFS) 4 | 5 | ![alt Experimental](https://img.shields.io/badge/Type-Experimental-red.svg) 6 | ![alt Rust](https://img.shields.io/badge/Language-Rust-orange.svg) 7 | ![alt Binary](https://img.shields.io/badge/Binary-Polymorphic-green.svg) 8 | 9 | ```shell 10 | 11 | ██████ ██████ ███████ ███████ 12 | ██ ██ ██ ██ ██ ██ 13 | ██████ ██ ██ █████ ███████ 14 | ██ ██ ██ ██ ██ ██ 15 | ██ ██ ██████ ██ ███████ 16 | 17 | a toy distributed file system 18 | ``` 19 | 20 | ## Intro 21 | 22 | Reading the original paper ["The Google File System"](https://pdos.csail.mit.edu/6.824/papers/gfs.pdf) 23 | was the inspiration for HDFS _(Hadoop Distributed File System)_ that later gave way to Amazon's "S3" which has 24 | become almost the "defacto" standard. Distributed file systems are super interesting and this project is 25 | an attempt to understand how distributed file systems work by building a toy version of the original GFS. 26 | 27 | ![img](GFS.png) 28 | 29 | I really like that idea of creating a single "polymorphic binary" that can act as the following: 30 | 31 | - Master node 32 | - Worker node 33 | - Client CLI 34 | 35 | ## Environment Variables 36 | 37 | This binary assumes that the following environemnt variables are present in order to setup the 38 | required global configuration: 39 | 40 | | Name | Example value | Description | 41 | | ------------- | ------------------------------------ | ------------------------------------------ | 42 | | RDFS_ENDPOINT | https://master-node-ip:8888 | where the master node can be reached | 43 | | RDFS_TOKEN | 7687a5ac-ed5a-4d69-8cc3-f78c119b3219 | the security token needed for this cluster | 44 | 45 | ## Usage: WARNING unstable will probably change 46 | 47 | ```shell 48 | rdfs 0.1.0 49 | Wat The Doodle 50 | 51 | 52 | ██████ ██████ ███████ ███████ 53 | ██ ██ ██ ██ ██ ██ 54 | ██████ ██ ██ █████ ███████ 55 | ██ ██ ██ ██ ██ ██ 56 | ██ ██ ██████ ██ ███████ 57 | 58 | a toy distributed file system 59 | 60 | 61 | Usage: rdfs [COMMAND] 62 | 63 | Commands: 64 | list List all remote files e.g rdfs list 65 | get Get a remote file e.g rdfs get foo.txt 66 | add Add a remote file e.g rdfs add foo.txt 67 | remove Remove a remote file e.g rdfs remove foo.txt 68 | mode Mode: run the binary in either as a "Master" or "Worker" node 69 | help Print this message or the help of the given subcommand(s) 70 | 71 | Options: 72 | -h, --help Print help 73 | -V, --version Print version 74 | 75 | ``` 76 | 77 | ## Authentication 78 | 79 | For all the HTTP calls we need to pass the token as a custom header value i.e. `x-rdfs-token`. This 80 | will be checked using an authentication middleware in axum. 81 | 82 | ## Test Harness 83 | 84 | Some of the local tests require us to call the worker or master http endpoints, we have a folder called `test-harness` that contains those tests. To run the test execute the following: 85 | 86 | ```shell 87 | $ deno task test 88 | ``` 89 | 90 | ## Simulating a cluster using docker 91 | 92 | In order to test our distributed cluster, instead of spinning up lots of heavy Virtual Machines, instead we can "simulate" it using lightweight containers. 93 | 94 | First we will need to build our container images via the following commnand: 95 | 96 | ```shell 97 | $ docker compose build 98 | ``` 99 | 100 | This will take some time but eventually once it completes we should have a custom docker image, we can check by doing the following: 101 | 102 | ```shell 103 | $ docker images 104 | 105 | REPOSITORY TAG IMAGE ID CREATED SIZE 106 | rdfs latest e9d6e2275c17 35 minutes ago 13.2MB 107 | ``` 108 | 109 | now we can spin up our inital "cluster" with only 1 master node and 1 worker node: 110 | 111 | ```shell 112 | $ docker compose up -d 113 | 114 | [+] Running 3/3 115 | ✔ Network rdfs_default Created 0.1s 116 | ✔ Container rdfs-master-1 Started 0.4s 117 | ✔ Container rdfs-worker-1 Started 0.3s 118 | ``` 119 | 120 | Now we can _scale_ the number of worker node simply by using the `scale` command, for example if we wanted to scale up to have 3 worker nodes: 121 | 122 | ```shell 123 | $ docker compose scale worker=3 124 | 125 | [+] Running 3/3 126 | ✔ Container rdfs-worker-1 Running 0.0s 127 | ✔ Container rdfs-worker-3 Started 0.6s 128 | ✔ Container rdfs-worker-2 Started 0.3s 129 | ``` 130 | 131 | If we wish to check out the logs we can do this by using the container names e.g: 132 | 133 | ```shell 134 | $ docker logs -f rdfs-master-1 135 | 136 | ██████ ██████ ███████ ███████ 137 | ██ ██ ██ ██ ██ ██ 138 | ██████ ██ ██ █████ ███████ 139 | ██ ██ ██ ██ ██ ██ 140 | ██ ██ ██████ ██ ███████ 141 | 142 | a toy distributed file system 143 | 144 | ==> launching node in [master] mode on port 8888... 145 | ==> got a heartbeat from worker node -> ...172.18.0.3:43640 146 | ``` 147 | 148 | Finally we can "tear down" our cluster simply by doing the following: 149 | 150 | ```shell 151 | $ docker compose down 152 | [+] Running 5/5 153 | ✔ Container rdfs-master-1 Removed 10.3s 154 | ✔ Container rdfs-worker-1 Removed 10.3s 155 | ✔ Container rdfs-worker-2 Removed 10.2s 156 | ✔ Container rdfs-worker-3 Removed 10.3s 157 | ✔ Network rdfs_default Removed 0.1s 158 | ``` 159 | -------------------------------------------------------------------------------- /src/master.rs: -------------------------------------------------------------------------------- 1 | use crate::auth; 2 | use crate::config; 3 | use crate::config::Config; 4 | use crate::worker::MetaChunk; 5 | use axum::extract; 6 | use axum::extract::{ConnectInfo, State}; 7 | use axum::http::StatusCode; 8 | use axum::middleware; 9 | use axum::response::{IntoResponse, Json, Response}; 10 | use axum::routing::post; 11 | use axum::Router; 12 | use chrono::{DateTime, Utc}; 13 | use rand::seq::SliceRandom; 14 | use serde::{Deserialize, Serialize}; 15 | use serde_json::json; 16 | use std::collections::{HashMap, HashSet}; 17 | use std::fs::{File, OpenOptions}; 18 | use std::io::{BufRead, BufReader, Write}; 19 | use std::net::SocketAddr; 20 | use std::path::Path; 21 | use std::sync::Mutex; 22 | use tracing::{error, info, warn}; 23 | 24 | const FILE_CHUNK_SIZE: u64 = 512; 25 | const TIMEOUT_IN_MINUTES: i64 = 5; 26 | const REPLICATION_FACTOR: usize = 3; 27 | 28 | #[derive(Deserialize, Serialize, Debug, Clone)] 29 | struct MetaStore { 30 | file_name: String, 31 | hash: String, 32 | chunk_id: i32, 33 | hosts: Vec, 34 | } 35 | 36 | #[derive(Deserialize, Serialize, Debug, Clone)] 37 | enum Status { 38 | Dirty, 39 | Synced, 40 | } 41 | 42 | #[derive(Deserialize, Serialize, Debug, Clone)] 43 | struct Host { 44 | ip: String, 45 | status: Status, 46 | } 47 | 48 | lazy_static! { 49 | static ref METASTATE: Mutex> = Mutex::new(vec![]); 50 | static ref HEARTBEAT: Mutex>> = Mutex::new(HashMap::new()); 51 | } 52 | 53 | pub async fn init(port: &i16) { 54 | println!("{}", crate::LOGO); 55 | 56 | if let Some(config) = config::get() { 57 | self::load_snapshot(); 58 | let _ = self::export_compacted_snapshot(); 59 | 60 | info!("launching node in [master] mode on port {}...", port); 61 | 62 | let app = Router::new() 63 | .route("/heartbeat", post(heartbeat)) 64 | .route("/list", post(list)) 65 | .route("/get", post(get)) 66 | .route("/upload", post(upload)) 67 | .route("/remove", post(remove)) 68 | .route("/sync", post(sync)) 69 | .route_layer(middleware::from_fn(auth::authorise)) 70 | .with_state(config.clone()); 71 | 72 | let listener = tokio::net::TcpListener::bind(format!("0.0.0.0:{}", port)) 73 | .await 74 | .unwrap(); 75 | 76 | axum::serve( 77 | listener, 78 | app.into_make_service_with_connect_info::(), 79 | ) 80 | .await 81 | .unwrap() 82 | } else { 83 | error!("Error: unable able to load the valid cluster configuration. Please make sure the ENV 'RDFS_ENDPOINT' and 'RDFS_TOKEN' are set"); 84 | } 85 | } 86 | 87 | async fn heartbeat(ConnectInfo(addr): ConnectInfo) -> String { 88 | info!("got a heartbeat from worker node -> ...{}", addr); 89 | 90 | if let Ok(mut heartbeat) = HEARTBEAT.lock() { 91 | let ts = chrono::Utc::now(); 92 | heartbeat 93 | .entry(addr.ip().to_string()) 94 | .and_modify(|x| *x = ts) 95 | .or_insert(ts); 96 | } 97 | 98 | "ok".to_string() 99 | } 100 | 101 | async fn sync(ConnectInfo(addr): ConnectInfo, extract::Json(payload): extract::Json) -> String { 102 | info!("got a chunk sync from worker node -> ...{}", addr); 103 | 104 | // TODO: find the specific chunk and update the Status from Dirty to Synced 105 | 106 | 107 | "ok".to_string() 108 | } 109 | 110 | #[derive(Deserialize, Serialize)] 111 | struct FileMeta { 112 | name: String, 113 | } 114 | 115 | #[derive(Deserialize, Serialize)] 116 | struct FileUploadMeta { 117 | name: String, 118 | hash: String, 119 | size: u64, 120 | } 121 | 122 | #[derive(Deserialize, Serialize)] 123 | struct ChunkSyncMeta { 124 | hash: String, 125 | chunk_id: i32 126 | } 127 | 128 | #[axum::debug_handler] 129 | async fn list() -> Response { 130 | info!("list all files"); 131 | 132 | let mut files: Vec = vec![]; 133 | 134 | if let Ok(memory) = METASTATE.lock() { 135 | files = memory 136 | .clone() 137 | .into_iter() 138 | .map(|x| x.file_name) 139 | .collect::>() 140 | .into_iter() 141 | .collect::>() 142 | .to_vec(); 143 | } 144 | 145 | if files.len() > 0 { 146 | return Json(files).into_response(); 147 | } 148 | 149 | StatusCode::INTERNAL_SERVER_ERROR.into_response() 150 | } 151 | 152 | #[axum::debug_handler] 153 | async fn get(extract::Json(payload): extract::Json) -> Response { 154 | info!("get file with name [{}]", &payload.name); 155 | 156 | let mut file: Vec = vec![]; 157 | 158 | if let Ok(memory) = METASTATE.lock() { 159 | file = memory 160 | .clone() 161 | .into_iter() 162 | .filter(|x| x.file_name == payload.name) 163 | .collect::>(); 164 | } 165 | 166 | if file.len() > 0 { 167 | return Json(file).into_response(); 168 | } 169 | 170 | StatusCode::INTERNAL_SERVER_ERROR.into_response() 171 | } 172 | 173 | #[axum::debug_handler] 174 | async fn upload(extract::Json(payload): extract::Json) -> Response { 175 | info!("upload file with name [{}]", &payload.name); 176 | 177 | /* --------------------------------------------------------------------------------------------- 178 | **CAUTION:** 179 | 180 | so this is the one of most complex part of the master node (the other would be re-balancing), 181 | here we need to take the total size of the file and split into 512kb chunks (we don't actually 182 | do the splitting here, just dealing with the meta information about the splitting.) 183 | 184 | then for each chunk we need to randomly allocate worker node(s) such that the replication factor 185 | is satisfied. 186 | 187 | We would then send back that entire meta information so that the client can then go ahead and 188 | split up the file and then send each chunk to all the various different worker nodes. 189 | 190 | We need to make sure that the selected worker nodes are "alive" based on the timestamped heart 191 | beat. For this we can use something like a defined upper "threshold" timeout limit, e.g 5 192 | minutes. 193 | 194 | of course this strategy is not robust and certainly NOT production grade, because there could be 195 | so many issues, for example the main one being that once the client gets the meta data back 196 | some of the worker nodes may fail to receive the file chunks, in this case we don't currently 197 | have any "fallback" way to retry. 198 | 199 | At the moment if the same file is called to be "uploaded" (with a "force" flag) it could simply 200 | re-calculate chunk distribution across random worker nodes, that itself is fine in terms of a 201 | retry mechanism, however it would mean we have the potential for ending up with "orhpaned" 202 | chunks, we don't care about "overlaping" chunks as that would simply overwrite existing chunks 203 | so no issues there. 204 | 205 | We may need to consider that "re-calculation" to be something abnormal, and only when the client 206 | fails to upload to worker nodes, so it maybe a good idea that under normal cases we simply check 207 | the metastore to see if we already have this file uploaded and return that if that is the case 208 | so that it's an "immutable" call. However we could potentially include an optional "force" flag 209 | in the payload which the client would only use in cases of retries/failures, in which case we 210 | skip any existing uploaded files and just re-calculate (we may need to issue a remove file first) 211 | 212 | These "orphaned" chunks would be chunks that exist on a worker node, but technically without any 213 | reference in the main master metastore. Over time these would take up waste disk space. 214 | 215 | One way to fix this could be to have some kind of "garbage" collection on the work nodes that 216 | would need the worker nodes to be able to identify that they have chunks that shouldn't exist, 217 | this could happen slowly when idle as it's not super important. 218 | 219 | Note (May 2025): 220 | The "Status" has been changed from "Unknown, Healthy, Dead" over to "Dirty, Synced", this 221 | makes much more sense, and we can use this status flag to track if a worker node has actually 222 | got a specific file chunk written to disk and not just the overal status of a worker node. 223 | When we "upload" a file, we only plan the chunk placement, hence at that point the chunks 224 | are "dirty" as in planned but not actually written to disk. 225 | When a chunck is written to disk, the worker node will need to send a chunk "Sync" over to 226 | the master, that can then update the internal meta store. This will require a new endpoint 227 | handler as well as changes to the snapshot reloading mechanism. 228 | --------------------------------------------------------------------------------------------- */ 229 | 230 | let mut worker_nodes: Vec = Vec::new(); 231 | let mut heartbeats = HashMap::new(); 232 | let now = chrono::Utc::now(); 233 | 234 | if let Ok(x) = HEARTBEAT.lock() { 235 | heartbeats = x.clone(); 236 | } 237 | 238 | worker_nodes = heartbeats 239 | .into_iter() 240 | .filter(|v| (now - v.1).num_minutes() <= TIMEOUT_IN_MINUTES) 241 | .map(|v| v.0) 242 | .collect(); 243 | 244 | if worker_nodes.len() >= REPLICATION_FACTOR { 245 | let chunks = payload.size / FILE_CHUNK_SIZE; 246 | let mut metastore: Vec = Vec::new(); 247 | 248 | // when file is less than 512kb 249 | if chunks == 0 { 250 | let hosts: Vec = worker_nodes 251 | .choose_multiple(&mut rand::thread_rng(), 3) 252 | .map(|x| Host { 253 | ip: x.to_string(), 254 | status: Status::Dirty, 255 | }) 256 | .collect(); 257 | 258 | metastore.push(MetaStore { 259 | file_name: payload.name.to_string(), 260 | hash: payload.hash.to_string(), 261 | chunk_id: 1, 262 | hosts: hosts, 263 | }); 264 | } 265 | 266 | for chunk in 1..chunks { 267 | // randomly pick X worker nodes 268 | let hosts: Vec = worker_nodes 269 | .choose_multiple(&mut rand::thread_rng(), 3) 270 | .map(|x| Host { 271 | ip: x.to_string(), 272 | status: Status::Dirty, 273 | }) 274 | .collect(); 275 | 276 | metastore.push(MetaStore { 277 | file_name: payload.name.to_string(), 278 | hash: payload.hash.to_string(), 279 | chunk_id: chunk as i32, 280 | hosts: hosts, 281 | }); 282 | } 283 | 284 | for line in metastore.iter() { 285 | self::append("snapshot", &format!("{}", json!(line))); 286 | } 287 | 288 | return Json(metastore).into_response(); 289 | } 290 | 291 | if worker_nodes.len() < REPLICATION_FACTOR { 292 | return StatusCode::SERVICE_UNAVAILABLE.into_response(); 293 | } 294 | 295 | StatusCode::INTERNAL_SERVER_ERROR.into_response() 296 | } 297 | 298 | #[axum::debug_handler] 299 | async fn remove( 300 | State(state): State, 301 | extract::Json(payload): extract::Json, 302 | ) -> Response { 303 | info!("remove file with name [{}]", &payload.name); 304 | 305 | let mut kill_list: Vec = vec![]; 306 | 307 | if let Ok(memory) = METASTATE.lock() { 308 | kill_list = memory 309 | .clone() 310 | .into_iter() 311 | .filter(|x| x.file_name == payload.name) 312 | .collect::>() 313 | .to_vec(); 314 | } 315 | 316 | let mut kill_hash = String::new(); 317 | 318 | if kill_list.len() > 0 { 319 | for chunk in kill_list { 320 | for worker in chunk.hosts { 321 | let chunk_id = format!("{}-{}", chunk.chunk_id, chunk.hash); 322 | kill_hash = chunk.hash.to_string(); 323 | let token = state.token.to_string(); 324 | let _ = tokio::task::spawn_blocking(move || { 325 | self::delete_remote_chunk(chunk_id, worker.ip, &token) 326 | }) 327 | .await; 328 | } 329 | } 330 | } 331 | 332 | if let Ok(mut memory) = METASTATE.lock() { 333 | memory.retain(|x| x.file_name != payload.name) 334 | } 335 | 336 | self::append("prune", &kill_hash); 337 | 338 | StatusCode::INTERNAL_SERVER_ERROR.into_response() 339 | } 340 | 341 | fn delete_remote_chunk(chunk_id: String, remote_ip: String, token: &str) { 342 | let data = MetaChunk { 343 | id: chunk_id.clone(), 344 | }; 345 | 346 | if let Ok(_) = ureq::post(&format!("http://{}:8888/delete-chunk", remote_ip)) 347 | .set("x-rdfs-token", token) 348 | .send_json(data) 349 | { 350 | info!("remote chunk deleted ({})", &chunk_id); 351 | } else { 352 | warn!("ERROR: unable to delete remote chunk ({})", &chunk_id); 353 | } 354 | } 355 | 356 | #[allow(dead_code)] 357 | fn create_dummy_snapshot() { 358 | info!("generating dummy snapshot data..."); 359 | 360 | let a = MetaStore { 361 | file_name: String::from("README.md"), 362 | hash: String::from("5c9d231c8b6d10f43fd0768ca80755d2"), 363 | chunk_id: 1, 364 | hosts: vec![ 365 | Host { 366 | ip: String::from("192.168.1.80"), 367 | status: Status::Dirty, 368 | }, 369 | Host { 370 | ip: String::from("192.168.1.83"), 371 | status: Status::Dirty, 372 | }, 373 | ], 374 | }; 375 | 376 | let b = MetaStore { 377 | file_name: String::from("README.md"), 378 | hash: String::from("5c9d231c8b6d10f43fd0768ca80755d2"), 379 | chunk_id: 2, 380 | hosts: vec![ 381 | Host { 382 | ip: String::from("192.168.1.81"), 383 | status: Status::Dirty, 384 | }, 385 | Host { 386 | ip: String::from("192.168.1.82"), 387 | status: Status::Dirty, 388 | }, 389 | ], 390 | }; 391 | 392 | let c = MetaStore { 393 | file_name: String::from("README.md"), 394 | hash: String::from("5c9d231c8b6d10f43fd0768ca80755d2"), 395 | chunk_id: 2, 396 | hosts: vec![ 397 | Host { 398 | ip: String::from("192.168.1.82"), 399 | status: Status::Dirty, 400 | }, 401 | Host { 402 | ip: String::from("192.168.1.255"), 403 | status: Status::Dirty, 404 | }, 405 | ], 406 | }; 407 | 408 | let mut w = File::create("snapshot").unwrap(); 409 | writeln!(&mut w, "{}", json!(a)).unwrap(); 410 | writeln!(&mut w, "{}", json!(b)).unwrap(); 411 | writeln!(&mut w, "{}", json!(c)).unwrap(); 412 | } 413 | 414 | fn load_snapshot() { 415 | /* --------------------------------------------------------------------------------------------- 416 | attempt to load from snapshot from disk into memory, we will need to also do 417 | compaction and then re-save the compacted snapshot back to disk. 418 | This will then allow the change events to be appended while the process is running. 419 | ---------------------------------------------------------------------------------------------- */ 420 | info!("attempting to load snapshot..."); 421 | 422 | // self::create_dummy_snapshot(); 423 | 424 | let mut prune = Vec::new(); 425 | 426 | if let Ok(v) = self::read_lines("prune") { 427 | prune = v; 428 | } 429 | 430 | if Path::new("snapshot").exists() { 431 | info!("existing snapshot detected!"); 432 | let snapshot = File::open("snapshot").unwrap(); 433 | let reader = BufReader::new(snapshot); 434 | 435 | let mut compactor: HashMap<(String, i32), MetaStore> = HashMap::new(); 436 | 437 | if let Ok(mut memory) = METASTATE.lock() { 438 | for line in reader.lines() { 439 | if let Ok(_line) = line { 440 | if let Ok(disk) = serde_json::from_str::(&_line) { 441 | if !prune.contains(&disk.hash) { 442 | compactor 443 | .entry((disk.hash.to_string(), disk.chunk_id)) 444 | .and_modify(|x| *x = disk.clone()) 445 | .or_insert(disk); 446 | } 447 | } 448 | } 449 | } 450 | for (_, v) in compactor { 451 | memory.push(v); 452 | } 453 | } 454 | 455 | if let Ok(memory) = METASTATE.lock() { 456 | info!( 457 | "total chunks loaded into memory after compaction: {}", 458 | memory.len() 459 | ) 460 | } 461 | } 462 | } 463 | 464 | fn export_compacted_snapshot() -> Result<(), std::io::Error> { 465 | info!("attempting to export compacted snapshot..."); 466 | // FIX: we're ignoring any errors, at some point we need to consider 467 | // handling them 468 | if let Ok(memory) = METASTATE.lock() { 469 | let mut w = File::create("snapshot.new")?; 470 | for v in &*memory { 471 | writeln!(&mut w, "{}", json!(v))?; 472 | } 473 | } 474 | std::fs::remove_file("snapshot")?; 475 | std::fs::rename("snapshot.new", "snapshot")?; 476 | std::fs::remove_file("prune")?; 477 | Ok(()) 478 | } 479 | 480 | fn append(f: &str, d: &str) { 481 | let mut h = OpenOptions::new().write(true).append(true).open(f).unwrap(); 482 | 483 | if let Err(e) = writeln!(h, "{}", d) { 484 | warn!("unable to append to file: {}", e); 485 | } 486 | } 487 | 488 | fn read_lines(p: &str) -> Result, std::io::Error> { 489 | let f = File::open(p)?; 490 | let r = BufReader::new(f); 491 | let mut v = Vec::new(); 492 | for l in r.lines() { 493 | if let Ok(l) = l { 494 | v.push(l); 495 | } 496 | } 497 | Ok(v) 498 | } 499 | -------------------------------------------------------------------------------- /Cargo.lock: -------------------------------------------------------------------------------- 1 | # This file is automatically @generated by Cargo. 2 | # It is not intended for manual editing. 3 | version = 3 4 | 5 | [[package]] 6 | name = "addr2line" 7 | version = "0.24.1" 8 | source = "registry+https://github.com/rust-lang/crates.io-index" 9 | checksum = "f5fb1d8e4442bd405fdfd1dacb42792696b0cf9cb15882e5d097b742a676d375" 10 | dependencies = [ 11 | "gimli", 12 | ] 13 | 14 | [[package]] 15 | name = "adler2" 16 | version = "2.0.0" 17 | source = "registry+https://github.com/rust-lang/crates.io-index" 18 | checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" 19 | 20 | [[package]] 21 | name = "android-tzdata" 22 | version = "0.1.1" 23 | source = "registry+https://github.com/rust-lang/crates.io-index" 24 | checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" 25 | 26 | [[package]] 27 | name = "android_system_properties" 28 | version = "0.1.5" 29 | source = "registry+https://github.com/rust-lang/crates.io-index" 30 | checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" 31 | dependencies = [ 32 | "libc", 33 | ] 34 | 35 | [[package]] 36 | name = "anstream" 37 | version = "0.6.15" 38 | source = "registry+https://github.com/rust-lang/crates.io-index" 39 | checksum = "64e15c1ab1f89faffbf04a634d5e1962e9074f2741eef6d97f3c4e322426d526" 40 | dependencies = [ 41 | "anstyle", 42 | "anstyle-parse", 43 | "anstyle-query", 44 | "anstyle-wincon", 45 | "colorchoice", 46 | "is_terminal_polyfill", 47 | "utf8parse", 48 | ] 49 | 50 | [[package]] 51 | name = "anstyle" 52 | version = "1.0.8" 53 | source = "registry+https://github.com/rust-lang/crates.io-index" 54 | checksum = "1bec1de6f59aedf83baf9ff929c98f2ad654b97c9510f4e70cf6f661d49fd5b1" 55 | 56 | [[package]] 57 | name = "anstyle-parse" 58 | version = "0.2.5" 59 | source = "registry+https://github.com/rust-lang/crates.io-index" 60 | checksum = "eb47de1e80c2b463c735db5b217a0ddc39d612e7ac9e2e96a5aed1f57616c1cb" 61 | dependencies = [ 62 | "utf8parse", 63 | ] 64 | 65 | [[package]] 66 | name = "anstyle-query" 67 | version = "1.1.1" 68 | source = "registry+https://github.com/rust-lang/crates.io-index" 69 | checksum = "6d36fc52c7f6c869915e99412912f22093507da8d9e942ceaf66fe4b7c14422a" 70 | dependencies = [ 71 | "windows-sys", 72 | ] 73 | 74 | [[package]] 75 | name = "anstyle-wincon" 76 | version = "3.0.4" 77 | source = "registry+https://github.com/rust-lang/crates.io-index" 78 | checksum = "5bf74e1b6e971609db8ca7a9ce79fd5768ab6ae46441c572e46cf596f59e57f8" 79 | dependencies = [ 80 | "anstyle", 81 | "windows-sys", 82 | ] 83 | 84 | [[package]] 85 | name = "async-trait" 86 | version = "0.1.83" 87 | source = "registry+https://github.com/rust-lang/crates.io-index" 88 | checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" 89 | dependencies = [ 90 | "proc-macro2", 91 | "quote", 92 | "syn", 93 | ] 94 | 95 | [[package]] 96 | name = "autocfg" 97 | version = "1.4.0" 98 | source = "registry+https://github.com/rust-lang/crates.io-index" 99 | checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" 100 | 101 | [[package]] 102 | name = "axum" 103 | version = "0.7.7" 104 | source = "registry+https://github.com/rust-lang/crates.io-index" 105 | checksum = "504e3947307ac8326a5437504c517c4b56716c9d98fac0028c2acc7ca47d70ae" 106 | dependencies = [ 107 | "async-trait", 108 | "axum-core", 109 | "axum-macros", 110 | "bytes", 111 | "futures-util", 112 | "http", 113 | "http-body", 114 | "http-body-util", 115 | "hyper", 116 | "hyper-util", 117 | "itoa", 118 | "matchit", 119 | "memchr", 120 | "mime", 121 | "percent-encoding", 122 | "pin-project-lite", 123 | "rustversion", 124 | "serde", 125 | "serde_json", 126 | "serde_path_to_error", 127 | "serde_urlencoded", 128 | "sync_wrapper 1.0.1", 129 | "tokio", 130 | "tower", 131 | "tower-layer", 132 | "tower-service", 133 | "tracing", 134 | ] 135 | 136 | [[package]] 137 | name = "axum-core" 138 | version = "0.4.5" 139 | source = "registry+https://github.com/rust-lang/crates.io-index" 140 | checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" 141 | dependencies = [ 142 | "async-trait", 143 | "bytes", 144 | "futures-util", 145 | "http", 146 | "http-body", 147 | "http-body-util", 148 | "mime", 149 | "pin-project-lite", 150 | "rustversion", 151 | "sync_wrapper 1.0.1", 152 | "tower-layer", 153 | "tower-service", 154 | "tracing", 155 | ] 156 | 157 | [[package]] 158 | name = "axum-macros" 159 | version = "0.4.2" 160 | source = "registry+https://github.com/rust-lang/crates.io-index" 161 | checksum = "57d123550fa8d071b7255cb0cc04dc302baa6c8c4a79f55701552684d8399bce" 162 | dependencies = [ 163 | "proc-macro2", 164 | "quote", 165 | "syn", 166 | ] 167 | 168 | [[package]] 169 | name = "backtrace" 170 | version = "0.3.74" 171 | source = "registry+https://github.com/rust-lang/crates.io-index" 172 | checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a" 173 | dependencies = [ 174 | "addr2line", 175 | "cfg-if", 176 | "libc", 177 | "miniz_oxide", 178 | "object", 179 | "rustc-demangle", 180 | "windows-targets", 181 | ] 182 | 183 | [[package]] 184 | name = "base64" 185 | version = "0.22.1" 186 | source = "registry+https://github.com/rust-lang/crates.io-index" 187 | checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" 188 | 189 | [[package]] 190 | name = "bumpalo" 191 | version = "3.16.0" 192 | source = "registry+https://github.com/rust-lang/crates.io-index" 193 | checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" 194 | 195 | [[package]] 196 | name = "byteorder" 197 | version = "1.5.0" 198 | source = "registry+https://github.com/rust-lang/crates.io-index" 199 | checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" 200 | 201 | [[package]] 202 | name = "bytes" 203 | version = "1.7.2" 204 | source = "registry+https://github.com/rust-lang/crates.io-index" 205 | checksum = "428d9aa8fbc0670b7b8d6030a7fadd0f86151cae55e4dbbece15f3780a3dfaf3" 206 | 207 | [[package]] 208 | name = "cc" 209 | version = "1.1.25" 210 | source = "registry+https://github.com/rust-lang/crates.io-index" 211 | checksum = "e8d9e0b4957f635b8d3da819d0db5603620467ecf1f692d22a8c2717ce27e6d8" 212 | dependencies = [ 213 | "shlex", 214 | ] 215 | 216 | [[package]] 217 | name = "cfg-if" 218 | version = "1.0.0" 219 | source = "registry+https://github.com/rust-lang/crates.io-index" 220 | checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" 221 | 222 | [[package]] 223 | name = "chrono" 224 | version = "0.4.38" 225 | source = "registry+https://github.com/rust-lang/crates.io-index" 226 | checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" 227 | dependencies = [ 228 | "android-tzdata", 229 | "iana-time-zone", 230 | "js-sys", 231 | "num-traits", 232 | "wasm-bindgen", 233 | "windows-targets", 234 | ] 235 | 236 | [[package]] 237 | name = "clap" 238 | version = "4.5.18" 239 | source = "registry+https://github.com/rust-lang/crates.io-index" 240 | checksum = "b0956a43b323ac1afaffc053ed5c4b7c1f1800bacd1683c353aabbb752515dd3" 241 | dependencies = [ 242 | "clap_builder", 243 | "clap_derive", 244 | ] 245 | 246 | [[package]] 247 | name = "clap_builder" 248 | version = "4.5.18" 249 | source = "registry+https://github.com/rust-lang/crates.io-index" 250 | checksum = "4d72166dd41634086d5803a47eb71ae740e61d84709c36f3c34110173db3961b" 251 | dependencies = [ 252 | "anstream", 253 | "anstyle", 254 | "clap_lex", 255 | "strsim", 256 | ] 257 | 258 | [[package]] 259 | name = "clap_derive" 260 | version = "4.5.18" 261 | source = "registry+https://github.com/rust-lang/crates.io-index" 262 | checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab" 263 | dependencies = [ 264 | "heck", 265 | "proc-macro2", 266 | "quote", 267 | "syn", 268 | ] 269 | 270 | [[package]] 271 | name = "clap_lex" 272 | version = "0.7.2" 273 | source = "registry+https://github.com/rust-lang/crates.io-index" 274 | checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" 275 | 276 | [[package]] 277 | name = "colorchoice" 278 | version = "1.0.2" 279 | source = "registry+https://github.com/rust-lang/crates.io-index" 280 | checksum = "d3fd119d74b830634cea2a0f58bbd0d54540518a14397557951e79340abc28c0" 281 | 282 | [[package]] 283 | name = "core-foundation-sys" 284 | version = "0.8.7" 285 | source = "registry+https://github.com/rust-lang/crates.io-index" 286 | checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" 287 | 288 | [[package]] 289 | name = "crc32fast" 290 | version = "1.4.2" 291 | source = "registry+https://github.com/rust-lang/crates.io-index" 292 | checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" 293 | dependencies = [ 294 | "cfg-if", 295 | ] 296 | 297 | [[package]] 298 | name = "flate2" 299 | version = "1.0.34" 300 | source = "registry+https://github.com/rust-lang/crates.io-index" 301 | checksum = "a1b589b4dc103969ad3cf85c950899926ec64300a1a46d76c03a6072957036f0" 302 | dependencies = [ 303 | "crc32fast", 304 | "miniz_oxide", 305 | ] 306 | 307 | [[package]] 308 | name = "fnv" 309 | version = "1.0.7" 310 | source = "registry+https://github.com/rust-lang/crates.io-index" 311 | checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" 312 | 313 | [[package]] 314 | name = "form_urlencoded" 315 | version = "1.2.1" 316 | source = "registry+https://github.com/rust-lang/crates.io-index" 317 | checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" 318 | dependencies = [ 319 | "percent-encoding", 320 | ] 321 | 322 | [[package]] 323 | name = "futures-channel" 324 | version = "0.3.30" 325 | source = "registry+https://github.com/rust-lang/crates.io-index" 326 | checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" 327 | dependencies = [ 328 | "futures-core", 329 | ] 330 | 331 | [[package]] 332 | name = "futures-core" 333 | version = "0.3.30" 334 | source = "registry+https://github.com/rust-lang/crates.io-index" 335 | checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" 336 | 337 | [[package]] 338 | name = "futures-task" 339 | version = "0.3.30" 340 | source = "registry+https://github.com/rust-lang/crates.io-index" 341 | checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" 342 | 343 | [[package]] 344 | name = "futures-util" 345 | version = "0.3.30" 346 | source = "registry+https://github.com/rust-lang/crates.io-index" 347 | checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" 348 | dependencies = [ 349 | "futures-core", 350 | "futures-task", 351 | "pin-project-lite", 352 | "pin-utils", 353 | ] 354 | 355 | [[package]] 356 | name = "getrandom" 357 | version = "0.2.15" 358 | source = "registry+https://github.com/rust-lang/crates.io-index" 359 | checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" 360 | dependencies = [ 361 | "cfg-if", 362 | "libc", 363 | "wasi", 364 | ] 365 | 366 | [[package]] 367 | name = "gimli" 368 | version = "0.31.0" 369 | source = "registry+https://github.com/rust-lang/crates.io-index" 370 | checksum = "32085ea23f3234fc7846555e85283ba4de91e21016dc0455a16286d87a292d64" 371 | 372 | [[package]] 373 | name = "heck" 374 | version = "0.5.0" 375 | source = "registry+https://github.com/rust-lang/crates.io-index" 376 | checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" 377 | 378 | [[package]] 379 | name = "hermit-abi" 380 | version = "0.3.9" 381 | source = "registry+https://github.com/rust-lang/crates.io-index" 382 | checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" 383 | 384 | [[package]] 385 | name = "http" 386 | version = "1.1.0" 387 | source = "registry+https://github.com/rust-lang/crates.io-index" 388 | checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" 389 | dependencies = [ 390 | "bytes", 391 | "fnv", 392 | "itoa", 393 | ] 394 | 395 | [[package]] 396 | name = "http-body" 397 | version = "1.0.1" 398 | source = "registry+https://github.com/rust-lang/crates.io-index" 399 | checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" 400 | dependencies = [ 401 | "bytes", 402 | "http", 403 | ] 404 | 405 | [[package]] 406 | name = "http-body-util" 407 | version = "0.1.2" 408 | source = "registry+https://github.com/rust-lang/crates.io-index" 409 | checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" 410 | dependencies = [ 411 | "bytes", 412 | "futures-util", 413 | "http", 414 | "http-body", 415 | "pin-project-lite", 416 | ] 417 | 418 | [[package]] 419 | name = "httparse" 420 | version = "1.9.4" 421 | source = "registry+https://github.com/rust-lang/crates.io-index" 422 | checksum = "0fcc0b4a115bf80b728eb8ea024ad5bd707b615bfed49e0665b6e0f86fd082d9" 423 | 424 | [[package]] 425 | name = "httpdate" 426 | version = "1.0.3" 427 | source = "registry+https://github.com/rust-lang/crates.io-index" 428 | checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" 429 | 430 | [[package]] 431 | name = "hyper" 432 | version = "1.4.1" 433 | source = "registry+https://github.com/rust-lang/crates.io-index" 434 | checksum = "50dfd22e0e76d0f662d429a5f80fcaf3855009297eab6a0a9f8543834744ba05" 435 | dependencies = [ 436 | "bytes", 437 | "futures-channel", 438 | "futures-util", 439 | "http", 440 | "http-body", 441 | "httparse", 442 | "httpdate", 443 | "itoa", 444 | "pin-project-lite", 445 | "smallvec", 446 | "tokio", 447 | ] 448 | 449 | [[package]] 450 | name = "hyper-util" 451 | version = "0.1.9" 452 | source = "registry+https://github.com/rust-lang/crates.io-index" 453 | checksum = "41296eb09f183ac68eec06e03cdbea2e759633d4067b2f6552fc2e009bcad08b" 454 | dependencies = [ 455 | "bytes", 456 | "futures-util", 457 | "http", 458 | "http-body", 459 | "hyper", 460 | "pin-project-lite", 461 | "tokio", 462 | "tower-service", 463 | ] 464 | 465 | [[package]] 466 | name = "iana-time-zone" 467 | version = "0.1.61" 468 | source = "registry+https://github.com/rust-lang/crates.io-index" 469 | checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220" 470 | dependencies = [ 471 | "android_system_properties", 472 | "core-foundation-sys", 473 | "iana-time-zone-haiku", 474 | "js-sys", 475 | "wasm-bindgen", 476 | "windows-core", 477 | ] 478 | 479 | [[package]] 480 | name = "iana-time-zone-haiku" 481 | version = "0.1.2" 482 | source = "registry+https://github.com/rust-lang/crates.io-index" 483 | checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" 484 | dependencies = [ 485 | "cc", 486 | ] 487 | 488 | [[package]] 489 | name = "idna" 490 | version = "0.5.0" 491 | source = "registry+https://github.com/rust-lang/crates.io-index" 492 | checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" 493 | dependencies = [ 494 | "unicode-bidi", 495 | "unicode-normalization", 496 | ] 497 | 498 | [[package]] 499 | name = "is_terminal_polyfill" 500 | version = "1.70.1" 501 | source = "registry+https://github.com/rust-lang/crates.io-index" 502 | checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" 503 | 504 | [[package]] 505 | name = "itoa" 506 | version = "1.0.11" 507 | source = "registry+https://github.com/rust-lang/crates.io-index" 508 | checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" 509 | 510 | [[package]] 511 | name = "js-sys" 512 | version = "0.3.72" 513 | source = "registry+https://github.com/rust-lang/crates.io-index" 514 | checksum = "6a88f1bda2bd75b0452a14784937d796722fdebfe50df998aeb3f0b7603019a9" 515 | dependencies = [ 516 | "wasm-bindgen", 517 | ] 518 | 519 | [[package]] 520 | name = "lazy_static" 521 | version = "1.5.0" 522 | source = "registry+https://github.com/rust-lang/crates.io-index" 523 | checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" 524 | 525 | [[package]] 526 | name = "libc" 527 | version = "0.2.159" 528 | source = "registry+https://github.com/rust-lang/crates.io-index" 529 | checksum = "561d97a539a36e26a9a5fad1ea11a3039a67714694aaa379433e580854bc3dc5" 530 | 531 | [[package]] 532 | name = "log" 533 | version = "0.4.22" 534 | source = "registry+https://github.com/rust-lang/crates.io-index" 535 | checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" 536 | 537 | [[package]] 538 | name = "matchit" 539 | version = "0.7.3" 540 | source = "registry+https://github.com/rust-lang/crates.io-index" 541 | checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" 542 | 543 | [[package]] 544 | name = "memchr" 545 | version = "2.7.4" 546 | source = "registry+https://github.com/rust-lang/crates.io-index" 547 | checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" 548 | 549 | [[package]] 550 | name = "mime" 551 | version = "0.3.17" 552 | source = "registry+https://github.com/rust-lang/crates.io-index" 553 | checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" 554 | 555 | [[package]] 556 | name = "miniz_oxide" 557 | version = "0.8.0" 558 | source = "registry+https://github.com/rust-lang/crates.io-index" 559 | checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" 560 | dependencies = [ 561 | "adler2", 562 | ] 563 | 564 | [[package]] 565 | name = "mio" 566 | version = "1.0.2" 567 | source = "registry+https://github.com/rust-lang/crates.io-index" 568 | checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec" 569 | dependencies = [ 570 | "hermit-abi", 571 | "libc", 572 | "wasi", 573 | "windows-sys", 574 | ] 575 | 576 | [[package]] 577 | name = "nu-ansi-term" 578 | version = "0.46.0" 579 | source = "registry+https://github.com/rust-lang/crates.io-index" 580 | checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" 581 | dependencies = [ 582 | "overload", 583 | "winapi", 584 | ] 585 | 586 | [[package]] 587 | name = "num-traits" 588 | version = "0.2.19" 589 | source = "registry+https://github.com/rust-lang/crates.io-index" 590 | checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" 591 | dependencies = [ 592 | "autocfg", 593 | ] 594 | 595 | [[package]] 596 | name = "object" 597 | version = "0.36.4" 598 | source = "registry+https://github.com/rust-lang/crates.io-index" 599 | checksum = "084f1a5821ac4c651660a94a7153d27ac9d8a53736203f58b31945ded098070a" 600 | dependencies = [ 601 | "memchr", 602 | ] 603 | 604 | [[package]] 605 | name = "once_cell" 606 | version = "1.20.1" 607 | source = "registry+https://github.com/rust-lang/crates.io-index" 608 | checksum = "82881c4be219ab5faaf2ad5e5e5ecdff8c66bd7402ca3160975c93b24961afd1" 609 | dependencies = [ 610 | "portable-atomic", 611 | ] 612 | 613 | [[package]] 614 | name = "overload" 615 | version = "0.1.1" 616 | source = "registry+https://github.com/rust-lang/crates.io-index" 617 | checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" 618 | 619 | [[package]] 620 | name = "percent-encoding" 621 | version = "2.3.1" 622 | source = "registry+https://github.com/rust-lang/crates.io-index" 623 | checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" 624 | 625 | [[package]] 626 | name = "pin-project-lite" 627 | version = "0.2.14" 628 | source = "registry+https://github.com/rust-lang/crates.io-index" 629 | checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" 630 | 631 | [[package]] 632 | name = "pin-utils" 633 | version = "0.1.0" 634 | source = "registry+https://github.com/rust-lang/crates.io-index" 635 | checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" 636 | 637 | [[package]] 638 | name = "portable-atomic" 639 | version = "1.9.0" 640 | source = "registry+https://github.com/rust-lang/crates.io-index" 641 | checksum = "cc9c68a3f6da06753e9335d63e27f6b9754dd1920d941135b7ea8224f141adb2" 642 | 643 | [[package]] 644 | name = "ppv-lite86" 645 | version = "0.2.20" 646 | source = "registry+https://github.com/rust-lang/crates.io-index" 647 | checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" 648 | dependencies = [ 649 | "zerocopy", 650 | ] 651 | 652 | [[package]] 653 | name = "proc-macro2" 654 | version = "1.0.86" 655 | source = "registry+https://github.com/rust-lang/crates.io-index" 656 | checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" 657 | dependencies = [ 658 | "unicode-ident", 659 | ] 660 | 661 | [[package]] 662 | name = "quote" 663 | version = "1.0.37" 664 | source = "registry+https://github.com/rust-lang/crates.io-index" 665 | checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" 666 | dependencies = [ 667 | "proc-macro2", 668 | ] 669 | 670 | [[package]] 671 | name = "rand" 672 | version = "0.8.5" 673 | source = "registry+https://github.com/rust-lang/crates.io-index" 674 | checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" 675 | dependencies = [ 676 | "libc", 677 | "rand_chacha", 678 | "rand_core", 679 | ] 680 | 681 | [[package]] 682 | name = "rand_chacha" 683 | version = "0.3.1" 684 | source = "registry+https://github.com/rust-lang/crates.io-index" 685 | checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" 686 | dependencies = [ 687 | "ppv-lite86", 688 | "rand_core", 689 | ] 690 | 691 | [[package]] 692 | name = "rand_core" 693 | version = "0.6.4" 694 | source = "registry+https://github.com/rust-lang/crates.io-index" 695 | checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" 696 | dependencies = [ 697 | "getrandom", 698 | ] 699 | 700 | [[package]] 701 | name = "rdfs" 702 | version = "0.1.0" 703 | dependencies = [ 704 | "axum", 705 | "base64", 706 | "chrono", 707 | "clap", 708 | "lazy_static", 709 | "rand", 710 | "serde", 711 | "serde_json", 712 | "tokio", 713 | "tracing", 714 | "tracing-subscriber", 715 | "ureq", 716 | ] 717 | 718 | [[package]] 719 | name = "ring" 720 | version = "0.17.8" 721 | source = "registry+https://github.com/rust-lang/crates.io-index" 722 | checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" 723 | dependencies = [ 724 | "cc", 725 | "cfg-if", 726 | "getrandom", 727 | "libc", 728 | "spin", 729 | "untrusted", 730 | "windows-sys", 731 | ] 732 | 733 | [[package]] 734 | name = "rustc-demangle" 735 | version = "0.1.24" 736 | source = "registry+https://github.com/rust-lang/crates.io-index" 737 | checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" 738 | 739 | [[package]] 740 | name = "rustls" 741 | version = "0.23.14" 742 | source = "registry+https://github.com/rust-lang/crates.io-index" 743 | checksum = "415d9944693cb90382053259f89fbb077ea730ad7273047ec63b19bc9b160ba8" 744 | dependencies = [ 745 | "log", 746 | "once_cell", 747 | "ring", 748 | "rustls-pki-types", 749 | "rustls-webpki", 750 | "subtle", 751 | "zeroize", 752 | ] 753 | 754 | [[package]] 755 | name = "rustls-pki-types" 756 | version = "1.9.0" 757 | source = "registry+https://github.com/rust-lang/crates.io-index" 758 | checksum = "0e696e35370c65c9c541198af4543ccd580cf17fc25d8e05c5a242b202488c55" 759 | 760 | [[package]] 761 | name = "rustls-webpki" 762 | version = "0.102.8" 763 | source = "registry+https://github.com/rust-lang/crates.io-index" 764 | checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" 765 | dependencies = [ 766 | "ring", 767 | "rustls-pki-types", 768 | "untrusted", 769 | ] 770 | 771 | [[package]] 772 | name = "rustversion" 773 | version = "1.0.17" 774 | source = "registry+https://github.com/rust-lang/crates.io-index" 775 | checksum = "955d28af4278de8121b7ebeb796b6a45735dc01436d898801014aced2773a3d6" 776 | 777 | [[package]] 778 | name = "ryu" 779 | version = "1.0.18" 780 | source = "registry+https://github.com/rust-lang/crates.io-index" 781 | checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" 782 | 783 | [[package]] 784 | name = "serde" 785 | version = "1.0.210" 786 | source = "registry+https://github.com/rust-lang/crates.io-index" 787 | checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a" 788 | dependencies = [ 789 | "serde_derive", 790 | ] 791 | 792 | [[package]] 793 | name = "serde_derive" 794 | version = "1.0.210" 795 | source = "registry+https://github.com/rust-lang/crates.io-index" 796 | checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" 797 | dependencies = [ 798 | "proc-macro2", 799 | "quote", 800 | "syn", 801 | ] 802 | 803 | [[package]] 804 | name = "serde_json" 805 | version = "1.0.128" 806 | source = "registry+https://github.com/rust-lang/crates.io-index" 807 | checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8" 808 | dependencies = [ 809 | "itoa", 810 | "memchr", 811 | "ryu", 812 | "serde", 813 | ] 814 | 815 | [[package]] 816 | name = "serde_path_to_error" 817 | version = "0.1.16" 818 | source = "registry+https://github.com/rust-lang/crates.io-index" 819 | checksum = "af99884400da37c88f5e9146b7f1fd0fbcae8f6eec4e9da38b67d05486f814a6" 820 | dependencies = [ 821 | "itoa", 822 | "serde", 823 | ] 824 | 825 | [[package]] 826 | name = "serde_urlencoded" 827 | version = "0.7.1" 828 | source = "registry+https://github.com/rust-lang/crates.io-index" 829 | checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" 830 | dependencies = [ 831 | "form_urlencoded", 832 | "itoa", 833 | "ryu", 834 | "serde", 835 | ] 836 | 837 | [[package]] 838 | name = "sharded-slab" 839 | version = "0.1.7" 840 | source = "registry+https://github.com/rust-lang/crates.io-index" 841 | checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" 842 | dependencies = [ 843 | "lazy_static", 844 | ] 845 | 846 | [[package]] 847 | name = "shlex" 848 | version = "1.3.0" 849 | source = "registry+https://github.com/rust-lang/crates.io-index" 850 | checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" 851 | 852 | [[package]] 853 | name = "smallvec" 854 | version = "1.13.2" 855 | source = "registry+https://github.com/rust-lang/crates.io-index" 856 | checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" 857 | 858 | [[package]] 859 | name = "socket2" 860 | version = "0.5.7" 861 | source = "registry+https://github.com/rust-lang/crates.io-index" 862 | checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c" 863 | dependencies = [ 864 | "libc", 865 | "windows-sys", 866 | ] 867 | 868 | [[package]] 869 | name = "spin" 870 | version = "0.9.8" 871 | source = "registry+https://github.com/rust-lang/crates.io-index" 872 | checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" 873 | 874 | [[package]] 875 | name = "strsim" 876 | version = "0.11.1" 877 | source = "registry+https://github.com/rust-lang/crates.io-index" 878 | checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" 879 | 880 | [[package]] 881 | name = "subtle" 882 | version = "2.6.1" 883 | source = "registry+https://github.com/rust-lang/crates.io-index" 884 | checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" 885 | 886 | [[package]] 887 | name = "syn" 888 | version = "2.0.79" 889 | source = "registry+https://github.com/rust-lang/crates.io-index" 890 | checksum = "89132cd0bf050864e1d38dc3bbc07a0eb8e7530af26344d3d2bbbef83499f590" 891 | dependencies = [ 892 | "proc-macro2", 893 | "quote", 894 | "unicode-ident", 895 | ] 896 | 897 | [[package]] 898 | name = "sync_wrapper" 899 | version = "0.1.2" 900 | source = "registry+https://github.com/rust-lang/crates.io-index" 901 | checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" 902 | 903 | [[package]] 904 | name = "sync_wrapper" 905 | version = "1.0.1" 906 | source = "registry+https://github.com/rust-lang/crates.io-index" 907 | checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" 908 | 909 | [[package]] 910 | name = "thread_local" 911 | version = "1.1.8" 912 | source = "registry+https://github.com/rust-lang/crates.io-index" 913 | checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" 914 | dependencies = [ 915 | "cfg-if", 916 | "once_cell", 917 | ] 918 | 919 | [[package]] 920 | name = "tinyvec" 921 | version = "1.8.0" 922 | source = "registry+https://github.com/rust-lang/crates.io-index" 923 | checksum = "445e881f4f6d382d5f27c034e25eb92edd7c784ceab92a0937db7f2e9471b938" 924 | dependencies = [ 925 | "tinyvec_macros", 926 | ] 927 | 928 | [[package]] 929 | name = "tinyvec_macros" 930 | version = "0.1.1" 931 | source = "registry+https://github.com/rust-lang/crates.io-index" 932 | checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" 933 | 934 | [[package]] 935 | name = "tokio" 936 | version = "1.40.0" 937 | source = "registry+https://github.com/rust-lang/crates.io-index" 938 | checksum = "e2b070231665d27ad9ec9b8df639893f46727666c6767db40317fbe920a5d998" 939 | dependencies = [ 940 | "backtrace", 941 | "libc", 942 | "mio", 943 | "pin-project-lite", 944 | "socket2", 945 | "tokio-macros", 946 | "windows-sys", 947 | ] 948 | 949 | [[package]] 950 | name = "tokio-macros" 951 | version = "2.4.0" 952 | source = "registry+https://github.com/rust-lang/crates.io-index" 953 | checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" 954 | dependencies = [ 955 | "proc-macro2", 956 | "quote", 957 | "syn", 958 | ] 959 | 960 | [[package]] 961 | name = "tower" 962 | version = "0.5.1" 963 | source = "registry+https://github.com/rust-lang/crates.io-index" 964 | checksum = "2873938d487c3cfb9aed7546dc9f2711d867c9f90c46b889989a2cb84eba6b4f" 965 | dependencies = [ 966 | "futures-core", 967 | "futures-util", 968 | "pin-project-lite", 969 | "sync_wrapper 0.1.2", 970 | "tokio", 971 | "tower-layer", 972 | "tower-service", 973 | "tracing", 974 | ] 975 | 976 | [[package]] 977 | name = "tower-layer" 978 | version = "0.3.3" 979 | source = "registry+https://github.com/rust-lang/crates.io-index" 980 | checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" 981 | 982 | [[package]] 983 | name = "tower-service" 984 | version = "0.3.3" 985 | source = "registry+https://github.com/rust-lang/crates.io-index" 986 | checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" 987 | 988 | [[package]] 989 | name = "tracing" 990 | version = "0.1.40" 991 | source = "registry+https://github.com/rust-lang/crates.io-index" 992 | checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" 993 | dependencies = [ 994 | "log", 995 | "pin-project-lite", 996 | "tracing-attributes", 997 | "tracing-core", 998 | ] 999 | 1000 | [[package]] 1001 | name = "tracing-attributes" 1002 | version = "0.1.27" 1003 | source = "registry+https://github.com/rust-lang/crates.io-index" 1004 | checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" 1005 | dependencies = [ 1006 | "proc-macro2", 1007 | "quote", 1008 | "syn", 1009 | ] 1010 | 1011 | [[package]] 1012 | name = "tracing-core" 1013 | version = "0.1.32" 1014 | source = "registry+https://github.com/rust-lang/crates.io-index" 1015 | checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" 1016 | dependencies = [ 1017 | "once_cell", 1018 | "valuable", 1019 | ] 1020 | 1021 | [[package]] 1022 | name = "tracing-log" 1023 | version = "0.2.0" 1024 | source = "registry+https://github.com/rust-lang/crates.io-index" 1025 | checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" 1026 | dependencies = [ 1027 | "log", 1028 | "once_cell", 1029 | "tracing-core", 1030 | ] 1031 | 1032 | [[package]] 1033 | name = "tracing-subscriber" 1034 | version = "0.3.18" 1035 | source = "registry+https://github.com/rust-lang/crates.io-index" 1036 | checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" 1037 | dependencies = [ 1038 | "nu-ansi-term", 1039 | "sharded-slab", 1040 | "smallvec", 1041 | "thread_local", 1042 | "tracing-core", 1043 | "tracing-log", 1044 | ] 1045 | 1046 | [[package]] 1047 | name = "unicode-bidi" 1048 | version = "0.3.17" 1049 | source = "registry+https://github.com/rust-lang/crates.io-index" 1050 | checksum = "5ab17db44d7388991a428b2ee655ce0c212e862eff1768a455c58f9aad6e7893" 1051 | 1052 | [[package]] 1053 | name = "unicode-ident" 1054 | version = "1.0.13" 1055 | source = "registry+https://github.com/rust-lang/crates.io-index" 1056 | checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" 1057 | 1058 | [[package]] 1059 | name = "unicode-normalization" 1060 | version = "0.1.24" 1061 | source = "registry+https://github.com/rust-lang/crates.io-index" 1062 | checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956" 1063 | dependencies = [ 1064 | "tinyvec", 1065 | ] 1066 | 1067 | [[package]] 1068 | name = "untrusted" 1069 | version = "0.9.0" 1070 | source = "registry+https://github.com/rust-lang/crates.io-index" 1071 | checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" 1072 | 1073 | [[package]] 1074 | name = "ureq" 1075 | version = "2.10.1" 1076 | source = "registry+https://github.com/rust-lang/crates.io-index" 1077 | checksum = "b74fc6b57825be3373f7054754755f03ac3a8f5d70015ccad699ba2029956f4a" 1078 | dependencies = [ 1079 | "base64", 1080 | "flate2", 1081 | "log", 1082 | "once_cell", 1083 | "rustls", 1084 | "rustls-pki-types", 1085 | "serde", 1086 | "serde_json", 1087 | "url", 1088 | "webpki-roots", 1089 | ] 1090 | 1091 | [[package]] 1092 | name = "url" 1093 | version = "2.5.2" 1094 | source = "registry+https://github.com/rust-lang/crates.io-index" 1095 | checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c" 1096 | dependencies = [ 1097 | "form_urlencoded", 1098 | "idna", 1099 | "percent-encoding", 1100 | ] 1101 | 1102 | [[package]] 1103 | name = "utf8parse" 1104 | version = "0.2.2" 1105 | source = "registry+https://github.com/rust-lang/crates.io-index" 1106 | checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" 1107 | 1108 | [[package]] 1109 | name = "valuable" 1110 | version = "0.1.0" 1111 | source = "registry+https://github.com/rust-lang/crates.io-index" 1112 | checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" 1113 | 1114 | [[package]] 1115 | name = "wasi" 1116 | version = "0.11.0+wasi-snapshot-preview1" 1117 | source = "registry+https://github.com/rust-lang/crates.io-index" 1118 | checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" 1119 | 1120 | [[package]] 1121 | name = "wasm-bindgen" 1122 | version = "0.2.95" 1123 | source = "registry+https://github.com/rust-lang/crates.io-index" 1124 | checksum = "128d1e363af62632b8eb57219c8fd7877144af57558fb2ef0368d0087bddeb2e" 1125 | dependencies = [ 1126 | "cfg-if", 1127 | "once_cell", 1128 | "wasm-bindgen-macro", 1129 | ] 1130 | 1131 | [[package]] 1132 | name = "wasm-bindgen-backend" 1133 | version = "0.2.95" 1134 | source = "registry+https://github.com/rust-lang/crates.io-index" 1135 | checksum = "cb6dd4d3ca0ddffd1dd1c9c04f94b868c37ff5fac97c30b97cff2d74fce3a358" 1136 | dependencies = [ 1137 | "bumpalo", 1138 | "log", 1139 | "once_cell", 1140 | "proc-macro2", 1141 | "quote", 1142 | "syn", 1143 | "wasm-bindgen-shared", 1144 | ] 1145 | 1146 | [[package]] 1147 | name = "wasm-bindgen-macro" 1148 | version = "0.2.95" 1149 | source = "registry+https://github.com/rust-lang/crates.io-index" 1150 | checksum = "e79384be7f8f5a9dd5d7167216f022090cf1f9ec128e6e6a482a2cb5c5422c56" 1151 | dependencies = [ 1152 | "quote", 1153 | "wasm-bindgen-macro-support", 1154 | ] 1155 | 1156 | [[package]] 1157 | name = "wasm-bindgen-macro-support" 1158 | version = "0.2.95" 1159 | source = "registry+https://github.com/rust-lang/crates.io-index" 1160 | checksum = "26c6ab57572f7a24a4985830b120de1594465e5d500f24afe89e16b4e833ef68" 1161 | dependencies = [ 1162 | "proc-macro2", 1163 | "quote", 1164 | "syn", 1165 | "wasm-bindgen-backend", 1166 | "wasm-bindgen-shared", 1167 | ] 1168 | 1169 | [[package]] 1170 | name = "wasm-bindgen-shared" 1171 | version = "0.2.95" 1172 | source = "registry+https://github.com/rust-lang/crates.io-index" 1173 | checksum = "65fc09f10666a9f147042251e0dda9c18f166ff7de300607007e96bdebc1068d" 1174 | 1175 | [[package]] 1176 | name = "webpki-roots" 1177 | version = "0.26.6" 1178 | source = "registry+https://github.com/rust-lang/crates.io-index" 1179 | checksum = "841c67bff177718f1d4dfefde8d8f0e78f9b6589319ba88312f567fc5841a958" 1180 | dependencies = [ 1181 | "rustls-pki-types", 1182 | ] 1183 | 1184 | [[package]] 1185 | name = "winapi" 1186 | version = "0.3.9" 1187 | source = "registry+https://github.com/rust-lang/crates.io-index" 1188 | checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" 1189 | dependencies = [ 1190 | "winapi-i686-pc-windows-gnu", 1191 | "winapi-x86_64-pc-windows-gnu", 1192 | ] 1193 | 1194 | [[package]] 1195 | name = "winapi-i686-pc-windows-gnu" 1196 | version = "0.4.0" 1197 | source = "registry+https://github.com/rust-lang/crates.io-index" 1198 | checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" 1199 | 1200 | [[package]] 1201 | name = "winapi-x86_64-pc-windows-gnu" 1202 | version = "0.4.0" 1203 | source = "registry+https://github.com/rust-lang/crates.io-index" 1204 | checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" 1205 | 1206 | [[package]] 1207 | name = "windows-core" 1208 | version = "0.52.0" 1209 | source = "registry+https://github.com/rust-lang/crates.io-index" 1210 | checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" 1211 | dependencies = [ 1212 | "windows-targets", 1213 | ] 1214 | 1215 | [[package]] 1216 | name = "windows-sys" 1217 | version = "0.52.0" 1218 | source = "registry+https://github.com/rust-lang/crates.io-index" 1219 | checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" 1220 | dependencies = [ 1221 | "windows-targets", 1222 | ] 1223 | 1224 | [[package]] 1225 | name = "windows-targets" 1226 | version = "0.52.6" 1227 | source = "registry+https://github.com/rust-lang/crates.io-index" 1228 | checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" 1229 | dependencies = [ 1230 | "windows_aarch64_gnullvm", 1231 | "windows_aarch64_msvc", 1232 | "windows_i686_gnu", 1233 | "windows_i686_gnullvm", 1234 | "windows_i686_msvc", 1235 | "windows_x86_64_gnu", 1236 | "windows_x86_64_gnullvm", 1237 | "windows_x86_64_msvc", 1238 | ] 1239 | 1240 | [[package]] 1241 | name = "windows_aarch64_gnullvm" 1242 | version = "0.52.6" 1243 | source = "registry+https://github.com/rust-lang/crates.io-index" 1244 | checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" 1245 | 1246 | [[package]] 1247 | name = "windows_aarch64_msvc" 1248 | version = "0.52.6" 1249 | source = "registry+https://github.com/rust-lang/crates.io-index" 1250 | checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" 1251 | 1252 | [[package]] 1253 | name = "windows_i686_gnu" 1254 | version = "0.52.6" 1255 | source = "registry+https://github.com/rust-lang/crates.io-index" 1256 | checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" 1257 | 1258 | [[package]] 1259 | name = "windows_i686_gnullvm" 1260 | version = "0.52.6" 1261 | source = "registry+https://github.com/rust-lang/crates.io-index" 1262 | checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" 1263 | 1264 | [[package]] 1265 | name = "windows_i686_msvc" 1266 | version = "0.52.6" 1267 | source = "registry+https://github.com/rust-lang/crates.io-index" 1268 | checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" 1269 | 1270 | [[package]] 1271 | name = "windows_x86_64_gnu" 1272 | version = "0.52.6" 1273 | source = "registry+https://github.com/rust-lang/crates.io-index" 1274 | checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" 1275 | 1276 | [[package]] 1277 | name = "windows_x86_64_gnullvm" 1278 | version = "0.52.6" 1279 | source = "registry+https://github.com/rust-lang/crates.io-index" 1280 | checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" 1281 | 1282 | [[package]] 1283 | name = "windows_x86_64_msvc" 1284 | version = "0.52.6" 1285 | source = "registry+https://github.com/rust-lang/crates.io-index" 1286 | checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" 1287 | 1288 | [[package]] 1289 | name = "zerocopy" 1290 | version = "0.7.35" 1291 | source = "registry+https://github.com/rust-lang/crates.io-index" 1292 | checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" 1293 | dependencies = [ 1294 | "byteorder", 1295 | "zerocopy-derive", 1296 | ] 1297 | 1298 | [[package]] 1299 | name = "zerocopy-derive" 1300 | version = "0.7.35" 1301 | source = "registry+https://github.com/rust-lang/crates.io-index" 1302 | checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" 1303 | dependencies = [ 1304 | "proc-macro2", 1305 | "quote", 1306 | "syn", 1307 | ] 1308 | 1309 | [[package]] 1310 | name = "zeroize" 1311 | version = "1.8.1" 1312 | source = "registry+https://github.com/rust-lang/crates.io-index" 1313 | checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" 1314 | --------------------------------------------------------------------------------