├── .github └── workflows │ └── build.yml ├── .gitignore ├── Cargo.toml ├── LICENSE-APACHE ├── LICENSE-MIT ├── Makefile ├── README.md ├── block-format ├── Cargo.toml ├── src │ ├── basic_block.rs │ ├── error.rs │ └── lib.rs └── tests │ └── lib.rs ├── datastore ├── Cargo.toml ├── rocksdb │ ├── Cargo.toml │ └── src │ │ ├── error.rs │ │ ├── lib.rs │ │ ├── tests.rs │ │ └── tx.rs └── src │ ├── basic_ds.rs │ ├── datastore.rs │ ├── error.rs │ ├── key.rs │ ├── keytransform │ ├── mod.rs │ └── transforms.rs │ ├── lib.rs │ ├── mount │ ├── async_results.rs │ ├── mod.rs │ └── sync_results.rs │ ├── namespace │ └── mod.rs │ ├── query │ ├── async_results.rs │ ├── filter.rs │ ├── mod.rs │ ├── order.rs │ └── sync_results.rs │ ├── singleton.rs │ └── tests │ ├── basic_ds_test.rs │ ├── common.rs │ ├── key_test.rs │ ├── keytransform_test.rs │ ├── mod.rs │ └── query │ ├── filter_test.rs │ ├── mod.rs │ ├── order_test.rs │ └── query_test.rs ├── fs-lock ├── Cargo.toml └── src │ └── lib.rs ├── ipfs └── blockstore │ ├── Cargo.toml │ └── src │ ├── error.rs │ └── lib.rs ├── ipld ├── amt │ ├── Cargo.toml │ └── src │ │ ├── blocks.rs │ │ ├── error.rs │ │ ├── lib.rs │ │ ├── node │ │ ├── iter.rs │ │ ├── mod.rs │ │ └── trait_impl.rs │ │ └── tests │ │ ├── amt_test.rs │ │ ├── cbor_test.rs │ │ └── mod.rs ├── core │ ├── Cargo.toml │ ├── benches │ │ └── benchmarks.rs │ ├── src │ │ ├── convert.rs │ │ ├── error.rs │ │ ├── lib.rs │ │ ├── node.rs │ │ └── value.rs │ └── tests │ │ ├── node_test.rs │ │ └── test_objects │ │ ├── QmRgutAxd8t7oGkSm4wmeuByG6M51wcTso6cubDdQtuEfL │ │ ├── array-link.cbor │ │ ├── array-link.json │ │ ├── empty-array.cbor │ │ ├── empty-array.json │ │ ├── empty-obj.cbor │ │ ├── empty-obj.json │ │ ├── expected.json │ │ ├── foo.cbor │ │ ├── foo.json │ │ ├── foo2.cbor │ │ ├── foo2.json │ │ ├── non-canon.cbor │ │ ├── obj-no-link.cbor │ │ ├── obj-no-link.json │ │ ├── obj-with-link.cbor │ │ └── obj-with-link.json ├── format │ ├── Cargo.toml │ ├── src │ │ ├── coding.rs │ │ ├── daghelpers.rs │ │ ├── error.rs │ │ ├── format.rs │ │ ├── lib.rs │ │ ├── merkledag.rs │ │ ├── navipld.rs │ │ └── walker.rs │ └── tests │ │ ├── common.rs │ │ ├── test_coding.rs │ │ └── test_walker.rs └── hamt │ ├── Cargo.toml │ └── src │ ├── error.rs │ ├── hash.rs │ ├── ipld.rs │ ├── lib.rs │ ├── node │ ├── mod.rs │ └── trait_impl.rs │ └── tests │ ├── cbor_test.rs │ ├── hamt_test.rs │ ├── hash_test.rs │ ├── ipld_test.rs │ └── mod.rs ├── paritylibs ├── kvdb-rocksdb │ ├── CHANGELOG.md │ ├── Cargo.toml │ ├── benches │ │ ├── .gitignore │ │ └── bench_read_perf.rs │ └── src │ │ ├── iter.rs │ │ ├── lib.rs │ │ └── stats.rs ├── kvdb-shared-tests │ ├── Cargo.toml │ └── src │ │ └── lib.rs └── kvdb │ ├── CHANGELOG.md │ ├── Cargo.toml │ └── src │ ├── io_stats.rs │ └── lib.rs └── rust-toolchain /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | on: [push, pull_request] 2 | 3 | name: build 4 | 5 | jobs: 6 | build: 7 | name: Lint and Test 8 | strategy: 9 | matrix: 10 | platform: [ubuntu-latest, macos-latest] 11 | toolchain: [nightly-2019-12-20] 12 | runs-on: ${{ matrix.platform }} 13 | 14 | steps: 15 | - name: Checkout sources 16 | uses: actions/checkout@v2 17 | 18 | - name: Install Rust toolchain 19 | uses: actions-rs/toolchain@v1 20 | with: 21 | profile: minimal 22 | toolchain: ${{ matrix.toolchain }} 23 | override: true 24 | components: rustfmt, clippy 25 | 26 | - name: Run cargo fmt 27 | uses: actions-rs/cargo@v1 28 | with: 29 | command: fmt 30 | args: --all -- --check 31 | 32 | - name: Run cargo clippy 33 | uses: actions-rs/cargo@v1 34 | with: 35 | command: clippy 36 | args: --all-targets --all-features -- -D warnings 37 | 38 | - name: Run cargo test 39 | uses: actions-rs/cargo@v1 40 | with: 41 | command: test 42 | 43 | - name: Run cargo test with all features 44 | uses: actions-rs/cargo@v1 45 | with: 46 | command: test 47 | args: --all-features 48 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | **/target/ 2 | **/*.rs.bk 3 | 4 | # Remove Cargo.lock when creating an executable, leave it for libraries 5 | # More information here http://doc.crates.io/guide.html#cargotoml-vs-cargolock 6 | Cargo.lock 7 | 8 | # Intellij IDEA config 9 | .idea/ 10 | .vscode/ 11 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | members = [ 3 | "block-format", 4 | "datastore", 5 | "datastore/rocksdb", 6 | "fs-lock", 7 | "ipfs/blockstore", 8 | # "ipld/amt", 9 | "ipld/core", 10 | "ipld/format", 11 | # "ipld/hamt", 12 | "paritylibs/kvdb", 13 | "paritylibs/kvdb-rocksdb", 14 | "paritylibs/kvdb-shared-tests", 15 | ] 16 | 17 | [replace] 18 | "cid:0.5.0" = { git = "https://github.com/PolkaX/rust-cid", branch = "impl-cbor-and-json" } 19 | -------------------------------------------------------------------------------- /LICENSE-MIT: -------------------------------------------------------------------------------- 1 | MIT License 2 | Copyright (c) 2016 The roaring-rs developers. 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining a copy of 5 | this software and associated documentation files (the "Software"), to deal in 6 | the Software without restriction, including without limitation the rights to 7 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies 8 | of the Software, and to permit persons to whom the Software is furnished to do 9 | so, subject to the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be included in all 12 | copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 20 | SOFTWARE. 21 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | SHELL := /bin/bash 2 | #ENABLE_FEATURES ?= 3 | 4 | default: dev 5 | 6 | .PHONY: all 7 | 8 | clean: 9 | cargo clean 10 | 11 | ## Development builds 12 | ## ------------------ 13 | 14 | all: format build test 15 | 16 | dev: format clippy test 17 | 18 | build: 19 | cargo build #--no-default-features --features "${ENABLE_FEATURES}" 20 | 21 | ## Release builds (optimized dev builds) 22 | ## ---------------------------- 23 | 24 | release: 25 | cargo build --release #--no-default-features --features "${ENABLE_FEATURES}" 26 | 27 | ## Testing 28 | ## ----- 29 | 30 | test: 31 | export LOG_LEVEL=DEBUG && \ 32 | export RUST_BACKTRACE=1 && \ 33 | cargo test --all ${EXTRA_CARGO_ARGS} -- --nocapture $(TEST_THREADS) 34 | 35 | ## Benchmarking 36 | ## ----- 37 | 38 | bench: 39 | export LOG_LEVEL=ERROR && \ 40 | export RUST_BACKTRACE=1 && \ 41 | cargo bench 42 | 43 | ## Static analysis 44 | ## --------------- 45 | 46 | unset-override: 47 | @# unset first in case of any previous overrides 48 | @if rustup override list | grep `pwd` > /dev/null; then rustup override unset; fi 49 | 50 | pre-format: unset-override 51 | @rustup component add rustfmt 52 | 53 | format: pre-format 54 | @cargo fmt --all -- --check >/dev/null || \ 55 | cargo fmt --all 56 | 57 | pre-clippy: unset-override 58 | @rustup component add clippy 59 | 60 | clippy: pre-clippy 61 | @cargo clippy --all --all-targets -- \ 62 | -A clippy::module_inception -A clippy::needless_pass_by_value -A clippy::cognitive_complexity \ 63 | -A clippy::unreadable_literal -A clippy::should_implement_trait -A clippy::verbose_bit_mask \ 64 | -A clippy::implicit_hasher -A clippy::large_enum_variant -A clippy::new_without_default \ 65 | -A clippy::neg_cmp_op_on_partial_ord -A clippy::too_many_arguments \ 66 | -A clippy::excessive_precision -A clippy::collapsible_if -A clippy::blacklisted_name \ 67 | -A clippy::needless_range_loop -A clippy::redundant_closure \ 68 | -A clippy::match_wild_err_arm -A clippy::blacklisted_name -A clippy::redundant_closure_call \ 69 | -A clippy::identity_conversion 70 | 71 | pre-audit: 72 | $(eval LATEST_AUDIT_VERSION := $(strip $(shell cargo search cargo-audit | head -n 1 | awk '{ gsub(/"/, "", $$3); print $$3 }'))) 73 | $(eval CURRENT_AUDIT_VERSION = $(strip $(shell (cargo audit --version 2> /dev/null || echo "noop 0") | awk '{ print $$2 }'))) 74 | @if [ "$(LATEST_AUDIT_VERSION)" != "$(CURRENT_AUDIT_VERSION)" ]; then \ 75 | cargo install cargo-audit --force; \ 76 | fi 77 | 78 | # Check for security vulnerabilities 79 | audit: pre-audit 80 | cargo audit 81 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # rust-ipfs 2 | 3 | [![Actions Status][ga-svg]][ga-url] 4 | [![GitHub License][license-svg]][license-url] 5 | [![Dependency Status][dep-svg]][dep-rs] 6 | 7 | [ga-svg]: https://github.com/PolkaX/rust-ipfs/workflows/build/badge.svg 8 | [ga-url]: https://github.com/PolkaX/rust-ipfs/actions 9 | [license-svg]: https://img.shields.io/badge/License-MIT/APACHE_v2.0-blue.svg 10 | [license-url]: https://github.com/PolkaX/rust-ipfs/blob/develop/LICENSE-MIT 11 | [dep-svg]: https://deps.rs/repo/github/PolkaX/rust-ipfs/status.svg 12 | [dep-rs]: https://deps.rs/repo/github/PolkaX/rust-ipfs 13 | 14 | ## License 15 | 16 | Licensed under either of 17 | 18 | * Apache License, Version 2.0 ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0) 19 | * MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT) 20 | 21 | at your option. 22 | 23 | ## Contribution 24 | 25 | Unless you explicitly state otherwise, any contribution intentionally submitted 26 | for inclusion in the work by you shall be dual licensed as above, without any 27 | additional terms or conditions. 28 | -------------------------------------------------------------------------------- /block-format/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "block-format" 3 | version = "0.1.0" 4 | authors = ["PolkaX "] 5 | edition = "2018" 6 | 7 | license = "MIT/Apache-2.0" 8 | repository = "https://github.com/PolkaX/rust-ipfs" 9 | description = "Implementation of the block format" 10 | keywords = ["ipfs", "block-format"] 11 | 12 | [dependencies] 13 | bytes = "0.5" 14 | cid = "0.5" 15 | multihash = "0.11" 16 | thiserror = "1.0" 17 | -------------------------------------------------------------------------------- /block-format/src/basic_block.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2020 PolkaX. Licensed under MIT or Apache-2.0. 2 | 3 | use bytes::Bytes; 4 | 5 | use cid::{Cid, ExtMultihashRef, IntoExt}; 6 | use multihash::Sha2_256; 7 | 8 | use crate::error::Result; 9 | 10 | /// The trait for getting raw data and cid of block. 11 | pub trait Block: AsRef { 12 | /// Get the raw data of block. 13 | fn raw_data(&self) -> &Bytes; 14 | 15 | /// Get the cid. 16 | fn cid(&self) -> &Cid { 17 | self.as_ref() 18 | } 19 | } 20 | 21 | impl Block for BasicBlock { 22 | fn raw_data(&self) -> &Bytes { 23 | &self.data 24 | } 25 | } 26 | 27 | impl AsRef for BasicBlock { 28 | fn as_ref(&self) -> &Cid { 29 | &self.cid 30 | } 31 | } 32 | 33 | /// The basic block. 34 | #[derive(Clone, Debug)] 35 | pub struct BasicBlock { 36 | cid: Cid, 37 | data: Bytes, 38 | } 39 | 40 | impl BasicBlock { 41 | /// Creates a new `BasicBlock` with given bytes, and its CID is version 0. 42 | pub fn new(data: Bytes) -> BasicBlock { 43 | let sha256_hash = Sha2_256::digest(data.as_ref()).into_ext(); 44 | BasicBlock { 45 | data, 46 | cid: Cid::new_v0(sha256_hash).expect("invalid hash for CIDv0"), 47 | } 48 | } 49 | 50 | /// Creates a new `BasicBlock` with given bytes and CID. 51 | pub fn new_with_cid(data: Bytes, cid: Cid) -> Result { 52 | use crate::error::BlockFormatError; 53 | let hash1 = cid.hash(); 54 | let hash2 = hash1.algorithm().digest(data.as_ref()); 55 | if hash1 != hash2 { 56 | return Err(BlockFormatError::WrongHash( 57 | hash1.as_bytes().to_vec(), 58 | hash2.as_bytes().to_vec(), 59 | )); 60 | } 61 | Ok(BasicBlock { data, cid }) 62 | } 63 | 64 | /// Get the multihash of cid of the basic block. 65 | pub fn multihash(&self) -> ExtMultihashRef { 66 | self.cid.hash() 67 | } 68 | } 69 | 70 | impl std::fmt::Display for BasicBlock { 71 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 72 | write!(f, "[Block {:?}]", self) 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /block-format/src/error.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2020 PolkaX. Licensed under MIT or Apache-2.0. 2 | 3 | /// Type alias to use this library's [`BlockFormatError`] type in a `Result`. 4 | pub type Result = std::result::Result; 5 | 6 | /// Errors generated from this library. 7 | #[derive(Debug, thiserror::Error)] 8 | pub enum BlockFormatError { 9 | /// The data of block is not match given hash. 10 | #[error("data is not match given hash, fst: {0:?}, snd: {1:?}")] 11 | WrongHash(Vec, Vec), 12 | /// Cid error. 13 | #[error("cid error: {0}")] 14 | CidError(#[from] cid::Error), 15 | /// Other type error. 16 | #[error("other err: {0}")] 17 | Other(#[from] Box), 18 | } 19 | -------------------------------------------------------------------------------- /block-format/src/lib.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2020 PolkaX. Licensed under MIT or Apache-2.0. 2 | 3 | //! Implementation of `block format` in Rust, 4 | //! which provides the `BasicBlock` structure and the `Block` trait. 5 | 6 | #![deny(missing_docs)] 7 | 8 | mod basic_block; 9 | mod error; 10 | 11 | pub use bytes::{Buf, Bytes}; 12 | pub use cid::Cid; 13 | pub use multihash::MultihashRef; 14 | 15 | pub use self::basic_block::{BasicBlock, Block}; 16 | pub use self::error::{BlockFormatError, Result}; 17 | -------------------------------------------------------------------------------- /block-format/tests/lib.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2020 PolkaX. Licensed under MIT or Apache-2.0. 2 | 3 | use multihash::Sha2_256; 4 | 5 | use block_format::{BasicBlock, Block}; 6 | use cid::IntoExt; 7 | 8 | #[test] 9 | fn test_blocks_basic() { 10 | // would not panic 11 | let empty = vec![]; 12 | BasicBlock::new(empty.into()); 13 | 14 | BasicBlock::new(b"Hello world!".as_ref().into()); 15 | } 16 | 17 | #[test] 18 | fn test_data() { 19 | let data = b"some data"; 20 | let block = BasicBlock::new(data.as_ref().into()); 21 | 22 | assert_eq!(block.raw_data().as_ref(), data.as_ref()) 23 | } 24 | 25 | #[test] 26 | fn test_hash() { 27 | let data = b"some other data"; 28 | let block = BasicBlock::new(data.as_ref().into()); 29 | 30 | let hash = Sha2_256::digest(data.as_ref()).into_ext(); 31 | 32 | assert_eq!(block.multihash(), hash); 33 | } 34 | 35 | #[test] 36 | fn test_cid() { 37 | let data = b"yet another data"; 38 | let block = BasicBlock::new(data.as_ref().into()); 39 | let cid = block.cid(); 40 | 41 | assert_eq!(block.multihash(), cid.hash()); 42 | } 43 | -------------------------------------------------------------------------------- /datastore/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "datastore" 3 | version = "0.1.0" 4 | authors = ["PolkaX "] 5 | edition = "2018" 6 | 7 | [dependencies] 8 | async-std = { version = "1.5", features = ["unstable"], optional = true} 9 | async-trait = { version = "0.1", optional = true } 10 | parking_lot = "0.10.0" 11 | path-clean = "0.1" 12 | serde = { version = "1.0", features = ["derive"] } 13 | thiserror = "1.0" 14 | uuid = { version = "0.8", features = ["v4"] } 15 | 16 | [dev-dependencies] 17 | matches = "0.1" 18 | rand = "0.7.2" 19 | serde_json = "1.0" 20 | 21 | [features] 22 | async = [ 23 | "async-std", 24 | "async-trait" 25 | ] 26 | -------------------------------------------------------------------------------- /datastore/rocksdb/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "ds-rocksdb" 3 | version = "0.1.0" 4 | authors = ["PolkaX "] 5 | edition = "2018" 6 | 7 | [dependencies] 8 | thiserror = "1.0" 9 | 10 | datastore = { path = ".." } 11 | kvdb = { path = "../../paritylibs/kvdb" } 12 | kvdb-rocksdb = { path = "../../paritylibs/kvdb-rocksdb" } 13 | 14 | [dev-dependencies] 15 | matches = "0.1" 16 | rand = "0.7" 17 | tempfile = "3.1" 18 | -------------------------------------------------------------------------------- /datastore/rocksdb/src/error.rs: -------------------------------------------------------------------------------- 1 | use std::io; 2 | use thiserror::Error; 3 | 4 | pub type Result = std::result::Result; 5 | #[derive(Error, Debug)] 6 | pub enum RocksDBError { 7 | #[error("rocksdb io error: {0:?}")] 8 | DBError(#[from] io::Error), 9 | 10 | #[error("datastore error: {0:?}")] 11 | DataStoreError(#[from] datastore::DSError), 12 | 13 | #[error("invalid column name: {0}")] 14 | InvalidColumnName(String), 15 | 16 | #[error("other err: {0}")] 17 | Other(#[from] Box), 18 | } 19 | -------------------------------------------------------------------------------- /datastore/rocksdb/src/lib.rs: -------------------------------------------------------------------------------- 1 | #![allow(clippy::or_fun_call)] 2 | 3 | mod error; 4 | #[cfg(test)] 5 | mod tests; 6 | mod tx; 7 | 8 | use std::collections::HashSet; 9 | use std::iter::FromIterator; 10 | use std::result; 11 | use std::sync::Arc; 12 | 13 | use datastore::{ 14 | key::{self, Key}, 15 | query, Batching, Datastore, Read, SyncQuery, TxnDatastore, Write, 16 | }; 17 | use error::*; 18 | // re-export 19 | pub use kvdb::DBTransaction; 20 | pub use kvdb_rocksdb::DEFAULT_COLUMN_NAME; 21 | pub use kvdb_rocksdb::{Database as RocksDatabase, DatabaseConfig}; 22 | 23 | pub use crate::tx::Transaction; 24 | 25 | pub type DSResult = result::Result; 26 | 27 | fn pre_process_key(cols: *const HashSet, key: &Key) -> (&str, &str) { 28 | let (prefix, k) = key.split_prefix(); 29 | // it's safe for if db is dropped, the process should come to end. 30 | // the cols's lifetime is same as db. 31 | let cols = unsafe { &*cols }; 32 | prefix 33 | .map(|p| { 34 | if cols.contains(p) { 35 | (p, k) 36 | } else { 37 | (DEFAULT_COLUMN_NAME, key.as_str()) 38 | } 39 | }) 40 | .unwrap_or((DEFAULT_COLUMN_NAME, key.as_str())) 41 | } 42 | 43 | pub(crate) struct Inner { 44 | db: RocksDatabase, 45 | cols: HashSet, 46 | } 47 | 48 | #[derive(Clone)] 49 | pub struct RocksDB { 50 | inner: Arc, 51 | } 52 | 53 | /// validate column name. a valid name should like "/blocks", start with "/" and only has one "/". 54 | /// column should not be empty 55 | #[inline] 56 | fn validate_column_name(col: &str, allow_default: bool) -> Result<()> { 57 | if col == DEFAULT_COLUMN_NAME { 58 | return if allow_default { 59 | Ok(()) 60 | } else { 61 | Err(RocksDBError::InvalidColumnName(format!( 62 | "not allow [{}] in this operation", 63 | DEFAULT_COLUMN_NAME 64 | ))) 65 | }; 66 | } 67 | let b = col.as_bytes(); 68 | // it means column name would not be empty, or just "/" 69 | if b.len() < 2 { 70 | return Err(RocksDBError::InvalidColumnName(format!( 71 | "column name should at least more then two chars, col:[{}]", 72 | col 73 | ))); 74 | } 75 | // should start with "/" 76 | if b[0] != key::LEFT_SLASH { 77 | return Err(RocksDBError::InvalidColumnName(format!( 78 | "column name should start with '/', col:[{}]", 79 | col 80 | ))); 81 | } 82 | // should not meet other "/", e.g. "/blocks/foo" 83 | if (&b[1..]).iter().any(|c| *c == key::LEFT_SLASH) { 84 | return Err(RocksDBError::InvalidColumnName(format!( 85 | "column name could only has one '/' char, col:[{}]", 86 | col 87 | ))); 88 | } 89 | Ok(()) 90 | } 91 | 92 | impl RocksDB { 93 | pub fn new(path: &str, config: &DatabaseConfig) -> Result { 94 | for col in config.columns.iter() { 95 | validate_column_name(col.as_str(), true)?; 96 | } 97 | 98 | let db = RocksDatabase::open(config, path)?; 99 | let columns = db.columns(); 100 | let inner = Inner { 101 | db, 102 | cols: HashSet::from_iter(columns.into_iter()), 103 | }; 104 | Ok(RocksDB { 105 | inner: Arc::from(inner), 106 | }) 107 | } 108 | 109 | pub fn new_with_default(path: &str) -> Result { 110 | Self::new(path, &Default::default()) 111 | } 112 | 113 | /// # Safety 114 | /// `add_column` should called before read/write database, please ensure don't call this 115 | /// function when other thread read/write data 116 | pub unsafe fn add_column(&self, col: &str) -> Result<()> { 117 | validate_column_name(col, false)?; 118 | 119 | if self.inner.cols.contains(col) { 120 | return Ok(()); 121 | } 122 | self.inner.db.add_column(col)?; 123 | let cols = &self.inner.cols as *const HashSet as *mut HashSet; 124 | let cols = &mut *cols; 125 | 126 | // dangerous!!! 127 | cols.insert(col.to_string()); 128 | Ok(()) 129 | } 130 | 131 | /// # Safety 132 | /// `remove_column` should called before read/write database, please ensure don't call this 133 | /// function when other thread read/write data 134 | pub unsafe fn remove_column(&self, col: &str) -> Result<()> { 135 | validate_column_name(col, false)?; 136 | 137 | if !self.inner.cols.contains(col) { 138 | return Ok(()); 139 | } 140 | self.inner.db.remove_column(col)?; 141 | let cols = &self.inner.cols as *const HashSet as *mut HashSet; 142 | let cols = &mut *cols; 143 | 144 | // dangerous!!! 145 | cols.remove(col); 146 | Ok(()) 147 | } 148 | } 149 | 150 | #[inline] 151 | fn inner_get(db: &RocksDB, key: &Key) -> DSResult> { 152 | let (prefix, key) = pre_process_key(&db.inner.cols, &key); 153 | let value = db.inner.db.get(prefix, key.as_bytes())?; 154 | value.ok_or(datastore::DSError::NotFound(key.to_string())) 155 | } 156 | 157 | impl Read for RocksDB { 158 | fn get(&self, key: &Key) -> DSResult> { 159 | inner_get(self, key).map(|v| v) 160 | } 161 | 162 | fn has(&self, key: &Key) -> DSResult { 163 | let (prefix, key) = pre_process_key(&self.inner.cols, &key); 164 | let value = self.inner.db.get(prefix, key.as_bytes())?; 165 | Ok(value.is_some()) 166 | } 167 | 168 | fn get_size(&self, key: &Key) -> DSResult { 169 | inner_get(self, key).map(|v| v.len()) 170 | } 171 | } 172 | 173 | impl SyncQuery for RocksDB { 174 | fn query(&self, _query: query::Query) -> DSResult { 175 | todo!() 176 | } 177 | } 178 | 179 | #[inline] 180 | fn inner_write(db: &RocksDB, key: &Key, f: F) -> DSResult<()> 181 | where 182 | F: Fn(&mut DBTransaction, &str, &str), 183 | { 184 | let (prefix, key) = pre_process_key(&db.inner.cols, &key); 185 | let mut tx = db.inner.db.transaction(); 186 | f(&mut tx, prefix, key); 187 | db.inner.db.write(tx)?; 188 | Ok(()) 189 | } 190 | 191 | impl Write for RocksDB { 192 | fn put(&self, key: Key, value: Vec) -> DSResult<()> { 193 | inner_write(self, &key, |tx, col, real_key| { 194 | tx.put(col, real_key.as_bytes(), &value); 195 | }) 196 | } 197 | 198 | fn delete(&self, key: &Key) -> DSResult<()> { 199 | inner_write(self, &key, |tx, col, real_key| { 200 | tx.delete(col, real_key.as_bytes()); 201 | }) 202 | } 203 | } 204 | 205 | impl Datastore for RocksDB { 206 | fn sync(&self, _prefix: &Key) -> DSResult<()> { 207 | self.inner.db.flush()?; 208 | Ok(()) 209 | } 210 | } 211 | 212 | impl Batching for RocksDB { 213 | type Txn = Transaction; 214 | 215 | fn batch(&self) -> DSResult { 216 | let inner = self.inner.db.transaction(); 217 | let cols = &self.inner.as_ref().cols as *const HashSet; 218 | // for cols is only readable, and lifetime is same as db, thus just pointer is enough 219 | // to avoid atomic copy for Arc casting 220 | let tx = Transaction::new(inner, cols); 221 | Ok(tx) 222 | } 223 | 224 | fn commit(&self, txn: Self::Txn) -> DSResult<()> { 225 | self.inner.db.write(txn.inner)?; 226 | Ok(()) 227 | } 228 | } 229 | 230 | impl TxnDatastore for RocksDB { 231 | fn new_transaction(&self, _read_only: bool) -> DSResult { 232 | let inner = self.inner.db.transaction(); 233 | let cols = &self.inner.as_ref().cols as *const HashSet; 234 | let tx = Transaction::new(inner, cols); 235 | Ok(tx) 236 | } 237 | } 238 | -------------------------------------------------------------------------------- /datastore/rocksdb/src/tx.rs: -------------------------------------------------------------------------------- 1 | use std::collections::HashSet; 2 | 3 | use datastore::{key::Key, Batch, DSError, Read, Txn}; 4 | use kvdb::{DBOp, DBTransaction}; 5 | 6 | use crate::{pre_process_key, DSResult}; 7 | 8 | pub struct Transaction { 9 | pub inner: DBTransaction, 10 | cols: *const HashSet, 11 | } 12 | 13 | impl Transaction { 14 | pub(crate) fn new(inner: DBTransaction, cols: *const HashSet) -> Self { 15 | Transaction { inner, cols } 16 | } 17 | fn inner_get(&self, k: &Key) -> DSResult<&[u8]> { 18 | for op in self.inner.ops.iter() { 19 | if let DBOp::Insert { col, key, value } = op { 20 | let (prefix, k) = pre_process_key(self.cols, k); 21 | // not fit col name 22 | if prefix != col.as_str() { 23 | continue; 24 | } 25 | if key.as_slice() == k.as_bytes() { 26 | return Ok(value); 27 | } 28 | } 29 | } 30 | Err(DSError::NotFound(k.to_string())) 31 | } 32 | } 33 | 34 | impl Read for Transaction { 35 | fn get(&self, key: &Key) -> DSResult> { 36 | self.inner_get(key).map(|b| b.to_vec()) 37 | } 38 | 39 | fn has(&self, key: &Key) -> DSResult { 40 | let r = self.inner_get(key); 41 | match r { 42 | Ok(_) => Ok(true), 43 | Err(DSError::NotFound(_)) => Ok(false), 44 | Err(e) => Err(e), 45 | } 46 | } 47 | 48 | fn get_size(&self, key: &Key) -> DSResult { 49 | self.inner_get(key).map(|b| b.len()) 50 | } 51 | } 52 | 53 | impl Batch for Transaction { 54 | fn put(&mut self, key: Key, value: Vec) -> DSResult<()> { 55 | let (prefix, k) = pre_process_key(self.cols, &key); 56 | self.inner.put(prefix, k.as_bytes(), &value); 57 | Ok(()) 58 | } 59 | 60 | fn delete(&mut self, key: &Key) -> DSResult<()> { 61 | let (prefix, k) = pre_process_key(self.cols, &key); 62 | self.inner.delete(prefix, k.as_bytes()); 63 | Ok(()) 64 | } 65 | } 66 | 67 | impl Txn for Transaction { 68 | fn discard(&mut self) { 69 | self.inner.ops.clear() 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /datastore/src/basic_ds.rs: -------------------------------------------------------------------------------- 1 | use std::collections::HashMap; 2 | 3 | use crate::singleton::SingletonDS; 4 | 5 | use crate::datastore::{Read, Write}; 6 | use crate::error::*; 7 | use crate::key::Key; 8 | use crate::{Batch, Batching, Datastore, Txn, TxnDatastore}; 9 | use std::ops::{Deref, DerefMut}; 10 | 11 | #[derive(Debug, Default)] 12 | pub struct InnerDB(HashMap>); 13 | 14 | pub type MapDatastore = SingletonDS; 15 | pub type BasicTxn = HashMap>>; 16 | 17 | impl Deref for InnerDB { 18 | type Target = HashMap>; 19 | 20 | fn deref(&self) -> &Self::Target { 21 | &(self.0) 22 | } 23 | } 24 | 25 | impl DerefMut for InnerDB { 26 | fn deref_mut(&mut self) -> &mut Self::Target { 27 | &mut self.0 28 | } 29 | } 30 | 31 | impl InnerDB { 32 | // do not need `RefCell` for in single thread, outside would not hold any ref for `HashMap` 33 | pub fn be_mutable(&self) -> &mut InnerDB { 34 | unsafe { 35 | let db = self as *const InnerDB as *mut InnerDB; 36 | &mut *db 37 | } 38 | } 39 | } 40 | 41 | impl Write for InnerDB { 42 | fn put(&self, key: Key, value: Vec) -> Result<()> { 43 | self.be_mutable().insert(key, value); 44 | Ok(()) 45 | } 46 | 47 | fn delete(&self, key: &Key) -> Result<()> { 48 | self.be_mutable().remove(key); 49 | Ok(()) 50 | } 51 | } 52 | 53 | impl Read for InnerDB { 54 | fn get(&self, key: &Key) -> Result> { 55 | (&**self) 56 | .get(key) 57 | .map(|v| v.to_owned()) 58 | .ok_or(DSError::NotFound(key.to_string())) 59 | } 60 | 61 | fn has(&self, key: &Key) -> Result { 62 | Ok(self.contains_key(key)) 63 | } 64 | 65 | fn get_size(&self, key: &Key) -> Result { 66 | (&**self) 67 | .get(key) 68 | .map(|v| v.len()) 69 | .ok_or(DSError::NotFound(key.to_string())) 70 | } 71 | } 72 | 73 | impl Datastore for InnerDB { 74 | fn sync(&self, _prefix: &Key) -> Result<()> { 75 | // do nothing 76 | Ok(()) 77 | } 78 | } 79 | 80 | impl Batching for InnerDB { 81 | type Txn = BasicTxn; 82 | 83 | fn batch(&self) -> Result { 84 | Ok(BasicTxn::default()) 85 | } 86 | 87 | fn commit(&self, txn: Self::Txn) -> Result<()> { 88 | for (k, v) in txn { 89 | match v { 90 | Some(d) => self.put(k, d)?, 91 | None => self.delete(&k)?, 92 | } 93 | } 94 | Ok(()) 95 | } 96 | } 97 | 98 | impl TxnDatastore for InnerDB { 99 | fn new_transaction(&self, _read_only: bool) -> Result { 100 | self.batch() 101 | } 102 | } 103 | 104 | impl Batch for BasicTxn { 105 | fn put(&mut self, key: Key, value: Vec) -> Result<()> { 106 | self.insert(key, Some(value)); 107 | Ok(()) 108 | } 109 | 110 | fn delete(&mut self, key: &Key) -> Result<()> { 111 | self.insert(key.clone(), None); 112 | Ok(()) 113 | } 114 | } 115 | 116 | impl Read for BasicTxn { 117 | fn get(&self, key: &Key) -> Result> { 118 | self.get(key) 119 | .ok_or(DSError::NotFound(key.to_string())) 120 | .and_then(|v| { 121 | v.as_ref() 122 | .cloned() 123 | .ok_or(DSError::NotFound(key.to_string())) 124 | }) 125 | } 126 | 127 | fn has(&self, key: &Key) -> Result { 128 | Ok(self.get(key).map(|v| v.is_some()).is_some()) 129 | } 130 | 131 | fn get_size(&self, key: &Key) -> Result { 132 | self.get(key) 133 | .ok_or(DSError::NotFound(key.to_string())) 134 | .and_then(|v| { 135 | v.as_ref() 136 | .map(|v| v.len()) 137 | .ok_or(DSError::NotFound(key.to_string())) 138 | }) 139 | } 140 | } 141 | 142 | impl Txn for BasicTxn { 143 | fn discard(&mut self) { 144 | self.clear(); 145 | } 146 | } 147 | 148 | pub fn new_map_datastore() -> MapDatastore { 149 | Default::default() 150 | } 151 | -------------------------------------------------------------------------------- /datastore/src/datastore.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2020 PolkaX. Licensed under MIT or Apache-2.0. 2 | 3 | use std::time::Duration; 4 | 5 | use crate::error::*; 6 | use crate::key::Key; 7 | use crate::query::{self, SyncResults}; 8 | 9 | /// Write is the write-side of the Datastore interface. 10 | pub trait Write { 11 | fn put(&self, key: Key, value: Vec) -> Result<()>; 12 | fn delete(&self, key: &Key) -> Result<()>; 13 | } 14 | 15 | /// Read is the read-side of the Datastore interface. 16 | pub trait Read { 17 | fn get(&self, key: &Key) -> Result>; 18 | fn has(&self, key: &Key) -> Result; 19 | fn get_size(&self, key: &Key) -> Result; 20 | } 21 | 22 | pub trait SyncQuery { 23 | fn query(&self, query: query::Query) -> Result; 24 | } 25 | 26 | #[cfg(feature = "async")] 27 | pub trait AsyncQuery { 28 | fn query(&self, query: query::Query) -> Result; 29 | } 30 | 31 | pub trait Datastore: Write + Read + Send + 'static { 32 | /// Sync guarantees that any Put or Delete calls under prefix that returned 33 | /// before Sync(prefix) was called will be observed after Sync(prefix) 34 | /// returns, even if the program crashes. If Put/Delete operations already 35 | /// satisfy these requirements then Sync may be a no-op. 36 | /// If the prefix fails to Sync this method returns an error. 37 | fn sync(&self, prefix: &Key) -> Result<()>; 38 | } 39 | 40 | pub trait CloneableDatastore: Datastore + Clone {} 41 | 42 | impl CloneableDatastore for T {} 43 | 44 | // TTLDatastore is an interface that should be implemented by datastores that 45 | // support expiring entries. 46 | pub trait TTLDatastore: Datastore + TTL {} 47 | 48 | impl TTLDatastore for T {} 49 | 50 | // TTL encapulates the methods that deal with entries with time-to-live. 51 | pub trait TTL { 52 | fn put_with_ttl(&self, key: Key, value: Vec, ttl: Duration) -> Result<()>; 53 | fn set_ttl(&self, key: Key, ttl: Duration) -> Result<()>; 54 | fn get_expiration(&self, key: &Key) -> Result; 55 | } 56 | 57 | pub trait CheckedDatastore: Datastore { 58 | fn check(&self) -> Result<()>; 59 | } 60 | 61 | pub trait Batch { 62 | fn put(&mut self, key: Key, value: Vec) -> Result<()>; 63 | fn delete(&mut self, key: &Key) -> Result<()>; 64 | } 65 | 66 | pub trait Batching: Datastore { 67 | type Txn: Batch; 68 | fn batch(&self) -> Result; 69 | fn commit(&self, txn: Self::Txn) -> Result<()>; 70 | } 71 | 72 | pub trait CloneableBatching: Batching + Clone {} 73 | 74 | impl CloneableBatching for T {} 75 | 76 | pub trait Txn: Read + Batch { 77 | fn discard(&mut self); 78 | } 79 | 80 | pub trait TxnDatastore: Batching 81 | where 82 | Self::Txn: Txn, 83 | { 84 | fn new_transaction(&self, read_only: bool) -> Result; 85 | } 86 | 87 | pub trait ScrubbedDatastore: Datastore { 88 | fn scrub(&self) -> Result<()>; 89 | } 90 | 91 | pub trait GCDatastore: Datastore { 92 | fn collect_garbage(&self) -> Result<()>; 93 | } 94 | 95 | pub trait PersistentDatastore: Datastore { 96 | fn disk_usage(&self) -> Result; 97 | } 98 | -------------------------------------------------------------------------------- /datastore/src/error.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2020 PolkaX. Licensed under MIT or Apache-2.0. 2 | 3 | use thiserror::Error; 4 | 5 | use std::io; 6 | 7 | pub type Result = std::result::Result; 8 | 9 | #[derive(Error, Debug)] 10 | pub enum DSError { 11 | #[error("not found for key: {0}")] 12 | NotFound(String), 13 | 14 | #[error("db io error: {0}")] 15 | DBIoErr(#[from] io::Error), 16 | 17 | #[error("other err: {0}")] 18 | Other(#[from] Box), 19 | } 20 | -------------------------------------------------------------------------------- /datastore/src/keytransform/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2020 PolkaX. Licensed under MIT or Apache-2.0. 2 | 3 | mod transforms; 4 | 5 | use std::ops::{Deref, DerefMut}; 6 | 7 | use crate::datastore::{Batch, Batching, Datastore as DatastoreT, Read, Write}; 8 | use crate::error::*; 9 | use crate::key::Key; 10 | 11 | use crate::Txn; 12 | pub use transforms::{KeyTransform, PrefixTransform}; 13 | 14 | pub fn wrap(child: D, key_transform: K) -> Datastore { 15 | Datastore { 16 | child, 17 | key_transform, 18 | } 19 | } 20 | 21 | pub struct Datastore { 22 | child: D, 23 | key_transform: K, 24 | } 25 | 26 | impl Deref for Datastore { 27 | type Target = D; 28 | 29 | fn deref(&self) -> &Self::Target { 30 | &self.child 31 | } 32 | } 33 | 34 | impl DerefMut for Datastore { 35 | fn deref_mut(&mut self) -> &mut Self::Target { 36 | &mut self.child 37 | } 38 | } 39 | 40 | impl Write for Datastore { 41 | fn put(&self, key: Key, value: Vec) -> Result<()> { 42 | self.child.put(self.key_transform.convert_key(key), value) 43 | } 44 | 45 | fn delete(&self, key: &Key) -> Result<()> { 46 | self.child.delete(&self.key_transform.convert_key(key)) 47 | } 48 | } 49 | 50 | impl Read for Datastore { 51 | fn get(&self, key: &Key) -> Result> { 52 | self.child.get(&self.key_transform.convert_key(key)) 53 | } 54 | 55 | fn has(&self, key: &Key) -> Result { 56 | self.child.has(&self.key_transform.convert_key(key)) 57 | } 58 | 59 | fn get_size(&self, key: &Key) -> Result { 60 | self.child.get_size(&self.key_transform.convert_key(key)) 61 | } 62 | } 63 | // TODO query 64 | 65 | impl DatastoreT for Datastore { 66 | fn sync(&self, prefix: &Key) -> Result<()> { 67 | self.child.sync(&self.key_transform.convert_key(prefix)) 68 | } 69 | } 70 | 71 | impl Batching for Datastore { 72 | type Txn = TransformBatch; 73 | fn batch(&self) -> Result { 74 | let child_batch = self.child.batch()?; 75 | Ok(TransformBatch { 76 | child_batch, 77 | transform: self.key_transform.clone(), 78 | }) 79 | } 80 | 81 | fn commit(&self, txn: Self::Txn) -> Result<()> { 82 | self.child.commit(txn.child_batch) 83 | } 84 | } 85 | 86 | pub struct TransformBatch { 87 | child_batch: B, 88 | transform: K, 89 | } 90 | 91 | impl Batch for TransformBatch { 92 | fn put(&mut self, key: Key, value: Vec) -> Result<()> { 93 | self.child_batch.put(self.transform.convert_key(key), value) 94 | } 95 | 96 | fn delete(&mut self, key: &Key) -> Result<()> { 97 | self.child_batch.delete(&self.transform.convert_key(key)) 98 | } 99 | } 100 | 101 | impl Read for TransformBatch { 102 | fn get(&self, key: &Key) -> Result> { 103 | self.child_batch.get(key) 104 | } 105 | 106 | fn has(&self, key: &Key) -> Result { 107 | self.child_batch.has(key) 108 | } 109 | 110 | fn get_size(&self, key: &Key) -> Result { 111 | self.child_batch.get_size(key) 112 | } 113 | } 114 | impl Txn for TransformBatch { 115 | fn discard(&mut self) { 116 | self.child_batch.discard() 117 | } 118 | } 119 | -------------------------------------------------------------------------------- /datastore/src/keytransform/transforms.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2020 PolkaX. Licensed under MIT or Apache-2.0. 2 | 3 | use crate::key::{Key, LEFT_SLASH_STR}; 4 | 5 | pub trait KeyTransform: Clone + Send + 'static { 6 | fn convert_key + Into>(&self, k: K) -> Key; 7 | fn invert_key + Into>(&self, k: K) -> Key; 8 | } 9 | 10 | /// Pair is a convince struct for constructing a key transform. 11 | #[derive(Clone)] 12 | pub struct Pair 13 | where 14 | A: Fn(&Key) -> Key + Clone + Send + 'static, 15 | B: Fn(&Key) -> Key + Clone + Send + 'static, 16 | { 17 | pub convert: A, 18 | pub invert: B, 19 | } 20 | 21 | impl KeyTransform for Pair 22 | where 23 | A: Fn(&Key) -> Key + Clone + Send + 'static, 24 | B: Fn(&Key) -> Key + Clone + Send + 'static, 25 | { 26 | fn convert_key + Into>(&self, k: K) -> Key { 27 | (self.convert)(k.as_ref()) 28 | } 29 | 30 | fn invert_key + Into>(&self, k: K) -> Key { 31 | (self.invert)(k.as_ref()) 32 | } 33 | } 34 | 35 | /// PrefixTransform constructs a KeyTransform with a pair of functions that 36 | /// add or remove the given prefix key. 37 | /// 38 | /// Warning: will panic if prefix not found when it should be there. This is 39 | /// to avoid insidious data inconsistency errors. 40 | #[derive(Clone)] 41 | pub struct PrefixTransform { 42 | pub prefix: Key, 43 | } 44 | 45 | impl KeyTransform for PrefixTransform { 46 | /// ConvertKey adds the prefix. 47 | fn convert_key + Into>(&self, k: K) -> Key { 48 | self.prefix.child(k) 49 | } 50 | 51 | /// InvertKey removes the prefix. panics if prefix not found. 52 | fn invert_key + Into>(&self, k: K) -> Key { 53 | if self.prefix.as_str() == LEFT_SLASH_STR { 54 | return k.into(); 55 | } 56 | if !self.prefix.is_ancestor_of(k.as_ref().as_str()) { 57 | panic!("expected prefix not found") 58 | } 59 | let (_, second) = k.as_ref().split_prefix(); 60 | Key::from_raw(second) 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /datastore/src/lib.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2020 PolkaX. Licensed under MIT or Apache-2.0. 2 | 3 | #![allow(clippy::or_fun_call, clippy::mut_from_ref)] 4 | 5 | mod datastore; 6 | mod error; 7 | #[cfg(test)] 8 | mod tests; 9 | 10 | pub mod basic_ds; 11 | pub mod key; 12 | pub mod keytransform; 13 | pub mod namespace; 14 | pub mod query; 15 | pub mod singleton; 16 | // TODO impl mount 17 | // pub mod mount; 18 | 19 | pub use self::datastore::*; 20 | pub use self::error::DSError; 21 | -------------------------------------------------------------------------------- /datastore/src/mount/async_results.rs: -------------------------------------------------------------------------------- 1 | // TODO 2 | -------------------------------------------------------------------------------- /datastore/src/mount/mod.rs: -------------------------------------------------------------------------------- 1 | //! mount provides a Datastore that has other Datastores 2 | //! mounted at various key prefixes and is threadsafe 3 | 4 | mod async_results; 5 | mod sync_results; 6 | 7 | use std::cmp::Ordering; 8 | 9 | use crate::datastore::{Datastore as DatastoreT, Read, Write}; 10 | use crate::error::DSError; 11 | use crate::key::Key; 12 | use crate::query::{self, QResult, Query}; 13 | 14 | pub struct Mount { 15 | pub prefix: Key, 16 | pub datastore: D, 17 | } 18 | 19 | pub struct Datastore { 20 | pub mounts: Vec>, 21 | } 22 | 23 | impl Datastore { 24 | pub fn new(mounts: Vec>) -> Datastore { 25 | let mut mounts = mounts; 26 | mounts.sort_by(|a, b| a.prefix.cmp(&b.prefix).reverse()); 27 | Datastore:: { mounts } 28 | } 29 | 30 | pub fn lookup(&self, key: &Key) -> Option<(D, Key, Key)> { 31 | for m in self.mounts.iter() { 32 | if &m.prefix == key || m.prefix.is_ancestor_of(key) { 33 | // trim prefix 34 | let s = &key.as_bytes()[..m.prefix.as_bytes().len()]; 35 | let s = unsafe { std::str::from_utf8_unchecked(s) }; 36 | let k = Key::new(s); 37 | // TODO 38 | // return Some(m.datastor, m.prefix.clone(), k) 39 | } 40 | } 41 | None 42 | } 43 | } 44 | // TODO 45 | /* 46 | struct QueryResults { 47 | mount: Key, 48 | results: Box, 49 | next: QResult, 50 | } 51 | 52 | fn advance(mount: &Key, results: &mut Box) -> Option { 53 | let mut r = results.next_sync(); 54 | match r { 55 | None => { 56 | // TODO set results? 57 | None 58 | } 59 | Some(mut query_result) => { 60 | if let Ok(ref mut entry) = query_result { 61 | // add mount prefix to entry.key 62 | let s: String = mount.child(Key::from_raw(entry.key.clone())).into(); 63 | entry.key = s; 64 | } 65 | Some(query_result) 66 | } 67 | } 68 | } 69 | impl QueryResults { 70 | fn new_with_advance(mount: Key, results: impl AsyncResults + 'static) -> Option { 71 | let mut results: Box = Box::new(results); 72 | advance(&mount, &mut results).map(|next| QueryResults { 73 | mount, 74 | results, 75 | next, 76 | }) 77 | } 78 | } 79 | 80 | struct QuerySet { 81 | query: Query, 82 | heads: Vec, 83 | } 84 | 85 | impl QuerySet { 86 | fn len(&self) -> usize { 87 | self.heads.len() 88 | } 89 | fn less(&self, i: usize, j: usize) -> bool { 90 | let i = self.heads[i].next.as_ref().expect(""); 91 | let j = self.heads[j].next.as_ref().expect(""); 92 | query::order::less(&self.query.orders, i, j) == Ordering::Less 93 | } 94 | 95 | fn swap(&mut self, i: usize, j: usize) { 96 | if i >= self.heads.len() || j >= self.heads.len() { 97 | return; 98 | } 99 | self.heads.swap(i, j); 100 | } 101 | fn push(&mut self, x: QueryResults) { 102 | self.heads.push(x); 103 | } 104 | fn pop(&mut self) -> Option { 105 | self.heads.pop() 106 | } 107 | fn add_results(&mut self, mount: Key, results: impl AsyncResults + 'static) { 108 | if let Some(r) = QueryResults::new_with_advance(mount, results) { 109 | self.push(r); 110 | } 111 | } 112 | 113 | fn fix(&mut self, i: usize) { 114 | if !self.down(i, self.len()) { 115 | self.up(i); 116 | } 117 | } 118 | fn remove(&mut self, i: usize) -> Option { 119 | if self.len() == 0 { 120 | return None; 121 | } 122 | let n = self.len() - 1; 123 | if n != i { 124 | self.swap(i, n); 125 | if !self.down(i, n) { 126 | self.up(i) 127 | } 128 | } 129 | self.pop() 130 | } 131 | 132 | fn down(&mut self, i0: usize, n: usize) -> bool { 133 | let mut i = i0; 134 | loop { 135 | let j1 = 2 * i + 1; 136 | if j1 >= n || j1 < 0 { 137 | // j1 < 0 after int overflow 138 | break; 139 | } 140 | let mut j = j1; // left child 141 | let j2 = j1 + 1; 142 | if j2 < n && self.less(j2, j1) { 143 | j = j2; // = 2*i + 2 // right child 144 | } 145 | if !self.less(j, i) { 146 | break; 147 | } 148 | self.swap(i, j); 149 | i = j; 150 | } 151 | i > i0 152 | } 153 | fn up(&mut self, j: usize) { 154 | let mut j = j; 155 | loop { 156 | let i = (j - 1) / 2; // parent 157 | if i == j || !self.less(j, i) { 158 | break; 159 | } 160 | self.swap(i, j); 161 | j = i 162 | } 163 | } 164 | } 165 | 166 | impl Iterator for QuerySet { 167 | type Item = QResult; 168 | 169 | fn next(&mut self) -> Option { 170 | if self.heads.is_empty() { 171 | return None; 172 | } 173 | let head = &mut self.heads[0]; 174 | let mut maybe = advance(&head.mount, &mut head.results); 175 | if let Some(mut r) = maybe { 176 | // use new advance next to replace old next, and return old, store new in `self.heads[0]` 177 | std::mem::swap(&mut r, &mut head.next); 178 | self.fix(0); 179 | Some(r) 180 | } else { 181 | self.remove(0).map(|r| r.next) 182 | } 183 | } 184 | } 185 | */ 186 | /* 187 | impl Read for Datastore { 188 | fn get(&self, key: &Key) -> Result< Vec, Error> { 189 | unimplemented!() 190 | } 191 | 192 | fn has(&self, key: &Key) -> Result { 193 | unimplemented!() 194 | } 195 | 196 | fn get_size(&self, key: &Key) -> Result { 197 | unimplemented!() 198 | } 199 | } 200 | impl Write for Datastore { 201 | fn put(&self, key: Key, value: Vec) -> Result<(), Error> { 202 | unimplemented!() 203 | } 204 | 205 | fn delete(&self, key: &Key) -> Result<(), Error> { 206 | unimplemented!() 207 | } 208 | } 209 | */ 210 | -------------------------------------------------------------------------------- /datastore/src/mount/sync_results.rs: -------------------------------------------------------------------------------- 1 | // TODO 2 | -------------------------------------------------------------------------------- /datastore/src/namespace/mod.rs: -------------------------------------------------------------------------------- 1 | //! namespace introduces a namespace Datastore Shim, which basically 2 | //! mounts the entire child datastore under a prefix. 3 | //! Use the Wrap function to wrap a datastore with any Key prefix. 4 | //! # For example: 5 | //! 6 | //! ```norun 7 | //! let db = /*...*/; 8 | //! let mut ns = wrap(db.clone(), Key("/foo/bar")); 9 | //! ns.put(Key("/beep"), "boop"); // now it's /foo/bar/boop 10 | //! let v2 = ns.get("beep").unwrap(); 11 | //! asset_eq!(&v2, "boop"); 12 | //! 13 | //! // and, in the underlying database 14 | //! v3 = db.get("/foo/bar/beep").unwrap(); 15 | //! asset_eq!(&v3, "boop"); 16 | //! ``` 17 | use crate::datastore::Datastore as DatastoreT; 18 | use crate::key::Key; 19 | use crate::keytransform; 20 | // re-export 21 | pub use crate::keytransform::{Datastore, PrefixTransform}; 22 | 23 | #[inline] 24 | pub fn prefix_transform(prefix: Key) -> PrefixTransform { 25 | PrefixTransform { prefix } 26 | } 27 | 28 | pub fn wrap(child: D, prefix: Key) -> Datastore { 29 | keytransform::wrap(child, prefix_transform(prefix)) 30 | } 31 | 32 | pub type NSDatastore = Datastore; 33 | -------------------------------------------------------------------------------- /datastore/src/query/async_results.rs: -------------------------------------------------------------------------------- 1 | use async_std::sync::{channel, Receiver, Sender}; 2 | use async_std::task; 3 | use async_trait::async_trait; 4 | 5 | use super::{Entry, QResult, Query}; 6 | use crate::error::*; 7 | 8 | #[async_trait] 9 | pub trait AsyncResults { 10 | fn query(&self) -> &Query; 11 | async fn next(&self) -> Option; 12 | fn next_sync(&self) -> Option; 13 | fn rest(&self) -> Result>; 14 | } 15 | 16 | pub struct AsyncResult { 17 | query: Query, 18 | res: Receiver, 19 | } 20 | 21 | #[async_trait] 22 | impl AsyncResults for AsyncResult { 23 | fn query(&self) -> &Query { 24 | &self.query 25 | } 26 | 27 | async fn next(&self) -> Option> { 28 | self.res.recv().await 29 | } 30 | 31 | fn next_sync(&self) -> Option> { 32 | task::block_on(self.next()) 33 | } 34 | 35 | fn rest(&self) -> Result> { 36 | let mut es = vec![]; 37 | while let Some(r) = self.next_sync() { 38 | let e = r?; 39 | es.push(e); 40 | } 41 | Ok(es) 42 | } 43 | } 44 | 45 | pub type ResultChannel = (Sender>, Receiver); 46 | 47 | pub struct AsyncResultBuilder { 48 | query: Query, 49 | output: ResultChannel, 50 | } 51 | 52 | const NORMAL_BUF_SIZE: usize = 1; 53 | impl AsyncResultBuilder { 54 | pub fn new(q: Query) -> Self { 55 | AsyncResultBuilder { 56 | query: q, 57 | output: channel(NORMAL_BUF_SIZE), 58 | } 59 | } 60 | pub fn results(self) -> AsyncResult { 61 | AsyncResult { 62 | query: self.query, 63 | res: self.output.1, 64 | } 65 | } 66 | } 67 | 68 | // TODO need more info to complete the rest 69 | -------------------------------------------------------------------------------- /datastore/src/query/filter.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2020 PolkaX. Licensed under MIT or Apache-2.0. 2 | 3 | use std::cmp::Ordering; 4 | use std::fmt; 5 | 6 | use super::Entry; 7 | 8 | pub trait Filter: fmt::Debug + Sync + Send { 9 | fn filter(&self, e: &Entry) -> bool; 10 | } 11 | 12 | pub type Op = &'static str; 13 | 14 | pub const EQUAL: Op = "=="; 15 | pub const NOT_EQUAL: Op = "!="; 16 | pub const GREATER_THAN: Op = ">"; 17 | pub const GREATER_THAN_OR_EQUAL: Op = ">="; 18 | pub const LESS_THAN: Op = "<"; 19 | pub const LESS_THAN_OR_EQUAL: Op = "<="; 20 | 21 | pub struct FilterValueCompare<'a> { 22 | op: Op, 23 | value: &'a [u8], 24 | } 25 | impl<'a> fmt::Debug for FilterValueCompare<'a> { 26 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 27 | write!( 28 | f, 29 | "VALUE {} {}", 30 | self.op, 31 | String::from_utf8_lossy(self.value) 32 | ) 33 | } 34 | } 35 | 36 | impl<'a> Filter for FilterValueCompare<'a> { 37 | fn filter(&self, e: &Entry) -> bool { 38 | let ordering = e.value.as_slice().cmp(self.value); 39 | match self.op { 40 | EQUAL => ordering == Ordering::Equal, 41 | NOT_EQUAL => ordering != Ordering::Equal, 42 | LESS_THAN => ordering < Ordering::Equal, 43 | LESS_THAN_OR_EQUAL => ordering <= Ordering::Equal, 44 | GREATER_THAN => ordering > Ordering::Equal, 45 | GREATER_THAN_OR_EQUAL => ordering >= Ordering::Equal, 46 | _ => unreachable!(format!("unknown operation: {}", self.op)), 47 | } 48 | } 49 | } 50 | 51 | pub struct FilterKeyCompare<'a> { 52 | op: Op, 53 | key: &'a str, 54 | } 55 | 56 | impl<'a> fmt::Debug for FilterKeyCompare<'a> { 57 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 58 | write!(f, "KEY {} {}", self.op, self.key) 59 | } 60 | } 61 | 62 | impl<'a> Filter for FilterKeyCompare<'a> { 63 | fn filter(&self, e: &Entry) -> bool { 64 | let ordering = e.key.as_str().cmp(self.key); 65 | match self.op { 66 | EQUAL => ordering == Ordering::Equal, 67 | NOT_EQUAL => ordering != Ordering::Equal, 68 | LESS_THAN => ordering < Ordering::Equal, 69 | LESS_THAN_OR_EQUAL => ordering <= Ordering::Equal, 70 | GREATER_THAN => ordering > Ordering::Equal, 71 | GREATER_THAN_OR_EQUAL => ordering >= Ordering::Equal, 72 | _ => unreachable!(format!("unknown operation: {}", self.op)), 73 | } 74 | } 75 | } 76 | 77 | pub struct FilterKeyPrefix<'a> { 78 | prefix: &'a str, 79 | } 80 | 81 | impl<'a> Filter for FilterKeyPrefix<'a> { 82 | fn filter(&self, e: &Entry) -> bool { 83 | e.key.starts_with(self.prefix) 84 | } 85 | } 86 | 87 | impl<'a> fmt::Debug for FilterKeyPrefix<'a> { 88 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 89 | write!(f, "PREFIX({})", self.prefix) 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /datastore/src/query/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2020 PolkaX. Licensed under MIT or Apache-2.0. 2 | 3 | #[cfg(feature = "async")] 4 | mod async_results; 5 | mod sync_results; 6 | 7 | pub mod filter; 8 | pub mod order; 9 | 10 | use std::fmt; 11 | 12 | use crate::error::*; 13 | use filter::Filter; 14 | use order::Order; 15 | 16 | // re-export 17 | #[cfg(feature = "async")] 18 | pub use async_results::{AsyncResult, AsyncResultBuilder, AsyncResults}; 19 | pub use sync_results::{SyncResult, SyncResults}; 20 | 21 | #[derive(Default, Clone)] 22 | pub struct Entry { 23 | pub key: String, 24 | pub value: Vec, 25 | pub size: usize, 26 | // expiration 27 | } 28 | 29 | pub struct Query { 30 | /// namespaces the query to results whose keys have Prefix 31 | pub prefix: String, 32 | /// filter results. apply sequentially 33 | pub filters: Vec>, 34 | /// order results. apply hierarchically 35 | pub orders: Vec>, 36 | /// maximum number of results 37 | pub limit: usize, 38 | /// skip given number of results 39 | pub offset: usize, 40 | /// return only keys. 41 | pub keys_only: bool, 42 | /// always return sizes. If not set, datastore impl can return 43 | /// it anyway if it doesn't involve a performance cost. If KeysOnly 44 | /// is not set, Size should always be set. 45 | pub returns_sizes: bool, 46 | // return expirations (see TTLDatastore) 47 | // expiration, 48 | } 49 | 50 | impl fmt::Debug for Query { 51 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 52 | write!(f, "SELECT keys")?; 53 | if !self.keys_only { 54 | write!(f, ",vals")?; 55 | } 56 | 57 | // if !self.expiration { 58 | // write!(f, ",exps") 59 | // } 60 | write!(f, " ")?; 61 | if !self.prefix.is_empty() { 62 | write!(f, "FROM {} ", self.prefix)?; 63 | } 64 | 65 | if !self.filters.is_empty() { 66 | write!(f, "FILTER [{:?}", self.filters[0])?; 67 | for filter in &self.filters[1..] { 68 | write!(f, ", {:?}", filter)?; 69 | } 70 | write!(f, "] ")?; 71 | } 72 | if !self.orders.is_empty() { 73 | write!(f, "ORDER [{:?}", self.orders[0])?; 74 | for order in &self.orders[1..] { 75 | write!(f, ", {:?}", order)?; 76 | } 77 | write!(f, "] ")?; 78 | } 79 | 80 | if self.offset > 0 { 81 | write!(f, "OFFSET {}", self.offset)?; 82 | } 83 | if self.limit > 0 { 84 | write!(f, "LIMIT {}", self.limit)?; 85 | } 86 | Ok(()) 87 | } 88 | } 89 | 90 | // TODO use stream? 91 | // Results is a set of Query results. This is the interface for clients. 92 | // Example: 93 | // 94 | // qr, _ := myds.Query(q) 95 | // for r := range qr.Next() { 96 | // if r.Error != nil { 97 | // // handle. 98 | // break 99 | // } 100 | // 101 | // fmt.Println(r.Entry.Key, r.Entry.Value) 102 | // } 103 | // 104 | // or, wait on all results at once: 105 | // 106 | // qr, _ := myds.Query(q) 107 | // es, _ := qr.Rest() 108 | // for _, e := range es { 109 | // fmt.Println(e.Key, e.Value) 110 | // } 111 | // 112 | 113 | pub type QResult = Result; 114 | -------------------------------------------------------------------------------- /datastore/src/query/order.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2020 PolkaX. Licensed under MIT or Apache-2.0. 2 | 3 | use std::cmp::Ordering; 4 | use std::fmt; 5 | 6 | use super::Entry; 7 | use std::ops::DerefMut; 8 | 9 | pub trait Order: fmt::Debug + Sync + Send { 10 | fn cmp(&self, a: &Entry, b: &Entry) -> Ordering; 11 | } 12 | 13 | impl Order for F 14 | where 15 | F: Fn(&Entry, &Entry) -> Ordering + fmt::Debug + Sync + Send, 16 | { 17 | fn cmp(&self, a: &Entry, b: &Entry) -> Ordering { 18 | self(a, b) 19 | } 20 | } 21 | 22 | /// OrderByValue is used to signal to datastores they should apply internal 23 | /// orderings. 24 | pub struct OrderByValue; 25 | 26 | impl Order for OrderByValue { 27 | fn cmp(&self, a: &Entry, b: &Entry) -> Ordering { 28 | a.value.cmp(&b.value) 29 | } 30 | } 31 | 32 | impl fmt::Debug for OrderByValue { 33 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 34 | write!(f, "VALUE") 35 | } 36 | } 37 | 38 | /// OrderByValueDescending is used to signal to datastores they 39 | /// should apply internal orderings. 40 | pub struct OrderByValueDescending; 41 | 42 | impl Order for OrderByValueDescending { 43 | fn cmp(&self, a: &Entry, b: &Entry) -> Ordering { 44 | a.value.cmp(&b.value).reverse() 45 | } 46 | } 47 | 48 | impl fmt::Debug for OrderByValueDescending { 49 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 50 | write!(f, "desc(VALUE)") 51 | } 52 | } 53 | 54 | /// OrderByKey 55 | pub struct OrderByKey; 56 | 57 | impl Order for OrderByKey { 58 | fn cmp(&self, a: &Entry, b: &Entry) -> Ordering { 59 | a.key.cmp(&b.key) 60 | } 61 | } 62 | 63 | impl fmt::Debug for OrderByKey { 64 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 65 | write!(f, "KEY") 66 | } 67 | } 68 | 69 | /// OrderByKeyDescending 70 | pub struct OrderByKeyDescending; 71 | 72 | impl Order for OrderByKeyDescending { 73 | fn cmp(&self, a: &Entry, b: &Entry) -> Ordering { 74 | a.key.cmp(&b.key).reverse() 75 | } 76 | } 77 | 78 | impl fmt::Debug for OrderByKeyDescending { 79 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 80 | write!(f, "KEY") 81 | } 82 | } 83 | 84 | pub fn less(orders: &[Box], a: &Entry, b: &Entry) -> Ordering { 85 | for cmp in orders.iter() { 86 | match cmp.cmp(a, b) { 87 | Ordering::Equal => {} 88 | Ordering::Less => return Ordering::Less, 89 | Ordering::Greater => return Ordering::Greater, 90 | } 91 | } 92 | // This gives us a *stable* sort for free. We don't care 93 | // preserving the order from the underlying datastore 94 | // because it's undefined. 95 | a.key.cmp(&b.key) 96 | } 97 | 98 | /// Sort sorts the given entries using the given orders. 99 | pub fn sort, L: DerefMut>(orders: &[Box], mut entries: L) { 100 | entries 101 | .deref_mut() 102 | .sort_by(|a, b| less(orders, a.as_ref(), b.as_ref())); 103 | } 104 | -------------------------------------------------------------------------------- /datastore/src/query/sync_results.rs: -------------------------------------------------------------------------------- 1 | use super::{Entry, Query}; 2 | 3 | pub trait SyncResults { 4 | fn query(&self) -> &Query; 5 | fn rest(self) -> Vec; 6 | } 7 | 8 | pub struct SyncResult { 9 | query: Query, 10 | res: Vec, 11 | } 12 | 13 | impl SyncResults for SyncResult { 14 | fn query(&self) -> &Query { 15 | &self.query 16 | } 17 | 18 | fn rest(self) -> Vec { 19 | self.res 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /datastore/src/singleton.rs: -------------------------------------------------------------------------------- 1 | use parking_lot::{RwLock, RwLockReadGuard, RwLockWriteGuard}; 2 | use std::sync::Arc; 3 | 4 | use crate::error::*; 5 | use crate::key::Key; 6 | use crate::{Batching, Datastore, Read, Txn, TxnDatastore, Write}; 7 | 8 | #[derive(Clone, Default, Debug)] 9 | pub struct SingletonDS { 10 | inner: Arc>, 11 | } 12 | impl SingletonDS {} 13 | 14 | impl SingletonDS { 15 | pub fn new(db: T) -> Self { 16 | SingletonDS:: { 17 | inner: Arc::new(RwLock::new(db)), 18 | } 19 | } 20 | fn read(&self) -> RwLockReadGuard { 21 | self.inner.read() 22 | } 23 | fn write(&self) -> RwLockWriteGuard { 24 | self.inner.write() 25 | } 26 | } 27 | 28 | impl Write for SingletonDS { 29 | fn put(&self, key: Key, value: Vec) -> Result<()> { 30 | self.write().put(key, value) 31 | } 32 | 33 | fn delete(&self, key: &Key) -> Result<()> { 34 | self.write().delete(key) 35 | } 36 | } 37 | impl Read for SingletonDS { 38 | fn get(&self, key: &Key) -> Result> { 39 | self.read().get(key) 40 | } 41 | 42 | fn has(&self, key: &Key) -> Result { 43 | self.read().has(key) 44 | } 45 | 46 | fn get_size(&self, key: &Key) -> Result { 47 | self.read().get_size(key) 48 | } 49 | } 50 | impl Datastore for SingletonDS { 51 | fn sync(&self, prefix: &Key) -> Result<()> { 52 | self.write().sync(prefix) 53 | } 54 | } 55 | 56 | impl Batching for SingletonDS { 57 | type Txn = T::Txn; 58 | 59 | fn batch(&self) -> Result { 60 | self.read().batch() 61 | } 62 | 63 | fn commit(&self, txn: Self::Txn) -> Result<()> { 64 | self.write().commit(txn) 65 | } 66 | } 67 | 68 | impl TxnDatastore for SingletonDS 69 | where 70 | Self::Txn: Txn, 71 | { 72 | fn new_transaction(&self, read_only: bool) -> Result { 73 | self.read().new_transaction(read_only) 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /datastore/src/tests/basic_ds_test.rs: -------------------------------------------------------------------------------- 1 | use super::*; 2 | use crate::basic_ds::*; 3 | 4 | #[test] 5 | fn test_basic_ds() { 6 | let ds = new_map_datastore(); 7 | basic_sub_tests(&ds); 8 | batch_sub_tests(&ds); 9 | } 10 | -------------------------------------------------------------------------------- /datastore/src/tests/common.rs: -------------------------------------------------------------------------------- 1 | use matches::matches; 2 | 3 | use super::*; 4 | use crate::key::Key; 5 | use crate::{Batching, Datastore}; 6 | 7 | pub fn test_basic_put_get(ds: &D) { 8 | let k = Key::new("foo"); 9 | let v = b"Hello Datastore!"; 10 | ds.put(k.clone(), v.to_vec()).unwrap(); 11 | 12 | let have = ds.has(&k).unwrap(); 13 | assert!(have); 14 | 15 | let size = ds.get_size(&k).unwrap(); 16 | assert_eq!(size, v.len()); 17 | 18 | let out = ds.get(&k).unwrap(); 19 | assert_eq!(out.as_slice(), v.as_ref()); 20 | // again after get 21 | let have = ds.has(&k).unwrap(); 22 | assert!(have); 23 | let size = ds.get_size(&k).unwrap(); 24 | assert_eq!(size, v.len()); 25 | 26 | ds.delete(&k).unwrap(); 27 | 28 | let have = ds.has(&k).unwrap(); 29 | assert!(!have); 30 | 31 | let r = ds.get_size(&k); 32 | assert!(matches!(r, Err(DSError::NotFound(_)))); 33 | } 34 | 35 | pub fn test_not_founds(ds: &D) { 36 | let badk = Key::new("notreal"); 37 | let r = ds.get(&badk); 38 | assert!(matches!(r, Err(DSError::NotFound(_)))); 39 | 40 | let has = ds.has(&badk).unwrap(); 41 | assert!(!has); 42 | 43 | let r = ds.get_size(&badk); 44 | assert!(matches!(r, Err(DSError::NotFound(_)))); 45 | } 46 | 47 | // TODO query limit 48 | 49 | // TODO query order 50 | 51 | // TODO manykeysandquery 52 | 53 | pub fn test_basic_sync(ds: &D) { 54 | ds.sync(&Key::new("foo")).unwrap(); 55 | 56 | ds.put(Key::new("/foo"), b"foo".to_vec()).unwrap(); 57 | 58 | ds.sync(&Key::new("/foo")).unwrap(); 59 | 60 | ds.put(Key::new("/foo/bar"), b"bar".to_vec()).unwrap(); 61 | 62 | ds.sync(&Key::new("/foo")).unwrap(); 63 | 64 | ds.sync(&Key::new("/foo/bar")).unwrap(); 65 | 66 | ds.sync(&Key::new("")).unwrap(); 67 | } 68 | 69 | // TODO query 70 | 71 | pub fn test_batch(ds: &D) { 72 | let mut batch = ds.batch().unwrap(); 73 | 74 | let mut blocks = vec![]; 75 | let mut keys = vec![]; 76 | for _ in 0..20 { 77 | let blk: [u8; 32] = random!(); 78 | 79 | let key = Key::new(String::from_utf8_lossy(&blk[..8])); 80 | keys.push(key.clone()); 81 | blocks.push(blk); 82 | 83 | batch.put(key, blk.to_vec()).unwrap(); 84 | } 85 | 86 | for k in keys.iter() { 87 | let r = ds.get(k); 88 | assert!(matches!(r, Err(DSError::NotFound(_)))); 89 | } 90 | 91 | ds.commit(batch).unwrap(); 92 | for (i, k) in keys.iter().enumerate() { 93 | let r = ds.get(k).unwrap(); 94 | assert_eq!(r.as_slice(), blocks[i].as_ref()) 95 | } 96 | } 97 | 98 | pub fn test_batch_delete(ds: &D) { 99 | let mut keys = vec![]; 100 | for _ in 0..20 { 101 | let blk: [u8; 16] = random!(); 102 | 103 | let key = Key::new(String::from_utf8_lossy(&blk[..8])); 104 | keys.push(key.clone()); 105 | 106 | ds.put(key, blk.to_vec()).unwrap(); 107 | } 108 | let mut batch = ds.batch().unwrap(); 109 | for k in keys.iter() { 110 | batch.delete(k).unwrap(); 111 | } 112 | 113 | ds.commit(batch).unwrap(); 114 | 115 | for k in keys.iter() { 116 | let r = ds.get(k); 117 | assert!(matches!(r, Err(DSError::NotFound(_)))); 118 | } 119 | } 120 | 121 | pub fn test_batch_put_and_delete(ds: &D) { 122 | let mut batch = ds.batch().unwrap(); 123 | 124 | let ka = Key::new("/a"); 125 | let kb = Key::new("/b"); 126 | 127 | batch.put(ka.clone(), [1_u8].to_vec()).unwrap(); 128 | batch.put(kb.clone(), [2_u8].to_vec()).unwrap(); 129 | 130 | batch.delete(&ka).unwrap(); 131 | batch.delete(&kb).unwrap(); 132 | 133 | batch.put(kb.clone(), [3_u8].to_vec()).unwrap(); 134 | 135 | ds.commit(batch).unwrap(); 136 | 137 | let out = ds.get(&kb).unwrap(); 138 | assert_eq!(out.as_slice(), [3_u8].as_ref()); 139 | } 140 | -------------------------------------------------------------------------------- /datastore/src/tests/key_test.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2020 PolkaX. Licensed under MIT or Apache-2.0. 2 | 3 | use crate::key::{namespace_type, namespace_value, Key}; 4 | 5 | #[test] 6 | fn test_namespace_type() { 7 | assert_eq!(namespace_type(""), ""); 8 | assert_eq!(namespace_type("123"), ""); 9 | assert_eq!(namespace_type(":"), ""); 10 | assert_eq!(namespace_type("123:"), "123"); 11 | assert_eq!(namespace_type("123:234"), "123"); 12 | assert_eq!(namespace_type("123:234:"), "123:234"); 13 | assert_eq!(namespace_type("123:234:345"), "123:234"); 14 | } 15 | 16 | #[test] 17 | fn test_namespace_value() { 18 | assert_eq!(namespace_value(""), ""); 19 | assert_eq!(namespace_value("123"), "123"); 20 | assert_eq!(namespace_value(":"), ""); 21 | assert_eq!(namespace_value("123:"), ""); 22 | assert_eq!(namespace_value("123:234"), "234"); 23 | assert_eq!(namespace_value("123:234:"), ""); 24 | assert_eq!(namespace_value("123:234:345"), "345"); 25 | } 26 | 27 | fn sub_test_key(s: &str) { 28 | let fixed = path_clean::clean(&format!("/{}", s)); 29 | let namespaces: Vec = fixed.split('/').skip(1).map(|s| s.to_string()).collect(); 30 | let last_namespace = namespaces.last().map(|s| s.to_owned()).unwrap(); 31 | let lnparts: Vec = last_namespace.split(':').map(|s| s.to_string()).collect(); 32 | let ktype = if lnparts.len() > 1 { 33 | lnparts[..lnparts.len() - 1].join(":") 34 | } else { 35 | "".to_string() 36 | }; 37 | let kname = lnparts[lnparts.len() - 1].to_owned(); 38 | 39 | let c = fixed.clone() + "/cchildd"; 40 | let kchild = path_clean::clean(&c); 41 | let kparent = "/".to_string() + &namespaces[..namespaces.len() - 1].join("/"); 42 | let c = kparent.clone() + "/" + &ktype; 43 | let kpath = path_clean::clean(&c); 44 | let kinstance = fixed.clone() + ":" + "inst"; 45 | println!("Testing: {}", Key::new(s.to_owned())); 46 | 47 | assert_eq!(Key::new(s).as_str(), fixed.as_str()); 48 | assert_eq!(Key::new(s), Key::new(s.to_owned())); 49 | assert_eq!(Key::new(s).as_str(), Key::new(s.to_owned()).as_str()); 50 | assert_eq!(Key::new(s).name(), kname.as_str()); 51 | assert_eq!(Key::new(s).type_(), ktype.as_str()); 52 | assert_eq!(Key::new(s).path().as_str(), kpath.as_str()); 53 | assert_eq!(Key::new(s).into_path().as_str(), kpath.as_str()); 54 | assert_eq!(Key::new(s).instance("inst").as_str(), kinstance.as_str()); 55 | assert_eq!( 56 | Key::new(s).into_instance("inst").as_str(), 57 | kinstance.as_str() 58 | ); 59 | 60 | assert_eq!(Key::new(s).child(Key::new("cchildd")).as_str(), kchild); 61 | assert_eq!(Key::new(s).into_child(Key::new("cchildd")).as_str(), kchild); 62 | assert_eq!( 63 | Key::new(s).child(Key::new("cchildd")).parent().as_str(), 64 | fixed 65 | ); 66 | assert_eq!( 67 | Key::new(s) 68 | .into_child(Key::new("cchildd")) 69 | .into_parent() 70 | .as_str(), 71 | fixed 72 | ); 73 | assert_eq!(Key::new(s).child_string("cchildd").as_str(), kchild); 74 | assert_eq!(Key::new(s).child_string("cchildd").parent().as_str(), fixed); 75 | assert_eq!(Key::new(s).parent().as_str(), kparent); 76 | assert_eq!(Key::new(s).list().len(), namespaces.len()); 77 | assert_eq!(Key::new(s).namespace().len(), namespaces.len()); 78 | 79 | assert_eq!( 80 | Key::new(s) 81 | .list() 82 | .into_iter() 83 | .map(|s| s.to_owned()) 84 | .collect::>(), 85 | namespaces 86 | ); 87 | 88 | assert_ne!(Key::new(s), Key::new("/fdsafdsa/".to_string() + s)); 89 | 90 | assert!(Key::new(s) >= Key::new(s).parent()); 91 | assert!(Key::new(s) < Key::new(s).into_child_string("foo")); 92 | } 93 | 94 | #[test] 95 | fn test_key_basic() { 96 | sub_test_key(""); 97 | sub_test_key("abcde"); 98 | sub_test_key("disahfidsalfhduisaufidsail"); 99 | sub_test_key("/fdisahfodisa/fdsa/fdsafdsafdsafdsa/fdsafdsa/"); 100 | sub_test_key("4215432143214321432143214321"); 101 | sub_test_key("/fdisaha////fdsa////fdsafdsafdsafdsa/fdsafdsa/"); 102 | sub_test_key("abcde:fdsfd"); 103 | sub_test_key("disahfidsalfhduisaufidsail:fdsa"); 104 | sub_test_key("/fdisahfodisa/fdsa/fdsafdsafdsafdsa/fdsafdsa/:"); 105 | sub_test_key("4215432143214321432143214321:"); 106 | sub_test_key("fdisaha////fdsa////fdsafdsafdsafdsa/fdsafdsa/f:fdaf"); 107 | } 108 | 109 | #[test] 110 | fn test_key_ancestry() { 111 | let k1 = Key::new("/A/B/C"); 112 | let k2 = Key::new("/A/B/C/D"); 113 | 114 | assert_eq!(k1.as_str(), "/A/B/C"); 115 | assert_eq!(k2.as_str(), "/A/B/C/D"); 116 | 117 | assert!(k1.is_ancestor_of(&k2)); 118 | assert!(k2.is_descendant_of(&k1)); 119 | 120 | assert!(Key::new("/A").is_ancestor_of(&k2)); 121 | assert!(Key::new("/A").is_ancestor_of(&k1)); 122 | assert!(!Key::new("/A").is_descendant_of(&k2)); 123 | assert!(!Key::new("/A").is_descendant_of(&k1)); 124 | 125 | assert!(k2.is_descendant_of(&Key::new("/A"))); 126 | assert!(k1.is_descendant_of(&Key::new("/A"))); 127 | assert!(!k2.is_ancestor_of(&Key::new("/A"))); 128 | assert!(!k1.is_ancestor_of(&Key::new("/A"))); 129 | 130 | assert!(!k2.is_ancestor_of(&k2)); 131 | assert!(!k1.is_ancestor_of(&k1)); 132 | 133 | assert_eq!(k1.child(Key::new("D")), k2); 134 | assert_eq!(k1.child_string("D"), k2); 135 | assert_eq!(k1, k2.parent()); 136 | assert_eq!(k1.path(), k2.parent().path()); 137 | } 138 | 139 | #[test] 140 | fn test_type() { 141 | let k1 = Key::new("/A/B/C:c"); 142 | let k2 = Key::new("/A/B/C:c/D:d"); 143 | 144 | assert!(k1.is_ancestor_of(&k2)); 145 | assert!(k2.is_descendant_of(&k1)); 146 | 147 | assert_eq!(k1.type_(), "C"); 148 | assert_eq!(k2.type_(), "D"); 149 | assert_eq!(k1.type_(), k2.parent().type_()); 150 | } 151 | 152 | #[test] 153 | fn test_random() { 154 | use std::collections::HashSet; 155 | let mut s = HashSet::with_capacity(1000); 156 | for _ in 0..1000 { 157 | let k = Key::random_key(); 158 | assert!(s.insert(k)) 159 | } 160 | } 161 | 162 | #[test] 163 | fn test_less() { 164 | fn check_less(a: Key, b: Key) { 165 | assert!(a < b); 166 | assert!(!(b < a)); 167 | } 168 | 169 | check_less(Key::new("/a/b/c"), Key::new("/a/b/c/d")); 170 | check_less(Key::new("/a/b"), Key::new("/a/b/c/d")); 171 | check_less(Key::new("/a"), Key::new("/a/b/c/d")); 172 | check_less(Key::new("/a/a/c"), Key::new("/a/b/c")); 173 | check_less(Key::new("/a/ab/c"), Key::new("/a/b/c")); 174 | check_less(Key::new("/a/a/d"), Key::new("/a/b/c")); 175 | check_less(Key::new("/a/b/c/d/e/f/g/h"), Key::new("/b")); 176 | check_less(Key::new("/"), Key::new("/a")); 177 | } 178 | 179 | struct Case { 180 | key: Key, 181 | data: Vec, 182 | } 183 | 184 | #[test] 185 | fn test_json() { 186 | let cases = vec![ 187 | Case { 188 | key: Key::new("/a/b/c"), 189 | data: br#""/a/b/c""#.to_vec(), 190 | }, 191 | Case { 192 | key: Key::new(r#"/shouldescapekey"/with/quote"#), 193 | data: br#""/shouldescapekey\"/with/quote""#.to_vec(), 194 | }, 195 | ]; 196 | 197 | for c in cases { 198 | let out = serde_json::to_string(&c.key).unwrap(); 199 | assert_eq!(out.as_bytes(), c.data.as_slice()); 200 | 201 | let k: Key = serde_json::from_str(&out).unwrap(); 202 | assert_eq!(k, c.key); 203 | } 204 | } 205 | -------------------------------------------------------------------------------- /datastore/src/tests/keytransform_test.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2020 PolkaX. Licensed under MIT or Apache-2.0. 2 | 3 | // use super::*; 4 | // TODO 5 | #[test] 6 | fn test_basic() {} 7 | -------------------------------------------------------------------------------- /datastore/src/tests/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2020 PolkaX. Licensed under MIT or Apache-2.0. 2 | 3 | mod basic_ds_test; 4 | mod common; 5 | mod key_test; 6 | mod keytransform_test; 7 | mod query; 8 | 9 | use super::*; 10 | #[macro_export] 11 | macro_rules! random { 12 | () => {{ 13 | use rand::distributions::Distribution; 14 | let mut rng = rand::rngs::OsRng; 15 | rand::distributions::Standard.sample(&mut rng) 16 | }}; 17 | } 18 | 19 | fn basic_sub_tests(ds: &D) { 20 | common::test_basic_put_get(ds); 21 | common::test_not_founds(ds); 22 | common::test_basic_sync(ds); 23 | } 24 | 25 | fn batch_sub_tests(ds: &D) { 26 | common::test_batch(ds); 27 | common::test_batch_delete(ds); 28 | common::test_batch_put_and_delete(ds); 29 | } 30 | -------------------------------------------------------------------------------- /datastore/src/tests/query/filter_test.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2020 PolkaX. Licensed under MIT or Apache-2.0. 2 | 3 | // use super::*; 4 | // use crate::query::filter::*; 5 | 6 | #[test] 7 | fn test_filter_key_compare() {} 8 | -------------------------------------------------------------------------------- /datastore/src/tests/query/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2020 PolkaX. Licensed under MIT or Apache-2.0. 2 | 3 | mod filter_test; 4 | mod order_test; 5 | mod query_test; 6 | 7 | // use super::*; 8 | // use crate::query::Entry; 9 | // common utils 10 | // TODO 11 | -------------------------------------------------------------------------------- /datastore/src/tests/query/order_test.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2020 PolkaX. Licensed under MIT or Apache-2.0. 2 | -------------------------------------------------------------------------------- /datastore/src/tests/query/query_test.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2020 PolkaX. Licensed under MIT or Apache-2.0. 2 | -------------------------------------------------------------------------------- /fs-lock/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "fs-lock" 3 | version = "0.1.0" 4 | authors = ["PolkaX "] 5 | edition = "2018" 6 | 7 | [dependencies] 8 | fs2 = "0.4" 9 | lazy_static = "1.4" 10 | log = "0.4" 11 | 12 | [dev-dependencies] 13 | tempfile = "3.1" 14 | -------------------------------------------------------------------------------- /fs-lock/src/lib.rs: -------------------------------------------------------------------------------- 1 | use std::fs; 2 | use std::io; 3 | use std::path::PathBuf; 4 | use std::sync::Mutex; 5 | 6 | // re-export it 7 | pub use fs2::FileExt; 8 | use lazy_static::lazy_static; 9 | use log::{debug, error}; 10 | 11 | pub const LOG_TARGET: &str = "lock"; 12 | 13 | lazy_static! { 14 | static ref GLOBAL_LOCK: Mutex<()> = Mutex::new(()); 15 | } 16 | 17 | pub fn lock(path: &PathBuf, lock_file: &str) -> io::Result { 18 | if !path.exists() { 19 | fs::create_dir_all(path.as_path())?; 20 | } 21 | 22 | let mut path = path.to_owned(); 23 | path.push(lock_file); 24 | 25 | let _lock = GLOBAL_LOCK.lock().unwrap(); 26 | let f = fs::File::create(path.as_path())?; 27 | f.try_lock_exclusive()?; 28 | Ok(f) 29 | } 30 | 31 | /// Unlock the file. if file is not locked, just return Ok(()). 32 | /// Thus the file could be unlocked more than once. 33 | pub fn unlock(file: &fs::File) -> io::Result<()> { 34 | let _lock = GLOBAL_LOCK.lock().unwrap(); 35 | file.unlock() 36 | } 37 | 38 | pub fn locked(path: &PathBuf, lock_file: &str) -> io::Result { 39 | debug!(target: LOG_TARGET, "Checking lock"); 40 | let mut path = path.to_owned(); 41 | path.push(lock_file); 42 | if !path.exists() { 43 | debug!(target: LOG_TARGET, "File doesn't exist: {:?}", path); 44 | return Ok(false); 45 | } 46 | 47 | let _lock = GLOBAL_LOCK.lock().unwrap(); 48 | let f = fs::OpenOptions::new().write(true).open(path.as_path())?; 49 | let r = f.try_lock_exclusive(); 50 | match r { 51 | Ok(_) => { 52 | debug!(target: LOG_TARGET, "No one has a lock"); 53 | Ok(false) 54 | } 55 | Err(e) => { 56 | debug!(target: LOG_TARGET, "try lock failed, err: {:?}", e); 57 | if e.kind() == io::ErrorKind::WouldBlock { 58 | Ok(true) 59 | } else { 60 | error!( 61 | target: LOG_TARGET, 62 | "lock failed due to other reason, err: {:?}", e 63 | ); 64 | Err(e) 65 | } 66 | } 67 | } 68 | } 69 | 70 | #[cfg(test)] 71 | mod tests { 72 | use super::*; 73 | 74 | fn assert_lock(path: &PathBuf, file: &str, expected: bool) { 75 | let is_locked = locked(path, file).unwrap(); 76 | assert_eq!(is_locked, expected) 77 | } 78 | 79 | #[test] 80 | fn test_lock_simple() { 81 | let lock_file = "my-test.lock"; 82 | let dir = tempfile::Builder::new().prefix("lock").tempdir().unwrap(); 83 | 84 | let mut p = dir.path().to_path_buf(); 85 | p.push(lock_file); 86 | 87 | // make sure we start clean 88 | let _ = fs::remove_file(p.as_path()); 89 | 90 | let path = dir.path().to_path_buf(); 91 | assert_lock(&path, lock_file, false); 92 | 93 | let file = lock(&path, lock_file).unwrap(); 94 | 95 | assert_eq!(locked(&path, lock_file).unwrap(), true); 96 | 97 | unlock(&file).unwrap(); 98 | 99 | assert_eq!(locked(&path, lock_file).unwrap(), false); 100 | 101 | // second round of locking 102 | let file2 = lock(&path, lock_file).unwrap(); 103 | assert_lock(&path, lock_file, true); 104 | unlock(&file2).unwrap(); 105 | assert_lock(&path, lock_file, false); 106 | } 107 | 108 | #[test] 109 | fn test_lock_multiple() { 110 | let lock_file1 = "test-1.lock"; 111 | let lock_file2 = "test-2.lock"; 112 | let dir = tempfile::Builder::new().prefix("lock").tempdir().unwrap(); 113 | 114 | // make sure we start clean 115 | let mut p = dir.path().to_path_buf(); 116 | p.push(lock_file1); 117 | let _ = fs::remove_file(p.as_path()); 118 | p.pop(); 119 | p.push(lock_file2); 120 | let _ = fs::remove_file(p.as_path()); 121 | 122 | let path = dir.path().to_path_buf(); 123 | let file1 = lock(&path, lock_file1).unwrap(); 124 | let file2 = lock(&path, lock_file2).unwrap(); 125 | 126 | assert_lock(&path, lock_file1, true); 127 | assert_lock(&path, lock_file2, true); 128 | 129 | unlock(&file1).unwrap(); 130 | 131 | assert_lock(&path, lock_file1, false); 132 | assert_lock(&path, lock_file2, true); 133 | 134 | unlock(&file2).unwrap(); 135 | 136 | assert_lock(&path, lock_file1, false); 137 | assert_lock(&path, lock_file2, false); 138 | } 139 | 140 | #[test] 141 | fn test_thread_safe() { 142 | use std::thread; 143 | 144 | let lock_file = "my-test.lock"; 145 | let dir = tempfile::Builder::new().prefix("lock").tempdir().unwrap(); 146 | let mut p = dir.path().to_path_buf(); 147 | p.push(lock_file); 148 | 149 | // make sure we start clean 150 | let _ = fs::remove_file(p.as_path()); 151 | 152 | let path = dir.path().to_path_buf(); 153 | for _ in 0..100 { 154 | let path_d = path.clone(); 155 | let file_name = lock_file.to_string(); 156 | let child = thread::spawn(move || lock(&path_d, &file_name)); 157 | 158 | let current_ret = lock(&path, lock_file); 159 | 160 | let res = child.join().unwrap(); 161 | 162 | assert!((current_ret.is_ok() && res.is_err()) || (current_ret.is_err() && res.is_ok())); 163 | 164 | if let Ok(r) = current_ret { 165 | unlock(&r).unwrap(); 166 | } 167 | 168 | if let Ok(r) = res { 169 | unlock(&r).unwrap(); 170 | } 171 | } 172 | } 173 | } 174 | -------------------------------------------------------------------------------- /ipfs/blockstore/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "ipfs-blockstore" 3 | version = "0.1.0" 4 | authors = ["PolkaX "] 5 | edition = "2018" 6 | 7 | license = "MIT/Apache-2.0" 8 | repository = "https://github.com/PolkaX/rust-ipfs" 9 | description = "Implementation of the ipfs blockstore" 10 | keywords = ["ipfs", "blockstore"] 11 | 12 | [dependencies] 13 | cid = { version = "0.5", features = ["cbor", "json"] } 14 | thiserror = "1.0" 15 | 16 | block-format = { path = "../../block-format" } 17 | -------------------------------------------------------------------------------- /ipfs/blockstore/src/error.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2020 PolkaX. Licensed under MIT or Apache-2.0. 2 | 3 | use cid::Cid; 4 | 5 | /// Type alias to use this library's [`BlockstoreError`] type in a `Result`. 6 | pub type Result = std::result::Result; 7 | 8 | /// Errors generated from this library. 9 | #[derive(Debug, thiserror::Error)] 10 | pub enum BlockstoreError { 11 | /// ErrNotFound is an error returned when a block is not found. 12 | #[error("blockstore: block not found. cid: {0}")] 13 | NotFound(Cid), 14 | 15 | /// ErrHashMismatch is an error returned when the hash of a block 16 | /// is different than expected. 17 | #[error("block in storage has different hash than requested")] 18 | HashMismatch, 19 | 20 | #[error("other err: {0}")] 21 | Other(#[from] Box), 22 | } 23 | -------------------------------------------------------------------------------- /ipfs/blockstore/src/lib.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2020 PolkaX. Licensed under MIT or Apache-2.0. 2 | 3 | mod error; 4 | 5 | use block_format::BasicBlock; 6 | use cid::Cid; 7 | 8 | pub use crate::error::*; 9 | 10 | /// Blockstore wraps a Datastore block-centered methods and provides a layer 11 | /// of abstraction which allows to add different caching strategies. 12 | // TODO need to decide BasicBlock or BlockT for params 13 | pub trait Blockstore { 14 | fn delete_block(&self, cid: &Cid) -> Result<()>; 15 | fn has(&self, cid: &Cid) -> Result<()>; 16 | fn get(&self, cid: &Cid) -> Result; 17 | 18 | /// GetSize returns the CIDs mapped BlockSize 19 | fn get_size(&self, cid: &Cid) -> Result; 20 | 21 | fn put(&mut self, block: BasicBlock) -> Result<()>; 22 | fn put_many(&mut self, block: &[BasicBlock]) -> Result<()>; 23 | fn hash_on_read(&mut self, enable: bool); 24 | } 25 | -------------------------------------------------------------------------------- /ipld/amt/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "ipld-amt" 3 | version = "0.1.0" 4 | authors = ["PolkaX "] 5 | edition = "2018" 6 | 7 | license = "MIT/Apache-2.0" 8 | repository = "https://github.com/PolkaX/rust-ipfs" 9 | description = "Implementation of the ipld amt" 10 | keywords = ["ipfs", "ipld", "amt"] 11 | 12 | [dependencies] 13 | cid = { version = "0.5", features = ["cbor", "json"] } 14 | multihash = "0.11" 15 | serde = "1.0" 16 | serde_bytes = "0.11" 17 | serde_cbor = { version = "0.11", features = ["tags"] } 18 | thiserror = "1.0" 19 | 20 | block-format = { path = "../../block-format" } 21 | ipfs-blockstore = { path = "../../ipfs/blockstore" } 22 | ipld-core = { path = "../core" } 23 | 24 | [dev-dependencies] 25 | matches = "0.1" 26 | rand = "0.7" 27 | -------------------------------------------------------------------------------- /ipld/amt/src/blocks.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2020 PolkaX. Licensed under MIT or Apache-2.0. 2 | 3 | use std::cell::RefCell; 4 | use std::rc::Rc; 5 | 6 | use cid::{Cid, Codec, Prefix, Version}; 7 | use serde::{de::DeserializeOwned, Serialize}; 8 | 9 | use block_format::{BasicBlock, Block as BlockT}; 10 | use ipfs_blockstore::Blockstore; 11 | 12 | use crate::error::*; 13 | 14 | pub trait Blocks { 15 | fn get(&self, cid: &Cid) -> Result; 16 | fn put(&mut self, v: Input) -> Result; 17 | } 18 | 19 | pub struct BStoreWrapper { 20 | bs: Rc>, 21 | } 22 | 23 | impl Clone for BStoreWrapper { 24 | fn clone(&self) -> Self { 25 | BStoreWrapper { 26 | bs: self.bs.clone(), 27 | } 28 | } 29 | } 30 | 31 | impl Blocks for BStoreWrapper { 32 | fn get(&self, cid: &Cid) -> Result { 33 | let r = self.bs.borrow().get(cid)?; 34 | let o: Output = serde_cbor::from_slice(r.raw_data().as_ref())?; 35 | Ok(o) 36 | } 37 | 38 | fn put(&mut self, v: Input) -> Result { 39 | let v = serde_cbor::to_vec(&v)?; 40 | let prefix = Prefix { 41 | version: Version::V1, 42 | codec: Codec::DagCBOR, 43 | mh_type: multihash::Code::Blake2b256, 44 | mh_len: 32, 45 | }; 46 | let cid = Cid::new_from_prefix(&prefix, v.as_ref()); 47 | 48 | let blk = BasicBlock::new_with_cid(v.into(), cid.clone())?; 49 | self.bs.borrow_mut().put(blk)?; 50 | Ok(cid) 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /ipld/amt/src/error.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2020 PolkaX. Licensed under MIT or Apache-2.0. 2 | 3 | pub type Result = std::result::Result; 4 | 5 | #[derive(Debug, thiserror::Error)] 6 | pub enum AmtIpldError { 7 | #[error("blockstore error, err:{0}")] 8 | Blockstore(#[from] blockstore::BlockstoreError), 9 | 10 | #[error("core de/serialize error: {0}")] 11 | Cbor(#[from] serde_cbor::Error), 12 | 13 | #[error("cid error: {0}")] 14 | Cid(#[from] cid::Error), 15 | 16 | #[error("block format error: {0}")] 17 | BlockFormat(#[from] block_format::BlockFormatError), 18 | 19 | #[error("ipld core error: {0}")] 20 | IpldCbor(#[from] ipld_core::IpldCoreError), 21 | 22 | #[error("not found for key: {0}")] 23 | NotFound(u64), 24 | 25 | #[error("no node found at (sub)index: {0}")] 26 | NoNodeForIndex(usize), 27 | 28 | #[error("other err: {0}")] 29 | Other(#[from] Box), 30 | } 31 | -------------------------------------------------------------------------------- /ipld/amt/src/lib.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2020 PolkaX. Licensed under MIT or Apache-2.0. 2 | #![allow(clippy::or_fun_call)] 3 | 4 | mod blocks; 5 | mod error; 6 | mod node; 7 | #[cfg(test)] 8 | mod tests; 9 | 10 | pub use crate::blocks::Blocks; 11 | pub use crate::error::*; 12 | pub use crate::node::Amt; 13 | -------------------------------------------------------------------------------- /ipld/amt/src/node/iter.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2020 PolkaX. Licensed under MIT or Apache-2.0. 2 | 3 | use serde_cbor::Value; 4 | 5 | use super::{Amt, Item, Node, BITS_PER_SUBKEY, WIDTH}; 6 | use crate::blocks::Blocks; 7 | use crate::error::*; 8 | 9 | impl Amt 10 | where 11 | B: Blocks, 12 | { 13 | /// this function would use in anywhere to traverse the trie, do not need flush first. 14 | pub fn for_each(&self, f: &mut F) -> Result<()> 15 | where 16 | F: FnMut(u64, &Value) -> Result<()>, 17 | { 18 | traversing(&self.bs, &self.root, self.height, 0, f) 19 | } 20 | 21 | /// Subtract removes all elements of 'or' from 'self' 22 | pub fn subtract(&mut self, or: Self) -> Result<()> { 23 | or.for_each(&mut |key, _| { 24 | // if not find, do not handle error 25 | match self.delete(key) { 26 | Ok(_) | Err(AmtIpldError::NotFound(_)) => Ok(()), 27 | Err(e) => Err(e), 28 | } 29 | }) 30 | } 31 | } 32 | 33 | fn traversing(bs: &B, node: &Node, height: u64, prefix_key: u64, f: &mut F) -> Result<()> 34 | where 35 | B: Blocks, 36 | F: FnMut(u64, &Value) -> Result<()>, 37 | { 38 | let prefix = prefix_key << BITS_PER_SUBKEY; 39 | if height == 0 { 40 | for i in 0..WIDTH { 41 | if node.get_bit(i) { 42 | let current_key = prefix + i as u64; 43 | let index = node.bit_to_index(i); 44 | f(current_key, &node.leafs[index])?; 45 | } 46 | } 47 | return Ok(()); 48 | } 49 | 50 | let mut branches = node.branches.borrow_mut(); 51 | 52 | for i in 0..WIDTH { 53 | if node.get_bit(i) { 54 | let current_key = prefix + i as u64; 55 | let index = node.bit_to_index(i); 56 | branches[index].load_item(bs)?; 57 | 58 | if let Item::Ptr(node) = &branches[index] { 59 | traversing(bs, node, height - 1, current_key, f)?; 60 | } else { 61 | unreachable!("after `load_item`, Item must be `Ptr`") 62 | } 63 | } 64 | } 65 | Ok(()) 66 | } 67 | 68 | impl Amt 69 | where 70 | B: Blocks, 71 | { 72 | /// `iter()` is equal to `for_each` now(could use before `flush()`). 73 | /// but `iter()` would cast more resource 74 | pub fn iter(&self) -> Iter { 75 | let node_ref = &self.root; 76 | 77 | let prefix_key_list = (0..WIDTH) 78 | .map(|prefix_key| (prefix_key, node_ref.get_bit(prefix_key))) 79 | .filter(|(_, need)| *need) 80 | .map(|(pref, _)| pref as u64) 81 | .collect::>(); 82 | 83 | let init_node = &self.root as *const Node; 84 | 85 | let init = if self.height == 0 { 86 | Traversing::Leaf(0, (prefix_key_list, init_node)) 87 | } else { 88 | Traversing::Branch(prefix_key_list, init_node) 89 | }; 90 | Iter { 91 | size: self.count, 92 | count: 0, 93 | stack: vec![init], 94 | bs: &self.bs, 95 | } 96 | } 97 | } 98 | 99 | /// this `Iter` only could be used for FlushedRoot, due to current module use child_cache to store child, 100 | /// and the child ref is under `RefCell`. So that we could only iterate the tree after flushing. 101 | /// if someone do not what iterating the free after flushing, could use `for_each`. 102 | pub struct Iter<'a, B> 103 | where 104 | B: Blocks, 105 | { 106 | size: u64, 107 | count: u64, 108 | stack: Vec, 109 | // blocks ref, use for load node from cid 110 | bs: &'a B, 111 | } 112 | 113 | enum Traversing { 114 | Leaf(usize, (Vec, *const Node)), 115 | Branch(Vec, *const Node), 116 | } 117 | 118 | impl<'a, B> Iterator for Iter<'a, B> 119 | where 120 | B: Blocks, 121 | { 122 | type Item = (u64, &'a Value); 123 | 124 | /// it's safe to use unsafe here, for except root node, every node is in heap, 125 | /// and be refered from root node. thus we use unsafe to avoid lifetime check 126 | /// and mutable check. 127 | /// notice iterator would load node for cid, thus after iter, all tree is in 128 | /// `Ptr` mode, in other word, is being expanded 129 | fn next(&mut self) -> Option { 130 | let last = match self.stack.pop() { 131 | Some(last) => last, 132 | None => { 133 | return None; 134 | } 135 | }; 136 | match last { 137 | Traversing::Leaf(pos, (keys, leaf_node)) => { 138 | let r = unsafe { (*leaf_node).leafs.get(pos).map(|v| (keys[pos], v)) }; 139 | match r { 140 | Some(v) => { 141 | let pos = pos + 1; 142 | self.stack.push(Traversing::Leaf(pos, (keys, leaf_node))); 143 | self.count += 1; 144 | Some(v) 145 | } 146 | None => self.next(), 147 | } 148 | } 149 | Traversing::Branch(keys, node_ref) => { 150 | unsafe { 151 | let node = &(*node_ref); 152 | let mut children = vec![]; 153 | for (b, key) in node.branches.borrow_mut().iter_mut().zip(keys.into_iter()) { 154 | b.load_item(self.bs).ok()?; 155 | if let Item::Ptr(child_node) = b { 156 | let prefix_key_list = (0..WIDTH) 157 | .map(|prefix_key| (prefix_key, child_node.get_bit(prefix_key))) 158 | .filter(|(_, need)| *need) 159 | .map(|(pref, _)| (key << BITS_PER_SUBKEY) + pref as u64) 160 | .collect::>(); 161 | 162 | let node_ptr = child_node.as_ref() as *const Node; 163 | if !child_node.leafs.is_empty() { 164 | children.push(Traversing::Leaf(0, (prefix_key_list, node_ptr))); 165 | } else { 166 | children.push(Traversing::Branch(prefix_key_list, node_ptr)); 167 | } 168 | } 169 | } 170 | self.stack.extend(children.into_iter().rev()); 171 | } 172 | self.next() 173 | } 174 | } 175 | } 176 | 177 | fn size_hint(&self) -> (usize, Option) { 178 | let hint = (self.size - self.count) as usize; 179 | (hint, Some(hint)) 180 | } 181 | } 182 | -------------------------------------------------------------------------------- /ipld/amt/src/node/trait_impl.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2020 PolkaX. Licensed under MIT or Apache-2.0. 2 | 3 | use std::fmt; 4 | use std::ops::Deref; 5 | use std::result; 6 | 7 | use serde::{de::Error, Deserialize, Deserializer, Serialize, Serializer}; 8 | use serde_cbor::Value; 9 | 10 | use super::{Amt, Item, Node}; 11 | use crate::blocks::Blocks; 12 | 13 | impl Serialize for Amt 14 | where 15 | B: Blocks, 16 | { 17 | fn serialize(&self, serializer: S) -> result::Result 18 | where 19 | S: Serializer, 20 | { 21 | (self.height, self.count, &self.root).serialize(serializer) 22 | } 23 | } 24 | 25 | impl Eq for Amt where B: Blocks {} 26 | 27 | impl PartialEq for Amt 28 | where 29 | B: Blocks, 30 | { 31 | fn eq(&self, other: &Self) -> bool { 32 | self.height.eq(&other.height) && self.count.eq(&other.count) && self.root.eq(&other.root) 33 | } 34 | } 35 | 36 | impl fmt::Debug for Amt 37 | where 38 | B: Blocks, 39 | { 40 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 41 | write!( 42 | f, 43 | "Root{{ height:{:}, count:{:}, node:{:?} }}", 44 | self.height, self.count, self.root 45 | ) 46 | } 47 | } 48 | 49 | #[derive(Debug, Deserialize)] 50 | pub struct PartAmt(pub u64, pub u64, pub Node); 51 | 52 | impl Serialize for Item { 53 | fn serialize(&self, serializer: S) -> Result<::Ok, ::Error> 54 | where 55 | S: Serializer, 56 | { 57 | match self { 58 | Item::Link(cid) => cid::ipld_dag_cbor::serialize(cid, serializer), 59 | Item::Ptr(_) => unreachable!("could not serialize `Ptr`, just allow `Link`"), 60 | } 61 | } 62 | } 63 | 64 | impl<'de> Deserialize<'de> for Item { 65 | fn deserialize(deserializer: D) -> Result>::Error> 66 | where 67 | D: Deserializer<'de>, 68 | { 69 | cid::ipld_dag_cbor::deserialize(deserializer).map(Item::Link) 70 | } 71 | } 72 | 73 | impl Serialize for Node { 74 | fn serialize(&self, serializer: S) -> result::Result 75 | where 76 | S: Serializer, 77 | { 78 | let r = self.bitmap.to_be_bytes(); 79 | let bytes: [u8; 1] = [r[r.len() - 1]; 1]; 80 | ( 81 | serde_bytes::Bytes::new(bytes.as_ref()), 82 | self.branches.borrow().deref(), 83 | &self.leafs, 84 | ) 85 | .serialize(serializer) 86 | } 87 | } 88 | 89 | #[derive(Deserialize)] 90 | struct NodeVisitor(serde_bytes::ByteBuf, Vec, Vec); 91 | impl<'de> Deserialize<'de> for Node { 92 | fn deserialize(deserializer: D) -> result::Result 93 | where 94 | D: Deserializer<'de>, 95 | { 96 | let visitor = NodeVisitor::deserialize(deserializer)?; 97 | if visitor.0.len() != 1 { 98 | return Err(D::Error::custom(format!( 99 | "node bitmap must be 1 byte, current is:{:?}", 100 | visitor.0 101 | ))); 102 | } 103 | Ok(Node::new_from_raw( 104 | visitor.0[0] as usize, 105 | visitor.1, 106 | visitor.2, 107 | )) 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /ipld/amt/src/tests/cbor_test.rs: -------------------------------------------------------------------------------- 1 | use super::*; 2 | use cid::IntoExt; 3 | 4 | #[test] 5 | fn node_test() { 6 | let cid = Cid::new_v0(multihash::Sha2_256::digest(b"something").into_ext()).unwrap(); 7 | let b = Item::Link(cid); 8 | let node = Node::new_from_raw(1, vec![b], vec![]); 9 | let v = serde_cbor::to_vec(&node).unwrap(); 10 | println!("{:?}", v); 11 | assert_eq!( 12 | v, 13 | vec![ 14 | 131, 65, 1, 129, 216, 42, 88, 35, 0, 18, 32, 63, 201, 182, 137, 69, 157, 115, 143, 140, 15 | 136, 163, 164, 138, 169, 227, 53, 66, 1, 107, 122, 64, 82, 224, 1, 170, 165, 54, 252, 16 | 167, 72, 19, 203, 128 17 | ] 18 | ); 19 | let n: Node = serde_cbor::from_slice(&v).unwrap(); 20 | assert_eq!(node, n); 21 | 22 | let node = Node::new_from_raw(255, vec![], vec![Value::Text("123".to_string())]); 23 | let v = serde_cbor::to_vec(&node).unwrap(); 24 | println!("{:?}", v); 25 | assert_eq!(v, vec![131, 65, 255, 128, 129, 99, 49, 50, 51]); 26 | let n: Node = serde_cbor::from_slice(&v).unwrap(); 27 | assert_eq!(node, n); 28 | } 29 | 30 | #[test] 31 | fn root_test() { 32 | let cid = Cid::new_v0(multihash::Sha2_256::digest(b"something").into_ext()).unwrap(); 33 | let b = Item::Link(cid); 34 | let node = Node::new_from_raw(1, vec![b], vec![]); 35 | let db = db(); 36 | let root = create_root(4, 100, node, db.clone()); 37 | let v = serde_cbor::to_vec(&root).unwrap(); 38 | println!("{:?}", v); 39 | 40 | assert_eq!( 41 | v, 42 | vec![ 43 | 131, 4, 24, 100, 131, 65, 1, 129, 216, 42, 88, 35, 0, 18, 32, 63, 201, 182, 137, 69, 44 | 157, 115, 143, 140, 136, 163, 164, 138, 169, 227, 53, 66, 1, 107, 122, 64, 82, 224, 1, 45 | 170, 165, 54, 252, 167, 72, 19, 203, 128 46 | ] 47 | ); 48 | 49 | let pr: PartAmt = serde_cbor::from_slice(&v).unwrap(); 50 | let r = Amt::from_part(pr, db); 51 | assert_eq!(root, r); 52 | } 53 | -------------------------------------------------------------------------------- /ipld/amt/src/tests/mod.rs: -------------------------------------------------------------------------------- 1 | mod amt_test; 2 | mod cbor_test; 3 | 4 | use std::cell::RefCell; 5 | use std::collections::HashMap; 6 | use std::rc::Rc; 7 | use std::result; 8 | 9 | use serde::{de::DeserializeOwned, Serialize}; 10 | use serde_cbor::Value; 11 | 12 | use blockstore::BlockstoreError; 13 | use cid::{Cid, Codec, Prefix, Version}; 14 | 15 | use crate::node::{create_root, Item, Node, PartAmt}; 16 | 17 | use super::*; 18 | 19 | #[derive(Default, Clone)] 20 | pub struct DB { 21 | db: Rc, Vec>>>, 22 | } 23 | 24 | impl Blocks for DB { 25 | fn get(&self, cid: &Cid) -> result::Result { 26 | let o = self 27 | .db 28 | .borrow() 29 | .get(&cid.to_bytes()) 30 | .ok_or(BlockstoreError::NotFound(cid.clone()).into()) 31 | .and_then(|v| serde_cbor::from_slice(v).map_err(AmtIpldError::Cbor))?; 32 | 33 | Ok(o) 34 | } 35 | 36 | fn put(&mut self, v: Input) -> result::Result { 37 | let v = serde_cbor::to_vec(&v)?; 38 | let prefix = Prefix { 39 | version: Version::V1, 40 | codec: Codec::DagCBOR, 41 | mh_type: multihash::Code::Blake2b256, 42 | mh_len: 32, 43 | }; 44 | let cid = Cid::new_from_prefix(&prefix, v.as_ref()); 45 | self.db.borrow_mut().insert(cid.to_bytes(), v); 46 | Ok(cid) 47 | } 48 | } 49 | 50 | pub fn db() -> DB { 51 | Default::default() 52 | } 53 | -------------------------------------------------------------------------------- /ipld/core/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "ipld-core" 3 | version = "0.1.0" 4 | authors = ["PolkaX "] 5 | edition = "2018" 6 | 7 | license = "MIT/Apache-2.0" 8 | repository = "https://github.com/PolkaX/rust-ipfs" 9 | description = "Implementation of the IPLD with CBOR and JSON serialization/deserialization" 10 | keywords = ["ipfs", "ipld", "cbor", "json"] 11 | 12 | [dependencies] 13 | bytes = { version = "0.5", features = ["serde"] } 14 | cid = { version = "0.5", features = ["cbor", "json"] } 15 | either = { version = "1.5", features = ["serde"] } 16 | minicbor = { version = "0.4", features = ["std", "half"] } 17 | multihash = "0.11" 18 | serde = { version = "1.0", features = ["derive"] } 19 | serde_json = "1.0" 20 | thiserror = "1.0" 21 | 22 | block-format = { path = "../../block-format" } 23 | ipld-format = { path = "../format" } 24 | 25 | [dev-dependencies] 26 | criterion = "0.3" 27 | hex = "0.4" 28 | maplit = "1.0" 29 | 30 | [[bench]] 31 | name = "benchmarks" 32 | path = "benches/benchmarks.rs" 33 | harness = false 34 | -------------------------------------------------------------------------------- /ipld/core/benches/benchmarks.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2020 PolkaX. Licensed under MIT or Apache-2.0. 2 | 3 | use std::collections::BTreeMap; 4 | 5 | use criterion::{black_box, criterion_group, criterion_main, Criterion}; 6 | use serde::{Deserialize, Serialize}; 7 | 8 | use ipld_core::{IpldNode, IpldValue}; 9 | 10 | #[derive(Serialize, Deserialize, Default, Clone, Debug)] 11 | struct MyStruct { 12 | items: BTreeMap, 13 | foo: String, 14 | bar: Vec, 15 | baz: Vec, 16 | } 17 | 18 | fn test_struct() -> MyStruct { 19 | let mut map = BTreeMap::new(); 20 | map.insert( 21 | "Foo".to_string(), 22 | MyStruct { 23 | foo: "Foo".to_string(), 24 | bar: b"Bar".to_vec(), 25 | baz: vec![1, 2, 3, 4], 26 | ..Default::default() 27 | }, 28 | ); 29 | map.insert( 30 | "Bar".to_string(), 31 | MyStruct { 32 | bar: b"Bar".to_vec(), 33 | baz: vec![1, 2, 3, 4], 34 | ..Default::default() 35 | }, 36 | ); 37 | 38 | MyStruct { 39 | items: map, 40 | baz: vec![5, 1, 2], 41 | ..Default::default() 42 | } 43 | } 44 | 45 | fn test_struct_obj() -> IpldValue { 46 | let obj = test_struct(); 47 | let json = serde_json::to_string(&obj).unwrap(); 48 | serde_json::from_str::(&json).unwrap() 49 | } 50 | 51 | fn bench_wrap_object(c: &mut Criterion) { 52 | let obj = test_struct_obj(); 53 | 54 | c.bench_function("wrap_object", |b| { 55 | b.iter(|| { 56 | let _ = 57 | black_box(IpldNode::wrap_object(&obj, multihash::Code::Sha2_256.into()).unwrap()); 58 | }) 59 | }); 60 | } 61 | 62 | fn bench_from_block(c: &mut Criterion) { 63 | let obj = test_struct_obj(); 64 | let node = IpldNode::wrap_object(&obj, multihash::Code::Sha2_256.into()).unwrap(); 65 | 66 | c.bench_function("from_block", |b| { 67 | b.iter(|| { 68 | let n = black_box(IpldNode::from_block(&node).unwrap()); 69 | assert_eq!(node, n); 70 | }) 71 | }); 72 | } 73 | 74 | fn bench_to_cbor(c: &mut Criterion) { 75 | let obj = test_struct_obj(); 76 | let node = IpldNode::wrap_object(&obj, multihash::Code::Sha2_256.into()).unwrap(); 77 | 78 | c.bench_function("to_cbor", |b| { 79 | b.iter(|| { 80 | let _ = black_box(node.to_cbor().unwrap()); 81 | }) 82 | }); 83 | } 84 | 85 | criterion_group!(benches, bench_wrap_object, bench_from_block, bench_to_cbor); 86 | criterion_main!(benches); 87 | -------------------------------------------------------------------------------- /ipld/core/src/convert.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2020 PolkaX. Licensed under MIT or Apache-2.0. 2 | 3 | use std::collections::BTreeMap; 4 | use std::convert::TryFrom; 5 | 6 | use cid::Cid; 7 | 8 | use crate::error::{IpldCborError, Result}; 9 | use crate::value::IpldValue; 10 | 11 | /// Convert structure to CBOR Value. 12 | pub fn struct_to_cbor_value(v: &S) -> Result { 13 | let s = serde_cbor::to_vec(&v)?; 14 | let value: serde_cbor::Value = serde_cbor::from_slice(&s)?; 15 | Ok(value) 16 | } 17 | 18 | /// Convert CBOR Value to structure. 19 | pub fn cbor_value_to_struct(v: serde_cbor::Value) -> Result { 20 | Ok(serde_cbor::value::from_value(v)?) 21 | } 22 | 23 | /// Convert Obj Integer to Obj Float for matching golang version. 24 | pub fn hack_convert_int_to_float(value: IpldValue) -> Result { 25 | let mut value = value; 26 | let mut func = |obj: &mut IpldValue| match obj { 27 | IpldValue::Integer(ref mut i) => { 28 | // all integer would convert into f64 29 | *obj = IpldValue::Float(*i as f64); 30 | Ok(()) 31 | } 32 | _ => Ok(()), 33 | }; 34 | traverse_obj_tree(&mut value, &mut func)?; 35 | Ok(value) 36 | } 37 | 38 | /// Convert Obj Float to Obj Integer for matching golang version. 39 | pub fn hack_convert_float_to_int(value: IpldValue) -> Result { 40 | let mut value = value; 41 | let mut func = |obj: &mut IpldValue| match obj { 42 | IpldValue::Float(ref mut f) => { 43 | if f.fract() == 0.0 { 44 | *obj = IpldValue::Integer(*f as i128); 45 | } 46 | Ok(()) 47 | } 48 | _ => Ok(()), 49 | }; 50 | traverse_obj_tree(&mut value, &mut func)?; 51 | Ok(value) 52 | } 53 | 54 | /// 55 | pub fn convert_to_cborish_obj(mut value: IpldValue) -> Result { 56 | let mut func = |obj: &mut IpldValue| match obj { 57 | IpldValue::Map(ref mut map) => { 58 | if map.len() == 1 { 59 | if let Some(link) = map.get("/") { 60 | match link { 61 | IpldValue::Text(s) => { 62 | let cid = Cid::try_from(s.as_str())?; 63 | *obj = IpldValue::Link(cid); 64 | } 65 | IpldValue::Link(cid) => { 66 | *obj = IpldValue::Link(cid.clone()); 67 | } 68 | _ => return Err(IpldCborError::NonStringLink), // should not happen 69 | } 70 | } 71 | } 72 | Ok(()) 73 | } 74 | _ => Ok(()), 75 | }; 76 | traverse_obj_tree(&mut value, &mut func)?; 77 | Ok(value) 78 | } 79 | 80 | /// 81 | pub fn convert_to_jsonish_obj(mut value: IpldValue) -> Result { 82 | let mut func = |obj: &mut IpldValue| { 83 | match obj { 84 | // change cid to map { "/", "string" } 85 | IpldValue::Link(cid) => { 86 | let link = IpldValue::Text(cid.to_string()); 87 | let mut map = BTreeMap::new(); 88 | map.insert("/".to_string(), link); 89 | *obj = IpldValue::Map(map); 90 | Ok(()) 91 | } 92 | IpldValue::Map(ref mut map) => { 93 | // if current map is like: { "/", cid }, change it to { "/": "string" } 94 | if map.len() == 1 { 95 | if let Some(ref mut obj) = map.get("/") { 96 | match obj { 97 | IpldValue::Link(cid) => *obj = &IpldValue::Text(cid.to_string()), 98 | IpldValue::Text(_) => {} // do nothing, 99 | _ => return Err(IpldCborError::NonStringLink), // should not happen 100 | } 101 | } 102 | } 103 | Ok(()) 104 | } 105 | _ => Ok(()), 106 | } 107 | }; 108 | traverse_obj_tree(&mut value, &mut func)?; 109 | Ok(value) 110 | } 111 | 112 | fn traverse_obj_tree(obj: &mut IpldValue, f: &mut F) -> Result<()> 113 | where 114 | F: FnMut(&mut IpldValue) -> Result<()>, 115 | { 116 | f(obj)?; 117 | match obj { 118 | IpldValue::Array(ref mut arr) => { 119 | for v in arr.iter_mut() { 120 | traverse_obj_tree(v, f)?; 121 | } 122 | Ok(()) 123 | } 124 | IpldValue::Map(ref mut m) => { 125 | for v in m.values_mut() { 126 | traverse_obj_tree(v, f)?; 127 | } 128 | Ok(()) 129 | } 130 | _ => Ok(()), 131 | } 132 | } 133 | 134 | /// Convert obj into json string. 135 | /// Just for testing. Please use the `to_json` method of `IpldNode`. 136 | #[inline] 137 | pub fn obj_to_json(obj: IpldValue) -> Result { 138 | let json_obj = convert_to_jsonish_obj(obj)?; 139 | // hack handle for rust, to match go 140 | let json_obj = hack_convert_float_to_int(json_obj)?; 141 | Ok(serde_json::to_string(&json_obj)?) 142 | } 143 | 144 | /// Convert json string into Obj. 145 | /// Just for testing. Please use the `from_json` method of `IpldNode`. 146 | #[inline] 147 | pub fn json_to_obj(json: &str) -> Result { 148 | let obj = serde_json::from_str::(json)?; 149 | // hack handle for rust, to match go 150 | let obj = hack_convert_int_to_float(obj)?; 151 | convert_to_cborish_obj(obj) 152 | } 153 | -------------------------------------------------------------------------------- /ipld/core/src/error.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2020 PolkaX. Licensed under MIT or Apache-2.0. 2 | 3 | /// Type alias to use this library's [`IpldCborError`] type in a `Result`. 4 | pub type Result = std::result::Result; 5 | 6 | /// Errors generated from this library. 7 | #[derive(Debug, thiserror::Error)] 8 | pub enum IpldCoreError { 9 | /// JSON serialization/deserialization error. 10 | #[error("json de/serialize error: {0}")] 11 | JsonCodecErr(#[from] serde_json::Error), 12 | /// CBOR serialization/deserialization error. 13 | #[error("core encode error: {0}")] 14 | CborEncodeErr(#[from] minicbor::encode::Error), 15 | /// CBOR deserialization error. 16 | #[error("core decode error: {0}")] 17 | CborDecodeErr(#[from] minicbor::decode::Error), 18 | /// CID error. 19 | #[error("cid error: {0}")] 20 | CidErr(#[from] cid::Error), 21 | /// Block format error. 22 | #[error("block format error: {0}")] 23 | BlockErr(#[from] block_format::BlockFormatError), 24 | /// Multihash encode error. 25 | #[error("multi hash error: {0}")] 26 | HashErr(#[from] multihash::EncodeError), 27 | /// No such link found. 28 | #[error("no such link found, path: {0}")] 29 | NoSuchLink(String), 30 | /// Non-link found at given path. 31 | #[error("non-link found at given path")] 32 | NonLink, 33 | /// Invalid link. 34 | #[error("link value should have been bytes or cid")] 35 | InvalidLink, 36 | /// No links 37 | #[error("tried to resolve through object that had no links")] 38 | NoLinks, 39 | /// Link is not a string. 40 | #[error("link should have been a string")] 41 | NonStringLink, 42 | /// Deserialize CID error. 43 | #[error("deserialize cid failed, reason: {0}")] 44 | DeserializeCid(String), 45 | /// Failure when converting to Obj. 46 | #[error("Failure when converting to Obj, reason: {0}")] 47 | ObjErr(String), 48 | /// Other error. 49 | #[error("other error: {0}")] 50 | Other(#[from] Box), 51 | } 52 | -------------------------------------------------------------------------------- /ipld/core/src/lib.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2020 PolkaX. Licensed under MIT or Apache-2.0. 2 | 3 | //! A CBOR implementation of `ipld format` in Rust. 4 | 5 | #![deny(missing_docs)] 6 | 7 | mod error; 8 | mod node; 9 | mod value; 10 | 11 | pub use ipld_format::{FormatError, Link, Node, NodeStat, Resolver}; 12 | 13 | pub use self::error::{IpldCoreError, Result}; 14 | pub use self::node::IpldNode; 15 | pub use self::value::IpldValue; 16 | -------------------------------------------------------------------------------- /ipld/core/tests/test_objects/QmRgutAxd8t7oGkSm4wmeuByG6M51wcTso6cubDdQtuEfL: -------------------------------------------------------------------------------- 1 | foobar 2 | -------------------------------------------------------------------------------- /ipld/core/tests/test_objects/array-link.cbor: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PolkaX/rust-ipfs/e1817f2ab3767426c4f4607560be160c58578f35/ipld/core/tests/test_objects/array-link.cbor -------------------------------------------------------------------------------- /ipld/core/tests/test_objects/array-link.json: -------------------------------------------------------------------------------- 1 | [{"/":"QmRgutAxd8t7oGkSm4wmeuByG6M51wcTso6cubDdQtuEfL"}] 2 | -------------------------------------------------------------------------------- /ipld/core/tests/test_objects/empty-array.cbor: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PolkaX/rust-ipfs/e1817f2ab3767426c4f4607560be160c58578f35/ipld/core/tests/test_objects/empty-array.cbor -------------------------------------------------------------------------------- /ipld/core/tests/test_objects/empty-array.json: -------------------------------------------------------------------------------- 1 | [] 2 | -------------------------------------------------------------------------------- /ipld/core/tests/test_objects/empty-obj.cbor: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PolkaX/rust-ipfs/e1817f2ab3767426c4f4607560be160c58578f35/ipld/core/tests/test_objects/empty-obj.cbor -------------------------------------------------------------------------------- /ipld/core/tests/test_objects/empty-obj.json: -------------------------------------------------------------------------------- 1 | {} 2 | -------------------------------------------------------------------------------- /ipld/core/tests/test_objects/expected.json: -------------------------------------------------------------------------------- 1 | { 2 | "array-link": {"/":"bafyreidj7kdv2gkmhycdq5bagiloawte5z6egwztbwovsk6v3xookbpyuu"}, 3 | "empty-array": {"/":"bafyreidwx2fvfdiaox32v2mnn6sxu3j4qoxeqcuenhtgrv5qv6litfnmoe"}, 4 | "empty-obj": {"/":"bafyreigbtj4x7ip5legnfznufuopl4sg4knzc2cof6duas4b3q2fy6swua"}, 5 | "foo": {"/":"bafyreid56m6jpnmimcltiqpdaujthpqfl722vd4kr2xjp6rpuhx66iri64"}, 6 | "foo2": {"/":"bafyreid56m6jpnmimcltiqpdaujthpqfl722vd4kr2xjp6rpuhx66iri64"}, 7 | "obj-with-link": {"/":"bafyreibvjvcv745gig4mvqs4hctx4zfkono4rjejm2ta6gtyzkqxfjeily"}, 8 | "obj-no-link": {"/":"bafyreiduyr3p3ppndxmoc4q2it3vmezjx2ndt5oakudtplfgyyfb2k7swm"} 9 | } 10 | -------------------------------------------------------------------------------- /ipld/core/tests/test_objects/foo.cbor: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PolkaX/rust-ipfs/e1817f2ab3767426c4f4607560be160c58578f35/ipld/core/tests/test_objects/foo.cbor -------------------------------------------------------------------------------- /ipld/core/tests/test_objects/foo.json: -------------------------------------------------------------------------------- 1 | { 2 | "foo": "bar", 3 | "cats": [ 4 | {"/": "QmRgutAxd8t7oGkSm4wmeuByG6M51wcTso6cubDdQtuEfL"}, 5 | { 6 | "something": "interesting" 7 | }, 8 | [ 9 | "fish", 10 | {"/": "QmRgutAxd8t7oGkSm4wmeuByG6M51wcTso6cubDdQtuEfL"}, 11 | 9.0 12 | ] 13 | ], 14 | "other": {"/": "QmRgutAxd8t7oGkSm4wmeuByG6M51wcTso6cubDdQtuEfL"} 15 | } 16 | -------------------------------------------------------------------------------- /ipld/core/tests/test_objects/foo2.cbor: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PolkaX/rust-ipfs/e1817f2ab3767426c4f4607560be160c58578f35/ipld/core/tests/test_objects/foo2.cbor -------------------------------------------------------------------------------- /ipld/core/tests/test_objects/foo2.json: -------------------------------------------------------------------------------- 1 | { 2 | "other": {"/": "QmRgutAxd8t7oGkSm4wmeuByG6M51wcTso6cubDdQtuEfL"}, 3 | "cats": [ 4 | {"/": "QmRgutAxd8t7oGkSm4wmeuByG6M51wcTso6cubDdQtuEfL"}, 5 | { 6 | "something": "interesting" 7 | }, 8 | [ 9 | "fish", 10 | {"/": "QmRgutAxd8t7oGkSm4wmeuByG6M51wcTso6cubDdQtuEfL"}, 11 | 9.0 12 | ] 13 | ], 14 | "foo": "bar" 15 | } 16 | -------------------------------------------------------------------------------- /ipld/core/tests/test_objects/non-canon.cbor: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PolkaX/rust-ipfs/e1817f2ab3767426c4f4607560be160c58578f35/ipld/core/tests/test_objects/non-canon.cbor -------------------------------------------------------------------------------- /ipld/core/tests/test_objects/obj-no-link.cbor: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PolkaX/rust-ipfs/e1817f2ab3767426c4f4607560be160c58578f35/ipld/core/tests/test_objects/obj-no-link.cbor -------------------------------------------------------------------------------- /ipld/core/tests/test_objects/obj-no-link.json: -------------------------------------------------------------------------------- 1 | { 2 | "sassafras": "and cats" 3 | } 4 | -------------------------------------------------------------------------------- /ipld/core/tests/test_objects/obj-with-link.cbor: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PolkaX/rust-ipfs/e1817f2ab3767426c4f4607560be160c58578f35/ipld/core/tests/test_objects/obj-with-link.cbor -------------------------------------------------------------------------------- /ipld/core/tests/test_objects/obj-with-link.json: -------------------------------------------------------------------------------- 1 | { 2 | "foo": {"/":"QmRgutAxd8t7oGkSm4wmeuByG6M51wcTso6cubDdQtuEfL"} 3 | } 4 | -------------------------------------------------------------------------------- /ipld/format/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "ipld-format" 3 | version = "0.1.0" 4 | authors = ["PolkaX "] 5 | edition = "2018" 6 | 7 | license = "MIT/Apache-2.0" 8 | repository = "https://github.com/PolkaX/rust-ipfs" 9 | description = "Implementation of the ipld format" 10 | keywords = ["ipfs", "ipld", "format"] 11 | 12 | [dependencies] 13 | cid = { version = "0.5", features = ["cbor", "json"] } 14 | lazy_static = "1.4" 15 | multihash = "0.11" 16 | thiserror = "1.0" 17 | 18 | block-format = { path = "../../block-format" } 19 | 20 | [dev-dependencies] 21 | bytes = "0.5" 22 | -------------------------------------------------------------------------------- /ipld/format/src/coding.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2020 PolkaX. Licensed under MIT or Apache-2.0. 2 | 3 | use std::collections::HashMap; 4 | use std::sync::Arc; 5 | use std::sync::RwLock; 6 | 7 | use block_format::Block; 8 | use cid::Codec; 9 | 10 | use crate::error::{FormatError, Result}; 11 | use crate::format::Node; 12 | 13 | lazy_static::lazy_static! { 14 | static ref BLOCK_DECODERS: RwLock>> = RwLock::new(HashMap::new()); 15 | } 16 | 17 | type DecodeBlockFunc = dyn Fn(&dyn Block) -> Result> + Send + Sync; 18 | 19 | /// Register decoder for all blocks with the passed codec. 20 | /// 21 | /// This will silently replace any existing registered block decoders. 22 | pub fn register(codec: Codec, decoder: F) 23 | where 24 | F: Fn(&dyn Block) -> Result> + Send + Sync + 'static, 25 | { 26 | let mut block_decoders = BLOCK_DECODERS 27 | .write() 28 | .expect("get instance write lock failed"); 29 | block_decoders.insert(codec, Arc::new(decoder)); 30 | } 31 | 32 | /// Decode block into node with the decode function corresponding to the codec of the block's CID. 33 | pub fn decode(block: &impl Block) -> Result> { 34 | let codec = block.cid().codec(); 35 | let decoder_func = { 36 | // just get lock and release, let decode function could be parallel 37 | let block_decoders = BLOCK_DECODERS 38 | .read() 39 | .expect("get instance read lock failed"); 40 | // get a copy of arc pointer 41 | block_decoders 42 | .get(&codec) 43 | .ok_or(FormatError::DecoderNotRegister(codec))? 44 | .clone() 45 | }; 46 | decoder_func(block) 47 | } 48 | -------------------------------------------------------------------------------- /ipld/format/src/daghelpers.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2020 PolkaX. Licensed under MIT or Apache-2.0. 2 | -------------------------------------------------------------------------------- /ipld/format/src/error.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2020 PolkaX. Licensed under MIT or Apache-2.0. 2 | 3 | /// Type alias to use this library's [`FormatError`] type in a `Result`. 4 | pub type Result = std::result::Result; 5 | 6 | /// Errors generated from this library. 7 | #[derive(Debug, thiserror::Error)] 8 | pub enum FormatError { 9 | /// The object is not support statistics for a node. 10 | #[error("this obj can't return NodeStat")] 11 | NotSupportStat, 12 | /// Failed to decode block into node. 13 | #[error("decode block into node error")] 14 | DecodeError, 15 | /// Cannot find the decoder corresponding to codec. 16 | #[error("this code has not register decoder: {0:?}")] 17 | DecoderNotRegister(cid::Codec), 18 | /// More than the depth of path. 19 | #[error("depth is larger than path, depth: {0}, path len: {1}")] 20 | DepthError(usize, usize), 21 | /// Depth is not init yet. 22 | #[error("depth is not init yet")] 23 | DepthNotInit, 24 | /// Cannot go down, no child. 25 | #[error("can't go down, the child does not exist, depth: {0}, index: {1}, child: {0}")] 26 | DownNoChild(usize, usize, usize), 27 | /// Cannot go up, already on root. 28 | #[error("can't go up, already on root")] 29 | UpOnRoot, 30 | /// No more child nodes. 31 | #[error("can't go to the next child, no more child nodes in this parent")] 32 | NextNoChild, 33 | /// No child exist at the index. 34 | #[error("child not exist for this index. index: {0}")] 35 | NoChild(usize), 36 | /// Link not found. 37 | #[error("no such link found")] 38 | NoSuchLink, 39 | /// Other error. 40 | #[error("other err: {0}")] 41 | Other(Box), 42 | } 43 | -------------------------------------------------------------------------------- /ipld/format/src/format.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2020 PolkaX. Licensed under MIT or Apache-2.0. 2 | 3 | use block_format::Block; 4 | use cid::Cid; 5 | 6 | use crate::error::Result; 7 | use crate::merkledag::NodeGetter; 8 | 9 | /// Node must support deep copy 10 | /// Node is the base interface all IPLD nodes must implement. 11 | /// 12 | /// Nodes are **Immutable** and all methods defined on the interface are **Thread Safe**. 13 | pub trait Node: Block { 14 | /// A helper function that calls resolve and asserts the output is a link. 15 | fn resolve_link(&self, path: &[&str]) -> Result<(Link, Vec)>; 16 | 17 | /// A helper function that returns all links within this object. 18 | fn links(&self) -> Vec<&Link>; 19 | 20 | /// A helper function that returns `NodeStat` ref. 21 | fn stat(&self) -> Result<&NodeStat>; 22 | 23 | /// Returns the size in bytes of the serialized object. 24 | fn size(&self) -> u64; 25 | } 26 | 27 | /// Resolver is the interface that operate path. 28 | pub trait Resolver { 29 | /// The found object by resolving a path through this node. 30 | type Output; 31 | 32 | /// Resolves a path through this node, stopping at any link boundary 33 | /// and returning the object found as well as the remaining path to traverse. 34 | fn resolve(&self, path: &[&str]) -> Result<(Self::Output, Vec)>; 35 | 36 | /// Lists all paths within the object under 'path', and up to the given depth. 37 | /// To list the entire object (similar to `find .`) pass "" and None. 38 | fn tree(&self, path: &str, depth: Option) -> Vec; 39 | } 40 | 41 | /// Link represents an IPFS Merkle DAG Link between Nodes. 42 | #[derive(PartialEq, Eq, Clone, Debug)] 43 | pub struct Link { 44 | /// It should be unique per object. 45 | pub name: String, 46 | /// The cumulative size of target object. 47 | pub size: u64, 48 | /// The CID of the target object. 49 | pub cid: Cid, 50 | } 51 | 52 | impl Link { 53 | /// Create a new `Link` with the given CID. 54 | pub fn new_with_cid(cid: Cid) -> Link { 55 | Link { 56 | name: "".to_string(), 57 | size: 0, 58 | cid, 59 | } 60 | } 61 | 62 | /// Creates a `Link` with the given node. 63 | pub fn new_with_node(node: T) -> Link { 64 | Link { 65 | name: Default::default(), 66 | size: node.size(), 67 | cid: node.cid().clone(), 68 | } 69 | } 70 | 71 | /// Returns the MerkleDAG Node that this link points to. 72 | pub fn node(&self, ng: &impl NodeGetter) -> Result { 73 | Ok(ng.get(&self.cid)) 74 | } 75 | } 76 | 77 | /// NodeStat is a statistics object for a Node. Mostly sizes. 78 | pub struct NodeStat { 79 | /// The multihash of node. 80 | pub hash: multihash::Multihash, 81 | /// The number of links in link table. 82 | pub num_links: usize, 83 | /// The size of the raw, encoded data. 84 | pub block_size: usize, 85 | /// The size of the links segment. 86 | pub links_size: usize, 87 | /// The size of the data segment. 88 | pub data_size: usize, 89 | /// The cumulative size of object and its references. 90 | pub cumulative_size: usize, 91 | } 92 | 93 | impl std::fmt::Debug for NodeStat { 94 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 95 | f.debug_struct("NodeStat") 96 | .field("NumLinks", &self.num_links) 97 | .field("BlockSize", &self.block_size) 98 | .field("LinksSize", &self.links_size) 99 | .field("DataSize", &self.data_size) 100 | .field("CumulativeSize", &self.cumulative_size) 101 | .finish() 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /ipld/format/src/lib.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2020 PolkaX. Licensed under MIT or Apache-2.0. 2 | 3 | //! IPLD Node and Resolver interfaces in Rust. 4 | //! Port from tht [Go implementation](https://github.com/ipfs/go-ipld-format). 5 | 6 | #![deny(missing_docs)] 7 | #![allow(unused)] 8 | 9 | /// Provides `register` and `decode` methods. 10 | pub mod coding; 11 | mod daghelpers; 12 | mod error; 13 | mod format; 14 | mod merkledag; 15 | mod navipld; 16 | mod walker; 17 | 18 | pub use self::error::{FormatError, Result}; 19 | pub use self::format::{Link, Node, NodeStat, Resolver}; 20 | pub use self::merkledag::{DAGService, LinkGetter, NodeAdder, NodeGetter}; 21 | pub use self::navipld::NavigableIpldNode; 22 | pub use self::walker::{NavigableNode, Walker}; 23 | -------------------------------------------------------------------------------- /ipld/format/src/merkledag.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2020 PolkaX. Licensed under MIT or Apache-2.0. 2 | 3 | use cid::Cid; 4 | 5 | use crate::error::Result; 6 | use crate::format::{Link, Node}; 7 | 8 | /// The basic Node resolution service. 9 | pub trait NodeGetter { 10 | /// Get retrieves nodes by CID. Depending on the NodeGetter 11 | /// implementation, this may involve fetching the Node from a remote 12 | /// machine; consider setting a deadline to stop it. 13 | fn get(&self, cid: &Cid) -> T; 14 | } 15 | 16 | /// NodeAdder adds nodes to a DAG. 17 | pub trait NodeAdder { 18 | /// Add adds a node to this DAG. 19 | fn add(&self, node: T); 20 | } 21 | 22 | /// NodeGetters can optionally implement this interface to make finding linked objects faster. 23 | pub trait LinkGetter: NodeGetter { 24 | /// returns the children of the node referred to by the given CID. 25 | fn get_links(node: &Cid) -> Result>; 26 | } 27 | 28 | /// DAGService is an IPFS Merkle DAG service. 29 | pub trait DAGService: NodeGetter + NodeAdder { 30 | /// remove a node, referred to by the given CID, from this DAG. 31 | fn remove(cid: &Cid) -> Result<()>; 32 | } 33 | -------------------------------------------------------------------------------- /ipld/format/src/navipld.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2020 PolkaX. Licensed under MIT or Apache-2.0. 2 | 3 | use cid::Cid; 4 | 5 | use crate::format::Node; 6 | use crate::merkledag::NodeGetter; 7 | 8 | /// NavigableIPLDNode implements the `NavigableNode` interface wrapping an IPLD `Node`. 9 | pub struct NavigableIpldNode> { 10 | node: N, 11 | node_getter: NG, 12 | child_cids: Vec, 13 | } 14 | 15 | impl> NavigableIpldNode { 16 | /// Create a `NavigableIpldNode` wrapping the provided `node`. 17 | pub fn new(node: N, node_getter: NG) -> Self { 18 | let child_cids = node 19 | .links() 20 | .into_iter() 21 | .map(|link| link.cid.clone()) 22 | .collect(); 23 | NavigableIpldNode { 24 | node, 25 | node_getter, 26 | child_cids, 27 | } 28 | } 29 | 30 | /// Return the IPLD `Node` wrapped into this structure. 31 | pub fn ipld_node(&self) -> &N { 32 | &self.node 33 | } 34 | 35 | /// Return the number of links (of child nodes) in this node. 36 | pub fn child_total(&self) -> usize { 37 | self.node.links().len() 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /ipld/format/src/walker.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2020 PolkaX. Licensed under MIT or Apache-2.0. 2 | 3 | use std::sync::Arc; 4 | 5 | use crate::error::{FormatError, Result}; 6 | 7 | /// An interface the nodes of a DAG need to implement in order to be traversed by the `Walker`. 8 | pub trait NavigableNode { 9 | /// Returns the number of children of the `ActiveNode`. 10 | fn child_total(&self) -> usize; 11 | 12 | /// Returns the child of this node pointed to by `child_index`. 13 | fn fetch_child(&self, child_index: usize) -> Result>; 14 | } 15 | 16 | /// Walker provides methods to move through a DAG of nodes that implement the `NavigableNode` 17 | /// interface. It uses iterative algorithms (instead of recursive ones) that expose the `path` 18 | /// of nodes from the root to the `active_node` it currently points to. 19 | /// 20 | /// It provides multiple ways to walk through the DAG. When using them, 21 | /// you provide a Visitor function that will be called for each node the Walker traverses. 22 | /// The Visitor can read data from those nodes and, optionally, direct the movement of the Walker. 23 | pub struct Walker { 24 | stack: Vec>, 25 | } 26 | 27 | impl Walker { 28 | /// Creates a new `Walker` from a `root` NavigableNode. 29 | pub fn new(root: Arc) -> Walker { 30 | Walker { stack: vec![root] } 31 | } 32 | 33 | /// Returns the `NavigableNode` that `Walker` is pointing 34 | /// to at the moment. It changes when `up` or `down` is called. 35 | pub fn active_node(&self) -> Result<&Arc> { 36 | self.stack.last().ok_or(FormatError::NextNoChild) 37 | } 38 | } 39 | 40 | impl Iterator for Walker { 41 | type Item = Arc; 42 | 43 | fn next(&mut self) -> Option { 44 | let node = self.stack.pop()?; 45 | for i in (0..node.child_total()).rev() { 46 | let node = node.fetch_child(i).ok()?; 47 | self.stack.push(node); 48 | } 49 | Some(node) 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /ipld/format/tests/common.rs: -------------------------------------------------------------------------------- 1 | use std::sync::Arc; 2 | 3 | use block_format::Block; 4 | use bytes::Bytes; 5 | use cid::{Cid, Codec, IntoExt}; 6 | 7 | use ipld_format::{FormatError, Link, NavigableNode, Node, NodeStat, Resolver, Result}; 8 | 9 | pub struct EmptyNode { 10 | cid: Cid, 11 | data: Bytes, 12 | } 13 | 14 | impl Default for EmptyNode { 15 | fn default() -> Self { 16 | EmptyNode { 17 | cid: Cid::new_v1(Codec::Raw, multihash::Identity::digest(b"").into_ext()), 18 | data: Bytes::from_static(b""), 19 | } 20 | } 21 | } 22 | 23 | impl EmptyNode { 24 | pub fn new() -> Self { 25 | Self::default() 26 | } 27 | } 28 | 29 | impl Node for EmptyNode { 30 | fn resolve_link(&self, _path: &[&str]) -> Result<(Link, Vec)> { 31 | unimplemented!() 32 | } 33 | 34 | fn links(&self) -> Vec<&Link> { 35 | unimplemented!() 36 | } 37 | 38 | fn stat(&self) -> Result<&NodeStat> { 39 | unimplemented!() 40 | } 41 | 42 | fn size(&self) -> u64 { 43 | unimplemented!() 44 | } 45 | } 46 | 47 | impl Block for EmptyNode { 48 | fn raw_data(&self) -> &Bytes { 49 | &self.data 50 | } 51 | } 52 | 53 | impl AsRef for EmptyNode { 54 | fn as_ref(&self) -> &Cid { 55 | &self.cid 56 | } 57 | } 58 | 59 | impl Resolver for EmptyNode { 60 | type Output = (); 61 | fn resolve(&self, _path: &[&str]) -> Result<(Self::Output, Vec)> { 62 | unimplemented!() 63 | } 64 | 65 | fn tree(&self, _path: &str, _depth: Option) -> Vec { 66 | unimplemented!() 67 | } 68 | } 69 | 70 | pub struct N { 71 | pub inner: EmptyNode, 72 | pub child: Vec>, 73 | } 74 | 75 | impl NavigableNode for N { 76 | fn child_total(&self) -> usize { 77 | self.child.len() 78 | } 79 | 80 | fn fetch_child(&self, child_index: usize) -> Result> { 81 | self.child 82 | .get(child_index) 83 | .cloned() 84 | .ok_or(FormatError::NoChild(child_index)) 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /ipld/format/tests/test_coding.rs: -------------------------------------------------------------------------------- 1 | mod common; 2 | 3 | use block_format::{BasicBlock, Block}; 4 | use cid::{Cid, Codec, IntoExt}; 5 | use ipld_format::coding::{decode, register}; 6 | use ipld_format::{Node, Result}; 7 | 8 | use self::common::EmptyNode; 9 | 10 | // coding 11 | fn init() { 12 | register(Codec::Raw, |_block| { 13 | let node = EmptyNode::new(); 14 | Ok(Box::new(node)) 15 | }); 16 | } 17 | 18 | fn decode_fu(_block: &dyn Block) -> Result> { 19 | let node = EmptyNode::new(); 20 | Ok(Box::new(node)) 21 | } 22 | 23 | fn init2() { 24 | register(Codec::Raw, decode_fu); 25 | } 26 | 27 | #[test] 28 | fn test_decode() { 29 | init(); 30 | let id = Cid::new_v1(Codec::Raw, multihash::Identity::digest(b"").into_ext()); 31 | let block = BasicBlock::new_with_cid(vec![].into(), id.clone()).unwrap(); 32 | let node = decode(&block).unwrap(); 33 | assert_eq!(node.cid(), &id); 34 | } 35 | 36 | #[test] 37 | fn test_init() { 38 | init2(); 39 | } 40 | -------------------------------------------------------------------------------- /ipld/format/tests/test_walker.rs: -------------------------------------------------------------------------------- 1 | mod common; 2 | 3 | use std::sync::Arc; 4 | 5 | use ipld_format::Walker; 6 | 7 | use self::common::{EmptyNode, N}; 8 | 9 | #[test] 10 | fn test_walker() { 11 | let n2_2 = Arc::new(N { 12 | inner: EmptyNode::new(), 13 | child: vec![], 14 | }); 15 | let n2_1 = Arc::new(N { 16 | inner: EmptyNode::new(), 17 | child: vec![], 18 | }); 19 | let n2 = Arc::new(N { 20 | inner: EmptyNode::new(), 21 | child: vec![n2_1, n2_2], 22 | }); 23 | let n1_1 = Arc::new(N { 24 | inner: EmptyNode::new(), 25 | child: vec![], 26 | }); 27 | let n1 = Arc::new(N { 28 | inner: EmptyNode::new(), 29 | child: vec![n1_1], 30 | }); 31 | let root = Arc::new(N { 32 | inner: EmptyNode::new(), 33 | child: vec![n1, n2], 34 | }); 35 | 36 | let counter = Walker::new(root).count(); 37 | assert_eq!(counter, 6); 38 | 39 | // root -> 1 -> 3 40 | // -> 2 ---^ 41 | // 3 should be seek twice 42 | let n3 = Arc::new(N { 43 | inner: EmptyNode::new(), 44 | child: vec![], 45 | }); 46 | let n2 = Arc::new(N { 47 | inner: EmptyNode::new(), 48 | child: vec![n3.clone()], 49 | }); 50 | let n1 = Arc::new(N { 51 | inner: EmptyNode::new(), 52 | child: vec![n3], 53 | }); 54 | let root = Arc::new(N { 55 | inner: EmptyNode::new(), 56 | child: vec![n1, n2], 57 | }); 58 | 59 | let counter = Walker::new(root).count(); 60 | assert_eq!(counter, 5); 61 | } 62 | -------------------------------------------------------------------------------- /ipld/hamt/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "ipld-hamt" 3 | version = "0.1.0" 4 | authors = ["PolkaX "] 5 | edition = "2018" 6 | 7 | license = "MIT/Apache-2.0" 8 | repository = "https://github.com/PolkaX/rust-ipfs" 9 | description = "Implementation of the ipld hamt" 10 | keywords = ["ipfs", "ipld", "hamt"] 11 | 12 | [dependencies] 13 | bigint = "4.4" 14 | bytes = { version = "0.5", features = ["serde"] } 15 | cid = { version = "0.5", features = ["cbor", "json"] } 16 | multihash = "0.11" 17 | murmur3 = "0.5" 18 | serde = { version = "1.0", features = ["derive"] } 19 | serde_bytes = "0.11" 20 | serde_cbor = { version = "0.11", features = ["tags"] } 21 | thiserror = "1.0" 22 | 23 | block-format = { path = "../../block-format" } 24 | ipld-core = { path = "../core" } 25 | 26 | [dev-dependencies] 27 | matches = "0.1" 28 | rand = "0.7" 29 | serde_json = "1.0" 30 | 31 | [features] 32 | test-hash = [] 33 | -------------------------------------------------------------------------------- /ipld/hamt/src/error.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2020 PolkaX. Licensed under MIT or Apache-2.0. 2 | 3 | use cid::Cid; 4 | use ipld_core::IpldCoreError; 5 | 6 | pub type Result = std::result::Result; 7 | 8 | #[derive(Debug, thiserror::Error)] 9 | pub enum Error { 10 | #[error("not found for this cid: {0:?}")] 11 | NotFoundForCid(Cid), 12 | 13 | #[error("ipld core error: {0:?}")] 14 | IpldCbor(#[from] IpldCoreError), 15 | 16 | #[error("reach hash buf max depth, attempted to traverse hamt beyond max depth")] 17 | MaxDepth, 18 | 19 | #[error("not found for key: {0}")] 20 | NotFound(String), 21 | 22 | #[error("incorrectly formed HAMT, corrupted some where")] 23 | InvalidFormatHAMT, 24 | 25 | #[error("other err: {0}")] 26 | Other(#[from] Box), 27 | } 28 | -------------------------------------------------------------------------------- /ipld/hamt/src/hash.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2020 PolkaX. Licensed under MIT or Apache-2.0. 2 | 3 | use std::cmp::Ordering; 4 | 5 | /// ```go 6 | /// func (d *digest64) Sum64() uint64 { 7 | /// h1, _ = (*digest128)(d).Sum128() 8 | /// return h1 9 | /// } 10 | /// ``` 11 | /// murmur3 hash for a bytes value. using hash128 but just pick half for result 12 | #[cfg(not(feature = "test-hash"))] 13 | pub fn hash>(v: T) -> [u8; 8] { 14 | let result = 15 | murmur3::murmur3_x64_128(&mut v.as_ref(), 0).expect("murmur3 hash shouldn't be fail"); 16 | // to big-ending sequence 17 | let all: [u8; 16] = result.to_be_bytes(); 18 | // digest64 is half a digest128. 19 | let mut bytes = [0_u8; 8]; 20 | // drop other half 21 | bytes.copy_from_slice(&all[8..]); 22 | bytes 23 | } 24 | 25 | /// replace hash function. jus for testing 26 | /// `identityHash` just copy v to output 27 | #[cfg(feature = "test-hash")] 28 | pub fn hash>(v: T) -> [u8; 32] { 29 | let mut bytes = [0_u8; 32]; 30 | for (index, byte) in v.as_ref().iter().take(32).enumerate() { 31 | bytes[index] = *byte; 32 | } 33 | bytes 34 | } 35 | 36 | /// hashBits is a helper that allows the reading of the 'next n bits' as an integer. 37 | /// e.g. bytes: [1, 66, 3], ([0b00000001, 0b01000010, 0b00000011]), read 10 bits would like 38 | /// [0b________, 0b__0000010, 0b00000011], and return 0b0000000001 = 2 (u32) 39 | #[derive(Debug, Clone)] 40 | pub struct HashBits<'a> { 41 | b: &'a [u8], 42 | consumed: u32, 43 | pub bit_width: u32, 44 | } 45 | 46 | impl<'a> HashBits<'a> { 47 | pub fn new(buf: &'a [u8], bit_width: u32) -> HashBits<'a> { 48 | Self::new_with_consumed(buf, 0, bit_width) 49 | } 50 | 51 | pub fn new_with_consumed(buf: &'a [u8], consumed: u32, bit_width: u32) -> HashBits<'a> { 52 | HashBits { 53 | b: buf, 54 | consumed, 55 | bit_width, 56 | } 57 | } 58 | 59 | pub fn consumed(&self) -> u32 { 60 | self.consumed 61 | } 62 | 63 | /// Next returns the next 'i' bits of the hashBits value as an u32, 64 | /// or `None `if there aren't enough bits. 65 | pub fn next(&mut self) -> Option { 66 | let i = self.bit_width; 67 | let new_consumed = self.consumed.checked_add(i)?; 68 | if new_consumed > self.b.len() as u32 * 8 { 69 | return None; 70 | } 71 | // return value is u32, couldn't pick over 32 72 | if i > 32 || i == 0 { 73 | return None; 74 | } 75 | 76 | let out = self.next_bit(i); 77 | Some(out) 78 | } 79 | 80 | fn next_bit(&mut self, i: u32) -> u32 { 81 | let cur_byte_index = (self.consumed / 8) as usize; 82 | let left_bit = 8 - (self.consumed % 8); // consumed % 8, left_bit is less and equal than 8 83 | 84 | let cur_byte = self.b[cur_byte_index]; 85 | match i.cmp(&left_bit) { 86 | Ordering::Equal => { 87 | // i and left_bit must less or equal than 8 88 | let out = mkmask(i) & cur_byte; 89 | self.consumed += i; 90 | out as u32 91 | } 92 | Ordering::Less => { 93 | // i must less than 8, left_bit must less or equal than 8 94 | // e.g. cur_byte: 0b11111111, self.consumed % 8=1, left_bit=7, i=2, then: 95 | // a=0b_1111111 96 | let a = cur_byte & mkmask(left_bit); // mask out the high bits we don't want, do not need consumed bits 97 | // b=0b_11_____ 98 | let b = a & (!mkmask(left_bit - i)); // mask out the low bits we don't want, do not need unused bits 99 | // c=0b______11 100 | let c = b as u32 >> (left_bit - i); // shift whats left down 101 | self.consumed += i; 102 | c 103 | } 104 | Ordering::Greater => { 105 | // must beyond current byte, pick all left_bit 106 | let mut out = (mkmask(left_bit) & cur_byte) as u32; 107 | out <<= i - left_bit; 108 | self.consumed += left_bit; 109 | out += self.next_bit(i - left_bit); 110 | out 111 | } 112 | } 113 | } 114 | } 115 | 116 | #[inline] 117 | fn mkmask(n: u32) -> u8 { 118 | ((1_u32 << n) - 1) as u8 119 | } 120 | -------------------------------------------------------------------------------- /ipld/hamt/src/ipld.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2020 PolkaX. Licensed under MIT or Apache-2.0. 2 | 3 | use block_format::Block as BlockT; 4 | use cid::{Cid, Codec}; 5 | use serde::{de::DeserializeOwned, Serialize}; 6 | 7 | use crate::error::*; 8 | 9 | pub trait Blockstore { 10 | fn get(&self, cid: &Cid) -> Result>; 11 | fn put(&mut self, block: impl BlockT) -> Result<()>; 12 | } 13 | 14 | pub trait CborIpldStore { 15 | fn get(&self, c: &Cid) -> Result; 16 | fn put(&mut self, v: T) -> Result; 17 | } 18 | 19 | /// A trait that represents whether a CID exists. 20 | pub trait HasCid { 21 | /// Whether a CID exists. 22 | fn has_cid(&self) -> Option<&Cid>; 23 | } 24 | 25 | impl HasCid for T { 26 | default fn has_cid(&self) -> Option<&Cid> { 27 | None 28 | } 29 | } 30 | 31 | impl> HasCid for T { 32 | fn has_cid(&self) -> Option<&Cid> { 33 | Some(self.as_ref()) 34 | } 35 | } 36 | 37 | #[derive(Debug)] 38 | #[cfg_attr(test, derive(Clone))] 39 | pub struct BasicCborIpldStore { 40 | blocks: B, 41 | } 42 | 43 | impl BasicCborIpldStore { 44 | pub fn new(b: B) -> Self { 45 | BasicCborIpldStore { blocks: b } 46 | } 47 | } 48 | impl CborIpldStore for BasicCborIpldStore { 49 | fn get(&self, c: &Cid) -> Result { 50 | let blk = self.blocks.get(c)?; 51 | let data = (*blk).raw_data(); 52 | let r = ipld_core::decode_into(data)?; 53 | Ok(r) 54 | } 55 | 56 | fn put(&mut self, v: T) -> Result { 57 | let mut hash_type = multihash::Code::Blake2b256; 58 | let mut codec = Codec::DagCBOR; 59 | 60 | // if this type has cid, would use this cid config 61 | let exp_cid_hash = if let Some(cid) = v.has_cid() { 62 | let perf = cid.prefix(); 63 | hash_type = perf.mh_type; 64 | codec = perf.codec; 65 | Some(cid.hash().to_owned()) 66 | } else { 67 | None 68 | }; 69 | 70 | let node = ipld_core::IpldNode::from_object_with_codec(v, hash_type, codec)?; 71 | let cid = node.cid().clone(); // this cid is calc from node 72 | self.blocks.put(node)?; 73 | 74 | if let Some(hash) = exp_cid_hash { 75 | // if has expected cid, then this expected hash 76 | assert_eq!(hash, cid.hash()); 77 | } 78 | 79 | Ok(cid) 80 | } 81 | } 82 | 83 | pub fn cst_from_bstore(bs: BS) -> BasicCborIpldStore { 84 | BasicCborIpldStore::new(bs) 85 | } 86 | -------------------------------------------------------------------------------- /ipld/hamt/src/lib.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2020 PolkaX. Licensed under MIT or Apache-2.0. 2 | 3 | //! A implementation of `ipld hamt` in Rust. 4 | 5 | #![allow(clippy::bool_comparison, clippy::type_complexity, clippy::or_fun_call)] 6 | #![feature(specialization)] 7 | 8 | mod error; 9 | mod hash; 10 | mod ipld; 11 | pub mod node; 12 | #[cfg(test)] 13 | mod tests; 14 | 15 | pub use self::ipld::{cst_from_bstore, BasicCborIpldStore, Blockstore, CborIpldStore, HasCid}; 16 | pub use self::node::{Hamt, DEFAULT_BIT_WIDTH}; 17 | -------------------------------------------------------------------------------- /ipld/hamt/src/node/trait_impl.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2020 PolkaX. Licensed under MIT or Apache-2.0. 2 | 3 | use std::collections::BTreeMap; 4 | use std::fmt; 5 | 6 | use bigint::U256; 7 | use cid::Cid; 8 | use serde::{de, ser, Deserialize, Serialize}; 9 | 10 | use super::{Hamt, Item, Node, KVT}; 11 | use crate::ipld::CborIpldStore; 12 | 13 | impl PartialEq for Hamt 14 | where 15 | B: CborIpldStore, 16 | { 17 | fn eq(&self, other: &Self) -> bool { 18 | self.root == other.root && self.bit_width == other.bit_width 19 | } 20 | } 21 | 22 | impl Eq for Hamt where B: CborIpldStore {} 23 | 24 | impl fmt::Debug for Hamt 25 | where 26 | B: CborIpldStore, 27 | { 28 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 29 | write!( 30 | f, 31 | "Hamt{{ root: {:?},\n bit_width:{:}}}", 32 | self.root, self.bit_width 33 | ) 34 | } 35 | } 36 | 37 | impl ser::Serialize for Node { 38 | fn serialize(&self, serializer: S) -> Result 39 | where 40 | S: ser::Serializer, 41 | { 42 | let mut bitmap_bytes = [0_u8; std::mem::size_of::()]; // u256 43 | self.bitfield.to_big_endian(&mut bitmap_bytes); 44 | // remove left 0 bytes, if all is 0, means an empty "" bytes. 45 | let index = bitmap_bytes 46 | .iter() 47 | .position(|i| *i != 0) 48 | .unwrap_or_else(|| std::mem::size_of_val(&self.bitfield)); 49 | let b = serde_bytes::Bytes::new(&bitmap_bytes[index..]); 50 | let tuple = (b, &self.items); 51 | tuple.serialize(serializer) 52 | } 53 | } 54 | 55 | impl<'de> de::Deserialize<'de> for Node { 56 | fn deserialize(deserializer: D) -> Result 57 | where 58 | D: de::Deserializer<'de>, 59 | { 60 | struct TupleVisitor; 61 | impl<'de> de::Visitor<'de> for TupleVisitor { 62 | type Value = (serde_bytes::ByteBuf, Vec); 63 | 64 | fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { 65 | write!(formatter, "tuple must be 2 item, bytes and Vec") 66 | } 67 | fn visit_seq(self, mut seq: A) -> Result 68 | where 69 | A: de::SeqAccess<'de>, 70 | { 71 | let first = seq 72 | .next_element()? 73 | .ok_or_else(|| de::Error::invalid_length(0, &self))?; 74 | let second = seq 75 | .next_element()? 76 | .ok_or_else(|| de::Error::invalid_length(1, &self))?; 77 | Ok((first, second)) 78 | } 79 | } 80 | let (byte_buf, items) = deserializer.deserialize_tuple(2, TupleVisitor)?; 81 | 82 | // it's big ending bytes, we copy value from end. 83 | // the buf is size of `u64` u8 array, notice could not out of bounds. 84 | let mut buf = [0_u8; std::mem::size_of::()]; 85 | let mut index = std::mem::size_of::(); 86 | for i in byte_buf.iter().rev() { 87 | index -= 1; 88 | buf[index] = *i; 89 | if index == 0 { 90 | break; 91 | } 92 | } 93 | // U256 receipt a big ending array 94 | let bitfield = buf.into(); 95 | Ok(Node::from_raw(bitfield, items)) 96 | } 97 | } 98 | 99 | #[derive(Serialize)] 100 | struct CborCid<'a>(#[serde(serialize_with = "cid::ipld_dag_cbor::serialize")] &'a Cid); 101 | 102 | impl ser::Serialize for Item { 103 | fn serialize(&self, serializer: S) -> Result 104 | where 105 | S: ser::Serializer, 106 | { 107 | match self { 108 | Item::Link(cid) => { 109 | let mut m = BTreeMap::new(); 110 | m.insert("0", CborCid(cid)); 111 | m.serialize(serializer) 112 | } 113 | Item::Leaf(kvs) => { 114 | let mut m = BTreeMap::new(); 115 | m.insert("1", kvs.iter().collect::>()); 116 | m.serialize(serializer) 117 | } 118 | Item::Ptr(_) => unreachable!("should not happen, could not serialize a node ptr"), 119 | } 120 | } 121 | } 122 | 123 | #[derive(Deserialize)] 124 | enum ItemRef { 125 | #[serde(rename = "0")] 126 | #[serde(deserialize_with = "cid::ipld_dag_cbor::deserialize")] 127 | Link(Cid), 128 | #[serde(rename = "1")] 129 | KVs(Vec), 130 | } 131 | 132 | impl<'de> de::Deserialize<'de> for Item { 133 | fn deserialize(deserializer: D) -> Result 134 | where 135 | D: de::Deserializer<'de>, 136 | { 137 | let item_ref = ItemRef::deserialize(deserializer)?; 138 | let i = match item_ref { 139 | ItemRef::Link(cid) => Item::from_link(cid), 140 | ItemRef::KVs(kvs) => Item::from_kvs(kvs), 141 | }; 142 | Ok(i) 143 | } 144 | } 145 | -------------------------------------------------------------------------------- /ipld/hamt/src/tests/cbor_test.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2020 PolkaX. Licensed under MIT or Apache-2.0. 2 | 3 | use super::*; 4 | 5 | use bytes::Bytes; 6 | use cid::{Cid, IntoExt}; 7 | use ipld_core::struct_to_cbor_value; 8 | 9 | #[test] 10 | fn test_kv() { 11 | let b: Bytes = vec![1_u8, 2, 3].into(); 12 | let v = struct_to_cbor_value(&b).unwrap(); 13 | let kv: KVT = ("123".to_string(), v); 14 | 15 | let r = serde_cbor::to_vec(&kv).unwrap(); 16 | println!("{:?}", r); 17 | 18 | let result = vec![130_u8, 99, 49, 50, 51, 67, 1, 2, 3]; 19 | assert_eq!(r, result); 20 | 21 | let kv2: KVT = serde_cbor::from_slice(&r).unwrap(); 22 | assert_eq!(kv, kv2); 23 | } 24 | 25 | #[test] 26 | fn test_pointer_and_node() { 27 | let b: Bytes = vec![1_u8, 2, 3].into(); 28 | let v = struct_to_cbor_value(&b).unwrap(); 29 | let kv: KVT = ("123".to_string(), v.clone()); 30 | let kv2: KVT = ("124".to_string(), v); 31 | let pointer = Item::from_kvs(vec![kv, kv2]); 32 | let r = serde_cbor::to_vec(&pointer).unwrap(); 33 | println!("{:?}", r); 34 | assert_eq!( 35 | r, 36 | vec![161, 97, 49, 130, 130, 99, 49, 50, 51, 67, 1, 2, 3, 130, 99, 49, 50, 52, 67, 1, 2, 3] 37 | ); 38 | 39 | let p2: Item = serde_cbor::from_slice(&r).unwrap(); 40 | assert_eq!(p2, pointer); 41 | 42 | let cid = Cid::new_v0(multihash::Sha2_256::digest(b"something").into_ext()).unwrap(); 43 | let pointer2 = Item::from_link(cid); 44 | let r = serde_cbor::to_vec(&pointer2).unwrap(); 45 | println!("{:?}", r); 46 | assert_eq!( 47 | r, 48 | vec![ 49 | 161, 97, 48, 216, 42, 88, 35, 0, 18, 32, 63, 201, 182, 137, 69, 157, 115, 143, 140, 50 | 136, 163, 164, 138, 169, 227, 53, 66, 1, 107, 122, 64, 82, 224, 1, 170, 165, 54, 252, 51 | 167, 72, 19, 203 52 | ] 53 | ); 54 | 55 | // bitfield is 0 56 | let node = test_node("0", vec![pointer.clone(), pointer2.clone()]); 57 | let r = serde_cbor::to_vec(&node).unwrap(); 58 | println!("{:?}", r); 59 | assert_eq!( 60 | r, 61 | vec![ 62 | 130, 64, 130, 161, 97, 49, 130, 130, 99, 49, 50, 51, 67, 1, 2, 3, 130, 99, 49, 50, 52, 63 | 67, 1, 2, 3, 161, 97, 48, 216, 42, 88, 35, 0, 18, 32, 63, 201, 182, 137, 69, 157, 115, 64 | 143, 140, 136, 163, 164, 138, 169, 227, 53, 66, 1, 107, 122, 64, 82, 224, 1, 170, 165, 65 | 54, 252, 167, 72, 19, 203 66 | ] 67 | ); 68 | let node: Node = serde_cbor::from_slice(&r).unwrap(); 69 | println!("{:?}", node); 70 | 71 | // bitfield is 9999 72 | let node = test_node("9999", vec![pointer.clone(), pointer2.clone()]); 73 | let r = serde_cbor::to_vec(&node).unwrap(); 74 | println!("{:?}", r); 75 | assert_eq!( 76 | r, 77 | vec![ 78 | 130, 66, 39, 15, 130, 161, 97, 49, 130, 130, 99, 49, 50, 51, 67, 1, 2, 3, 130, 99, 49, 79 | 50, 52, 67, 1, 2, 3, 161, 97, 48, 216, 42, 88, 35, 0, 18, 32, 63, 201, 182, 137, 69, 80 | 157, 115, 143, 140, 136, 163, 164, 138, 169, 227, 53, 66, 1, 107, 122, 64, 82, 224, 1, 81 | 170, 165, 54, 252, 167, 72, 19, 203 82 | ] 83 | ); 84 | let node: Node = serde_cbor::from_slice(&r).unwrap(); 85 | println!("{:?}", node); 86 | 87 | // bitfield is 0x12345678 88 | let node = test_node("305419896", vec![pointer.clone(), pointer2.clone()]); 89 | let r = serde_cbor::to_vec(&node).unwrap(); 90 | println!("{:?}", r); 91 | assert_eq!( 92 | r, 93 | vec![ 94 | 130, 68, 18, 52, 86, 120, 130, 161, 97, 49, 130, 130, 99, 49, 50, 51, 67, 1, 2, 3, 130, 95 | 99, 49, 50, 52, 67, 1, 2, 3, 161, 97, 48, 216, 42, 88, 35, 0, 18, 32, 63, 201, 182, 96 | 137, 69, 157, 115, 143, 140, 136, 163, 164, 138, 169, 227, 53, 66, 1, 107, 122, 64, 82, 97 | 224, 1, 170, 165, 54, 252, 167, 72, 19, 203 98 | ] 99 | ); 100 | let node: Node = serde_cbor::from_slice(&r).unwrap(); 101 | println!("{:?}", node); 102 | 103 | let node = test_node( 104 | "11579208923731619542357098500868790785326998466564056403945758400791312", 105 | vec![pointer, pointer2], 106 | ); 107 | let r = serde_cbor::to_vec(&node).unwrap(); 108 | let node: Node = serde_cbor::from_slice(&r).unwrap(); 109 | println!("{:?}", node); 110 | } 111 | -------------------------------------------------------------------------------- /ipld/hamt/src/tests/hamt_test.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2020 PolkaX. Licensed under MIT or Apache-2.0. 2 | 3 | use std::time::Instant; 4 | 5 | use matches::matches; 6 | use rand::distributions::Alphanumeric; 7 | use rand::{thread_rng, Rng}; 8 | 9 | use super::*; 10 | 11 | #[cfg(not(feature = "test-hash"))] 12 | fn rand_string() -> String { 13 | let rand_string: String = thread_rng().sample_iter(&Alphanumeric).take(18).collect(); 14 | rand_string 15 | } 16 | 17 | fn rand_value() -> Vec { 18 | let rand_string: String = thread_rng().sample_iter(&Alphanumeric).take(30).collect(); 19 | rand_string.into_bytes() 20 | } 21 | 22 | #[cfg(feature = "test-hash")] 23 | fn node_equal(h1: &mut Hamt, h2: &mut Hamt) 24 | where 25 | B: CborIpldStore, 26 | { 27 | let cid1 = h1.flush().unwrap(); 28 | let cid2 = h2.flush().unwrap(); 29 | assert_eq!(cid1, cid2); 30 | } 31 | 32 | #[cfg(feature = "test-hash")] 33 | fn add_and_remove_keys(bit_width: u32, keys: &[&str], extra_keys: &[&str]) { 34 | let all: Vec<(&str, Vec)> = keys.iter().map(|k| (*k, rand_value())).collect(); 35 | 36 | let cs = new_cbor_store(); 37 | let mut begin_node = Hamt::new_with_bitwidth(cs.clone(), bit_width); 38 | for (k, v) in all.iter() { 39 | begin_node.set(k, v.clone()).unwrap(); 40 | } 41 | println!("start flush"); 42 | let now = Instant::now(); 43 | let cid = begin_node.flush().unwrap(); 44 | println!("flush took: {}", now.elapsed().as_nanos()); 45 | 46 | let mut node = Hamt::load_with_bitwidth(cs.clone(), &cid, bit_width).unwrap(); 47 | 48 | for (k, v) in all { 49 | let v2: Vec = node.find(k).unwrap(); 50 | assert_eq!(v, v2); 51 | } 52 | 53 | // create second hamt by adding and deleting the extra keys 54 | for k in extra_keys.iter() { 55 | begin_node.set(k, rand_value()).unwrap(); 56 | } 57 | for k in extra_keys.iter() { 58 | begin_node.delete(k).unwrap(); 59 | } 60 | let cid2 = begin_node.flush().unwrap(); 61 | let mut node2 = Hamt::load(cs, &cid2).unwrap(); 62 | node_equal(&mut node, &mut node2); 63 | } 64 | 65 | #[cfg(feature = "test-hash")] 66 | #[test] 67 | fn test_hash_canonical_structure() { 68 | let k1 = ["K"]; 69 | let k2 = ["B"]; 70 | add_and_remove_keys(DEFAULT_BIT_WIDTH, &k1, &k2); 71 | let k1 = ["K0", "K1", "KAA1", "KAA2", "KAA3"]; 72 | let k2 = ["KAA4"]; 73 | add_and_remove_keys(DEFAULT_BIT_WIDTH, &k1, &k2); 74 | } 75 | 76 | #[cfg(feature = "test-hash")] 77 | #[test] 78 | fn test_hash_canonical_structure_alternate_bit_width() { 79 | add_and_remove_keys(7, ["K"].as_ref(), ["B"].as_ref()); 80 | add_and_remove_keys( 81 | 7, 82 | ["K0", "K1", "KAA1", "KAA2", "KAA3"].as_ref(), 83 | ["KAA4"].as_ref(), 84 | ); 85 | add_and_remove_keys(6, ["K"].as_ref(), ["B"].as_ref()); 86 | add_and_remove_keys( 87 | 6, 88 | ["K0", "K1", "KAA1", "KAA2", "KAA3"].as_ref(), 89 | ["KAA4"].as_ref(), 90 | ); 91 | add_and_remove_keys(5, ["K"].as_ref(), ["B"].as_ref()); 92 | add_and_remove_keys( 93 | 5, 94 | ["K0", "K1", "KAA1", "KAA2", "KAA3"].as_ref(), 95 | ["KAA4"].as_ref(), 96 | ); 97 | } 98 | 99 | #[cfg(feature = "test-hash")] 100 | #[test] 101 | fn test_hash_overflow() { 102 | let keys = [ 103 | "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA0", 104 | "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA1", 105 | "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA2", 106 | "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA3", 107 | ]; 108 | let cs = new_cbor_store(); 109 | let mut node = Hamt::new(cs); 110 | for k in &keys[..3] { 111 | node.set(k, b"foobar".to_vec()).unwrap(); 112 | } 113 | 114 | let res = node.set(keys[3], b"foobar".to_vec()); 115 | assert!(matches!(res, Err(Error::MaxDepth))); 116 | // Try forcing the depth beyond 32 117 | node.set(&keys[3][1..], b"foobar".to_vec()).unwrap(); 118 | } 119 | 120 | #[cfg(feature = "test-hash")] 121 | #[test] 122 | fn test_hash_delete() { 123 | let cs = new_cbor_store(); 124 | let mut hamt = Hamt::new(cs); 125 | hamt.set("K00", "K00").unwrap(); 126 | hamt.set("K01", "K01").unwrap(); 127 | hamt.set("K02", "K02").unwrap(); 128 | hamt.set("K03", "K03").unwrap(); 129 | hamt.set("K04", "K04").unwrap(); 130 | 131 | hamt.delete("K00").unwrap(); 132 | let cid = hamt.flush().unwrap(); 133 | assert_eq!( 134 | &cid.to_string(), 135 | "bafy2bzacedro73y3jw6pty567op7zen2nwns3vder2zl6bfiglwdnthzym3jq" 136 | ); 137 | 138 | hamt.delete("K01").unwrap(); 139 | let cid = hamt.flush().unwrap(); 140 | assert_eq!( 141 | &cid.to_string(), 142 | "bafy2bzacea6u342rxxfa73nd7kymr4bnsmeupko5o22b5ysdnjkns7z7sjzri" 143 | ); 144 | 145 | hamt.delete("K02").unwrap(); 146 | let cid = hamt.flush().unwrap(); 147 | assert_eq!( 148 | &cid.to_string(), 149 | "bafy2bzacedwuc7klbz327nddmckpdpdtc4z3fhdbe5rcsvzr4w23mrxcyouvu" 150 | ); 151 | 152 | hamt.delete("K03").unwrap(); 153 | let cid = hamt.flush().unwrap(); 154 | assert_eq!( 155 | &cid.to_string(), 156 | "bafy2bzacebouo3627hr7jm2bxryhbazqttw4h4jcxuhouu7llgpxaqjyh3jvc" 157 | ); 158 | 159 | hamt.delete("K04").unwrap(); 160 | let cid = hamt.flush().unwrap(); 161 | assert_eq!( 162 | &cid.to_string(), 163 | "bafy2bzaceamp42wmmgr2g2ymg46euououzfyck7szknvfacqscohrvaikwfay" 164 | ); 165 | } 166 | 167 | #[cfg(not(feature = "test-hash"))] 168 | #[test] 169 | fn test_basic() { 170 | let cs = new_cbor_store(); 171 | let mut begin_node = Hamt::new(cs.clone()); 172 | let val = b"cat dog bear".to_vec(); 173 | let key = "foo"; 174 | begin_node.set(key, val.clone()).unwrap(); 175 | 176 | for _ in 0..1000 { 177 | let k = rand_string(); 178 | begin_node.set(&k, rand_value()).unwrap() 179 | } 180 | 181 | let cid = begin_node.flush().unwrap(); 182 | 183 | let node = Hamt::load(cs, &cid).unwrap(); 184 | let v: Vec = node.find(key).unwrap(); 185 | assert_eq!(v, val); 186 | } 187 | 188 | #[cfg(not(feature = "test-hash"))] 189 | #[test] 190 | fn test_set_get() { 191 | use crate::node::stats; 192 | 193 | let mut map: HashMap> = HashMap::new(); 194 | for _ in 0..100_000 { 195 | map.insert(rand_string(), rand_value()); 196 | } 197 | 198 | let cs = new_cbor_store(); 199 | let mut begin_node = Hamt::new(cs.clone()); 200 | for (k, v) in map.iter() { 201 | begin_node.set(k, v.clone()).unwrap(); 202 | } 203 | 204 | let size = begin_node.check_size().unwrap(); 205 | let map_size = map 206 | .iter() 207 | .fold(0, |last, item| last + (item.0.len() + item.1.len())); 208 | println!( 209 | "Total size is: {}, size of keys+vals: {}, overhead: {:.2}", 210 | size, 211 | map_size, 212 | size as f64 / map_size as f64 213 | ); 214 | println!("stats:{:?}", stats(&begin_node)); 215 | 216 | println!("start flush"); 217 | let now = Instant::now(); 218 | let cid = begin_node.flush().unwrap(); 219 | println!("flush took: {}", now.elapsed().as_millis()); 220 | 221 | let mut node = Hamt::load_with_bitwidth(cs, &cid, begin_node.bit_width()).unwrap(); 222 | 223 | let now = Instant::now(); 224 | for (k, v) in map.iter() { 225 | let map_v: Vec = node.find(k).unwrap(); 226 | assert_eq!(map_v, *v); 227 | } 228 | println!("finds took: {}", now.elapsed().as_millis()); 229 | 230 | for _ in 0..100 { 231 | let r = rand_string(); 232 | let result = node.find::>(&r); 233 | assert!(matches!(result, Err(Error::NotFound(_)))); 234 | } 235 | 236 | map.iter_mut().for_each(|(k, v)| { 237 | let new_v = rand_value(); 238 | *v = new_v.clone(); 239 | node.set(k, new_v).unwrap(); 240 | }); 241 | 242 | map.iter().for_each(|(k, v)| { 243 | let node_v: Vec = node.find(k).unwrap(); 244 | assert_eq!(node_v, *v); 245 | }); 246 | 247 | for _ in 0..100 { 248 | let r = rand_string(); 249 | let result = node.delete(&r); 250 | assert!(matches!(result, Err(Error::NotFound(_)))); 251 | } 252 | 253 | for (k, _) in map { 254 | node.delete(&k).unwrap(); 255 | let result = node.find::>(&k); 256 | assert!(matches!(result, Err(Error::NotFound(_)))); 257 | } 258 | } 259 | 260 | #[cfg(not(feature = "test-hash"))] 261 | #[test] 262 | fn test_reload_empty() { 263 | let cs = new_cbor_store(); 264 | let mut n = Hamt::new(cs.clone()); 265 | let c = n.flush().unwrap(); 266 | let mut on = Hamt::load(cs, &c).unwrap(); 267 | on.set("foo", b"bar".to_vec()).unwrap(); 268 | } 269 | 270 | #[cfg(not(feature = "test-hash"))] 271 | #[test] 272 | fn test_value_linking() { 273 | use std::collections::BTreeMap; 274 | 275 | use ipld_core::IpldValue; 276 | 277 | let mut cs = new_cbor_store(); 278 | let mut thingy1 = HashMap::new(); 279 | thingy1.insert("cat".to_string(), "dog".to_string()); 280 | let c1 = cs.put(thingy1).unwrap(); 281 | 282 | let c = IpldValue::Link(c1); 283 | let mut hash = BTreeMap::new(); 284 | hash.insert("one".into(), c); 285 | hash.insert("foo".into(), IpldValue::String("bar".to_string())); 286 | let thingy2 = IpldValue::Map(hash); 287 | 288 | let mut n = Hamt::new(cs); 289 | n.set("cat", thingy2).unwrap(); 290 | let tcid = n.flush().unwrap(); 291 | 292 | assert_eq!( 293 | &tcid.to_string(), 294 | "bafy2bzacedrwmwbquhdfs2ivq4dhejgyo2jxhlw2xdrrjwcarbimpxuvmx3e4" 295 | ); 296 | } 297 | -------------------------------------------------------------------------------- /ipld/hamt/src/tests/hash_test.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2020 PolkaX. Licensed under MIT or Apache-2.0. 2 | 3 | use crate::hash::HashBits; 4 | 5 | #[cfg(not(feature = "test-hash"))] 6 | #[test] 7 | fn test_hash() { 8 | use crate::hash::hash; 9 | 10 | let h1 = hash("abcd"); 11 | let h2 = hash("abce"); 12 | 13 | let first = [184_u8, 123, 183, 214, 70, 86, 205, 79]; 14 | let second = [5, 245, 47, 205, 203, 64, 38, 66]; 15 | 16 | assert_eq!(h1, first); 17 | assert_eq!(h2, second); 18 | 19 | let h3 = hash("hello world"); 20 | assert_eq!(h3, [83, 63, 96, 70, 235, 127, 97, 14]); 21 | 22 | let h4 = hash(b"hello world"); 23 | assert_eq!(h4, [83, 63, 96, 70, 235, 127, 97, 14]); 24 | 25 | let h5 = hash("一二三"); 26 | assert_eq!(h5, [37, 41, 47, 183, 147, 168, 177, 225]); 27 | } 28 | 29 | #[test] 30 | fn test_hash_bits_overflow() { 31 | let buf = [255_u8]; 32 | let bit_width = 1; 33 | let mut hb = HashBits::new(buf.as_ref(), bit_width); 34 | for _i in 0..8 { 35 | let bit = hb.next().unwrap(); 36 | assert_eq!(bit, 1); 37 | } 38 | assert_eq!(hb.next(), None) 39 | } 40 | 41 | #[test] 42 | fn test_hash_bits_uneven() { 43 | let buf = [255_u8, 127, 79, 45, 116, 99, 35, 17]; 44 | let bit_width = 4; 45 | let mut hb = HashBits::new(buf.as_ref(), bit_width); 46 | let v = hb.next(); 47 | assert_eq!(v, Some(15)); 48 | 49 | let v = hb.next(); 50 | assert_eq!(v, Some(15)); 51 | 52 | hb.bit_width = 3; 53 | let v = hb.next(); 54 | assert_eq!(v, Some(3)); 55 | 56 | let v = hb.next(); 57 | assert_eq!(v, Some(7)); 58 | 59 | let v = hb.next(); 60 | assert_eq!(v, Some(6)); 61 | 62 | hb.bit_width = 15; 63 | let v = hb.next(); 64 | assert_eq!(v, Some(20269)); 65 | } 66 | -------------------------------------------------------------------------------- /ipld/hamt/src/tests/ipld_test.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2020 PolkaX. Licensed under MIT or Apache-2.0. 2 | 3 | use std::string::ToString; 4 | 5 | use bigint::U256; 6 | use ipld_core::struct_to_cbor_value; 7 | 8 | use super::*; 9 | use crate::node::set_bit; 10 | 11 | #[test] 12 | fn test_roundtrip() { 13 | let mut cs = new_cbor_store(); 14 | 15 | let mut bitmap: U256 = Default::default(); 16 | 17 | set_bit(&mut bitmap, 5); 18 | set_bit(&mut bitmap, 7); 19 | set_bit(&mut bitmap, 18); 20 | 21 | let v = struct_to_cbor_value(&vec![0x83_u8, 0x01, 0x02, 0x03]).unwrap(); 22 | let kv: KVT = ("foo".to_string(), v); 23 | let p = Item::from_kvs(vec![kv]); 24 | 25 | let n = test_node(&bitmap.to_string(), vec![p]); 26 | 27 | let cid = cs.put(n).unwrap(); 28 | let n2: Node = cs.get(&cid).unwrap(); 29 | 30 | let c2 = cs.put(n2).unwrap(); 31 | assert_eq!(cid, c2); 32 | } 33 | 34 | #[test] 35 | fn test_basic_bytes_loading() { 36 | let b = b"cats and dogs are taking over".to_vec(); 37 | let o = ipld_core::dump_object::>(b.as_ref()).unwrap(); 38 | let s: Vec = ipld_core::decode_into(&o).unwrap(); 39 | assert_eq!(b, s); 40 | } 41 | 42 | #[cfg(not(feature = "test-hash"))] 43 | #[test] 44 | fn test_kv() { 45 | use ipld_core::IpldValue; 46 | use std::collections::BTreeMap; 47 | 48 | let mut cs = new_cbor_store(); 49 | let mut thingy1 = HashMap::new(); 50 | thingy1.insert("cat".to_string(), "dog".to_string()); 51 | let c1 = cs.put(thingy1).unwrap(); 52 | 53 | let c = IpldValue::Link(c1); 54 | let mut hash = BTreeMap::new(); 55 | hash.insert("one".into(), c); 56 | hash.insert("foo".into(), IpldValue::String("bar".to_string())); 57 | let thingy2 = IpldValue::Map(hash); 58 | 59 | let b = ipld_core::dump_object(&thingy2).unwrap(); 60 | println!("{:?}", b); 61 | 62 | let mut node = Hamt::new(cs); 63 | node.set("cat", thingy2).unwrap(); 64 | 65 | let b = ipld_core::dump_object(node.root()).unwrap(); 66 | println!("{:?} {}", b, b.len()); 67 | 68 | assert_eq!( 69 | b, 70 | vec![ 71 | 130, 88, 30, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72 | 0, 0, 0, 0, 0, 129, 161, 97, 49, 129, 130, 99, 99, 97, 116, 162, 99, 102, 111, 111, 99, 73 | 98, 97, 114, 99, 111, 110, 101, 216, 42, 88, 39, 0, 1, 113, 160, 228, 2, 32, 236, 82, 74 | 22, 81, 235, 94, 233, 82, 25, 143, 252, 235, 234, 106, 54, 182, 180, 21, 78, 124, 8, 75 | 62, 200, 123, 158, 248, 162, 126, 171, 22, 88, 64 76 | ] 77 | ) 78 | } 79 | -------------------------------------------------------------------------------- /ipld/hamt/src/tests/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2020 PolkaX. Licensed under MIT or Apache-2.0. 2 | 3 | mod cbor_test; 4 | mod hamt_test; 5 | mod hash_test; 6 | mod ipld_test; 7 | 8 | use std::cell::RefCell; 9 | use std::collections::HashMap; 10 | use std::rc::Rc; 11 | 12 | use block_format::{BasicBlock, Block as BlockT}; 13 | use bytes::Bytes; 14 | use cid::Cid; 15 | 16 | use super::*; 17 | use crate::error::*; 18 | use crate::ipld::Blockstore; 19 | use crate::node::{test_node, Item, Node, KVT}; 20 | 21 | #[derive(Clone, Default)] 22 | pub struct MockBlocks { 23 | data: Rc>>>, 24 | } 25 | 26 | impl Blockstore for MockBlocks { 27 | fn get(&self, cid: &Cid) -> Result> { 28 | let blk = self 29 | .data 30 | .borrow() 31 | .get(cid) 32 | .map(|data| BasicBlock::new(Bytes::copy_from_slice(data))) 33 | .ok_or_else(|| Error::NotFoundForCid(cid.clone()))?; 34 | Ok(Box::new(blk)) 35 | } 36 | 37 | fn put(&mut self, block: impl BlockT) -> Result<()> { 38 | let cid = block.cid().clone(); 39 | self.data 40 | .borrow_mut() 41 | .insert(cid, block.raw_data().to_vec()); 42 | Ok(()) 43 | } 44 | } 45 | 46 | pub fn new_cbor_store() -> BasicCborIpldStore { 47 | BasicCborIpldStore::new(MockBlocks::default()) 48 | } 49 | -------------------------------------------------------------------------------- /paritylibs/kvdb-rocksdb/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | The format is based on [Keep a Changelog]. 4 | 5 | [Keep a Changelog]: http://keepachangelog.com/en/1.0.0/ 6 | 7 | ## [Unreleased] 8 | 9 | ## [0.5.0] - 2019-02-05 10 | - Bump parking_lot to 0.10. [#332](https://github.com/paritytech/parity-common/pull/332 11 | 12 | ## [0.4.2] - 2019-02-04 13 | ### Fixes 14 | - Fixed `iter_from_prefix` being slow. [#326](https://github.com/paritytech/parity-common/pull/326) 15 | 16 | ## [0.4.1] - 2019-01-06 17 | - Updated features and feature dependencies. [#307](https://github.com/paritytech/parity-common/pull/307) 18 | 19 | ## [0.4.0] - 2019-01-03 20 | - Add I/O statistics for RocksDB. [#294](https://github.com/paritytech/parity-common/pull/294) 21 | - Support querying memory footprint via `MallocSizeOf` trait. [#292](https://github.com/paritytech/parity-common/pull/292) 22 | 23 | ## [0.3.0] - 2019-12-19 24 | - Use `get_pinned` API to save one allocation for each call to `get()` (See [PR #274](https://github.com/paritytech/parity-common/pull/274) for details) 25 | - Rename `drop_column` to `remove_last_column` (See [PR #274](https://github.com/paritytech/parity-common/pull/274) for details) 26 | - Rename `get_cf` to `cf` (See [PR #274](https://github.com/paritytech/parity-common/pull/274) for details) 27 | - Default column support removed from the API (See [PR #278](https://github.com/paritytech/parity-common/pull/278) for details) 28 | - Column argument type changed from `Option` to `u32` 29 | - Migration 30 | - Column index `None` -> unsupported, `Some(0)` -> `0`, `Some(1)` -> `1`, etc. 31 | - Database must be opened with at least one column and existing DBs has to be opened with a number of columns increased by 1 to avoid having to migrate the data, e.g. before: `Some(9)`, after: `10`. 32 | - `DatabaseConfig::default()` defaults to 1 column 33 | - `Database::with_columns` still accepts `u32`, but panics if `0` is provided 34 | - `Database::open` panics if configuration with 0 columns is provided 35 | - Add `num_keys(col)` to get an estimate of the number of keys in a column (See [PR #285](https://github.com/paritytech/parity-common/pull/285)). 36 | - Remove `ElasticArray` and use the new `DBValue` (alias for `Vec`) and `DBKey` types from `kvdb`. (See [PR #282](https://github.com/paritytech/parity-common/pull/282/files)) 37 | 38 | ## [0.2.0] - 2019-11-28 39 | - Switched away from using [parity-rocksdb](https://crates.io/crates/parity-rocksdb) in favour of upstream [rust-rocksdb](https://crates.io/crates/rocksdb) (see [PR #257](https://github.com/paritytech/parity-common/pull/257) for details) 40 | - Revamped configuration handling, allowing per-column memory budgeting (see [PR #256](https://github.com/paritytech/parity-common/pull/256) for details) 41 | ### Dependencies 42 | - rust-rocksdb v0.13 43 | 44 | ## [0.1.6] - 2019-10-24 45 | - Updated to 2018 edition idioms (https://github.com/paritytech/parity-common/pull/237) 46 | ### Dependencies 47 | - Updated dependencies (https://github.com/paritytech/parity-common/pull/239) 48 | -------------------------------------------------------------------------------- /paritylibs/kvdb-rocksdb/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "kvdb-rocksdb" 3 | version = "0.5.0" 4 | authors = ["Parity Technologies "] 5 | repository = "https://github.com/paritytech/parity-common" 6 | description = "kvdb implementation backed by RocksDB" 7 | license = "GPL-3.0" 8 | edition = "2018" 9 | 10 | [[bench]] 11 | name = "bench_read_perf" 12 | harness = false 13 | 14 | [dependencies] 15 | smallvec = "1.0.0" 16 | fs-swap = "0.2.4" 17 | interleaved-ordered = "0.1.1" 18 | kvdb = { path = "../kvdb", version = "0.4" } 19 | log = "0.4.8" 20 | num_cpus = "1.10.1" 21 | parking_lot = "0.10.0" 22 | regex = "1.3.1" 23 | rocksdb = { version = "0.13", features = ["snappy"], default-features = false } 24 | owning_ref = "0.4.0" 25 | 26 | [dev-dependencies] 27 | alloc_counter = "0.0.4" 28 | criterion = "0.3" 29 | kvdb-shared-tests = { path = "../kvdb-shared-tests", version = "0.2" } 30 | rand = "0.7.2" 31 | tempfile = "3.1" 32 | -------------------------------------------------------------------------------- /paritylibs/kvdb-rocksdb/benches/.gitignore: -------------------------------------------------------------------------------- 1 | _rocksdb_bench_get 2 | -------------------------------------------------------------------------------- /paritylibs/kvdb-rocksdb/benches/bench_read_perf.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2019 Parity Technologies (UK) Ltd. 2 | // This file is part of Parity Ethereum. 3 | 4 | // Parity Ethereum is free software: you can redistribute it and/or modify 5 | // it under the terms of the GNU General Public License as published by 6 | // the Free Software Foundation, either version 3 of the License, or 7 | // (at your option) any later version. 8 | 9 | // Parity Ethereum is distributed in the hope that it will be useful, 10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | // GNU General Public License for more details. 13 | 14 | // You should have received a copy of the GNU General Public License 15 | // along with Parity Ethereum. If not, see . 16 | 17 | //! Benchmark RocksDB read performance. 18 | //! The benchmark setup consists in writing `NEEDLES * NEEDLES_TO_HAYSTACK_RATIO` 32-bytes random 19 | //! keys with random values 150 +/- 30 bytes long. With 10 000 keys and a ratio of 100 we get one 20 | //! million keys; ideally the db should be deleted for each benchmark run but in practice it has 21 | //! little impact on the performance numbers for these small database sizes. 22 | //! Allocations (on the Rust side) are counted and printed. 23 | //! 24 | //! Note that this benchmark is not a good way to measure the performance of the database itself; 25 | //! its purpose is to be a tool to gauge the performance of the glue code, or work as a starting point 26 | //! for a more elaborate benchmark of a specific workload. 27 | 28 | const NEEDLES: usize = 10_000; 29 | const NEEDLES_TO_HAYSTACK_RATIO: usize = 100; 30 | 31 | use std::io; 32 | use std::time::{Duration, Instant}; 33 | 34 | use alloc_counter::{count_alloc, AllocCounterSystem}; 35 | use criterion::{black_box, criterion_group, criterion_main, Criterion}; 36 | use rand::{distributions::Uniform, seq::SliceRandom, Rng}; 37 | 38 | use kvdb_rocksdb::{Database, DatabaseConfig, DEFAULT_COLUMN_NAME}; 39 | 40 | type H256 = [u8; 32]; 41 | 42 | #[global_allocator] 43 | static A: AllocCounterSystem = AllocCounterSystem; 44 | 45 | criterion_group!(benches, get, iter); 46 | criterion_main!(benches); 47 | 48 | /// Opens (or creates) a RocksDB database in the `benches/` folder of the crate with one column 49 | /// family and default options. Needs manual cleanup. 50 | fn open_db() -> Database { 51 | let tempdir_str = "./benches/_rocksdb_bench_get"; 52 | let cfg = DatabaseConfig::with_columns(vec!["1".to_string()]); 53 | Database::open(&cfg, tempdir_str).expect("rocksdb works") 54 | } 55 | 56 | /// Generate `n` random bytes +/- 20%. 57 | /// The variability in the payload size lets us simulate payload allocation patterns: `DBValue` is 58 | /// an `ElasticArray128` so sometimes we save on allocations. 59 | fn n_random_bytes(n: usize) -> Vec { 60 | let mut rng = rand::thread_rng(); 61 | let variability: i64 = rng.gen_range(0, (n / 5) as i64); 62 | let plus_or_minus: i64 = if variability % 2 == 0 { 1 } else { -1 }; 63 | let range = Uniform::from(0..u8::max_value()); 64 | rng.sample_iter(&range) 65 | .take((n as i64 + plus_or_minus * variability) as usize) 66 | .collect() 67 | } 68 | 69 | /// Writes `NEEDLES * NEEDLES_TO_HAYSTACK_RATIO` keys to the DB. Keys are random, 32 bytes long and 70 | /// values are random, 120-180 bytes long. Every `NEEDLES_TO_HAYSTACK_RATIO` keys are kept and 71 | /// returned in a `Vec` for and used to benchmark point lookup performance. Keys are sorted 72 | /// lexicographically in the DB, and the benchmark keys are random bytes making the needles are 73 | /// effectively random points in the key set. 74 | fn populate(db: &Database) -> io::Result> { 75 | let random = || -> H256 { 76 | use rand::distributions::Distribution; 77 | let mut rng = rand::rngs::OsRng; 78 | rand::distributions::Standard.sample(&mut rng) 79 | }; 80 | 81 | let mut needles = Vec::with_capacity(NEEDLES); 82 | let mut batch = db.transaction(); 83 | for i in 0..NEEDLES * NEEDLES_TO_HAYSTACK_RATIO { 84 | let key = random(); 85 | if i % NEEDLES_TO_HAYSTACK_RATIO == 0 { 86 | needles.push(key.clone()); 87 | if i % 100_000 == 0 && i > 0 { 88 | println!("[populate] {} keys", i); 89 | } 90 | } 91 | // In ethereum keys are mostly 32 bytes and payloads ~140bytes. 92 | batch.put(DEFAULT_COLUMN_NAME, &key.as_ref(), &n_random_bytes(140)); 93 | } 94 | db.write(batch)?; 95 | // Clear the overlay 96 | db.flush()?; 97 | Ok(needles) 98 | } 99 | 100 | fn get(c: &mut Criterion) { 101 | let db = open_db(); 102 | let needles = populate(&db).expect("rocksdb works"); 103 | 104 | let mut total_iterations = 0; 105 | let mut total_allocs = 0; 106 | 107 | c.bench_function("get key", |b| { 108 | b.iter_custom(|iterations| { 109 | total_iterations += iterations; 110 | let mut elapsed = Duration::new(0, 0); 111 | // NOTE: counts allocations on the Rust side only 112 | let (alloc_stats, _) = count_alloc(|| { 113 | let start = Instant::now(); 114 | for _ in 0..iterations { 115 | // This has no measurable impact on performance (~30ns) 116 | let needle = needles 117 | .choose(&mut rand::thread_rng()) 118 | .expect("needles is not empty"); 119 | black_box(db.get(DEFAULT_COLUMN_NAME, needle.as_ref()).unwrap()); 120 | } 121 | elapsed = start.elapsed(); 122 | }); 123 | total_allocs += alloc_stats.0; 124 | elapsed 125 | }); 126 | }); 127 | if total_iterations > 0 { 128 | println!( 129 | "[get key] total: iterations={}, allocations={}; allocations per iter={:.2}\n", 130 | total_iterations, 131 | total_allocs, 132 | total_allocs as f64 / total_iterations as f64 133 | ); 134 | } 135 | 136 | total_iterations = 0; 137 | total_allocs = 0; 138 | c.bench_function("get key by prefix", |b| { 139 | b.iter_custom(|iterations| { 140 | total_iterations += iterations; 141 | let mut elapsed = Duration::new(0, 0); 142 | // NOTE: counts allocations on the Rust side only 143 | let (alloc_stats, _) = count_alloc(|| { 144 | let start = Instant::now(); 145 | for _ in 0..iterations { 146 | // This has no measurable impact on performance (~30ns) 147 | let needle = needles 148 | .choose(&mut rand::thread_rng()) 149 | .expect("needles is not empty"); 150 | black_box( 151 | db.get_by_prefix(DEFAULT_COLUMN_NAME, &needle.as_ref()[..8]) 152 | .unwrap(), 153 | ); 154 | } 155 | elapsed = start.elapsed(); 156 | }); 157 | total_allocs += alloc_stats.0; 158 | elapsed 159 | }); 160 | }); 161 | if total_iterations > 0 { 162 | println!( 163 | "[get key by prefix] total: iterations={}, allocations={}; allocations per iter={:.2}\n", 164 | total_iterations, 165 | total_allocs, 166 | total_allocs as f64 / total_iterations as f64 167 | ); 168 | } 169 | } 170 | 171 | fn iter(c: &mut Criterion) { 172 | let db = open_db(); 173 | let mut total_iterations = 0; 174 | let mut total_allocs = 0; 175 | 176 | c.bench_function("iterate over 1k keys", |b| { 177 | b.iter_custom(|iterations| { 178 | total_iterations += iterations; 179 | let mut elapsed = Duration::new(0, 0); 180 | // NOTE: counts allocations on the Rust side only 181 | let (alloc_stats, _) = count_alloc(|| { 182 | let start = Instant::now(); 183 | for _ in 0..iterations { 184 | black_box(db.iter(DEFAULT_COLUMN_NAME).take(1000).collect::>()); 185 | } 186 | elapsed = start.elapsed(); 187 | }); 188 | total_allocs += alloc_stats.0; 189 | elapsed 190 | }); 191 | }); 192 | if total_iterations > 0 { 193 | println!( 194 | "[iterate over 1k keys] total: iterations={}, allocations={}; allocations per iter={:.2}\n", 195 | total_iterations, 196 | total_allocs, 197 | total_allocs as f64 / total_iterations as f64 198 | ); 199 | } 200 | 201 | total_allocs = 0; 202 | total_iterations = 0; 203 | c.bench_function("single key from iterator", |b| { 204 | b.iter_custom(|iterations| { 205 | total_iterations += iterations; 206 | let mut elapsed = Duration::new(0, 0); 207 | // NOTE: counts allocations on the Rust side only 208 | let (alloc_stats, _) = count_alloc(|| { 209 | let start = Instant::now(); 210 | for _ in 0..iterations { 211 | black_box(db.iter(DEFAULT_COLUMN_NAME).next().unwrap()); 212 | } 213 | elapsed = start.elapsed(); 214 | }); 215 | total_allocs += alloc_stats.0; 216 | elapsed 217 | }); 218 | }); 219 | if total_iterations > 0 { 220 | println!( 221 | "[single key from iterator] total: iterations={}, allocations={}; allocations per iter={:.2}\n", 222 | total_iterations, 223 | total_allocs, 224 | total_allocs as f64 / total_iterations as f64 225 | ); 226 | } 227 | } 228 | -------------------------------------------------------------------------------- /paritylibs/kvdb-rocksdb/src/iter.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Parity Technologies (UK) Ltd. 2 | // This file is part of Parity. 3 | 4 | // Parity is free software: you can redistribute it and/or modify 5 | // it under the terms of the GNU General Public License as published by 6 | // the Free Software Foundation, either version 3 of the License, or 7 | // (at your option) any later version. 8 | 9 | // Parity is distributed in the hope that it will be useful, 10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | // GNU General Public License for more details. 13 | 14 | // You should have received a copy of the GNU General Public License 15 | // along with Parity. If not, see . 16 | 17 | //! This module contains an implementation of a RocksDB iterator 18 | //! wrapped inside a `RwLock`. Since `RwLock` "owns" the inner data, 19 | //! we're using `owning_ref` to work around the borrowing rules of Rust. 20 | //! 21 | //! Note: this crate does not use "Prefix Seek" mode which means that the prefix iterator 22 | //! will return keys not starting with the given prefix as well (as long as `key >= prefix`). 23 | //! To work around this we filter the data returned by rocksdb to ensure that 24 | //! all data yielded by the iterator does start with the given prefix. 25 | //! See https://github.com/facebook/rocksdb/wiki/Prefix-Seek-API-Changes for details. 26 | 27 | use crate::DBAndColumns; 28 | use owning_ref::{OwningHandle, StableAddress}; 29 | use parking_lot::RwLockReadGuard; 30 | use rocksdb::{DBIterator, Direction, IteratorMode, ReadOptions}; 31 | use std::ops::{Deref, DerefMut}; 32 | 33 | /// A tuple holding key and value data, used as the iterator item type. 34 | pub type KeyValuePair = (Box<[u8]>, Box<[u8]>); 35 | 36 | /// Iterator with built-in synchronization. 37 | pub struct ReadGuardedIterator<'a, I, T> { 38 | inner: OwningHandle>, DerefWrapper>>, 39 | } 40 | 41 | // We can't implement `StableAddress` for a `RwLockReadGuard` 42 | // directly due to orphan rules. 43 | #[repr(transparent)] 44 | struct UnsafeStableAddress<'a, T>(RwLockReadGuard<'a, T>); 45 | 46 | impl<'a, T> Deref for UnsafeStableAddress<'a, T> { 47 | type Target = T; 48 | 49 | fn deref(&self) -> &Self::Target { 50 | self.0.deref() 51 | } 52 | } 53 | 54 | // RwLockReadGuard dereferences to a stable address; qed 55 | unsafe impl<'a, T> StableAddress for UnsafeStableAddress<'a, T> {} 56 | 57 | struct DerefWrapper(T); 58 | 59 | impl Deref for DerefWrapper { 60 | type Target = T; 61 | 62 | fn deref(&self) -> &Self::Target { 63 | &self.0 64 | } 65 | } 66 | 67 | impl DerefMut for DerefWrapper { 68 | fn deref_mut(&mut self) -> &mut Self::Target { 69 | &mut self.0 70 | } 71 | } 72 | 73 | impl<'a, I: Iterator, T> Iterator for ReadGuardedIterator<'a, I, T> { 74 | type Item = I::Item; 75 | 76 | fn next(&mut self) -> Option { 77 | self.inner.deref_mut().as_mut().and_then(|iter| iter.next()) 78 | } 79 | } 80 | 81 | /// Instantiate iterators yielding `KeyValuePair`s. 82 | pub trait IterationHandler { 83 | type Iterator: Iterator; 84 | 85 | /// Create an `Iterator` over a `ColumnFamily` corresponding to the passed index. Takes a 86 | /// reference to a `ReadOptions` to allow configuration of the new iterator (see 87 | /// https://github.com/facebook/rocksdb/blob/master/include/rocksdb/options.h#L1169). 88 | fn iter(&self, col: &str, read_opts: &ReadOptions) -> Self::Iterator; 89 | /// Create an `Iterator` over a `ColumnFamily` corresponding to the passed index. Takes a 90 | /// reference to a `ReadOptions` to allow configuration of the new iterator (see 91 | /// https://github.com/facebook/rocksdb/blob/master/include/rocksdb/options.h#L1169). 92 | /// The iterator starts from the first key having the provided `prefix`. 93 | fn iter_from_prefix(&self, col: &str, prefix: &[u8], read_opts: &ReadOptions) 94 | -> Self::Iterator; 95 | } 96 | 97 | impl<'a, T> ReadGuardedIterator<'a, <&'a T as IterationHandler>::Iterator, T> 98 | where 99 | &'a T: IterationHandler, 100 | { 101 | /// Creates a new `ReadGuardedIterator` that maps `RwLock` to `RwLock`, 102 | /// where `DBIterator` iterates over all keys. 103 | pub fn new( 104 | read_lock: RwLockReadGuard<'a, Option>, 105 | col: &str, 106 | read_opts: &ReadOptions, 107 | ) -> Self { 108 | Self { 109 | inner: Self::new_inner(read_lock, |db| db.iter(col, read_opts)), 110 | } 111 | } 112 | 113 | /// Creates a new `ReadGuardedIterator` that maps `RwLock` to `RwLock`, 114 | /// where `DBIterator` iterates over keys >= prefix. 115 | pub fn new_from_prefix( 116 | read_lock: RwLockReadGuard<'a, Option>, 117 | col: &str, 118 | prefix: &[u8], 119 | read_opts: &ReadOptions, 120 | ) -> Self { 121 | Self { 122 | inner: Self::new_inner(read_lock, |db| db.iter_from_prefix(col, prefix, read_opts)), 123 | } 124 | } 125 | 126 | fn new_inner( 127 | rlock: RwLockReadGuard<'a, Option>, 128 | f: impl FnOnce(&'a T) -> <&'a T as IterationHandler>::Iterator, 129 | ) -> OwningHandle< 130 | UnsafeStableAddress<'a, Option>, 131 | DerefWrapper::Iterator>>, 132 | > { 133 | OwningHandle::new_with_fn(UnsafeStableAddress(rlock), move |rlock| { 134 | let rlock = unsafe { rlock.as_ref().expect("initialized as non-null; qed") }; 135 | DerefWrapper(rlock.as_ref().map(f)) 136 | }) 137 | } 138 | } 139 | 140 | impl<'a> IterationHandler for &'a DBAndColumns { 141 | type Iterator = DBIterator<'a>; 142 | 143 | fn iter(&self, col: &str, read_opts: &ReadOptions) -> Self::Iterator { 144 | self.db 145 | .iterator_cf_opt(self.cf(col), read_opts, IteratorMode::Start) 146 | .expect("iterator params are valid; qed") 147 | } 148 | 149 | fn iter_from_prefix( 150 | &self, 151 | col: &str, 152 | prefix: &[u8], 153 | read_opts: &ReadOptions, 154 | ) -> Self::Iterator { 155 | self.db 156 | .iterator_cf_opt( 157 | self.cf(col), 158 | read_opts, 159 | IteratorMode::From(prefix, Direction::Forward), 160 | ) 161 | .expect("iterator params are valid; qed") 162 | } 163 | } 164 | -------------------------------------------------------------------------------- /paritylibs/kvdb-rocksdb/src/stats.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2020 Parity Technologies (UK) Ltd. 2 | // This file is part of Parity. 3 | 4 | // Parity is free software: you can redistribute it and/or modify 5 | // it under the terms of the GNU General Public License as published by 6 | // the Free Software Foundation, either version 3 of the License, or 7 | // (at your option) any later version. 8 | 9 | // Parity is distributed in the hope that it will be useful, 10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | // GNU General Public License for more details. 13 | 14 | // You should have received a copy of the GNU General Public License 15 | // along with Parity. If not, see . 16 | 17 | use parking_lot::RwLock; 18 | use std::sync::atomic::{AtomicU64, Ordering as AtomicOrdering}; 19 | use std::time::Instant; 20 | 21 | pub struct RawDbStats { 22 | pub reads: u64, 23 | pub writes: u64, 24 | pub bytes_written: u64, 25 | pub bytes_read: u64, 26 | pub transactions: u64, 27 | } 28 | 29 | impl RawDbStats { 30 | fn combine(&self, other: &RawDbStats) -> Self { 31 | RawDbStats { 32 | reads: self.reads + other.reads, 33 | writes: self.writes + other.writes, 34 | bytes_written: self.bytes_written + other.bytes_written, 35 | bytes_read: self.bytes_read + other.bytes_written, 36 | transactions: self.transactions + other.transactions, 37 | } 38 | } 39 | } 40 | 41 | struct OverallDbStats { 42 | stats: RawDbStats, 43 | last_taken: Instant, 44 | started: Instant, 45 | } 46 | 47 | impl OverallDbStats { 48 | fn new() -> Self { 49 | OverallDbStats { 50 | stats: RawDbStats { 51 | reads: 0, 52 | writes: 0, 53 | bytes_written: 0, 54 | bytes_read: 0, 55 | transactions: 0, 56 | }, 57 | last_taken: Instant::now(), 58 | started: Instant::now(), 59 | } 60 | } 61 | } 62 | 63 | pub struct RunningDbStats { 64 | reads: AtomicU64, 65 | writes: AtomicU64, 66 | bytes_written: AtomicU64, 67 | bytes_read: AtomicU64, 68 | transactions: AtomicU64, 69 | overall: RwLock, 70 | } 71 | 72 | pub struct TakenDbStats { 73 | pub raw: RawDbStats, 74 | pub started: Instant, 75 | } 76 | 77 | impl RunningDbStats { 78 | pub fn new() -> Self { 79 | Self { 80 | reads: 0.into(), 81 | bytes_read: 0.into(), 82 | writes: 0.into(), 83 | bytes_written: 0.into(), 84 | transactions: 0.into(), 85 | overall: OverallDbStats::new().into(), 86 | } 87 | } 88 | 89 | pub fn tally_reads(&self, val: u64) { 90 | self.reads.fetch_add(val, AtomicOrdering::Relaxed); 91 | } 92 | 93 | pub fn tally_bytes_read(&self, val: u64) { 94 | self.bytes_read.fetch_add(val, AtomicOrdering::Relaxed); 95 | } 96 | 97 | pub fn tally_writes(&self, val: u64) { 98 | self.writes.fetch_add(val, AtomicOrdering::Relaxed); 99 | } 100 | 101 | pub fn tally_bytes_written(&self, val: u64) { 102 | self.bytes_written.fetch_add(val, AtomicOrdering::Relaxed); 103 | } 104 | 105 | pub fn tally_transactions(&self, val: u64) { 106 | self.transactions.fetch_add(val, AtomicOrdering::Relaxed); 107 | } 108 | 109 | fn take_current(&self) -> RawDbStats { 110 | RawDbStats { 111 | reads: self.reads.swap(0, AtomicOrdering::Relaxed), 112 | writes: self.writes.swap(0, AtomicOrdering::Relaxed), 113 | bytes_written: self.bytes_written.swap(0, AtomicOrdering::Relaxed), 114 | bytes_read: self.bytes_read.swap(0, AtomicOrdering::Relaxed), 115 | transactions: self.transactions.swap(0, AtomicOrdering::Relaxed), 116 | } 117 | } 118 | 119 | fn peek_current(&self) -> RawDbStats { 120 | RawDbStats { 121 | reads: self.reads.load(AtomicOrdering::Relaxed), 122 | writes: self.writes.load(AtomicOrdering::Relaxed), 123 | bytes_written: self.bytes_written.load(AtomicOrdering::Relaxed), 124 | bytes_read: self.bytes_read.load(AtomicOrdering::Relaxed), 125 | transactions: self.transactions.load(AtomicOrdering::Relaxed), 126 | } 127 | } 128 | 129 | pub fn since_previous(&self) -> TakenDbStats { 130 | let mut overall_lock = self.overall.write(); 131 | 132 | let current = self.take_current(); 133 | 134 | overall_lock.stats = overall_lock.stats.combine(¤t); 135 | 136 | let stats = TakenDbStats { 137 | raw: current, 138 | started: overall_lock.last_taken, 139 | }; 140 | 141 | overall_lock.last_taken = Instant::now(); 142 | 143 | stats 144 | } 145 | 146 | pub fn overall(&self) -> TakenDbStats { 147 | let overall_lock = self.overall.read(); 148 | 149 | let current = self.peek_current(); 150 | 151 | TakenDbStats { 152 | raw: overall_lock.stats.combine(¤t), 153 | started: overall_lock.started, 154 | } 155 | } 156 | } 157 | -------------------------------------------------------------------------------- /paritylibs/kvdb-shared-tests/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "kvdb-shared-tests" 3 | version = "0.2.0" 4 | authors = ["Parity Technologies "] 5 | edition = "2018" 6 | description = "Shared tests for kvdb functionality, to be executed against actual implementations" 7 | license = "GPL-3.0" 8 | 9 | [dependencies] 10 | kvdb = { path = "../kvdb", version = "0.4" } 11 | -------------------------------------------------------------------------------- /paritylibs/kvdb-shared-tests/src/lib.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Parity Technologies (UK) Ltd. 2 | // This file is part of Parity. 3 | 4 | // Parity is free software: you can redistribute it and/or modify 5 | // it under the terms of the GNU General Public License as published by 6 | // the Free Software Foundation, either version 3 of the License, or 7 | // (at your option) any later version. 8 | 9 | // Parity is distributed in the hope that it will be useful, 10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | // GNU General Public License for more details. 13 | 14 | // You should have received a copy of the GNU General Public License 15 | // along with Parity. If not, see . 16 | 17 | //! Shared tests for kvdb functionality, to be executed against actual implementations. 18 | 19 | use kvdb::{IoStatsKind, KeyValueDB}; 20 | use std::io; 21 | 22 | const DEFAULT_COLUMN_NAME: &str = "default"; 23 | 24 | /// A test for `KeyValueDB::get`. 25 | pub fn test_put_and_get(db: &dyn KeyValueDB) -> io::Result<()> { 26 | let key1 = b"key1"; 27 | 28 | let mut transaction = db.transaction(); 29 | transaction.put(DEFAULT_COLUMN_NAME, key1, b"horse"); 30 | db.write_buffered(transaction); 31 | assert_eq!(&*db.get(DEFAULT_COLUMN_NAME, key1)?.unwrap(), b"horse"); 32 | Ok(()) 33 | } 34 | 35 | /// A test for `KeyValueDB::get`. 36 | pub fn test_delete_and_get(db: &dyn KeyValueDB) -> io::Result<()> { 37 | let key1 = b"key1"; 38 | 39 | let mut transaction = db.transaction(); 40 | transaction.put(DEFAULT_COLUMN_NAME, key1, b"horse"); 41 | db.write_buffered(transaction); 42 | assert_eq!(&*db.get(DEFAULT_COLUMN_NAME, key1)?.unwrap(), b"horse"); 43 | 44 | let mut transaction = db.transaction(); 45 | transaction.delete(DEFAULT_COLUMN_NAME, key1); 46 | db.write_buffered(transaction); 47 | assert!(db.get(DEFAULT_COLUMN_NAME, key1)?.is_none()); 48 | Ok(()) 49 | } 50 | 51 | /// A test for `KeyValueDB::get`. 52 | /// Assumes the `db` has only 1 column. 53 | pub fn test_get_fails_with_non_existing_column(db: &dyn KeyValueDB) -> io::Result<()> { 54 | assert!(db.get("1", &[]).is_err()); 55 | Ok(()) 56 | } 57 | 58 | /// A test for `KeyValueDB::write`. 59 | pub fn test_write_clears_buffered_ops(db: &dyn KeyValueDB) -> io::Result<()> { 60 | let mut batch = db.transaction(); 61 | batch.put(DEFAULT_COLUMN_NAME, b"foo", b"bar"); 62 | db.write_buffered(batch); 63 | 64 | assert_eq!(db.get(DEFAULT_COLUMN_NAME, b"foo")?.unwrap(), b"bar"); 65 | 66 | let mut batch = db.transaction(); 67 | batch.put(DEFAULT_COLUMN_NAME, b"foo", b"baz"); 68 | db.write(batch)?; 69 | 70 | assert_eq!(db.get(DEFAULT_COLUMN_NAME, b"foo")?.unwrap(), b"baz"); 71 | Ok(()) 72 | } 73 | 74 | /// A test for `KeyValueDB::iter`. 75 | pub fn test_iter(db: &dyn KeyValueDB) -> io::Result<()> { 76 | let key1 = b"key1"; 77 | let key2 = b"key2"; 78 | 79 | let mut transaction = db.transaction(); 80 | transaction.put(DEFAULT_COLUMN_NAME, key1, key1); 81 | transaction.put(DEFAULT_COLUMN_NAME, key2, key2); 82 | db.write_buffered(transaction); 83 | 84 | let contents: Vec<_> = db.iter(DEFAULT_COLUMN_NAME).collect(); 85 | assert_eq!(contents.len(), 2); 86 | assert_eq!(&*contents[0].0, key1); 87 | assert_eq!(&*contents[0].1, key1); 88 | assert_eq!(&*contents[1].0, key2); 89 | assert_eq!(&*contents[1].1, key2); 90 | Ok(()) 91 | } 92 | 93 | /// A test for `KeyValueDB::iter_from_prefix`. 94 | pub fn test_iter_from_prefix(db: &dyn KeyValueDB) -> io::Result<()> { 95 | let key1 = b"0"; 96 | let key2 = b"ab"; 97 | let key3 = b"abc"; 98 | let key4 = b"abcd"; 99 | 100 | let mut batch = db.transaction(); 101 | batch.put(DEFAULT_COLUMN_NAME, key1, key1); 102 | batch.put(DEFAULT_COLUMN_NAME, key2, key2); 103 | batch.put(DEFAULT_COLUMN_NAME, key3, key3); 104 | batch.put(DEFAULT_COLUMN_NAME, key4, key4); 105 | db.write(batch)?; 106 | 107 | // empty prefix 108 | let contents: Vec<_> = db.iter_from_prefix(DEFAULT_COLUMN_NAME, b"").collect(); 109 | assert_eq!(contents.len(), 4); 110 | assert_eq!(&*contents[0].0, key1); 111 | assert_eq!(&*contents[1].0, key2); 112 | assert_eq!(&*contents[2].0, key3); 113 | assert_eq!(&*contents[3].0, key4); 114 | 115 | // prefix a 116 | let contents: Vec<_> = db.iter_from_prefix(DEFAULT_COLUMN_NAME, b"a").collect(); 117 | assert_eq!(contents.len(), 3); 118 | assert_eq!(&*contents[0].0, key2); 119 | assert_eq!(&*contents[1].0, key3); 120 | assert_eq!(&*contents[2].0, key4); 121 | 122 | // prefix abc 123 | let contents: Vec<_> = db.iter_from_prefix(DEFAULT_COLUMN_NAME, b"abc").collect(); 124 | assert_eq!(contents.len(), 2); 125 | assert_eq!(&*contents[0].0, key3); 126 | assert_eq!(&*contents[1].0, key4); 127 | 128 | // prefix abcde 129 | let contents: Vec<_> = db.iter_from_prefix(DEFAULT_COLUMN_NAME, b"abcde").collect(); 130 | assert_eq!(contents.len(), 0); 131 | 132 | // prefix 0 133 | let contents: Vec<_> = db.iter_from_prefix(DEFAULT_COLUMN_NAME, b"0").collect(); 134 | assert_eq!(contents.len(), 1); 135 | assert_eq!(&*contents[0].0, key1); 136 | Ok(()) 137 | } 138 | 139 | /// A test for `KeyValueDB::io_stats`. 140 | /// Assumes that the `db` has at least 3 columns. 141 | pub fn test_io_stats(db: &dyn KeyValueDB) -> io::Result<()> { 142 | let key1 = b"kkk"; 143 | let mut batch = db.transaction(); 144 | batch.put("0", key1, key1); 145 | batch.put("1", key1, key1); 146 | batch.put("2", key1, key1); 147 | 148 | for _ in 0..10 { 149 | db.get("0", key1)?; 150 | } 151 | 152 | db.write(batch)?; 153 | 154 | let io_stats = db.io_stats(IoStatsKind::SincePrevious); 155 | assert_eq!(io_stats.transactions, 1); 156 | assert_eq!(io_stats.writes, 3); 157 | assert_eq!(io_stats.bytes_written, 18); 158 | assert_eq!(io_stats.reads, 10); 159 | assert_eq!(io_stats.bytes_read, 30); 160 | 161 | let new_io_stats = db.io_stats(IoStatsKind::SincePrevious); 162 | // Since we taken previous statistic period, 163 | // this is expected to be totally empty. 164 | assert_eq!(new_io_stats.transactions, 0); 165 | 166 | // but the overall should be there 167 | let new_io_stats = db.io_stats(IoStatsKind::Overall); 168 | assert_eq!(new_io_stats.bytes_written, 18); 169 | 170 | let mut batch = db.transaction(); 171 | batch.delete("0", key1); 172 | batch.delete("1", key1); 173 | batch.delete("2", key1); 174 | 175 | // transaction is not commited yet 176 | assert_eq!(db.io_stats(IoStatsKind::SincePrevious).writes, 0); 177 | 178 | db.write(batch)?; 179 | // now it is, and delete is counted as write 180 | assert_eq!(db.io_stats(IoStatsKind::SincePrevious).writes, 3); 181 | Ok(()) 182 | } 183 | 184 | /// A complex test. 185 | pub fn test_complex(db: &dyn KeyValueDB) -> io::Result<()> { 186 | let key1 = b"02c69be41d0b7e40352fc85be1cd65eb03d40ef8427a0ca4596b1ead9a00e9fc"; 187 | let key2 = b"03c69be41d0b7e40352fc85be1cd65eb03d40ef8427a0ca4596b1ead9a00e9fc"; 188 | let key3 = b"04c00000000b7e40352fc85be1cd65eb03d40ef8427a0ca4596b1ead9a00e9fc"; 189 | let key4 = b"04c01111110b7e40352fc85be1cd65eb03d40ef8427a0ca4596b1ead9a00e9fc"; 190 | let key5 = b"04c02222220b7e40352fc85be1cd65eb03d40ef8427a0ca4596b1ead9a00e9fc"; 191 | 192 | let mut batch = db.transaction(); 193 | batch.put(DEFAULT_COLUMN_NAME, key1, b"cat"); 194 | batch.put(DEFAULT_COLUMN_NAME, key2, b"dog"); 195 | batch.put(DEFAULT_COLUMN_NAME, key3, b"caterpillar"); 196 | batch.put(DEFAULT_COLUMN_NAME, key4, b"beef"); 197 | batch.put(DEFAULT_COLUMN_NAME, key5, b"fish"); 198 | db.write(batch)?; 199 | 200 | assert_eq!(&*db.get(DEFAULT_COLUMN_NAME, key1)?.unwrap(), b"cat"); 201 | 202 | let contents: Vec<_> = db.iter(DEFAULT_COLUMN_NAME).collect(); 203 | assert_eq!(contents.len(), 5); 204 | assert_eq!(contents[0].0.to_vec(), key1.to_vec()); 205 | assert_eq!(&*contents[0].1, b"cat"); 206 | assert_eq!(contents[1].0.to_vec(), key2.to_vec()); 207 | assert_eq!(&*contents[1].1, b"dog"); 208 | 209 | let mut prefix_iter = db.iter_from_prefix(DEFAULT_COLUMN_NAME, b"04c0"); 210 | assert_eq!(*prefix_iter.next().unwrap().1, b"caterpillar"[..]); 211 | assert_eq!(*prefix_iter.next().unwrap().1, b"beef"[..]); 212 | assert_eq!(*prefix_iter.next().unwrap().1, b"fish"[..]); 213 | 214 | let mut batch = db.transaction(); 215 | batch.delete(DEFAULT_COLUMN_NAME, key1); 216 | db.write(batch)?; 217 | 218 | assert!(db.get(DEFAULT_COLUMN_NAME, key1)?.is_none()); 219 | 220 | let mut batch = db.transaction(); 221 | batch.put(DEFAULT_COLUMN_NAME, key1, b"cat"); 222 | db.write(batch)?; 223 | 224 | let mut transaction = db.transaction(); 225 | transaction.put(DEFAULT_COLUMN_NAME, key3, b"elephant"); 226 | transaction.delete(DEFAULT_COLUMN_NAME, key1); 227 | db.write(transaction)?; 228 | assert!(db.get(DEFAULT_COLUMN_NAME, key1)?.is_none()); 229 | assert_eq!(&*db.get(DEFAULT_COLUMN_NAME, key3)?.unwrap(), b"elephant"); 230 | 231 | assert_eq!( 232 | &*db.get_by_prefix(DEFAULT_COLUMN_NAME, key3).unwrap(), 233 | b"elephant" 234 | ); 235 | assert_eq!( 236 | &*db.get_by_prefix(DEFAULT_COLUMN_NAME, key2).unwrap(), 237 | b"dog" 238 | ); 239 | 240 | let mut transaction = db.transaction(); 241 | transaction.put(DEFAULT_COLUMN_NAME, key1, b"horse"); 242 | transaction.delete(DEFAULT_COLUMN_NAME, key3); 243 | db.write_buffered(transaction); 244 | assert!(db.get(DEFAULT_COLUMN_NAME, key3)?.is_none()); 245 | assert_eq!(&*db.get(DEFAULT_COLUMN_NAME, key1)?.unwrap(), b"horse"); 246 | 247 | db.flush()?; 248 | assert!(db.get(DEFAULT_COLUMN_NAME, key3)?.is_none()); 249 | assert_eq!(&*db.get(DEFAULT_COLUMN_NAME, key1)?.unwrap(), b"horse"); 250 | Ok(()) 251 | } 252 | -------------------------------------------------------------------------------- /paritylibs/kvdb/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | The format is based on [Keep a Changelog]. 4 | 5 | [Keep a Changelog]: http://keepachangelog.com/en/1.0.0/ 6 | 7 | ## [Unreleased] 8 | 9 | ## [0.4.0] - 2019-01-06 10 | - Bump parking_lot to 0.10. [#332](https://github.com/paritytech/parity-common/pull/332) 11 | 12 | ## [0.3.1] - 2019-01-06 13 | - Updated features and feature dependencies. [#307](https://github.com/paritytech/parity-common/pull/307) 14 | 15 | ## [0.3.0] - 2020-01-03 16 | - I/O statistics API. [#294](https://github.com/paritytech/parity-common/pull/294) 17 | - Removed `KeyValueDBHandler` trait. [#304](https://github.com/paritytech/parity-common/pull/304) 18 | 19 | ## [0.2.0] - 2019-12-19 20 | ### Changed 21 | - Default column support removed from the API 22 | - Column argument type changed from `Option` to `u32` 23 | - Migration `None` -> unsupported, `Some(0)` -> `0`, `Some(1)` -> `1`, etc. 24 | - Remove `ElasticArray` and change `DBValue` to be a type alias for `Vec` and add a `DBKey` backed by a `SmallVec`. (See [PR #282](https://github.com/paritytech/parity-common/pull/282/files)) 25 | 26 | ## [0.1.1] - 2019-10-24 27 | ### Dependencies 28 | - Updated dependencies (https://github.com/paritytech/parity-common/pull/239) 29 | ### Changed 30 | - Migrated to 2018 edition (https://github.com/paritytech/parity-common/pull/205) 31 | -------------------------------------------------------------------------------- /paritylibs/kvdb/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "kvdb" 3 | version = "0.4.0" 4 | authors = ["Parity Technologies "] 5 | repository = "https://github.com/paritytech/parity-common" 6 | description = "Generic key-value trait" 7 | license = "GPL-3.0" 8 | edition = "2018" 9 | 10 | [dependencies] 11 | smallvec = "1.0.0" 12 | lazy_static = "1.4" -------------------------------------------------------------------------------- /paritylibs/kvdb/src/io_stats.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2019 Parity Technologies (UK) Ltd. 2 | // This file is part of Parity. 3 | 4 | // Parity is free software: you can redistribute it and/or modify 5 | // it under the terms of the GNU General Public License as published by 6 | // the Free Software Foundation, either version 3 of the License, or 7 | // (at your option) any later version. 8 | 9 | // Parity is distributed in the hope that it will be useful, 10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | // GNU General Public License for more details. 13 | 14 | // You should have received a copy of the GNU General Public License 15 | // along with Parity. If not, see . 16 | 17 | //! Generic statistics for key-value databases 18 | 19 | /// Statistic kind to query. 20 | pub enum Kind { 21 | /// Overall statistics since start. 22 | Overall, 23 | /// Statistics since previous query. 24 | SincePrevious, 25 | } 26 | 27 | /// Statistic for the `span` period 28 | #[derive(Debug, Clone)] 29 | pub struct IoStats { 30 | /// Number of transaction. 31 | pub transactions: u64, 32 | /// Number of read operations. 33 | pub reads: u64, 34 | /// Number of reads resulted in a read from cache. 35 | pub cache_reads: u64, 36 | /// Number of write operations. 37 | pub writes: u64, 38 | /// Number of bytes read 39 | pub bytes_read: u64, 40 | /// Number of bytes read from cache 41 | pub cache_read_bytes: u64, 42 | /// Number of bytes write 43 | pub bytes_written: u64, 44 | /// Start of the statistic period. 45 | pub started: std::time::Instant, 46 | /// Total duration of the statistic period. 47 | pub span: std::time::Duration, 48 | } 49 | 50 | impl IoStats { 51 | /// Empty statistic report. 52 | pub fn empty() -> Self { 53 | Self { 54 | transactions: 0, 55 | reads: 0, 56 | cache_reads: 0, 57 | writes: 0, 58 | bytes_read: 0, 59 | cache_read_bytes: 0, 60 | bytes_written: 0, 61 | started: std::time::Instant::now(), 62 | span: std::time::Duration::default(), 63 | } 64 | } 65 | 66 | /// Average batch (transaction) size (writes per transaction) 67 | pub fn avg_batch_size(&self) -> f64 { 68 | if self.writes == 0 { 69 | return 0.0; 70 | } 71 | self.transactions as f64 / self.writes as f64 72 | } 73 | 74 | /// Read operations per second. 75 | pub fn reads_per_sec(&self) -> f64 { 76 | if self.span.as_secs_f64() == 0.0 { 77 | return 0.0; 78 | } 79 | 80 | self.reads as f64 / self.span.as_secs_f64() 81 | } 82 | 83 | pub fn byte_reads_per_sec(&self) -> f64 { 84 | if self.span.as_secs_f64() == 0.0 { 85 | return 0.0; 86 | } 87 | 88 | self.bytes_read as f64 / self.span.as_secs_f64() 89 | } 90 | 91 | /// Write operations per second. 92 | pub fn writes_per_sec(&self) -> f64 { 93 | if self.span.as_secs_f64() == 0.0 { 94 | return 0.0; 95 | } 96 | 97 | self.writes as f64 / self.span.as_secs_f64() 98 | } 99 | 100 | pub fn byte_writes_per_sec(&self) -> f64 { 101 | if self.span.as_secs_f64() == 0.0 { 102 | return 0.0; 103 | } 104 | 105 | self.bytes_written as f64 / self.span.as_secs_f64() 106 | } 107 | 108 | /// Total number of operations per second. 109 | pub fn ops_per_sec(&self) -> f64 { 110 | if self.span.as_secs_f64() == 0.0 { 111 | return 0.0; 112 | } 113 | 114 | (self.writes as f64 + self.reads as f64) / self.span.as_secs_f64() 115 | } 116 | 117 | /// Transactions per second. 118 | pub fn transactions_per_sec(&self) -> f64 { 119 | if self.span.as_secs_f64() == 0.0 { 120 | return 0.0; 121 | } 122 | 123 | (self.transactions as f64) / self.span.as_secs_f64() 124 | } 125 | 126 | pub fn avg_transaction_size(&self) -> f64 { 127 | if self.transactions == 0 { 128 | return 0.0; 129 | } 130 | 131 | self.bytes_written as f64 / self.transactions as f64 132 | } 133 | 134 | pub fn cache_hit_ratio(&self) -> f64 { 135 | if self.reads == 0 { 136 | return 0.0; 137 | } 138 | 139 | self.cache_reads as f64 / self.reads as f64 140 | } 141 | } 142 | -------------------------------------------------------------------------------- /paritylibs/kvdb/src/lib.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2019 Parity Technologies (UK) Ltd. 2 | // This file is part of Parity. 3 | 4 | // Parity is free software: you can redistribute it and/or modify 5 | // it under the terms of the GNU General Public License as published by 6 | // the Free Software Foundation, either version 3 of the License, or 7 | // (at your option) any later version. 8 | 9 | // Parity is distributed in the hope that it will be useful, 10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | // GNU General Public License for more details. 13 | 14 | // You should have received a copy of the GNU General Public License 15 | // along with Parity. If not, see . 16 | #![allow(clippy::type_complexity)] 17 | //! Key-Value store abstraction. 18 | 19 | use smallvec::SmallVec; 20 | use std::collections::HashMap; 21 | use std::io; 22 | use std::sync::{Arc, RwLock}; 23 | 24 | use lazy_static::lazy_static; 25 | 26 | mod io_stats; 27 | 28 | /// Required length of prefixes. 29 | pub const PREFIX_LEN: usize = 12; 30 | 31 | /// Database value. 32 | pub type DBValue = Vec; 33 | /// Database keys. 34 | pub type DBKey = SmallVec<[u8; 32]>; 35 | 36 | pub use io_stats::{IoStats, Kind as IoStatsKind}; 37 | 38 | /// Write transaction. Batches a sequence of put/delete operations for efficiency. 39 | #[derive(Default, Clone, PartialEq)] 40 | pub struct DBTransaction { 41 | /// Database operations. 42 | pub ops: Vec, 43 | } 44 | 45 | lazy_static! { 46 | // use `HashMap` rather than `HashSet` due to can't use `get(&str)` for `HashSet>` 47 | static ref CACHE: RwLock>> = Default::default(); 48 | } 49 | 50 | pub fn init_cache, L: AsRef<[S]>>(cols: L) { 51 | let mut cache = CACHE.write().unwrap(); 52 | for c in cols.as_ref().iter() { 53 | let col = c.as_ref(); 54 | cache.insert(col.to_owned(), Arc::new(col.to_owned())); 55 | } 56 | } 57 | 58 | fn column(col: &str) -> Arc { 59 | if let Some(s) = CACHE.read().unwrap().get(col) { 60 | return s.to_owned(); 61 | } 62 | // put new col name into cache. if col not in database column, would panic in database functions 63 | let s: Arc = Arc::new(col.to_owned()); 64 | CACHE.write().unwrap().insert(col.to_owned(), s.clone()); 65 | s 66 | } 67 | 68 | /// Database operation. 69 | #[derive(Clone, PartialEq)] 70 | pub enum DBOp { 71 | Insert { 72 | col: Arc, 73 | key: DBKey, 74 | value: DBValue, 75 | }, 76 | Delete { 77 | col: Arc, 78 | key: DBKey, 79 | }, 80 | } 81 | 82 | impl DBOp { 83 | /// Returns the key associated with this operation. 84 | pub fn key(&self) -> &[u8] { 85 | match *self { 86 | DBOp::Insert { ref key, .. } => key, 87 | DBOp::Delete { ref key, .. } => key, 88 | } 89 | } 90 | 91 | /// Returns the column associated with this operation. 92 | pub fn col(&self) -> &str { 93 | match self { 94 | DBOp::Insert { col, .. } => &col, 95 | DBOp::Delete { col, .. } => &col, 96 | } 97 | } 98 | } 99 | 100 | impl DBTransaction { 101 | /// Create new transaction. 102 | pub fn new() -> DBTransaction { 103 | DBTransaction::with_capacity(256) 104 | } 105 | 106 | /// Create new transaction with capacity. 107 | pub fn with_capacity(cap: usize) -> DBTransaction { 108 | DBTransaction { 109 | ops: Vec::with_capacity(cap), 110 | } 111 | } 112 | 113 | /// Insert a key-value pair in the transaction. Any existing value will be overwritten upon write. 114 | pub fn put(&mut self, col: &str, key: &[u8], value: &[u8]) { 115 | let col = column(col); 116 | self.ops.push(DBOp::Insert { 117 | col, 118 | key: DBKey::from_slice(key), 119 | value: value.to_vec(), 120 | }) 121 | } 122 | 123 | /// Insert a key-value pair in the transaction. Any existing value will be overwritten upon write. 124 | pub fn put_vec(&mut self, col: &str, key: &[u8], value: DBValue) { 125 | let col = column(col); 126 | self.ops.push(DBOp::Insert { 127 | col, 128 | key: DBKey::from_slice(key), 129 | value, 130 | }); 131 | } 132 | 133 | /// Delete value by key. 134 | pub fn delete(&mut self, col: &str, key: &[u8]) { 135 | let col = column(col); 136 | self.ops.push(DBOp::Delete { 137 | col, 138 | key: DBKey::from_slice(key), 139 | }); 140 | } 141 | } 142 | 143 | /// Generic key-value database. 144 | /// 145 | /// This makes a distinction between "buffered" and "flushed" values. Values which have been 146 | /// written can always be read, but may be present in an in-memory buffer. Values which have 147 | /// been flushed have been moved to backing storage, like a RocksDB instance. There are certain 148 | /// operations which are only guaranteed to operate on flushed data and not buffered, 149 | /// although implementations may differ in this regard. 150 | /// 151 | /// The contents of an interior buffer may be explicitly flushed using the `flush` method. 152 | /// 153 | /// The `KeyValueDB` also deals in "column families", which can be thought of as distinct 154 | /// stores within a database. Keys written in one column family will not be accessible from 155 | /// any other. The number of column families must be specified at initialization, with a 156 | /// differing interface for each database. The `None` argument in place of a column index 157 | /// is always supported. 158 | /// 159 | /// The API laid out here, along with the `Sync` bound implies interior synchronization for 160 | /// implementation. 161 | pub trait KeyValueDB: Sync + Send { 162 | /// Helper to create a new transaction. 163 | fn transaction(&self) -> DBTransaction { 164 | DBTransaction::new() 165 | } 166 | 167 | /// Get a value by key. 168 | fn get(&self, col: &str, key: &[u8]) -> io::Result>; 169 | 170 | /// Get a value by partial key. Only works for flushed data. 171 | fn get_by_prefix(&self, col: &str, prefix: &[u8]) -> Option>; 172 | 173 | /// Write a transaction of changes to the buffer. 174 | fn write_buffered(&self, transaction: DBTransaction); 175 | 176 | /// Write a transaction of changes to the backing store. 177 | fn write(&self, transaction: DBTransaction) -> io::Result<()> { 178 | self.write_buffered(transaction); 179 | self.flush() 180 | } 181 | 182 | /// Flush all buffered data. 183 | fn flush(&self) -> io::Result<()>; 184 | 185 | /// Iterate over flushed data for a given column. 186 | fn iter<'a>(&'a self, col: &str) -> Box, Box<[u8]>)> + 'a>; 187 | 188 | /// Iterate over flushed data for a given column, starting from a given prefix. 189 | fn iter_from_prefix<'a>( 190 | &'a self, 191 | col: &str, 192 | prefix: &'a [u8], 193 | ) -> Box, Box<[u8]>)> + 'a>; 194 | 195 | /// Attempt to replace this database with a new one located at the given path. 196 | fn restore(&self, new_db: &str) -> io::Result<()>; 197 | 198 | /// Query statistics. 199 | /// 200 | /// Not all kvdb implementations are able or expected to implement this, so by 201 | /// default, empty statistics is returned. Also, not all kvdb implementation 202 | /// can return every statistic or configured to do so (some statistics gathering 203 | /// may impede the performance and might be off by default). 204 | fn io_stats(&self, _kind: IoStatsKind) -> IoStats { 205 | IoStats::empty() 206 | } 207 | } 208 | -------------------------------------------------------------------------------- /rust-toolchain: -------------------------------------------------------------------------------- 1 | nightly-2019-12-20 2 | --------------------------------------------------------------------------------