├── store ├── src │ ├── api │ │ ├── error.rs │ │ ├── mod.rs │ │ ├── transaction.rs │ │ └── block.rs │ ├── verify │ │ ├── mod.rs │ │ └── header.rs │ ├── db │ │ ├── db_block.rs │ │ └── mod.rs │ ├── header.rs │ ├── lib.rs │ ├── pow │ │ └── mod.rs │ ├── util.rs │ ├── record.rs │ ├── hash.rs │ └── transaction.rs ├── Cargo.toml └── tests │ ├── empty_db.rs │ ├── get_tx.rs │ ├── util.rs │ ├── import.rs │ └── blk_file.rs ├── serde_network ├── .gitignore ├── README.md ├── Cargo.toml ├── src │ ├── lib.rs │ └── error.rs ├── tests │ └── test.rs └── benches │ └── bincode.rs ├── doc ├── api.md └── bitcrust-db.md ├── test-load ├── monitor ├── src │ └── main.rs └── Cargo.toml ├── serde_json ├── .gitignore ├── rustfmt.toml ├── tests │ ├── ui │ │ ├── unexpected_colon.stderr │ │ ├── unexpected_comma.stderr │ │ ├── parse_key.stderr │ │ ├── not_found.stderr │ │ ├── missing_colon.stderr │ │ ├── missing_value.stderr │ │ ├── parse_expr.stderr │ │ ├── missing_colon.rs │ │ ├── missing_value.rs │ │ ├── not_found.rs │ │ ├── parse_expr.rs │ │ ├── parse_key.rs │ │ ├── unexpected_colon.rs │ │ ├── unexpected_comma.rs │ │ └── update-references.sh │ ├── deps │ │ ├── Cargo.toml │ │ └── src │ │ │ └── lib.rs │ ├── preserve_order.rs │ ├── compiletest.rs │ ├── macros │ │ └── mod.rs │ └── stream.rs ├── .travis.yml ├── appveyor.yml ├── LICENSE-MIT ├── travis.sh ├── Cargo.toml ├── CONTRIBUTING.md └── src │ ├── iter.rs │ └── value │ └── partial_eq.rs ├── hashstore ├── .gitignore ├── Cargo.toml ├── src │ ├── timer.rs │ ├── lib.rs │ ├── values.rs │ └── header.rs ├── README.md └── tests │ └── main.rs ├── site ├── assets │ ├── qr.png │ ├── storage.png │ ├── tomasvdw.jpeg │ ├── bc_blocklogo.png │ ├── storage.0x260.png │ ├── storage.200x0.png │ ├── tomasvdw.0x260.jpeg │ ├── tomasvdw.200x0.jpeg │ ├── bc_blocklogo.200x0.png │ ├── bitcrust_top_logo.png │ ├── bitcrust_landing_logo.png │ ├── bitcrust_top_logo.200x0.png │ ├── bitcrust_top_logo.260x0.png │ ├── bitcrust_top_logo_notype.png │ ├── bitcrust_landing_logo.0x260.png │ ├── bitcrust_landing_logo.200x0.png │ └── bitcrust_top_logo_notype.200x0.png ├── script │ └── main.js ├── results.html ├── features.html └── faq.html ├── .gitignore ├── bench-init ├── encode-derive ├── Cargo.toml ├── tests │ └── test.rs └── src │ └── lib.rs ├── src ├── script │ ├── mod.rs │ └── context.rs ├── lib.rs ├── README.md ├── store │ ├── blockheaderptr.rs │ ├── README.md │ ├── txptr.rs │ ├── prune.rs │ ├── spend_index.rs │ └── tips.rs ├── util.rs ├── config.rs ├── api.rs ├── ffi.rs ├── merkle_tree.rs ├── metrics.rs └── hash.rs ├── bitcrustd ├── Cargo.toml └── src │ ├── client_message.rs │ ├── db_query.rs │ ├── util.rs │ └── main.rs ├── net ├── Cargo.toml └── src │ ├── message │ ├── inv_message.rs │ ├── header_message.rs │ ├── getdata_message.rs │ ├── notfound_message.rs │ ├── getblocks_message.rs │ ├── getheaders_message.rs │ ├── sendcmpct_message.rs │ ├── block_message.rs │ ├── bitcrust │ │ └── mod.rs │ ├── addr_message.rs │ ├── transaction_message.rs │ └── version_message.rs │ ├── inventory_vector.rs │ ├── lib.rs │ ├── block_header.rs │ ├── transactions.rs │ ├── var_int.rs │ ├── services.rs │ └── net_addr.rs ├── Cargo.toml ├── .travis.yml ├── bench-next ├── LICENSE.txt ├── tests ├── blk_file.rs ├── test_bench.rs ├── test_file1.rs └── compare_core.rs ├── README.md └── tool └── gen_compare.py /store/src/api/error.rs: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /store/src/verify/mod.rs: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /serde_network/.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | /Cargo.lock 3 | -------------------------------------------------------------------------------- /doc/api.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | # API 5 | 6 | ## Tip management 7 | 8 | 9 | -------------------------------------------------------------------------------- /store/src/verify/header.rs: -------------------------------------------------------------------------------- 1 | 2 | 3 | //! Block header verification 4 | -------------------------------------------------------------------------------- /test-load: -------------------------------------------------------------------------------- 1 | cargo test file_large --release -- --ignored --nocapture 2 | -------------------------------------------------------------------------------- /monitor/src/main.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | println!("Hello, world!"); 3 | } 4 | -------------------------------------------------------------------------------- /serde_json/.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | **/*.rs.bk 3 | *.sw[po] 4 | Cargo.lock 5 | -------------------------------------------------------------------------------- /hashstore/.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | **/*.rs.bk 3 | Cargo.lock 4 | .idea 5 | testdb/ 6 | -------------------------------------------------------------------------------- /store/src/api/mod.rs: -------------------------------------------------------------------------------- 1 | 2 | 3 | pub mod transaction; 4 | pub mod block; 5 | 6 | 7 | -------------------------------------------------------------------------------- /site/assets/qr.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tomasvdw/bitcrust/HEAD/site/assets/qr.png -------------------------------------------------------------------------------- /site/assets/storage.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tomasvdw/bitcrust/HEAD/site/assets/storage.png -------------------------------------------------------------------------------- /site/assets/tomasvdw.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tomasvdw/bitcrust/HEAD/site/assets/tomasvdw.jpeg -------------------------------------------------------------------------------- /site/assets/bc_blocklogo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tomasvdw/bitcrust/HEAD/site/assets/bc_blocklogo.png -------------------------------------------------------------------------------- /site/assets/storage.0x260.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tomasvdw/bitcrust/HEAD/site/assets/storage.0x260.png -------------------------------------------------------------------------------- /site/assets/storage.200x0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tomasvdw/bitcrust/HEAD/site/assets/storage.200x0.png -------------------------------------------------------------------------------- /site/assets/tomasvdw.0x260.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tomasvdw/bitcrust/HEAD/site/assets/tomasvdw.0x260.jpeg -------------------------------------------------------------------------------- /site/assets/tomasvdw.200x0.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tomasvdw/bitcrust/HEAD/site/assets/tomasvdw.200x0.jpeg -------------------------------------------------------------------------------- /site/assets/bc_blocklogo.200x0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tomasvdw/bitcrust/HEAD/site/assets/bc_blocklogo.200x0.png -------------------------------------------------------------------------------- /site/assets/bitcrust_top_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tomasvdw/bitcrust/HEAD/site/assets/bitcrust_top_logo.png -------------------------------------------------------------------------------- /site/assets/bitcrust_landing_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tomasvdw/bitcrust/HEAD/site/assets/bitcrust_landing_logo.png -------------------------------------------------------------------------------- /site/assets/bitcrust_top_logo.200x0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tomasvdw/bitcrust/HEAD/site/assets/bitcrust_top_logo.200x0.png -------------------------------------------------------------------------------- /site/assets/bitcrust_top_logo.260x0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tomasvdw/bitcrust/HEAD/site/assets/bitcrust_top_logo.260x0.png -------------------------------------------------------------------------------- /site/assets/bitcrust_top_logo_notype.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tomasvdw/bitcrust/HEAD/site/assets/bitcrust_top_logo_notype.png -------------------------------------------------------------------------------- /site/assets/bitcrust_landing_logo.0x260.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tomasvdw/bitcrust/HEAD/site/assets/bitcrust_landing_logo.0x260.png -------------------------------------------------------------------------------- /site/assets/bitcrust_landing_logo.200x0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tomasvdw/bitcrust/HEAD/site/assets/bitcrust_landing_logo.200x0.png -------------------------------------------------------------------------------- /site/assets/bitcrust_top_logo_notype.200x0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tomasvdw/bitcrust/HEAD/site/assets/bitcrust_top_logo_notype.200x0.png -------------------------------------------------------------------------------- /serde_network/README.md: -------------------------------------------------------------------------------- 1 | # serde_network 2 | 3 | Serialization format for Bitcoin Transactions and Blocks 4 | 5 | Only supports the few types necessary 6 | -------------------------------------------------------------------------------- /serde_json/rustfmt.toml: -------------------------------------------------------------------------------- 1 | fn_args_layout = "Block" 2 | array_layout = "Block" 3 | where_style = "Rfc" 4 | generics_indent = "Block" 5 | fn_call_style = "Block" 6 | -------------------------------------------------------------------------------- /monitor/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "bitcrust-monitor" 3 | version = "0.1.0" 4 | authors = ["Tomas van der Wansem "] 5 | 6 | [dependencies] 7 | -------------------------------------------------------------------------------- /serde_json/tests/ui/unexpected_colon.stderr: -------------------------------------------------------------------------------- 1 | error: expected expression, found `:` 2 | --> $DIR/unexpected_colon.rs:13:13 3 | | 4 | 13 | json!({ : true }); 5 | | ^ 6 | 7 | -------------------------------------------------------------------------------- /serde_json/tests/ui/unexpected_comma.stderr: -------------------------------------------------------------------------------- 1 | error: expected expression, found `,` 2 | --> $DIR/unexpected_comma.rs:13:17 3 | | 4 | 13 | json!({ "a" , "b": true }); 5 | | ^ 6 | 7 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | target 3 | /test-lmdb/ 4 | core-blocks 5 | data-bench 6 | 7 | Cargo.lock 8 | launch.json 9 | .vscode 10 | *.swp 11 | 12 | tmp/ 13 | data/ 14 | .idea/ 15 | .cargo/ 16 | bitcrust.iml 17 | 18 | store/tst-* 19 | -------------------------------------------------------------------------------- /store/src/db/db_block.rs: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | //! db_block embeds which transactions are included in a block. 5 | //! It is represents as a vector of 64-bit Records 6 | 7 | use record::Record; 8 | 9 | type DbBlock = Vec; 10 | -------------------------------------------------------------------------------- /serde_json/tests/ui/parse_key.stderr: -------------------------------------------------------------------------------- 1 | error[E0609]: no field `s` on type `&'static str` 2 | --> $DIR/parse_key.rs:13:16 3 | | 4 | 13 | json!({ "".s : true }); 5 | | ^ 6 | 7 | error: aborting due to previous error 8 | 9 | -------------------------------------------------------------------------------- /serde_json/tests/deps/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "serde_test_suite_deps" 3 | version = "0.0.0" 4 | authors = ["David Tolnay "] 5 | publish = false 6 | 7 | [workspace] 8 | 9 | [dependencies] 10 | serde_json = { path = "../.." } 11 | -------------------------------------------------------------------------------- /serde_json/tests/ui/not_found.stderr: -------------------------------------------------------------------------------- 1 | error[E0425]: cannot find value `x` in this scope 2 | --> $DIR/not_found.rs:13:19 3 | | 4 | 13 | json!({ "a" : x }); 5 | | ^ not found in this scope 6 | 7 | error: aborting due to previous error 8 | 9 | -------------------------------------------------------------------------------- /hashstore/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "hashstore" 3 | version = "0.1.0" 4 | authors = ["Tomas van der Wansem "] 5 | 6 | [dependencies] 7 | memmap = "0.5" 8 | bincode = "*" 9 | serde = "*" 10 | serde_derive = "*" 11 | 12 | [dev-dependencies] 13 | rand = "0.3" 14 | 15 | -------------------------------------------------------------------------------- /bench-init: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | 4 | ln -s ~/.bitcoin/blocks core-blocks 5 | 6 | BITCRUST_STORE=data-bench \ 7 | cargo test --release load_bench_init -- --ignored --nocapture 8 | 9 | BITCRUST_NOCLEAR=1 BITCRUST_STORE=data-bench \ 10 | cargo test --release prune_tx_index -- --ignored --nocapture 11 | 12 | -------------------------------------------------------------------------------- /encode-derive/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "encode-derive" 3 | version = "0.1.0" 4 | authors = ["Chris MacNaughton "] 5 | 6 | [dependencies] 7 | syn = "0.11.11" 8 | quote = "0.3.15" 9 | 10 | [lib] 11 | proc-macro = true 12 | 13 | [dev-dependencies] 14 | bitcrust-net = {path = "../net"} -------------------------------------------------------------------------------- /serde_json/tests/ui/missing_colon.stderr: -------------------------------------------------------------------------------- 1 | error: unexpected end of macro invocation 2 | --> $DIR/missing_colon.rs:13:5 3 | | 4 | 13 | json!({ "a" }); 5 | | ^^^^^^^^^^^^^^^ 6 | | 7 | = note: this error originates in a macro outside of the current crate (run with -Z external-macro-backtrace for more info) 8 | 9 | error: aborting due to previous error 10 | 11 | -------------------------------------------------------------------------------- /serde_json/tests/ui/missing_value.stderr: -------------------------------------------------------------------------------- 1 | error: unexpected end of macro invocation 2 | --> $DIR/missing_value.rs:13:5 3 | | 4 | 13 | json!({ "a" : }); 5 | | ^^^^^^^^^^^^^^^^^ 6 | | 7 | = note: this error originates in a macro outside of the current crate (run with -Z external-macro-backtrace for more info) 8 | 9 | error: aborting due to previous error 10 | 11 | -------------------------------------------------------------------------------- /serde_json/tests/ui/parse_expr.stderr: -------------------------------------------------------------------------------- 1 | error: unexpected end of macro invocation 2 | --> $DIR/parse_expr.rs:13:5 3 | | 4 | 13 | json!({ "a" : ~ }); 5 | | ^^^^^^^^^^^^^^^^^^^ 6 | | 7 | = note: this error originates in a macro outside of the current crate (run with -Z external-macro-backtrace for more info) 8 | 9 | error: aborting due to previous error 10 | 11 | -------------------------------------------------------------------------------- /serde_network/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "serde_network" 3 | version = "0.0.1" 4 | authors = ["Tomas van der Wansem "] 5 | 6 | [dependencies] 7 | 8 | byteorder = "1" 9 | itertools = "0.5.1" 10 | ring = "0.12.1" 11 | 12 | serde = "*" 13 | serde_derive = "*" 14 | 15 | serde_json = { path = "../serde_json" } 16 | serde_network = { path = "../serde_network" } 17 | 18 | hashstore = { path = "../hashstore" } 19 | -------------------------------------------------------------------------------- /src/script/mod.rs: -------------------------------------------------------------------------------- 1 | //! SCRIPTING interface 2 | //! 3 | //! CURRENTLY ONLY USED FOR DISPLAY 4 | //! 5 | //! libbitcoinconsensus is used for script verification 6 | 7 | 8 | pub mod context; 9 | 10 | pub mod opcode; 11 | 12 | mod opcode_pushdata; 13 | 14 | pub mod stack; 15 | 16 | #[derive(Debug, PartialEq, Eq)] 17 | pub enum ScriptError { 18 | StackUnderflow, 19 | NumericOverflow, 20 | 21 | UnexpectedEndOfScript, 22 | 23 | InvalidOpcode, 24 | 25 | PushdataTooLarge 26 | } 27 | -------------------------------------------------------------------------------- /serde_json/tests/ui/missing_colon.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Serde Developers 2 | // 3 | // Licensed under the Apache License, Version 2.0 or the MIT license 5 | // , at your 6 | // option. This file may not be copied, modified, or distributed 7 | // except according to those terms. 8 | 9 | #[macro_use] 10 | extern crate serde_json; 11 | 12 | fn main() { 13 | json!({ "a" }); 14 | } 15 | -------------------------------------------------------------------------------- /serde_json/tests/ui/missing_value.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Serde Developers 2 | // 3 | // Licensed under the Apache License, Version 2.0 or the MIT license 5 | // , at your 6 | // option. This file may not be copied, modified, or distributed 7 | // except according to those terms. 8 | 9 | #[macro_use] 10 | extern crate serde_json; 11 | 12 | fn main() { 13 | json!({ "a" : }); 14 | } 15 | -------------------------------------------------------------------------------- /serde_json/tests/ui/not_found.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Serde Developers 2 | // 3 | // Licensed under the Apache License, Version 2.0 or the MIT license 5 | // , at your 6 | // option. This file may not be copied, modified, or distributed 7 | // except according to those terms. 8 | 9 | #[macro_use] 10 | extern crate serde_json; 11 | 12 | fn main() { 13 | json!({ "a" : x }); 14 | } 15 | -------------------------------------------------------------------------------- /serde_json/tests/ui/parse_expr.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Serde Developers 2 | // 3 | // Licensed under the Apache License, Version 2.0 or the MIT license 5 | // , at your 6 | // option. This file may not be copied, modified, or distributed 7 | // except according to those terms. 8 | 9 | #[macro_use] 10 | extern crate serde_json; 11 | 12 | fn main() { 13 | json!({ "a" : ~ }); 14 | } 15 | -------------------------------------------------------------------------------- /serde_json/tests/ui/parse_key.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Serde Developers 2 | // 3 | // Licensed under the Apache License, Version 2.0 or the MIT license 5 | // , at your 6 | // option. This file may not be copied, modified, or distributed 7 | // except according to those terms. 8 | 9 | #[macro_use] 10 | extern crate serde_json; 11 | 12 | fn main() { 13 | json!({ "".s : true }); 14 | } 15 | -------------------------------------------------------------------------------- /serde_json/tests/ui/unexpected_colon.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Serde Developers 2 | // 3 | // Licensed under the Apache License, Version 2.0 or the MIT license 5 | // , at your 6 | // option. This file may not be copied, modified, or distributed 7 | // except according to those terms. 8 | 9 | #[macro_use] 10 | extern crate serde_json; 11 | 12 | fn main() { 13 | json!({ : true }); 14 | } 15 | -------------------------------------------------------------------------------- /store/src/api/transaction.rs: -------------------------------------------------------------------------------- 1 | 2 | use db::*; 3 | 4 | 5 | pub fn transaction_get(db: &mut Db, hash: &[u8;32]) -> Result, DbError> { 6 | let r = db_transaction::read_transaction(db, hash)?; 7 | Ok(r) 8 | } 9 | 10 | pub enum TransactionPutFlags { 11 | 12 | } 13 | 14 | pub enum TransactionPutOk { 15 | ValidationError, 16 | OkOrphan(Vec<[u8;32]>), 17 | Ok 18 | } 19 | 20 | pub fn transaction_put(_tx: ::Transaction ) -> Result<(), DbError> { 21 | unimplemented!() 22 | } -------------------------------------------------------------------------------- /serde_json/tests/ui/unexpected_comma.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Serde Developers 2 | // 3 | // Licensed under the Apache License, Version 2.0 or the MIT license 5 | // , at your 6 | // option. This file may not be copied, modified, or distributed 7 | // except according to those terms. 8 | 9 | #[macro_use] 10 | extern crate serde_json; 11 | 12 | fn main() { 13 | json!({ "a" , "b": true }); 14 | } 15 | -------------------------------------------------------------------------------- /bitcrustd/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "bitcrustd" 3 | version = "0.1.0" 4 | authors = ["Chris MacNaughton "] 5 | 6 | [dependencies] 7 | bitcrust-net = {path = "../net"} 8 | store = {path = "../store"} 9 | clap = "~2.25" 10 | simple_logger = "*" 11 | log = "0.3" 12 | multiqueue = "~0.3.2" 13 | ring = "0.12" 14 | serde_derive = "1.0" 15 | serde = "1.0" 16 | toml = "0.4" 17 | rusqlite = "~0.11" 18 | 19 | serde_json = {path = "../serde_json"} 20 | 21 | [dev-dependencies] 22 | tempfile = "2.2" 23 | -------------------------------------------------------------------------------- /serde_json/tests/deps/src/lib.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Serde Developers 2 | // 3 | // Licensed under the Apache License, Version 2.0 or the MIT license 5 | // , at your 6 | // option. This file may not be copied, modified, or distributed 7 | // except according to those terms. 8 | 9 | #![feature(/*=============================================] 10 | #![=== Serde test suite requires a nightly compiler. ===] 11 | #![====================================================*/)] 12 | -------------------------------------------------------------------------------- /serde_json/appveyor.yml: -------------------------------------------------------------------------------- 1 | environment: 2 | matrix: 3 | - APPVEYOR_RUST_CHANNEL: stable 4 | - APPVEYOR_RUST_CHANNEL: nightly 5 | 6 | install: 7 | # Install rust, x86_64-pc-windows-msvc host 8 | - appveyor DownloadFile https://win.rustup.rs/ -FileName rustup-init.exe 9 | - rustup-init.exe -y --default-host x86_64-pc-windows-msvc --default-toolchain %APPVEYOR_RUST_CHANNEL% 10 | - set PATH=C:\msys64\usr\bin;%PATH%;C:\Users\appveyor\.cargo\bin 11 | - rustc -vV 12 | - cargo -vV 13 | 14 | build: false 15 | 16 | test_script: 17 | - sh -c 'PATH=`rustc --print sysroot`/bin:$PATH ./travis.sh' 18 | -------------------------------------------------------------------------------- /store/src/header.rs: -------------------------------------------------------------------------------- 1 | 2 | use hash::*; 3 | 4 | use serde_network; 5 | 6 | /// Header represents the header of a block 7 | #[derive(Debug, Clone, Serialize, Deserialize)] 8 | pub struct Header { 9 | 10 | // db and network 11 | pub version: u32, 12 | pub prev_hash: Hash, 13 | pub merkle_root: Hash, 14 | pub time: u32, 15 | pub bits: u32, 16 | pub nonce: u32, 17 | } 18 | 19 | 20 | impl Header { 21 | pub fn new(raw_header: &[u8]) -> Result { 22 | serde_network::deserialize(raw_header) 23 | } 24 | 25 | } 26 | 27 | 28 | -------------------------------------------------------------------------------- /net/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "bitcrust-net" 3 | version = "0.1.0" 4 | authors = [ 5 | "Chris MacNaughton ", 6 | "Tomas van der Wansem " 7 | ] 8 | 9 | [dependencies] 10 | bitflags = "0.7" 11 | circular = "~0.2" 12 | byteorder = "1" 13 | log = "0.3" 14 | multiqueue = "~0.3.2" 15 | # nom = "^2.2" 16 | rand = "0.3" 17 | regex = "*" 18 | ring = "0.12" 19 | rusqlite = "~0.11" 20 | sha2 = "0.5" 21 | encode-derive = {path = "../encode-derive"} 22 | 23 | [dependencies.nom] 24 | version = "^2.2" 25 | features = ["verbose-errors", "regexp"] 26 | 27 | [dev-dependencies] 28 | env_logger = "0.4.2" -------------------------------------------------------------------------------- /store/src/lib.rs: -------------------------------------------------------------------------------- 1 | 2 | extern crate itertools; 3 | extern crate hashstore; 4 | 5 | extern crate serde; 6 | #[macro_use] 7 | extern crate serde_derive; 8 | 9 | extern crate serde_network; 10 | 11 | mod api; 12 | mod db; 13 | mod util; 14 | mod hash; 15 | mod record; 16 | mod transaction; 17 | mod header; 18 | mod pow; 19 | 20 | pub use transaction::Transaction; 21 | pub use header::Header; 22 | 23 | pub use db::db_transaction::DbTransaction; 24 | pub use db::db_header::DbHeader; 25 | 26 | use hashstore::ValuePtr; 27 | 28 | 29 | pub use api::transaction::*; 30 | pub use api::block::*; 31 | 32 | pub use db::{Db, DbError, init, init_empty}; 33 | 34 | pub use hash::double_sha256; 35 | 36 | -------------------------------------------------------------------------------- /hashstore/src/timer.rs: -------------------------------------------------------------------------------- 1 | 2 | 3 | use std::time; 4 | use std::sync::atomic; 5 | 6 | /// Struct for maintaining timings of operations 7 | pub struct Timer { 8 | start: time::Instant, 9 | stat: &'static atomic::AtomicU64 10 | } 11 | 12 | impl Timer { 13 | pub fn new(stat: &'static atomic::AtomicU64) -> Self { 14 | Timer { 15 | start: time::Instant::now(), 16 | stat: stat 17 | } 18 | } 19 | } 20 | 21 | impl Drop for Timer { 22 | fn drop(&mut self) { 23 | let elapsed = self.start.elapsed(); 24 | let elapsed = (elapsed.as_secs() as u64 * 1_000_000_000) 25 | + elapsed.subsec_nanos() as u64; 26 | self.stat.fetch_add(elapsed, atomic::Ordering::Relaxed); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /net/src/message/inv_message.rs: -------------------------------------------------------------------------------- 1 | use inventory_vector::InventoryVector; 2 | use {Encode, VarInt}; 3 | 4 | #[cfg(test)] 5 | mod tests { 6 | use super::*; 7 | 8 | #[test] 9 | fn it_implements_types_required_for_protocol() { 10 | let m = InvMessage::default(); 11 | assert_eq!(m.name(), "inv"); 12 | assert_eq!(m.len(), 8); 13 | } 14 | } 15 | #[derive(Debug, Default, Encode, PartialEq)] 16 | pub struct InvMessage { 17 | #[count] 18 | pub inventory: Vec, 19 | } 20 | 21 | impl InvMessage { 22 | #[inline] 23 | pub fn len(&self) -> usize { 24 | 8 + (36 * self.inventory.len()) 25 | } 26 | 27 | #[inline] 28 | pub fn name(&self) -> &'static str { 29 | "inv" 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /serde_network/src/lib.rs: -------------------------------------------------------------------------------- 1 | extern crate byteorder; 2 | 3 | extern crate serde; 4 | use serde::{Serialize, Deserialize}; 5 | 6 | mod ser; 7 | pub use ser::Serializer; 8 | mod de; 9 | pub use de::Deserializer; 10 | mod error; 11 | pub use error::{Error, Result}; 12 | pub use ser::encode_compact_size; 13 | pub use de::decode_compact_size; 14 | 15 | 16 | pub fn serialize(out: &mut Vec, value: &T) 17 | where T: Serialize 18 | { 19 | let mut ser = Serializer::new(out); 20 | Serialize::serialize(value, &mut ser) 21 | .expect("Internal error in serializer"); // can't happen writing to buffer 22 | } 23 | 24 | pub fn deserialize<'de, T>(bytes: &'de [u8]) -> Result 25 | where T: Deserialize<'de> 26 | { 27 | Deserializer::new(bytes).deserialize() 28 | } 29 | -------------------------------------------------------------------------------- /net/src/message/header_message.rs: -------------------------------------------------------------------------------- 1 | use {Encode, VarInt}; 2 | use block_header::BlockHeader; 3 | 4 | #[cfg(test)] 5 | mod tests { 6 | use super::*; 7 | 8 | #[test] 9 | fn it_implements_types_required_for_protocol() { 10 | let m = HeaderMessage::default(); 11 | assert_eq!(m.name(), "headers"); 12 | assert_eq!(m.len(), 8); 13 | } 14 | } 15 | 16 | #[derive(Debug, Default, Encode, PartialEq)] 17 | pub struct HeaderMessage { 18 | pub count: VarInt, 19 | pub headers: Vec, 20 | } 21 | 22 | impl HeaderMessage { 23 | #[inline] 24 | pub fn len(&self) -> usize { 25 | 8 + (81 * self.headers.len()) 26 | } 27 | 28 | #[inline] 29 | pub fn name(&self) -> &'static str { 30 | "headers" 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /net/src/message/getdata_message.rs: -------------------------------------------------------------------------------- 1 | use inventory_vector::InventoryVector; 2 | use {Encode, VarInt}; 3 | 4 | #[cfg(test)] 5 | mod tests { 6 | use super::*; 7 | 8 | #[test] 9 | fn it_implements_types_required_for_protocol() { 10 | let m = GetdataMessage::default(); 11 | assert_eq!(m.name(), "getdata"); 12 | assert_eq!(m.len(), 8); 13 | } 14 | } 15 | 16 | #[derive(Debug, Default, Encode, PartialEq)] 17 | pub struct GetdataMessage { 18 | #[count] 19 | pub inventory: Vec, 20 | } 21 | 22 | impl GetdataMessage { 23 | #[inline] 24 | pub fn len(&self) -> usize { 25 | 8 + (36 * self.inventory.len()) 26 | } 27 | 28 | #[inline] 29 | pub fn name(&self) -> &'static str { 30 | "getdata" 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /net/src/message/notfound_message.rs: -------------------------------------------------------------------------------- 1 | use inventory_vector::InventoryVector; 2 | use {Encode, VarInt}; 3 | 4 | #[cfg(test)] 5 | mod tests { 6 | use super::*; 7 | 8 | #[test] 9 | fn it_implements_types_required_for_protocol() { 10 | let m = NotfoundMessage::default(); 11 | assert_eq!(m.name(), "notfound"); 12 | assert_eq!(m.len(), 8); 13 | } 14 | } 15 | #[derive(Debug, Default, Encode, PartialEq)] 16 | pub struct NotfoundMessage { 17 | #[count] 18 | pub inventory: Vec, 19 | } 20 | 21 | impl NotfoundMessage { 22 | #[inline] 23 | pub fn len(&self) -> usize { 24 | 8 + (36 * self.inventory.len()) 25 | } 26 | 27 | #[inline] 28 | pub fn name(&self) -> &'static str { 29 | "notfound" 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /net/src/message/getblocks_message.rs: -------------------------------------------------------------------------------- 1 | use {Encode, VarInt}; 2 | 3 | #[cfg(test)] 4 | mod tests { 5 | use super::*; 6 | 7 | #[test] 8 | fn it_implements_types_required_for_protocol() { 9 | let m = GetblocksMessage::default(); 10 | assert_eq!(m.name(), "getblocks"); 11 | assert_eq!(m.len(), 45); 12 | } 13 | } 14 | 15 | #[derive(Debug, Default, Encode, PartialEq)] 16 | pub struct GetblocksMessage { 17 | pub version: u32, 18 | #[count] 19 | pub locator_hashes: Vec<[u8; 32]>, 20 | pub hash_stop: [u8; 32], 21 | } 22 | 23 | impl GetblocksMessage { 24 | #[inline] 25 | pub fn len(&self) -> usize { 26 | 4 + 9 + ( self.locator_hashes.len() * 32 ) + 32 27 | } 28 | 29 | #[inline] 30 | pub fn name(&self) -> &'static str { 31 | "getblocks" 32 | } 33 | } -------------------------------------------------------------------------------- /net/src/message/getheaders_message.rs: -------------------------------------------------------------------------------- 1 | use {Encode, VarInt}; 2 | 3 | #[cfg(test)] 4 | mod tests { 5 | use super::*; 6 | 7 | #[test] 8 | fn it_implements_types_required_for_protocol() { 9 | let m = GetheadersMessage::default(); 10 | assert_eq!(m.name(), "getheaders"); 11 | assert_eq!(m.len(), 45); 12 | } 13 | } 14 | 15 | #[derive(Debug, Default, Encode, PartialEq)] 16 | pub struct GetheadersMessage { 17 | pub version: u32, 18 | #[count] 19 | pub locator_hashes: Vec<[u8; 32]>, 20 | pub hash_stop: [u8; 32], 21 | } 22 | 23 | impl GetheadersMessage { 24 | #[inline] 25 | pub fn len(&self) -> usize { 26 | 4 + 9 + ( self.locator_hashes.len() * 32 ) + 32 27 | } 28 | 29 | #[inline] 30 | pub fn name(&self) -> &'static str { 31 | "getheaders" 32 | } 33 | } -------------------------------------------------------------------------------- /store/tests/empty_db.rs: -------------------------------------------------------------------------------- 1 | 2 | 3 | extern crate store; 4 | extern crate serde_json; 5 | mod util; 6 | 7 | fn hash_from_slice(slice: &[u8]) -> [u8;32] { 8 | let mut result = [0;32]; 9 | result.copy_from_slice(&slice[0..32]); 10 | result 11 | } 12 | 13 | 14 | #[test] 15 | fn test_empty() { 16 | let mut db = store::init_empty("tst-empty").unwrap(); 17 | 18 | // get genesis coinbase tx 19 | let dbtx = store::transaction_get(&mut db, &hash_from_slice(&util::from_hex_rev( 20 | "4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b") 21 | )).unwrap().unwrap(); 22 | 23 | let tx = dbtx.as_tx().unwrap(); 24 | println!("{}", serde_json::to_string_pretty(&tx).unwrap()); 25 | assert_eq!(tx.version, 1); 26 | assert_eq!(tx.txs_out[0].value, 50 * 100_000_000); 27 | } 28 | 29 | //10mb/sec => 13000 -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "bitcrust" 3 | version = "0.1.0" 4 | authors = ["Tomas van der Wansem "] 5 | 6 | 7 | [lib] 8 | name = "bitcrust_lib" 9 | path = "src/lib.rs" 10 | 11 | 12 | [dependencies] 13 | byteorder = "0.5" 14 | 15 | itertools = "0.5.1" 16 | memmap = "0.4" 17 | libc = "0.2.30" 18 | rand = "0.3" 19 | ring = "0.12" 20 | 21 | serde = "1" 22 | serde_derive = "1" 23 | toml = "0.4" 24 | 25 | slog = { version = "1.3.2", features = ["max_level_trace", "release_max_level_info"] } 26 | slog-term = "1.3.2" 27 | 28 | rayon = "0.6" 29 | 30 | 31 | [dev-dependencies] 32 | tempdir = "0.3" 33 | 34 | 35 | 36 | 37 | 38 | [profile.release] 39 | lto = true 40 | opt-level = 3 41 | 42 | [features] 43 | default = [] 44 | 45 | [workspace] 46 | members = [ "bitcrustd", "monitor", "net", "encode-derive", "store", "hashstore"] 47 | 48 | -------------------------------------------------------------------------------- /net/src/inventory_vector.rs: -------------------------------------------------------------------------------- 1 | use Encode; 2 | 3 | #[derive(Debug, Default, Encode, PartialEq)] 4 | pub struct InventoryVector { 5 | flags: InvFlags, 6 | pub hash: [u8; 32], 7 | } 8 | 9 | bitflags! { 10 | #[derive(Default, Encode)] 11 | flags InvFlags: u32 { 12 | const ERROR = 0b0, 13 | const MSG_TX = 0b00000001, 14 | const MSG_BLOCK = 0b00000010, 15 | const MSG_FILTERED_BLOCK = 0b00000100, 16 | const MSG_CMPCT_BLOCK = 0b00001000 17 | } 18 | } 19 | 20 | impl InventoryVector { 21 | pub fn new(flags: u32, hash: &[u8]) -> InventoryVector { 22 | debug_assert!(hash.len() == 32); 23 | let mut a: [u8; 32] = Default::default(); 24 | a.copy_from_slice(&hash); 25 | InventoryVector { 26 | flags: InvFlags { bits: flags }, 27 | hash: a, 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /net/src/message/sendcmpct_message.rs: -------------------------------------------------------------------------------- 1 | use Encode; 2 | 3 | #[cfg(test)] 4 | mod tests { 5 | use super::*; 6 | 7 | #[test] 8 | fn it_implements_types_required_for_protocol() { 9 | let m = SendCmpctMessage::default(); 10 | assert_eq!(m.name(), "sendcmpct"); 11 | assert_eq!(m.len(), 9); 12 | } 13 | } 14 | /// 15 | /// https://github.com/bitcoin/bips/blob/master/bip-0152.mediawiki 16 | /// 17 | /// Setting the send_compact field to 1 enables the high-bandwidth 18 | /// mode specified in the above bip. 19 | 20 | #[derive(Debug, Default, Encode, PartialEq)] 21 | pub struct SendCmpctMessage { 22 | pub send_compact: bool, 23 | pub version: u64, 24 | } 25 | 26 | impl SendCmpctMessage { 27 | #[inline] 28 | pub fn len(&self) -> usize { 29 | 9 30 | } 31 | 32 | #[inline] 33 | pub fn name(&self) -> &'static str { 34 | "sendcmpct" 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | 2 | 3 | #![feature(link_args)] 4 | #![feature(custom_derive, plugin)] 5 | #![feature(integer_atomics)] 6 | 7 | 8 | 9 | 10 | extern crate memmap; 11 | extern crate itertools; 12 | extern crate rand; 13 | extern crate ring; 14 | extern crate rayon; 15 | 16 | #[macro_use] 17 | pub extern crate slog ; 18 | extern crate slog_term ; 19 | 20 | 21 | /// Macro to create and empty a storage folder; used by tests 22 | macro_rules! test_cfg { 23 | () => (::config::Config::new_empty(format!("{}-{}", file!(), line!()))) 24 | } 25 | 26 | 27 | 28 | mod hash; 29 | 30 | #[macro_use] 31 | mod builders; 32 | 33 | pub mod metrics; 34 | pub mod transaction; 35 | pub mod block; 36 | pub mod script; 37 | 38 | mod ffi; 39 | mod buffer; 40 | mod util; 41 | mod store; 42 | mod config; 43 | mod merkle_tree; 44 | mod block_add; 45 | mod api; 46 | 47 | 48 | pub use store::Store; 49 | 50 | 51 | pub use api::*; 52 | 53 | -------------------------------------------------------------------------------- /src/README.md: -------------------------------------------------------------------------------- 1 | # Bitcurst-db source 2 | 3 | These sources provide the library that stores, verifies and retrieves blocks and 4 | transactions. It is intended to be used by bitcoin network node software, mining software and 5 | utility tools. 6 | 7 | The interface itself is not yet complete; it only provides an [add_block](lib.rs#L71-74) method. 8 | 9 | This procedure is handled in [block_add.rs](block_add.rs) and uses a [store](store/) to access the filesystem. 10 | 11 | Concurrent access is allowed by using multiple Store instances both from different threads as well as from 12 | different processes; multiple processes can concurrently read and write to the same datafiles. 13 | 14 | Script validation is handled by libbitcoinconsensus via [ffi](ffi.rs) 15 | 16 | The library uses a *deserialize-only* model supported by [buffer](buffer.rs); a reference to the binary block is kept 17 | such that serialization is not needed. -------------------------------------------------------------------------------- /src/store/blockheaderptr.rs: -------------------------------------------------------------------------------- 1 | 2 | 3 | //! A BlockheaderPtr is a pointer to a blockheader in the blockheader-content fileset 4 | //! It contains a file_number, a file offset that together point to the data 5 | 6 | //! 7 | //! It is not stored directly, but can be extracted from the Record structure, that is stored 8 | //! in the Spend-Tree 9 | 10 | 11 | use super::flatfileset::FlatFilePtr; 12 | 13 | #[derive(Debug, Copy, Clone)] 14 | pub struct BlockHeaderPtr { 15 | file_offset: u32, 16 | file_number: i16, 17 | } 18 | 19 | 20 | impl FlatFilePtr for BlockHeaderPtr { 21 | 22 | fn new(file_number: i16, file_offset: u64) -> Self { 23 | BlockHeaderPtr { 24 | file_offset: file_offset as u32, 25 | file_number: file_number 26 | } 27 | } 28 | 29 | fn get_file_number(self) -> i16 { self.file_number } 30 | 31 | fn get_file_offset(self) -> u64 { self.file_offset as u64 } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /serde_json/tests/preserve_order.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Serde Developers 2 | // 3 | // Licensed under the Apache License, Version 2.0 or the MIT license 5 | // , at your 6 | // option. This file may not be copied, modified, or distributed 7 | // except according to those terms. 8 | 9 | extern crate serde_json; 10 | 11 | use serde_json::{from_str, Value}; 12 | 13 | #[test] 14 | fn test_map_order() { 15 | // Sorted order 16 | #[cfg(not(feature = "preserve_order"))] 17 | const EXPECTED: &[&str] = &["a", "b", "c"]; 18 | 19 | // Insertion order 20 | #[cfg(feature = "preserve_order")] 21 | const EXPECTED: &[&str] = &["b", "a", "c"]; 22 | 23 | let v: Value = from_str(r#"{"b":null,"a":null,"c":null}"#).unwrap(); 24 | let keys: Vec<_> = v.as_object().unwrap().keys().collect(); 25 | assert_eq!(keys, EXPECTED); 26 | } 27 | -------------------------------------------------------------------------------- /serde_json/tests/compiletest.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Serde Developers 2 | // 3 | // Licensed under the Apache License, Version 2.0 or the MIT license 5 | // , at your 6 | // option. This file may not be copied, modified, or distributed 7 | // except according to those terms. 8 | 9 | extern crate compiletest_rs as compiletest; 10 | 11 | use std::env; 12 | 13 | fn run_mode(mode: &'static str) { 14 | let mut config = compiletest::Config::default(); 15 | 16 | config.mode = mode.parse().expect("invalid mode"); 17 | config.target_rustcflags = Some("-L tests/deps/target/debug/deps".to_owned()); 18 | if let Ok(name) = env::var("TESTNAME") { 19 | config.filter = Some(name); 20 | } 21 | config.src_base = format!("tests/{}", mode).into(); 22 | 23 | compiletest::run_tests(&config); 24 | } 25 | 26 | #[test] 27 | fn ui() { 28 | run_mode("ui"); 29 | } 30 | -------------------------------------------------------------------------------- /bitcrustd/src/client_message.rs: -------------------------------------------------------------------------------- 1 | use bitcrust_net::NetAddr; 2 | 3 | #[cfg(test)] 4 | mod tests { 5 | use super::*; 6 | 7 | #[test] 8 | fn it_is_clonable_with_peers() { 9 | let input = ClientMessage::PeersConnected(12); 10 | let output = input.clone(); 11 | 12 | assert_eq!(input, output); 13 | } 14 | 15 | #[test] 16 | fn it_is_clonable_with_addrs() { 17 | let input = ClientMessage::Addrs(vec![]); 18 | let output = input.clone(); 19 | 20 | assert_eq!(input, output); 21 | } 22 | 23 | #[test] 24 | fn it_is_clonable_with_closing() { 25 | let input = ClientMessage::Closing("localhost:8192".into()); 26 | let output = input.clone(); 27 | 28 | assert_eq!(input, output); 29 | } 30 | } 31 | 32 | #[derive(Clone, Debug, PartialEq)] 33 | pub enum ClientMessage { 34 | Addrs(Vec), 35 | PeersConnected(u64), 36 | /// Expects a hostname argument 37 | Closing(String), 38 | } 39 | -------------------------------------------------------------------------------- /net/src/lib.rs: -------------------------------------------------------------------------------- 1 | #![feature(tcpstream_connect_timeout)] 2 | 3 | #[macro_use] 4 | extern crate bitflags; 5 | extern crate byteorder; 6 | extern crate circular; 7 | #[macro_use] 8 | extern crate log; 9 | extern crate multiqueue; 10 | #[macro_use] 11 | extern crate nom; 12 | extern crate rand; 13 | extern crate ring; 14 | extern crate regex; 15 | extern crate rusqlite; 16 | extern crate sha2; 17 | 18 | #[macro_use] 19 | extern crate encode_derive; 20 | 21 | pub mod bitcoin_network_connection; 22 | mod block_header; 23 | mod encode; 24 | // pub mod client_message; 25 | mod parser; 26 | mod message; 27 | mod inventory_vector; 28 | mod net_addr; 29 | mod services; 30 | mod transactions; 31 | mod var_int; 32 | 33 | pub use encode::Encode; 34 | pub use message::*; 35 | pub use net_addr::NetAddr; 36 | // pub use client_message::ClientMessage; 37 | pub use bitcoin_network_connection::{BitcoinNetworkConnection, BitcoinNetworkError}; 38 | pub use block_header::BlockHeader; 39 | pub use services::Services; 40 | pub use var_int::VarInt; 41 | -------------------------------------------------------------------------------- /site/script/main.js: -------------------------------------------------------------------------------- 1 | // Sticky Header 2 | $(window).scroll(function() { 3 | 4 | if ($(window).scrollTop() > 400) { 5 | $('.main_h').addClass('sticky'); 6 | } else { 7 | $('.main_h').removeClass('sticky'); 8 | } 9 | }); 10 | 11 | // Mobile Navigation 12 | $('.mobile-toggle').click(function() { 13 | if ($('.main_h').hasClass('open-nav')) { 14 | $('.main_h').removeClass('open-nav'); 15 | } else { 16 | $('.main_h').addClass('open-nav'); 17 | } 18 | }); 19 | 20 | $('.main_h li a').click(function() { 21 | if ($('.main_h').hasClass('open-nav')) { 22 | $('.navigation').removeClass('open-nav'); 23 | $('.main_h').removeClass('open-nav'); 24 | } 25 | }); 26 | 27 | // Navigation Scroll 28 | $('nav a').click(function(event) { 29 | var id = $(this).attr("href"); 30 | var offset = 20; 31 | var target = $(id).offset().top - offset; 32 | $('html, body').animate({ 33 | scrollTop: target 34 | }, 500); 35 | event.preventDefault(); 36 | }); 37 | 38 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: true 2 | dist: trusty 3 | 4 | language: rust 5 | 6 | cache: 7 | - apt 8 | - cargo 9 | 10 | rust: 11 | - nightly 12 | 13 | before_install: 14 | - sudo apt-get update -q 15 | - sudo apt-get install libcurl4-openssl-dev libelf-dev libdw-dev libssl-dev libevent-dev libboost-all-dev binutils-dev 16 | 17 | install: 18 | - git clone https://github.com/bitcoin/bitcoin 19 | - cd bitcoin 20 | - ./autogen.sh && ./configure --without-daemon --without-utils --without-gui --disable-wallet --disable-bench --disable-zmq --disable-man --disable-tests --disable-gui-tests && make && sudo make install 21 | - sudo ldconfig /usr/local/lib 22 | - cd .. 23 | 24 | # load travis-cargo 25 | before_script: 26 | - | 27 | cargo install cargo-travis && 28 | export PATH=$HOME/.cargo/bin:$PATH 29 | 30 | # the main build 31 | script: 32 | - | 33 | cargo build --all && 34 | cargo test --all 35 | 36 | after_success: 37 | - cargo coveralls --exclude-pattern tests/,script/,bitcoin/src/,encode-derive/ --all 38 | 39 | 40 | -------------------------------------------------------------------------------- /net/src/block_header.rs: -------------------------------------------------------------------------------- 1 | use Encode; 2 | use VarInt; 3 | 4 | #[derive(Debug, Encode, PartialEq)] 5 | /// 4 version int32_t Block version information (note, this is signed) 6 | /// 32 prev_block char[32] The hash value of the previous block this particular block references 7 | /// 32 merkle_root char[32] The reference to a Merkle tree collection which is a hash of all transactions related to this block 8 | /// 4 timestamp uint32_t A timestamp recording when this block was created (Will overflow in 2106[2]) 9 | /// 4 bits uint32_t The calculated difficulty target being used for this block 10 | /// 4 nonce uint32_t The nonce used to generate this block… to allow variations of the header and compute different hashes 11 | /// 1 txn_count var_int Number of transaction entries, this value is always 0 12 | pub struct BlockHeader { 13 | pub version: i32, 14 | pub prev_block: [u8; 32], 15 | pub merkle_root: [u8; 32], 16 | pub timestamp: u32, 17 | pub bits: u32, 18 | pub nonce: u32, 19 | /// txn_count is a var_int on the wire 20 | pub txn_count: VarInt, 21 | } 22 | -------------------------------------------------------------------------------- /store/src/pow/mod.rs: -------------------------------------------------------------------------------- 1 | 2 | 3 | mod u256; 4 | 5 | pub use self::u256::U256; 6 | 7 | // Converts a header "nbits" representation to a U256 difficulty target 8 | // This doesn't check errors; the correctness of a compact target u32 (nbits) is 9 | // is tested in header-validation 10 | pub fn from_compact(compact_target: u32) -> U256 { 11 | 12 | let size = compact_target as usize >> 24; 13 | let mut word = U256::from((compact_target as u64) & 0x007f_ffff); 14 | let x = if size <= 3 { 15 | word >> (8 * (3 - size)) 16 | } 17 | else { 18 | word << (8 * (size - 3)) 19 | }; 20 | x 21 | 22 | } 23 | 24 | // Converts the difficulty target (= maximum hash to find) to work, 25 | // which is its reciprocal 26 | // we multiply by constant 2^256 to keep ensure the results are integral 27 | pub fn difficulty_target_to_work(target: U256) -> U256 { 28 | // We find: 29 | // (2^256) / (target+1) 30 | // = ((2^256 - (target+1))/ (target+1)) - 1 31 | // = (!target / (target+1)) + 1 32 | ((!target) / (target + U256::one())) + U256::one() 33 | } 34 | -------------------------------------------------------------------------------- /bench-next: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | if ! [ -d data-bench ] 4 | then 5 | echo "Run bench-init first" 6 | exit 1 7 | fi 8 | 9 | if ! [ -d data-bench2 ] 10 | then 11 | mkdir data-bench2 12 | 13 | # We link the transactions to save space 14 | cd data-bench2 15 | ln -s ../data-bench/transactions1 16 | ln -s ../data-bench/transactions2 17 | cd .. 18 | 19 | fi 20 | 21 | echo "Removing" 22 | rm -rf data-bench2/block-index 23 | rm -rf data-bench2/headers 24 | rm -rf data-bench2/spend-tree 25 | rm -rf data-bench2/spend-index 26 | rm -rf data-bench2/tx-index 27 | 28 | echo "Copying" 29 | cp -r data-bench/block-index data-bench2/block-index 30 | cp -r data-bench/headers data-bench2/headers 31 | cp -r data-bench/spend-index data-bench2/spend-index 32 | cp -r data-bench/spend-tree data-bench2/spend-tree 33 | cp -r data-bench/tx-index data-bench2/tx-index 34 | 35 | ln -s ~/.bitcoin/blocks core-blocks 36 | 37 | echo "Copy complete. benching" 38 | 39 | BITCRUST_NOCLEAR=1 BITCRUST_STORE=data-bench2 \ 40 | cargo test --release load_bench_next -- --ignored --nocapture 41 | 42 | 43 | -------------------------------------------------------------------------------- /serde_network/src/error.rs: -------------------------------------------------------------------------------- 1 | use std::{self, error, io, str}; 2 | use std::fmt::{self, Display}; 3 | use serde; 4 | 5 | #[derive(Debug)] 6 | pub enum Error { 7 | EndOfBufferError, 8 | IOError(io::Error) 9 | } 10 | 11 | pub type Result = std::result::Result; 12 | 13 | 14 | 15 | 16 | impl error::Error for Error { 17 | fn description(&self) -> &str { 18 | match *self { 19 | Error::EndOfBufferError => "Unexpected end of buffer", 20 | Error::IOError(ref io) => io.description() 21 | } 22 | } 23 | } 24 | 25 | impl serde::de::Error for Error { 26 | fn custom(_desc: T) -> Error { 27 | Error::EndOfBufferError 28 | } 29 | } 30 | 31 | impl serde::ser::Error for Error { 32 | fn custom(_msg: T) -> Self { 33 | Error::EndOfBufferError 34 | } 35 | } 36 | 37 | impl Display for Error { 38 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 39 | f.write_str("Serde_network error") 40 | } 41 | } 42 | 43 | 44 | impl From for Error { 45 | fn from(err: io::Error) -> Self { 46 | Error::IOError(err) 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2017 The Bitcrust developers 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /serde_json/LICENSE-MIT: -------------------------------------------------------------------------------- 1 | Copyright (c) 2014 The Rust Project Developers 2 | 3 | Permission is hereby granted, free of charge, to any 4 | person obtaining a copy of this software and associated 5 | documentation files (the "Software"), to deal in the 6 | Software without restriction, including without 7 | limitation the rights to use, copy, modify, merge, 8 | publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software 10 | is furnished to do so, subject to the following 11 | conditions: 12 | 13 | The above copyright notice and this permission notice 14 | shall be included in all copies or substantial portions 15 | of the Software. 16 | 17 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF 18 | ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED 19 | TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 20 | PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT 21 | SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY 22 | CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 23 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR 24 | IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 25 | DEALINGS IN THE SOFTWARE. 26 | -------------------------------------------------------------------------------- /src/util.rs: -------------------------------------------------------------------------------- 1 | //! General utility functions 2 | 3 | 4 | 5 | /// Used mainly for tests; found somewhere (rustc_serialize I think) 6 | pub fn from_hex(str: &str) -> Vec { 7 | 8 | // This may be an overestimate if there is any whitespace 9 | let mut b = Vec::with_capacity(str.len() / 2); 10 | let mut modulus = 0; 11 | let mut buf = 08; 12 | 13 | for byte in str.bytes() { 14 | buf <<= 4; 15 | 16 | match byte { 17 | b'A'...b'F' => buf |= byte - b'A' + 10, 18 | b'a'...b'f' => buf |= byte - b'a' + 10, 19 | b'0'...b'9' => buf |= byte - b'0', 20 | b' '|b'\r'|b'\n'|b'\t' => { 21 | buf >>= 4; 22 | continue 23 | } 24 | _ => { 25 | panic!("Invalid hex char"); 26 | } 27 | } 28 | 29 | modulus += 1; 30 | if modulus == 2 { 31 | modulus = 0; 32 | b.push(buf); 33 | } 34 | } 35 | 36 | b.into_iter().collect() 37 | } 38 | 39 | /// Useful to keep hashes in the same format as usually printed 40 | pub fn from_hex_rev(str: &str) -> Vec { 41 | let mut v = from_hex(str); 42 | v.reverse(); 43 | v 44 | } 45 | -------------------------------------------------------------------------------- /serde_json/travis.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 6 | 7 | channel() { 8 | if [ -n "${TRAVIS}" ]; then 9 | if [ "${TRAVIS_RUST_VERSION}" = "${CHANNEL}" ]; then 10 | pwd 11 | (set -x; cargo "$@") 12 | fi 13 | elif [ -n "${APPVEYOR}" ]; then 14 | if [ "${APPVEYOR_RUST_CHANNEL}" = "${CHANNEL}" ]; then 15 | pwd 16 | (set -x; cargo "$@") 17 | fi 18 | else 19 | pwd 20 | (set -x; cargo "+${CHANNEL}" "$@") 21 | fi 22 | } 23 | 24 | if [ -n "${CLIPPY}" ]; then 25 | # cached installation will not work on a later nightly 26 | if [ -n "${TRAVIS}" ] && ! cargo install clippy --debug --force; then 27 | echo "COULD NOT COMPILE CLIPPY, IGNORING CLIPPY TESTS" 28 | exit 29 | fi 30 | 31 | cargo clippy -- -Dclippy 32 | else 33 | CHANNEL=nightly 34 | channel clean 35 | channel build 36 | (cd "$DIR/tests/deps" && channel build) 37 | channel test 38 | channel test --features preserve_order 39 | 40 | for CHANNEL in stable 1.15.0 1.16.0 1.17.0 beta; do 41 | channel clean 42 | channel build 43 | channel build --features preserve_order 44 | done 45 | fi 46 | -------------------------------------------------------------------------------- /encode-derive/tests/test.rs: -------------------------------------------------------------------------------- 1 | 2 | #[macro_use] 3 | extern crate encode_derive; 4 | extern crate bitcrust_net; 5 | 6 | use bitcrust_net::{Encode, VarInt}; 7 | 8 | #[derive(Encode)] 9 | struct TestStructWithCount { 10 | name: String, 11 | #[count] 12 | data: Vec 13 | } 14 | 15 | #[derive(Encode)] 16 | struct TestStruct { 17 | name: String, 18 | data: [u8; 32] 19 | } 20 | 21 | #[test] 22 | fn it_encodes() { 23 | let t = TestStruct { 24 | name: "TestStruct".into(), 25 | data: [0x00; 32], 26 | }; 27 | let mut encoded = vec![]; 28 | let _ = t.encode(&mut encoded); 29 | assert_eq!(encoded, vec![ 30 | // name 31 | 84, 101, 115, 116, 83, 116, 114, 117, 99, 116, 32 | // data 33 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]); 34 | } 35 | 36 | #[test] 37 | fn it_encodes_with_count() { 38 | let t = TestStructWithCount { 39 | name: "TestStruct".into(), 40 | data: vec![0x00], 41 | }; 42 | let mut encoded = vec![]; 43 | let _ = t.encode(&mut encoded); 44 | assert_eq!(encoded, vec![ 45 | // name 46 | 84, 101, 115, 116, 83, 116, 114, 117, 99, 116, 47 | // len 48 | 1, 49 | // data 50 | 0]); 51 | } -------------------------------------------------------------------------------- /bitcrustd/src/db_query.rs: -------------------------------------------------------------------------------- 1 | 2 | use clap::ArgMatches; 3 | use config::Config; 4 | use util::*; 5 | use store; 6 | 7 | use serde_json; 8 | 9 | pub fn db_query(matches: &ArgMatches, config: &Config) { 10 | 11 | let db = &mut store::init(&config.data_dir).unwrap(); 12 | 13 | 14 | match matches.subcommand() { 15 | ("get-transaction", Some(txhash)) => { 16 | let txhash = txhash.value_of("tx-hash").unwrap(); 17 | let tx_db = store::transaction_get(db, &hash_from_hex(txhash)) 18 | .unwrap() 19 | .expect("Not found"); 20 | 21 | let tx = tx_db.as_tx().unwrap(); 22 | 23 | println!("{}", serde_json::to_string_pretty(&tx).unwrap()); 24 | 25 | 26 | }, 27 | ("get-block", Some(block_hash)) => { 28 | let block_hash = block_hash.value_of("block-hash").unwrap(); 29 | let hdr_db = store::header_get(db, &hash_from_hex(block_hash)) 30 | .unwrap() 31 | .expect("Not found"); 32 | 33 | println!("{}", serde_json::to_string_pretty(&hdr_db).unwrap()); 34 | 35 | 36 | }, 37 | ("", None) => println!("No subcommand was used"), // If no subcommand was usd it'll match the tuple ("", None) 38 | _ => unreachable!(), // If all subcommands are defined above, anything else is unreachabe!() 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /serde_json/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "serde_json" 3 | version = "1.0.6" # remember to update html_root_url 4 | authors = ["Erick Tryzelaar ", "David Tolnay "] 5 | license = "MIT/Apache-2.0" 6 | description = "A JSON serialization file format" 7 | repository = "https://github.com/serde-rs/json" 8 | documentation = "http://docs.serde.rs/serde_json/" 9 | keywords = ["json", "serde", "serialization"] 10 | categories = ["encoding"] 11 | readme = "README.md" 12 | include = ["Cargo.toml", "src/**/*.rs", "README.md", "LICENSE-APACHE", "LICENSE-MIT"] 13 | 14 | [badges] 15 | travis-ci = { repository = "serde-rs/json" } 16 | appveyor = { repository = "serde-rs/json" } 17 | 18 | [dependencies] 19 | serde = "1.0" 20 | num-traits = "0.1.32" 21 | linked-hash-map = { version = "0.5", optional = true } 22 | itoa = "0.3" 23 | dtoa = "0.4" 24 | 25 | [dev-dependencies] 26 | compiletest_rs = "0.2" 27 | serde_bytes = "0.10" 28 | serde_derive = "1.0" 29 | 30 | 31 | ### FEATURES ################################################################# 32 | 33 | [features] 34 | default = [] 35 | 36 | # Use LinkedHashMap rather than BTreeMap as the map type of serde_json::Value. 37 | # This allows data to be read into a Value and written back to a JSON string 38 | # while preserving the order of map keys in the input. 39 | preserve_order = ["linked-hash-map"] 40 | -------------------------------------------------------------------------------- /hashstore/src/lib.rs: -------------------------------------------------------------------------------- 1 | #![feature(integer_atomics)] 2 | //! 3 | //! Key/value store with specific design properties 4 | //! optimized for storing the transactions of a blockchain 5 | //! 6 | //! Apart from getting and setting values, it allows for 7 | //! getting *dependent* values. If a dependency X of Y is requested 8 | //! and X is missing, an anchor will be inserted to ensure that the 9 | //! dependency is resolved before X is inserted. 10 | //! 11 | //! This allows a caller that wants to insert B with a dependency A to 12 | //! 13 | //! * Get dependency A 14 | //! * If successful, verify the dependency 15 | //! * If not, ignore and store B anyway. 16 | //! * When A comes in later and is stored, set will fail and the dependency A->B 17 | //! can still be verified 18 | //! 19 | //! Other design considerations 20 | //! 21 | //! * Keys are always 32-byte hashes 22 | //! * Concurrent, lock-free R/W access across processes 23 | //! * Append only 24 | //! * Allows seeking only recent keys 25 | //! 26 | //! The implementation uses single root hash table, and MRU liked list of objects 27 | //! to solve collisions. 28 | //! 29 | //! See [HashStore](struct.HashStore.html) for examples 30 | 31 | 32 | #[macro_use] 33 | extern crate serde_derive; 34 | extern crate bincode; 35 | 36 | mod header; 37 | mod io; 38 | mod values; 39 | mod timer; 40 | mod hashstore; 41 | 42 | pub use hashstore::{HashStoreError, HashStore, SearchDepth}; 43 | pub use values::ValuePtr; 44 | 45 | 46 | -------------------------------------------------------------------------------- /net/src/transactions.rs: -------------------------------------------------------------------------------- 1 | use Encode; 2 | use VarInt; 3 | 4 | #[derive(Debug, Encode, PartialEq)] 5 | pub struct TransactionInput { 6 | pub previous: Outpoint, 7 | #[count] 8 | pub script: Vec, 9 | pub sequence: u32, 10 | } 11 | 12 | impl TransactionInput { 13 | pub fn len(&self) -> usize { 14 | 36 + // previous Outpoint 15 | 4 + // length of script 16 | self.script.len() + 17 | 4 // sequence 18 | } 19 | } 20 | 21 | #[derive(Debug, Encode, PartialEq)] 22 | pub struct Outpoint { 23 | pub hash: [u8; 32], 24 | pub index: u32, 25 | } 26 | 27 | impl Outpoint { 28 | pub fn new(index: u32, hash: &[u8]) -> Outpoint { 29 | debug_assert!(hash.len() == 32); 30 | let mut a: [u8; 32] = Default::default(); 31 | a.copy_from_slice(&hash); 32 | Outpoint { 33 | index: index, 34 | hash: a, 35 | } 36 | } 37 | } 38 | 39 | #[derive(Debug, Encode, PartialEq)] 40 | pub struct TransactionOutput{ 41 | pub value: i64, 42 | #[count] 43 | pub pk_script: Vec, 44 | } 45 | 46 | impl TransactionOutput { 47 | pub fn len(&self) -> usize { 48 | 8 + // Transaction Value 49 | 4 + // length of script 50 | self.pk_script.len() 51 | } 52 | } 53 | 54 | #[derive(Debug, Encode, PartialEq)] 55 | pub struct Witness { 56 | #[count] 57 | pub component: Vec 58 | } 59 | 60 | impl Witness { 61 | pub fn len(&self) -> usize { 62 | 8 * self.component.len() 63 | } 64 | } -------------------------------------------------------------------------------- /hashstore/src/values.rs: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | /// A persistent pointer to a value in the database 6 | /// It contains the bitfields: 7 | /// 8 | /// * bit 0-47 file position 9 | /// * bit 48-53 size of object: X such that size is at most 1 << X bytes 10 | /// 11 | /// This mod are some helper functions to encode/decode dataptrs 12 | 13 | pub type ValuePtr = u64; 14 | 15 | 16 | // A prefix for every value in the database 17 | #[derive(Default, Serialize, Deserialize)] 18 | pub struct ValuePrefix { 19 | pub key: [u8; 32], 20 | pub prev_pos: u64, 21 | pub size: u32, 22 | pub time: u32 23 | } 24 | 25 | 26 | 27 | pub fn ptr_new(filepos: u64, sz: usize) -> ValuePtr { 28 | 29 | // compress size: find S such that size is at least 2^(S+6) 30 | let s = (sz as u64) 31 | .next_power_of_two() 32 | .trailing_zeros() as u64; 33 | 34 | filepos | (s << 48) 35 | } 36 | 37 | // Returns an *estimate* of the size of the object 38 | pub fn ptr_size_est(dataptr: ValuePtr) -> usize { 39 | (1 << ((dataptr >> 48) & 0x1F)) as usize 40 | } 41 | 42 | 43 | pub fn ptr_file_pos(dataptr: ValuePtr) -> u64 { 44 | dataptr & 0xFFFF_FFFF_FFFF 45 | } 46 | 47 | #[cfg(test)] 48 | mod tests { 49 | use super::*; 50 | 51 | #[test] 52 | fn test_size() { 53 | // test size encode/decode 54 | for n in 0..2000000 { 55 | let dp = ptr_new(0, n); 56 | let sz = ptr_size_est(dp); 57 | assert!(sz >= n, format!("n={}", n)); 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /hashstore/src/header.rs: -------------------------------------------------------------------------------- 1 | 2 | 3 | use bincode; 4 | use std::io; 5 | use std::io::{Read,Write}; 6 | 7 | 8 | 9 | #[derive(Serialize, Deserialize, Copy,Clone)] 10 | pub struct Header { 11 | magic_file_id: u64, 12 | pub root_bits: u8, 13 | _reserved: [u8;7], 14 | pub extrema: [u64;8], 15 | pub stats: [u64;8] 16 | } 17 | 18 | pub const MAGIC_FILE_ID: u64 = 0x485348_53544f5231; 19 | 20 | pub fn header_size_u64() -> usize { 21 | return ::std::mem::size_of::
() / 8; 22 | } 23 | 24 | pub fn stats_offset_u64() -> usize { 25 | return 1+ 1 + 8; // field offset of stats 26 | } 27 | 28 | pub fn extrema_offset_u64() -> usize { 29 | return 1+ 1 ; // field offset of extrema 30 | } 31 | 32 | 33 | impl Header { 34 | 35 | pub fn new(root_bits: u8) -> Self { 36 | Header { 37 | magic_file_id: MAGIC_FILE_ID, 38 | root_bits: root_bits, 39 | _reserved: [0u8;7], 40 | extrema: [0; 8], 41 | stats: [0;8] 42 | } 43 | } 44 | 45 | 46 | pub fn is_correct_fileid(&self) -> bool { 47 | self.magic_file_id == MAGIC_FILE_ID 48 | } 49 | 50 | pub fn read(rdr: &mut R) -> Result { 51 | bincode::deserialize_from(rdr, bincode::Infinite) 52 | .map_err(|err| io::Error::new(io::ErrorKind::Other, err)) 53 | } 54 | 55 | pub fn write(wrt: &mut W, hdr: &Header) -> Result<(), io::Error> { 56 | bincode::serialize_into(wrt, hdr, bincode::Infinite) 57 | .map_err(|err| io::Error::new(io::ErrorKind::Other, err)) 58 | } 59 | } -------------------------------------------------------------------------------- /tests/blk_file.rs: -------------------------------------------------------------------------------- 1 | //! 2 | //! Module to access bitcoin-core style blk files 3 | //! that store the integral blockchain 4 | 5 | 6 | extern crate byteorder; 7 | 8 | use byteorder::{ReadBytesExt, LittleEndian}; 9 | 10 | 11 | use std::io; 12 | 13 | /// Magic number stored at the start of each block 14 | const MAGIC: u32 = 0xD9B4BEF9; 15 | 16 | 17 | /// Reads a block from a blk_file as used by 18 | /// bitcoin-core and various other implementations 19 | pub fn read_block(rdr: &mut io::Read) -> Result>, io::Error> { 20 | 21 | loop { 22 | let magicnr = rdr.read_u32::(); 23 | match magicnr { 24 | Err(_) => return Ok(None), // assume EOF 25 | Ok(m) => match m { 26 | 27 | // TODO investigate; // Can't really find it in the cpp. 28 | // this happens on bitcrust-1 at block 451327 29 | // file blk000760, file pos 54391594 30 | // first 8 zero-bytes before magicnr 31 | // for now we skip them; not too important as we 32 | // might not want to support this type of import anyway 33 | 0 => continue, 34 | 35 | MAGIC => break, 36 | _ =>return Err(io::Error::new(io::ErrorKind::InvalidData, "Incorrect magic number")) 37 | } 38 | } 39 | 40 | } 41 | 42 | 43 | let length = try!(rdr.read_u32::()); 44 | let mut buffer = vec![0; length as usize]; 45 | 46 | 47 | try!(rdr.read_exact(&mut buffer)); 48 | 49 | 50 | Ok(Some(buffer)) 51 | 52 | 53 | } 54 | 55 | 56 | 57 | 58 | -------------------------------------------------------------------------------- /serde_network/tests/test.rs: -------------------------------------------------------------------------------- 1 | 2 | 3 | extern crate serde_network; 4 | 5 | #[macro_use] 6 | extern crate serde_derive; 7 | 8 | extern crate serde; 9 | 10 | 11 | #[derive(Serialize, Deserialize, PartialEq, Debug)] 12 | struct Test { 13 | pub primitive: u32, 14 | pub fixed_size: [u32;2], 15 | 16 | pub var_size: Vec, 17 | pub t2: Test2 18 | } 19 | 20 | #[derive(Serialize, Deserialize, PartialEq, Debug)] 21 | struct Test2 { 22 | pub primitive: u32 23 | } 24 | 25 | #[test] 26 | fn test_encode_decode() { 27 | 28 | let val = Test { 29 | primitive: 17, 30 | fixed_size: [15,16], 31 | var_size: vec![21,22,23], 32 | t2: Test2 { primitive: 31 } 33 | }; 34 | 35 | let buf = &mut Vec::new(); 36 | serde_network::serialize(buf, &val); 37 | assert_eq!(buf[0], 17u8); 38 | assert_eq!(buf[1], 0); 39 | 40 | assert_eq!(buf[4], 15); 41 | assert_eq!(buf[5], 0); 42 | assert_eq!(buf[8], 16); 43 | assert_eq!(buf[9], 0); 44 | 45 | assert_eq!(buf[12], 3); 46 | assert_eq!(buf[13], 21); 47 | assert_eq!(buf[14], 0); 48 | assert_eq!(buf[17], 22); 49 | assert_eq!(buf[21], 23); 50 | assert_eq!(buf[25], 31); 51 | 52 | 53 | let back = serde_network::deserialize(buf).unwrap(); 54 | assert_eq!(val, back); 55 | } 56 | 57 | #[test] 58 | fn test_deserialize() { 59 | 60 | // test deserialization by parts 61 | let x = vec![12u8,13u8]; 62 | 63 | let mut de = serde_network::Deserializer::new(&x); 64 | 65 | let u1: u8 = de.deserialize().unwrap() ; 66 | let u2: u8 = de.deserialize().unwrap() ; 67 | assert_eq!(12u8, u1); 68 | assert_eq!(13u8, u2); 69 | 70 | } 71 | -------------------------------------------------------------------------------- /store/src/util.rs: -------------------------------------------------------------------------------- 1 | //! General utility functions 2 | 3 | 4 | pub fn to_hex(buf: &[u8]) -> String { 5 | let x = buf 6 | .iter() 7 | .map(|n| format!("{:02x}", n)) 8 | .collect::>() 9 | .concat(); 10 | 11 | x.to_owned() 12 | } 13 | 14 | pub fn to_hex_rev(buf: &[u8]) -> String { 15 | let x = buf 16 | .iter() 17 | .rev() 18 | .map(|n| format!("{:02x}", n)) 19 | .collect::>() 20 | .concat(); 21 | 22 | x.to_owned() 23 | } 24 | 25 | 26 | 27 | /// Used mainly for tests; found somewhere (rustc_serialize I think) 28 | pub fn from_hex(str: &str) -> Vec { 29 | 30 | // This may be an overestimate if there is any whitespace 31 | let mut b = Vec::with_capacity(str.len() / 2); 32 | let mut modulus = 0; 33 | let mut buf = 08; 34 | 35 | for byte in str.bytes() { 36 | buf <<= 4; 37 | 38 | match byte { 39 | b'A'...b'F' => buf |= byte - b'A' + 10, 40 | b'a'...b'f' => buf |= byte - b'a' + 10, 41 | b'0'...b'9' => buf |= byte - b'0', 42 | b' '|b'\r'|b'\n'|b'\t' => { 43 | buf >>= 4; 44 | continue 45 | } 46 | _ => { 47 | panic!("Invalid hex char"); 48 | } 49 | } 50 | 51 | modulus += 1; 52 | if modulus == 2 { 53 | modulus = 0; 54 | b.push(buf); 55 | } 56 | } 57 | 58 | b.into_iter().collect() 59 | } 60 | 61 | /// Useful to keep hashes in the same format as usually printed 62 | pub fn from_hex_rev(str: &str) -> Vec { 63 | let mut v = from_hex(str); 64 | v.reverse(); 65 | v 66 | } 67 | -------------------------------------------------------------------------------- /encode-derive/src/lib.rs: -------------------------------------------------------------------------------- 1 | 2 | extern crate proc_macro; 3 | extern crate syn; 4 | #[macro_use] 5 | extern crate quote; 6 | 7 | use proc_macro::TokenStream; 8 | 9 | #[proc_macro_derive(Encode, attributes(count))] 10 | pub fn derive_encode(input: TokenStream) -> TokenStream { 11 | // Construct a string representation of the type definition 12 | let s = input.to_string(); 13 | 14 | // Parse the string representation 15 | let ast = syn::parse_derive_input(&s).unwrap(); 16 | 17 | // Build the impl 18 | let gen = impl_encode(&ast); 19 | 20 | // Return the generated impl 21 | gen.parse().unwrap() 22 | } 23 | 24 | fn impl_encode(ast: &syn::DeriveInput) -> quote::Tokens { 25 | let name = &ast.ident; 26 | let fields = match ast.body { 27 | syn::Body::Struct(ref data) => data.fields(), 28 | syn::Body::Enum(_) => panic!("#[derive(Encode)] can only be used with structs"), 29 | }; 30 | let fields = generate_fields(&fields); 31 | quote! { 32 | impl Encode for #name { 33 | fn encode(&self, mut buff: &mut Vec) -> Result<(), ::std::io::Error> { 34 | #(#fields)* 35 | Ok(()) 36 | } 37 | } 38 | } 39 | 40 | } 41 | 42 | fn generate_fields(fields: &[syn::Field]) -> Vec { 43 | let mut result = Vec::new(); 44 | for field in fields { 45 | let ident = &field.ident; 46 | if field.attrs.iter().any(|f| f.value.name() == "count") { 47 | result.push(quote!{ 48 | VarInt::new(self.#ident.len() as u64).encode(&mut buff)?; 49 | }); 50 | } 51 | result.push(quote!{ 52 | self.#ident.encode(&mut buff)?; 53 | }); 54 | } 55 | result 56 | } -------------------------------------------------------------------------------- /store/tests/get_tx.rs: -------------------------------------------------------------------------------- 1 | extern crate store; 2 | 3 | extern crate serde_json; 4 | 5 | mod util; 6 | 7 | 8 | #[test] 9 | fn test_get() { 10 | 11 | 12 | let db = &mut store::init("tst-import").unwrap(); 13 | 14 | let hdr = store::header_get(db, 15 | &util::hash_from_hex("000000000000034a7dedef4a161fa058a2d67a173a90155f3a2fe6fc132e0ebf")) 16 | .unwrap().unwrap(); 17 | 18 | let tx = store::transaction_get(db, 19 | &util::hash_from_hex("4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b")) 20 | .unwrap().unwrap(); 21 | 22 | println!("{}", serde_json::to_string_pretty(&hdr.header).unwrap()); 23 | println!("{}", serde_json::to_string_pretty(&tx.as_tx().unwrap()).unwrap()); 24 | } 25 | 26 | #[test] 27 | fn test_get_all_headers() { 28 | // just browse through all imported headers; 29 | let db = &mut store::init("tst-import").unwrap(); 30 | 31 | let mut hash = store::header_get_best(db).unwrap(); 32 | 33 | println!("{:?}", hash); 34 | loop { 35 | let hdr = store::header_get(db, &hash).unwrap().unwrap(); 36 | 37 | if hdr.height == 0 { 38 | break; 39 | } 40 | 41 | hash = hdr.header.prev_hash; 42 | 43 | } 44 | } 45 | 46 | #[test] 47 | fn test_locator() { 48 | // just browse through all imported headers; 49 | let db = &mut store::init("tst-import").unwrap(); 50 | 51 | let mut hash = store::header_get_best(db).unwrap(); 52 | 53 | println!("{:?}", hash); 54 | loop { 55 | let hdr = store::header_get(db, &hash).unwrap().unwrap(); 56 | 57 | if hdr.height == 0 { 58 | break; 59 | } 60 | 61 | hash = hdr.header.prev_hash; 62 | 63 | } 64 | } 65 | 66 | 67 | 68 | 69 | 70 | 71 | fn main() { 72 | println!("Hallo"); 73 | } 74 | -------------------------------------------------------------------------------- /bitcrustd/src/util.rs: -------------------------------------------------------------------------------- 1 | //! General utility functions 2 | 3 | 4 | pub fn to_hex(buf: &[u8]) -> String { 5 | let x = buf 6 | .iter() 7 | .map(|n| format!("{:02x}", n)) 8 | .collect::>() 9 | .concat(); 10 | 11 | x.to_owned() 12 | } 13 | 14 | pub fn to_hex_rev(buf: &[u8]) -> String { 15 | let x = buf 16 | .iter() 17 | .rev() 18 | .map(|n| format!("{:02x}", n)) 19 | .collect::>() 20 | .concat(); 21 | 22 | x.to_owned() 23 | } 24 | 25 | #[allow(dead_code)] 26 | pub fn hash_from_hex(str: &str) -> [u8;32] { 27 | let mut result = [0;32]; 28 | result.copy_from_slice(&from_hex_rev(str)[..]); 29 | result 30 | } 31 | 32 | 33 | /// Used mainly for tests; found somewhere (rustc_serialize I think) 34 | pub fn from_hex(str: &str) -> Vec { 35 | 36 | // This may be an overestimate if there is any whitespace 37 | let mut b = Vec::with_capacity(str.len() / 2); 38 | let mut modulus = 0; 39 | let mut buf = 08; 40 | 41 | for byte in str.bytes() { 42 | buf <<= 4; 43 | 44 | match byte { 45 | b'A'...b'F' => buf |= byte - b'A' + 10, 46 | b'a'...b'f' => buf |= byte - b'a' + 10, 47 | b'0'...b'9' => buf |= byte - b'0', 48 | b' '|b'\r'|b'\n'|b'\t' => { 49 | buf >>= 4; 50 | continue 51 | } 52 | _ => { 53 | panic!("Invalid hex char"); 54 | } 55 | } 56 | 57 | modulus += 1; 58 | if modulus == 2 { 59 | modulus = 0; 60 | b.push(buf); 61 | } 62 | } 63 | 64 | b.into_iter().collect() 65 | } 66 | 67 | /// Useful to keep hashes in the same format as usually printed 68 | pub fn from_hex_rev(str: &str) -> Vec { 69 | let mut v = from_hex(str); 70 | v.reverse(); 71 | v 72 | } 73 | -------------------------------------------------------------------------------- /tests/test_bench.rs: -------------------------------------------------------------------------------- 1 | 2 | extern crate bitcrust_lib; 3 | 4 | extern crate byteorder; 5 | 6 | 7 | use std::io::BufReader; 8 | use std::fs::File; 9 | 10 | mod blk_file; 11 | 12 | use std::time::{Instant}; 13 | extern crate rayon; 14 | 15 | 16 | 17 | #[test] 18 | #[ignore] 19 | fn load_bench_init() { 20 | 21 | let mut store = bitcrust_lib::init(); 22 | 23 | for fileno in 0..750 { 24 | let name = format!("./core-blocks/blk{:05}.dat", fileno); 25 | println!("Processing {}", name); 26 | let f = File::open(name).unwrap(); 27 | let mut rdr = BufReader::new(f); 28 | 29 | let mut blocks = 0; 30 | loop { 31 | let blk = blk_file::read_block(&mut rdr).unwrap(); 32 | 33 | if blk.is_none() { 34 | break; 35 | } 36 | 37 | bitcrust_lib::add_block(&mut store, &blk.unwrap()); 38 | 39 | blocks += 1; 40 | } 41 | } 42 | 43 | } 44 | 45 | #[test] 46 | #[ignore] 47 | fn load_bench_next() { 48 | let mut store = bitcrust_lib::init(); 49 | store.initial_sync = false; 50 | 51 | let fileno = 750; 52 | 53 | let name = format!("./core-blocks/blk{:05}.dat", fileno); 54 | println!("Processing {}", name); 55 | let f = File::open(name).unwrap(); 56 | 57 | let mut rdr = BufReader::new(f); 58 | 59 | let mut blocks = 0; 60 | let start = Instant::now(); 61 | loop { 62 | let blk = blk_file::read_block(&mut rdr).unwrap(); 63 | 64 | if blk.is_none() { 65 | break; 66 | } 67 | 68 | bitcrust_lib::add_block(&mut store, &blk.unwrap()); 69 | 70 | blocks += 1; 71 | } 72 | 73 | let elapsed = Instant::now() - start; 74 | let elapsed = elapsed.as_secs() as u64 * 1000 + elapsed.subsec_nanos() as u64 / 1000_000; 75 | println!("Processes {} blocks in {} ms ({} ms/block)", blocks, elapsed, elapsed / blocks ) 76 | 77 | } 78 | 79 | -------------------------------------------------------------------------------- /src/config.rs: -------------------------------------------------------------------------------- 1 | //! Stub configutation 2 | //! Meant to wrap toml-config files 3 | 4 | use std::fs; 5 | use std::env; 6 | use std::path::PathBuf; 7 | 8 | 9 | // Overrides the store directory to use 10 | pub const ENV_BITCRUST_STORE: &'static str = "BITCRUST_STORE"; 11 | 12 | // Set to "1" will prevent the data-folder to be cleared 13 | pub const ENV_BITCRUST_NOCLEAR: &'static str = "BITCRUST_NOCLEAR"; 14 | 15 | 16 | 17 | #[derive(Clone)] 18 | pub struct Config { 19 | pub root: PathBuf 20 | } 21 | 22 | 23 | impl Config { 24 | 25 | pub fn root() -> PathBuf { 26 | unimplemented!() 27 | } 28 | } 29 | impl Config { 30 | 31 | pub fn new(path: &str) -> Config { 32 | 33 | let path = PathBuf::from(path); 34 | Config { root: path } 35 | 36 | } 37 | 38 | /// Creates and empties a store 39 | /// Uses the given name except when overriden by env-var BITCRUST_STORE 40 | /// The store is cleared unless BITCRUST_NOCLEAR=1 41 | /// This is used from tests 42 | pub fn new_empty>(name: T) -> Config { 43 | 44 | let path = env::var(ENV_BITCRUST_STORE) 45 | 46 | .map(|s| PathBuf::from(s)) 47 | .unwrap_or_else(|_| { 48 | 49 | let mut path = PathBuf::from("tmp"); 50 | let name: String = name.into() 51 | .replace("bitcrust-lib/","") 52 | .replace("/", "-"); 53 | path.push(name); 54 | path 55 | } 56 | ); 57 | println!("Using store {:?}", path); 58 | 59 | if env::var(ENV_BITCRUST_NOCLEAR).unwrap_or("0".to_string()) != "1" { 60 | let _ = fs::remove_dir_all(path.clone()); 61 | } 62 | Config { root: path } 63 | } 64 | 65 | 66 | pub fn new_persist() -> Config { 67 | 68 | let path = PathBuf::from("prs"); 69 | Config { root: path } 70 | 71 | } 72 | } 73 | 74 | -------------------------------------------------------------------------------- /store/src/record.rs: -------------------------------------------------------------------------------- 1 | 2 | use std::fmt; 3 | 4 | // Record layout 5 | // ------------- 6 | // 7 | // TRANSACTION: 8 | // bits 0 -45 valueptr of transaction >> 3 9 | // bits 46-62 zero 10 | // bits 63 1 11 | // 12 | // PREVOUT: 13 | // bits 0 -45 valueptr of transaction >> 3 14 | // bits 46-62 output index + 1 15 | // bits 63 1 16 | // 17 | // PREVOUT COINBASE: 18 | // bits 0 -45 0 19 | // bits 46-62 0 20 | // bits 63 1 21 | // 22 | // START-Of BLOCK 23 | // bits 1-62 record-count 24 | // bits 63 0 25 | 26 | // 27 | // 28 | // fileoffset == 0 => transaction not found at the time 29 | // fileoffset == -1 => script validation failed 30 | 31 | 32 | #[derive(Clone,Copy,PartialEq, Serialize, Deserialize)] 33 | pub struct Record(pub u64); 34 | 35 | impl fmt::Debug for Record { 36 | fn fmt(&self, fmt: &mut fmt::Formatter) -> Result<(), fmt::Error> { 37 | write!(fmt, "REC {0:016X} ", self.0) 38 | } 39 | } 40 | 41 | impl Record { 42 | 43 | // Various constructors 44 | pub fn new_transaction(tx_ptr: ::ValuePtr) -> Record { 45 | 46 | Record((1<<63) | ((tx_ptr >> 3) & 0x1fff_ffff_ffff)) 47 | } 48 | 49 | pub fn new_coinbase() -> Record { 50 | 51 | Record((1<<63)) 52 | } 53 | 54 | pub fn new_output(tx_ptr: ::ValuePtr, output: u32) -> Record { 55 | Record((1<<63) | ((tx_ptr >> 3) & 0x1fff_ffff_ffff) | ((output as u64) << 46)) 56 | } 57 | 58 | pub fn new_start_of_block(record_count: usize) -> Record { 59 | Record(record_count as u64) 60 | } 61 | 62 | pub fn to_bytes<'a>(records: &'a Vec) -> &'a[u8] { 63 | let size = records.len() * 8; 64 | unsafe { ::std::slice::from_raw_parts(records.as_ptr() as *const u8, size) } 65 | } 66 | 67 | pub fn get_output_index(&self) -> u32 { 68 | debug_assert_eq!(self.0 & (1<<63), (1<<63)); 69 | (self.0 >> 46 & ((1 << 17)-1)) as u32 70 | } 71 | } 72 | 73 | -------------------------------------------------------------------------------- /store/tests/util.rs: -------------------------------------------------------------------------------- 1 | //! General utility functions 2 | 3 | #[allow(dead_code)] 4 | pub fn to_hex(buf: &[u8]) -> String { 5 | let x = buf 6 | .iter() 7 | .map(|n| format!("{:02x}", n)) 8 | .collect::>() 9 | .concat(); 10 | 11 | x.to_owned() 12 | } 13 | 14 | 15 | 16 | #[allow(dead_code)] 17 | pub fn to_hex_rev(buf: &[u8]) -> String { 18 | let x = buf 19 | .iter() 20 | .rev() 21 | .map(|n| format!("{:02x}", n)) 22 | .collect::>() 23 | .concat(); 24 | 25 | x.to_owned() 26 | } 27 | 28 | #[allow(dead_code)] 29 | pub fn hash_from_hex(str: &str) -> [u8;32] { 30 | let mut result = [0;32]; 31 | result.copy_from_slice(&from_hex_rev(str)[..]); 32 | result 33 | } 34 | 35 | /// Used mainly for tests; found somewhere (rustc_serialize I think) 36 | #[allow(dead_code)] 37 | pub fn from_hex(str: &str) -> Vec { 38 | 39 | // This may be an overestimate if there is any whitespace 40 | let mut b = Vec::with_capacity(str.len() / 2); 41 | let mut modulus = 0; 42 | let mut buf = 08; 43 | 44 | for byte in str.bytes() { 45 | buf <<= 4; 46 | 47 | match byte { 48 | b'A'...b'F' => buf |= byte - b'A' + 10, 49 | b'a'...b'f' => buf |= byte - b'a' + 10, 50 | b'0'...b'9' => buf |= byte - b'0', 51 | b' '|b'\r'|b'\n'|b'\t' => { 52 | buf >>= 4; 53 | continue 54 | } 55 | _ => { 56 | panic!("Invalid hex char"); 57 | } 58 | } 59 | 60 | modulus += 1; 61 | if modulus == 2 { 62 | modulus = 0; 63 | b.push(buf); 64 | } 65 | } 66 | 67 | b.into_iter().collect() 68 | } 69 | 70 | /// Useful to keep hashes in the same format as usually printed 71 | pub fn from_hex_rev(str: &str) -> Vec { 72 | let mut v = from_hex(str); 73 | v.reverse(); 74 | v 75 | } 76 | -------------------------------------------------------------------------------- /store/src/hash.rs: -------------------------------------------------------------------------------- 1 | //! Hashing functions 2 | //! For now: Hash32 acts as a reference wrapper 3 | //! and HashBuf acts as an owned hash 4 | //! TODO; move to AsRef trait instead of explicit conversions 5 | 6 | extern crate ring; 7 | 8 | 9 | pub type Hash = [u8; 32]; 10 | 11 | /// Copies a slice into an owned buffer 12 | pub fn hash_from_slice(slice: &[u8]) -> Hash { 13 | let mut result = [0;32]; 14 | result.copy_from_slice(&slice[0..32]); 15 | result 16 | } 17 | 18 | /// Hashes the input twice with SHA256 and returns an owned buffer; 19 | pub fn double_sha256(input: &[u8]) -> Hash { 20 | let digest1 = ring::digest::digest(&ring::digest::SHA256, input); 21 | let digest2 = ring::digest::digest(&ring::digest::SHA256, digest1.as_ref()); 22 | 23 | hash_from_slice(digest2.as_ref()) 24 | } 25 | 26 | /// Hashes the input twice with SHA256 and returns an owned buffer; 27 | /// Can be extracted as an Hash32 using as_ref() 28 | pub fn double_sha256_from_pair(first: &Hash, second: &Hash) -> Hash { 29 | let mut v: Vec = Vec::new(); 30 | v.extend(first.iter()); 31 | v.extend(second.iter()); 32 | 33 | double_sha256(&v) 34 | } 35 | 36 | 37 | #[cfg(test)] 38 | mod tests { 39 | use util::*; 40 | use super::*; 41 | 42 | 43 | #[test] 44 | fn test_double_hash() { 45 | 46 | 47 | const HASH1: &'static str = "212300e77d897f2f059366ed03c8bf2757bc2b1dd30df15d34f6f1ee521e58e8"; 48 | const HASH2: &'static str = "4feec9316077e49b59bc23173303e13be9e9f5f9fa0660a58112a04a65a84ef1"; 49 | const HASH3: &'static str = "03b750bf691caf40b7e33d8e15f64dd16becf944b39a82710d6d257159361b93"; 50 | 51 | let hash1 = hash_from_slice(&from_hex_rev(HASH1)); 52 | let hash2 = hash_from_slice(&from_hex_rev(HASH2)); 53 | let hash3 = hash_from_slice(&from_hex_rev(HASH3)); 54 | 55 | let paired = double_sha256_from_pair(&hash1, &hash2); 56 | 57 | assert_eq!(hash3, paired); 58 | 59 | } 60 | } 61 | 62 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Build Status](https://travis-ci.org/tomasvdw/bitcrust.svg?branch=master)](https://travis-ci.org/tomasvdw/bitcrust) 2 | [![Coverage Status](https://coveralls.io/repos/github/tomasvdw/bitcrust/badge.svg)](https://coveralls.io/github/tomasvdw/bitcrust) 3 | 4 | # Bitcrust 5 | 6 | 7 | **Bitcrust** is a full software suite for bitcoin in development. Currently the focus and progress is on 8 | _[bitcrust-db](doc/bitcrust-db.md)_: a storage-engine which uses a novel approach to block-storage to 9 | provides a high performance, _lock-free_ concurrent R/W access, and 10 | fully parallel block verification. 11 | 12 | More information is available at [bitcrust.org](https://bitcrust.org) 13 | 14 | 15 | ## Table of Contents 16 | 17 | - [Install](#install) 18 | - [Components](#components) 19 | - [Contribute](#contribute) 20 | 21 | ## Install 22 | 23 | Bitcrust depends on libbitcoinconsensus which can be created by building 24 | [bitcoin-core](https://github.com/bitcoin/bitcoin) from source per its instructions. 25 | 26 | 27 | After that you can build and test the bitcrust libraries with 28 | 29 | ``` 30 | cargo test 31 | ``` 32 | 33 | 34 | ## Components 35 | 36 | Bitcrust is planned to have the following components: 37 | 38 | * _[bitcrust-db](doc/bitcrust-db.md)_ Bitcrust-db is the library component that verifies and stores blocks 39 | and transactions. 40 | * _bitcrust-net (planned)_ P2P bitcoin protocol implementation 41 | * _bitcrust-mapreduce (planned)_ A scriptable indexing framework 42 | * _bitcrust-monitor (planned)_ Terminal based query and monitoring tool 43 | * _bitcrust-node (planned)_ 44 | * _bitcrust-wallet (planned)_ 45 | 46 | 47 | ![Components of bitcrust](https://cdn.rawgit.com/tomasvdw/bitcrust/master/doc/components.svg "Dependencies") 48 | 49 | 50 | ## Contribute 51 | 52 | Help is very much wanted; if you have a fix or improvement, please submit a pull request. 53 | 54 | Or contact the maintainer for suggestions on work to be done. 55 | -------------------------------------------------------------------------------- /serde_json/tests/ui/update-references.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Copyright 2015 The Rust Project Developers. See the COPYRIGHT 4 | # file at the top-level directory of this distribution and at 5 | # http://rust-lang.org/COPYRIGHT. 6 | # 7 | # Licensed under the Apache License, Version 2.0 or the MIT license 9 | # , at your 10 | # option. This file may not be copied, modified, or distributed 11 | # except according to those terms. 12 | 13 | # A script to update the references for particular tests. The idea is 14 | # that you do a run, which will generate files in the build directory 15 | # containing the (normalized) actual output of the compiler. This 16 | # script will then copy that output and replace the "expected output" 17 | # files. You can then commit the changes. 18 | # 19 | # If you find yourself manually editing a foo.stderr file, you're 20 | # doing it wrong. 21 | 22 | if [[ "$1" == "--help" || "$1" == "-h" || "$1" == "" || "$2" == "" ]]; then 23 | echo "usage: $0 " 24 | echo "" 25 | echo "For example:" 26 | echo " $0 ../../../build/x86_64-apple-darwin/test/ui *.rs */*.rs" 27 | fi 28 | 29 | MYDIR=$(dirname $0) 30 | 31 | BUILD_DIR="$1" 32 | shift 33 | 34 | while [[ "$1" != "" ]]; do 35 | STDERR_NAME="${1/%.rs/.stderr}" 36 | STDOUT_NAME="${1/%.rs/.stdout}" 37 | shift 38 | if [ -f $BUILD_DIR/$STDOUT_NAME ] && \ 39 | ! (diff $BUILD_DIR/$STDOUT_NAME $MYDIR/$STDOUT_NAME >& /dev/null); then 40 | echo updating $MYDIR/$STDOUT_NAME 41 | cp $BUILD_DIR/$STDOUT_NAME $MYDIR/$STDOUT_NAME 42 | fi 43 | if [ -f $BUILD_DIR/$STDERR_NAME ] && \ 44 | ! (diff $BUILD_DIR/$STDERR_NAME $MYDIR/$STDERR_NAME >& /dev/null); then 45 | echo updating $MYDIR/$STDERR_NAME 46 | cp $BUILD_DIR/$STDERR_NAME $MYDIR/$STDERR_NAME 47 | fi 48 | done 49 | 50 | 51 | -------------------------------------------------------------------------------- /net/src/message/block_message.rs: -------------------------------------------------------------------------------- 1 | use Encode; 2 | use VarInt; 3 | use super::TransactionMessage; 4 | 5 | #[cfg(test)] 6 | mod tests { 7 | use super::*; 8 | 9 | #[test] 10 | fn it_implements_types_required_for_protocol() { 11 | let m = BlockMessage::default(); 12 | assert_eq!(m.name(), "block"); 13 | assert_eq!(m.len(), 84); 14 | } 15 | } 16 | 17 | #[derive(Debug, Default, Encode, PartialEq)] 18 | pub struct BlockMessage { 19 | pub version: i32, 20 | pub previous_block: [u8; 32], 21 | pub merkle_root: [u8; 32], 22 | pub timestamp: u32, 23 | pub bits: u32, 24 | pub nonce: u32, 25 | #[count] 26 | pub transactions: Vec 27 | } 28 | 29 | impl BlockMessage { 30 | #[inline] 31 | pub fn len(&self) -> usize { 32 | 4usize + // version 33 | 32usize + // previous block 34 | 32usize + // merkle root 35 | 4usize + // timestamp 36 | 4usize + // bits 37 | 4usize + // nonce 38 | 4usize + // count of transactions 39 | self.transactions.iter().map(|i| i.len()).sum::() // transactions 40 | } 41 | 42 | #[inline] 43 | pub fn name(&self) -> &'static str { 44 | "block" 45 | } 46 | 47 | pub fn new(version: i32, prev_block: &[u8], merkle_root: &[u8], timestamp: u32, 48 | bits: u32, nonce: u32, transactions: Vec) -> BlockMessage { 49 | debug_assert!(prev_block.len() == 32); 50 | let mut a: [u8; 32] = Default::default(); 51 | a.copy_from_slice(&prev_block); 52 | debug_assert!(merkle_root.len() == 32); 53 | let mut b: [u8; 32] = Default::default(); 54 | b.copy_from_slice(&merkle_root); 55 | BlockMessage { 56 | version: version, 57 | previous_block: a, 58 | merkle_root: b, 59 | timestamp: timestamp, 60 | bits: bits, 61 | nonce: nonce, 62 | transactions: transactions, 63 | } 64 | } 65 | } -------------------------------------------------------------------------------- /net/src/message/bitcrust/mod.rs: -------------------------------------------------------------------------------- 1 | use rand::{self, Rng}; 2 | use ring::hmac; 3 | 4 | use Encode; 5 | 6 | #[cfg(test)] 7 | mod tests { 8 | use ring::digest; 9 | use super::*; 10 | 11 | #[test] 12 | fn it_implements_types_required_for_protocol() { 13 | let m = AuthenticatedBitcrustMessage::default(); 14 | assert_eq!(m.name(), "bcr_pcr"); 15 | assert_eq!(m.len(), 40); 16 | } 17 | 18 | #[test] 19 | fn it_creates_and_validates() { 20 | let key = hmac::SigningKey::new(&digest::SHA256, &[0x00; 32]); 21 | let m = AuthenticatedBitcrustMessage::create(&key); 22 | assert!(m.valid(&key)); 23 | } 24 | } 25 | 26 | #[derive(Debug, Default, Encode, PartialEq)] 27 | pub struct AuthenticatedBitcrustMessage { 28 | nonce: [u8; 8], 29 | signature: [u8; 32], 30 | } 31 | 32 | impl AuthenticatedBitcrustMessage { 33 | pub fn create(key: &hmac::SigningKey) -> 34 | AuthenticatedBitcrustMessage { 35 | let mut rng = rand::thread_rng(); 36 | 37 | let nonce: u64 = rng.gen(); 38 | 39 | let mut nonce_vec = Vec::with_capacity(8); 40 | let _ = nonce.encode(&mut nonce_vec); 41 | let signature = hmac::sign(&key, &nonce_vec); 42 | AuthenticatedBitcrustMessage::with_signature(signature.as_ref(), &nonce_vec) 43 | } 44 | 45 | pub fn with_signature(input: &[u8], nonce: &[u8]) -> AuthenticatedBitcrustMessage{ 46 | let mut a: [u8; 32] = [0; 32]; 47 | a.copy_from_slice(&input); 48 | let mut b: [u8; 8] = [0; 8]; 49 | b.copy_from_slice(&nonce); 50 | AuthenticatedBitcrustMessage { 51 | nonce: b, 52 | signature: a 53 | } 54 | } 55 | pub fn valid(&self, key: &hmac::SigningKey) -> bool { 56 | hmac::verify_with_own_key(key, &self.nonce, &self.signature).is_ok() 57 | } 58 | 59 | pub fn len(&self) -> usize { 60 | 40 61 | } 62 | 63 | #[inline] 64 | pub fn name(&self) -> &'static str { 65 | "bcr_pcr" 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /src/api.rs: -------------------------------------------------------------------------------- 1 | //! 2 | //! Public API for bitcrust-db 3 | //! 4 | //! 5 | //! 6 | //! 7 | 8 | 9 | use config; 10 | use store; 11 | use store::Store; 12 | use block_add; 13 | 14 | 15 | 16 | // Creates a store; mock interface 17 | pub fn init() -> Store { 18 | 19 | let config = test_cfg!(); 20 | let store = Store::new(&config); 21 | 22 | info!(store.logger, "Store intitalized"; "dir" => config.root.to_str().unwrap()); 23 | 24 | store 25 | } 26 | 27 | // Creates a store; mock interface 28 | pub fn init_prs() -> Store { 29 | 30 | let config = config::Config::new("prs"); 31 | let store = Store::new(&config); 32 | 33 | info!(store.logger, "Store intitalized"; "dir" => config.root.to_str().unwrap()); 34 | 35 | store 36 | } 37 | 38 | // This is a preliminary interface. 39 | pub fn add_block(store: &mut store::Store, buffer: &[u8]) { 40 | block_add::add_block(store, buffer) 41 | } 42 | 43 | pub fn add_transaction(_: &[u8]) { 44 | 45 | } 46 | 47 | 48 | 49 | pub fn get_block(_: [u8; 32]) { 50 | 51 | } 52 | 53 | 54 | 55 | #[cfg(test)] 56 | mod tests { 57 | 58 | use util::*; 59 | use super::*; 60 | 61 | #[test] 62 | pub fn test_add_block() { 63 | let hex = "0100000000000000000000000000000000000000000000000000000000000000\ 64 | 000000003ba3edfd7a7b12b27ac72c3e67768f617fc81bc3888a51323a9fb8aa\ 65 | 4b1e5e4a29ab5f49ffff001d1dac2b7c01010000000100000000000000000000\ 66 | 00000000000000000000000000000000000000000000ffffffff4d04ffff001d\ 67 | 0104455468652054696d65732030332f4a616e2f32303039204368616e63656c\ 68 | 6c6f72206f6e206272696e6b206f66207365636f6e64206261696c6f75742066\ 69 | 6f722062616e6b73ffffffff0100f2052a01000000434104678afdb0fe554827\ 70 | 1967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4\ 71 | f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5fac00000000"; 72 | 73 | 74 | let slice = &from_hex(hex); 75 | let mut store = init(); 76 | 77 | add_block(&mut store, slice); 78 | 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /serde_json/tests/macros/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Serde Developers 2 | // 3 | // Licensed under the Apache License, Version 2.0 or the MIT license 5 | // , at your 6 | // option. This file may not be copied, modified, or distributed 7 | // except according to those terms. 8 | 9 | macro_rules! json_str { 10 | ([]) => { 11 | "[]" 12 | }; 13 | ([ $e1:tt $(, $e:tt)* ]) => { 14 | concat!("[", 15 | json_str!($e1), 16 | $(",", json_str!($e),)* 17 | "]") 18 | }; 19 | ({}) => { 20 | "{}" 21 | }; 22 | ({ $k1:tt : $v1:tt $(, $k:tt : $v:tt)* }) => { 23 | concat!("{", 24 | stringify!($k1), ":", json_str!($v1), 25 | $(",", stringify!($k), ":", json_str!($v),)* 26 | "}") 27 | }; 28 | (($other:tt)) => { 29 | $other 30 | }; 31 | ($other:tt) => { 32 | stringify!($other) 33 | }; 34 | } 35 | 36 | macro_rules! pretty_str { 37 | ($json:tt) => { 38 | pretty_str_impl!("", $json) 39 | }; 40 | } 41 | 42 | macro_rules! pretty_str_impl { 43 | ($indent:expr, []) => { 44 | "[]" 45 | }; 46 | ($indent:expr, [ $e1:tt $(, $e:tt)* ]) => { 47 | concat!("[\n ", 48 | $indent, pretty_str_impl!(concat!(" ", $indent), $e1), 49 | $(",\n ", $indent, pretty_str_impl!(concat!(" ", $indent), $e),)* 50 | "\n", $indent, "]") 51 | }; 52 | ($indent:expr, {}) => { 53 | "{}" 54 | }; 55 | ($indent:expr, { $k1:tt : $v1:tt $(, $k:tt : $v:tt)* }) => { 56 | concat!("{\n ", 57 | $indent, stringify!($k1), ": ", pretty_str_impl!(concat!(" ", $indent), $v1), 58 | $(",\n ", $indent, stringify!($k), ": ", pretty_str_impl!(concat!(" ", $indent), $v),)* 59 | "\n", $indent, "}") 60 | }; 61 | ($indent:expr, ($other:tt)) => { 62 | $other 63 | }; 64 | ($indent:expr, $other:tt) => { 65 | stringify!($other) 66 | }; 67 | } 68 | -------------------------------------------------------------------------------- /net/src/var_int.rs: -------------------------------------------------------------------------------- 1 | use std::io; 2 | 3 | use Encode; 4 | 5 | #[cfg(test)] 6 | mod tests { 7 | use super::{Encode, VarInt}; 8 | 9 | #[test] 10 | fn it_encodes_a_u8_varint() { 11 | let input = VarInt::new(12); 12 | let mut var_int = vec![]; 13 | let _ = input.encode(&mut var_int); 14 | assert_eq!(var_int, &[12 as u8]); 15 | } 16 | 17 | #[test] 18 | fn it_encodes_a_u16_varint() { 19 | let input = VarInt::new(0xFFFF); 20 | let mut var_int = vec![]; 21 | let _ = input.encode(&mut var_int); 22 | assert_eq!(var_int, &[0xFD, 0xFF, 0xFF]); 23 | } 24 | 25 | #[test] 26 | fn it_encodes_a_u32_varint() { 27 | let input = VarInt::new(0xFFFFFFFF); 28 | let mut var_int = vec![]; 29 | let _ = input.encode(&mut var_int); 30 | assert_eq!(var_int, &[0xFE, 0xFF, 0xFF, 0xFF, 0xFF]); 31 | } 32 | 33 | #[test] 34 | fn it_encodes_a_u64_varint() { 35 | let input = VarInt::new(0xFFFFFFFF + 1); 36 | let mut var_int = vec![]; 37 | let _ = input.encode(&mut var_int); 38 | assert_eq!(var_int, 39 | &[0xFF, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00]); 40 | } 41 | } 42 | 43 | #[derive(Debug, Default, PartialEq)] 44 | pub struct VarInt { 45 | value: u64 46 | } 47 | 48 | impl VarInt { 49 | pub fn new(i: u64) -> VarInt { 50 | VarInt { 51 | value: i 52 | } 53 | } 54 | } 55 | 56 | impl Encode for VarInt { 57 | fn encode(&self, mut buff: &mut Vec) -> Result<(), io::Error> { 58 | if self.value < 0xFD { 59 | (self.value as u8).encode(&mut buff)?; 60 | return Ok(()); 61 | } 62 | if self.value <= 0xFFFF { 63 | 0xFDu8.encode(&mut buff)?; 64 | (self.value as u16).encode(&mut buff)?; 65 | return Ok(()); 66 | } 67 | if self.value <= 0xFFFFFFFF { 68 | 0xFEu8.encode(&mut buff)?; 69 | (self.value as u32).encode(&mut buff)?; 70 | return Ok(()); 71 | } 72 | 0xFFu8.encode(&mut buff)?; 73 | self.value.encode(&mut buff)?; 74 | Ok(()) 75 | } 76 | } -------------------------------------------------------------------------------- /serde_network/benches/bincode.rs: -------------------------------------------------------------------------------- 1 | #![feature(test)] 2 | 3 | extern crate byteorder; 4 | 5 | #[macro_use] 6 | extern crate serde_derive; 7 | 8 | extern crate bincode; 9 | extern crate serde; 10 | extern crate serde_bench; 11 | extern crate test; 12 | 13 | use bincode::Infinite; 14 | use byteorder::NetworkEndian; 15 | use serde::{Serialize, Deserialize}; 16 | use test::Bencher; 17 | 18 | #[derive(Serialize, Deserialize)] 19 | struct Foo { 20 | bar: String, 21 | baz: u64, 22 | derp: bool, 23 | } 24 | 25 | impl Default for Foo { 26 | fn default() -> Self { 27 | Foo { 28 | bar: "hello".into(), 29 | baz: 1337u64, 30 | derp: true, 31 | } 32 | } 33 | } 34 | 35 | #[bench] 36 | fn bincode_deserialize(b: &mut Bencher) { 37 | let foo = Foo::default(); 38 | let mut bytes = Vec::with_capacity(128); 39 | type BincodeSerializer = bincode::internal::Serializer; 40 | foo.serialize(&mut BincodeSerializer::new(&mut bytes)).unwrap(); 41 | 42 | b.iter(|| { 43 | type BincodeDeserializer = bincode::internal::Deserializer; 44 | let read = bincode::read_types::SliceReader::new(&bytes); 45 | let mut de = BincodeDeserializer::new(read, Infinite); 46 | Foo::deserialize(&mut de).unwrap() 47 | }) 48 | } 49 | 50 | #[bench] 51 | fn bincode_serialize(b: &mut Bencher) { 52 | let foo = Foo::default(); 53 | 54 | b.iter(|| { 55 | let mut bytes = Vec::with_capacity(128); 56 | type BincodeSerializer = bincode::internal::Serializer; 57 | foo.serialize(&mut BincodeSerializer::new(&mut bytes)).unwrap() 58 | }) 59 | } 60 | 61 | #[bench] 62 | fn serde_deserialize(b: &mut Bencher) { 63 | let foo = Foo::default(); 64 | let mut bytes = Vec::new(); 65 | serde_bench::serialize(&mut bytes, &foo).unwrap(); 66 | 67 | b.iter(|| { 68 | serde_bench::deserialize::(&bytes).unwrap() 69 | }) 70 | } 71 | 72 | #[bench] 73 | fn serde_serialize(b: &mut Bencher) { 74 | let foo = Foo::default(); 75 | 76 | b.iter(|| { 77 | let mut bytes = Vec::with_capacity(128); 78 | serde_bench::serialize(&mut bytes, &foo).unwrap() 79 | }) 80 | } 81 | -------------------------------------------------------------------------------- /store/tests/import.rs: -------------------------------------------------------------------------------- 1 | 2 | 3 | extern crate store; 4 | 5 | 6 | use std::time::Instant; 7 | 8 | mod util; 9 | mod blk_file; 10 | 11 | 12 | 13 | #[test] 14 | #[ignore] 15 | fn test_import() { 16 | 17 | let mut db = store::init_empty("tst-import").unwrap(); 18 | let mut orphans = std::collections::HashMap::new(); 19 | let now = Instant::now(); 20 | let mut blocks = 0; 21 | 22 | for blk in blk_file::read_blocks() { 23 | 24 | // we only use the header 25 | let mut raw_hdr = blk; 26 | let mut hash = store::double_sha256(&raw_hdr[0..80]); 27 | 28 | let header = store::Header::new(&raw_hdr[0..80]).unwrap(); 29 | let prev_hash = header.prev_hash; 30 | 31 | 32 | let add_result = store::header_add(&mut db, &hash, header).unwrap(); 33 | if let store::HeaderAddResult::Orphan(parent) = add_result { 34 | 35 | let header = store::Header::new(&raw_hdr[0..80]).unwrap(); 36 | 37 | 38 | //println!("Marking orphan {} waiting for {}", util::to_hex_rev(&hash[..]), util::to_hex_rev(&parent[..])); 39 | assert!(orphans.insert(parent, (hash,header)).is_none()); 40 | 41 | 42 | } else { 43 | //println!("Added header {} with prev {}", util::to_hex_rev(&hash[..]), util::to_hex_rev(&prev_hash[..])); 44 | 45 | 46 | while let Some(&(ref orphan_hash, ref orphan_header)) = orphans.get(&hash) { 47 | //println!("Adding decendent {} of {}", util::to_hex_rev(&orphan_hash[..]), util::to_hex_rev(&hash[..])); 48 | let add_result = store::header_add(&mut db, &orphan_hash, orphan_header.clone()).unwrap(); 49 | 50 | if let store::HeaderAddResult::Orphan(_) = add_result { 51 | panic!("{} should not be orphan anymore", util::to_hex_rev(&hash[..])); 52 | } 53 | hash = *orphan_hash; 54 | } 55 | } 56 | 57 | blocks += 1; 58 | } 59 | 60 | let elapsed = now.elapsed().as_secs() * 1000 + now.elapsed().subsec_nanos() as u64 / 1000_000; 61 | let ms_block = elapsed / blocks; 62 | println!("Processed {} in {} ms, {} ms/block", blocks, elapsed, ms_block); 63 | 64 | let _ = store::header_get(&mut db, 65 | &util::hash_from_hex("000000000000034a7dedef4a161fa058a2d67a173a90155f3a2fe6fc132e0ebf")) 66 | .unwrap().unwrap(); 67 | 68 | 69 | } 70 | 71 | -------------------------------------------------------------------------------- /serde_json/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to Serde 2 | 3 | Serde welcomes contribution from everyone in the form of suggestions, bug 4 | reports, pull requests, and feedback. This document gives some guidance if you 5 | are thinking of helping us. 6 | 7 | Please reach out here in a GitHub issue or in the #serde IRC channel on 8 | [`irc.mozilla.org`] if we can do anything to help you contribute. 9 | 10 | [`irc.mozilla.org`]: https://wiki.mozilla.org/IRC 11 | 12 | ## Submitting bug reports and feature requests 13 | 14 | Serde development is spread across lots of repositories. In general, prefer to 15 | open issues against the main [serde-rs/serde] repository unless the topic is 16 | clearly specific to JSON. 17 | 18 | [serde-rs/serde]: https://github.com/serde-rs/serde 19 | 20 | When reporting a bug or asking for help, please include enough details so that 21 | the people helping you can reproduce the behavior you are seeing. For some tips 22 | on how to approach this, read about how to produce a [Minimal, Complete, and 23 | Verifiable example]. 24 | 25 | [Minimal, Complete, and Verifiable example]: https://stackoverflow.com/help/mcve 26 | 27 | When making a feature request, please make it clear what problem you intend to 28 | solve with the feature, any ideas for how Serde could support solving that 29 | problem, any possible alternatives, and any disadvantages. 30 | 31 | ## Running the test suite 32 | 33 | We encourage you to check that the test suite passes locally before submitting a 34 | pull request with your changes. If anything does not pass, typically it will be 35 | easier to iterate and fix it locally than waiting for the CI servers to run 36 | tests for you. 37 | 38 | The test suite requires a nightly compiler. 39 | 40 | ##### In the [`tests/deps`] directory 41 | 42 | ```sh 43 | # This is a prerequisite for running the full test suite 44 | cargo clean && cargo update && cargo build 45 | ``` 46 | 47 | ##### In the top level serde-json directory 48 | 49 | ```sh 50 | # Run the full test suite, including doc test and compile-tests 51 | cargo test 52 | ``` 53 | 54 | [`tests/deps`]: https://github.com/serde-rs/json/tree/master/tests/deps 55 | 56 | ## Conduct 57 | 58 | In all Serde-related forums, we follow the [Rust Code of Conduct]. For 59 | escalation or moderation issues please contact Erick (erick.tryzelaar@gmail.com) 60 | instead of the Rust moderation team. 61 | 62 | [Rust Code of Conduct]: https://www.rust-lang.org/conduct.html 63 | -------------------------------------------------------------------------------- /hashstore/README.md: -------------------------------------------------------------------------------- 1 | # hashstore 2 | Key/Value store optimized for storing blockchain data 3 | 4 | ## Abstract 5 | 6 | This key/value store can be used as write/purge only store (no updates) and is suitable to store transactions and blockheaders. 7 | 8 | It has rather specific properties 9 | 10 | * Keys are 32-byte hashes 11 | * Values are arbitrary length byte sequences 12 | * Append only. No updates of existsing data. 13 | * Allows atoming receiving and storing of dependencies 14 | * Very fast writes. Very fast MRU reads. Slow old reads. 15 | 16 | ## Architecture 17 | 18 | The storage file starts with a header followed by a large hash-table of filepointers to values. After the hashtable, the values are appended. 19 | 20 | The values form a LIFO linked list to resolve hash collisions. 21 | 22 | ## Usage 23 | 24 | use hashstore::*; 25 | 26 | // open or create `myfile` with a 24-bits hashtable (= 8*(2^24) bytes). 27 | let hs = HashStore::new("myfile", 24).unwrap(); 28 | 29 | // fast append (timestamp=10) 30 | hs.set(mykey1, myvalue1, 10).unwrap(); 31 | 32 | assert_eq!(hs.get(mykey1, SearchDepth::FullSearch).unwrap(), Some(myvalue1)); 33 | 34 | 35 | ## Timestamps 36 | 37 | Values are added with a "timestamp", which is just an opaque increasing number. On retrieving values, the search can be limited to 38 | after specific timestamp. For instance, for blockchains, heights can be used as timestamps, and only the last X blocks can be searched 39 | to allow deprecation of transactions. 40 | 41 | // May fail as we're searching after mykey1's timestamp 42 | assert_eq!(hs.get(mykey1, SearchDepth::After(20)).unwrap(), Some(myvalue1)); 43 | 44 | 45 | ## Dependencies 46 | 47 | Values can be retrieved as a required dependency of another. If A is retrieved as dependency of B, and A isn't found, 48 | a dependency A->B is atomically inserted. This will block insertion of A until the dependency is resolved. 49 | 50 | This can be used to process out-of-order (orphan) transactions and blocks. 51 | 52 | // failed lookup mykey2 doesn't exist 53 | assert!(hs.get_dependency(mykey2, mykey1, 20).unwrap().is_none()); 54 | 55 | // failed set of mykey2 56 | assert!(hs.set(mykey2, myvalue2, vec![], SearchDepth::FullSearch, 30).unwrap().is_none()); 57 | 58 | // ... verify mykey1->mykey2 59 | // now set and declare dependecy as met 60 | assert!(hs.set(mykey2, myvalue2, vec![mykey2], SearchDepth::FullSearch, 30).unwrap().is_none()); 61 | 62 | 63 | -------------------------------------------------------------------------------- /serde_json/src/iter.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Serde Developers 2 | // 3 | // Licensed under the Apache License, Version 2.0 or the MIT license 5 | // , at your 6 | // option. This file may not be copied, modified, or distributed 7 | // except according to those terms. 8 | 9 | use std::io; 10 | 11 | pub struct LineColIterator { 12 | iter: I, 13 | 14 | /// Index of the current line. Characters in the first line of the input 15 | /// (before the first newline character) are in line 1. 16 | line: usize, 17 | 18 | /// Index of the current column. The first character in the input and any 19 | /// characters immediately following a newline character are in column 1. 20 | /// The column is 0 immediately after a newline character has been read. 21 | col: usize, 22 | 23 | /// Byte offset of the start of the current line. This is the sum of lenghts 24 | /// of all previous lines. Keeping track of things this way allows efficient 25 | /// computation of the current line, column, and byte offset while only 26 | /// updating one of the counters in `next()` in the common case. 27 | start_of_line: usize, 28 | } 29 | 30 | impl LineColIterator 31 | where 32 | I: Iterator>, 33 | { 34 | pub fn new(iter: I) -> LineColIterator { 35 | LineColIterator { 36 | iter: iter, 37 | line: 1, 38 | col: 0, 39 | start_of_line: 0, 40 | } 41 | } 42 | 43 | pub fn line(&self) -> usize { 44 | self.line 45 | } 46 | 47 | pub fn col(&self) -> usize { 48 | self.col 49 | } 50 | 51 | pub fn byte_offset(&self) -> usize { 52 | self.start_of_line + self.col 53 | } 54 | } 55 | 56 | impl Iterator for LineColIterator 57 | where 58 | I: Iterator>, 59 | { 60 | type Item = io::Result; 61 | 62 | fn next(&mut self) -> Option> { 63 | match self.iter.next() { 64 | None => None, 65 | Some(Ok(b'\n')) => { 66 | self.start_of_line += self.col + 1; 67 | self.line += 1; 68 | self.col = 0; 69 | Some(Ok(b'\n')) 70 | } 71 | Some(Ok(c)) => { 72 | self.col += 1; 73 | Some(Ok(c)) 74 | } 75 | Some(Err(e)) => Some(Err(e)), 76 | } 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /src/ffi.rs: -------------------------------------------------------------------------------- 1 | //! Interface to C-libs. 2 | //! 3 | //! Currently only libbitcoinconsensus 4 | 5 | 6 | extern crate libc; 7 | 8 | #[link(name = "bitcoinconsensus")] 9 | extern { 10 | 11 | /* EXPORT_SYMBOL int bitcoinconsensus_verify_script(const unsigned char *scriptPubKey, unsigned int scriptPubKeyLen, 12 | const unsigned char *txTo , unsigned int txToLen, 13 | unsigned int nIn, unsigned int flags, bitcoinconsensus_error* err); 14 | */ 15 | 16 | pub fn bitcoinconsensus_verify_script( 17 | prevout_script: *const u8, 18 | prevout_script_size: u32, 19 | transaction: *const u8, 20 | transaction_size: u32, 21 | tx_input_index: u32, 22 | flags: u32, 23 | err: *mut i32 24 | ) 25 | 26 | -> i32; 27 | } 28 | 29 | // typedef enum bitcoinconsensus_error_t 30 | // { 31 | // bitcoinconsensus_ERR_OK = 0, 32 | // bitcoinconsensus_ERR_TX_INDEX, 33 | // bitcoinconsensus_ERR_TX_SIZE_MISMATCH, 34 | // bitcoinconsensus_ERR_TX_DESERIALIZE, 35 | // bitcoinconsensus_ERR_AMOUNT_REQUIRED, 36 | // bitcoinconsensus_ERR_INVALID_FLAGS, 37 | // } bitcoinconsensus_error; 38 | 39 | #[derive(Debug)] 40 | pub enum VerifyScriptError { 41 | Index, 42 | SizeMismatch, 43 | Deserialize, 44 | AmountRequired, 45 | InvalidFlags, 46 | } 47 | 48 | /// Verifies whether the given `input` of the transaction spends the given `output` 49 | /// using libbitcoin-consensus 50 | 51 | pub fn verify_script(previous_tx_out: &[u8], transaction: &[u8], input: u32) -> Result<(), VerifyScriptError> { 52 | let flags = 0; 53 | let mut err: i32 = 0; 54 | let result = unsafe { bitcoinconsensus_verify_script( 55 | previous_tx_out.as_ptr(), 56 | previous_tx_out.len() as u32, 57 | transaction.as_ptr(), 58 | transaction.len() as u32, 59 | input as u32, 60 | flags, 61 | &mut err 62 | ) }; 63 | 64 | if result == 1 { 65 | Ok(()) 66 | } 67 | else { 68 | Err(match err { 69 | 0 => VerifyScriptError::Index, 70 | 1 => VerifyScriptError::SizeMismatch, 71 | 2 => VerifyScriptError::Deserialize, 72 | 3 => VerifyScriptError::AmountRequired, 73 | 4 => VerifyScriptError::InvalidFlags, 74 | _ => unreachable!() 75 | }) 76 | } 77 | } 78 | 79 | 80 | 81 | #[cfg(test)] 82 | mod tests { 83 | 84 | } 85 | -------------------------------------------------------------------------------- /store/src/api/block.rs: -------------------------------------------------------------------------------- 1 | 2 | use db::*; 3 | use Header; 4 | pub enum BlockAddHeaderOk { 5 | Invalid, 6 | Orphan, 7 | 8 | } 9 | 10 | pub enum VerifyFlags { 11 | NoVerifySignatures, 12 | VerifyAll 13 | } 14 | 15 | 16 | 17 | pub enum HeaderAddResult { 18 | Ok, 19 | AlreadyExists, 20 | Invalid, 21 | Orphan([u8;32]) 22 | } 23 | /// Adds a header 24 | /// 25 | pub fn header_add(db: &mut Db, hash: &[u8;32], header: Header) -> Result { 26 | 27 | if let Some(_) = db_header::get(db, &hash)? { 28 | Ok(HeaderAddResult::AlreadyExists) 29 | 30 | } else if let Some((parent_ptr, parent)) = db_header::get(db, &header.prev_hash)? { 31 | 32 | let db_header = db_header::DbHeader::new(parent, parent_ptr, header); 33 | db_header::write_header(db, hash, db_header)?; 34 | Ok(HeaderAddResult::Ok) 35 | 36 | } else { 37 | 38 | Ok(HeaderAddResult::Orphan(header.prev_hash)) 39 | } 40 | } 41 | 42 | 43 | pub enum BlockExistsOk { 44 | NotFound, 45 | FoundHeaderOrphan, 46 | FoundHeader, 47 | FoundHeaderAndData 48 | } 49 | 50 | 51 | pub fn block_add_transactions(_db: &mut Db, _block_data: &[u8], _validate: bool) -> Result<(), DbError> 52 | { 53 | Ok(()) 54 | } 55 | 56 | 57 | 58 | pub fn block_exists(_blockhash: &[u8;32]) -> Result { 59 | unimplemented!() 60 | } 61 | 62 | /// Returns the hash of the block header with the most accumulated work 63 | pub fn header_get_best(db: &mut Db) -> Result<[u8;32], DbError> { 64 | 65 | Ok(db_header::get_best_header(db)?) 66 | } 67 | 68 | /// Returns a set of block hashes of which no records are known 69 | pub fn block_needs_download(_db: &mut Db, _count: usize) -> Result, DbError> { 70 | 71 | /*let best_header = db_header::get_best_header(db)?; 72 | if best_header.has_transactions() { 73 | return Ok(vec![]); 74 | } else { 75 | 76 | }*/ 77 | 78 | unimplemented!() //Ok(db_header::get_best_block(db)?) 79 | } 80 | 81 | 82 | pub fn header_get(db: &mut Db, hash: &[u8;32]) -> Result, DbError> { 83 | Ok(db_header::get(db, hash)? 84 | .map(|(_, db_hdr)| db_hdr)) 85 | } 86 | 87 | 88 | /// Constructs a locator object for the given block hash 89 | /// 90 | /// This consists of the blockhash and at most 32 hashes ancestor hashes, 91 | /// ending in Genesis 92 | pub fn block_get_locator(db: &mut Db, blockhash: &[u8;32]) -> Result, DbError> { 93 | 94 | Ok(db_header::get_locator(db, blockhash)?) 95 | } 96 | 97 | 98 | -------------------------------------------------------------------------------- /net/src/message/addr_message.rs: -------------------------------------------------------------------------------- 1 | use net_addr::NetAddr; 2 | use Encode; 3 | use VarInt; 4 | 5 | #[cfg(test)] 6 | mod tests { 7 | use super::*; 8 | use ::Services; 9 | 10 | #[test] 11 | fn it_encodes_an_addr_message() { 12 | let input = vec![ 13 | // Payload: 14 | 0x01, // 1 address in this message 15 | // Address: 16 | 0xE2, 17 | 0x15, 18 | 0x10, 19 | 0x4D, // Mon Dec 20 21:50:10 EST 2010 (only when version is >= 31402) 20 | 0x01, 21 | 0x00, 22 | 0x00, 23 | 0x00, 24 | 0x00, 25 | 0x00, 26 | 0x00, 27 | 0x00, // 1 (NODE_NETWORK service - see version message) 28 | 0x00, 29 | 0x00, 30 | 0x00, 31 | 0x00, 32 | 0x00, 33 | 0x00, 34 | 0x00, 35 | 0x00, 36 | 0x00, 37 | 0x00, 38 | 0xFF, 39 | 0xFF, 40 | 0x0A, 41 | 0x00, 42 | 0x00, 43 | 0x01, // IPv4: 10.0.0.1, IPv6: ::ffff:10.0.0.1 (IPv4-mapped IPv6 address) 44 | 0x20, 45 | 0x8D]; 46 | let addr = AddrMessage { 47 | addrs: vec![ 48 | NetAddr { 49 | time: Some(1292899810), 50 | services: Services::from(1), 51 | ip: "::ffff:10.0.0.1".parse().unwrap(), 52 | port: 8333 }] }; 53 | let mut encoded = vec![]; 54 | addr.encode(&mut encoded).expect("Failed to encode Addr"); 55 | assert_eq!(input, encoded); 56 | } 57 | 58 | #[test] 59 | fn it_implements_types_required_for_protocol() { 60 | let m = AddrMessage::default(); 61 | assert_eq!(m.name(), "addr"); 62 | assert_eq!(m.len(), 8); 63 | } 64 | } 65 | 66 | /// addr message 67 | #[derive(Debug, Default, Encode, PartialEq)] 68 | pub struct AddrMessage { 69 | #[count] 70 | pub addrs: Vec, 71 | } 72 | 73 | impl AddrMessage { 74 | #[inline] 75 | pub fn len(&self) -> usize { 76 | 8 + (30 * self.addrs.len()) 77 | } 78 | 79 | #[inline] 80 | pub fn name(&self) -> &'static str { 81 | "addr" 82 | } 83 | } -------------------------------------------------------------------------------- /serde_json/src/value/partial_eq.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Serde Developers 2 | // 3 | // Licensed under the Apache License, Version 2.0 or the MIT license 5 | // , at your 6 | // option. This file may not be copied, modified, or distributed 7 | // except according to those terms. 8 | 9 | use super::Value; 10 | 11 | impl PartialEq for Value { 12 | fn eq(&self, other: &str) -> bool { 13 | self.as_str().map_or(false, |s| s == other) 14 | } 15 | } 16 | 17 | impl<'a> PartialEq<&'a str> for Value { 18 | fn eq(&self, other: &&str) -> bool { 19 | self.as_str().map_or(false, |s| s == *other) 20 | } 21 | } 22 | 23 | impl PartialEq for str { 24 | fn eq(&self, other: &Value) -> bool { 25 | other.as_str().map_or(false, |s| s == self) 26 | } 27 | } 28 | 29 | impl<'a> PartialEq for &'a str { 30 | fn eq(&self, other: &Value) -> bool { 31 | other.as_str().map_or(false, |s| s == *self) 32 | } 33 | } 34 | 35 | impl PartialEq for Value { 36 | fn eq(&self, other: &String) -> bool { 37 | self.as_str().map_or(false, |s| s == other) 38 | } 39 | } 40 | 41 | 42 | impl PartialEq for String { 43 | fn eq(&self, other: &Value) -> bool { 44 | other.as_str().map_or(false, |s| s == self) 45 | } 46 | } 47 | 48 | macro_rules! partialeq_numeric { 49 | ($([$($ty:ty)*], $conversion:ident, $base:ty)*) => { 50 | $($( 51 | impl PartialEq<$ty> for Value { 52 | fn eq(&self, other: &$ty) -> bool { 53 | self.$conversion().map_or(false, |i| i == (*other as $base)) 54 | } 55 | } 56 | 57 | impl PartialEq for $ty { 58 | fn eq(&self, other: &Value) -> bool { 59 | other.$conversion().map_or(false, |i| i == (*self as $base)) 60 | } 61 | } 62 | 63 | impl<'a> PartialEq<$ty> for &'a Value { 64 | fn eq(&self, other: &$ty) -> bool { 65 | self.$conversion().map_or(false, |i| i == (*other as $base)) 66 | } 67 | } 68 | 69 | impl<'a> PartialEq<$ty> for &'a mut Value { 70 | fn eq(&self, other: &$ty) -> bool { 71 | self.$conversion().map_or(false, |i| i == (*other as $base)) 72 | } 73 | } 74 | )*)* 75 | } 76 | } 77 | 78 | partialeq_numeric! { 79 | [i8 i16 i32 i64 isize], as_i64, i64 80 | [u8 u16 u32 u64 usize], as_u64, u64 81 | [f32 f64], as_f64, f64 82 | [bool], as_bool, bool 83 | } 84 | -------------------------------------------------------------------------------- /net/src/services.rs: -------------------------------------------------------------------------------- 1 | use std::fmt::{self, Debug}; 2 | use std::io; 3 | 4 | use Encode; 5 | 6 | bitflags! { 7 | #[derive(Default)] 8 | flags ServiceFlags: u64 { 9 | const NETWORK = 0b00000001, 10 | const UTXO = 0b00000010, 11 | const BLOOM = 0b00000100, 12 | } 13 | } 14 | 15 | /// The following services are currently assigned: 16 | /// 17 | /// Value Name Description 18 | /// 1 NODE_NETWORK This node can be asked for full blocks instead of just headers. 19 | /// 2 NODE_GETUTXO See [BIP 0064](https://github.com/bitcoin/bips/blob/master/bip-0064.mediawiki) 20 | /// 4 NODE_BLOOM See [BIP 0111](https://github.com/bitcoin/bips/blob/master/bip-0111.mediawiki) 21 | #[derive(Clone, Copy, Default, PartialEq, Eq, Hash)] 22 | pub struct Services { 23 | flags: ServiceFlags, 24 | } 25 | 26 | impl Debug for Services { 27 | fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> { 28 | 29 | write!(f, 30 | r"Services {{ 31 | network: {}, 32 | utxo: {}, 33 | bloom: {}}}", 34 | self.network(), 35 | self.utxo(), 36 | self.bloom()) 37 | } 38 | } 39 | 40 | impl Services { 41 | pub fn as_i64(&self) -> i64 { 42 | self.flags.bits as i64 43 | } 44 | 45 | pub fn from(input: u64) -> Services { 46 | Services { flags: ServiceFlags { bits: input } } 47 | } 48 | 49 | pub fn network(&self) -> bool { 50 | self.flags.contains(NETWORK) 51 | } 52 | 53 | pub fn utxo(&self) -> bool { 54 | self.flags.contains(UTXO) 55 | } 56 | 57 | pub fn bloom(&self) -> bool { 58 | self.flags.contains(BLOOM) 59 | } 60 | } 61 | 62 | impl Encode for Services { 63 | fn encode(&self, mut buff: &mut Vec) -> Result<(), io::Error> { 64 | self.flags.bits.encode(&mut buff) 65 | } 66 | } 67 | 68 | #[cfg(test)] 69 | mod tests { 70 | use super::*; 71 | 72 | #[test] 73 | fn it_identifies_a_network_node() { 74 | let svc = Services::from(0b00000001u64); 75 | println!("{:?}", svc); 76 | assert!(svc.network()); 77 | } 78 | 79 | #[test] 80 | fn it_identifies_a_utxo_node() { 81 | let svc = Services::from(0b00000010u64); 82 | println!("{:?}", svc); 83 | assert!(svc.utxo()); 84 | } 85 | 86 | #[test] 87 | fn it_identifies_a_bloom_node() { 88 | let svc = Services::from(0b00000100u64); 89 | println!("{:?}", svc); 90 | assert!(svc.bloom()); 91 | } 92 | 93 | #[test] 94 | fn it_identifies_a_bloom_network_node() { 95 | let svc = Services::from(0b00000101u64); 96 | println!("{:?}", svc); 97 | assert!(svc.network()); 98 | assert!(svc.bloom()); 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /src/store/README.md: -------------------------------------------------------------------------------- 1 | 2 | # Flatfiles 3 | 4 | Various data is stored in *flatfilesets* [(src)](flatfileset.rs). 5 | A datatype is associate with a path and a prefix and the data is stored 6 | as sequential records in files of the format path/prefix-XXXX. 7 | 8 | Here XXXX is a hex signed file-sequence number and the maximum file size; the files are of fixed size to allow 9 | memory mapped access. The sequence number is signed such that files can be pruning or cleaned to lower numbers. 10 | 11 | Each file starts with a 16 byte header: 12 | 13 | 4 bytes magic number 14 | 4 bytes write-pointer; this is the location where the next transanction will be written in the file 15 | (both in native endian) 16 | 8 bytes reserved. 17 | 18 | In most operation the files are append-only and writes and reads occur lock-free. Writes will first increase the 19 | write-pointer (using atomic compare-and-swap) and then write the record at the location of the previous write pointer. 20 | 21 | Data in the files can be identified with pointer that implements the *flatfileptr* Trait. 22 | 23 | This provides a file number (max 16-bit) and a file offset (max 64 bit), 24 | but the different filesets use different semantics as seen in [TxPtr](txptr.rs) 25 | and [BlockHeaderPtr](blockheaderptr.rs). 26 | 27 | 28 | ## Block Content 29 | 30 | 31 | Transactions and blockheaders are stored in flatfiles `transactions/tx-XXXX` 32 | and `headers/bh-0000`. 33 | 34 | Both are prefixed with a 4-byte length and written in network format. 35 | Blockheaders are not length-prefixed, and also stored in network format. 36 | 37 | 38 | ## Hash Index 39 | 40 | Hashes of blocks and transactions are looked in two hash-indexes [(src)](hash_index.rs). 41 | They are stored in flat_files `tx-index/ti-XXXX` and `block-index/bi-XXXX`. The first 64mb of the flatfileset is 42 | the root node; it is a hash-table to resolve the first 24-bits of a hash. This points to a append-only unbalanced 43 | binary tree. 44 | 45 | This set-up ensures a nice temporal locality of reference, as only the 64mb root node and recent tree-branches are 46 | needed in RAM. 47 | 48 | ## Spend-tree 49 | 50 | Files with the name `spend-tree/st-XXXX` [(src)](spend_tree/mod.rs) contain the spend-tree; Records are 16 byte long. 51 | 52 | 53 | A block is added to the spend_tree by first adding a start-of-block record, then for each transanction a transaction record 54 | and for each input of the transaction an output-spend record. At the end an end-of-block record is added. 55 | 56 | Each output-spend is verified by scanning backwards using the parent pointers, to ensure that the same output-spend is 57 | not found before the spend transaction record is found. 58 | 59 | This ensures that 60 | 61 | * the transaction that is being spent exists on this branch of the tree and 62 | * the output of the transaction was not yet spent. 63 | 64 | ## Spend-index 65 | 66 | The spend index `spend-index/si-XXXX` [(src)](spend_index.rs) catches seeks earlier in the chain 67 | and uses a simple concurrent bit-index to look them up. 68 | -------------------------------------------------------------------------------- /store/tests/blk_file.rs: -------------------------------------------------------------------------------- 1 | //! 2 | //! Module to access bitcoin-core style blk files 3 | //! that store the integral blockchain 4 | 5 | 6 | extern crate byteorder; 7 | extern crate store; 8 | 9 | use self::byteorder::{ReadBytesExt, LittleEndian}; 10 | use std::io::BufReader; 11 | use std::fs::File; 12 | use std::path::Path; 13 | 14 | use std::io; 15 | 16 | 17 | /// Magic number stored at the start of each block 18 | const MAGIC: u32 = 0xD9B4BEF9; 19 | 20 | 21 | pub struct BlkFileIterator { 22 | file_nr: usize, 23 | reader: BufReader 24 | 25 | } 26 | 27 | impl Iterator for BlkFileIterator { 28 | type Item = Vec; 29 | 30 | fn next(&mut self) -> Option> { 31 | 32 | let blk = read_block(&mut self.reader).unwrap(); 33 | if blk.is_none() { 34 | self.file_nr += 1; 35 | let sname = format!("../core-blocks/blk{:05}.dat", self.file_nr); 36 | let name = Path::new(&sname); 37 | if !name.exists() { 38 | return None; 39 | } 40 | let f = File::open(name).unwrap(); 41 | self.reader = BufReader::new(f); 42 | read_block(&mut self.reader).unwrap() 43 | 44 | } 45 | else { 46 | blk 47 | } 48 | } 49 | } 50 | 51 | 52 | pub fn read_blocks() -> BlkFileIterator { 53 | let sname = format!("../core-blocks/blk{:05}.dat", 0); 54 | let name = Path::new(& sname); 55 | if !name.exists() { 56 | panic!("No blk-files found at ./core-blocks. \ 57 | Use 'ln - ~/.bitcoin/blocks ./core-blocks' to link the directory."); 58 | } 59 | let f = File::open(name).unwrap(); 60 | let rdr = BufReader::new(f); 61 | 62 | BlkFileIterator { file_nr: 0, reader: rdr } 63 | } 64 | 65 | /// Reads a block from a blk_file as used by 66 | /// bitcoin-core and various other implementations 67 | fn read_block(rdr: &mut io::Read) -> Result>, io::Error> { 68 | 69 | loop { 70 | let magicnr = rdr.read_u32::(); 71 | match magicnr { 72 | Err(_) => return Ok(None), // assume EOF 73 | Ok(m) => match m { 74 | 75 | // TODO investigate; // Can't really find it in the cpp. 76 | // this happens on bitcrust-1 at block 451327 77 | // file blk000760, file pos 54391594 78 | // first 8 zero-bytes before magicnr 79 | // for now we skip them; not too important as we 80 | // might not want to support this type of import anyway 81 | 0 => continue, 82 | 83 | MAGIC => break, 84 | _ =>return Err(io::Error::new(io::ErrorKind::InvalidData, "Incorrect magic number")) 85 | } 86 | } 87 | 88 | } 89 | 90 | 91 | let length = try!(rdr.read_u32::()); 92 | let mut buffer = vec![0; length as usize]; 93 | 94 | 95 | try!(rdr.read_exact(&mut buffer)); 96 | 97 | 98 | Ok(Some(buffer)) 99 | 100 | 101 | } 102 | 103 | 104 | 105 | 106 | -------------------------------------------------------------------------------- /net/src/message/transaction_message.rs: -------------------------------------------------------------------------------- 1 | use std::io; 2 | 3 | use Encode; 4 | use VarInt; 5 | use transactions::*; 6 | 7 | #[cfg(test)] 8 | mod tests { 9 | use super::*; 10 | 11 | #[test] 12 | fn it_implements_types_required_for_protocol() { 13 | let m = TransactionMessage::default(); 14 | assert_eq!(m.name(), "tx"); 15 | assert_eq!(m.len(), 18); 16 | } 17 | } 18 | 19 | /// 20 | /// Field Size Description Data type Comments 21 | /// 4 version int32_t Transaction data format version (note, this is signed) 22 | /// 0 or 2 flag optional uint8_t[2] If present, always 0001, and indicates the presence of witness data 23 | /// 1+ tx_in count var_int Number of Transaction inputs (never zero) 24 | /// 41+ tx_in tx_in[] A list of 1 or more transaction inputs or sources for coins 25 | /// 1+ tx_out count var_int Number of Transaction outputs 26 | /// 9+ tx_out tx_out[] A list of 1 or more transaction outputs or destinations for coins 27 | /// 0+ tx_witnesses tx_witness[] A list of witnesses, one for each input; omitted if flag is omitted above 28 | /// 4 lock_time uint32_t The block number or timestamp at which this transaction is unlocked: 29 | /// Value Description 30 | /// 0 Not locked 31 | /// < 500000000 Block number at which this transaction is unlocked 32 | /// >= 500000000 UNIX timestamp at which this transaction is unlocked 33 | /// If all TxIn inputs have final (0xffffffff) sequence numbers then lock_time is irrelevant. Otherwise, the transaction may not be added to a block until after lock_time (see NLockTime). 34 | /// 35 | #[derive(Debug, Default, PartialEq)] 36 | pub struct TransactionMessage { 37 | pub version: i32, 38 | pub inputs: Vec, 39 | pub outputs: Vec, 40 | pub witnesses: Vec, 41 | pub lock_time: u32, 42 | } 43 | 44 | impl TransactionMessage { 45 | #[inline] 46 | pub fn len(&self) -> usize { 47 | 4usize // version 48 | + 2usize // flag for witness data 49 | + 4usize // len of inputs 50 | + self.inputs.iter().map(|i| i.len()).sum::() // inputs 51 | + 4usize // len of outputs 52 | + self.outputs.iter().map(|i| i.len()).sum::() // outputs 53 | + self.witnesses.iter().map(|i| i.len()).sum::() // witnesses 54 | + 4usize // lock time 55 | } 56 | 57 | #[inline] 58 | pub fn name(&self) -> &'static str { 59 | "tx" 60 | } 61 | } 62 | 63 | impl Encode for TransactionMessage { 64 | fn encode(&self, mut buff: &mut Vec) -> Result<(), io::Error> { 65 | self.version.encode(&mut buff)?; 66 | 67 | if self.witnesses.len() > 0 { 68 | [0x00, 0x01].encode(&mut buff)?; 69 | } 70 | VarInt::new(self.inputs.len() as u64).encode(&mut buff)?; 71 | self.inputs.encode(&mut buff)?; 72 | 73 | VarInt::new(self.outputs.len() as u64).encode(&mut buff)?; 74 | self.outputs.encode(&mut buff)?; 75 | 76 | if self.witnesses.len() > 0 { 77 | VarInt::new(self.witnesses.len() as u64).encode(&mut buff)?; 78 | self.witnesses.encode(&mut buff)?; 79 | } 80 | 81 | self.lock_time.encode(&mut buff)?; 82 | Ok(()) 83 | } 84 | } -------------------------------------------------------------------------------- /src/store/txptr.rs: -------------------------------------------------------------------------------- 1 | 2 | 3 | //! A TxPtr is a pointer to a transaction in the transaction-content 4 | //! It contains a file_number, a file offset that together point to the data 5 | //! And optionally a input-index. 6 | //! 7 | //! TxPtr's are stored in the TransactionIndex; if they contain an input index 8 | //! they are used as `guard`: When a transaction is stored but the prev_tx needed 9 | //! for validation is not found, a link to the input that still awaits validation 10 | //! is stored as a guard in the prev_tx location in the TransactionIndex 11 | 12 | 13 | use super::flatfileset::FlatFilePtr; 14 | use super::hash_index::HashIndexGuard; 15 | 16 | const INPUT_INDEX_NULL: u16 = 0xFFFF; 17 | 18 | #[derive(Debug, PartialEq, Clone, Copy, Hash, Eq)] 19 | pub struct TxPtr { 20 | file_offset: u32, 21 | file_number: i16, 22 | input_index: u16 23 | } 24 | 25 | 26 | impl FlatFilePtr for TxPtr { 27 | 28 | fn new(file_number: i16, file_offset: u64) -> Self { 29 | TxPtr { 30 | file_offset: file_offset as u32, 31 | file_number: file_number, 32 | input_index: INPUT_INDEX_NULL 33 | } 34 | } 35 | 36 | fn get_file_number(self) -> i16 { self.file_number } 37 | 38 | fn get_file_offset(self) -> u64 { self.file_offset as u64 } 39 | 40 | } 41 | 42 | impl HashIndexGuard for TxPtr { 43 | 44 | fn is_guard(self) -> bool { self.input_index != INPUT_INDEX_NULL } 45 | } 46 | 47 | impl TxPtr { 48 | pub fn get_input_index(self) -> u16 { 49 | 50 | debug_assert!(self.is_guard()); 51 | 52 | self.input_index 53 | } 54 | 55 | pub fn to_input(self, input_index: u16) -> TxPtr { 56 | 57 | debug_assert!(input_index != INPUT_INDEX_NULL); 58 | 59 | TxPtr { 60 | file_offset: self.file_offset, 61 | file_number: self.file_number, 62 | input_index: input_index 63 | } 64 | } 65 | 66 | pub fn first() -> TxPtr { 67 | TxPtr { 68 | file_number: 0, 69 | file_offset: super::flatfile::INITIAL_WRITEPOS as u32, 70 | input_index: INPUT_INDEX_NULL 71 | } 72 | } 73 | 74 | pub fn offset(self, offset: u32) -> TxPtr { 75 | if self.file_offset + offset > super::MAX_CONTENT_SIZE as u32 { 76 | println!("Next file!"); 77 | TxPtr { 78 | file_number: self.file_number + 1, 79 | file_offset: super::flatfile::INITIAL_WRITEPOS as u32, 80 | input_index: INPUT_INDEX_NULL 81 | } 82 | } else { 83 | TxPtr { 84 | file_number: self.file_number, 85 | file_offset: self.file_offset + offset, 86 | input_index: INPUT_INDEX_NULL 87 | } 88 | } 89 | } 90 | } 91 | 92 | #[cfg(test)] 93 | mod tests { 94 | 95 | use super::*; 96 | use super::super::flatfile::INITIAL_WRITEPOS; 97 | #[test] 98 | fn test_skip() 99 | { 100 | let x = TxPtr::first(); 101 | let y = x.offset(1000); 102 | assert_eq!(x.file_number, 0); 103 | assert_eq!(y.file_offset, 1000 + INITIAL_WRITEPOS as u32); 104 | 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /src/script/context.rs: -------------------------------------------------------------------------------- 1 | 2 | /// Context provides an execution environment for scripts 3 | /// Currently unused 4 | 5 | use super::stack; 6 | use super::ScriptError; 7 | use std::fmt; 8 | use std::io; 9 | use std::io::Write; 10 | 11 | use script::opcode::OPCODES; 12 | 13 | pub struct Context<'a> { 14 | pub stack: stack::Stack, 15 | pub alt_stack: stack::Stack, 16 | 17 | pub script1: &'a[u8], 18 | pub ip: usize 19 | } 20 | 21 | 22 | impl<'a> Context<'a> { 23 | 24 | pub fn new(script: &'a[u8]) -> Context<'a> 25 | { 26 | Context { 27 | stack: stack::Stack::new(), 28 | alt_stack: stack::Stack::new(), 29 | script1: script, 30 | ip: 0 31 | } 32 | } 33 | 34 | pub fn run(&mut self) -> Result<(), ScriptError> { 35 | unimplemented!() 36 | } 37 | 38 | 39 | /// Returns the bytes of the script pointed to by the current 40 | /// ip (instruction pointer), and increases the ip to the last 41 | /// byte returned 42 | /// 43 | /// Can return a UnexpectedEndOfScript if not enough bytes are available 44 | pub fn next_bytes(&mut self, count: u64) -> Result<&[u8], ScriptError> { 45 | if self.script1.len() < self.ip + count as usize + 1 { 46 | return Err(ScriptError::UnexpectedEndOfScript); 47 | } 48 | 49 | let old_ip = self.ip; 50 | self.ip += count as usize; 51 | Ok(&self.script1[old_ip + 1 .. self.ip + 1]) 52 | 53 | } 54 | 55 | pub fn next_uint(&mut self, count: u64) -> Result { 56 | let bytes = try!(self.next_bytes(count)); 57 | 58 | // parse as little endian 59 | Ok(bytes.iter().enumerate().fold(0, 60 | |sum, (n, byte)| sum + ((*byte as u64) << (n * 8)) 61 | )) 62 | } 63 | 64 | } 65 | 66 | 67 | impl<'a> fmt::Debug for Context<'a> { 68 | 69 | fn fmt(&self, fmt: &mut fmt::Formatter) -> Result<(), fmt::Error> { 70 | 71 | // create mutable copy 72 | let mut copied_context = Context::new(self.script1); 73 | 74 | // target to write the script to 75 | let buf: Vec = Vec::new(); 76 | let mut cursor = io::Cursor::new(buf); 77 | 78 | 79 | while copied_context.ip < copied_context.script1.len() { 80 | let opcode = copied_context.script1[copied_context.ip] as usize; 81 | 82 | // OpCodes display function 83 | (OPCODES[opcode].display)(&mut copied_context, &mut cursor).unwrap(); 84 | 85 | write!(&mut cursor, " ").unwrap(); 86 | copied_context.ip += 1; 87 | } 88 | 89 | // write to output 90 | // we know we're not writing invalid utf so we can unwrap 91 | write!(fmt, "{}", &String::from_utf8(cursor.into_inner()).unwrap()) 92 | 93 | } 94 | } 95 | 96 | 97 | 98 | mod tests { 99 | #![cfg(test)] 100 | use ::script::context::Context; 101 | 102 | 103 | #[test] 104 | fn test_op_false() 105 | { 106 | let script = vec![0x00]; 107 | let _ = Context::new(&script); 108 | 109 | 110 | 111 | 112 | } 113 | } -------------------------------------------------------------------------------- /src/store/prune.rs: -------------------------------------------------------------------------------- 1 | 2 | 3 | //! Preliminairy pruning of tx-index 4 | //! This is probably less needed when using HAMT 5 | 6 | // this is needed because of #[ignore] 7 | #![allow(unused_imports)] 8 | #![allow(dead_code)] 9 | 10 | use std::env; 11 | use std::fs; 12 | use std::path; 13 | 14 | use hash::*; 15 | use buffer::*; 16 | use config::Config; 17 | use store::hash_index; 18 | use store::{Store,TxPtr,Record}; 19 | use transaction::Transaction; 20 | 21 | 22 | /// Prunes the tx-index of the store to a separate tx-index-pruned folder 23 | fn prune_to_new_index() { 24 | 25 | let cfg = test_cfg!(); 26 | let mut store = Store::new(&cfg); 27 | 28 | 29 | let mut new_tx_index: hash_index::HashIndex 30 | = hash_index::HashIndex::new(&cfg, "tx-index-pruned"); 31 | 32 | let mut tx_ptr = TxPtr::first(); 33 | let mut count: u64 = 0; 34 | let mut count_purged: u64 = 0; 35 | loop { 36 | let result = store.transactions.next(tx_ptr); 37 | 38 | if result == None { 39 | break; 40 | } 41 | let (tx_raw, next_ptr) = result.unwrap(); 42 | 43 | // we parse it only for the output_count 44 | let tx = Transaction::parse(&mut Buffer::new(tx_raw.as_slice())).unwrap(); 45 | 46 | let input_count = tx.txs_out.len() as u32; 47 | 48 | let tx_ptr_copy = tx_ptr; 49 | let spend_outputs = (0..input_count) 50 | .map(|n| Record::new_output(tx_ptr_copy, n)) 51 | .map(|rec| rec.hash()) 52 | .filter(|hash| store.spend_index.exists(*hash)) 53 | .count() as u32; 54 | 55 | if spend_outputs < input_count { 56 | 57 | // we still need this one 58 | let hash = Hash32Buf::double_sha256(tx.to_raw()); 59 | 60 | assert_eq!(store.tx_index.get(hash.as_ref()).len(),1); 61 | 62 | new_tx_index.set(hash.as_ref(), tx_ptr, &[], true); 63 | } 64 | else { 65 | // all inputs are spend; don't add it to the new-index 66 | count_purged += 1; 67 | } 68 | 69 | 70 | count = count + 1; 71 | if count % 1000 == 0 { 72 | println!("Done: {}; purged {} %", count, count_purged as u64 * 100 / count as u64); 73 | } 74 | 75 | tx_ptr = next_ptr; 76 | } 77 | 78 | println!("Done"); 79 | println!(" {} transactions", count); 80 | println!(" {} purged ({} %)", count_purged , count_purged as u64 * 100 / count as u64); 81 | println!(" {} remain", count - count_purged ,); 82 | 83 | } 84 | 85 | #[ignore] 86 | #[test] 87 | fn prune_tx_index() { 88 | 89 | 90 | let store_path = env::var(::config::ENV_BITCRUST_STORE). 91 | expect(&format!("Use {} env var to specify a store to prune", ::config::ENV_BITCRUST_STORE)); 92 | 93 | // add pruned tx-index to store_path 94 | let pruned_path = path::PathBuf::from(&store_path).join("tx-index-pruned"); 95 | let tx_index_path = path::PathBuf::from(&store_path).join("tx-index"); 96 | let _ = fs::remove_dir_all(&pruned_path); // if exists 97 | 98 | prune_to_new_index(); 99 | 100 | // move the new index into position 101 | fs::remove_dir_all(&tx_index_path).expect("Couldn't remove old tx-index"); 102 | fs::rename(pruned_path, &tx_index_path).expect("Failed to move tx-index after pruning"); 103 | 104 | } -------------------------------------------------------------------------------- /site/results.html: -------------------------------------------------------------------------------- 1 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 33 | 34 | 35 | 36 | 39 | 40 | BITCRUST 41 | 42 | 43 | 44 |
45 | 47 |
48 | 49 | 50 | 51 |
52 | 53 | 57 |
58 |
59 | 60 |
61 |
62 | 63 | 64 |
65 |

66 | The test results below show a comparison of recent blocks between the full block processing of Core, the full processing of Bitcrust, as well as the order processing of Bitcrust. 67 |

68 |

69 | The comparison is between a finished product and an unfinished product and as such only provide an indication of what to expect. 70 |

71 |

72 | Both Core (v0.13.2) and Bitcrust are currently running on the same machine (32gb/8 cores). CPU competition is prevented by not verifying at the same time, but competition for the OS cache is causing some extra variance. 73 |

74 |

75 | Blocks only mode is used in Core and Bitcrust; the tx-index of Bitcrust is manually pruned to best match Core's defaults. 76 |

77 |
78 |
79 |
80 | 81 | 82 |
83 | 84 | 85 | 86 | 87 | 88 |
89 | 90 | 91 | 92 | 93 | 94 | 95 | -------------------------------------------------------------------------------- /net/src/message/version_message.rs: -------------------------------------------------------------------------------- 1 | use std::io; 2 | 3 | use Encode; 4 | use net_addr::NetAddr; 5 | use services::Services; 6 | 7 | #[cfg(test)] 8 | mod tests { 9 | use std::net::Ipv6Addr; 10 | use std::str::FromStr; 11 | use super::*; 12 | #[test] 13 | fn it_parses_a_version_message() {} 14 | 15 | #[test] 16 | fn it_encodes_a_version_message() { 17 | let v = VersionMessage { 18 | version: 60002, 19 | services: Services::from(1), 20 | timestamp: 1495102309, 21 | addr_recv: NetAddr { 22 | time: None, 23 | services: Services::from(1), 24 | ip: Ipv6Addr::from_str("::ffff:127.0.0.1").unwrap(), 25 | port: 8333, 26 | }, 27 | addr_send: NetAddr { 28 | time: None, 29 | services: Services::from(1), 30 | ip: Ipv6Addr::from_str("::ffff:127.0.0.1").unwrap(), 31 | port: 8333, 32 | }, 33 | nonce: 1, 34 | user_agent: "bitcrust".into(), 35 | start_height: 0, 36 | relay: false, 37 | }; 38 | let mut encoded = vec![]; 39 | v.encode(&mut encoded).unwrap(); 40 | let expected: Vec = 41 | vec![98, 234, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 101, 115, 29, 89, 0, 0, 0, 0, 1, 0, 0, 0, 42 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 255, 127, 0, 0, 1, 32, 141, 1, 0, 43 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 255, 127, 0, 0, 1, 32, 141, 44 | 1, 0, 0, 0, 0, 0, 0, 0, 8, 98, 105, 116, 99, 114, 117, 115, 116, 0, 0, 0, 0]; 45 | assert_eq!(expected, encoded); 46 | } 47 | 48 | #[test] 49 | fn it_implements_types_required_for_protocol() { 50 | let m = VersionMessage::default(); 51 | assert_eq!(m.name(), "version"); 52 | assert_eq!(m.len(), 86); 53 | } 54 | 55 | } 56 | 57 | #[derive(Debug, Default, PartialEq)] 58 | pub struct VersionMessage { 59 | pub version: i32, 60 | pub services: Services, 61 | pub timestamp: i64, 62 | pub addr_recv: NetAddr, 63 | pub addr_send: NetAddr, 64 | pub nonce: u64, 65 | pub user_agent: String, 66 | pub start_height: i32, 67 | pub relay: bool, 68 | } 69 | 70 | impl VersionMessage { 71 | #[inline] 72 | pub fn len(&self) -> usize { 73 | 86 + self.user_agent.len() 74 | } 75 | 76 | #[inline] 77 | pub fn name(&self) -> &'static str { 78 | "version" 79 | } 80 | } 81 | 82 | impl Encode for VersionMessage { 83 | fn encode(&self, mut buff: &mut Vec) -> Result<(), io::Error> { 84 | let _ = self.version.encode(&mut buff); 85 | let _ = self.services.encode(&mut buff); 86 | let _ = self.timestamp.encode(&mut buff); 87 | let _ = self.addr_recv.encode(&mut buff); 88 | if self.version >= 106 { 89 | let _ = self.addr_send.encode(&mut buff); 90 | let _ = self.nonce.encode(&mut buff); 91 | let _ = (self.user_agent.bytes().len() as u8).encode(&mut buff); 92 | let _ = self.user_agent.encode(&mut buff); 93 | let _ = self.start_height.encode(&mut buff); 94 | if self.version >= 70001 { 95 | if self.relay { 96 | buff.push(1); 97 | } else { 98 | buff.push(0); 99 | } 100 | } 101 | } 102 | Ok(()) 103 | } 104 | } -------------------------------------------------------------------------------- /src/store/spend_index.rs: -------------------------------------------------------------------------------- 1 | //! Index that stores transactions and spend outputs 2 | //! 3 | //! This serves as a broom-wagon for the spend-tree. 4 | //! After X blocks, outputs and transactions are stored here, 5 | //! and when outputs aren't found in the spend tree for X blocks, 6 | //! they are searched here. 7 | //! 8 | //! The data-structure here is a simple bit-index where each transaction and each spend-output 9 | //! are given a unique bit which is set if the given transaction or spend exists 10 | 11 | use std::sync::atomic::{AtomicU64,Ordering}; 12 | 13 | use config; 14 | use store::flatfileset::FlatFileSet; 15 | use store::RecordPtr; 16 | 17 | 18 | const MB: u64 = 1024 * 1024; 19 | const FILE_SIZE: u64 = 16 * 1024 * MB ; 20 | const MAX_CONTENT_SIZE: u64 = FILE_SIZE - 10 * MB ; 21 | 22 | // TODO; make this dynamic using fileset continuation; 23 | // this isn't hastily needed as the OS does not actually allocate 24 | // all the space; (compare ls with du). 25 | const VEC_SIZE: usize = 500_000_000; 26 | 27 | /// Index to lookup spends 28 | /// 29 | /// Internally uses fileset 30 | /// 31 | pub struct SpendIndex { 32 | 33 | #[allow(dead_code)] 34 | fileset: FlatFileSet, 35 | 36 | bitvector: &'static [AtomicU64] 37 | 38 | } 39 | 40 | unsafe impl Sync for SpendIndex {} 41 | 42 | impl SpendIndex 43 | { 44 | /// Opens the spend_index at the location given in the config 45 | /// 46 | /// Creates a new fileset if needed 47 | pub fn new(cfg: &config::Config) -> SpendIndex { 48 | let dir = &cfg.root.clone().join("spend-index"); 49 | 50 | let mut fileset = FlatFileSet::new( 51 | dir, "si-", FILE_SIZE, MAX_CONTENT_SIZE); 52 | 53 | let bitvector = fileset.read_mut_slice(RecordPtr::new(0), VEC_SIZE); 54 | SpendIndex { 55 | fileset: fileset, 56 | bitvector: bitvector 57 | } 58 | } 59 | 60 | 61 | 62 | 63 | /// Tests if the given hash exists. 64 | pub fn exists(&self, hash: u64) -> bool { 65 | 66 | let idx = (hash >> 6) as usize; 67 | (self.bitvector[idx].load(Ordering::Relaxed) & (1 << (hash & 0x3F))) > 0 68 | } 69 | 70 | 71 | /// Stores a record hash; this should uniquely identify an output or a transaction 72 | pub fn set(&mut self, hash: u64) { 73 | 74 | // CAS-loop 75 | loop { 76 | let idx = (hash >> 6) as usize; 77 | let org = self.bitvector[idx].load(Ordering::Acquire); 78 | let new = org | (1 << (hash & 0x3F)); 79 | 80 | if self.bitvector[idx].compare_and_swap(org, new, Ordering::Release) == org { 81 | break; 82 | } 83 | } 84 | } 85 | } 86 | 87 | 88 | #[cfg(test)] 89 | mod tests { 90 | 91 | extern crate rand; 92 | use std::collections::HashSet; 93 | 94 | use super::*; 95 | 96 | 97 | #[test] 98 | fn test_seq() { 99 | 100 | let mut idx: SpendIndex = SpendIndex::new(& test_cfg!() ); 101 | 102 | 103 | let mut set = HashSet::new(); 104 | 105 | for n in 0..60000_u64 { 106 | if n % 3 == 0 { 107 | set.insert(n); 108 | idx.set(n); 109 | } 110 | 111 | } 112 | 113 | for n in 0..60000 { 114 | if n % 3 == 0 { 115 | assert!( idx.exists(n)); 116 | } 117 | else { 118 | assert!( !idx.exists(n)); 119 | } 120 | } 121 | } 122 | } -------------------------------------------------------------------------------- /net/src/net_addr.rs: -------------------------------------------------------------------------------- 1 | use std::default::Default; 2 | use std::hash::{Hash, Hasher}; 3 | use std::io; 4 | use std::net::{IpAddr, Ipv6Addr, SocketAddr}; 5 | use std::str::FromStr; 6 | use std::time::{SystemTime, UNIX_EPOCH}; 7 | 8 | use byteorder::{BigEndian, WriteBytesExt}; 9 | 10 | use Encode; 11 | use services::Services; 12 | 13 | #[cfg(test)] 14 | mod tests { 15 | use std::str::FromStr; 16 | use Encode; 17 | use super::*; 18 | use std::net::Ipv4Addr; 19 | 20 | #[test] 21 | fn it_parses_a_net_address() {} 22 | 23 | #[test] 24 | fn it_encodes_a_net_address() { 25 | let addr = NetAddr { 26 | time: None, 27 | services: Services::from(1), 28 | ip: Ipv6Addr::from_str("::ffff:10.0.0.1").unwrap(), 29 | port: 8333, 30 | }; 31 | 32 | let mut encoded = vec![]; 33 | addr.encode(&mut encoded).unwrap(); 34 | let expected = vec![ 35 | 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // services 36 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xFF, 0xFF, 0x0A, 0x00, 0x00, 0x01, // IP 37 | 0x20, 0x8d // port 38 | ]; 39 | assert_eq!(expected, encoded); 40 | } 41 | 42 | #[test] 43 | fn it_builds_from_socket_addr() { 44 | let socket_addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8080); 45 | let addr = NetAddr::from_socket_addr( 46 | socket_addr 47 | ); 48 | assert_eq!(addr.port, 8080); 49 | match socket_addr.ip() { 50 | IpAddr::V4(a) => assert_eq!(addr.ip, a.to_ipv6_mapped()), 51 | IpAddr::V6(_) => unreachable!() 52 | }; 53 | 54 | } 55 | } 56 | 57 | #[derive(Clone, Copy, Debug, PartialEq, Eq)] 58 | pub struct NetAddr { 59 | pub time: Option, 60 | pub services: Services, 61 | pub ip: Ipv6Addr, 62 | pub port: u16, 63 | } 64 | 65 | impl NetAddr { 66 | pub fn from_socket_addr(addr: SocketAddr) -> NetAddr { 67 | let start = SystemTime::now(); 68 | let now = start.duration_since(UNIX_EPOCH) 69 | .expect("Time went backwards").as_secs(); 70 | let ip = match addr.ip() { 71 | IpAddr::V4(a) => a.to_ipv6_mapped(), 72 | IpAddr::V6(a) => a 73 | }; 74 | NetAddr { 75 | time: Some(now as u32), 76 | services: Services::from(0), 77 | ip: ip, 78 | port: addr.port(), 79 | } 80 | } 81 | 82 | pub fn to_host(&self) -> String { 83 | format!("{}:{}", self.ip, self.port) 84 | } 85 | } 86 | 87 | impl Encode for NetAddr { 88 | fn encode(&self, mut v: &mut Vec) -> Result<(), io::Error> { 89 | // write time 90 | self.time.encode(&mut v)?; 91 | // write services 92 | self.services.encode(&mut v)?; 93 | // write IP 94 | self.ip.encode(&mut v)?; 95 | // write port 96 | v.write_u16::(self.port)?; 97 | Ok(()) 98 | } 99 | } 100 | 101 | impl Hash for NetAddr { 102 | fn hash(&self, state: &mut H) { 103 | self.services.hash(state); 104 | self.ip.hash(state); 105 | self.port.hash(state); 106 | } 107 | } 108 | 109 | impl Default for NetAddr { 110 | fn default() -> NetAddr { 111 | NetAddr { 112 | time: None, 113 | services: Services::default(), 114 | ip: Ipv6Addr::from_str("::ffff:10.0.0.1").unwrap(), 115 | port: 8333, 116 | } 117 | } 118 | } 119 | -------------------------------------------------------------------------------- /src/store/tips.rs: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | //! These rules manage the set of tips 5 | //! 6 | //! * When genesis is added, it is stored as tip 0 7 | //! * When a block is added to a block with a tip, the tip is updated 8 | //! * When a block is added to a block without a tip, a tip is added 9 | //! * Tips are eventually pruned TBD 10 | 11 | //! Tips do not use flatfileset. Instead, each tip is a file on disk 12 | //! as these cover the requirements better and nicely allow atomic replacement 13 | 14 | //! A tip is referenced by the block hash and it contains 15 | //! * The block hash 16 | //! * The difficulty target 17 | //! * Softfork info 18 | //! 19 | //! This is a draft implementation; the format is TBD 20 | 21 | 22 | 23 | use std::path::{Path,PathBuf}; 24 | use std::fs; 25 | use std::fs::File; 26 | use std::io; 27 | use std::io::prelude::*; 28 | 29 | use util::*; 30 | use hash::*; 31 | use config; 32 | 33 | 34 | pub struct Tips { 35 | 36 | path: PathBuf 37 | } 38 | 39 | pub fn add_tip( 40 | tips: &Tips, 41 | block_hash: Hash32Buf, 42 | previous_hash: Option, 43 | difficulty: u64, 44 | height: u64) 45 | { 46 | 47 | 48 | let tip = Tip { 49 | block_hash: block_hash, 50 | difficulty: difficulty, 51 | height: height 52 | }; 53 | let path = tips.path.join(format!("{}", tip.filename())); 54 | 55 | println!("ADD TIP: {:?}", path); 56 | 57 | let path = tips.path.join(format!("{}", tip.filename())); 58 | 59 | let mut file = fs::File::create(path) 60 | .expect("Cannot create files in store"); 61 | 62 | tip.write(&mut file); 63 | } 64 | 65 | 66 | impl Tips { 67 | 68 | pub fn new(cfg: &config::Config) -> Tips { 69 | let path = &cfg.root.clone().join("tips"); 70 | 71 | if !path.exists() { 72 | fs::create_dir_all(path) 73 | .expect(&format!("Could not create {:?}", path)); 74 | } 75 | 76 | Tips { 77 | path: PathBuf::from(path) 78 | } 79 | } 80 | 81 | 82 | pub fn get_tips() -> Vec { 83 | vec![] 84 | } 85 | 86 | pub fn get_most_work_tip() -> Tip { 87 | unimplemented!() 88 | } 89 | 90 | pub fn remove_tip(tip: Tip) { 91 | unimplemented!() 92 | } 93 | 94 | pub fn get_all() { 95 | 96 | } 97 | } 98 | 99 | pub struct Tip { 100 | 101 | block_hash: Hash32Buf, 102 | 103 | difficulty: u64, 104 | height: u64 105 | 106 | 107 | // softfork rules 108 | 109 | } 110 | 111 | impl Tip { 112 | 113 | 114 | pub fn new(block_hash: Hash32Buf, difficulty: u64, height: u64) -> Tip { 115 | 116 | Tip { 117 | block_hash: block_hash, 118 | difficulty: difficulty, 119 | height: height 120 | } 121 | } 122 | 123 | fn write(&self, writer: &mut W) { 124 | 125 | write!(writer,"{},{}", self.difficulty, self.height).unwrap(); 126 | } 127 | 128 | fn filename(&self) -> String { 129 | 130 | self.block_hash 131 | .as_ref().0 132 | .iter() 133 | .rev() 134 | .map(|n| format!("{:02x}", n)) 135 | .collect::>() 136 | .concat() 137 | 138 | } 139 | } 140 | 141 | #[cfg(test)] 142 | mod tests { 143 | 144 | use super::*; 145 | use util::*; 146 | 147 | #[test] 148 | fn test_create_tip() { 149 | 150 | const HASH1: &'static str = "212300e77d897f2f059366ed03c8bf2757bc2b1dd30df15d34f6f1ee521e58e8"; 151 | const HASH2: &'static str = "4feec9316077e49b59bc23173303e13be9e9f5f9fa0660a58112a04a65a84ef1"; 152 | 153 | let tips = Tips::new(&test_cfg!()); 154 | 155 | add_tip(&tips, Hash32Buf::from_slice(&from_hex_rev(HASH1)), None, 1, 2); 156 | add_tip(&tips, Hash32Buf::from_slice(&from_hex_rev(HASH2)), None, 3, 4); 157 | 158 | } 159 | } -------------------------------------------------------------------------------- /serde_json/tests/stream.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Serde Developers 2 | // 3 | // Licensed under the Apache License, Version 2.0 or the MIT license 5 | // , at your 6 | // option. This file may not be copied, modified, or distributed 7 | // except according to those terms. 8 | 9 | #![cfg(not(feature = "preserve_order"))] 10 | 11 | extern crate serde; 12 | 13 | #[macro_use] 14 | extern crate serde_json; 15 | 16 | use serde_json::{Deserializer, Value}; 17 | 18 | macro_rules! test_stream { 19 | ($data:expr, $ty:ty, |$stream:ident| $test:block) => { 20 | { 21 | let de = Deserializer::from_str($data); 22 | let mut $stream = de.into_iter::<$ty>(); 23 | assert_eq!($stream.byte_offset(), 0); 24 | $test 25 | } 26 | { 27 | let de = Deserializer::from_slice($data.as_bytes()); 28 | let mut $stream = de.into_iter::<$ty>(); 29 | assert_eq!($stream.byte_offset(), 0); 30 | $test 31 | } 32 | { 33 | let mut bytes = $data.as_bytes(); 34 | let de = Deserializer::from_reader(&mut bytes); 35 | let mut $stream = de.into_iter::<$ty>(); 36 | assert_eq!($stream.byte_offset(), 0); 37 | $test 38 | } 39 | } 40 | } 41 | 42 | #[test] 43 | fn test_json_stream_newlines() { 44 | let data = "{\"x\":39} {\"x\":40}{\"x\":41}\n{\"x\":42}"; 45 | 46 | test_stream!( 47 | data, Value, |stream| { 48 | assert_eq!(stream.next().unwrap().unwrap()["x"], 39); 49 | assert_eq!(stream.byte_offset(), 8); 50 | 51 | assert_eq!(stream.next().unwrap().unwrap()["x"], 40); 52 | assert_eq!(stream.byte_offset(), 17); 53 | 54 | assert_eq!(stream.next().unwrap().unwrap()["x"], 41); 55 | assert_eq!(stream.byte_offset(), 25); 56 | 57 | assert_eq!(stream.next().unwrap().unwrap()["x"], 42); 58 | assert_eq!(stream.byte_offset(), 34); 59 | 60 | assert!(stream.next().is_none()); 61 | assert_eq!(stream.byte_offset(), 34); 62 | } 63 | ); 64 | } 65 | 66 | #[test] 67 | fn test_json_stream_trailing_whitespaces() { 68 | let data = "{\"x\":42} \t\n"; 69 | 70 | test_stream!( 71 | data, Value, |stream| { 72 | assert_eq!(stream.next().unwrap().unwrap()["x"], 42); 73 | assert_eq!(stream.byte_offset(), 8); 74 | 75 | assert!(stream.next().is_none()); 76 | assert_eq!(stream.byte_offset(), 11); 77 | } 78 | ); 79 | } 80 | 81 | #[test] 82 | fn test_json_stream_truncated() { 83 | let data = "{\"x\":40}\n{\"x\":"; 84 | 85 | test_stream!( 86 | data, Value, |stream| { 87 | assert_eq!(stream.next().unwrap().unwrap()["x"], 40); 88 | assert_eq!(stream.byte_offset(), 8); 89 | 90 | assert!(stream.next().unwrap().unwrap_err().is_eof()); 91 | assert_eq!(stream.byte_offset(), 9); 92 | } 93 | ); 94 | } 95 | 96 | #[test] 97 | fn test_json_stream_empty() { 98 | let data = ""; 99 | 100 | test_stream!( 101 | data, Value, |stream| { 102 | assert!(stream.next().is_none()); 103 | assert_eq!(stream.byte_offset(), 0); 104 | } 105 | ); 106 | } 107 | 108 | #[test] 109 | fn test_json_stream_primitive() { 110 | let data = "{} true"; 111 | 112 | test_stream!( 113 | data, Value, |stream| { 114 | assert_eq!(stream.next().unwrap().unwrap(), json!({})); 115 | assert_eq!(stream.byte_offset(), 2); 116 | 117 | let second = stream.next().unwrap().unwrap_err(); 118 | assert_eq!(second.to_string(), "expected `{` or `[` at line 1 column 4"); 119 | } 120 | ); 121 | } 122 | -------------------------------------------------------------------------------- /doc/bitcrust-db.md: -------------------------------------------------------------------------------- 1 | 2 | # Bitcrust-db 3 | 4 | - [Introduction](#introduction) 5 | - [Block content](#block-content) 6 | - [Spend tree instead of a UTXO-set](#spend-tree) 7 | - [Concurrent block validation](#concurrent-validation) 8 | 9 | ## Introduction 10 | 11 | Bitcoin-core uses a linearized model of the block tree. On-disk, only a main chain 12 | is stored to ensure that there is always one authorative UTXO set. 13 | 14 | Bitcrust uses a tree-structure and indexes spends instead of unspends. This has several key 15 | advantages in terms of performance, minimal memory requirement, simplicity and most importantly, concurrency. 16 | 17 | The first results are very positive and show that this approach addresses Core's major bottlenecks in 18 | block verification. 19 | 20 | ## Block content 21 | 22 | Transactions are stored on disk after they have been verified even when not yet in a block. 23 | This means they are only written once. 24 | Unconfirmed transactions are not necessarily kept in memory, as 25 | they only pollute precious RAM space. After the scripts 26 | are verified and transactions are relayed, the transaction contents is only rarely needed. 27 | 28 | Blockheaders are stored in the same data-files (to ensure rough ordering), and both transactions 29 | and blocks are referenced by 48-bit file-pointers. 30 | 31 | For more details, check the [store](../src/store/) documentation 32 | 33 | 34 | ## Spend tree 35 | 36 | When transactions are stored, only their scripts are validated. When blocks come in, 37 | we need to verify for each input of each transaction whether the referenced output exists and is unspend before 38 | this one. Instead of a UTXO-set we use the spend-tree. 39 | 40 | This is a table (stored in a flatfileset) consisting of four types of records: block-start, block-end, transactions and spends. 41 | 42 | Here we see a spend-tree with two blocks, where the third transaction has one input referencing the output of the first transaction (purple). 43 | 44 | 45 | ![Spend tree example 1](https://cdn.rawgit.com/tomasvdw/bitcrust/master/doc/spend-tree1.svg "Spend-tree example") 46 | 47 | If another block (2b) comes with the same block 1 as parent this can be simply appended with the proper pointer: 48 | 49 | ![Spend tree example 2](https://cdn.rawgit.com/tomasvdw/bitcrust/master/doc/spend-tree2.svg "Spend-tree example 2") 50 | 51 | The rule for verification is simple: A spend (purple) record can only be added if, when browsing back through the 52 | records, we will find the corresponding transaction before we find the same spend. This ensures both the existence 53 | of the referenced transaction, and it being unspend. 54 | 55 | Obviously, with hundreds of millions transactions, simply scanning won't do. 56 | This is where the spent-index comes in to play. This is a very compact bit-index of spends that 57 | lags behind the tips and serves as a broom wagon. When scanning the spend-tree reaches the broom-wagon, 58 | the order can be verifies with two simple lookups. 59 | 60 | 61 | ## Concurrent validation 62 | 63 | One major cause for sleepless nights for nodes and miners is the idea of a _toxic block_ or transaction. 64 | The flexibility of bitcoin allows one to create blocks that will cause a huge amount of time and effort to be processed and can thereby choke or even crash other 65 | nodes and miners, especially smaller ones. A simple example being a non-segwit transaction with a huge amount of inputs which abuses quadratic hashing. 66 | 67 | By its architecture, bitcrust is insensitive for such malice; blocks and transaction can be processed fully in parallel: 68 | 69 | ![Parallel validation](https://cdn.rawgit.com/tomasvdw/bitcrust/master/doc/parallel-validation.svg "Parellel validation") 70 | 71 | The long-lasting validation of block A does not at any point 72 | block the validation of block B, C and D. 73 | 74 | The actual orphaning and breaking of the connection (as well as deprioritizing) 75 | can be implemented using the same per peer cost/benefit analysis as other DOS protection. 76 | -------------------------------------------------------------------------------- /store/src/db/mod.rs: -------------------------------------------------------------------------------- 1 | 2 | use std::fs; 3 | use std::path::Path; 4 | use hashstore::*; 5 | use hash::*; 6 | use record::Record; 7 | use serde_network; 8 | 9 | pub mod db_transaction; 10 | pub mod db_header; 11 | 12 | mod db_block; 13 | 14 | const ROOT_BITS_TX : u8 = 26; 15 | const ROOT_BITS_SIG: u8 = 0; 16 | 17 | const ROOT_BITS_HDR: u8 = 20; 18 | const ROOT_BITS_BLK: u8 = 0; 19 | 20 | pub const EXTREMUM_BEST_HEADER: usize = 1; 21 | pub const EXTREMUM_BEST_BLOCK: usize = 1; 22 | 23 | /// All DbErrors are unrecoverable data corruption errors 24 | #[derive(Debug)] 25 | pub enum DbError { 26 | HashStoreError(HashStoreError), 27 | EndOfBufferError, 28 | ParentNotFound, 29 | HeaderFileCorrupted 30 | } 31 | 32 | 33 | pub type DbResult = Result; 34 | 35 | impl From for DbError { 36 | fn from(err: HashStoreError) -> DbError { 37 | DbError::HashStoreError(err) 38 | } 39 | } 40 | impl From for DbError { 41 | fn from(_: serde_network::Error) -> DbError { 42 | DbError::EndOfBufferError 43 | } 44 | } 45 | 46 | 47 | pub struct Db { 48 | tx: HashStore, 49 | sig: HashStore, 50 | 51 | hdr: HashStore, 52 | blk: HashStore, 53 | } 54 | 55 | // useful for testing 56 | pub fn init_empty>(db_path: P) -> Result { 57 | let db_path = db_path.as_ref(); 58 | let exists = db_path.exists(); 59 | if exists { 60 | // temporary useful for testing 61 | fs::remove_dir_all(db_path).unwrap(); 62 | } 63 | init(db_path) 64 | } 65 | 66 | 67 | pub fn init>(db_path: P) -> Result { 68 | let db_path = db_path.as_ref(); 69 | let exists = db_path.exists(); 70 | let mut db = Db { 71 | tx : HashStore::new(Path::join(db_path, "tx"), ROOT_BITS_TX)?, 72 | sig: HashStore::new(Path::join(db_path, "sig"), ROOT_BITS_SIG)?, 73 | hdr: HashStore::new(Path::join(db_path, "hdr"), ROOT_BITS_HDR)?, 74 | blk: HashStore::new(Path::join(db_path, "blk"), ROOT_BITS_BLK)?, 75 | }; 76 | 77 | if !exists { 78 | add_genesis(&mut db)?; 79 | } 80 | Ok(db) 81 | } 82 | 83 | const GENESIS_BLOCK: &'static str = "\ 84 | 0100000000000000000000000000000000000000000000000000000000000000\ 85 | 000000003BA3EDFD7A7B12B27AC72C3E67768F617FC81BC3888A51323A9FB8AA\ 86 | 4B1E5E4A29AB5F49FFFF001D1DAC2B7C01010000000100000000000000000000\ 87 | 00000000000000000000000000000000000000000000FFFFFFFF4D04FFFF001D\ 88 | 0104455468652054696D65732030332F4A616E2F32303039204368616E63656C\ 89 | 6C6F72206F6E206272696E6B206F66207365636F6E64206261696C6F75742066\ 90 | 6F722062616E6B73FFFFFFFF0100F2052A01000000434104678AFDB0FE554827\ 91 | 1967F1A67130B7105CD6A828E03909A67962E0EA1F61DEB649F6BC3F4CEF38C4\ 92 | F35504E51EC112DE5C384DF7BA0B8D578A4C702B6BF11D5FAC00000000"; 93 | 94 | /// Add genesis tx and block to the db 95 | fn add_genesis(db: &mut Db) -> Result<(), DbError> { 96 | 97 | let genesis = ::util::from_hex(GENESIS_BLOCK); 98 | 99 | let block_hash = double_sha256(&genesis[0..80]); 100 | let tx_hash = double_sha256(&genesis[81..]); 101 | let hdr = ::Header::new(&genesis). 102 | expect("Hardcoded genesis is invalid"); 103 | 104 | // tx count = 1 105 | assert_eq!(genesis[80], 1, "Hardcoded genesis is invalid"); 106 | 107 | let tx= ::Transaction::decode(&genesis[81..]). 108 | expect("Hardcoded genesis is invalid"); 109 | 110 | let tx_ptr = db_transaction::write_transaction(db, &tx_hash, &tx, vec![Record::new_coinbase()])?; 111 | 112 | assert_eq!(&tx_hash[..], 113 | &::util::from_hex_rev("4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b")[..]); 114 | 115 | // Create our blk-records 116 | let records = vec![ 117 | Record::new_start_of_block(1), Record::new_transaction(tx_ptr)]; 118 | 119 | let blk_ptr = db.blk.set_value(Record::to_bytes(&records))?; 120 | 121 | let _ = db_header::write_genesis(db, &block_hash, hdr, blk_ptr)?; 122 | 123 | Ok(()) 124 | } 125 | 126 | -------------------------------------------------------------------------------- /src/merkle_tree.rs: -------------------------------------------------------------------------------- 1 | //! Merkle tree implementation 2 | //! 3 | 4 | // minimum number of hashes to use parallel hashing 5 | const PARALLEL_HASHING_THRESHOLD: usize = 60; 6 | 7 | use rayon::prelude::*; 8 | use hash::*; 9 | 10 | /// This halves the merkle tree leaves, taking it one level up 11 | /// 12 | /// Calls itself recursively until one is left 13 | fn shrink_merkle_tree(hashes: Vec) -> Vec { 14 | 15 | if hashes.len() == 1 { 16 | return hashes; 17 | } 18 | 19 | // the result is half the size rounded up 20 | let count = (hashes.len() + 1 ) / 2; 21 | 22 | // closure to hash n*2 and n*2+1 23 | let reduce = |n| { 24 | let ref first: Hash32Buf = hashes[n * 2]; 25 | 26 | // we double the first one if we have an odd number of hashes 27 | // in this layer 28 | let ref second = hashes.get(n * 2 + 1).unwrap_or(first); 29 | 30 | Hash32Buf::double_sha256_from_pair( 31 | first.as_ref(), 32 | second.as_ref() 33 | ) 34 | }; 35 | 36 | let result = if count > PARALLEL_HASHING_THRESHOLD 37 | { (0..count).into_par_iter().map(reduce).collect() } 38 | else 39 | { (0..count).into_iter().map(reduce).collect() }; 40 | 41 | shrink_merkle_tree(result) 42 | } 43 | 44 | /// Calculates the merkle root for the given set of hashes 45 | pub fn get_merkle_root(hashes: Vec) -> Hash32Buf { 46 | 47 | shrink_merkle_tree(hashes)[0] 48 | } 49 | 50 | 51 | 52 | 53 | #[cfg(test)] 54 | mod tests { 55 | 56 | use super::*; 57 | use util::*; 58 | 59 | 60 | 61 | #[test] 62 | fn test_merkle1() { 63 | 64 | const HASH1: &'static str = "212300e77d897f2f059366ed03c8bf2757bc2b1dd30df15d34f6f1ee521e58e8"; 65 | const HASH2: &'static str = "4feec9316077e49b59bc23173303e13be9e9f5f9fa0660a58112a04a65a84ef1"; 66 | const EXP_MERKLE: &'static str = "03b750bf691caf40b7e33d8e15f64dd16becf944b39a82710d6d257159361b93"; 67 | 68 | let hash1 = Hash32Buf::from_slice(&from_hex_rev(HASH1)); 69 | let hash2 = Hash32Buf::from_slice(&from_hex_rev(HASH2)); 70 | let exp_merkle = Hash32Buf::from_slice(&from_hex_rev(EXP_MERKLE)); 71 | 72 | let mut merkle = Vec::new(); 73 | merkle.push(hash1); 74 | merkle.push(hash2); 75 | 76 | 77 | let merkle_root = get_merkle_root(merkle); 78 | 79 | assert_eq!(exp_merkle, merkle_root); 80 | 81 | 82 | } 83 | 84 | #[test] 85 | fn test_merkle2() { 86 | 87 | const HASH_SET : [&'static str;12] = [ 88 | "30803bc3fefa999bf187cda4fff3647a78db6b957fcf5a579270c0535ec1601e", 89 | "101d83a3e4739640fbb6279883478bb6a2814e6fcd58322f0b1d3bf03983268a", 90 | "d646c47be5581891fc8d098e0db6e288efa22962d175f6d8c77913c2f898c0aa", 91 | "61419189bed89e85689442ee144e0aafc8b1ae6ed813e15f241b0750b97886ec", 92 | "09e5c37fe017605a15ae1fb139b2c186c76013c8cb78e6524261b29aed0b0424", 93 | "8226f6778f0f900ed38d9ca7314cfc55b28cc9c809e99570e3499ffeaa57759f", 94 | "0568ba35848086f4b3352fb468338ff727411fbf0e49fbe46eaf926929d205fe", 95 | "4e9dc6455ee50181a12e43b46de1bc72b201e627294dea7abe2f928b3e86bdb3", 96 | "6780b0e7801554d92d409f30b19328948ee3bcfd53764be30db7f1146fc12890", 97 | "dbcf135133afb7c3a256f87d09ae15994c5358bffd3e3993b56efb43742dce1e", 98 | "29454d128096bf66ebefe3f80a38edba4befac78c0eebfd351b7099f01933e2f", 99 | "c8cb5078d0faae9dc7bfcd150207c15d331e12161ec0e7c66c744d1201e08f3b"]; 100 | 101 | const HASH_SET_MERKLE: &'static str = 102 | "1ba9abf54ae4cb022f53e767669931bacdd42c783fe063c5738ca49d29f1fbe3"; 103 | 104 | 105 | let exp_merkle = Hash32Buf::from_slice(&from_hex_rev(HASH_SET_MERKLE)); 106 | 107 | let mut merkle = Vec::new(); 108 | for tx in HASH_SET.iter() { 109 | let txh = Hash32Buf::from_slice(&from_hex_rev(tx)); 110 | merkle.push(txh); 111 | } 112 | 113 | let merkle_root = get_merkle_root(merkle); 114 | 115 | assert_eq!(exp_merkle, merkle_root); 116 | 117 | 118 | } 119 | } -------------------------------------------------------------------------------- /tests/test_file1.rs: -------------------------------------------------------------------------------- 1 | 2 | extern crate bitcrust_lib; 3 | 4 | extern crate byteorder; 5 | 6 | 7 | use std::io::BufReader; 8 | use std::fs::File; 9 | 10 | mod blk_file; 11 | 12 | use std::time::{Instant}; 13 | extern crate rayon; 14 | 15 | 16 | #[test] 17 | #[ignore] 18 | fn load_file1() { 19 | 20 | let mut store = bitcrust_lib::init(); 21 | 22 | let fileno = 0; 23 | let name = format!("./data/blk{:05}.dat", fileno); 24 | println!("Processing {}", name); 25 | let f = File::open(name).unwrap(); 26 | let mut rdr = BufReader::new(f); 27 | 28 | let mut blocks = 0; 29 | loop { 30 | let blk = blk_file::read_block(&mut rdr).unwrap(); 31 | 32 | if blk.is_none() { 33 | break; 34 | } 35 | 36 | bitcrust_lib::add_block(&mut store, &blk.unwrap()); 37 | 38 | blocks += 1; 39 | 40 | 41 | 42 | 43 | if blocks == 2 { 44 | break; 45 | } 46 | 47 | } 48 | 49 | 50 | 51 | 52 | } 53 | 54 | 55 | 56 | 57 | 58 | #[test] 59 | #[ignore] 60 | fn load_file_large() { 61 | let start = Instant::now(); 62 | const BLOCK_COUNT: u64 = 450000; 63 | 64 | let mut blocks = 0; 65 | 66 | let mut store = bitcrust_lib::init(); 67 | 68 | store.initial_sync = true; 69 | for fileno in 0..999 { 70 | let name = format!("./core-blocks/blk{:05}.dat", fileno); 71 | println!("Processing {}", name); 72 | let f = File::open(name).unwrap(); 73 | let mut rdr = BufReader::new(f); 74 | 75 | loop { 76 | let blk = blk_file::read_block(&mut rdr).unwrap(); 77 | 78 | if blk.is_none() { 79 | break; 80 | } 81 | 82 | bitcrust_lib::add_block(&mut store, &blk.unwrap()); 83 | 84 | 85 | blocks += 1; 86 | 87 | if blocks % 100 == 0 { 88 | println!("Processed {} blocks in {} sec at {}/s", blocks, start.elapsed().as_secs(), 89 | blocks / (start.elapsed().as_secs() + 1)); 90 | } 91 | 92 | if blocks >= BLOCK_COUNT { 93 | break; 94 | } 95 | } 96 | 97 | if blocks >= BLOCK_COUNT { 98 | break; 99 | } 100 | } 101 | 102 | println!("DONE: Processed {} blocks in {} sec at {}/s", blocks, start.elapsed().as_secs(), 103 | blocks / (start.elapsed().as_secs() + 1)); 104 | } 105 | 106 | use std::thread; 107 | #[test] 108 | #[ignore] 109 | fn load_large_concurrent() { 110 | const THREADS: usize = 5; 111 | const BLOCK_COUNT: u64 = 400000; 112 | 113 | 114 | 115 | let handles: Vec<_> = (0..THREADS).map(|n| { 116 | //let mut store = store_b.clone(); 117 | thread::spawn(move || { 118 | let mut store = bitcrust_lib::init_prs(); 119 | 120 | let start = Instant::now(); 121 | let mut blocks = 0; 122 | for fileno_b in 0..999 { 123 | let fileno = fileno_b * THREADS + n; 124 | let name = format!("./data/blk{:05}.dat", fileno); 125 | println!("Processing {}", name); 126 | let f = File::open(name).unwrap(); 127 | let mut rdr = BufReader::new(f); 128 | 129 | loop { 130 | let blk = blk_file::read_block(&mut rdr).unwrap(); 131 | 132 | if blk.is_none() { 133 | break; 134 | } 135 | 136 | bitcrust_lib::add_block(&mut store, &blk.unwrap()); 137 | 138 | 139 | blocks += 1; 140 | 141 | if blocks % 100 == 0 { 142 | println!("Processed {} blocks in {} sec at {}/s", blocks, start.elapsed().as_secs(), 143 | blocks / (start.elapsed().as_secs() + 1)); 144 | } 145 | 146 | if blocks >= BLOCK_COUNT { 147 | break; 148 | } 149 | } 150 | 151 | if blocks >= BLOCK_COUNT { 152 | break; 153 | } 154 | } 155 | }) 156 | }).collect(); 157 | 158 | for h in handles { 159 | h.join().unwrap(); 160 | } 161 | } -------------------------------------------------------------------------------- /hashstore/tests/main.rs: -------------------------------------------------------------------------------- 1 | 2 | extern crate hashstore; 3 | extern crate rand; 4 | 5 | 6 | use hashstore::*; 7 | use std::time::{Instant}; 8 | use std::collections::HashMap; 9 | 10 | use self::rand::Rng; 11 | 12 | fn random_value(rng: &mut R) -> Vec { 13 | 14 | let size = if rng.next_u32() & 100 == 1 { 15 | 100 + (rng.next_u32() % 200_000) 16 | } 17 | else { 18 | 100 + (rng.next_u32() % 600) 19 | }; 20 | let mut value = vec![0; size as usize]; 21 | rng.fill_bytes(&mut value); 22 | value 23 | } 24 | 25 | fn random_key(rng: &mut R) -> [u8; 32] { 26 | let mut key = [0; 32]; 27 | rng.fill_bytes(&mut key); 28 | key 29 | } 30 | 31 | fn ms(start: Instant) -> u64 { 32 | let d = Instant::now() - start; 33 | (d.as_secs() * 1000) as u64 + (d.subsec_nanos() / 1_000_000) as u64 34 | } 35 | 36 | 37 | 38 | #[test] 39 | fn test_exists() { 40 | 41 | // we use a root hashtable of size one to test search depth 42 | let mut hs = HashStore::new_empty("./testdb/exists", 0).unwrap(); 43 | 44 | hs.set(&[1;32], &[2;8], 10).unwrap(); 45 | hs.set(&[3;32], &[4;8], 20).unwrap(); 46 | hs.set(&[5;32], &[6;8], 30).unwrap(); 47 | 48 | assert!(hs.exists(&[1;32], SearchDepth::FullSearch).unwrap().is_some()); 49 | 50 | // after(20) still reaches 1, as we only stop *after* and element has t<20 51 | assert!(hs.exists(&[1;32], SearchDepth::SearchAfter(20)).unwrap().is_some()); 52 | assert!(hs.exists(&[1;32], SearchDepth::SearchAfter(21)).unwrap().is_none()); 53 | 54 | assert!(hs.exists(&[2;32], SearchDepth::FullSearch).unwrap().is_none()); 55 | assert!(hs.exists(&[2;32], SearchDepth::SearchAfter(25)).unwrap().is_none()); 56 | 57 | assert!(hs.exists(&[5;32], SearchDepth::FullSearch).unwrap().is_some()); 58 | assert!(hs.exists(&[5;32], SearchDepth::SearchAfter(45)).unwrap().is_some()); 59 | 60 | } 61 | 62 | #[test] 63 | #[ignore] 64 | fn test_big() { 65 | let mut rng = rand::weak_rng(); 66 | let mut hs = HashStore::new_empty("./testdb/big", 26).unwrap(); 67 | 68 | let mut block1 = HashMap::new(); 69 | let mut blockend = HashMap::new(); 70 | 71 | let block_count = 20000; 72 | // load block 1 73 | println!("Block 1"); 74 | for _ in 0..100000 { 75 | let k1 = random_key(&mut rng); 76 | let v1 = random_value(&mut rng); 77 | block1.insert(k1, v1.clone()); 78 | hs.set(&k1, &v1, 1).unwrap(); 79 | } 80 | let b1 = block1.clone(); 81 | let l = block1.len(); 82 | let tm = Instant::now(); 83 | for (k, _) in b1.into_iter() { 84 | 85 | let _ = hs.get(&k, SearchDepth::FullSearch).unwrap().unwrap(); 86 | 87 | } 88 | println!("block 1 {} lookups in {}ms", l, ms(tm)); 89 | 90 | // load 20_000 91 | println!("Next {}", block_count); 92 | let tm = Instant::now(); 93 | for block in 2..(block_count+2) { 94 | for _ in 0..2000 { 95 | let k = random_key(&mut rng); 96 | let v = random_value(&mut rng); 97 | hs.set(&k, &v, block).unwrap(); 98 | } 99 | } 100 | 101 | println!("{} blocks in {}ms", block_count, ms(tm)); 102 | let b1 = block1.clone(); 103 | let l = block1.len(); 104 | let tm = Instant::now(); 105 | for (k, _) in b1.into_iter() { 106 | 107 | let _ = hs.get(&k, SearchDepth::FullSearch).unwrap().unwrap(); 108 | 109 | } 110 | println!("block 1 {} lookups in {}ms", l, ms(tm)); 111 | 112 | for _ in 0..100000 { 113 | let k1 = random_key(&mut rng); 114 | let v1 = random_value(&mut rng); 115 | blockend.insert(k1, v1.clone()); 116 | hs.set(&k1, &v1, 1).unwrap(); 117 | } 118 | println!("Block-end loaded"); 119 | 120 | let b1 = block1.clone(); 121 | let l = block1.len(); 122 | let tm = Instant::now(); 123 | for (k, _) in b1.into_iter() { 124 | 125 | let _ = hs.get(&k, SearchDepth::FullSearch).unwrap().unwrap(); 126 | 127 | } 128 | println!("block 1 {} lookups in {}ms", l, ms(tm)); 129 | let tm = Instant::now(); 130 | let l = blockend.len(); 131 | for (k, _) in blockend.into_iter() { 132 | 133 | let _ = hs.get(&k, SearchDepth::FullSearch).unwrap().unwrap(); 134 | 135 | } 136 | println!("block end {} lookups in {}ms", l, ms(tm)); 137 | } 138 | -------------------------------------------------------------------------------- /src/metrics.rs: -------------------------------------------------------------------------------- 1 | //! Small metrics library 2 | //! Probably obsolete; we now use structured logging which seems sufficient 3 | //! 4 | //! Besides; the lib is now hyper-fast anyways :) 5 | //! 6 | 7 | use std::collections::HashMap; 8 | use std::time::{Instant,Duration}; 9 | 10 | 11 | use std::sync::Arc; 12 | use std::cell::RefCell; 13 | 14 | /// Represents a single named countable value 15 | struct Metric { 16 | time: Duration, 17 | count: usize, 18 | ticker: usize 19 | } 20 | 21 | /// A handle to a timed key; this adds itself to the referenced metric 22 | /// when going out-of-scope 23 | pub struct RunningMetric<'a>{ 24 | metric: &'a Metrics, 25 | name: &'static str, 26 | started: Instant, 27 | ticker: usize, 28 | is_stopped: bool, 29 | } 30 | 31 | pub struct Metrics { 32 | 33 | metrics: Arc>> 34 | 35 | } 36 | 37 | 38 | 39 | impl Metrics { 40 | pub fn new() -> Metrics { 41 | Metrics { 42 | metrics: Arc::new(RefCell::new(HashMap::new())) 43 | } 44 | } 45 | 46 | /// Start measuring time at the given tag name. 47 | /// Stops when the result go out of scope 48 | /// 49 | /// Hence the results must be saved in a temporary variable. Use a _ prefix. 50 | /// 51 | /// # Examples 52 | /// 53 | /// ``` 54 | /// use bitcrust_lib::metrics; 55 | /// 56 | /// let metrics = metrics::Metrics::new(); 57 | /// let _m = metrics.start("mymetric"); 58 | /// ``` 59 | /// 60 | pub fn start(&self, name: &'static str) -> RunningMetric { 61 | 62 | RunningMetric { 63 | started: Instant::now(), 64 | metric: self, 65 | name: name, 66 | ticker: 0, 67 | is_stopped: false 68 | 69 | } 70 | } 71 | } 72 | 73 | 74 | impl Drop for Metrics { 75 | 76 | /// We dump everything on exit 77 | fn drop(&mut self) { 78 | 79 | let metrics = self.metrics.borrow(); 80 | println!("METRICS:"); 81 | 82 | for k in metrics.keys() { 83 | 84 | let v = metrics.get(k).unwrap(); 85 | 86 | 87 | if v.count == 0 { 88 | continue; 89 | } 90 | 91 | let micros: usize = v.time.as_secs() as usize * 1000_000 + v.time.subsec_nanos() as usize / 1_000 as usize; 92 | println!("{}", k); 93 | 94 | 95 | println!(" count={}", v.count); 96 | println!(" durat={}", micros); 97 | println!(" avg={}", micros / v.count); 98 | println!(" tickr={}", v.ticker); 99 | println!(" avg={}", v.ticker / v.count); 100 | 101 | 102 | } 103 | } 104 | } 105 | 106 | impl<'a> RunningMetric<'a> { 107 | 108 | // Close this metric and add the numbers to the main metrics table 109 | fn stop(&mut self) { 110 | 111 | if self.is_stopped { 112 | return; 113 | } 114 | 115 | let mut metrics = self.metric.metrics.borrow_mut(); 116 | 117 | let metric = metrics.entry(self.name).or_insert( 118 | Metric { 119 | time: Duration::new(0, 0), 120 | count: 0, 121 | ticker: 0 122 | } 123 | ); 124 | metric.time += self.started.elapsed(); 125 | metric.count += 1; 126 | metric.ticker += self.ticker; 127 | 128 | self.is_stopped = true; 129 | } 130 | 131 | pub fn set_ticker(&mut self, ticker: usize) { 132 | self.ticker = ticker; 133 | } 134 | } 135 | 136 | impl<'a> Drop for RunningMetric<'a> { 137 | 138 | // This is where we add the result of this running metric to our hashmap 139 | fn drop(&mut self) { 140 | self.stop(); 141 | } 142 | } 143 | 144 | 145 | #[cfg(test)] 146 | mod tests { 147 | 148 | use super::*; 149 | use std::thread; 150 | 151 | use std::time; 152 | #[test] 153 | fn test_metric() { 154 | let m = Metrics::new(); 155 | 156 | { 157 | let _q = m.start("test"); 158 | thread::sleep(time::Duration::from_millis(5)); 159 | } 160 | assert!(m.metrics.borrow().get("test").unwrap().count == 1); 161 | assert!(m.metrics.borrow().get("test").unwrap().time.subsec_nanos() >= 5_000_000); 162 | } 163 | } -------------------------------------------------------------------------------- /site/features.html: -------------------------------------------------------------------------------- 1 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 33 | 34 | 35 | 36 | 39 | 40 | BITCRUST 41 | 42 | 43 | 44 |
45 | 46 |
47 | 48 | 49 | 50 |
51 | 52 | 56 |
57 |
58 | 59 |
60 |
61 |
62 | 63 | 64 | 65 | 66 | 67 |

Modular

68 |

Bitcrust is build using modular approach that makes it easy to write different components for different use cases. 69 | The wallet, the miner, the full and SPV node all use a the same storage back-end but can use different network behaviour optimized for their case.

70 |

Parallel verification

71 |

Blocks can be verified in parallel. If block processing takes a long time because it is made excessively heavy, for example due to quadratic hashing, Bitcrust can accept and verify an incoming lighter block in parallel and use that instead. This way the cost of creating heavy blocks is primarily imposed on the sender instead of the receiver. 72 |

Minimized peak load

73 |

The storage engine is optimized from ground up for xthin/compact-block 74 | synchronization. This ensures that when the majority of transactions are 75 | already synced, incoming blocks can be verified at minimal resources using 76 | order-validation only.

77 |

Reverse syncing

78 |

Due to the low cost of order validation, block connections are extremely fast. This makes it possible to reverse the synchronization of the chain without much additional overhead. That way the Node could become partially availble very quickly.

79 |

Efficient SPV node

80 |

The storage engine is written to handle both full node and SPV node use cases. SPV nodes can use the Merkle tree to run a node with very low resource requirements with only slightly reduced security.

81 |

Extensible indices

82 |

By using a concurrently accessable data store, Bitcrust allows external index engines to be added without much overhead. One such might be a map-reduce engine tuned to temporal blockchain data. This would allow for efficient custom queries for blockchain analysis and monitoring.

83 |
84 |
85 | 86 | 87 |
88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | -------------------------------------------------------------------------------- /tool/gen_compare.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | # Tool that generates comparison data from the core log and the bitcrust log 4 | 5 | import tailer 6 | import pprint 7 | import re 8 | import pystache 9 | 10 | from dateutil.parser import parse 11 | 12 | CORE_LOG = "/home/tomas/.bitcoin/debug.log" 13 | BC_LOG = "/home/tomas/bitcrust/cmp1" 14 | 15 | RE_CORE_CONNECT = re.compile(r'^(?P