├── clean.sh ├── .dir-locals.el ├── rebar ├── .editorconfig ├── .gitignore ├── src ├── trie.app.src ├── ids.erl ├── trie_app.erl ├── delete.erl ├── cfg.erl ├── prune.erl ├── get.erl ├── trie_sup.erl ├── verify.erl ├── leaf.erl ├── stem.erl ├── trie.erl ├── store.erl └── test_trie.erl ├── docs ├── hashdepth.md └── todo.md ├── rebar.config ├── start.sh ├── README.md ├── .travis.yml ├── test ├── trie_test_utils.erl ├── prop_trie_arbitrary_put_and_delete.erl ├── prop_trie_root_hash_independent_from_order_of_operations.erl ├── prop_trie_history_readable_at_any_height.erl └── trie_tests.erl ├── scripts └── load_test │ ├── trie.config.template │ └── basho_bench_driver_trie.erl ├── unused ├── ram_store.erl └── ram_trie.erl ├── ci └── garbage.erl /clean.sh: -------------------------------------------------------------------------------- 1 | rm data/*.db 2 | -------------------------------------------------------------------------------- /.dir-locals.el: -------------------------------------------------------------------------------- 1 | ((erlang-mode . ((indent-tabs-mode . t)))) 2 | -------------------------------------------------------------------------------- /rebar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zack-bitcoin/MerkleTrie/HEAD/rebar -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*.{erl,hrl}] 4 | indent_size = 4 5 | tab_width = 8 6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | deps/ 2 | /ebin/ 3 | /rel/ 4 | *.db 5 | *~ 6 | .#* 7 | \#* 8 | rebar_source 9 | .rebar 10 | erl_crash.dump 11 | keys_backup -------------------------------------------------------------------------------- /src/trie.app.src: -------------------------------------------------------------------------------- 1 | {application, trie, 2 | [ 3 | {description, ""}, 4 | {vsn, "1"}, 5 | {registered, []}, 6 | {applications, [ 7 | kernel, 8 | stdlib 9 | ]}, 10 | {mod, { trie_app, []}}, 11 | {env, []} 12 | ]}. 13 | -------------------------------------------------------------------------------- /docs/hashdepth.md: -------------------------------------------------------------------------------- 1 | bitcoin hash rate is 2 million terahashes per second. 2 | 2*10^6 * 10^12 3 | 4 | 3.15*10^8 seconds in a year. 5 | 6 | so 6.3*10^26 hashes per year 7 | 8 | is 89 bits of collision per year. 9 | 10 | So to get 96 bits, we would need 32 years of the entire bitcoin network's hashpower. -------------------------------------------------------------------------------- /src/ids.erl: -------------------------------------------------------------------------------- 1 | -module(ids). 2 | -export([main_id/1, leaf/1, main/1, stem/1, bits/1, ram/1]). 3 | 4 | 5 | leaf(CFG) -> list_to_atom(atom_to_list(cfg:id(CFG)) ++ "_leaf"). 6 | stem(CFG) -> list_to_atom(atom_to_list(cfg:id(CFG)) ++ "_stem"). 7 | main(CFG) -> main_id(cfg:id(CFG)). 8 | bits(CFG) -> list_to_atom(atom_to_list(cfg:id(CFG)) ++ "_bits"). 9 | main_id(ID) -> list_to_atom(atom_to_list(ID) ++ "_main"). 10 | ram(CFG) -> list_to_atom(atom_to_list(cfg:id(CFG)) ++ "_ram"). 11 | -------------------------------------------------------------------------------- /rebar.config: -------------------------------------------------------------------------------- 1 | %{erl_opts, [native, {hipe, [verbose]}, warnings_as_errors, debug_info]}. 2 | 3 | {deps, [ 4 | % {pink_hash, "1", {git, "https://github.com/BumblebeeBat/pink_crypto", {branch, "partial_hash_problem"}}}, 5 | {pink_hash, "1", {git, "https://github.com/BumblebeeBat/pink_crypto", {tag, "master"}}}, 6 | {dump, "1", {git, "https://github.com/BumblebeeBat/dump", {tag, "master"}}} 7 | ]}. 8 | 9 | %{plugins, [rebar3_proper]}. 10 | {profiles, [{test, [{deps, [{proper, "1.2.0"}]}]}]}. 11 | -------------------------------------------------------------------------------- /src/trie_app.erl: -------------------------------------------------------------------------------- 1 | -module(trie_app). 2 | 3 | -behaviour(application). 4 | 5 | %% Application callbacks 6 | -export([start/2, stop/1]). 7 | 8 | %start(_StartType, _StartArgs) -> 9 | start(normal, []) -> 10 | Size = 2, 11 | %Max = 20000000000, 12 | ID = trie01, 13 | KeyLength = 5,%in bytes 14 | Amount = 1000000, 15 | Mode = ram, 16 | Meta = 2, 17 | HashSize = 32, 18 | trie_sup:start_link(KeyLength, Size, ID, Amount, Meta, HashSize, Mode, ""). 19 | 20 | stop(_State) -> 21 | ok. 22 | -------------------------------------------------------------------------------- /src/delete.erl: -------------------------------------------------------------------------------- 1 | -module(delete). 2 | -export([delete/3]). 3 | 4 | -spec delete(leaf:key(), stem:stem_p(), cfg:cfg()) -> stem:stem_p(). 5 | delete(ID, Root, CFG) -> 6 | Path = leaf:path_maker(ID, CFG), 7 | case store:get_branch(Path, 0, Root, [], CFG) of 8 | {_, _, _} -> %already no leaf with the specified path 9 | Root; 10 | Branch -> 11 | X = cfg:hash_size(CFG)*8, 12 | EmptyHash = <<0:X>>, 13 | {_, NewRoot, _} = store:store_branch(Branch, Path, 0, 0, EmptyHash, CFG), 14 | NewRoot 15 | end. 16 | 17 | -------------------------------------------------------------------------------- /start.sh: -------------------------------------------------------------------------------- 1 | #if you want to use a different port, then start like this: 2 | # sh start 3666 3 | 4 | #sh clean.sh #this deletes the database so every time we re-start, we have 0 blocks again. only needed during testing. 5 | ./rebar get-deps 6 | sh clean.sh #this deletes the database so every time we re-start, we have 0 blocks again. only needed during testing. 7 | ./rebar compile #this line checks if any modules were modified, and recompiles them if they were. only needed during testing. 8 | mkdir data 9 | erl -pa ebin deps/*/ebin/ -eval "application:ensure_all_started(trie)" 10 | 11 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Merkle Trie 2 | 3 | This merkel tree database is sparse. That means you can make a proof to prove the non-existance of some data. 4 | 5 | This tree can be configured to use either ram or hard drive for storage. 6 | 7 | The root hash of a trie is deterministicly derived from the contents, order of insertion/deletion doesn't matter. 8 | 9 | This is an order 16 radix tree. every node of the tree has 16 children. 10 | 11 | 12 | Install process: 13 | 14 | First you need erlang installed. 15 | 16 | Now to run the software simply: ```sh start.sh``` 17 | 18 | For examples on how to use it, look at [this](src/test_trie.erl) 19 | 20 | here is an example of a project that uses this MerkleTrie: https://github.com/zack-bitcoin/amoveo 21 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | dist: trusty 2 | sudo: required 3 | language: erlang 4 | otp_release: 5 | - 19.3.6.1 6 | - 18.3 7 | before_install: 8 | - if test smokeloadtest = "${JOB:?}"; then sudo apt-get -qq update && sudo apt-get install -y r-cran-plyr r-cran-ggplot2; fi 9 | - ./ci before_install "${JOB:?}" "${TRAVIS_BUILD_DIR:?}" 10 | install: 11 | - ./ci install "${JOB:?}" "${TRAVIS_BUILD_DIR:?}" 12 | before_script: 13 | - ./ci before_script "${JOB:?}" "${TRAVIS_BUILD_DIR:?}" 14 | script: 15 | - ./ci script "${JOB:?}" "${TRAVIS_BUILD_DIR:?}" 16 | matrix: 17 | exclude: 18 | - otp_release: 18.3 19 | env: JOB=smokeloadtest 20 | allow_failures: 21 | - env: JOB=dialyzer 22 | - env: JOB=xref 23 | fast_finish: true 24 | env: 25 | - JOB=eunit 26 | - JOB=dialyzer 27 | - JOB=xref 28 | - JOB=smokeloadtest 29 | cache: 30 | directories: 31 | - .plt 32 | -------------------------------------------------------------------------------- /test/trie_test_utils.erl: -------------------------------------------------------------------------------- 1 | -module(trie_test_utils). 2 | 3 | -export([cleanup_alive_sup/1, 4 | setup_for_clean_data_dir/1, cleanup_for_clean_data_dir/1]). 5 | 6 | -include_lib("stdlib/include/assert.hrl"). 7 | 8 | cleanup_alive_sup(Sup) when is_pid(Sup) -> 9 | SupMonRef = erlang:monitor(process, Sup), 10 | unlink(Sup), 11 | exit(Sup, Reason = shutdown), 12 | receive 13 | {'DOWN', SupMonRef, process, Sup, R} -> 14 | Reason = R, 15 | ok 16 | end, 17 | ok. 18 | 19 | setup_for_clean_data_dir(DataDir) -> 20 | ?assert(filelib:is_dir(DataDir)), 21 | ?assertEqual({ok, []}, file:list_dir_all(DataDir)), 22 | DataDir. 23 | 24 | cleanup_for_clean_data_dir(DataDir) -> 25 | DbFiles = filelib:wildcard(filename:join([DataDir, "*.db"])), 26 | lists:foreach(fun(F) -> {ok, _} = {file:delete(F), F} end, DbFiles), 27 | ?assertEqual({ok, []}, file:list_dir_all(DataDir)), 28 | ok. 29 | -------------------------------------------------------------------------------- /scripts/load_test/trie.config.template: -------------------------------------------------------------------------------- 1 | %% -*- mode: erlang;-*- 2 | {mode, {rate, 10}}. %% `max`, i.e. as many ops/s as possible, or `{rate, N}`, i.e. N ops/s, with exponentially distributed interarrival times. 3 | {concurrent, 1}. %% number of concurrent workers 4 | {duration, 1}. %% minutes 5 | 6 | {code_paths, CODE_PATHS_AS_STRING_LIST}. %% Paths of additional Erlang code e.g. `{code_paths, ["./foo/ebin", "./bar/ebin"]}.` 7 | 8 | {driver, basho_bench_driver_trie}. 9 | 10 | {operations, %% weighted list of operations the driver will run 11 | [ {root_hash, 1} 12 | , {put, 1} 13 | , {get, 1} 14 | , {get_all, 1} 15 | , {delete, 1} 16 | , {garbage, 1} 17 | ]}. 18 | 19 | {key_generator, {uniform_int, 100}}. %% 1..N 20 | {value_generator, {fixed_bin, 2}}. %% random binary of size N bytes 21 | 22 | %% Driver-specific configuration below. 23 | {trie_id, trieLoadTest}. 24 | {trie_key_size_bytes, 9}. 25 | {trie_value_size_bytes, 2}. 26 | {trie_meta_size_bytes, 0}. 27 | {trie_hash_size_bytes, 12}. 28 | {trie_amount, 1000000}. 29 | {trie_mode, hd}. 30 | {trie_initial_empty_root, 0}. 31 | %% TODO parametrize trie storage location. E.g. `{trie_dir, "./tmp/trie.bench"}.` 32 | -------------------------------------------------------------------------------- /unused/ram_store.erl: -------------------------------------------------------------------------------- 1 | -module(ram_store). 2 | -export([store/3]). 3 | 4 | store(Leaf, Trie, CFG) -> 5 | P = leaf:path(Leaf, CFG), 6 | LH = leaf:hash(Leaf, CFG), 7 | Weight = leaf:weight(Leaf), 8 | B = case get_branch(P, 0, leaf:value(Leaf), Trie, [], CFG) of 9 | {Leaf2, LP2, Branch} -> %split leaf, add stem(s) 10 | {A, N2} = path_match(P, leaf:path(Leaf2, CFG), 0), 11 | [H|T] = empty_stems(A-length(Branch)+1), 12 | LH2 = leaf:hash(Leaf2, CFG), 13 | W2 = leaf:weight(Leaf2), 14 | H2 = stem:add(H, N2, 2, LP2, W2, LH2), 15 | [H2|T]++Branch; 16 | Branch -> %overwrite 17 | Branch 18 | end, 19 | store_branch(B, P, 2, Leaf, CFG). 20 | 21 | get_branch(_,_,_,_,_,_) -> 22 | ok. 23 | store_branch(_,_,_,_,_) -> 24 | ok. 25 | 26 | add(L) -> add(L, 0). 27 | add([], X) -> X; 28 | add([H|T], X) -> add(T, H+X). 29 | path_match(LP, LP2, N) -> %returns {convergense_length, next nibble} 30 | NN = N*4, 31 | <<_:NN, A:4, _/bitstring>> = LP, 32 | <<_:NN, B:4, _/bitstring>> = LP2, 33 | if 34 | A == B -> path_match(LP, LP2, N+1); 35 | true -> {N, B}%+leaf:weight(Leaf)} 36 | end. 37 | empty_stems(0) -> []; 38 | empty_stems(N) -> [stem:new_empty()|empty_stems(N-1)]. 39 | -------------------------------------------------------------------------------- /src/cfg.erl: -------------------------------------------------------------------------------- 1 | -module(cfg). 2 | -compile(export_all). 3 | -export_type([cfg/0,path/0,value/0,id/0,meta/0,hash_size/0]). 4 | -record(cfg, { path 5 | , value 6 | , id 7 | , meta 8 | , hash_size 9 | , mode 10 | , empty_root 11 | }). 12 | -opaque cfg() :: #cfg{}. 13 | -type path() :: pos_integer(). 14 | -type value() :: non_neg_integer(). 15 | -type id() :: atom(). 16 | -type meta() :: non_neg_integer(). 17 | -type hash_size() :: pos_integer(). 18 | %-spec new(path(), value(), id(), meta(), hash_size()) -> cfg(). 19 | empty(X) when is_record(X, cfg) -> 20 | X#cfg.empty_root; 21 | empty(X) -> 22 | io:fwrite(X), 23 | 1=2, 24 | ok. 25 | set_empty(X, E) -> X#cfg{empty_root = E}. 26 | new(P, V, ID, M, H, X) -> #cfg{path = P, value = V, 27 | id = ID, meta = M, 28 | hash_size = H , mode = X}. 29 | -spec path(cfg()) -> path(). 30 | path(X) -> X#cfg.path. %how many bytes to store the path (defaul is 5) 31 | mode(X) -> X#cfg.mode. 32 | -spec value(cfg()) -> value(). 33 | value(X) -> X#cfg.value.%how many bytes to store the value. 34 | -spec meta(cfg()) -> meta(). 35 | meta(X) -> X#cfg.meta. %how many bytes to store the meta data that isn't hashed into the merkle tree. 36 | leaf(X) -> path(X) + value(X) + meta(X). 37 | -spec id(cfg()) -> id(). 38 | id(X) -> X#cfg.id. 39 | -spec hash_size(cfg()) -> hash_size(). 40 | hash_size(X) -> X#cfg.hash_size. 41 | -------------------------------------------------------------------------------- /src/prune.erl: -------------------------------------------------------------------------------- 1 | -module(prune). 2 | -export([stem/3, garbage/3]). 3 | %stem/3 is for pruning an old part of our history we no longer care about. 4 | %Each batch needs to be pruned in order to completely clear the memory. 5 | stem(Old, New, CFG) -> 6 | Stem1 = stem:get(Old, CFG), 7 | Stem2 = stem:get(New, CFG), 8 | dump:delete(Old, ids:stem(CFG)), 9 | loop(CFG, Stem1, Stem2, 1, [], fun(A, B, C) -> stem(A, B, C) end). 10 | 11 | %garbage/3 is for undoing a batch, to recover the space. 12 | %For example, if you are mining on a blockchain, and the block you are mining on does not get included. You want to recover the memory that was used by this block. 13 | garbage(Trash, Keep, CFG) -> 14 | TStem = stem:get(Trash, CFG), 15 | KStem = stem:get(Keep, CFG), 16 | if 17 | Trash == Keep -> ok; 18 | true -> dump:delete(Trash, ids:stem(CFG)) 19 | end, 20 | loop(CFG, TStem, KStem, 1, [], fun(A, B, C) -> garbage(A, B, C) end). 21 | loop(_, _, _, 17, R, _) -> R; 22 | loop(CFG, S1, S2, N, R, Stem) -> 23 | T1 = stem:type(N, S1), 24 | T2 = stem:type(N, S2), 25 | H1 = element(N, stem:hashes(S1)), 26 | H2 = element(N, stem:hashes(S2)), 27 | P1 = stem:pointer(N, S1), 28 | P2 = stem:pointer(N, S2), 29 | R2 = if 30 | (T1 == 0) or (H1 == H2) -> [];%nothing to clean 31 | true -> 32 | case {T1, T2} of%{trash, keep} 33 | {1, 1} -> Stem(P1, P2, CFG); 34 | {1, _} -> Stem(P1, 0, CFG); 35 | {2, _} -> dump:delete(P1, ids:leaf(CFG)), [{P1, P2}] 36 | end 37 | end, 38 | loop(CFG, S1, S2, N+1, R ++ R2, Stem). 39 | -------------------------------------------------------------------------------- /docs/todo.md: -------------------------------------------------------------------------------- 1 | The situation we care about is like this. 2 | Something has been stored in 3-3, and there is nothing else stored in 3, and we want to prove that 3-4 is empty. 3 | In order to prove this, we need to show that a leaf is stored in 3, and that the leaf doesn't point to 4. 4 | 5 | currently when you try to access something that doesn't exist, it returns 'empty'. 6 | But this is no good, if it is empty then we need to know the closest leaf above it to prove that it is empty. 7 | So get should probably return 4 things. besides returning a leaf, it should have a flag to tell us if this is the exact leaf, or the closest we could find. 8 | 9 | 10 | test_trie:test(6, _) used to do a test where we restored info to a trie that was no longer existing. 11 | This test was never passing, it needs to be fixed. 12 | 13 | 14 | 15 | check out this explanation: http://blog.notdot.net/2009/12/Damn-Cool-Algorithms-Log-structured-storagexs 16 | 17 | store:store/5 needs to be fixed. there are comments on how. 18 | 19 | 20 | get:get. 21 | If we garbage collect part of the trie, and try to get from it, we need a useful error message. 22 | It should not look the same as reading from an address that isn't written to. 23 | 24 | 25 | 26 | We need a totally ram version of the trie and the ability to use the ram trie to update our hard drive trie efficiently. This is a batch-write. 27 | We should also be able to read a RAM trie out of the hard drive given a root and list of leaves. 28 | 29 | 30 | 31 | %ram version 32 | {23372646,{2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}} 33 | 34 | %hard drive version 35 | {26785770,{2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}} 36 | 37 | based on measurements of the dump, the ram version should be about twice as fast. I am not sure why it isn't. -------------------------------------------------------------------------------- /src/get.erl: -------------------------------------------------------------------------------- 1 | -module(get). 2 | -export([get/3, same_end/3, test/0]). 3 | -export_type([proof/0]). 4 | 5 | -type proof() :: [stem:hashes(), ...]. % the last element is the 16-hashes-tuple contained in the root 6 | 7 | -spec get(leaf:path(), stem:stem_p(), cfg:cfg()) -> 8 | {RootHash::stem:hash(), Value, proof()} 9 | when Value :: empty | leaf:leaf(). 10 | get(Path, Root, CFG) -> 11 | S = stem:get(Root, CFG), 12 | H = stem:hash(S, CFG), 13 | case get2(Path, S, [stem:hashes(S)], CFG) of 14 | {unknown, Proof} -> {H, unknown, Proof}; 15 | {empty, Proof} -> {H, empty, Proof}; 16 | {A, Proof} -> {H, A, Proof} 17 | end. 18 | get2([<> | Path], Stem, Proof, CFG) -> 19 | NextType = stem:type(N+1, Stem), 20 | PN = stem:pointer(N+1, Stem), 21 | if 22 | NextType == 0 -> %empty 23 | %Next = stem:get(PN, CFG), 24 | {empty, Proof}; 25 | PN == 0 -> {unknown, Proof}; 26 | NextType == 1 -> %another stem 27 | Next = stem:get(PN, CFG), 28 | get2(Path, Next, [stem:hashes(Next)|Proof], CFG); 29 | NextType == 2 -> %leaf 30 | Leaf2 = leaf:get(PN, CFG), 31 | LPath = leaf:path(Leaf2, CFG), 32 | B = same_end(LPath, Path, CFG), 33 | LV = leaf:key(Leaf2), 34 | if 35 | B -> {Leaf2, Proof}; 36 | LV == 0 -> 37 | {empty, Proof}; 38 | true -> 39 | {empty, [leaf:serialize(Leaf2, CFG)|Proof]} 40 | end 41 | end. 42 | same_end(LPath, Path, _CFG) -> 43 | S = length(Path)*4, 44 | LS = (length(LPath)*4) - S, 45 | Path2 = tl_times(LS div 4, LPath), 46 | Path2 == Path. 47 | tl_times(N, L) when N < 1 -> L; 48 | tl_times(N, L) -> 49 | tl_times(N-1, tl(L)). 50 | 51 | test() -> 52 | CFG = trie:cfg(trie01), 53 | A = [1,2,3,4,5], 54 | B = [3,4,5] ++ A, 55 | true = same_end(B, A, CFG), 56 | success. 57 | -------------------------------------------------------------------------------- /src/trie_sup.erl: -------------------------------------------------------------------------------- 1 | -module(trie_sup). 2 | -behaviour(supervisor). 3 | -export([start_link/8,init/1,stop/1]). 4 | -define(CHILD(I, Type), {I, {I, start_link, []}, permanent, 5000, Type, [I]}). 5 | start_link(KeyLength, Size, ID, Amount, Meta, HashSize, Mode, Location) -> 6 | %keylength is the number of bytes to encode the path that you follow on the trie. 7 | CFG = cfg:new(KeyLength, Size, ID, Meta, HashSize, Mode), 8 | supervisor:start_link({global, cfg:id(CFG)}, ?MODULE, [CFG, Amount, Mode, Location]). 9 | stop(ID) -> 10 | CFG = trie:cfg(ID), 11 | supervisor:terminate_child({global, ID}, ids:main(CFG)), 12 | dump_sup:stop(ids:stem(CFG)), 13 | supervisor:terminate_child({global, ID}, ids:stem(CFG)), 14 | dump_sup:stop(ids:leaf(CFG)), 15 | supervisor:terminate_child({global, ID}, ids:leaf(CFG)), 16 | supervisor:terminate_child({global, ID}, ids:bits(CFG)), 17 | ok. 18 | 19 | %trie01_main). 20 | %halt(). 21 | init([CFG, Amount, Mode, Location]) -> 22 | %Size is the size of the data we store in the trie. 23 | KeyLength = cfg:path(CFG), 24 | HashSize = cfg:hash_size(CFG), 25 | Size = cfg:value(CFG)+cfg:meta(CFG), 26 | ID = cfg:id(CFG), 27 | IDS = atom_to_list(ID), 28 | A2 = list_to_atom(IDS++"_bits"), 29 | A3 = ids:leaf(CFG), 30 | A4 = ids:stem(CFG), 31 | A5 = ids:main(CFG), 32 | L2 = Location ++ "data/" ++ IDS ++ "_trie_bits.db", 33 | Children = [{A3, {dump_sup, start_link, [A3, KeyLength+Size, Amount, Mode, Location]}, permanent, 5000, supervisor, [dump_sup]}, 34 | {A4, {dump_sup, start_link, [A4, 4+(16*(HashSize + KeyLength)), Amount, Mode, Location]}, permanent, 5000, supervisor, [dump_sup]}, 35 | %{A2, {bits, start_link, [A2, L2, Amount]}, permanent, 5000, worker, [bits]}, 36 | {A5, {trie, start_link, [CFG]}, permanent, 5000, worker, [trie]} 37 | ], 38 | {ok, { {one_for_one, 5, 10}, Children} }. 39 | -------------------------------------------------------------------------------- /unused/ram_trie.erl: -------------------------------------------------------------------------------- 1 | -module(ram_trie). 2 | -behaviour(gen_server). 3 | -export([start_link/1,code_change/3,handle_call/3,handle_cast/2,handle_info/2,init/1,terminate/2, cfg/1,get/3,put/4,garbage/2,garbage_leaves/2]). 4 | init(CFG) -> {ok, CFG}. 5 | start_link(CFG) -> gen_server:start_link({global, ids:ram(CFG)}, ?MODULE, CFG, []). 6 | code_change(_OldVsn, State, _Extra) -> {ok, State}. 7 | terminate(_, _) -> io:format("died!"), ok. 8 | handle_info(_, X) -> {noreply, X}. 9 | handle_cast({garbage, Keepers}, CFG) -> 10 | ram_garbage:garbage(Keepers, CFG), 11 | {noreply, CFG}; 12 | handle_cast({garbage_leaves, KLS}, CFG) -> 13 | ram_garbage:garbage_leaves(KLS, CFG), 14 | {noreply, CFG}; 15 | handle_cast(_, X) -> {noreply, X}. 16 | handle_call({put, Value, Root, Weight}, _From, CFG) -> 17 | ID = ids:ram(cfg:id(CFG)), 18 | Key = bits:top(ID), 19 | Leaf = leaf:new(Key, Weight, Value), 20 | {_, NewRoot, _} = ram_store:store(Leaf, Root, CFG), 21 | bits:write(ID), 22 | {reply, {Key, NewRoot}, CFG}; 23 | handle_call({get, Key, Root}, _From, CFG) -> 24 | P = leaf:path_maker(Key, CFG), 25 | {RootHash, Leaf, Proof} = ram_get:get(P, Root, CFG), 26 | {reply, {RootHash, Leaf, Proof}, CFG}; 27 | handle_call({garbage, Keepers}, _From, CFG) -> 28 | ram_garbage:garbage(Keepers, CFG), 29 | {reply, ok, CFG}; 30 | handle_call({garbage_leaves, KLS}, _From, CFG) -> 31 | ram_garbage:garbage_leaves(KLS, CFG), 32 | {reply, ok, CFG}; 33 | handle_call(cfg, _From, CFG) -> 34 | {reply, CFG, CFG}. 35 | cfg(ID) when is_atom(ID) -> gen_server:call({global, ids:main_id(ID)}, cfg). 36 | put(Value, Root, Weight, ID) -> 37 | gen_server:call({global, ids:main_id(ID)}, {put, Value, Root, Weight}). 38 | get(Key, Root, ID) -> gen_server:call({global, ids:main_id(ID)}, {get, Key, Root}). 39 | garbage(Keepers, ID) -> 40 | io:fwrite("trie garbage \n"), 41 | gen_server:cast({global, ids:main_id(ID)}, {garbage, Keepers}). 42 | garbage_leaves(KLS, ID) -> 43 | gen_server:cast({global, ids:main_id(ID)}, {garbage_leaves, KLS}). 44 | 45 | 46 | -------------------------------------------------------------------------------- /test/prop_trie_arbitrary_put_and_delete.erl: -------------------------------------------------------------------------------- 1 | -module(prop_trie_arbitrary_put_and_delete). 2 | -include_lib("stdlib/include/assert.hrl"). 3 | -include_lib("proper/include/proper.hrl"). 4 | 5 | -export([command/1, initial_state/0, next_state/3, 6 | precondition/2, postcondition/3]). 7 | 8 | -define(ID, triePropTest). 9 | 10 | prop_from_data_dir_as_it_is() -> 11 | ?FORALL(Cmds, commands(?MODULE), 12 | begin 13 | R = setup_trie(?ID), 14 | {History, State, Result} = run_commands(?MODULE, Cmds), 15 | cleanup_trie(R), 16 | ?WHENFAIL(io:format("History: ~p~nState: ~p~nResult: ~p~n", 17 | [History, State, Result]), 18 | aggregate(command_names(Cmds), Result =:= ok)) 19 | end). 20 | 21 | prop_from_clean_data_dir() -> 22 | {ok, Cwd} = file:get_cwd(), 23 | DataDir = filename:join([Cwd, "data"]), 24 | trie_test_utils:cleanup_for_clean_data_dir(DataDir), 25 | ?FORALL(Cmds, commands(?MODULE), 26 | begin 27 | R1 = trie_test_utils:setup_for_clean_data_dir(DataDir), 28 | R2 = setup_trie(?ID), 29 | {History, State, Result} = run_commands(?MODULE, Cmds), 30 | cleanup_trie(R2), 31 | trie_test_utils:cleanup_for_clean_data_dir(R1), 32 | ?WHENFAIL(io:format("History: ~p~nState: ~p~nResult: ~p~n", 33 | [History, State, Result]), 34 | aggregate(command_names(Cmds), Result =:= ok)) 35 | end). 36 | 37 | setup_trie(Id) -> 38 | {ok, SupPid} = trie_sup:start_link( 39 | _KeyLength = 9, 40 | _Size = 2, 41 | _ID = Id, 42 | _Amount = 1000000, 43 | _Meta = 2, 44 | _HashSize = 12, 45 | _Mode = hd), 46 | SupPid. 47 | 48 | cleanup_trie(SupPid) -> 49 | trie_test_utils:cleanup_alive_sup(SupPid). 50 | 51 | -record(state, {root}). 52 | 53 | initial_state() -> 54 | #state{root = 0}. 55 | 56 | command(State) -> 57 | Root = State#state.root, 58 | oneof([{call, trie, put, [leaf_key(), <<1,1>>, 0, Root, ?ID]}, 59 | {call, trie, delete, [leaf_key(), Root, ?ID]} 60 | ]). 61 | 62 | precondition(#state{}, {call, _Mod, _Fun, _Args}) -> 63 | true. 64 | 65 | leaf_key() -> 66 | integer(1, 100). 67 | 68 | next_state(State, Res, {call, trie, put, _Args}) -> 69 | State#state{root = Res}; 70 | next_state(State, Res, {call, trie, delete, _Args}) -> 71 | State#state{root = Res}. 72 | 73 | postcondition(_State, {call, _Mod, _Fun, _Args}, _Res) -> 74 | true. 75 | -------------------------------------------------------------------------------- /test/prop_trie_root_hash_independent_from_order_of_operations.erl: -------------------------------------------------------------------------------- 1 | -module(prop_trie_root_hash_independent_from_order_of_operations). 2 | 3 | -include_lib("stdlib/include/assert.hrl"). 4 | -include_lib("proper/include/proper.hrl"). 5 | 6 | -define(ID, triePropTest). 7 | 8 | prop_put_each_key_only_once_and_no_delete() -> 9 | {ok, Cwd} = file:get_cwd(), 10 | DataDir = filename:join([Cwd, "data"]), 11 | trie_test_utils:cleanup_for_clean_data_dir(DataDir), 12 | ?FORALL({DistinctLeafKeys, ShuffledDistinctLeafKeys}, 13 | distinct_leaf_keys_and_shuffled(0, 99), 14 | begin 15 | ?assertEqual(length(DistinctLeafKeys), 16 | length(ShuffledDistinctLeafKeys)), 17 | RH1 = trie_put_all_and_compute_root_hash_in_clean_data_dir( 18 | DataDir, ?ID, 0, DistinctLeafKeys), 19 | RH2 = trie_put_all_and_compute_root_hash_in_clean_data_dir( 20 | DataDir, ?ID, 0, ShuffledDistinctLeafKeys), 21 | %% io:format( 22 | %% "Put ~p elements in two trees and comparing their root hashes ~w and ~w.~nInitial elements in first tree are:~n ~w~nInitial elements in second tree are:~n ~w~n", 23 | %% [length(DistinctLeafKeys), 24 | %% RH1, RH1, 25 | %% lists:sublist(DistinctLeafKeys, 3), 26 | %% lists:sublist(ShuffledDistinctLeafKeys, 3)]), 27 | RH1 == RH2 28 | end). 29 | 30 | trie_put_all_and_compute_root_hash_in_clean_data_dir(DataDir, Id, Root, Keys) -> 31 | R1 = trie_test_utils:setup_for_clean_data_dir(DataDir), 32 | R2 = setup_trie(Id), 33 | NewRoot = 34 | lists:foldl( 35 | fun(K, R) -> 36 | trie:put(K, _V = <>, _M = 0, R, Id) 37 | end, 38 | Root, 39 | Keys), 40 | RH = trie:root_hash(Id, NewRoot), 41 | cleanup_trie(R2), 42 | trie_test_utils:cleanup_for_clean_data_dir(R1), 43 | RH. 44 | 45 | setup_trie(Id) -> 46 | {ok, SupPid} = trie_sup:start_link( 47 | _KeyLength = 9, 48 | _Size = 2, 49 | _ID = Id, 50 | _Amount = 1000000, 51 | _Meta = 2, 52 | _HashSize = 12, 53 | _Mode = hd), 54 | SupPid. 55 | 56 | cleanup_trie(SupPid) -> 57 | trie_test_utils:cleanup_alive_sup(SupPid). 58 | 59 | distinct_leaf_keys_and_shuffled(Low, High) -> 60 | ?LET(Ks, distinct_leaf_keys(Low, High), {Ks, shuffle(Ks)}). 61 | 62 | distinct_leaf_keys(Low, High) -> 63 | ulist(leaf_key(Low, High)). 64 | 65 | leaf_key(Low, High) -> 66 | integer(Low, High). 67 | 68 | %% From https://github.com/manopapad/proper/blob/v1.2/test/proper_tests.erl#L1457-L1458 69 | ulist(ElemType) -> 70 | ?SUCHTHAT(L, list(ElemType), no_duplicates(L)). 71 | %% From https://github.com/manopapad/proper/blob/v1.2/test/proper_tests.erl#L1219-L1220 72 | no_duplicates(L) -> 73 | length(lists:usort(L)) =:= length(L). 74 | 75 | %% From https://github.com/manopapad/proper/blob/v1.2/test/proper_tests.erl#L1306-L1309 76 | shuffle([]) -> 77 | []; 78 | shuffle(L) -> 79 | ?LET(X, elements(L), [X | shuffle(lists:delete(X,L))]). 80 | -------------------------------------------------------------------------------- /scripts/load_test/basho_bench_driver_trie.erl: -------------------------------------------------------------------------------- 1 | -module(basho_bench_driver_trie). 2 | 3 | -export([new/1, 4 | run/4]). 5 | 6 | %% Macros compatible with header `basho_bench.hrl` - in order not to 7 | %% require including it for compiling this module: 8 | -define(DEBUG(Str, Args), error_logger:info_msg(Str, Args)). 9 | -define(INFO(Str, Args), error_logger:info_msg(Str, Args)). 10 | -define(WARN(Str, Args), error_logger:warning_msg(Str, Args)). 11 | -define(ERROR(Str, Args), error_logger:error_msg(Str, Args)). 12 | 13 | -record(state, { 14 | trie_id, 15 | root 16 | }). 17 | 18 | new({_,_,Id}) -> 19 | TrieId = basho_bench_config:get(trie_id), 20 | KeySizeBytes = basho_bench_config:get(trie_key_size_bytes), 21 | ValueSizeBytes = basho_bench_config:get(trie_value_size_bytes), 22 | MetaSizeBytes = 0 = basho_bench_config:get(trie_meta_size_bytes), 23 | HashSizeBytes = basho_bench_config:get(trie_hash_size_bytes), 24 | Amount = basho_bench_config:get(trie_amount), 25 | Mode = basho_bench_config:get(trie_mode), 26 | {ok, TrieSupPid} = 27 | trie_sup:start_link( %% TODO review linking 28 | KeySizeBytes, 29 | ValueSizeBytes, 30 | TrieId, 31 | Amount, 32 | MetaSizeBytes, 33 | HashSizeBytes, 34 | Mode), 35 | ?INFO("Worker ~p (~p) using trie with id ~p and sup ~p.\n", [Id, self(), TrieId, TrieSupPid]), 36 | Root = basho_bench_config:get(trie_initial_empty_root), 37 | {[], _} = {trie:get_all(Root, TrieId), {{worker_id, Id}, {trie_initial_allegedly_empy_root, Root}}}, 38 | State = #state{trie_id = TrieId, root = Root}, 39 | {ok, State}. 40 | 41 | run(root_hash, _KeyGen, _ValueGen, State = #state{}) -> 42 | _ = trie:root_hash(State#state.trie_id, State#state.root), 43 | {ok, State}; 44 | run(put, KeyGen, ValueGen, State = #state{}) -> 45 | Key = KeyGen(), 46 | Value = ValueGen(), 47 | NewRoot = trie:put(Key, Value, _Meta = 0, State#state.root, State#state.trie_id), 48 | NewState = State#state{root = NewRoot}, 49 | {ok, NewState}; 50 | run(get, KeyGen, _ValueGen, State = #state{}) -> 51 | Key = KeyGen(), 52 | case trie:get(Key, State#state.root, State#state.trie_id) of 53 | {_, empty, _} -> 54 | {ok, State}; 55 | {_, _, _} -> 56 | {ok, State}; 57 | Unknown -> 58 | {error, Unknown, State} 59 | end; 60 | run(get_all, _KeyGen, _ValueGen, State = #state{}) -> 61 | case trie:get_all(State#state.root, State#state.trie_id) of 62 | L when is_list(L) -> 63 | {ok, State}; 64 | Unknown -> 65 | {error, Unknown, State} 66 | end; 67 | run(delete, KeyGen, _ValueGen, State = #state{}) -> 68 | Key = KeyGen(), 69 | NewRoot = trie:delete(Key, State#state.root, State#state.trie_id), 70 | NewState = State#state{root = NewRoot}, 71 | {ok, NewState}; 72 | run(garbage, _KeyGen, _ValueGen, State = #state{}) -> 73 | case trie:garbage([State#state.root], State#state.trie_id) of 74 | ok -> 75 | {ok, State}; 76 | Unknown -> 77 | {error, Unknown, State} 78 | end. 79 | -------------------------------------------------------------------------------- /ci: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -ev # Ref https://docs.travis-ci.com/user/customizing-the-build/#Implementing-Complex-Build-Steps 4 | 5 | case "${1:?}"-"${2:?}" in 6 | before_install-*) 7 | ## Travis CI does not support rebar3 yet. See https://github.com/travis-ci/travis-ci/issues/6506#issuecomment-275189490 8 | BuildDir="${3:?}" 9 | curl -fL -o "${BuildDir:?}"/rebar3 https://github.com/erlang/rebar3/releases/download/3.4.2/rebar3 10 | chmod +x "${BuildDir:?}"/rebar3 11 | ;; 12 | install-dialyzer) 13 | BuildDir="${3:?}" 14 | ( cd "${BuildDir:?}" && ./rebar3 tree; ) 15 | ( cd "${BuildDir:?}" && ./rebar3 dialyzer -u true -s false; ) 16 | ;; 17 | install-smokeloadtest) 18 | BuildDir="${3:?}" 19 | WorkDir="${BuildDir:?}"/tmp 20 | mkdir "${WorkDir:?}" ## create working folder 21 | ( cd "${WorkDir:?}" && git clone https://github.com/mrallen1/basho_bench.git && cd basho_bench && git checkout c92c71c09b17f5f2c983a31d2a6ec2db40de9bfe && PATH="${BuildDir:?}":"$PATH" make && file basho_bench; ) ## build load test tool "basho_bench" 22 | mkdir "${WorkDir:?}"/R_libs ## create location for R libraries used by graph generation in basho_bench 23 | ;; 24 | install-*) 25 | true 26 | ;; 27 | before_script-eunit) 28 | BuildDir="${3:?}" 29 | mkdir "${BuildDir:?}"/data 30 | ;; 31 | before_script-smokeloadtest) 32 | BuildDir="${3:?}" 33 | WorkDir="${BuildDir:?}"/tmp 34 | ( cd "${BuildDir:?}" && ./rebar3 compile && ls "${BuildDir:?}"/_build/default/lib; ) ## compile trie library 35 | mkdir "${WorkDir:?}"/ebin ## create location for trie library-specific module for basho_bench 36 | mkdir "${WorkDir:?}"/basho_bench/data ## create location for data persisted by trie library 37 | ;; 38 | before_script-*) 39 | true 40 | ;; 41 | script-eunit) 42 | BuildDir="${3:?}" 43 | ( cd "${BuildDir:?}" && ./rebar3 eunit && ./rebar3 proper; ) 44 | ;; 45 | script-dialyzer) 46 | BuildDir="${3:?}" 47 | ( cd "${BuildDir:?}" && ./rebar3 dialyzer; ) 48 | ;; 49 | script-xref) 50 | BuildDir="${3:?}" 51 | ( cd "${BuildDir:?}" && ./rebar3 xref; ) 52 | ;; 53 | script-smokeloadtest) 54 | BuildDir="${3:?}" 55 | WorkDir="${BuildDir:?}"/tmp 56 | erlc -o "${WorkDir:?}"/ebin "${BuildDir:?}"/scripts/load_test/basho_bench_driver_trie.erl && file "${WorkDir:?}"/ebin/basho_bench_driver_trie.beam ## compile trie library-specific module for basho_bench 57 | sed -e "s|CODE_PATHS_AS_STRING_LIST|[\"${WorkDir:?}/ebin\", \"${BuildDir:?}/_build/default/lib/dump/ebin\", \"${BuildDir:?}/_build/default/lib/encrypter/ebin\", \"${BuildDir:?}/_build/default/lib/pink_hash/ebin\", \"${BuildDir:?}/_build/default/lib/trie/ebin\"]|" < "${BuildDir:?}"/scripts/load_test/trie.config.template > "${WorkDir:?}"/trie.config && cat "${WorkDir:?}"/trie.config ## instantiate load test configuration and show it 58 | ( cd "${WorkDir:?}"/basho_bench && ./basho_bench --results-dir ./tests "${WorkDir:?}"/trie.config; ) ## run load test 59 | ( cd "${WorkDir:?}"/basho_bench && ( R_LIBS="${WorkDir:?}"/R_libs && export R_LIBS && make results; ) && ( cd ./tests/current && ls -l; ) && file ./tests/current/summary.png; ) ## generate load test results graph 60 | ;; 61 | esac 62 | -------------------------------------------------------------------------------- /src/verify.erl: -------------------------------------------------------------------------------- 1 | -module(verify). 2 | -export([proof/4, update_proof/3, update_proofs/2]). 3 | 4 | -spec proof(stem:hash(), leaf:leaf(), get:proof(), cfg:cfg()) -> boolean(). 5 | 6 | update_proof(L, Proof, CFG) -> 7 | LP = leaf:path(L, CFG), 8 | %take the slice of the path we will use, and reverse it. 9 | N = length(Proof), 10 | {LP2, _} = lists:split(N, LP), 11 | LP3 = lists:reverse(LP2), 12 | LH = leaf:hash(L, CFG), 13 | Proof2 = update_internal(LP3, LH, Proof, CFG), 14 | Proof2. 15 | 16 | update_internal(_, _, [], _) -> []; 17 | update_internal([<> | M], LH, Proof, CFG) -> 18 | P1 = hd(Proof), 19 | %Hash = element(N+1, P1), 20 | P2 = setelement(N+1, P1, LH), 21 | NH = stem:hash(P2, CFG), 22 | [P2|update_internal(M, NH, tl(Proof), CFG)]. 23 | 24 | update_proofs(X, CFG) -> 25 | update_proofs(X, CFG, dict:new(), []). 26 | update_proofs([], _, D, L) -> 27 | L2 = lists:reverse(L), 28 | lists:map(fun(X) ->%do this to every list in the list of lists. 29 | lists:map(fun(Y) ->%update every element of the list 30 | merge_find_helper(Y, D) 31 | 32 | end, X) 33 | end, L2); 34 | update_proofs([{Leaf, Proof}|T], CFG, D, L) -> 35 | %use D to remember which stems have been updated already. 36 | LP = leaf:path(Leaf, CFG), 37 | N = length(Proof), 38 | {LP2, _} = lists:split(N, LP), 39 | LP3 = lists:reverse(LP2), 40 | LH = leaf:hash(Leaf, CFG), 41 | {D2, NewProof} = update_proofs2(LP3, LH, Proof, D, CFG, []), 42 | update_proofs(T, CFG, D2, [NewProof|L]). 43 | 44 | merge_find_helper(P, D) -> 45 | io:fwrite("merge find helper\n"), 46 | case dict:find(P, D) of 47 | error -> P; 48 | {ok, error} -> 1=2; 49 | {ok, P2} -> 50 | merge_find_helper(P2, D) 51 | end. 52 | 53 | update_proofs2(_, _, [], D, _, Proof) -> 54 | {D, lists:reverse(Proof)}; 55 | update_proofs2([<>|M], LH, Proof, D, CFG, Proof2) -> 56 | P1 = hd(Proof), 57 | P = merge_find_helper(P1, D), 58 | P2 = setelement(N+1, P, LH), 59 | D2 = dict:store(P1, P2, D), 60 | D3 = dict:store(P, P2, D), 61 | NH = stem:hash(P2, CFG), 62 | update_proofs2(M, NH, tl(Proof), D3, CFG, [P2|Proof2]). 63 | 64 | proof(RootHash, L, Proof, CFG) -> 65 | [H|F] = lists:reverse(Proof), 66 | %[H|F] = Proof, 67 | SH = stem:hash(H, CFG), 68 | if 69 | SH == RootHash -> 70 | proof_internal(leaf:path(L, CFG), L, [H|F], CFG); 71 | true -> 72 | io:fwrite("false 1\n"), 73 | false 74 | end. 75 | 76 | -spec proof_internal(leaf:path(), leaf:leaf(), get:proof(), cfg:cfg()) -> boolean(). 77 | proof_internal([<> | M], Leaf, P, CFG) when length(P) == 1-> 78 | P1 = hd(P), 79 | Hash = element(N+1, P1), 80 | V = leaf:value(Leaf), 81 | LH = leaf:hash(Leaf, CFG), 82 | Hash == LH; 83 | proof_internal([<>| Path ], Leaf, [P1, P2 | Proof], CFG) -> 84 | %if leaf is empty, and P2 is a leaf, then we do a different test. 85 | %pass if hash(leaf) is in P1, and N does _not_ point to leaf P2. 86 | LB = leaf:is_serialized_leaf(P2, CFG), 87 | LV = leaf:value(Leaf), 88 | if 89 | (LV == empty) and LB -> 90 | Leaf2 = leaf:deserialize(P2, CFG), 91 | LH = leaf:hash(Leaf2, CFG), 92 | is_in(LH, tuple_to_list(P1)) 93 | and not(get:same_end(leaf:path(Leaf2, CFG), 94 | [<>|Path], 95 | CFG)); 96 | true -> 97 | Hash = element(N+1, P1), 98 | case stem:hash(P2, CFG) of 99 | Hash -> proof_internal(Path, Leaf, [P2 | Proof], CFG); 100 | X -> 101 | io:fwrite("false 3\n"), 102 | %io:fwrite({X, Hash, [P1, P2|Proof]}), 103 | false 104 | end 105 | end; 106 | proof_internal(_, _, _, _) -> 107 | io:fwrite("false 2\n"), 108 | false. 109 | is_in(X, []) -> false; 110 | is_in(X, [X|_]) -> true; 111 | is_in(X, [A|T]) -> is_in(X, T). 112 | -------------------------------------------------------------------------------- /src/leaf.erl: -------------------------------------------------------------------------------- 1 | -module(leaf). 2 | -export([new/4, key/1, value/1, meta/1, path/2, path_maker/2, hash/2, put/2, get/2, serialize/2, deserialize/2, 3 | put_batch/2, 4 | is_serialized_leaf/2, test/0]). 5 | -export_type([leaf/0,key/0,value/0,meta/0,leaf_p/0,path/0]). 6 | -record(leaf, { key :: key() 7 | , value :: value() 8 | , meta :: meta() %meta is data we want to remember that doesn't get hashed into the merkle tree. 9 | }). 10 | -opaque leaf() :: #leaf{}. 11 | -type key() :: non_neg_integer(). 12 | -type value() :: binary(). 13 | -type meta() :: non_neg_integer(). 14 | -opaque leaf_p() :: non_neg_integer(). 15 | -type path() :: path(cfg:path()). 16 | -type path(_CfgPathSizeBytes) :: [nib(), ...]. % non-empty because configured path size positive 17 | -type nib() :: <<_:4>>. 18 | is_serialized_leaf(X, CFG) -> 19 | P = cfg:path(CFG), 20 | M = cfg:meta(CFG), 21 | S = cfg:value(CFG), 22 | size(X) == (P + M + S). 23 | %is_record(X, leaf). 24 | serialize(X, CFG) -> 25 | P = cfg:path(CFG) * 8, 26 | M = cfg:meta(CFG) * 8, 27 | S = cfg:value(CFG), 28 | S = size(X#leaf.value), 29 | %io:fwrite({CFG, X}), 30 | <<(X#leaf.key):P, 31 | (X#leaf.meta):M, 32 | (X#leaf.value)/binary>>. 33 | deserialize(A, CFG) -> 34 | L = cfg:value(CFG) * 8, 35 | P = cfg:path(CFG) * 8, 36 | MS = cfg:meta(CFG) * 8, 37 | <> = A, 40 | #leaf{key = Key, value = <>, meta = Meta}. 41 | -spec new(key(), value(), meta(), cfg:cfg()) -> leaf(). 42 | new(Key, Value, Meta, CFG) -> 43 | P = cfg:path(CFG), 44 | ok = check_key(Key, P), 45 | L = cfg:value(CFG) * 8, 46 | case Value of 47 | empty -> ok; 48 | <<_:L>> -> ok; 49 | _ -> io:fwrite({value_is, size(Value), L div 8}) 50 | end, 51 | %L = cfg:value(CFG) * 8, 52 | %<<_:L>> = Value, 53 | #leaf{key = Key, value = Value, meta = Meta}. 54 | check_key(Key, LBytes) when is_integer(Key), 55 | Key >= 0, 56 | Key < (1 bsl (LBytes * 8)) -> 57 | ok; 58 | check_key(Key, _) when is_integer(Key) -> 59 | {error, key_out_of_range}; 60 | check_key(_, _) -> 61 | {error, key_not_integer}. 62 | -spec key(leaf()) -> key(). 63 | key(L) -> L#leaf.key. 64 | -spec path(leaf(), cfg:cfg()) -> path(). 65 | path(L, CFG) -> 66 | K = key(L), 67 | path_maker(K, CFG). 68 | -spec path_maker(key(), cfg:cfg()) -> path(). 69 | path_maker(K, CFG) -> 70 | T = cfg:path(CFG)*8, 71 | lists:reverse([<>||<> <= <>]). 72 | 73 | -spec value(leaf()) -> value(). 74 | value(L) -> L#leaf.value. 75 | -spec meta(leaf()) -> meta(). 76 | meta(X) -> X#leaf.meta. 77 | -spec put(leaf(), cfg:cfg()) -> leaf_p(). 78 | put_batch(Leaves, CFG) -> 79 | SL = serialize_leaves(Leaves, CFG), 80 | dump:put_batch(SL, ids:leaf(CFG)). 81 | serialize_leaves([], _) -> 82 | []; 83 | serialize_leaves([{N, L}| T], CFG) -> 84 | [{N, serialize(L, CFG)}|serialize_leaves(T, CFG)]. 85 | 86 | put(Leaf, CFG) -> 87 | dump:put(serialize(Leaf, CFG), 88 | ids:leaf(CFG)). 89 | -spec get(leaf_p(), cfg:cfg()) -> leaf(). 90 | get(Pointer, CFG) -> 91 | L = dump:get(Pointer, ids:leaf(CFG)), 92 | deserialize(L, CFG). 93 | -spec hash(leaf(), cfg:cfg()) -> stem:hash(). 94 | hash(L, CFG) -> 95 | HS = cfg:hash_size(CFG)*8, 96 | case L#leaf.value of 97 | empty -> <<0:HS>>; 98 | V -> 99 | P = cfg:path(CFG) * 8,%8 times bigger than necessary. :( 100 | HS2 = cfg:hash_size(CFG), 101 | hash:doit(<<(L#leaf.key):P, V/binary>>, HS2) 102 | end. 103 | test() -> 104 | CFG = trie:cfg(trie01), 105 | %{cfg,5,2,trie01,2,32} path, value, id, meta, hash_size 106 | %HS = cfg:hash_size(CFG)*8, 107 | X = new(1, <<0:16>>, 0, CFG), 108 | SX = serialize(X, CFG), 109 | io:fwrite(integer_to_list(size(SX))),%9 110 | X = deserialize(serialize(X, CFG), CFG), 111 | true = is_serialized_leaf(SX, CFG), 112 | success. 113 | 114 | -------------------------------------------------------------------------------- /garbage.erl: -------------------------------------------------------------------------------- 1 | %choices on how to fix garbage_leaves 2 | %1 we could cycle through every stem in the trie, check every pointer, and see if it is deleted. 3 | %2 we could rethink how garbage and garbage_leaves work, so keepers is combined with delete_stuff. Instead of cycling through every stem, we walk down the 4 | 5 | -module(garbage). 6 | -export([garbage/2, garbage_leaves/2]). 7 | garbage_leaves(KeeperLeaves, CFG) -> 8 | {KeeperStems, KL} = keepers_backwards(KeeperLeaves, CFG), 9 | %We need to update all the Keeper stems, so that the empty things aren't pointed to. 10 | DeletedLeaves = delete_stuff(1, KL, ids:leaf(CFG)), 11 | DeletedStems = delete_stuff(1, [1|KeeperStems], ids:stem(CFG)),%maybe delete_stuff should return the locations of all deleted stems and leaves, that way we know what to replace with 0s. 12 | remove_bad_pointers(1, [1|KeeperStems], DeletedStems, CFG),%1 is for stems 13 | remove_bad_pointers(2, [1|KeeperStems], DeletedLeaves, CFG),%2 is for leaves 14 | ok. 15 | remove_bad_pointers(_, [], _, _) -> ok; 16 | remove_bad_pointers(PT, [K|KT], DS, CFG) -> 17 | Stem = stem:get(K, CFG), 18 | Pointers = stem:pointers(Stem), 19 | Types = stem:types(Stem), 20 | NewPointers = rbp2(PT, Pointers, Types, DS), 21 | Stem2 = stem:update_pointers(Stem, NewPointers), 22 | stem:update(K, Stem2, CFG), 23 | remove_bad_pointers(PT, KT, DS, CFG). 24 | rbp2(PT, Pointers, Types, DS) -> 25 | X = rbp3(tuple_to_list(Pointers), 26 | tuple_to_list(Types), 27 | DS, PT), 28 | list_to_tuple(X). 29 | rbp3([], [], _, _) -> []; 30 | rbp3([P|PT], [T|TT], DS, PointerType) -> 31 | B = (lists:member(P, DS)) and (T == PointerType), 32 | case B of 33 | true -> 34 | [0|rbp3(PT, TT, DS, PointerType)]; 35 | false -> [P|rbp3(PT, TT, DS, PointerType)] 36 | end. 37 | 38 | -spec garbage([stem:stem_p()], cfg:cfg()) -> ok. 39 | garbage(KeeperRoots, CFG) -> 40 | {KeeperStems, KeeperLeaves} = keepers(KeeperRoots, CFG), 41 | delete_stuff(1, KeeperStems, ids:stem(CFG)), 42 | delete_stuff(1, KeeperLeaves, ids:leaf(CFG)), 43 | ok. 44 | keepers_backwards(X, CFG) -> keepers_backwards(X, {[],[]}, CFG). 45 | keepers_backwards([], X, _) -> X; 46 | keepers_backwards([{Path, Root}|Leaves], {KS, KL}, CFG) -> 47 | S = stem:get(Root, CFG), 48 | {Stems, Leaf} = kb2(Path, S, [Root], CFG), 49 | keepers_backwards(Leaves, 50 | {append_no_repeats(Stems, KS), 51 | append_no_repeats([Leaf], KL)}, 52 | CFG). 53 | kb2([<> | Path], Stem, Keepers, CFG) -> 54 | NextType = stem:type(N+1, Stem), 55 | PN = stem:pointer(N+1, Stem), 56 | case NextType of 57 | 1 -> %another stem 58 | Next = stem:get(PN, CFG), 59 | kb2(Path, Next, append_no_repeats([PN], Keepers), CFG); 60 | 2 -> %leaf 61 | {Keepers, PN} 62 | end. 63 | keepers([], _) -> {[], []}; 64 | keepers([R|Roots], CFG) -> %returns {keeperstems, keeperleaves} 65 | case stem:get(R, CFG) of 66 | error -> 67 | {A, B} = keepers(Roots, CFG), 68 | {[R|A],B}; 69 | S -> 70 | {X, Y, MoreRoots} = stem_keepers(S), 71 | {A, B} = keepers(MoreRoots++Roots, CFG), 72 | {[R|append_no_repeats(X, A)], 73 | append_no_repeats(Y, B)} 74 | end. 75 | append_no_repeats([],X) -> X; 76 | append_no_repeats([A|Ta],X) -> 77 | Bool2 = lists:member(A, X), 78 | if 79 | Bool2 -> append_no_repeats(Ta, X); 80 | true -> append_no_repeats(Ta, [A|X]) 81 | end. 82 | stem_keepers(S) -> 83 | stem_keepers(S, 1, [], [], []). 84 | stem_keepers(_, 17, Stems, Leaves, Roots) -> {Stems,Leaves, Roots}; 85 | stem_keepers(S, N, Stems, Leaves, MoreRoots) -> 86 | P = stem:pointer(N, S), 87 | {NewStems, NewLeaves, NewMoreRoots} = 88 | case stem:type(N, S) of 89 | 0 -> {Stems, Leaves, MoreRoots}; 90 | 1 -> 91 | {[P|Stems], Leaves,[P|MoreRoots]}; 92 | 2 -> {Stems, [P|Leaves], MoreRoots} 93 | end, 94 | stem_keepers(S, N+1, NewStems, NewLeaves, NewMoreRoots). 95 | delete_stuff(_, Keepers, ID) -> 96 | S = dump:highest(ID) div dump:word(ID), 97 | delete_stuff(S, 1, Keepers, ID, []). 98 | delete_stuff(S, N, Keepers, Id, Out) -> 99 | Bool = lists:member(N, Keepers), 100 | if 101 | N>S -> Out;%we should go through the list of keepers and update any pointers that point to deleted data to instead point to 0. 102 | Bool -> 103 | delete_stuff(S, N+1, Keepers, Id, Out); 104 | true -> 105 | dump:delete(N, Id), 106 | delete_stuff(S, N+1, Keepers, Id, [N|Out]) 107 | end. 108 | -------------------------------------------------------------------------------- /src/stem.erl: -------------------------------------------------------------------------------- 1 | %The purpose of this file is to define stems as a data structure in ram, and give some simple functions to operate on them. 2 | 3 | -module(stem). 4 | -export([test/0,get/2,put/2,type/2,hash/2,pointers/1, 5 | types/1,hashes/1,pointer/2,new/5,add/5, 6 | new_empty/1,recover/6, empty_hashes/1, 7 | update_pointers/2, empty_tuple/0, 8 | make/3, make/2, update/3, onify2/2, 9 | put_batch/2, serialize/2, 10 | empty_trie/2]). 11 | -export_type([stem/0,types/0,empty_t/0,stem_t/0,leaf_t/0,pointers/0,empty_p/0,hashes/0,hash/0,empty_hash/0,stem_p/0,nibble/0]). 12 | -record(stem, { types = empty_tuple() :: types() 13 | , pointers = empty_tuple() :: pointers() 14 | , hashes :: hashes() 15 | }). 16 | -opaque stem() :: #stem{}. 17 | -type types() :: {t(),t(),t(),t(), 18 | t(),t(),t(),t(), 19 | t(),t(),t(),t(), 20 | t(),t(),t(),t()}. 21 | -type t() :: type(). 22 | -type type() :: empty_t() | stem_t() | leaf_t(). 23 | -type empty_t() :: 0. 24 | -type stem_t() :: 1. 25 | -type leaf_t() :: 2. 26 | -type pointers() :: {p(),p(),p(),p(), 27 | p(),p(),p(),p(), 28 | p(),p(),p(),p(), 29 | p(),p(),p(),p()}. 30 | -type p() :: pointer(). 31 | -type pointer() :: empty_p() | stem_p() | leaf:leaf_p(). 32 | -type empty_p() :: 0. 33 | -type hashes() :: {h(),h(),h(),h(), 34 | h(),h(),h(),h(), 35 | h(),h(),h(),h(), 36 | h(),h(),h(),h()}. 37 | -type h() :: hash(). 38 | -type hash() :: hash(cfg:hash_size()). 39 | -type hash(_CfgHashSizeBytes) :: non_empty_binary(). % non-empty because configured hash size positive 40 | -type empty_hash() :: hash(). 41 | -opaque stem_p() :: non_neg_integer(). 42 | -type nibble() :: 0..15. 43 | -type non_empty_binary() :: <<_:8, _:_*8>>. 44 | empty_tuple() -> {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}. 45 | -spec add(stem(), nibble(), leaf_t(), leaf:leaf_p(), hash()) -> stem(); 46 | (stem(), nibble(), stem_t(), stem_p(), hash()) -> stem(); 47 | (stem(), nibble(), empty_t(), empty_p(), empty_hash()) -> stem(). 48 | add(S, N, T, P, H) -> 49 | M = N+1, 50 | Ty = S#stem.types, 51 | Po = S#stem.pointers, 52 | Ha = S#stem.hashes, 53 | T2 = setelement(M, Ty, T), 54 | P2 = setelement(M, Po, P), 55 | H2 = setelement(M, Ha, H), 56 | #stem{types = T2, pointers = P2, hashes = H2}. 57 | -spec new_empty(cfg:cfg()) -> stem(). 58 | new_empty(CFG) -> #stem{hashes = empty_hashes(CFG)}. 59 | recover(M, T, P, H, Hashes, CFG) -> 60 | Types = onify2(Hashes, CFG), 61 | %Types = list_to_tuple(onify(tuple_to_list(Hashes), CFG)), 62 | S = #stem{hashes = Hashes, types = Types}, 63 | add(S, M, T, P, H). 64 | onify2(H, CFG) -> 65 | list_to_tuple(onify(tuple_to_list(H), CFG)). 66 | onify([], _) -> []; 67 | onify([H|T], CFG) -> 68 | HS = cfg:hash_size(CFG)*8, 69 | <> = H, 70 | case X of 71 | 0 -> [0|onify(T, CFG)]; 72 | _ -> [1|onify(T, CFG)] 73 | end. 74 | 75 | %onify([<<0:_>>|T]) -> [0|onify(T)]; 76 | %onify([_|T]) -> [1|onify(T)]. 77 | make(Hashes, ID) -> 78 | CFG = trie:cfg(ID), 79 | Types = onify2(Hashes, CFG), 80 | Pointers = empty_tuple(), 81 | make(Types, Pointers, Hashes). 82 | make(Types, Pointers, Hashes) -> 83 | #stem{types = Types, 84 | pointers = Pointers, 85 | hashes = Hashes}. 86 | new(M, T, P, H, CFG) -> 87 | %N is the nibble being pointed to. 88 | %T is the type, P is the pointer, H is the Hash 89 | S = new_empty(CFG), 90 | add(S, M, T, P, H). 91 | -spec pointers(stem()) -> pointers(). 92 | pointers(R) -> R#stem.pointers. 93 | update_pointers(Stem, NP) -> 94 | Stem#stem{pointers = NP}. 95 | -spec types(stem()) -> types(). 96 | types(R) -> R#stem.types. 97 | -spec hashes(stem()) -> hashes(). 98 | hashes(R) -> R#stem.hashes. 99 | -spec pointer(1..16, stem()) -> pointer(). 100 | pointer(N, R) -> 101 | T = pointers(R), 102 | element(N, T). 103 | -spec type(1..16, stem()) -> type(). 104 | type(N, R) -> 105 | T = types(R), 106 | element(N, T). 107 | serialize(S, CFG) -> 108 | Path = cfg:path(CFG)*8, 109 | P = S#stem.pointers, 110 | H = S#stem.hashes, 111 | T = S#stem.types, 112 | X = serialize(P, H, T, Path, 1), 113 | X. 114 | serialize(_, _, _, _, N) when N>16 -> <<>>; 115 | serialize(P, H, T, Path, N) -> 116 | P1 = element(N, P), 117 | H1 = element(N, H), 118 | T1 = element(N, T), 119 | D = serialize(P, H, T, Path, N+1), 120 | << T1:2, P1:Path, H1/binary, D/bitstring >>. 121 | deserialize(B, CFG) -> 122 | X = empty_tuple(), 123 | %deserialize(1,X,X,cfg:path(CFG)*8,hash:hash_depth()*8,X, B). 124 | HS = cfg:hash_size(CFG), 125 | deserialize(1,X,X,cfg:path(CFG)*8,HS*8,X, B). 126 | deserialize(17, T,P,_,_,H, <<>>) -> 127 | #stem{types = T, pointers = P, hashes = H}; 128 | deserialize(N, T0,P0,Path,HashDepth,H0,X) when N < 17 -> 129 | <> = X, 130 | T1 = setelement(N, T0, T), 131 | P1 = setelement(N, P0, P), 132 | H1 = setelement(N, H0, <>), 133 | deserialize(N+1, T1, P1, Path, HashDepth,H1, D). 134 | empty_hashes(CFG) -> 135 | HS = cfg:hash_size(CFG), 136 | %X = hash:hash_depth()*8, 137 | X = HS * 8, 138 | {<<0:X>>,<<0:X>>,<<0:X>>,<<0:X>>, 139 | <<0:X>>,<<0:X>>,<<0:X>>,<<0:X>>, 140 | <<0:X>>,<<0:X>>,<<0:X>>,<<0:X>>, 141 | <<0:X>>,<<0:X>>,<<0:X>>,<<0:X>>}. 142 | 143 | -spec hash(Hashes, cfg:cfg()) -> hash() when 144 | Hashes :: SerializedStem | hashes() | stem(), 145 | SerializedStem :: binary(). 146 | hash(S, CFG) when is_binary(S) -> 147 | hash(deserialize(S, CFG), CFG); 148 | hash(S, CFG) when is_tuple(S) and (size(S) == 16)-> 149 | hash2(1, S, <<>>, CFG); 150 | hash(S, CFG) -> 151 | H = S#stem.hashes, 152 | hash2(1, H, <<>>, CFG). 153 | hash2(17, _, X, CFG) -> 154 | HS = cfg:hash_size(CFG), 155 | hash:doit(X, HS); 156 | hash2(N, H, X, CFG) -> 157 | A = element(N, H), 158 | HS = cfg:hash_size(CFG), 159 | HS = size(A), 160 | hash2(N+1, H, <>, CFG). 161 | -spec put(stem(), cfg:cfg()) -> stem_p(). 162 | update(Location, Stem, CFG) -> 163 | dump:update(Location, serialize(Stem, CFG), ids:stem(CFG)). 164 | put(Stem, CFG) -> 165 | dump:put(serialize(Stem, CFG), ids:stem(CFG)). 166 | put_batch(Leaves, CFG) -> 167 | SL = serialize_stems(Leaves, CFG), 168 | dump:put_batch(SL, ids:stem(CFG)). 169 | serialize_stems([], _) -> []; 170 | serialize_stems([{N, L}| T], CFG) -> 171 | [{N, serialize(L, CFG)}|serialize_stems(T, CFG)]. 172 | -spec get(stem_p(), cfg:cfg()) -> stem(). 173 | get(Pointer, CFG) -> 174 | true = Pointer > 0, 175 | S = dump:get(Pointer, ids:stem(CFG)), 176 | deserialize(S, CFG). 177 | empty_trie(Root, CFG) -> 178 | Stem = get(Root, CFG), 179 | update_pointers(Stem, empty_tuple()). 180 | 181 | test() -> 182 | P = {6,5,4,3,7,8,9,4,5,3,2,6,7,8,3,4}, 183 | T = {0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0}, 184 | %CFG = cfg:new(1, 9, 2, trie), %path value id meta hash_size 185 | CFG = trie:cfg(trie01), 186 | %{cfg,5,2,trie01,2,32} path, value, id, meta, hash_size 187 | %596 total, average 37.25 188 | H = empty_hashes(CFG), 189 | S = #stem{types = T, pointers = P, hashes = H}, 190 | S2 = serialize(S, CFG), 191 | io:fwrite(integer_to_list(size(S2))), 192 | S = deserialize(S2, CFG), 193 | Hash = hash(S, CFG), 194 | add(S, 3, 1, 5, Hash), 195 | success. 196 | 197 | -------------------------------------------------------------------------------- /test/prop_trie_history_readable_at_any_height.erl: -------------------------------------------------------------------------------- 1 | -module(prop_trie_history_readable_at_any_height). 2 | 3 | -export([command/1, initial_state/0, next_state/3, 4 | precondition/2, postcondition/3]). 5 | -export([apply_ops_and_gc/4]). 6 | 7 | -include_lib("stdlib/include/assert.hrl"). 8 | -include_lib("proper/include/proper.hrl"). 9 | 10 | -define(DEBUG, false). 11 | 12 | -define(INITIAL_EMPTY_ROOT, 0). 13 | -define(ID, triePropTest). 14 | -define(VALUE_SIZE, 2). 15 | 16 | prop_from_clean_data_dir() -> 17 | {ok, Cwd} = file:get_cwd(), 18 | DataDir = filename:join([Cwd, "data"]), 19 | trie_test_utils:cleanup_for_clean_data_dir(DataDir), 20 | numtests( 21 | 25, %% Default is 100. 22 | ?FORALL(Cmds, commands(?MODULE), 23 | begin 24 | R1 = trie_test_utils:setup_for_clean_data_dir(DataDir), 25 | R2 = 26 | setup_trie( 27 | ?ID, 28 | %% This function test produces and keeps a 29 | %% large number of leaves and stems. It 30 | %% avoids hitting the amount passed at trie 31 | %% instantiation by choosing a high value. 32 | _Amount = 1000000), 33 | debug_fmt("Going to run commands: ~p~n", [Cmds]), 34 | {History, State, Result} = run_commands(?MODULE, Cmds), 35 | cleanup_trie(R2), 36 | trie_test_utils:cleanup_for_clean_data_dir(R1), 37 | ?WHENFAIL(io:format("Commands: ~p~nHistory: ~p~nState: ~p~nResult: ~p~n", 38 | [Cmds, History, State, Result]), 39 | aggregate(command_names(Cmds), Result =:= ok)) 40 | end)). 41 | 42 | setup_trie(Id, Amount) -> 43 | {ok, SupPid} = trie_sup:start_link( 44 | _KeyLength = 9, 45 | _Size = ?VALUE_SIZE, 46 | _ID = Id, 47 | Amount, 48 | _Meta = 2, 49 | _HashSize = 12, 50 | _Mode = hd), 51 | SupPid. 52 | 53 | cleanup_trie(SupPid) -> 54 | trie_test_utils:cleanup_alive_sup(SupPid). 55 | 56 | -type height() :: non_neg_integer(). 57 | -record(state_at_height, 58 | {root :: stem:stem_p(), 59 | kvs :: dict:dict(leaf:key(), height()) %% Record only key and height at which key was written - in order to save memory if configuring large value size. 60 | }). 61 | -record(state, {chain :: dict:dict(height(), 62 | #state_at_height{}), 63 | height :: height()}). 64 | 65 | initial_state() -> 66 | debug_fmt("In initial_state. (Pid ~p.)~n", [self()]), 67 | H = 0, 68 | StateAtHeight0 = #state_at_height{root = ?INITIAL_EMPTY_ROOT, 69 | kvs = dict:new()}, 70 | #state{chain = dict:from_list([{H, StateAtHeight0}]), 71 | height = H}. 72 | 73 | command(State) -> 74 | CurrentHeight = State#state.height, 75 | CurrentRoot = (dict:fetch(CurrentHeight, State#state.chain) 76 | )#state_at_height.root, 77 | PreviousRoots = 78 | lists:map( 79 | fun({_H, SaH}) -> SaH#state_at_height.root end, 80 | dict:to_list( 81 | dict:filter( 82 | fun(H, _) -> H =/= CurrentHeight end, 83 | State#state.chain))), 84 | oneof([{call, 85 | ?MODULE, apply_ops_and_gc, 86 | [list({oneof([put, delete]), leaf_key()}), 87 | CurrentHeight, CurrentRoot, 88 | PreviousRoots 89 | ]} 90 | ]). 91 | 92 | precondition(#state{}, {call, _Mod, _Fun, _Args}) -> 93 | true. 94 | 95 | leaf_key() -> 96 | integer(1, 100). 97 | 98 | apply_ops_and_gc(Ops, CurrentHeight, CurrentRoot, PreviousRoots) -> 99 | NewHeight = CurrentHeight + 1, 100 | NewRoot = 101 | lists:foldl( 102 | fun 103 | ({put, K}, R) -> 104 | V = leaf_value_from_key_and_height(K, NewHeight), 105 | debug_fmt("(Current height ~p, new height ~p) Going to put leaf key ~p value ~p.~n", [CurrentHeight, NewHeight, K, V]), 106 | trie:put(K, V, _M = 0, R, ?ID); 107 | ({delete, K}, R) -> 108 | debug_fmt("(Current height ~p, new height ~p) Going to delete leaf key ~p.~n", [CurrentHeight, NewHeight, K]), 109 | trie:delete(K, R, ?ID) 110 | end, 111 | CurrentRoot, 112 | Ops), 113 | ok = trie:garbage([NewRoot, CurrentRoot | PreviousRoots], ?ID), 114 | NewRoot. 115 | 116 | leaf_value_from_key_and_height(K, H) -> 117 | <<(K+H):(?VALUE_SIZE*8)>>. 118 | 119 | postcondition(State, 120 | {call, ?MODULE, apply_ops_and_gc, Args = [_, H, _, _]}, 121 | _Res = NewRoot) when H =:= State#state.height -> 122 | debug_fmt("(Height ~p) In postcondition. (Pid ~p.)~n", [State#state.height, self()]), 123 | [Ops, _, _, _] = Args, 124 | NewState = next_state_internal(State, Ops, NewRoot), 125 | is_history_readable_at_any_height(NewState). 126 | 127 | next_state(State, 128 | _Res = NewRoot, 129 | {call, ?MODULE, apply_ops_and_gc, Args}) -> 130 | debug_fmt("(Height ~p) In next_state. (Pid ~p.)~n", [State#state.height, self()]), 131 | [Ops, _, _, _] = Args, 132 | next_state_internal(State, Ops, NewRoot). 133 | 134 | next_state_internal(State, Ops, NewRoot) -> 135 | #state{chain = Chain, height = Height} = State, 136 | #state_at_height{kvs = KVs} = dict:fetch(Height, Chain), 137 | NewHeight = Height + 1, 138 | NewKVs = 139 | lists:foldl( 140 | fun 141 | ({put, K}, AccKVs) -> 142 | dict:store(K, NewHeight, AccKVs); 143 | ({delete, K}, AccKVs) -> 144 | dict:erase(K, AccKVs) 145 | end, 146 | KVs, 147 | Ops), 148 | ?assertEqual(error, dict:find(NewHeight, Chain)), 149 | NewChain = dict:store(NewHeight, 150 | #state_at_height{root = NewRoot, 151 | kvs = NewKVs}, 152 | Chain), 153 | State#state{chain = NewChain, 154 | height = NewHeight}. 155 | 156 | is_history_readable_at_any_height(State) -> 157 | lists:all( 158 | fun({H, SaH}) -> 159 | are_datum_and_proof_of_each_key_readable_and_correct( 160 | _TopHeight = State#state.height, 161 | H, SaH) 162 | end, 163 | dict:to_list(State#state.chain)). 164 | 165 | are_datum_and_proof_of_each_key_readable_and_correct(TopHeight, Height, SaH) -> 166 | #state_at_height{root = Root, kvs = KVs} = SaH, 167 | lists:all( 168 | fun({K, HeightAtWhichKeyWasPut}) -> 169 | are_datum_and_proof_of_key_readable_and_correct( 170 | TopHeight, 171 | Height, 172 | K, HeightAtWhichKeyWasPut, 173 | Root) 174 | end, 175 | dict:to_list(KVs)). 176 | 177 | are_datum_and_proof_of_key_readable_and_correct( 178 | TopHeight, 179 | Height, 180 | Key, HeightAtWhichKeyWasPut, 181 | Root) -> 182 | Value = leaf_value_from_key_and_height(Key, HeightAtWhichKeyWasPut), 183 | {RootHash, Leaf, Proof} = trie:get(Key, Root, ?ID), 184 | DatumKey = leaf:key(Leaf), 185 | IsDatumKeyOk = (DatumKey =:= Key), 186 | case IsDatumKeyOk of 187 | true -> ok; 188 | false -> fmt("(Top height ~p, height ~p) Bad datum key: expected ~p, actual ~p.~n", [TopHeight, Height, Key, DatumKey]) 189 | end, 190 | DatumValue = leaf:value(Leaf), 191 | IsDatumValueOk = (DatumValue =:= Value), 192 | case IsDatumValueOk of 193 | true -> ok; 194 | false -> fmt("(Top height ~p, height ~p) Bad datum value for key ~p: expected ~p, actual ~p.~n", [TopHeight, Height, Key, Value, DatumValue]) 195 | end, 196 | IsProofOk = verify:proof(RootHash, Leaf, Proof, trie:cfg(?ID)), 197 | case IsProofOk of 198 | true -> ok; 199 | false -> fmt("(Top height ~p, height ~p) Bad proof for key ~p: root hash ~p, leaf ~p, proof ~p.~n", [TopHeight, Height, Key, RootHash, Leaf, Proof]) 200 | end, 201 | IsDatumKeyOk and IsDatumValueOk and IsProofOk. 202 | 203 | debug_fmt(Fmt, Data) when ?DEBUG =:= true -> 204 | fmt(Fmt, Data); 205 | debug_fmt(_, _) when ?DEBUG =:= false -> 206 | ok. 207 | 208 | fmt(Fmt, Data) -> 209 | io:format(Fmt, Data). 210 | -------------------------------------------------------------------------------- /src/trie.erl: -------------------------------------------------------------------------------- 1 | -module(trie). 2 | -behaviour(gen_server). 3 | -export([start_link/1,code_change/3,handle_call/3,handle_cast/2,handle_info/2,init/1,terminate/2, root_hash/2,cfg/1,get/3,put/5,put_batch/3,delete/3,%garbage/2,garbage_leaves/2, 4 | get_all/2,new_trie/2, restore/5,restore/7, 5 | empty/1, 6 | quick_save/1, %the copy of the ets currently in ram, it uses this to update the copy stored on the hard drive. 7 | reload_ets/1, %grabs the copy of the ets from the hard drive, loads it into ram 8 | clean_ets/2, %deletes everything from the merkel tree database, except for what can be proved from this single state root. 9 | prune/3, garbage/3]). 10 | init(CFG) -> 11 | process_flag(trap_exit, true), 12 | %ID = cfg:id(CFG), 13 | Empty = stem:put(stem:new_empty(CFG), CFG), 14 | %CFG2 = CFG#cfg{empty = Empty}, 15 | CFG2 = cfg:set_empty(CFG, Empty), 16 | {ok, CFG2}. 17 | start_link(CFG) -> %keylength, or M is the size outputed by hash:doit(_). 18 | gen_server:start_link({global, ids:main(CFG)}, ?MODULE, CFG, []). 19 | code_change(_OldVsn, State, _Extra) -> {ok, State}. 20 | terminate(_, CFG) -> 21 | io:fwrite("tree "), 22 | ID = cfg:id(CFG), 23 | io:fwrite(ID), 24 | io:fwrite(" died \n"), 25 | ok. 26 | handle_info(_, X) -> {noreply, X}. 27 | handle_cast(reload_ets, CFG) -> 28 | A3 = ids:leaf(CFG), 29 | A4 = ids:stem(CFG), 30 | dump:reload_ets(A3), 31 | dump:reload_ets(A4), 32 | Empty = stem:put(stem:new_empty(CFG), CFG), 33 | CFG2 = cfg:set_empty(CFG, Empty), 34 | {noreply, CFG2}; 35 | handle_cast(_, X) -> {noreply, X}. 36 | handle_call(quick_save, _, CFG) -> 37 | A3 = ids:leaf(CFG), 38 | A4 = ids:stem(CFG), 39 | dump:quick_save(A3), 40 | dump:quick_save(A4), 41 | {reply, ok, CFG}; 42 | handle_call({clean_ets, Pointer}, _, CFG) -> 43 | %A3 = ids:leaf(CFG), 44 | %A4 = ids:stem(CFG), 45 | LID = ids:leaf(CFG), 46 | SID = ids:stem(CFG), 47 | TempLID = list_to_atom(atom_to_list(LID) ++ "_temp"), 48 | TempSID = list_to_atom(atom_to_list(SID) ++ "_temp"), 49 | case ets:info(TempLID) of 50 | undefined -> 51 | ets:new(TempLID, [set, named_table, {write_concurrency, false}, compressed]); 52 | _ -> ets:delete_all_objects(TempLID) 53 | end, 54 | case ets:info(TempSID) of 55 | undefined -> 56 | ets:new(TempSID, [set, named_table, {write_concurrency, false}, compressed]); 57 | _ -> ets:delete_all_objects(TempSID) 58 | end, 59 | 60 | %depth first search over all stems and leaves from Pointer, store all of them in the new temp ets databases. 61 | clean_ets_internal(Pointer, CFG, TempSID, TempLID), 62 | dump:delete_all(SID), 63 | dump:delete_all(LID), 64 | 65 | ets:foldr(fun({Pt, Stem}, _) -> 66 | dump:update(Pt, Stem, SID) 67 | end, 0, TempSID), 68 | ets:foldr(fun({Pt, Leaf}, _) -> 69 | dump:update(Pt, Leaf, LID) 70 | end, 0, TempLID), 71 | 72 | Empty = stem:put(stem:new_empty(CFG), CFG), 73 | CFG2 = cfg:set_empty(CFG, Empty), 74 | {reply, ok, CFG2}; 75 | handle_call({garbage, NewRoot, OldRoot}, _From, CFG) ->%prune new 76 | X = prune:garbage(NewRoot, OldRoot, CFG), 77 | {reply, X, CFG}; 78 | %handle_call({prune, OldRoot, NewRoot}, _From, CFG) ->%prune old 79 | % 1=2, 80 | % X = prune:stem(OldRoot, NewRoot, CFG), 81 | % {reply, X, CFG}; 82 | handle_call({delete, Key, Root}, _From, CFG) -> 83 | valid_key(Key), 84 | NewRoot = delete:delete(Key, Root, CFG), 85 | {reply, NewRoot, CFG}; 86 | handle_call({restore, Key, Value, Meta, Hash, Proof, Root}, _From, CFG) -> 87 | valid_key(Key), 88 | Leaf = leaf:new(Key, Value, Meta, CFG), 89 | {Hash, NewRoot, _} = store:restore(Leaf, Hash, Proof, Root, CFG), 90 | {reply, NewRoot, CFG}; 91 | handle_call({put, Key, Value, Meta, Root}, _From, CFG) -> 92 | valid_key(Key), 93 | Leaf = leaf:new(Key, Value, Meta, CFG), 94 | {_, NewRoot, _} = store:store(Leaf, Root, CFG), 95 | {reply, NewRoot, CFG}; 96 | handle_call({put_batch, Leaves, Root}, _From, CFG) -> 97 | {Hash, NewRoot} = store:batch(Leaves, Root, CFG), 98 | {reply, NewRoot, CFG}; 99 | handle_call({get, Key, RootPointer}, _From, CFG) -> 100 | valid_key(Key), 101 | P = leaf:path_maker(Key, CFG), 102 | {RootHash, L, Proof} = get:get(P, RootPointer, CFG), 103 | L2 = if 104 | L == empty -> empty; 105 | L == unknown -> unknown; 106 | true -> 107 | Key2 = leaf:key(L), 108 | if 109 | Key == Key2 -> L; 110 | true -> empty 111 | end 112 | end, 113 | {reply, {RootHash, L2, Proof}, CFG}; 114 | handle_call({get_all, Root}, _From, CFG) -> 115 | X = get_all_internal(Root, CFG), 116 | {reply, X, CFG}; 117 | handle_call(empty, _, CFG) -> 118 | {reply, cfg:empty(CFG), CFG}; 119 | handle_call({new_trie, RootStem}, _From, CFG) -> 120 | %Stem = stem:empty_trie(Root, CFG), 121 | Stem = stem:update_pointers(RootStem, stem:empty_tuple()), 122 | X = stem:put(Stem, CFG), 123 | {reply, X, CFG}; 124 | handle_call({root_hash, RootPointer}, _From, CFG) -> 125 | S = stem:get(RootPointer, CFG), 126 | H = stem:hash(S, CFG), 127 | {reply, H, CFG}; 128 | handle_call(cfg, _From, CFG) -> 129 | {reply, CFG, CFG}. 130 | 131 | save_table(ID, Loc) -> 132 | case ets:tab2file(ID, Loc, [{sync, true}]) of 133 | ok -> ok; 134 | {error, R} -> 135 | save_table(ID, Loc) 136 | end. 137 | 138 | cfg(ID) when is_atom(ID) -> 139 | gen_server:call({global, ids:main_id(ID)}, cfg). 140 | new_trie(ID, RootStem) when is_atom(ID) -> 141 | gen_server:call({global, ids:main_id(ID)}, {new_trie, RootStem}). 142 | clean_ets(ID, Pointer) -> 143 | %deletes everything from the merkel tree database, except for what can be proved from this single state root. 144 | %used for loading a checkpoint. 145 | gen_server:call({global, ids:main_id(ID)}, {clean_ets, Pointer}). 146 | 147 | reload_ets(ID) -> 148 | %reloads the ram databases from the hard drive copy. 149 | gen_server:cast({global, ids:main_id(ID)}, reload_ets). 150 | quick_save(ID) -> 151 | gen_server:call({global, ids:main_id(ID)}, quick_save). 152 | empty(ID) when is_atom(ID) -> 153 | gen_server:call({global, ids:main_id(ID)}, empty). 154 | -spec root_hash(atom(), stem:stem_p()) -> stem:hash(). 155 | root_hash(ID, RootPointer) when (is_atom(ID) and is_integer(RootPointer))-> 156 | gen_server:call({global, ids:main_id(ID)}, {root_hash, RootPointer}). 157 | -spec put(leaf:key(), leaf:value(), leaf:meta(), stem:stem_p(), atom()) -> 158 | stem:stem_p(). 159 | restore(Leaf, Hash, Path, Root, ID) -> 160 | restore(leaf:key(Leaf), leaf:value(Leaf), leaf:meta(Leaf), 161 | Hash, Path, Root, ID). 162 | restore(Key, Value, Meta, Hash, Path, Root, ID) -> 163 | gen_server:call({global, ids:main_id(ID)}, {restore, Key, Value, Meta, Hash, Path, Root}). 164 | put_batch([], Root, _) -> Root; 165 | put_batch(Leaves, Root, ID) -> 166 | gen_server:call({global, ids:main_id(ID)}, {put_batch, Leaves, Root}). 167 | put(Key, Value, Meta, Root, ID) -> 168 | gen_server:call({global, ids:main_id(ID)}, {put, Key, Value, Meta, Root}). 169 | -spec get(leaf:key(), stem:stem_p(), atom()) -> 170 | {stem:hash(), empty | leaf:leaf(), get:proof()}. 171 | get(Key, Root, ID) -> gen_server:call({global, ids:main_id(ID)}, {get, Key, Root}). 172 | -spec get_all(stem:stem_p(), atom()) -> [leaf:leaf()]. 173 | get_all(Root, ID) -> gen_server:call({global, ids:main_id(ID)}, {get_all, Root}). 174 | -spec delete(leaf:key(), stem:stem_p(), atom()) -> stem:stem_p(). 175 | delete(Key, Root, ID) -> gen_server:call({global, ids:main_id(ID)}, {delete, Key, Root}). 176 | garbage(NewRoot, OldRoot, ID) ->%removes new 177 | gen_server:call({global, ids:main_id(ID)}, {garbage, NewRoot, OldRoot}). 178 | prune(OldRoot, NewRoot, ID) ->%removes old 179 | gen_server:call({global, ids:main_id(ID)}, {prune, OldRoot, NewRoot}). 180 | 181 | get_all_internal(Root, CFG) -> 182 | S = stem:get(Root, CFG), 183 | P = tuple_to_list(stem:pointers(S)), 184 | T = tuple_to_list(stem:types(S)), 185 | get_all_internal2(P, T, CFG). 186 | get_all_internal2([], [], _) -> []; 187 | get_all_internal2([A|AT], [T|TT], CFG) -> 188 | B = case T of 189 | 0 -> [];%empty 190 | 1 -> get_all_internal(A, CFG);%another stem 191 | 2 -> [leaf:get(A, CFG)]%a leaf 192 | end, 193 | B++get_all_internal2(AT, TT, CFG). 194 | clean_ets_internal(Pointer, CFG, SID, LID) -> 195 | S = stem:get(Pointer, CFG), 196 | P = tuple_to_list(stem:pointers(S)), 197 | T = tuple_to_list(stem:types(S)), 198 | H = tuple_to_list(stem:hashes(S)), 199 | clean_ets_internal2(P, T, H, CFG, SID, LID), 200 | SS = stem:serialize(S, CFG), 201 | ets:insert(SID, {Pointer, SS}), 202 | stem:hash(S, CFG). 203 | 204 | clean_ets_internal2([], [], _, _, _, _) -> []; 205 | clean_ets_internal2([Pointer|PT], [Type|TT], [Hash|HT], CFG, SID, LID) -> 206 | case Type of 207 | 0 -> %empty 208 | Hash = <<0:256>>; 209 | 1 -> %another stem 210 | Hash = clean_ets_internal(Pointer, CFG, SID, LID); 211 | 2 -> %a leaf 212 | Leaf = leaf:get(Pointer, CFG), 213 | Hash = leaf:hash(Leaf, CFG), 214 | SL = leaf:serialize(Leaf, CFG), 215 | ets:insert(LID, {Pointer, SL}) 216 | end, 217 | clean_ets_internal2(PT, TT, HT, CFG, SID, LID). 218 | 219 | valid_key(Key) -> 220 | true = is_integer(Key), 221 | true = Key > 0. 222 | 223 | -------------------------------------------------------------------------------- /test/trie_tests.erl: -------------------------------------------------------------------------------- 1 | -module(trie_tests). 2 | -include_lib("eunit/include/eunit.hrl"). 3 | 4 | -define(ID, trieUnitTest). 5 | 6 | api_smoke_test_() -> 7 | {foreach, 8 | fun() -> 9 | ?debugFmt("~nCurrent working directory: ~p~n", 10 | [begin {ok, Cwd} = file:get_cwd(), Cwd end]), 11 | {ok, SupPid} = 12 | trie_sup:start_link( 13 | _KeyLength = 9, 14 | _Size = 2, 15 | _ID = ?ID, 16 | _Amount = 1000000, 17 | _Meta = 2, 18 | _HashSize = 12, 19 | _Mode = hd), 20 | ?debugFmt("~nTrie sup pid: ~p~n", [SupPid]), 21 | SupPid 22 | end, 23 | fun(SupPid) -> 24 | ?assert(is_process_alive(SupPid)), 25 | trie_test_utils:cleanup_alive_sup(SupPid), 26 | ?assertNot(is_process_alive(SupPid)), 27 | ok 28 | end, 29 | [ {"Initialization produces empty trie", 30 | ?_test(assert_trie_empty(_Root = 0, ?ID))} 31 | , {"Put - happy path", fun put_happy_path/0} 32 | , {"Get - case empty", fun get_empty/0} 33 | , {"Delete - happy path", fun delete_happy_path/0} 34 | , {"Garbage collection keeping a root (i.e. a stem)", fun gc_keeping_root/0} 35 | , {"Garbage collection keeping a leaf", fun gc_keeping_leaf/0} 36 | ] 37 | }. 38 | 39 | root_hash_test_() -> 40 | {foreach, 41 | fun() -> 42 | ?debugFmt("~nCurrent working directory: ~p~n", 43 | [begin {ok, Cwd} = file:get_cwd(), Cwd end]), 44 | {ok, SupPid} = 45 | trie_sup:start_link( 46 | _KeyLength = 9, 47 | _Size = 2, 48 | _ID = ?ID, 49 | _Amount = 1000000, 50 | _Meta = 2, 51 | _HashSize = 12, 52 | _Mode = hd), 53 | ?debugFmt("~nTrie sup pid: ~p~n", [SupPid]), 54 | assert_trie_empty(0, ?ID), 55 | SupPid 56 | end, 57 | fun(SupPid) -> 58 | ?assert(is_process_alive(SupPid)), 59 | trie_test_utils:cleanup_alive_sup(SupPid), 60 | ?assertNot(is_process_alive(SupPid)), 61 | ok 62 | end, 63 | [ {"Hash of root of empty tree", 64 | fun() -> 65 | Root = 0, 66 | RH = trie:root_hash(?ID, Root), 67 | ?assertMatch({RH, empty, _}, trie:get(_Key = 1, Root, ?ID)), 68 | ok 69 | end} 70 | , {"Hash of tree root changes with tree content", 71 | fun() -> 72 | Root = 0, 73 | RH = trie:root_hash(?ID, Root), 74 | Key = 1, 75 | Root2 = trie:put(Key, _V = <<1,1>>, _Meta = 0, Root, ?ID), 76 | RH2 = trie:root_hash(?ID, Root2), 77 | ?assertMatch({RH2, _, _}, trie:get(Key, Root2, ?ID)), 78 | ?assertNotEqual(RH, RH2), 79 | ok 80 | end} 81 | , {"Hash of tree root changes with leaf key", 82 | fun() -> 83 | Root = 0, 84 | K1 = 1, 85 | K2 = 2, 86 | V = <<1,1>>, 87 | Meta = 0, 88 | Root2A = trie:put(K1, V, Meta, Root, ?ID), 89 | Root2B = trie:put(K2, V, Meta, Root, ?ID), 90 | ?assertNotEqual(trie:root_hash(?ID, Root2A), 91 | trie:root_hash(?ID, Root2B)), 92 | ok 93 | end} 94 | , {"Hash of tree root changes with leaf value", 95 | fun() -> 96 | Root = 0, 97 | K = 1, 98 | V1 = <<1,1>>, 99 | V2 = <<1,2>>, 100 | Meta = 0, 101 | Root2A = trie:put(K, V1, Meta, Root, ?ID), 102 | Root2B = trie:put(K, V2, Meta, Root, ?ID), 103 | ?assertNotEqual(trie:root_hash(?ID, Root2A), 104 | trie:root_hash(?ID, Root2B)), 105 | ok 106 | end} 107 | , {"Hash of tree root does not change with leaf meta", 108 | fun() -> 109 | Root = 0, 110 | K = 1, 111 | V = <<1,1>>, 112 | Meta1 = 0, 113 | Meta2 = 1, 114 | Root2A = trie:put(K, V, Meta1, Root, ?ID), 115 | Root2B = trie:put(K, V, Meta2, Root, ?ID), 116 | ?assertEqual(trie:root_hash(?ID, Root2A), 117 | trie:root_hash(?ID, Root2B)), 118 | ok 119 | end} 120 | , {"Hash of tree root depends on tree content rather than order of operations - case empty tree", 121 | fun() -> 122 | Root = 0, 123 | Key = 1, 124 | Root2 = trie:put(Key, _V = <<1,1>>, _Meta = 0, Root, ?ID), 125 | Root3 = trie:delete(Key, Root2, ?ID), 126 | ?assertEqual(trie:root_hash(?ID, Root), 127 | trie:root_hash(?ID, Root3)), 128 | ok 129 | end} 130 | , {"Hash of tree root depends on tree content rather than order of operations - case non-empty tree", 131 | fun() -> 132 | Root = 0, 133 | K1 = 1, 134 | K2 = 2, 135 | Meta = 0, 136 | Root2 = trie:put(K1, _V1 = <<1,1>>, Meta, Root, ?ID), 137 | Root3 = trie:put(K2, _V2 = <<1,2>>, Meta, Root2, ?ID), 138 | Root4 = trie:delete(K2, Root3, ?ID), 139 | ?assertEqual(trie:root_hash(?ID, Root2), 140 | trie:root_hash(?ID, Root4)), 141 | ok 142 | end} 143 | ] 144 | }. 145 | 146 | key_range_good_test_() -> 147 | {foreach, 148 | fun() -> 149 | ?debugFmt("~nCurrent working directory: ~p~n", 150 | [begin {ok, Cwd} = file:get_cwd(), Cwd end]), 151 | {ok, SupPid} = 152 | trie_sup:start_link( 153 | _KeyLength = 9, 154 | _Size = 2, 155 | _ID = ?ID, 156 | _Amount = 1000000, 157 | _Meta = 2, 158 | _HashSize = 12, 159 | _Mode = hd), 160 | ?debugFmt("~nTrie sup pid: ~p~n", [SupPid]), 161 | assert_trie_empty(0, ?ID), 162 | SupPid 163 | end, 164 | fun(SupPid) -> 165 | ?assert(is_process_alive(SupPid)), 166 | trie_test_utils:cleanup_alive_sup(SupPid), 167 | ?assertNot(is_process_alive(SupPid)), 168 | ok 169 | end, 170 | [ {"Put key range - case min key", 171 | ?_test(put_and_assert_key(_Key = 0, _Root = 0, ?ID))} 172 | , {"Put key range - case max key", 173 | ?_test(put_and_assert_key( 174 | _Key = (1 bsl (cfg:path(trie:cfg(?ID)) * 8)) - 1, 175 | _Root = 0, 176 | ?ID))} 177 | ] 178 | }. 179 | 180 | key_range_bad_test_() -> 181 | {foreach, 182 | fun() -> 183 | ?debugFmt("~nCurrent working directory: ~p~n", 184 | [begin {ok, Cwd} = file:get_cwd(), Cwd end]), 185 | {ok, SupPid} = 186 | trie_sup:start_link( 187 | _KeyLength = 9, 188 | _Size = 2, 189 | _ID = ?ID, 190 | _Amount = 1000000, 191 | _Meta = 2, 192 | _HashSize = 12, 193 | _Mode = hd), 194 | ?debugFmt("~nTrie sup pid: ~p~n", [SupPid]), 195 | SupMonRef = erlang:monitor(process, SupPid), 196 | unlink(SupPid), 197 | assert_trie_empty(0, ?ID), 198 | {SupPid, SupMonRef} 199 | end, 200 | fun({SupPid, SupMonRef}) -> 201 | receive 202 | {'DOWN', SupMonRef, process, SupPid, Reason} -> 203 | ?debugFmt("~nTrie sup ~p exited for reason ~p~n", 204 | [SupPid, Reason]), 205 | ok 206 | end, 207 | ?assertNot(is_process_alive(SupPid)), 208 | ok 209 | end, 210 | [ {"Put key range - case negative key", 211 | ?_assertException( 212 | _, _, 213 | trie:put(_Key = -1, _Value = <<1,1>>, _Meta = 0, _Root = 0, ?ID))} 214 | , {"Put key range - case too big key", 215 | ?_assertException( 216 | _, _, 217 | trie:put(_Key = 1 bsl (cfg:path(trie:cfg(?ID)) * 8), 218 | _Value = <<1,1>>, _Meta = 0, _Root = 0, ?ID))} 219 | ] 220 | }. 221 | 222 | delete_unexistent_key_test_() -> 223 | {foreach, 224 | fun() -> 225 | ?debugFmt("~nCurrent working directory: ~p~n", 226 | [begin {ok, Cwd} = file:get_cwd(), Cwd end]), 227 | {ok, SupPid} = 228 | trie_sup:start_link( 229 | _KeyLength = 9, 230 | _Size = 2, 231 | _ID = ?ID, 232 | _Amount = 1000000, 233 | _Meta = 2, 234 | _HashSize = 12, 235 | _Mode = hd), 236 | ?debugFmt("~nTrie sup pid: ~p~n", [SupPid]), 237 | assert_trie_empty(0, ?ID), 238 | SupPid 239 | end, 240 | fun(SupPid) -> 241 | ?assert(is_process_alive(SupPid)), 242 | trie_test_utils:cleanup_alive_sup(SupPid), 243 | ?assertNot(is_process_alive(SupPid)), 244 | ok 245 | end, 246 | [ {"Delete from empty tree", 247 | ?_test(trie:delete(_Key = 0, _Root = 0, ?ID))} 248 | , {"Delete unexistent key from non-empty tree", 249 | fun() -> 250 | %% Test case identified by property-based testing. 251 | K1 = 90, 252 | K2 = 26, 253 | %% The keys share the first nibble of the path. 254 | ?assertEqual(hd(leaf:path_maker(K1, trie:cfg(?ID))), 255 | hd(leaf:path_maker(K2, trie:cfg(?ID)))), 256 | trie:delete( 257 | K2, 258 | trie:put(K1, _V = <<1,1>>, _Meta = 1, _Root = 0, ?ID), 259 | ?ID), 260 | ok 261 | end} 262 | ] 263 | }. 264 | 265 | gc_test_() -> 266 | {foreach, 267 | fun() -> 268 | ?debugFmt("~nCurrent working directory: ~p~n", 269 | [begin {ok, Cwd} = file:get_cwd(), Cwd end]), 270 | {ok, SupPid} = 271 | trie_sup:start_link( 272 | _KeyLength = 9, 273 | _Size = 2, 274 | _ID = ?ID, 275 | _Amount = 1000000, 276 | _Meta = 2, 277 | _HashSize = 12, 278 | _Mode = hd), 279 | ?debugFmt("~nTrie sup pid: ~p~n", [SupPid]), 280 | assert_trie_empty(0, ?ID), 281 | SupPid 282 | end, 283 | fun(SupPid) -> 284 | ?assert(is_process_alive(SupPid)), 285 | trie_test_utils:cleanup_alive_sup(SupPid), 286 | ?assertNot(is_process_alive(SupPid)), 287 | ok 288 | end, 289 | [ {"Garbage collection keeping roots (i.e. stems) deletes nothing if specified stem is root", 290 | fun() -> 291 | Root = 0, 292 | Key = 1, 293 | V = <<1,1>>, 294 | Meta = 0, 295 | Root2 = trie:put(Key, V, Meta, Root, ?ID), 296 | [Leaf] = trie:get_all(Root2, ?ID), 297 | ?assertEqual(ok, trie:garbage([Root2], ?ID)), 298 | ?assertEqual([Leaf], trie:get_all(Root2, ?ID)), 299 | ok 300 | end} 301 | , {"Garbage collection keeping roots (i.e. stems) deletes things that are not descendent of specified stems", 302 | fun() -> 303 | Root = 0, 304 | K1 = 1, 305 | V1 = <<1,1>>, 306 | K2 = 2, 307 | V2 = <<1,2>>, 308 | Meta = 0, 309 | Root2A = trie:put(K1, V1, Meta, Root, ?ID), 310 | Root2B = trie:put(K2, V2, Meta, Root, ?ID), 311 | {_, L2, _} = trie:get(K2, Root2B, ?ID), 312 | ?assertEqual(V2, leaf:value(L2)), 313 | {_, L1, _} = trie:get(K1, Root2A, ?ID), 314 | ?assertEqual(V1, leaf:value(L1)), 315 | ?assertEqual(ok, trie:garbage([Root2B], ?ID)), 316 | ?assertMatch({_, empty, _}, trie:get(K1, Root2A, ?ID)), 317 | ?assertEqual([], trie:get_all(Root2A, ?ID)), 318 | ?assertMatch({_, L2, _}, trie:get(K2, Root2B, ?ID)), 319 | ?assertMatch([_], trie:get_all(Root2B, ?ID)), 320 | ok 321 | end} 322 | ] 323 | }. 324 | 325 | assert_trie_empty(Root = 0, Id) -> 326 | ?assertEqual([], trie:get_all(Root, Id)), 327 | ok. 328 | 329 | put_and_assert_key(Key, Root, Id) -> 330 | V = <<1,1>>, 331 | Meta = 0, 332 | Root2 = trie:put(Key, V, Meta, Root, Id), 333 | {_, Leaf, _} = trie:get(Key, Root2, Id), 334 | ?assertEqual(Key, leaf:key(Leaf)), 335 | ok. 336 | 337 | put_happy_path() -> 338 | Root = 0, 339 | assert_trie_empty(Root, ?ID), 340 | Key = 1, 341 | V = <<1,1>>, 342 | Meta = 0, 343 | Root2 = trie:put(Key, V, Meta, Root, ?ID), 344 | {Root2Hash, Leaf, Proof} = trie:get(Key, Root2, ?ID), 345 | ?assertEqual(V, leaf:value(Leaf)), 346 | ?assert(verify:proof(Root2Hash, Leaf, Proof, trie:cfg(?ID))), 347 | ?assertEqual([Leaf], trie:get_all(Root2, ?ID)), 348 | ok. 349 | 350 | get_empty() -> 351 | Root = 0, 352 | assert_trie_empty(Root, ?ID), 353 | ?assertMatch({_, empty, _}, trie:get(_Key = 1, Root, ?ID)), 354 | ok. 355 | 356 | delete_happy_path() -> 357 | Root = 0, 358 | assert_trie_empty(Root, ?ID), 359 | Key = 1, 360 | V = <<1,1>>, 361 | Meta = 0, 362 | Root2 = trie:put(Key, V, Meta, Root, ?ID), 363 | {_, Leaf, _} = trie:get(Key, Root2, ?ID), 364 | ?assertEqual(V, leaf:value(Leaf)), 365 | Root3 = trie:delete(Key, Root2, ?ID), 366 | ?assertMatch({_, empty, _}, trie:get(Key, Root3, ?ID)), 367 | ?assertEqual([], trie:get_all(Root3, ?ID)), 368 | ok. 369 | 370 | gc_keeping_root() -> 371 | Root = 0, 372 | assert_trie_empty(Root, ?ID), 373 | Key = 1, 374 | V = <<1,1>>, 375 | Meta = 0, 376 | Root2 = trie:put(Key, V, Meta, Root, ?ID), 377 | ?assertEqual(ok, trie:garbage([Root2], ?ID)), 378 | {Root2Hash, Leaf, Proof} = trie:get(Key, Root2, ?ID), 379 | ?assert(verify:proof(Root2Hash, Leaf, Proof, trie:cfg(?ID))), 380 | ok. 381 | 382 | gc_keeping_leaf() -> 383 | Root = 0, 384 | assert_trie_empty(Root, ?ID), 385 | Cfg = trie:cfg(?ID), 386 | K1 = 1, 387 | V1 = <<1,1>>, 388 | K2 = 2, 389 | ?assert(K2 > K1), 390 | V2 = <<1,2>>, 391 | Meta = 0, 392 | Root2 = trie:put(K1, V1, Meta, Root, ?ID), 393 | Root3 = trie:put(K2, V2, Meta, Root2, ?ID), 394 | {_, Leaf2, _} = trie:get(K2, Root3, ?ID), 395 | ?assertEqual(ok, trie:garbage_leaves([{leaf:path(Leaf2, Cfg), Root3}], ?ID)), 396 | {RootHash, Leaf2, Proof} = trie:get(K2, Root3, ?ID), 397 | ?assert(verify:proof(RootHash, Leaf2, Proof, Cfg)), 398 | ok. 399 | -------------------------------------------------------------------------------- /src/store.erl: -------------------------------------------------------------------------------- 1 | -module(store). 2 | -export([store/3, restore/5, get_branch/5, store_branch/6, batch/3]). 3 | -export_type([branch/0, nonempty_branch/0]). 4 | 5 | -type branch() :: [stem:stem()]. % first element is most distant from root i.e. closest to leaf (if any) 6 | -type nonempty_branch() :: [stem:stem(), ...]. 7 | 8 | restore(Leaf, Hash, Proof, Root, CFG) -> %this restores information to the merkle trie that had been garbage collected. 9 | %We take the existing branch, and the proof2branch, and mix them together. We want the new branch to contain pointers to the existing data. 10 | true = verify:proof(Hash, Leaf, Proof, CFG), 11 | HSE = cfg:hash_size(CFG) * 8, 12 | B1 = (leaf:value(Leaf) == empty), 13 | B2 = (is_binary(hd(Proof))),%A leaf is stored with the proof. 14 | {LPointer, LH, Path, Type, Proof2} = 15 | if 16 | B1 and B2 -> 17 | L2 = leaf:deserialize(hd(Proof), CFG), 18 | {leaf:put(L2, CFG), 19 | leaf:hash(L2, CFG), 20 | leaf:path(L2, CFG), 21 | 2, 22 | tl(Proof)}; 23 | B1 -> 24 | {0, <<0:HSE>>, leaf:path(Leaf, CFG), 0, Proof}; 25 | not B2 -> 26 | {leaf:put(Leaf, CFG), 27 | leaf:hash(Leaf, CFG), 28 | leaf:path(Leaf, CFG), 29 | 2, 30 | Proof} 31 | end, 32 | ReversePath = lists:reverse(first_n(length(Proof2), Path)), 33 | Branch = proof2branch(Proof2, Type, LPointer, LH, 34 | ReversePath, 35 | CFG),%branch only proves one thing. We want to combine. 36 | Branch2 = get_branch(Path, 0, Root, [], CFG),%branch2 proves everything else. 37 | Branch3 = combine_branches(Branch, Branch2), 38 | %Key = leaf:key(Leaf), 39 | store_branch(Branch3, Path, Type, LPointer, LH, CFG). 40 | first_n(N, [H|T]) when N > 0 -> 41 | [H|first_n(N-1, T)]; 42 | first_n(_, _) -> []. 43 | combine_branches([], []) -> []; 44 | combine_branches(A, B) when length(A) > length(B) -> 45 | [hd(A)|combine_branches(tl(A), B)]; 46 | combine_branches(_, B) ->%The second one has many pointers we care about. The first one has 1 leaf-pointer we care about. 47 | B. 48 | proof2branch([],_,_,_, _, _) -> []; 49 | proof2branch([H|T], _, _, Hash, _, CFG) when is_binary(H) -> 50 | L = leaf:deserialize(H, CFG), 51 | Path = leaf:path(L, CFG), 52 | Pointer = leaf:put(L, CFG), 53 | proof2branch(T, 2, Pointer, Hash, Path, CFG); 54 | proof2branch([H|T], Type, Pointer, Hash, Path, CFG) -> 55 | [<> | NewPath] = Path, 56 | S = stem:recover(Nibble, Type, Pointer, Hash, H, CFG), 57 | NewPointer = stem:put(S, CFG), 58 | NewHash = stem:hash(S, CFG), 59 | [S|proof2branch(T, 1, NewPointer, NewHash, NewPath, CFG)]. 60 | 61 | -spec store(leaf:leaf(), stem:stem_p(), cfg:cfg()) -> 62 | {RootHash, RootPointer, get:proof()} 63 | when RootHash :: stem:hash(), 64 | RootPointer :: stem:stem_p(). 65 | max_list([X]) -> X; 66 | max_list([A|[B|T]]) -> 67 | max_list([max(A, B)|T]). 68 | batch(Leaves, Root, CFG) -> 69 | %first we should sort the leaves by path. This way if any of the proofs can be combines, they will be adjacent in the list. So we can combine all the proofs by comparing pairs of adjacent proofs. 70 | Leaves2 = sort_by_path(Leaves, CFG), 71 | case cfg:mode(CFG) of 72 | ram -> 73 | Top = dump:highest(ids:leaf(CFG)), 74 | {BranchData0, ToStore} = store_batch_helper_ram(Leaves2, CFG, [], Root, Top, []),%store leaves 75 | leaf:put_batch(ToStore, CFG), 76 | BranchData = extra_stem(BranchData0, CFG), 77 | BStart = max_list( 78 | lists:map( 79 | fun(Branch) -> 80 | {_,_,_,B,_} = Branch, 81 | length(B) 82 | end, BranchData)), 83 | StemTop = dump:highest(ids:stem(CFG)), 84 | {FHash, FLoc, SToStore} = batch2_ram(BStart, BranchData, CFG, StemTop, []), 85 | %io:fwrite("store batch ram mode 2\n"), 86 | stem:put_batch(SToStore, CFG),%store stems 87 | %io:fwrite("store batch ram mode 3\n"), 88 | {FHash, FLoc}; 89 | 90 | hd -> 91 | BranchData0 = store_batch_helper(Leaves2, CFG, [], Root), 92 | BranchData = extra_stem(BranchData0, CFG), 93 | %leaf-pointer, leaf-hash, leaf-path, branch, type 94 | BStart = max_list( 95 | lists:map( 96 | fun(Branch) -> 97 | {_, _, _, B, _} = Branch, 98 | length(B) 99 | end, BranchData)), 100 | batch2(BStart, BranchData, CFG) 101 | end. 102 | extra_stem([], _) -> []; 103 | extra_stem([X], _) -> [X]; 104 | extra_stem([A|[B|T]], CFG) -> 105 | {Pa,Ha,Aa,Ra,Tya} = A, 106 | {Pb,Hb,Ab,Rb,Tyb} = B, 107 | LRA = length(Ra), 108 | LRB = length(Rb), 109 | S = min(LRA, LRB), 110 | {Pta, _} = lists:split(S, Aa), 111 | {Ptb, _} = lists:split(S, Ab), 112 | Bool = Pta == Ptb, 113 | if 114 | Bool -> %add extra stem to the one(s) that have only S stems. recurse to check if they still match. 115 | Ra2 = if 116 | LRA == S -> empty_stems(1, CFG) ++ Ra;%add extra stem 117 | true -> Ra 118 | end, 119 | Rb2 = if 120 | LRB == S -> empty_stems(1, CFG) ++ Rb;%add extra stem 121 | true -> Rb 122 | end, 123 | A2 = {Pa,Ha,Aa,Ra2,Tya}, 124 | B2 = {Pb,Hb,Ab,Rb2,Tyb}, 125 | extra_stem([A2|[B2|T]], CFG); 126 | true -> [A|extra_stem([B|T], CFG)] 127 | end. 128 | %{pointer, hash, path, branch, type} 129 | batch2_ram(1, Branches, _CFG, Pointer, T) -> 130 | 131 | {_, Hash, _, [Stem], _} = hd(Branches), 132 | Stem2 = batch3(Stem, 1, Branches), 133 | H = {Pointer, Stem2}, 134 | %Loc = stem:put(Stem2, CFG), 135 | {Hash, Pointer, [H|T]}; 136 | batch2_ram(N, Branches, CFG, Pointer, T) -> 137 | %If the first N-1 nibbles of the path are the same, then they should be combined using batch3. 138 | {NewBranches, Pointer2, T2} = branch2helper_ram(N, Branches, CFG, Pointer, T, []), 139 | %Store the nth thing in each branch onto the blockchain, update the pointer and hash etc 140 | batch2_ram(N-1, NewBranches, CFG, Pointer2, T2). 141 | 142 | batch2(1, Branches, CFG) -> 143 | {_, Hash, _, [Stem], _} = hd(Branches), 144 | Stem2 = batch3(Stem, 1, Branches), 145 | Loc = stem:put(Stem2, CFG), 146 | {Hash, Loc}; 147 | batch2(N, Branches, CFG) -> 148 | %If the first N-1 nibbles of the path are the same, then they should be combined using batch3. 149 | NewBranches = branch2helper(N, Branches, CFG), 150 | %Store the nth thing in each branch onto the blockchain, update the pointer and hash etc 151 | batch2(N-1, NewBranches, CFG). 152 | branch2helper(_, [], _) -> []; 153 | branch2helper(N, Branches, CFG) -> 154 | {_, _, Path, [Stem|ST], _} = hd(Branches), 155 | if 156 | length([Stem|ST]) == N -> 157 | {M, _} = lists:split(N-1, Path), 158 | {Combine, Rest} = batch4(Branches, M, N, []), 159 | Stem2 = batch3(Stem, N, Combine), 160 | Loc = stem:put(Stem2, CFG), 161 | Hash = stem:hash(Stem2, CFG), 162 | [{Loc, Hash, Path, ST, 1}|branch2helper(N, Rest, CFG)]; 163 | length([Stem|ST]) < N -> 164 | [hd(Branches)|branch2helper(N, tl(Branches), CFG)] 165 | end. 166 | batch4([], _, _, Out) -> {lists:reverse(Out), []}; 167 | batch4([B|Branches], M, N, Out) -> 168 | {_, _, Path, _, _} = B, 169 | {M2, _} = lists:split(N-1, Path), 170 | case M2 of 171 | M -> batch4(Branches, M, N, [B|Out]); 172 | _ -> {lists:reverse(Out), [B|Branches]} 173 | end. 174 | batch3(Stem, _, []) -> Stem; 175 | batch3(Stem, N, [{Pointer, Hash, Path, _, Type}|T]) -> 176 | <> = lists:nth(N, Path), 177 | S2 = stem:add(Stem, A, Type, Pointer, Hash), 178 | batch3(S2, N, T). 179 | %we will look at pairs at the same time, and delete stuff from the older of the two. That way we still have everything when we move on to the next pair. 180 | %use stem:add(branch, direction, type, pointer, hash, cfg) to to add a child to a stem. Remember, you cannot know the pointer until the child stem is already added. The root of the trie is the last thing we can calculate. 181 | %So every time we iterate over the list of branches, some of the branches might combine, until eventually there is only 1 branch left, which is the root branch. 182 | 183 | sort_by_path([], _) -> []; 184 | sort_by_path([X], _) -> [X]; 185 | sort_by_path([Pivot|List], CFG) -> 186 | Key = leaf:path_maker(leaf:key(Pivot), CFG), 187 | {Below, Above} = pivot_split(Key, List, [], [], CFG), 188 | sort_by_path(Below, CFG) ++ 189 | [Pivot] ++ 190 | sort_by_path(Above, CFG). 191 | pivot_split(_, [], Below, Above, _) -> {Below, Above}; 192 | pivot_split(PKey, [H|T], Below, Above, CFG) -> 193 | Key = leaf:path_maker(leaf:key(H), CFG), 194 | B = compare_keys(PKey, Key), 195 | {C, D} = case B of 196 | true -> {[H|Below], Above}; 197 | false -> {Below, [H|Above]} 198 | end, 199 | pivot_split(PKey, T, C, D, CFG). 200 | compare_keys([<>|AT], [<>|BT]) -> 201 | if 202 | A > B -> true; 203 | B > A -> false; 204 | true -> compare_keys(AT, BT) 205 | end. 206 | branch2helper_ram(_, [], _, P, T, B) -> 207 | {B, P, T}; 208 | branch2helper_ram(N, Branches, CFG, P, T, B0) -> 209 | {_, _, Path, [Stem|ST], _} = hd(Branches), 210 | if 211 | length([Stem|ST]) == N -> 212 | {M, _} = lists:split(N-1, Path), 213 | {Combine, Rest} = batch4(Branches, M, N, []), 214 | Stem2 = batch3(Stem, N, Combine), 215 | %Loc = stem:put(Stem2, CFG), 216 | Hash = stem:hash(Stem2, CFG), 217 | B1 = B0 ++ [{P, Hash, Path, ST, 1}], 218 | %[{Loc, Hash, Path, ST, 1}|branch2helper(N, Rest, CFG)]; 219 | branch2helper_ram(N, Rest, CFG, P+1, [{P, Stem2}|T], B1); 220 | length([Stem|ST]) < N -> 221 | B1 = B0 ++ [hd(Branches)], 222 | branch2helper_ram(N, tl(Branches), CFG, P, T, B1) 223 | %[hd(Branches)|branch2helper(N, tl(Branches), CFG)] 224 | end. 225 | 226 | store_batch_helper_ram([], _, X, _, _Pointer, L) -> 227 | 228 | %io:fwrite("store batch helper ram done\n"), 229 | {X, L}; 230 | store_batch_helper_ram([H|T], CFG, BD, Root, Pointer, L) -> 231 | %io:fwrite("sbhr 0\n"), 232 | Path = leaf:path(H, CFG), 233 | GB = get_branch(Path, 0, Root, [], CFG), 234 | %io:fwrite("sbhr 1\n"), 235 | case leaf:value(H) of 236 | empty -> 237 | case GB of 238 | {_, _, _} -> store_batch_helper_ram(T, CFG, BD, Root, Pointer, L); %if you are deleting something that doesn't exist, then you don't have to do anything. 239 | Branch0 -> 240 | X = cfg:hash_size(CFG)*8, 241 | EmptyHash = <<0:X>>, 242 | store_batch_helper_ram(T, CFG, [{0, <<0:X>>, Path, Branch0, 0}|BD], Root, Pointer, L) 243 | end; 244 | _ -> 245 | %NLP = leaf:put(H, CFG), 246 | NLH = leaf:hash(H, CFG), 247 | {Br, NewPointer, L2} = 248 | case GB of 249 | {Leaf2, _LP1, Branch} ->%split leaf, add stem(s) 250 | %LP2 = leaf:put(Leaf2, CFG), 251 | %need to add 1 or more stems. 252 | {A, N2} = path_match(Path, leaf:path(Leaf2, CFG)), 253 | [Hp|Tp] = empty_stems(max(1, A-length(Branch)+1), CFG), 254 | LH2 = leaf:hash(Leaf2, CFG), 255 | H2 = stem:add(Hp, N2, 2, Pointer + 1, LH2), 256 | {[H2|Tp]++Branch, Pointer + 2, [{Pointer + 1, Leaf2}|[{Pointer, H}|L]]}; 257 | AB -> %overwrite 258 | {AB, Pointer + 1, [{Pointer, H}|L]} 259 | end, 260 | store_batch_helper_ram(T, CFG, [{Pointer, NLH, Path, Br, 2}|BD], Root, NewPointer, L2) 261 | end. 262 | store_batch_helper([], _, X, _) -> X; 263 | store_batch_helper([H|T], CFG, BD, Root) -> 264 | Path = leaf:path(H, CFG), 265 | GB = get_branch(Path, 0, Root, [], CFG), 266 | case leaf:value(H) of 267 | empty -> 268 | case GB of 269 | {_, _, _} -> store_batch_helper(T, CFG, BD, Root); %if you are deleting something that doesn't exist, then you don't have to do anything. 270 | Branch0 -> 271 | X = cfg:hash_size(CFG)*8, 272 | EmptyHash = <<0:X>>, 273 | store_batch_helper(T, CFG, [{0, <<0:X>>, Path, Branch0, 0}|BD], Root) 274 | end; 275 | _ -> 276 | NLP = leaf:put(H, CFG), 277 | NLH = leaf:hash(H, CFG), 278 | Br = case GB of 279 | {Leaf2, _LP1, Branch} ->%split leaf, add stem(s) 280 | LP2 = leaf:put(Leaf2, CFG), 281 | %need to add 1 or more stems. 282 | {A, N2} = path_match(Path, leaf:path(Leaf2, CFG)), 283 | [Hp|Tp] = empty_stems(max(1, A-length(Branch)+1), CFG), 284 | LH2 = leaf:hash(Leaf2, CFG), 285 | H2 = stem:add(Hp, N2, 2, LP2, LH2), 286 | [H2|Tp]++Branch; 287 | AB -> %overwrite 288 | AB 289 | end, 290 | store_batch_helper(T, CFG, [{NLP, NLH, Path, Br, 2}|BD], Root) 291 | end. 292 | store(Leaf, Root, CFG) -> 293 | %we could make it faster if the input was like [{Key1, Value1}, {Key2, Value2}...] 294 | LPointer = leaf:put(Leaf, CFG), 295 | LH = leaf:hash(Leaf, CFG), 296 | P = leaf:path(Leaf, CFG), 297 | B = case get_branch(P, 0, Root, [], CFG) of 298 | {Leaf2, LP2, Branch} ->%split leaf, add stem(s) 299 | %need to add 1 or more stems. 300 | {A, N2} = path_match(P, leaf:path(Leaf2, CFG)), 301 | [H|T] = empty_stems(max(1, A-length(Branch)+1), CFG), 302 | LH2 = leaf:hash(Leaf2, CFG), 303 | H2 = stem:add(H, N2, 2, LP2, LH2), 304 | [H2|T]++Branch; 305 | Branch -> %overwrite 306 | Branch 307 | end, 308 | store_branch(B, P, 2, LPointer, LH, CFG). 309 | -type path_nibble_index() :: path_nibble_index(cfg:path()). 310 | -type path_nibble_index(_CfgPathSizeBytes) :: non_neg_integer(). % 0..((cfg:path() * 2) - 1) 311 | -spec get_branch(Path::leaf:path(), StartInPath::path_nibble_index(), 312 | stem:stem_p(), branch(), cfg:cfg()) -> 313 | {leaf:leaf(), leaf:leaf_p(), % leaf (and corresponding pointer) at returned branch and containing path different from the specified one 314 | Branch::nonempty_branch()} | 315 | nonempty_branch(). % branch either (1) without leaf or (2) with leaf containing specified path 316 | get_branch(Path, N, Parent, Trail, CFG) -> 317 | %gather the branch as it currently looks. 318 | M = N+1, 319 | <> = lists:nth(M, Path), % TODO this could be turned into hd (head) 320 | R = stem:get(Parent, CFG), 321 | Pointer = stem:pointer(A+1, R), 322 | RP = [R|Trail], 323 | ST = stem:type(A+1, R), 324 | if 325 | ST == 0 -> RP; 326 | Pointer == 0 -> RP; 327 | ST == 1 -> get_branch(Path, M, Pointer, RP, CFG); 328 | ST == 2 -> 329 | Leaf = leaf:get(Pointer, CFG), 330 | case leaf:path(Leaf, CFG) of 331 | Path -> %overwrite 332 | RP; 333 | _ -> %split leaf, add stem(s) 334 | {Leaf, Pointer, RP} 335 | end 336 | end. 337 | store_branch([], Path, _, Pointer, _, CFG) -> 338 | %Instead of getting the thing, we can build it up while doing store. 339 | {Hash, _, Proof} = get:get(Path, Pointer, CFG), 340 | {Hash, Pointer, Proof}; 341 | 342 | store_branch([B|Branch], Path, Type, Pointer, Hash, CFG) -> 343 | S = length(Branch), 344 | <> = lists:nth(S+1,Path), 345 | S1 = stem:add(B, A, Type, Pointer, Hash), 346 | Loc = stem:put(S1, CFG), 347 | SH = stem:hash(S1, CFG), 348 | store_branch(Branch, Path, 1, Loc, SH, CFG). 349 | -spec path_match(Path1::leaf:path(), Path2::leaf:path()) -> 350 | {ConvergenceLength::path_nibble_index(), 351 | NextNibbleInPath2::stem:nibble()}. 352 | path_match(P1, P2) -> path_match(P1, P2, 0). 353 | path_match([<> | P1], [<> | P2], N) -> 354 | if 355 | A == B -> path_match(P1, P2, N+1); 356 | true -> {N, B} 357 | end. 358 | empty_stems(0, _) -> []; 359 | empty_stems(N, CFG) -> [stem:new_empty(CFG)|empty_stems(N-1, CFG)]. 360 | -------------------------------------------------------------------------------- /src/test_trie.erl: -------------------------------------------------------------------------------- 1 | -module(test_trie). 2 | -export([test/0, test/2]). 3 | 4 | -define(ID, trie01). 5 | 6 | test() -> 7 | CFG = trie:cfg(?ID), 8 | %V = [1,2,3,4,5,7,8,9,10,11,12,13,14,16,17], 9 | V = [1,2,3,5,7,8,9,10,11,12,13,14,16,17,18], 10 | %V = [5, 6, 12, 13], 11 | %V = [18], 12 | test_helper(V, CFG). 13 | test_helper([], _) -> success; 14 | test_helper([N|T], CFG) -> 15 | io:fwrite("test "), 16 | io:fwrite(integer_to_list(N)), 17 | io:fwrite("\n"), 18 | success = test(N, CFG), 19 | test_helper(T, CFG). 20 | 21 | test(1, CFG) -> 22 | leaf:new(1, empty, 0, CFG), 23 | Nib1 = 4, 24 | Nib2 = 2, 25 | L = [<>,<>,<<0:4>>,<<0:4>>,<<0:4>>,<<0:4>>,<<0:4>>,<<0:4>>,<<0:4>>,<<0:4>>], 26 | Lflip = lists:reverse(L), 27 | Lb = <<255,255>>, 28 | <> = list_to_bitstring(Lflip), 29 | Meta = 0, 30 | LeafAB = leaf:new(Laa, Lb, Meta, CFG), 31 | %io:fwrite("leafAB is "), 32 | %io:fwrite(LeafAB), 33 | %called as io:format(<0.26.0>,{leaf,36,<<"\377\377">>,0},[]) 34 | Lc = leaf:serialize(LeafAB, CFG), 35 | %io:fwrite("lc is "), 36 | %io:fwrite(size(Lc)), 37 | Loc1 = dump:put(Lc, ids:leaf(CFG)), 38 | LH = leaf:hash(LeafAB, CFG), 39 | S1 = stem:new(Nib2, 2, Loc1, LH, CFG), 40 | Loc2 = stem:put(S1, CFG), 41 | SH = stem:hash(S1, CFG), 42 | S = stem:new(Nib1, 1, Loc2, SH, CFG), 43 | Loc3 = stem:put(S, CFG), 44 | %Starts with a 2-level tree with a single leaf at the end. 45 | RootHash = stem:hash(S, CFG), 46 | X = {RootHash, LeafAB, [stem:hashes(S1), stem:hashes(S)]}, 47 | Proof = [stem:hashes(S1), stem:hashes(S)], 48 | {RootHash, LeafAB, _} = get:get(L, Loc3, CFG),%Path, Root 49 | X = get:get(L, Loc3, CFG),%Path, Root 50 | {_, LeafAB, Proof} = X, 51 | true = verify:proof(RootHash, LeafAB, Proof, CFG), 52 | %Now we add a second element. 53 | Nib3 = 5, 54 | Nib4 = 10, 55 | L2 = [<>,<>,<<0:4>>,<<0:4>>,<<0:4>>,<<0:4>>,<<0:4>>,<<0:4>>,<<0:4>>,<<0:4>>], 56 | L2flip = lists:reverse(L2), 57 | L2b = <<255,255>>, 58 | <> = list_to_bitstring(L2flip), 59 | Leafbb = leaf:new(Lbb, L2b, Meta, CFG), 60 | L2c = leaf:serialize(Leafbb, CFG), 61 | Loc4 = dump:put(L2c, ids:leaf(CFG)), 62 | LH2 = leaf:hash(Leafbb, CFG), 63 | S2 = stem:new(Nib4, 2, Loc4, LH2, CFG), 64 | Loc5 = stem:put(S2, CFG), 65 | SH2 = stem:hash(S2, CFG), 66 | S3 = stem:add(S, Nib3, 1, Loc5, SH2), 67 | Loc6 = stem:put(S3, CFG), 68 | RootHash2 = stem:hash(S3, CFG), 69 | Proof2 = [stem:hashes(S2), stem:hashes(S3)], 70 | X2 = {RootHash2, Leafbb, Proof2}, 71 | X2 = get:get(L2, Loc6, CFG), 72 | true = verify:proof(RootHash2, Leafbb, Proof2, CFG), 73 | Nib5 = 4, 74 | Nib6 = 2, 75 | L3 = [<>,<>,<<0:4>>,<<1:4>>,<<0:4>>,<<0:4>>,<<0:4>>,<<0:4>>,<<0:4>>,<<0:4>>], 76 | L3flip = lists:reverse(L3), 77 | L3b = <<255,255>>, 78 | <> = list_to_bitstring(L3flip), 79 | Leafcc = leaf:new(L3abc, L3b, Meta, CFG), 80 | {Root7, Loc7, _} = store:store(Leafcc, Loc6, CFG), 81 | {Root7, _, _} = store:store(Leafcc, Loc6, CFG), 82 | %trie:garbage([Loc7], ?ID), 83 | %timer:sleep(100), 84 | trie:cfg(?ID), 85 | ReplaceStem = <<0:(8*(dump:word(ids:stem(CFG))))>>, 86 | %1 = dump:put(ReplaceStem, ids:stem(CFG)), 87 | {PP4,Leafcc,PP5} = get:get(L3, Loc7, CFG), 88 | true = verify:proof(PP4,Leafcc,PP5,CFG), 89 | success; 90 | 91 | test(2, CFG) -> 92 | Loc = 1, 93 | %L = <<0:4,0:4,0:4,0:4,0,0,0>>, 94 | La = <<255, 0>>, 95 | Leaf = leaf:new(1, La, 0, CFG), 96 | store:store(Leaf, Loc, CFG), 97 | success; 98 | 99 | test(3, CFG) -> 100 | Loc = 1, 101 | Times = 1000, 102 | NewLoc = test3a(Times, Times, Loc), 103 | test3b(Times, NewLoc, CFG), 104 | success; 105 | 106 | test(4, CFG) -> 107 | Size = dump:word(ids:leaf(CFG)), 108 | Size1 = cfg:leaf(CFG), 109 | %io:fwrite({Size, Size1}), 110 | %Size = Size1, 111 | Data0 = <<11:(8*Size)>>, 112 | Data1 = <<2:(8*Size)>>, 113 | Data2 = <<3:(8*Size)>>, 114 | Data3 = <<4:(8*Size)>>, 115 | IDSL = ids:leaf(CFG), 116 | IDSS = ids:stem(CFG), 117 | A0 = dump:put(Data0, IDSL), 118 | Data0 = dump:get(A0, IDSL), 119 | A1 = dump:put(Data1, IDSL), 120 | Data1 = dump:get(A1, IDSL), 121 | dump:delete(A0, IDSL), 122 | A0 = dump:put(Data1, IDSL), 123 | Data1 = dump:get(A0, IDSL), 124 | A2 = dump:put(Data2, IDSL), 125 | Data1 = dump:get(A0, IDSL), 126 | A3 = dump:put(Data3, IDSL), 127 | Data1 = dump:get(A0, IDSL), 128 | Data1 = dump:get(A1, IDSL), 129 | Data2 = dump:get(A2, IDSL), 130 | Data3 = dump:get(A3, IDSL), 131 | 132 | Size2 = dump:word(IDSS), 133 | Data02 = <<11:(8*Size2)>>, 134 | Data12 = <<2:(8*Size2)>>, 135 | Data22 = <<3:(8*Size2)>>, 136 | Data32 = <<4:(8*Size2)>>, 137 | A02 = dump:put(Data02, IDSS), 138 | A12 = dump:put(Data12, IDSS), 139 | A22 = dump:put(Data22, IDSS), 140 | A32 = dump:put(Data32, IDSS), 141 | Data02 = dump:get(A02, IDSS), 142 | Data12 = dump:get(A12, IDSS), 143 | Data22 = dump:get(A22, IDSS), 144 | Data32 = dump:get(A32, IDSS), 145 | dump:delete(A02, IDSS), 146 | success; 147 | 148 | test(5, CFG) -> 149 | %Root0 = 1, 150 | Root0 = cfg:empty(CFG), 151 | V1 = <<1,1>>, 152 | V2 = <<1,2>>, 153 | V3 = <<1,3>>, 154 | <> = <<1,0,0,0,0>>, 155 | <> = <<0, 0,16,0,0>>, 156 | <> = <<0, 0,1,0,0>>, 157 | Meta = 0, 158 | Leaf1 = leaf:new(L1, V1, Meta, CFG), 159 | Leaf2 = leaf:new(L2, V2, Meta, CFG), 160 | Leaf3 = leaf:new(L3, V3, Meta, CFG), 161 | Leaf4 = leaf:new(L3, V1, Meta, CFG), 162 | {_, Root1, _} = store:store(Leaf1, Root0, CFG), 163 | {Hash, Root2, Proof2} = store:store(Leaf2, Root1, CFG), 164 | {Hash, Root3, Proof3} = store:store(Leaf2, Root2, CFG), 165 | {Hash2, Root4, Proof4} = store:store(Leaf3, Root3, CFG), 166 | {Hash2, Leaf2, Proof7} = get:get(leaf:path(Leaf2, CFG), Root4, CFG), 167 | {Hash2, Leaf3, Proof8} = get:get(leaf:path(Leaf3, CFG), Root4, CFG), 168 | Lpath1 = leaf:path(Leaf1, CFG), 169 | X = [{Lpath1, Root4}], 170 | %garbage:garbage_leaves(X, CFG),%After we do garbage leaves we can't insert things into the merkle tree normally. 171 | %many stems are missing, so we can't make proofs of anything we don't save, but we can still verify them. 172 | %We need a merkle proof of it's previous state in order to update. 173 | %timer:sleep(500), 174 | %timer:sleep(500), 175 | {Hash2, Leaf1, Proof} = get:get(Lpath1, Root4, CFG), 176 | true = verify:proof(Hash2, Leaf1, Proof, CFG), 177 | true = verify:proof(Hash, Leaf2, Proof2, CFG), 178 | %{Hash2, unknown, _} = get:get(leaf:path(Leaf2, CFG), Root4, CFG), 179 | {Hash2, Root5, _} = store:restore(Leaf2, Hash2, Proof7, Root4, CFG), 180 | %{Hash2, unknown, _} = get:get(leaf:path(Leaf3, CFG), Root5, CFG), 181 | {Hash2, Root6, Proof5} = store:restore(Leaf3, Hash2, Proof8, Root5, CFG), 182 | %we need to be able to add proofs for things into an empty database. 183 | true = verify:proof(Hash2, Leaf3, Proof5, CFG), 184 | {Hash5, _, Proof6} = store:store(Leaf4, Root6, CFG), %overwrite the same spot. 185 | true = verify:proof(Hash5, Leaf4, Proof6, CFG), 186 | success; 187 | 188 | test(6, CFG) -> 189 | %The purpose of this test is to test merge. 190 | % The full merkel trie will be too big, most people wont keep track of it all. 191 | % sometimes parts of the trie get updated that we aren't keeping track of. We need to look at the proof of their update, and update our state root accordingly. 192 | % We don't get a proof of the final state. We only get a proof of the initial state, and the final state. It is possible to calculate the new proof from this. The reason we don't get the new proof is because depending on which txs get accepted into the block, the root hash of the new state will be different 193 | %Root0 = 1, 194 | Root0 = cfg:empty(CFG), 195 | V1 = <<1,1>>, 196 | V2 = <<1,2>>, 197 | V3 = <<1,3>>, 198 | <> = <<0,0,0,0,1>>, 199 | <> = <<0,16,0,0,0>>, 200 | Meta = 0, 201 | Leafa = leaf:new(L1, V1, Meta, CFG), 202 | {_, Root1, _} = store:store(Leafa, Root0, CFG), 203 | {Hash0bb, Leafa, Proofa} = get:get(leaf:path(Leafa, CFG), Root1, CFG), 204 | true = verify:proof(Hash0bb, Leafa, Proofa, CFG), 205 | Leafb = leaf:new(L2, V2, Meta, CFG), 206 | {_, Root2, _} = store:store(Leafb, Root1, CFG), 207 | {Hash0, Leafb, Proofb} = get:get(leaf:path(Leafb, CFG), Root2, CFG), 208 | true = verify:proof(Hash0, Leafb, Proofb, CFG), 209 | Leafc = leaf:new(L2, V3, Meta, CFG), 210 | {Hash, Root3, _} = store:store(Leafc, Root2, CFG), 211 | {Hasha, _, _} = store:store(Leafc, Root1, CFG), 212 | Hasha = Hash, 213 | {Hash, Leafc, Proofc} = get:get(leaf:path(Leafc, CFG), Root3, CFG), 214 | true = verify:proof(Hash, Leafc, Proofc, CFG), 215 | {Hash, Root6, Proofc} = store:store(Leafc, Root1, CFG), 216 | GL = [{leaf:path(Leafa, CFG), Root6}], 217 | {_, Leafa, _} = get:get(leaf:path(Leafa, CFG), Root6, CFG), 218 | garbage:garbage_leaves(GL, CFG), 219 | timer:sleep(1000), 220 | {Hash3, Leafa, _} = get:get(leaf:path(Leafa, CFG), Root6, CFG), 221 | %{Hash3, empty, _} = get:get(leaf:path(Leafc, CFG), Root6, CFG), 222 | %RootStem = stem:update_pointers(stem:get(Root6, CFG), 223 | % stem:empty_tuple()), 224 | %Root7 = trie:new_trie(trie01, RootStem), 225 | %RootStem = stem:empty_trie(stem:get(Root6, CFG), CFG), 226 | Root7 = trie:new_trie(trie01, stem:get(Root6, CFG)), 227 | Hash3 = trie:root_hash(trie01, Root7), 228 | {Hash3, unknown, _} = get:get(leaf:path(Leafc, CFG), Root7, CFG), 229 | {Hash3, Root8, _} = store:restore(Leafc, Hash, Proofc, Root7, CFG), %it is restoring the deleted leaf to the database. 230 | %{Hash, Leafa, _B2} = get:get(leaf:path(Leafa, CFG), Root5, CFG), 231 | {Hash3, Leafc, _} = get:get(leaf:path(Leafc, CFG), Root8, CFG), 232 | %true = verify:proof(Hash, Leafa, B2, CFG), 233 | 234 | 235 | % the current implementation is very innefficient. It stores the entire proof onto the hard drive 236 | success; 237 | 238 | test(7, CFG) -> 239 | %Root0 = 1, 240 | Root0 = cfg:empty(CFG), 241 | V1 = <<1,1>>, 242 | V2 = <<1,2>>, 243 | <> = <<0,0,0,0,2>>, 244 | <> = <<0,16,0,0,0>>, 245 | Meta = 0, 246 | Leaf1 = leaf:new(L1, V1, Meta, CFG), 247 | Leaf2 = leaf:new(L2, V2, Meta, CFG), 248 | {_, Root1, _} = store:store(Leaf1, Root0, CFG), 249 | {Hash0bb, Leaf1, B0bb} = get:get(leaf:path(Leaf1, CFG), Root1, CFG), 250 | true = verify:proof(Hash0bb, Leaf1, B0bb, CFG), 251 | {_, Root2, _} = store:store(Leaf2, Root1, CFG), 252 | {Hash0, Leaf2, B0} = get:get(leaf:path(Leaf2, CFG), Root2, CFG), 253 | true = verify:proof(Hash0, Leaf2, B0, CFG), 254 | success; 255 | 256 | test(8, CFG) -> 257 | V1 = <<1,1>>, 258 | Root = 1, 259 | Key = 1, 260 | Meta = 0, 261 | Root2 = trie:put(Key, V1, Meta, Root, trie01), 262 | {RootHash, empty, Proof} = trie:get(2, Root2, trie01), 263 | {RootHash, empty, _} = trie:get(3, Root2, trie01), 264 | {_, empty, _} = trie:get(4, Root2, trie01), 265 | {_, Leaf, _} = trie:get(Key, Root2, trie01), 266 | V1 = leaf:value(Leaf), 267 | true = verify:proof(RootHash, leaf:new(2, empty, 0, CFG), 268 | Proof, CFG), 269 | success; 270 | 271 | test(9, CFG) -> 272 | %Root0 = 1, 273 | %io:fwrite(CFG), 274 | Root0 = cfg:empty(CFG), 275 | S = stem:get(Root0, CFG), 276 | V1 = <<2,3>>, 277 | Key = 5, 278 | RH = trie:root_hash(trie01, Root0), 279 | Meta = 0, 280 | Root2 = trie:put(Key, V1, Meta, Root0, trie01), 281 | {_, Leaf, _Proof1} = trie:get(Key, Root2, trie01), 282 | V1 = leaf:value(Leaf), 283 | Root3 = trie:delete(Key, Root2, trie01), 284 | {RH, empty, _Proof} = trie:get(Key, Root3, trie01), 285 | S = stem:get(Root0, CFG), 286 | success; 287 | 288 | test(10, _CFG) -> 289 | trie:get_all(1, trie01), 290 | success; 291 | test(11, CFG) -> 292 | Meta = 0, 293 | V1 = <<2,3>>, 294 | Key1 = 1, 295 | Key2 = 2, 296 | Key3 = 257, 297 | Key4 = 513, 298 | %Root0 = 1, 299 | Root0 = cfg:empty(CFG), 300 | Root = trie:put(Key1, V1, Meta, Root0, trie01), 301 | %Root = trie:put(Key3, V1, Meta, Root01, trie01), 302 | {RootHash, Leaf1, Proof1} = trie:get(Key1, Root, trie01), 303 | {RootHash, empty, Proof2} = trie:get(Key2, Root, trie01), 304 | {RootHash, empty, Proof3} = trie:get(Key3, Root, trie01), 305 | {RootHash, empty, Proof4} = trie:get(Key4, Root, trie01), 306 | true = verify:proof(RootHash, Leaf1, Proof1, CFG), 307 | true = verify:proof(RootHash, leaf:new(Key1, V1, 0, CFG), Proof1, CFG), 308 | true = verify:proof(RootHash, leaf:new(Key2, empty, 0, CFG), Proof2, CFG), 309 | %io:fwrite({proofs, Proof2, Proof3}), 310 | true = verify:proof(RootHash, leaf:new(Key3, empty, 0, CFG), Proof3, CFG), 311 | true = verify:proof(RootHash, leaf:new(Key4, empty, 0, CFG), Proof4, CFG), 312 | success; 313 | test(12, CFG) -> 314 | %Times = 257, 315 | Times = 25, 316 | %Times = 5, 317 | ID = 2, 318 | %Root0 = 1, 319 | Root0 = cfg:empty(CFG), 320 | L2 = test3a(Times, Times, Root0), 321 | Root1 = trie:new_trie(trie01, stem:get(L2, CFG)), 322 | Hash = trie:root_hash(trie01, Root1), 323 | Hash = trie:root_hash(trie01, L2), 324 | 325 | %Restore data to trie. 326 | {Hash, unknown, _} = trie:get(ID, Root1, trie01), 327 | {Leaf, Root2} = restore(ID, L2, Root1), 328 | {Hash, Leaf, Proof} = trie:get(ID, Root2, trie01), 329 | 330 | %Restore our knowledge of the lack of data that ends with a leaf. 331 | {Hash, empty, Proof2} = trie:get(Times+1, L2, trie01), 332 | EmptyLeaf = leaf:new(Times, empty, 0, CFG), 333 | true = verify:proof(Hash, EmptyLeaf, Proof2, CFG), 334 | {Hash, unknown, _} = trie:get(Times+1, Root2, trie01), 335 | Root3 = trie:restore(EmptyLeaf, Hash, Proof2, Root2, trie01), 336 | {Hash, empty, _} = trie:get(Times+1, Root3, trie01), 337 | 338 | %Restore our knowledge of the lack of data that ends with a stem. 339 | Root4 = test3a(2, 2, Root0), 340 | Root5 = trie:new_trie(trie01, stem:get(Root4, CFG)), 341 | {Hash4, empty, Proof4} = trie:get(17, Root4, trie01), 342 | EmptyLeaf2 = leaf:new(17, empty, 0, CFG), 343 | true = verify:proof(Hash4, EmptyLeaf2, Proof4, CFG), 344 | {Hash4, unknown, _} = trie:get(17, Root5, trie01), 345 | Root6 = trie:restore(EmptyLeaf2, Hash4, Proof4, Root5, trie01), 346 | {Hash4, empty, _} = trie:get(17, Root6, trie01), 347 | success; 348 | 349 | test(13, CFG) -> 350 | 351 | %Restore our knowledge of various things, and then check that the information is remembered correctly. 352 | Times = 25, 353 | %Root0 = 1, 354 | Root0 = cfg:empty(CFG), 355 | L2 = test3a(Times, Times, Root0), 356 | Root1 = trie:new_trie(trie01, stem:get(L2, CFG)), 357 | [ID1, ID2, ID3] = [1, 17, 18],%adding an 18 and a 1 to this list makes it break. 358 | {Leaf7, Root7} = restore(ID1, L2, Root1), 359 | {Leaf8, Root8} = restore(ID2, L2, Root7), 360 | {Leaf9, Root9} = restore(ID3, L2, Root8), 361 | 362 | Hash = trie:root_hash(trie01, Root7), 363 | Hash = trie:root_hash(trie01, Root8), 364 | Hash = trie:root_hash(trie01, Root9), 365 | 366 | {Hash, Leaf7, _} = trie:get(ID1, Root7, trie01), 367 | {Hash, Leaf7, _} = trie:get(ID1, Root8, trie01), 368 | {Hash, Leaf7, _} = trie:get(ID1, Root9, trie01), 369 | {Hash, Leaf8, _} = trie:get(ID2, Root9, trie01), 370 | {Hash, Leaf9, _} = trie:get(ID3, Root9, trie01), 371 | success; 372 | test(14, CFG) -> 373 | Loc0 = 1, 374 | La = <<255, 0>>, 375 | Lb = <<255, 1>>, 376 | Loc1 = trie:put(2, La, 0, Loc0, trie01), 377 | Loc = trie:put(1, Lb, 0, Loc1, trie01), 378 | Leaves = [leaf:new(1, La, 0, CFG), 379 | leaf:new(2, empty, 0, CFG), 380 | %leaf:new(33, La, 0, CFG), 381 | leaf:new(17, La, 0, CFG)], 382 | %Leaves = [Leaf, Leaf3], 383 | %io:fwrite(packer:pack(store:batch(Leaves, Loc, CFG))), 384 | trie:put_batch(Leaves, Loc, trie01), 385 | %root hash <<89,127,205,119,243,7,208,239,239,229,27,12,178,241,27, 386 | success; 387 | test(15, CFG) -> 388 | Loc = 1, 389 | La = <<255, 0>>, 390 | Leaf = leaf:new(1, La, 0, CFG), 391 | %Leaf2 = leaf:new(33, La, 0, CFG), 392 | Leaf3 = leaf:new(17, La, 0, CFG), 393 | %Leaves = [Leaf, Leaf2, Leaf3], 394 | Leaves = [Leaf, Leaf3], 395 | Loc2 = trie:put(1, La, 0, Loc, trie01), 396 | Loc3 = trie:put(17, La, 0, Loc2, trie01), 397 | io:fwrite("loc 3 is "), 398 | io:fwrite(integer_to_list(Loc3)), 399 | io:fwrite("\n"), 400 | %root hash matches test 14. {<<89,127,205,119,243,7,208,239,239,229,27,12,178,241,27, 401 | success; 402 | test(16, CFG) -> 403 | %prune test. 404 | %Root0 = 1, 405 | Root0 = cfg:empty(CFG), 406 | La = <<255, 0>>, 407 | Lb = <<255, 1>>, 408 | Leaves1 = [leaf:new(1, La, 0, CFG), 409 | leaf:new(2, La, 0, CFG), 410 | %leaf:new(33, La, 0, CFG), 411 | leaf:new(17, La, 0, CFG)], 412 | Leaves2 = [leaf:new(1, Lb, 0, CFG), 413 | leaf:new(3, Lb, 0, CFG)], 414 | Leaves3 = [leaf:new(1, La, 0, CFG), 415 | leaf:new(4, La, 0, CFG)], 416 | Old = trie:put_batch(Leaves1, Root0, trie01), 417 | New = trie:put_batch(Leaves2, Old, trie01), 418 | %insert a batch to get oldroot old, 419 | %insert a batch to get new 420 | Ls = trie:garbage(Old, New, trie01), 421 | io:fwrite("prune removed these "), 422 | io:fwrite(packer:pack(Ls)), 423 | io:fwrite("\n"), 424 | io:fwrite(packer:pack(element(2, trie:get(1, New, trie01)))), 425 | Final = trie:put_batch(Leaves3, New, trie01), 426 | io:fwrite(packer:pack(element(2, trie:get(1, Final, trie01)))), 427 | %make sure we can still look up stuff from New. 428 | success; 429 | test(17, CFG) -> 430 | %prune test. 431 | %Root0 = 1, 432 | Root0 = cfg:empty(CFG), 433 | La = <<255, 0>>, 434 | Lb = <<255, 1>>, 435 | Leaves1 = [leaf:new(1, La, 0, CFG), 436 | leaf:new(2, La, 0, CFG), 437 | %leaf:new(33, La, 0, CFG), 438 | leaf:new(17, La, 0, CFG)], 439 | Leaves2 = [leaf:new(1, Lb, 0, CFG), 440 | leaf:new(3, Lb, 0, CFG)], 441 | Leaves3 = [leaf:new(1, La, 0, CFG), 442 | leaf:new(4, La, 0, CFG)], 443 | Old = trie:put_batch(Leaves1, Root0, trie01), 444 | New = trie:put_batch(Leaves2, Old, trie01), 445 | %insert a batch to get oldroot old, 446 | %insert a batch to get new 447 | Ls = trie:garbage(New, Old, trie01), 448 | %Ls = prune2:stem(New, Old, trie:cfg(trie01)), 449 | io:fwrite("garbage removed these "), 450 | io:fwrite(packer:pack(Ls)), 451 | io:fwrite("\n"), 452 | io:fwrite(packer:pack(element(2, trie:get(1, Old, trie01)))), 453 | Final = trie:put_batch(Leaves3, Old, trie01), 454 | io:fwrite(packer:pack(element(2, trie:get(1, Final, trie01)))), 455 | io:fwrite("\n"), 456 | io:fwrite("final pointer "), 457 | io:fwrite(integer_to_list(Final)), 458 | io:fwrite("\n"), 459 | %make sure we can still look up stuff from New. 460 | success; 461 | test(18, CFG) -> 462 | %Proof2 = verify:update_proof(Leaf2, Proof, CFG), 463 | Loc = 1, 464 | Times = 1000, 465 | NewLoc = test3a(Times, Times, Loc), 466 | %test3b(Times, NewLoc, CFG), 467 | {Hash4, Value4, Proof4} = trie:get(4, NewLoc, ?ID), 468 | {Hash5, Value5, Proof5} = trie:get(5, NewLoc, ?ID), 469 | Leaf = leaf:new(5, <<0, 1>>, 0, CFG), 470 | Proof2 = verify:update_proof(Leaf, Proof5, CFG), 471 | NewRoot = stem:hash(hd(lists:reverse(Proof2)), CFG), 472 | true = verify:proof(NewRoot, Leaf, Proof2, CFG), 473 | 474 | [Proof3|_] = verify:update_proofs( 475 | [{Leaf, Proof5}| 476 | [{leaf:new(4, <<0, 1>>, 0, CFG), 477 | Proof4}|[]]], CFG), 478 | NewRoot2 = stem:hash(hd(lists:reverse(Proof3)), CFG), 479 | true = verify:proof(NewRoot2, Leaf, Proof3, CFG), 480 | 481 | success. 482 | 483 | 484 | 485 | 486 | restore(ID, FilledTree, NewTree) -> 487 | {Hash, Leaf, Proof} = trie:get(ID, FilledTree, trie01), 488 | %{Hash, unknown, _} = trie:get(ID, NewTree, trie01), 489 | {Leaf, trie:restore(Leaf, Hash, Proof, NewTree, trie01)}. 490 | 491 | 492 | test3a(0, _, L) -> L; 493 | test3a(N, Times, Loc) -> %load up the trie 494 | if 495 | (N rem 100) == 0 -> 496 | io:fwrite(integer_to_list(N)), 497 | io:fwrite("\n"); 498 | true -> ok 499 | end, 500 | Meta = 0, 501 | NewLoc = trie:put(Times + 1 - N, <>, Meta, Loc, ?ID), 502 | test3a(N-1, Times, NewLoc). 503 | 504 | test3b(0, L, _CFG) -> L; 505 | test3b(N, Loc, CFG) -> %check that everything is in the trie 506 | if 507 | (N rem 100) == 0 -> 508 | io:fwrite(integer_to_list(N)), 509 | io:fwrite("\n"); 510 | true -> ok 511 | end, 512 | {Hash, Value, Proof} = trie:get(N, Loc, ?ID), 513 | true = verify:proof(Hash, Value, Proof, CFG), 514 | test3b(N-1, Loc, CFG). 515 | 516 | --------------------------------------------------------------------------------