├── doc ├── erlang.png ├── rkvs_util.md ├── stylesheet.css ├── rkvs_storage_backend.md ├── rkvs_bitcask.md ├── rkvs_hanoidb.md ├── rkvs_ets.md ├── rkvs_rocksdb.md ├── rkvs_leveldb.md ├── overview.edoc ├── README.md └── rkvs.md ├── .gitignore ├── NOTICE ├── src ├── rkvs.app.src ├── rkvs_util.erl ├── rkvs_storage_backend.erl ├── rkvs.erl ├── rkvs_hanoidb.erl ├── rkvs_ets.erl ├── rkvs_rocksdb.erl ├── rkvs_leveldb.erl └── rkvs_bitcask.erl ├── NEWS.md ├── include └── rkvs.hrl ├── rebar.config ├── test ├── rkvs_tests.hrl ├── rkvs_bitcask_tests.erl └── rkvs_tests.erl ├── Makefile ├── rebar_dev.config ├── README.md └── LICENSE /doc/erlang.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/refuge/rkvs/HEAD/doc/erlang.png -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ebin 2 | *.swp 3 | *.dump 4 | edoc-info 5 | .DS_Store 6 | deps/ 7 | .eunit 8 | .rebar 9 | test/temp 10 | log 11 | .eunit 12 | .rebar 13 | test/sst* 14 | test/LOG* 15 | test/MANIFEST* 16 | test/*.log 17 | test/LOCK 18 | test/CURRENT 19 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | rkvs 2 | ---- 3 | Copyright 2014-2015 - Benoit Chesneau & rkvs Authors 4 | 5 | The rkvs Source Code is subject to the terms of the Mozilla Public 6 | License, v. 2.0. If a copy of the MPL was not distributed with this 7 | file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | -------------------------------------------------------------------------------- /src/rkvs.app.src: -------------------------------------------------------------------------------- 1 | {application, rkvs, 2 | [ 3 | {description, "simple K/Vs interface"}, 4 | {vsn, "0.2.1"}, 5 | {registered, []}, 6 | {applications, [ 7 | kernel, 8 | stdlib 9 | ]}, 10 | {include_apps, [eleveldb]}, 11 | {env, []} 12 | ]}. 13 | -------------------------------------------------------------------------------- /NEWS.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## 0.2.0 - 2015/05/03 4 | 5 | - add rocksdb backend 6 | - add hanoidb backend 7 | - add bitcask backend 8 | - improve folding: add new gt, gte, lt, lte options 9 | - support different key and value encoding 10 | - add fill_cache option to leveldb backend 11 | 12 | ## 0.1.0 - 2014/11/14 13 | 14 | - initial release 15 | -------------------------------------------------------------------------------- /include/rkvs.hrl: -------------------------------------------------------------------------------- 1 | %-*- mode: erlang -*- 2 | %% 3 | %% This Source Code Form is subject to the terms of the Mozilla Public 4 | %% License, v. 2.0. If a copy of the MPL was not distributed with this 5 | %% file, You can obtain one at http://mozilla.org/MPL/2.0/. 6 | 7 | -record(engine, {name, 8 | mod, 9 | ref, 10 | key_enc, 11 | val_enc, 12 | options}). 13 | 14 | -record(fold_options, {start_key=first, 15 | end_key=nil, 16 | gt=nil, 17 | gte=nil, 18 | lt=nil, 19 | lte=nil, 20 | max=0, 21 | key_enc, 22 | val_enc}). 23 | 24 | -type engine() :: #engine{}. 25 | -export_type([engine/0]). 26 | -------------------------------------------------------------------------------- /doc/rkvs_util.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Module rkvs_util # 4 | * [Function Index](#index) 5 | * [Function Details](#functions) 6 | 7 | 8 | 9 | 10 | ## Function Index ## 11 | 12 | 13 |
dec/3
enc/3
fold_options/2
14 | 15 | 16 | 17 | 18 | ## Function Details ## 19 | 20 | 21 | 22 | ### dec/3 ### 23 | 24 | `dec(T, V, E) -> any()` 25 | 26 | 27 | 28 | 29 | ### enc/3 ### 30 | 31 | `enc(T, V, E) -> any()` 32 | 33 | 34 | 35 | 36 | ### fold_options/2 ### 37 | 38 | `fold_options(Rest, Options) -> any()` 39 | 40 | 41 | -------------------------------------------------------------------------------- /rebar.config: -------------------------------------------------------------------------------- 1 | %% -*- mode: erlang -*- 2 | {cover_enabled, true}. 3 | {edoc_opts, [{preprocess, true}]}. 4 | 5 | {erl_opts, [ 6 | warnings_as_errors 7 | ]}. 8 | {xref_checks, [undefined_function_calls]}. 9 | 10 | 11 | {eunit_opts, [verbose]}. 12 | 13 | {require_otp_vsn, "R16|17|18"}. 14 | 15 | {erl_first_files, [ "src/rkvs_storage_backend.erl"]}. 16 | 17 | {clean_files, ["*~","*/*~","*/*.xfm","test/*.beam", "test/temp/*"]}. 18 | 19 | {deps, [ 20 | %% {eleveldb, ".*", {git, "https://github.com/basho/eleveldb.git", 21 | %% {tag, "2.1.0"}}}, 22 | %% 23 | %% {erocksdb, ".*", {git, "https://github.com/leo-project/erocksdb.git", 24 | %% {tag, 0.4}}}, 25 | %% 26 | %% {hanoidb, ".*", {git, "https://github.com/krestenkrab/hanoidb.git", 27 | %% "4e82d5f81ab087f038bfd13354ff48ee9113f459"}}, 28 | %% 29 | %% {bitcask, ".*", {git, "https://github.com/basho/bitcask.git", 30 | %% {tag, "2.0.0"}}}, 31 | ]}. 32 | -------------------------------------------------------------------------------- /test/rkvs_tests.hrl: -------------------------------------------------------------------------------- 1 | %%-*- mode: erlang -*- 2 | %% 3 | %% This Source Code Form is subject to the terms of the Mozilla Public 4 | %% License, v. 2.0. If a copy of the MPL was not distributed with this 5 | %% file, You can obtain one at http://mozilla.org/MPL/2.0/. 6 | 7 | 8 | -include_lib("eunit/include/eunit.hrl"). 9 | 10 | -define(BUILDDIR, filename:absname( 11 | filename:join([ 12 | filename:dirname(code:which(?MODULE)), 13 | ".."]))). 14 | 15 | -define(TEMPDIR, 16 | filename:join([?BUILDDIR, "test", "temp"])). 17 | 18 | -define(tempfile, 19 | fun() -> 20 | {A, B, C} = erlang:now(), 21 | N = node(), 22 | FileName = lists:flatten(io_lib:format("~p-~p.~p.~p", [N, A, B, C])), 23 | filename:join([?TEMPDIR, FileName]) 24 | end). 25 | 26 | -define(tempdb, 27 | fun() -> 28 | Nums = tuple_to_list(erlang:now()), 29 | Prefix = "eunit-test-db", 30 | Suffix = lists:concat([integer_to_list(Num) || Num <- Nums]), 31 | Prefix ++ "-" ++ Suffix 32 | end). 33 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | BASE_DIR = $(shell pwd) 2 | SUPPORT_DIR=$(BASE_DIR)/support 3 | ERLC ?= $(shell which erlc) 4 | ESCRIPT ?= $(shell which escript) 5 | ERL ?= $(shell which erl) 6 | APP := enkidb 7 | REBAR?= rebar 8 | 9 | $(if $(ERLC),,$(warning "Warning: No Erlang found in your path, this will probably not work")) 10 | 11 | $(if $(ESCRIPT),,$(warning "Warning: No escript found in your path, this will probably not work")) 12 | 13 | .PHONY: deps doc test 14 | 15 | all: deps compile 16 | 17 | compile: 18 | @$(REBAR) compile 19 | 20 | deps: 21 | @$(REBAR) get-deps 22 | 23 | doc: dev 24 | $(REBAR) -C rebar_dev.config doc skip_deps=true 25 | 26 | test: dev 27 | $(REBAR) -C rebar_dev.config eunit skip_deps=true 28 | 29 | dev: dev-deps 30 | $(REBAR) -C rebar_dev.config compile 31 | 32 | dev-deps: 33 | $(REBAR) -C rebar_dev.config get-deps 34 | 35 | clean: 36 | @$(REBAR) clean 37 | @rm -f t/*.beam t/temp.* 38 | @rm -f doc/*.html doc/*.css doc/edoc-info doc/*.png 39 | 40 | distclean: clean 41 | @$(REBAR) delete-deps 42 | @rm -rf deps 43 | 44 | dialyzer: compile 45 | @dialyzer -Wno_return -c ebin 46 | -------------------------------------------------------------------------------- /doc/stylesheet.css: -------------------------------------------------------------------------------- 1 | /* standard EDoc style sheet */ 2 | body { 3 | font-family: Verdana, Arial, Helvetica, sans-serif; 4 | margin-left: .25in; 5 | margin-right: .2in; 6 | margin-top: 0.2in; 7 | margin-bottom: 0.2in; 8 | color: #000000; 9 | background-color: #ffffff; 10 | } 11 | h1,h2 { 12 | margin-left: -0.2in; 13 | } 14 | div.navbar { 15 | background-color: #add8e6; 16 | padding: 0.2em; 17 | } 18 | h2.indextitle { 19 | padding: 0.4em; 20 | background-color: #add8e6; 21 | } 22 | h3.function,h3.typedecl { 23 | background-color: #add8e6; 24 | padding-left: 1em; 25 | } 26 | div.spec { 27 | margin-left: 2em; 28 | background-color: #eeeeee; 29 | } 30 | a.module,a.package { 31 | text-decoration:none 32 | } 33 | a.module:hover,a.package:hover { 34 | background-color: #eeeeee; 35 | } 36 | ul.definitions { 37 | list-style-type: none; 38 | } 39 | ul.index { 40 | list-style-type: none; 41 | background-color: #eeeeee; 42 | } 43 | 44 | /* 45 | * Minor style tweaks 46 | */ 47 | ul { 48 | list-style-type: square; 49 | } 50 | table { 51 | border-collapse: collapse; 52 | } 53 | td { 54 | padding: 3 55 | } 56 | -------------------------------------------------------------------------------- /doc/rkvs_storage_backend.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Module rkvs_storage_backend # 4 | * [Data Types](#types) 5 | 6 | __This module defines the `rkvs_storage_backend` behaviour.__
Required callback functions: `open/2`, `close/1`, `destroy/1`, `contains/2`, `get/2`, `put/3`, `clear/2`, `write_batch/2`, `scan/4`, `clear_range/4`, `fold_keys/4`, `fold/4`, `is_empty/1`. 7 | 8 | 9 | 10 | ## Data Types ## 11 | 12 | 13 | 14 | 15 | ### fold_options() ### 16 | 17 | 18 | 19 | 

20 | fold_options() = [{start_key, binary()} | {end_key, binary()} | {gt, binary()} | {gte, binary()} | {lt, binary()} | {lte, binary()} | {max, integer()}]
21 |
22 | 23 | 24 | 25 | 26 | 27 | ### kv() ### 28 | 29 | 30 | 31 | 

32 | kv() = {Key::binary(), Value::binary()}
33 |
34 | 35 | 36 | 37 | 38 | 39 | ### kvs() ### 40 | 41 | 42 | 43 | 

44 | kvs() = [kv()]
45 |
46 | 47 | 48 | 49 | 50 | 51 | ### write_ops() ### 52 | 53 | 54 | 55 | 

56 | write_ops() = [{put, Key::binary(), Value::any()} | {delete, Key::binary()} | clear]
57 |
58 | 59 | 60 | -------------------------------------------------------------------------------- /rebar_dev.config: -------------------------------------------------------------------------------- 1 | %% -*- mode: erlang -*- 2 | {cover_enabled, true}. 3 | 4 | {erl_opts, [ 5 | warnings_as_errors, 6 | debug_info, 7 | {i, "include"}, 8 | {i, "deps"}, 9 | {src_dirs, ["src"]} 10 | ]}. 11 | 12 | {xref_checks, [undefined_function_calls]}. 13 | 14 | 15 | {eunit_opts, [verbose]}. 16 | 17 | {require_otp_vsn, "R16|17|18"}. 18 | 19 | {erl_first_files, [ "src/rkvs_storage_backend.erl"]}. 20 | 21 | {clean_files, ["*~","*/*~","*/*.xfm","test/*.beam", "test/temp/*"]}. 22 | 23 | {deps, [ 24 | {eleveldb, ".*", {git, "https://github.com/basho/eleveldb.git", 25 | {tag, "2.1.0"}}}, 26 | 27 | {erocksdb, ".*", {git, "https://github.com/leo-project/erocksdb.git", 28 | {tag, "0.4"}}}, 29 | 30 | {hanoidb, ".*", {git, "https://github.com/krestenkrab/hanoidb.git", 31 | "4e82d5f81ab087f038bfd13354ff48ee9113f459"}}, 32 | 33 | {bitcask, ".*", {git, "https://github.com/basho/bitcask.git", 34 | {tag, "2.0.0"}}}, 35 | 36 | {edown, ".*", 37 | {git, "git://github.com/esl/edown.git", "HEAD"}} 38 | 39 | ]}. 40 | 41 | {edoc_opts, [{application, ["rkvs"]}, 42 | {doclet, edown_doclet}, 43 | {subpackages, false}, 44 | {top_level_readme, 45 | {"./README.md", "http://github.com/refuge/rkvs"}}]}. 46 | -------------------------------------------------------------------------------- /src/rkvs_util.erl: -------------------------------------------------------------------------------- 1 | %%-*- mode: erlang -*- 2 | %% 3 | %% This Source Code Form is subject to the terms of the Mozilla Public 4 | %% License, v. 2.0. If a copy of the MPL was not distributed with this 5 | %% file, You can obtain one at http://mozilla.org/MPL/2.0/. 6 | 7 | -module(rkvs_util). 8 | 9 | -include("rkvs.hrl"). 10 | 11 | -export([fold_options/2]). 12 | -export([enc/3, dec/3]). 13 | 14 | fold_options([], Options) -> 15 | Options; 16 | fold_options([{start_key, Start} | Rest], Options) 17 | when is_binary(Start) or (Start =:= first) -> 18 | fold_options(Rest, Options#fold_options{gte=Start}); 19 | fold_options([{end_key, End} | Rest], Options) 20 | when is_binary(End) or (End == nil) -> 21 | fold_options(Rest, Options#fold_options{lte=End}); 22 | fold_options([{gt, GT} | Rest], Options) 23 | when is_binary(GT) or (GT =:= first) -> 24 | fold_options(Rest, Options#fold_options{gt=GT}); 25 | fold_options([{gte, GT} | Rest], Options) 26 | when is_binary(GT) or (GT =:= first) -> 27 | fold_options(Rest, Options#fold_options{gte=GT}); 28 | fold_options([{lt, LT} | Rest], Options) 29 | when is_binary(LT) or (LT == nil) -> 30 | fold_options(Rest, Options#fold_options{lt=LT}); 31 | fold_options([{lte, LT} | Rest], Options) 32 | when is_binary(LT) or (LT == nil) -> 33 | fold_options(Rest, Options#fold_options{lte=LT}); 34 | fold_options([{max, Max} | Rest], Options) -> 35 | fold_options(Rest, Options#fold_options{max=Max}); 36 | fold_options([_ | Rest], Options) -> 37 | fold_options(Rest, Options). 38 | 39 | enc(_, V, raw) -> V; 40 | enc(_, V, term) -> term_to_binary(V); 41 | enc(_, V, {term, Opts}) -> term_to_binary(V, Opts); 42 | enc(_, V, sext) -> sext:encode(V); 43 | enc(T, V, E) -> erlang:error({cannot_encode, [T, V, E]}). 44 | 45 | dec(_, V, raw) -> V; 46 | dec(_, V, term) -> binary_to_term(V); 47 | dec(_, V, {term, _}) -> binary_to_term(V); 48 | dec(_, V, sext) -> sext:decode(V); 49 | dec(T, V, E) -> erlang:error({cannot_decode, [T, V, E]}). 50 | -------------------------------------------------------------------------------- /src/rkvs_storage_backend.erl: -------------------------------------------------------------------------------- 1 | %%-*- mode: erlang -*- 2 | %% 3 | %% This Source Code Form is subject to the terms of the Mozilla Public 4 | %% License, v. 2.0. If a copy of the MPL was not distributed with this 5 | %% file, You can obtain one at http://mozilla.org/MPL/2.0/. 6 | 7 | -module(rkvs_storage_backend). 8 | 9 | -include("rkvs.hrl"). 10 | 11 | -type write_ops() :: [{put, Key :: binary(), Value :: any()} | 12 | {delete, Key :: binary()} | 13 | clear]. 14 | 15 | -type kv() :: {Key:: binary(), Value::binary()}. 16 | -type kvs() :: [kv()]. 17 | 18 | -type fold_options() :: [{start_key, binary()} | 19 | {end_key, binary()} | 20 | {gt, binary()} | 21 | {gte, binary()} | 22 | {lt, binary()} | 23 | {lte, binary()} | 24 | {max, integer()}]. 25 | 26 | 27 | -export_type([write_ops/0, 28 | kv/0, 29 | kvs/0, 30 | fold_options/0]). 31 | 32 | -callback open(Name :: binary(), Options :: list()) -> 33 | {ok, Engine :: engine()} 34 | | {error, Reason :: any()}. 35 | 36 | -callback close(Engine :: engine()) -> 37 | ok 38 | | {error, Reason :: any()}. 39 | 40 | -callback destroy(Engine :: engine()) -> 41 | ok 42 | | {error, Reason :: any()}. 43 | 44 | 45 | -callback contains(Engine :: engine(), Key :: binary()) -> 46 | true 47 | | false. 48 | 49 | 50 | -callback get(Engine :: engine(), Key :: binary()) -> 51 | {ok, Value :: any()} 52 | | {error, Reason :: any()}. 53 | 54 | -callback put(Engine :: engine(), Key :: binary(), Value :: any()) -> 55 | ok 56 | | {error, Reason :: any()}. 57 | 58 | 59 | -callback clear(Engine :: engine(), Key :: binary()) -> 60 | ok 61 | | {error, Reason :: any()}. 62 | 63 | 64 | -callback write_batch(Engine :: engine(), Ops :: write_ops()) -> 65 | ok 66 | | {error, Reason :: any()}. 67 | 68 | 69 | -callback scan(Engine :: engine(), Start :: binary(), End :: binary(), 70 | Max :: integer()) -> 71 | KeyValues :: kvs() 72 | | {error, Reason :: any()}. 73 | 74 | -callback clear_range(Engine :: engine(), Start :: binary(), End :: binary(), 75 | Max :: integer()) -> 76 | ok 77 | | {error, Reason :: any()}. 78 | 79 | 80 | -callback fold_keys(Engine :: engine(), Fun :: function(), AccIn :: any(), 81 | Options :: fold_options()) -> 82 | AccOut :: any() 83 | | {error, Reason :: any()}. 84 | 85 | -callback fold(Engine :: engine(), Fun :: function(), AccIn :: any(), 86 | Options :: fold_options()) -> 87 | AccOut :: any() 88 | | {error, Reason :: any()}. 89 | 90 | -callback is_empty(Engine :: engine()) -> 91 | boolean() 92 | | {error, term()}. 93 | -------------------------------------------------------------------------------- /doc/rkvs_bitcask.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Module rkvs_bitcask # 4 | * [Description](#description) 5 | * [Function Index](#index) 6 | * [Function Details](#functions) 7 | 8 | 9 | bitcask backend. 10 | __Behaviours:__ [`rkvs_storage_backend`](rkvs_storage_backend.md). 11 | 12 | 13 | ## Description ## 14 | 15 | 16 | 17 | You can pass any options from 18 | [bitcask](https://github.com/basho/bitcask) 19 | when opening the database using the db_opts settings. 20 | 21 | 22 | Optionnaly you can pass a db_dir option to set the path of the database. 23 | 24 | ## Function Index ## 25 | 26 | 27 |
clear/2
clear_range/4
close/1
contains/2
destroy/1
fold/4
fold_keys/4
get/2
is_empty/1
open/2
put/3
scan/4
write_batch/2
28 | 29 | 30 | 31 | 32 | ## Function Details ## 33 | 34 | 35 | 36 | ### clear/2 ### 37 | 38 | `clear(Engine, Key) -> any()` 39 | 40 | 41 | 42 | 43 | ### clear_range/4 ### 44 | 45 | `clear_range(Engine, Start, End, Max) -> any()` 46 | 47 | 48 | 49 | 50 | ### close/1 ### 51 | 52 | `close(Engine) -> any()` 53 | 54 | 55 | 56 | 57 | ### contains/2 ### 58 | 59 | `contains(Engine, Key) -> any()` 60 | 61 | 62 | 63 | 64 | ### destroy/1 ### 65 | 66 | `destroy(Engine) -> any()` 67 | 68 | 69 | 70 | 71 | ### fold/4 ### 72 | 73 | `fold(Engine, Fun, Acc, Opts) -> any()` 74 | 75 | 76 | 77 | 78 | ### fold_keys/4 ### 79 | 80 | `fold_keys(Engine, Fun, Acc, Opts) -> any()` 81 | 82 | 83 | 84 | 85 | ### get/2 ### 86 | 87 | `get(Engine, Key) -> any()` 88 | 89 | 90 | 91 | 92 | ### is_empty/1 ### 93 | 94 | `is_empty(Engine) -> any()` 95 | 96 | 97 | 98 | 99 | ### open/2 ### 100 | 101 | `open(Name, Options) -> any()` 102 | 103 | 104 | 105 | 106 | ### put/3 ### 107 | 108 | `put(Engine, Key, Value) -> any()` 109 | 110 | 111 | 112 | 113 | ### scan/4 ### 114 | 115 | `scan(Engine, Start, End, Max) -> any()` 116 | 117 | 118 | 119 | 120 | ### write_batch/2 ### 121 | 122 | `write_batch(Engine, Ops) -> any()` 123 | 124 | 125 | -------------------------------------------------------------------------------- /doc/rkvs_hanoidb.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Module rkvs_hanoidb # 4 | * [Description](#description) 5 | * [Function Index](#index) 6 | * [Function Details](#functions) 7 | 8 | 9 | hanoidb backend. 10 | __Behaviours:__ [`rkvs_storage_backend`](rkvs_storage_backend.md). 11 | 12 | 13 | ## Description ## 14 | 15 | 16 | 17 | You can pass any options from 18 | [hanoidb](https://github.com/bkrestenkra/erocksdb) 19 | when opening the database using the db_opts settings. 20 | 21 | 22 | Optionnaly you can pass a db_dir option to set the path of the database. 23 | 24 | ## Function Index ## 25 | 26 | 27 |
clear/2
clear_range/4
close/1
contains/2
destroy/1
fold/4
fold_keys/4
get/2
is_empty/1
open/2
put/3
scan/4
write_batch/2
28 | 29 | 30 | 31 | 32 | ## Function Details ## 33 | 34 | 35 | 36 | ### clear/2 ### 37 | 38 | `clear(Engine, Key) -> any()` 39 | 40 | 41 | 42 | 43 | ### clear_range/4 ### 44 | 45 | `clear_range(Engine, Start, End, Max) -> any()` 46 | 47 | 48 | 49 | 50 | ### close/1 ### 51 | 52 | `close(Engine) -> any()` 53 | 54 | 55 | 56 | 57 | ### contains/2 ### 58 | 59 | `contains(Engine, Key) -> any()` 60 | 61 | 62 | 63 | 64 | ### destroy/1 ### 65 | 66 | `destroy(Engine) -> any()` 67 | 68 | 69 | 70 | 71 | ### fold/4 ### 72 | 73 | `fold(Engine, Fun, Acc0, Opts) -> any()` 74 | 75 | 76 | 77 | 78 | ### fold_keys/4 ### 79 | 80 | `fold_keys(Engine, Fun, Acc0, Opts) -> any()` 81 | 82 | 83 | 84 | 85 | ### get/2 ### 86 | 87 | `get(Engine, Key) -> any()` 88 | 89 | 90 | 91 | 92 | ### is_empty/1 ### 93 | 94 | `is_empty(Engine) -> any()` 95 | 96 | 97 | 98 | 99 | ### open/2 ### 100 | 101 | `open(Name, Options) -> any()` 102 | 103 | 104 | 105 | 106 | ### put/3 ### 107 | 108 | `put(Engine, Key, Value) -> any()` 109 | 110 | 111 | 112 | 113 | ### scan/4 ### 114 | 115 | `scan(Engine, Start, End, Max) -> any()` 116 | 117 | 118 | 119 | 120 | ### write_batch/2 ### 121 | 122 | `write_batch(Engine, Ops0) -> any()` 123 | 124 | 125 | -------------------------------------------------------------------------------- /doc/rkvs_ets.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Module rkvs_ets # 4 | * [Description](#description) 5 | * [Function Index](#index) 6 | * [Function Details](#functions) 7 | 8 | 9 | ETS backend 10 | Note: value encoding is ignored since everything is encoded as a term. 11 | __Behaviours:__ [`rkvs_storage_backend`](rkvs_storage_backend.md). 12 | 13 | 14 | ## Function Index ## 15 | 16 | 17 |
clear/2
clear_range/4
close/1
contains/2
destroy/1
fold/4
fold_keys/4
get/2
is_empty/1Returns true if this backend contains any values; otherwise returns false.
open/2
put/3
scan/4
write_batch/2
18 | 19 | 20 | 21 | 22 | ## Function Details ## 23 | 24 | 25 | 26 | ### clear/2 ### 27 | 28 | `clear(Engine, Key) -> any()` 29 | 30 | 31 | 32 | 33 | ### clear_range/4 ### 34 | 35 | `clear_range(Engine, Start, End, Max) -> any()` 36 | 37 | 38 | 39 | 40 | ### close/1 ### 41 | 42 | `close(Engine) -> any()` 43 | 44 | 45 | 46 | 47 | ### contains/2 ### 48 | 49 | `contains(Engine, Key) -> any()` 50 | 51 | 52 | 53 | 54 | ### destroy/1 ### 55 | 56 | `destroy(Engine) -> any()` 57 | 58 | 59 | 60 | 61 | ### fold/4 ### 62 | 63 | `fold(Engine, Fun, Acc0, Opts) -> any()` 64 | 65 | 66 | 67 | 68 | ### fold_keys/4 ### 69 | 70 | `fold_keys(Engine, Fun, Acc0, Opts) -> any()` 71 | 72 | 73 | 74 | 75 | ### get/2 ### 76 | 77 | `get(Engine, Key) -> any()` 78 | 79 | 80 | 81 | 82 | ### is_empty/1 ### 83 | 84 | 85 | 

 86 | is_empty(Engine::engine()) -> boolean() | {error, term()}
 87 |
88 |
89 | 90 | Returns true if this backend contains any values; otherwise returns false. 91 | 92 | 93 | ### open/2 ### 94 | 95 | `open(Name, Options) -> any()` 96 | 97 | 98 | 99 | 100 | ### put/3 ### 101 | 102 | `put(Engine, Key, Value) -> any()` 103 | 104 | 105 | 106 | 107 | ### scan/4 ### 108 | 109 | `scan(Engine, Start, End, Max) -> any()` 110 | 111 | 112 | 113 | 114 | ### write_batch/2 ### 115 | 116 | `write_batch(Engine, Ops) -> any()` 117 | 118 | 119 | -------------------------------------------------------------------------------- /doc/rkvs_rocksdb.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Module rkvs_rocksdb # 4 | * [Description](#description) 5 | * [Function Index](#index) 6 | * [Function Details](#functions) 7 | 8 | 9 | erocksdb backend. 10 | __Behaviours:__ [`rkvs_storage_backend`](rkvs_storage_backend.md). 11 | 12 | 13 | ## Description ## 14 | 15 | 16 | 17 | You can pass any options from 18 | [erocksdb](https://github.com/leo-project/erocksdb) 19 | when opening the database using the db_opts settings. 20 | 21 | 22 | Optionnaly you can pass a db_dir option to set the path of the database. 23 | 24 | ## Function Index ## 25 | 26 | 27 |
clear/2
clear_range/4
close/1
contains/2
destroy/1
fold/4
fold_keys/4
get/2
is_empty/1Returns true if this backend contains any values; otherwise returns false.
open/2
put/3
scan/4
write_batch/2
28 | 29 | 30 | 31 | 32 | ## Function Details ## 33 | 34 | 35 | 36 | ### clear/2 ### 37 | 38 | `clear(Engine, Key) -> any()` 39 | 40 | 41 | 42 | 43 | ### clear_range/4 ### 44 | 45 | `clear_range(Engine, Start, End, Max) -> any()` 46 | 47 | 48 | 49 | 50 | ### close/1 ### 51 | 52 | `close(Engine) -> any()` 53 | 54 | 55 | 56 | 57 | ### contains/2 ### 58 | 59 | `contains(Engine, Key) -> any()` 60 | 61 | 62 | 63 | 64 | ### destroy/1 ### 65 | 66 | `destroy(Engine) -> any()` 67 | 68 | 69 | 70 | 71 | ### fold/4 ### 72 | 73 | `fold(Engine, Fun, Acc0, Opts) -> any()` 74 | 75 | 76 | 77 | 78 | ### fold_keys/4 ### 79 | 80 | `fold_keys(Engine, Fun, Acc0, Opts) -> any()` 81 | 82 | 83 | 84 | 85 | ### get/2 ### 86 | 87 | `get(Engine, Key) -> any()` 88 | 89 | 90 | 91 | 92 | ### is_empty/1 ### 93 | 94 | 95 | 

 96 | is_empty(Engine::engine()) -> boolean() | {error, term()}
 97 |
98 |
99 | 100 | Returns true if this backend contains any values; otherwise returns false. 101 | 102 | 103 | ### open/2 ### 104 | 105 | `open(Name, Options) -> any()` 106 | 107 | 108 | 109 | 110 | ### put/3 ### 111 | 112 | `put(Engine, Key, Value) -> any()` 113 | 114 | 115 | 116 | 117 | ### scan/4 ### 118 | 119 | `scan(Engine, Start, End, Max) -> any()` 120 | 121 | 122 | 123 | 124 | ### write_batch/2 ### 125 | 126 | `write_batch(Engine, Ops0) -> any()` 127 | 128 | 129 | -------------------------------------------------------------------------------- /doc/rkvs_leveldb.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Module rkvs_leveldb # 4 | * [Description](#description) 5 | * [Function Index](#index) 6 | * [Function Details](#functions) 7 | 8 | 9 | eleveldb backendi. 10 | __Behaviours:__ [`rkvs_storage_backend`](rkvs_storage_backend.md). 11 | 12 | 13 | ## Description ## 14 | 15 | 16 | 17 | You can pass any options from 18 | [eleveldb](https://github.com/basho/eleveldb/blob/develop/src/eleveldb.erl) 19 | when opening the database using the db_opts settings. 20 | 21 | 22 | Optionnaly you can pass a db_dir option to set the path of the database. 23 | 24 | ## Function Index ## 25 | 26 | 27 |
clear/2
clear_range/4
close/1
contains/2
destroy/1
fold/4
fold_keys/4
get/2
is_empty/1Returns true if this backend contains any values; otherwise returns false.
open/2
put/3
scan/4
write_batch/2
28 | 29 | 30 | 31 | 32 | ## Function Details ## 33 | 34 | 35 | 36 | ### clear/2 ### 37 | 38 | `clear(Engine, Key) -> any()` 39 | 40 | 41 | 42 | 43 | ### clear_range/4 ### 44 | 45 | `clear_range(Engine, Start, End, Max) -> any()` 46 | 47 | 48 | 49 | 50 | ### close/1 ### 51 | 52 | `close(Engine) -> any()` 53 | 54 | 55 | 56 | 57 | ### contains/2 ### 58 | 59 | `contains(Engine, Key) -> any()` 60 | 61 | 62 | 63 | 64 | ### destroy/1 ### 65 | 66 | `destroy(Engine) -> any()` 67 | 68 | 69 | 70 | 71 | ### fold/4 ### 72 | 73 | `fold(Engine, Fun, Acc0, Opts) -> any()` 74 | 75 | 76 | 77 | 78 | ### fold_keys/4 ### 79 | 80 | `fold_keys(Engine, Fun, Acc0, Opts) -> any()` 81 | 82 | 83 | 84 | 85 | ### get/2 ### 86 | 87 | `get(Engine, Key) -> any()` 88 | 89 | 90 | 91 | 92 | ### is_empty/1 ### 93 | 94 | 95 | 

 96 | is_empty(Engine::engine()) -> boolean() | {error, term()}
 97 |
98 |
99 | 100 | Returns true if this backend contains any values; otherwise returns false. 101 | 102 | 103 | ### open/2 ### 104 | 105 | `open(Name, Options) -> any()` 106 | 107 | 108 | 109 | 110 | ### put/3 ### 111 | 112 | `put(Engine, Key, Value) -> any()` 113 | 114 | 115 | 116 | 117 | ### scan/4 ### 118 | 119 | `scan(Engine, Start, End, Max) -> any()` 120 | 121 | 122 | 123 | 124 | ### write_batch/2 ### 125 | 126 | `write_batch(Engine, Ops0) -> any()` 127 | 128 | 129 | -------------------------------------------------------------------------------- /doc/overview.edoc: -------------------------------------------------------------------------------- 1 | %% This Source Code Form is subject to the terms of the Mozilla Public 2 | %% License, v. 2.0. If a copy of the MPL was not distributed with this 3 | %% file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | @copyright 2014-2015 Benoît Chesneau. 6 | @version 0.2.1 7 | @title rkvs - Simple Erlang Key/Values interface 8 | 9 | @doc 10 | 11 | rkvs is a simple Key-Value database interface. It offers for now a frontend to 12 | the following K/V storages: ets, leveldb, rocksdb, hanoidb, bitcask. 13 | 14 | 15 | ## Example of usage 16 | 17 | ### Enable a backend 18 | 19 | To enable one of the backend, add one of the following line to your rebar or 20 | make the depency availavle in ERL_LIBS: 21 | 22 | ``` 23 | {leveldb, ".*", {git, "https://github.com/basho/eleveldb.git", 24 | {tag, "2.1.0"}}}, 25 | 26 | {erocksdb, ".*", {git, "https://github.com/leo-project/erocksdb.git", 27 | {branch, "develop"}}}, 28 | 29 | {hanoidb, ".*", {git, "https://github.com/krestenkrab/hanoidb.git", 30 | "4e82d5f81ab087f038bfd13354ff48ee9113f459"}}, 31 | 32 | {bitcask, ".*", {git, "https://github.com/basho/bitcask.git", 33 | {tag, "2.0.0"}}}, 34 | ''' 35 | 36 | ### Create a database: 37 | 38 | ``` 39 | Name = "mydb", 40 | {ok, Engine} = rkvs:open(Name, [{backend, rkvs_leveldb}]). 41 | ''' 42 | 43 | 5 backends are available: 44 | 45 | - `rkvs_ets': ETS backend 46 | - `rkvs_leveldb': LevelDB backend using [eleveldb](https://github.com/basho/eleveldb) from Basho. 47 | - `rkvs_rocksdb': [rocksdb](http://rocksdb.org) backend using 48 | [erocksdb](https://github.com/leo-project/erocksdb). 49 | - `rkvs_hanoidb': [hanoidb](https://github.com/krestenkrab/hanoidb) backend 50 | - `rkvs_bitcask': [bitcask](https://github.com/basho/bitcask) backend 51 | 52 | ### Store a values 53 | 54 | Storing a value associated to a key using `rkvs:put/3': 55 | 56 | ``` 57 | Key = <<"a">>, 58 | Value = 1, 59 | ok = rkvs:put(Engine, Key, Value). 60 | ''' 61 | 62 | ### Retrieve a value 63 | 64 | Use the `rkvs:get/2' function to retrieve a value. 65 | 66 | ``` 67 | Value = rkvs:get(Engine, Key). 68 | ''' 69 | 70 | Value should be 1 71 | 72 | > Note: you can use `rkvs:contains/2'. 73 | 74 | ### Delete a value 75 | 76 | Use `rkvs:clear/2' to delete a value: 77 | 78 | ``` 79 | ok = rkvs:clear(Engine, Key). 80 | ''' 81 | 82 | 83 | ### Store and write multiples values in one pass: 84 | 85 | Using `rkvs:write_batch/2' you can write and delete multiple values in one 86 | pass: 87 | 88 | ``` 89 | ok = rkvs:write_batch(Engine, [{put, <<"a">>, 1}, 90 | {put, <<"b">>, 2}, 91 | {put, <<"c">>, 3}]), 92 | 93 | ok = rkvs:write_batch(Engine, [{put, <<"d">>, 4}, 94 | {delete, <<"b">>}, 95 | {put, <<"e">>, 5}]). 96 | ''' 97 | 98 | 99 | ### Retrieve multiple values in a range: 100 | 101 | Use `rkvs:scan/4' to retrieve multiples K/Vs as a list: 102 | 103 | ``` 104 | Result = rkvs:scan(Engine, first, nil, 0). 105 | ''' 106 | 107 | Result should be `[{<<"a">>,1},{<<"c">>,3},{<<"d">>,4},{<<"e">>,5}]' 108 | 109 | > Use `rkvs:fold/5' to pass a function to keys in a range instead of retrieving 110 | > all the values as a list. `rkvs:clear_range/4' can be used to clear all the 111 | > values in a range. 112 | 113 | 114 | ### Close a storage 115 | 116 | Close a storage using `rkvs:close/1': 117 | 118 | ``` 119 | rkvs:close(Engine) 120 | ''' 121 | 122 | > You can use `rkvs:destroy/1' to close and delete the full storage 123 | 124 | 125 | 126 | ## Ownership and License 127 | 128 | The contributors are listed in AUTHORS. This project uses the MPL v2 129 | license, see LICENSE. 130 | 131 | rkvs uses the [C4.1 (Collective Code Construction 132 | Contract)](http://rfc.zeromq.org/spec:22) process for contributions. 133 | 134 | ## Development 135 | 136 | Under C4.1 process, you are more than welcome to help us by: 137 | 138 | * join the discussion over anything from design to code style try out 139 | * and [submit issue reports](https://github.com/refuge/rkvs/issues/new) 140 | * or feature requests pick a task in 141 | * [issues](https://github.com/refuge/rkvs/issues) and get it done fork 142 | * the repository and have your own fixes send us pull requests and even 143 | * star this project ^_^ 144 | 145 | To run the test suite: 146 | 147 | ``` 148 | make test 149 | ''' 150 | 151 | @end 152 | -------------------------------------------------------------------------------- /test/rkvs_bitcask_tests.erl: -------------------------------------------------------------------------------- 1 | %%-*- mode: erlang -*- 2 | %% 3 | %% This Source Code Form is subject to the terms of the Mozilla Public 4 | %% License, v. 2.0. If a copy of the MPL was not distributed with this 5 | %% file, You can obtain one at http://mozilla.org/MPL/2.0/. 6 | %% 7 | -module(rkvs_bitcask_tests). 8 | 9 | -include_lib("include/rkvs.hrl"). 10 | -include("rkvs_tests.hrl"). 11 | 12 | 13 | basic_test() -> 14 | {ok, Engine} = rkvs:open(?tempdb(), [{backend, rkvs_bitcask}]), 15 | ?assertMatch(ok, rkvs:put(Engine, <<"a">>, 1)), 16 | ?assertMatch(1, rkvs:get(Engine, <<"a">>)), 17 | rkvs:destroy(Engine). 18 | 19 | contains_test() -> 20 | {ok, Engine} = rkvs:open(?tempdb(), [{backend, rkvs_bitcask}]), 21 | ?assert(rkvs:contains(Engine, <<"a">>) =:= false), 22 | ?assertMatch({error, not_found}, rkvs:get(Engine, <<"a">>)), 23 | ok = rkvs:put(Engine, <<"a">>, 1), 24 | ?assert(rkvs:contains(Engine, <<"a">>)), 25 | rkvs:destroy(Engine). 26 | 27 | delete_test() -> 28 | {ok, Engine} = rkvs:open(?tempdb(), [{backend, rkvs_bitcask}]), 29 | ok = rkvs:put(Engine, <<"a">>, 1), 30 | ?assertMatch(ok, rkvs:clear(Engine, <<"a">>)), 31 | ?assertMatch(false, rkvs:contains(Engine, <<"a">>)), 32 | ?assertMatch(ok, rkvs:clear(Engine, <<"a">>)), 33 | rkvs:destroy(Engine). 34 | 35 | update_test() -> 36 | {ok, Engine} = rkvs:open(?tempdb(), [{backend, rkvs_bitcask}]), 37 | ok = rkvs:put(Engine, <<"a">>, 1), 38 | ok = rkvs:put(Engine, <<"a">>, 2), 39 | ?assertMatch(2, rkvs:get(Engine, <<"a">>)), 40 | rkvs:destroy(Engine). 41 | 42 | fold_test() -> 43 | {ok, Engine} = rkvs:open(?tempdb(), [{backend, rkvs_bitcask}]), 44 | ok = rkvs:put(Engine, <<"a">>, 1), 45 | ok = rkvs:put(Engine, <<"b">>, 2), 46 | ok = rkvs:put(Engine, <<"c">>, 3), 47 | 48 | %% test range scanning 49 | ?assertMatch([{<<"a">>, 1}, {<<"b">>, 2}, {<<"c">>, 3}], 50 | rkvs:scan(Engine, first, nil, 0)), 51 | 52 | ?assertMatch([{<<"b">>, 2}, {<<"c">>, 3}], 53 | rkvs:scan(Engine, <<"b">>, nil, 0)), 54 | 55 | 56 | %% test max 57 | ok = rkvs:put(Engine, <<"d">>, 4), 58 | 59 | ?assertMatch([{<<"b">>, 2}, {<<"c">>, 3}], 60 | rkvs:scan(Engine, <<"b">>, <<"c">>, 0)), 61 | 62 | 63 | ?assertMatch([{<<"a">>, 1}, {<<"b">>, 2}], 64 | rkvs:scan(Engine, first, nil, 2)), 65 | 66 | 67 | ?assertMatch([{<<"b">>, 2}], 68 | rkvs:scan(Engine, <<"b">>, <<"c">>, 1)), 69 | 70 | 71 | %% test gt, lt, ge, lte 72 | AccFun = fun({K, V}, Acc) -> 73 | [{K, V} | Acc] 74 | end, 75 | 76 | ?assertMatch([{<<"b">>, 2}, {<<"c">>, 3}, {<<"d">>, 4}], 77 | lists:reverse(rkvs:fold(Engine, AccFun, [], [{gt, <<"a">>}]))), 78 | 79 | 80 | ?assertMatch([{<<"a">>, 1}, {<<"b">>, 2}, {<<"c">>, 3}], 81 | lists:reverse(rkvs:fold(Engine, AccFun, [], [{lt, <<"d">>}]))), 82 | 83 | ?assertMatch([{<<"b">>, 2}, {<<"c">>, 3}], 84 | lists:reverse(rkvs:fold(Engine, AccFun, [], [{gt, <<"a">>}, 85 | {lt, <<"d">>}]))), 86 | 87 | ?assertMatch([{<<"b">>, 2}], 88 | rkvs:fold(Engine, AccFun, [], [{gt, <<"a">>}, 89 | {lt, <<"d">>}, 90 | {max, 1}])), 91 | 92 | rkvs:destroy(Engine). 93 | 94 | clear_range_test() -> 95 | {ok, Engine} = rkvs:open(?tempdb(), [{backend, rkvs_bitcask}]), 96 | ok = rkvs:put(Engine, <<"a">>, 1), 97 | ok = rkvs:put(Engine, <<"b">>, 2), 98 | ok = rkvs:put(Engine, <<"c">>, 3), 99 | ok = rkvs:clear_range(Engine, first, nil, 0), 100 | ?assertMatch([], rkvs:scan(Engine, first, nil, 0)), 101 | rkvs:destroy(Engine). 102 | 103 | write_batch_test() -> 104 | {ok, Engine} = rkvs:open(?tempdb(), [{backend, rkvs_bitcask}]), 105 | ok = rkvs:write_batch(Engine, [{put, <<"a">>, 1}, 106 | {put, <<"b">>, 2}, 107 | {put, <<"c">>, 3}]), 108 | ?assertMatch([{<<"a">>, 1}, {<<"b">>, 2}, {<<"c">>, 3}], 109 | rkvs:scan(Engine, first, nil, 0)), 110 | rkvs:destroy(Engine). 111 | 112 | write_delete_batch_test() -> 113 | {ok, Engine} = rkvs:open(?tempdb(), [{backend, rkvs_bitcask}]), 114 | ok = rkvs:write_batch(Engine, [{put, <<"a">>, 1}, 115 | {put, <<"b">>, 2}, 116 | {put, <<"c">>, 3}]), 117 | 118 | ok = rkvs:write_batch(Engine, [{put, <<"d">>, 4}, 119 | {delete, <<"b">>}, 120 | {put, <<"e">>, 5}]), 121 | 122 | ?assertMatch([{<<"a">>, 1}, {<<"c">>, 3}, {<<"d">>, 4}, {<<"e">>, 5}], 123 | rkvs:scan(Engine, first, nil, 0)), 124 | rkvs:destroy(Engine). 125 | -------------------------------------------------------------------------------- /doc/README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # rkvs - Simple Erlang Key/Values interface # 4 | 5 | Copyright (c) 2014-2015 Benoît Chesneau. 6 | 7 | __Version:__ 0.2.1 8 | 9 | rkvs is a simple Key-Value database interface. It offers for now a frontend to 10 | the following K/V storages: ets, leveldb, rocksdb, hanoidb, bitcask. 11 | 12 | ## Example of usage 13 | 14 | ### Enable a backend 15 | 16 | To enable one of the backend, add one of the following line to your rebar or 17 | make the depency availavle in ERL_LIBS: 18 | 19 | ``` 20 | {leveldb, ".*", {git, "https://github.com/basho/eleveldb.git", 21 | {tag, "2.1.0"}}}, 22 | 23 | {erocksdb, ".*", {git, "https://github.com/leo-project/erocksdb.git", 24 | {branch, "develop"}}}, 25 | 26 | {hanoidb, ".*", {git, "https://github.com/krestenkrab/hanoidb.git", 27 | "4e82d5f81ab087f038bfd13354ff48ee9113f459"}}, 28 | 29 | {bitcask, ".*", {git, "https://github.com/basho/bitcask.git", 30 | {tag, "2.0.0"}}}, 31 | ``` 32 | 33 | ### Create a database: 34 | 35 | ``` 36 | Name = "mydb", 37 | {ok, Engine} = rkvs:open(Name, [{backend, rkvs_leveldb}]). 38 | ``` 39 | 40 | 5 backends are available: 41 | 42 | - `rkvs_ets`: ETS backend 43 | - `rkvs_leveldb`: LevelDB backend using [eleveldb](https://github.com/basho/eleveldb) from Basho. 44 | - `rkvs_rocksdb`: [rocksdb](http://rocksdb.org) backend using 45 | [erocksdb](https://github.com/leo-project/erocksdb). 46 | - `rkvs_hanoidb`: [hanoidb](https://github.com/krestenkrab/hanoidb) backend 47 | - `rkvs_bitcask`: [bitcask](https://github.com/basho/bitcask) backend 48 | 49 | ### Store a values 50 | 51 | Storing a value associated to a key using `rkvs:put/3`: 52 | 53 | ``` 54 | Key = <<"a">>, 55 | Value = 1, 56 | ok = rkvs:put(Engine, Key, Value). 57 | ``` 58 | 59 | ### Retrieve a value 60 | 61 | Use the `rkvs:get/2` function to retrieve a value. 62 | 63 | ``` 64 | Value = rkvs:get(Engine, Key). 65 | ``` 66 | 67 | Value should be 1 68 | 69 | > Note: you can use `rkvs:contains/2`. 70 | 71 | ### Delete a value 72 | 73 | Use `rkvs:clear/2` to delete a value: 74 | 75 | ``` 76 | ok = rkvs:clear(Engine, Key). 77 | ``` 78 | 79 | ### Store and write multiples values in one pass: 80 | 81 | Using `rkvs:write_batch/2` you can write and delete multiple values in one 82 | pass: 83 | 84 | ``` 85 | ok = rkvs:write_batch(Engine, [{put, <<"a">>, 1}, 86 | {put, <<"b">>, 2}, 87 | {put, <<"c">>, 3}]), 88 | 89 | ok = rkvs:write_batch(Engine, [{put, <<"d">>, 4}, 90 | {delete, <<"b">>}, 91 | {put, <<"e">>, 5}]). 92 | ``` 93 | 94 | ### Retrieve multiple values in a range: 95 | 96 | Use `rkvs:scan/4` to retrieve multiples K/Vs as a list: 97 | 98 | ``` 99 | Result = rkvs:scan(Engine, first, nil, 0). 100 | ``` 101 | 102 | Result should be `[{<<"a">>,1},{<<"c">>,3},{<<"d">>,4},{<<"e">>,5}]` 103 | 104 | > Use `rkvs:fold/5` to pass a function to keys in a range instead of retrieving 105 | > all the values as a list. `rkvs:clear_range/4` can be used to clear all the 106 | > values in a range. 107 | 108 | ### Close a storage 109 | 110 | Close a storage using `rkvs:close/1`: 111 | 112 | ``` 113 | rkvs:close(Engine) 114 | ``` 115 | 116 | > You can use `rkvs:destroy/1` to close and delete the full storage 117 | 118 | ## Ownership and License 119 | 120 | The contributors are listed in AUTHORS. This project uses the MPL v2 121 | license, see LICENSE. 122 | 123 | rkvs uses the [C4.1 (Collective Code Construction 124 | Contract)](http://rfc.zeromq.org/spec:22) process for contributions. 125 | 126 | ## Development 127 | 128 | Under C4.1 process, you are more than welcome to help us by: 129 | 130 | * join the discussion over anything from design to code style try out 131 | * and [submit issue reports](https://github.com/refuge/rkvs/issues/new) 132 | * or feature requests pick a task in 133 | * [issues](https://github.com/refuge/rkvs/issues) and get it done fork 134 | * the repository and have your own fixes send us pull requests and even 135 | * star this project ^_^ 136 | 137 | To run the test suite: 138 | 139 | ``` 140 | make test 141 | ``` 142 | 143 | 144 | 145 | ## Modules ## 146 | 147 | 148 | 149 | 150 | 151 | 152 | 153 | 154 | 155 | 156 |
rkvs
rkvs_bitcask
rkvs_ets
rkvs_hanoidb
rkvs_leveldb
rkvs_rocksdb
rkvs_storage_backend
rkvs_util
157 | 158 | -------------------------------------------------------------------------------- /src/rkvs.erl: -------------------------------------------------------------------------------- 1 | -module(rkvs). 2 | 3 | -export([open/2, 4 | close/1, 5 | destroy/1, 6 | contains/2, 7 | get/2, 8 | put/3, 9 | clear/2, 10 | write_batch/2, 11 | scan/4, 12 | clear_range/4, 13 | fold/4, 14 | fold_keys/4, 15 | is_empty/1]). 16 | 17 | -include("rkvs.hrl"). 18 | 19 | -type key() :: term(). 20 | -type keys() :: [key()]. 21 | -type value() :: term(). 22 | -type kvs() :: [{key(), value()}]. 23 | -type ops_kvs() :: [{put, key(), value()} 24 | | {delete, key(), value()}]. 25 | -type fold_options() :: [{start_key, key()} | 26 | {end_key, key()} | 27 | {gt, key()} | {gte, key()} | 28 | {lt, key()} | {lte, key()} | 29 | {max, integer()} | 30 | {fill_cache, true | false}]. 31 | 32 | -export_type([key/0, keys/0, 33 | value/0, 34 | kvs/0, 35 | ops_kvs/0]). 36 | 37 | %% @doc open a storage, amd pass options to the backend. 38 | %% The following optinos can be used: 39 | %% 48 | -spec open(Name::binary(), Options::list()) -> 49 | {ok, engine()} | {error, any()}. 50 | open(Name, Options) -> 51 | Mod = proplists:get_value(backend, Options, rkvs_ets), 52 | Mod:open(Name, Options). 53 | 54 | 55 | %% @doc close a storage 56 | -spec close(engine()) -> ok | {error, any()}. 57 | close(#engine{mod=Mod}=Engine) -> 58 | Mod:close(Engine). 59 | 60 | %% @doc close a storage and remove all the data 61 | -spec destroy(engine()) -> ok | {error, any()}. 62 | destroy(#engine{mod=Mod}=Engine) -> 63 | Mod:destroy(Engine). 64 | 65 | %% @doc is the key exists in the storage 66 | -spec contains(engine(), key()) -> true | false. 67 | contains(#engine{mod=Mod}=Engine, Key) -> 68 | Mod:contains(Engine, Key). 69 | 70 | %% @doc get the value associated to the key 71 | -spec get(engine(), key()) -> any() | {error, term()}. 72 | get(#engine{mod=Mod}=Engine, Key) -> 73 | Mod:get(Engine, Key). 74 | 75 | %% @doc store the value associated to the key. 76 | -spec put(engine(), key(), value()) -> ok | {error, term()}. 77 | put(#engine{mod=Mod}=Engine, Key, Value) -> 78 | Mod:put(Engine, Key, Value). 79 | 80 | %% @doc delete the value associated to the key 81 | -spec clear(engine(), key()) -> ok | {error, term()}. 82 | clear(#engine{mod=Mod}=Engine, Key) -> 83 | Mod:clear(Engine, Key). 84 | 85 | %% @doc do multiple operations on the backend. 86 | -spec write_batch(engine(), ops_kvs()) -> ok | {error, term()}. 87 | write_batch(#engine{mod=Mod}=Engine, Ops) -> 88 | Mod:write_batch(Engine, Ops). 89 | 90 | %% @doc retrieve a list of Key/Value in a range 91 | -spec scan(engine(), key(), key(), integer()) -> kvs() | {error, term()}. 92 | scan(#engine{mod=Mod}=Engine, Start, End, Max) -> 93 | Mod:scan(Engine, Start, End, Max). 94 | 95 | %% @doc delete all K/Vs in a range 96 | -spec clear_range(engine(), key(), key(), integer()) -> ok | {error, term()}. 97 | clear_range(#engine{mod=Mod}=Engine, Start, End, Max) -> 98 | Mod:clear_range(Engine, Start, End, Max). 99 | 100 | %% @doc fold all keys with a function 101 | %% same parameters as in the fold function. 102 | -spec fold_keys(engine(), fun(), any(), fold_options()) -> any() | {error, term()}. 103 | fold_keys(#engine{mod=Mod}=Engine, Fun, Acc0, Opts) -> 104 | Mod:fold_keys(Engine, Fun, Acc0, Opts). 105 | 106 | %% @doc fold all K/Vs with a function 107 | %% Additionnaly you can pass the following options: 108 | %% 121 | -spec fold(engine(), function(), any(), fold_options()) -> 122 | any() | {error, term()}. 123 | fold(#engine{mod=Mod}=Engine, Fun, Acc0, Opts) -> 124 | Mod:fold(Engine, Fun, Acc0, Opts). 125 | 126 | %% @doc Returns true if this backend contains any values; otherwise returns false. 127 | -spec is_empty(engine()) -> boolean() | {error, term()}. 128 | is_empty(#engine{mod=Mod}=Engine) -> 129 | Mod:is_empty(Engine). 130 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # rkvs - Simple Erlang Key/Values interface # 4 | 5 | Copyright (c) 2014-2015 Benoît Chesneau. 6 | 7 | __Version:__ 0.2.1 8 | 9 | rkvs is a simple Key-Value database interface. It offers for now a frontend to 10 | the following K/V storages: ets, leveldb, rocksdb, hanoidb, bitcask. 11 | 12 | ## Example of usage 13 | 14 | ### Enable a backend 15 | 16 | To enable one of the backend, add one of the following line to your rebar or 17 | make the depency availavle in ERL_LIBS: 18 | 19 | ``` 20 | {leveldb, ".*", {git, "https://github.com/basho/eleveldb.git", 21 | {tag, "2.1.0"}}}, 22 | 23 | {erocksdb, ".*", {git, "https://github.com/leo-project/erocksdb.git", 24 | {branch, "develop"}}}, 25 | 26 | {hanoidb, ".*", {git, "https://github.com/krestenkrab/hanoidb.git", 27 | "4e82d5f81ab087f038bfd13354ff48ee9113f459"}}, 28 | 29 | {bitcask, ".*", {git, "https://github.com/basho/bitcask.git", 30 | {tag, "2.0.0"}}}, 31 | ``` 32 | 33 | ### Create a database: 34 | 35 | ``` 36 | Name = "mydb", 37 | {ok, Engine} = rkvs:open(Name, [{backend, rkvs_leveldb}]). 38 | ``` 39 | 40 | 5 backends are available: 41 | 42 | - `rkvs_ets`: ETS backend 43 | - `rkvs_leveldb`: LevelDB backend using [eleveldb](https://github.com/basho/eleveldb) from Basho. 44 | - `rkvs_rocksdb`: [rocksdb](http://rocksdb.org) backend using 45 | [erocksdb](https://github.com/leo-project/erocksdb). 46 | - `rkvs_hanoidb`: [hanoidb](https://github.com/krestenkrab/hanoidb) backend 47 | - `rkvs_bitcask`: [bitcask](https://github.com/basho/bitcask) backend 48 | 49 | ### Store a values 50 | 51 | Storing a value associated to a key using `rkvs:put/3`: 52 | 53 | ``` 54 | Key = <<"a">>, 55 | Value = 1, 56 | ok = rkvs:put(Engine, Key, Value). 57 | ``` 58 | 59 | ### Retrieve a value 60 | 61 | Use the `rkvs:get/2` function to retrieve a value. 62 | 63 | ``` 64 | Value = rkvs:get(Engine, Key). 65 | ``` 66 | 67 | Value should be 1 68 | 69 | > Note: you can use `rkvs:contains/2`. 70 | 71 | ### Delete a value 72 | 73 | Use `rkvs:clear/2` to delete a value: 74 | 75 | ``` 76 | ok = rkvs:clear(Engine, Key). 77 | ``` 78 | 79 | ### Store and write multiples values in one pass: 80 | 81 | Using `rkvs:write_batch/2` you can write and delete multiple values in one 82 | pass: 83 | 84 | ``` 85 | ok = rkvs:write_batch(Engine, [{put, <<"a">>, 1}, 86 | {put, <<"b">>, 2}, 87 | {put, <<"c">>, 3}]), 88 | 89 | ok = rkvs:write_batch(Engine, [{put, <<"d">>, 4}, 90 | {delete, <<"b">>}, 91 | {put, <<"e">>, 5}]). 92 | ``` 93 | 94 | ### Retrieve multiple values in a range: 95 | 96 | Use `rkvs:scan/4` to retrieve multiples K/Vs as a list: 97 | 98 | ``` 99 | Result = rkvs:scan(Engine, first, nil, 0). 100 | ``` 101 | 102 | Result should be `[{<<"a">>,1},{<<"c">>,3},{<<"d">>,4},{<<"e">>,5}]` 103 | 104 | > Use `rkvs:fold/5` to pass a function to keys in a range instead of retrieving 105 | > all the values as a list. `rkvs:clear_range/4` can be used to clear all the 106 | > values in a range. 107 | 108 | ### Close a storage 109 | 110 | Close a storage using `rkvs:close/1`: 111 | 112 | ``` 113 | rkvs:close(Engine) 114 | ``` 115 | 116 | > You can use `rkvs:destroy/1` to close and delete the full storage 117 | 118 | ## Ownership and License 119 | 120 | The contributors are listed in AUTHORS. This project uses the MPL v2 121 | license, see LICENSE. 122 | 123 | rkvs uses the [C4.1 (Collective Code Construction 124 | Contract)](http://rfc.zeromq.org/spec:22) process for contributions. 125 | 126 | ## Development 127 | 128 | Under C4.1 process, you are more than welcome to help us by: 129 | 130 | * join the discussion over anything from design to code style try out 131 | * and [submit issue reports](https://github.com/refuge/rkvs/issues/new) 132 | * or feature requests pick a task in 133 | * [issues](https://github.com/refuge/rkvs/issues) and get it done fork 134 | * the repository and have your own fixes send us pull requests and even 135 | * star this project ^_^ 136 | 137 | To run the test suite: 138 | 139 | ``` 140 | make test 141 | ``` 142 | 143 | 144 | 145 | ## Modules ## 146 | 147 | 148 | 149 | 150 | 151 | 152 | 153 | 154 | 155 | 156 |
rkvs
rkvs_bitcask
rkvs_ets
rkvs_hanoidb
rkvs_leveldb
rkvs_rocksdb
rkvs_storage_backend
rkvs_util
157 | 158 | -------------------------------------------------------------------------------- /src/rkvs_hanoidb.erl: -------------------------------------------------------------------------------- 1 | %%-*- mode: erlang -*- 2 | %% 3 | %% This Source Code Form is subject to the terms of the Mozilla Public 4 | %% License, v. 2.0. If a copy of the MPL was not distributed with this 5 | %% file, You can obtain one at http://mozilla.org/MPL/2.0/. 6 | 7 | %% @doc hanoidb backend 8 | %% 9 | %% You can pass any options from 10 | %% [hanoidb](https://github.com/bkrestenkra/erocksdb) 11 | %% when opening the database using the db_opts settings. 12 | %% 13 | %% Optionnaly you can pass a db_dir option to set the path of the database. 14 | 15 | -module(rkvs_hanoidb). 16 | -behaviour(rkvs_storage_backend). 17 | 18 | -include("rkvs.hrl"). 19 | 20 | -include_lib("hanoidb/include/hanoidb.hrl"). 21 | 22 | 23 | -export([open/2, 24 | close/1, 25 | destroy/1, 26 | contains/2, 27 | get/2, 28 | put/3, 29 | clear/2, 30 | write_batch/2, 31 | scan/4, 32 | clear_range/4, 33 | fold/4, 34 | fold_keys/4, 35 | is_empty/1]). 36 | 37 | -import(rkvs_util, [enc/3, dec/3]). 38 | 39 | open(Name, Options) -> 40 | Path = case proplists:get_value(db_dir, Options) of 41 | undefined -> Name; 42 | Dir -> filename:join(Dir, Name) 43 | end, 44 | 45 | DbOpts = proplists:get_value(db_opts, Options, [{sync_strategy, sync}]), 46 | KeyEncoding = proplists:get_value(key_encoding, Options, raw), 47 | ValueEncoding = proplists:get_value(value_encoding, Options, term), 48 | case hanoidb:open(Path, DbOpts) of 49 | {ok, Ref} -> 50 | {ok, #engine{name=Name, 51 | mod=?MODULE, 52 | ref=Ref, 53 | key_enc=KeyEncoding, 54 | val_enc=ValueEncoding, 55 | options=Options}}; 56 | Error -> 57 | Error 58 | end. 59 | 60 | close(#engine{ref=Ref}) -> 61 | hanoidb:close(Ref). 62 | 63 | destroy(#engine{ref=Ref}) -> 64 | hanoidb:destroy(Ref). 65 | 66 | contains(Engine, Key) -> 67 | Fun = fun(_K, _Acc) -> true end, 68 | fold_keys(Engine, Fun, false, [{start_key, Key}, 69 | {end_key, Key}, 70 | {max, 1}]). 71 | 72 | 73 | get(#engine{ref=Ref, key_enc=KE, val_enc=VE}, Key) -> 74 | case hanoidb:get(Ref, enc(key, Key, KE)) of 75 | {ok, Val} -> dec(value, Val, VE); 76 | not_found -> {error, not_found}; 77 | Error -> Error 78 | end. 79 | 80 | put(#engine{ref=Ref, key_enc=KE, val_enc=VE}, Key, Value) -> 81 | hanoidb:put(Ref, enc(key, Key, KE), enc(value, Value, VE)). 82 | 83 | clear(#engine{ref=Ref, key_enc=KE}, Key) -> 84 | hanoidb:delete(Ref, enc(key, Key, KE)). 85 | 86 | write_batch(#engine{ref=Ref, key_enc=KE, val_enc=VE}, Ops0) -> 87 | Ops = lists:reverse(lists:foldl(fun 88 | ({put, K, V}, Acc) -> 89 | [{put, enc(key, K, KE), enc(value, V, VE)} | Acc]; 90 | ({delete, K}, Acc) -> 91 | [{delete, enc(key, K, KE)} | Acc] 92 | end, [], Ops0)), 93 | 94 | hanoidb:transact(Ref, Ops). 95 | 96 | 97 | scan(Engine, Start, End, Max) -> 98 | AccFun = fun({K, V}, Acc) -> 99 | [{K, V} | Acc] 100 | end, 101 | AccOut = fold(Engine, AccFun, [], [{gte, Start}, 102 | {lte, End}, 103 | {max, Max}]), 104 | lists:reverse(AccOut). 105 | 106 | clear_range(Engine, Start, End, Max) -> 107 | AccFun = fun(K, Acc) -> 108 | [{delete, K} | Acc] 109 | end, 110 | Ops = fold_keys(Engine, AccFun, [], [{gte, Start}, 111 | {lte, End}, 112 | {max, Max}]), 113 | write_batch(Engine, Ops). 114 | 115 | 116 | fold_keys(#engine{ref=Ref, key_enc=KE}, Fun, Acc0, Opts) -> 117 | KeyRange = key_range(Opts, KE), 118 | WrapperFun = fun(K, _V, Acc1) -> 119 | Fun(dec(key, K, KE), Acc1) 120 | end, 121 | hanoidb:fold_range(Ref, WrapperFun, Acc0, KeyRange). 122 | 123 | 124 | fold(#engine{ref=Ref, key_enc=KE, val_enc=VE}, Fun, Acc0, Opts) -> 125 | KeyRange = key_range(Opts, KE), 126 | WrapperFun = fun(K, V, Acc1) -> 127 | Fun({dec(key, K, KE), dec(value, V, VE)}, Acc1) 128 | end, 129 | hanoidb:fold_range(Ref, WrapperFun, Acc0, KeyRange). 130 | 131 | is_empty(Engine) -> 132 | case scan(Engine, <<>>, undefined, 1) of 133 | [] -> true; 134 | _ -> false 135 | end. 136 | 137 | key_range(Opts, KE) -> 138 | key_range(Opts, KE, #key_range{}). 139 | 140 | key_range([], _KE, KR) -> 141 | KR; 142 | key_range([{start_key, Start} | Rest], KE, KR) -> 143 | key_range(Rest, KE, KR#key_range{from_key=enc(key, Start, KE), 144 | from_inclusive=true}); 145 | key_range([{end_key, End} | Rest], KE, KR) -> 146 | key_range(Rest, KE, KR#key_range{to_key=enc(key, End, KE), 147 | to_inclusive=true}); 148 | key_range([{gt, Start} | Rest], KE, KR) -> 149 | key_range(Rest, KE, KR#key_range{from_key=enc(key, Start, KE), 150 | from_inclusive=false}); 151 | key_range([{gte, Start} | Rest], KE, KR) -> 152 | key_range(Rest, KE, KR#key_range{from_key=enc(key, Start, KE), 153 | from_inclusive=true}); 154 | key_range([{lt, End} | Rest], KE, KR) -> 155 | key_range(Rest, KE, KR#key_range{to_key=enc(key, End, KE), 156 | to_inclusive=false}); 157 | key_range([{lte, End} | Rest], KE, KR) -> 158 | key_range(Rest, KE, KR#key_range{to_key=enc(key, End, KE), 159 | to_inclusive=true}); 160 | key_range([{max, Max} | Rest], KE, KR) -> 161 | key_range(Rest, KE, KR#key_range{limit=Max}); 162 | key_range([_ | Rest], KE, KR) -> 163 | key_range(Rest, KE, KR). 164 | -------------------------------------------------------------------------------- /src/rkvs_ets.erl: -------------------------------------------------------------------------------- 1 | %%-*- mode: erlang -*- 2 | %% 3 | %% This Source Code Form is subject to the terms of the Mozilla Public 4 | %% License, v. 2.0. If a copy of the MPL was not distributed with this 5 | %% file, You can obtain one at http://mozilla.org/MPL/2.0/. 6 | 7 | %% @doc ETS backend 8 | %% Note: value encoding is ignored since everything is encoded as a term 9 | 10 | -module(rkvs_ets). 11 | -behaviour(rkvs_storage_backend). 12 | 13 | -include("rkvs.hrl"). 14 | 15 | -export([open/2, 16 | close/1, 17 | destroy/1, 18 | contains/2, 19 | get/2, 20 | put/3, 21 | clear/2, 22 | write_batch/2, 23 | scan/4, 24 | clear_range/4, 25 | fold/4, 26 | fold_keys/4, 27 | is_empty/1]). 28 | 29 | -import(rkvs_util, [enc/3, dec/3]). 30 | 31 | open(Name, Options) -> 32 | Ets = ets:new(kvdb_ets, [ordered_set,public]), 33 | KeyEncoding = proplists:get_value(key_encoding, Options, raw), 34 | 35 | {ok, #engine{name=Name, 36 | mod=?MODULE, 37 | ref=Ets, 38 | key_enc=KeyEncoding, 39 | options=Options}}. 40 | 41 | 42 | close(_Engine) -> 43 | ok. 44 | 45 | destroy(#engine{ref=Ref}) -> 46 | true = ets:delete(Ref), 47 | ok. 48 | 49 | contains(#engine{ref=Ref}, Key) -> 50 | ets:member(Ref, Key). 51 | 52 | get(#engine{ref=Ref, key_enc=KE}, Key) -> 53 | case ets:lookup(Ref, enc(key, Key, KE)) of 54 | [] -> {error, not_found}; 55 | [{Key, Value}] -> Value 56 | end. 57 | 58 | put(#engine{ref=Ref, key_enc=KE}, Key, Value) -> 59 | true = ets:insert(Ref, {enc(key, Key, KE), Value}), 60 | ok. 61 | 62 | clear(#engine{ref=Ref, key_enc=KE}, Key) -> 63 | true = ets:delete(Ref, enc(key, Key, KE)), 64 | ok. 65 | 66 | write_batch(#engine{ref=Ref, key_enc=KE}, Ops) -> 67 | lists:foreach(fun 68 | ({put, K, V}) -> 69 | true = ets:insert(Ref, {enc(key, K, KE), V}); 70 | ({delete, K}) -> 71 | true = ets:delete(Ref, enc(key, K, KE)) 72 | end, Ops). 73 | 74 | scan(Engine, Start, End, Max) -> 75 | AccFun = fun({K, V}, Acc) -> 76 | [{K, V} | Acc] 77 | end, 78 | AccOut = fold(Engine, AccFun, [], [{gte, Start}, 79 | {lte, End}, 80 | {max, Max}]), 81 | lists:reverse(AccOut). 82 | 83 | clear_range(Engine, Start, End, Max) -> 84 | AccFun = fun(K, Acc) -> 85 | [{delete, K} | Acc] 86 | end, 87 | Ops = fold_keys(Engine, AccFun, [], [{gte, Start}, 88 | {lte, End}, 89 | {max, Max}]), 90 | write_batch(Engine, Ops). 91 | 92 | fold_keys(#engine{ref=Ref, key_enc=KE}, Fun, Acc0, Opts) -> 93 | do_fold(Ref, Fun, Acc0, rkvs_util:fold_options(Opts, 94 | #fold_options{key_enc=KE}), 95 | keys_only). 96 | 97 | fold(#engine{ref=Ref, key_enc=KE}, Fun, Acc0, Opts) -> 98 | do_fold(Ref, Fun, Acc0, rkvs_util:fold_options(Opts, 99 | #fold_options{key_enc=KE}), 100 | values). 101 | 102 | do_fold(Ref, Fun, Acc0, #fold_options{gt=GT, gte=GTE, key_enc=KE}=Opts, Type) -> 103 | %% define start key and the precondition 104 | {Start, GType} = case {GT, GTE} of 105 | {nil, nil} -> {first, gte}; 106 | {nil, K} when K /= nil -> {enc(key, K, KE), gte}; 107 | {K, _} -> {enc(key, K, KE), gt} 108 | end, 109 | 110 | FoldKey = case Start of 111 | first -> 112 | %% if first and condition is greater or equal then start 113 | %% with the first key if any. 114 | ets:first(Ref); 115 | _ when GType =:= gte -> 116 | %% does the key exists ? if true, starts with it, else 117 | %% start with the next one if any 118 | case ets:member(Ref, Start) of 119 | true -> Start; 120 | false -> ets:next(Ref, Start) 121 | end; 122 | _ -> 123 | %% if condition is greater, just start with the next 124 | %% one. 125 | ets:next(Ref, Start) 126 | end, 127 | fold_loop(FoldKey, Ref, Fun, Acc0, 0, Opts, Type). 128 | 129 | 130 | fold_loop('$end_of_table', _Ref, _Fun, Acc, _N, _Opts, _Type) -> 131 | Acc; 132 | fold_loop(Key, Ref, Fun, Acc0, N0, #fold_options{lt=End}=Opts, Type) 133 | when End /= nil orelse Key < End -> 134 | fold_loop1(Key, Ref, Fun, Acc0, N0, Opts, Type); 135 | fold_loop(Key, Ref, Fun, Acc0, N0, #fold_options{lte=End}=Opts, Type) 136 | when End =:= nil orelse Key < End -> 137 | fold_loop1(Key, Ref, Fun, Acc0, N0, Opts, Type); 138 | fold_loop(Key, _Ref, Fun, Acc0, _N0, #fold_options{lt=nil, lte=Key, key_enc=KE}, 139 | keys_only) -> 140 | Fun(dec(key, Key, KE), Acc0); 141 | fold_loop(Key, Ref, Fun, Acc0, _N0, #fold_options{lt=nil, 142 | lte=Key, 143 | key_enc=KE}, values) -> 144 | case ets:lookup(Ref, Key) of 145 | [] -> Acc0; 146 | [{Key, Val}] -> Fun({dec(key, Key, KE), Val}, Acc0) 147 | end; 148 | fold_loop(_Key, _Ref, _Fun, Acc, _N, _Opts, _Type) -> 149 | Acc. 150 | 151 | fold_loop1(Key, Ref, Fun, Acc0, N0, #fold_options{max=Max}=Opts, Type) -> 152 | #fold_options{key_enc=KE}=Opts, 153 | {Acc, N} = case Type of 154 | keys_only -> {Fun(dec(key, Key, KE), Acc0), N0 + 1}; 155 | values -> 156 | case ets:lookup(Ref, Key) of 157 | [] -> {Acc0, N0 + 1}; 158 | [{Key, Val}] -> {Fun({dec(key, Key, KE), Val}, Acc0), N0 + 1} 159 | end 160 | end, 161 | if ((Max =:=0) orelse (N < Max)) -> 162 | fold_loop(ets:next(Ref, Key), Ref, Fun, Acc, N, Opts, Type); 163 | true -> 164 | Acc 165 | end. 166 | 167 | 168 | %% @doc Returns true if this backend contains any values; otherwise returns false. 169 | -spec is_empty(engine()) -> boolean() | {error, term()}. 170 | is_empty(#engine{ref=Ref}) -> 171 | case ets:info(Ref, size) of 172 | undefined -> true; 173 | Size -> Size =:= 0 174 | end. 175 | -------------------------------------------------------------------------------- /src/rkvs_rocksdb.erl: -------------------------------------------------------------------------------- 1 | %%-*- mode: erlang -*- 2 | %% 3 | %% This Source Code Form is subject to the terms of the Mozilla Public 4 | %% License, v. 2.0. If a copy of the MPL was not distributed with this 5 | %% file, You can obtain one at http://mozilla.org/MPL/2.0/. 6 | 7 | %% @doc erocksdb backend 8 | %% 9 | %% You can pass any options from 10 | %% [erocksdb](https://github.com/leo-project/erocksdb) 11 | %% when opening the database using the db_opts settings. 12 | %% 13 | %% Optionnaly you can pass a db_dir option to set the path of the database. 14 | -module(rkvs_rocksdb). 15 | -behaviour(rkvs_storage_backend). 16 | 17 | -include("rkvs.hrl"). 18 | 19 | -export([open/2, 20 | close/1, 21 | destroy/1, 22 | contains/2, 23 | get/2, 24 | put/3, 25 | clear/2, 26 | write_batch/2, 27 | scan/4, 28 | clear_range/4, 29 | fold/4, 30 | fold_keys/4, 31 | is_empty/1]). 32 | 33 | -import(rkvs_util, [enc/3, dec/3]). 34 | 35 | open(Name, Options) -> 36 | Path = case proplists:get_value(db_dir, Options) of 37 | undefined -> Name; 38 | Dir -> filename:join(Dir, Name) 39 | end, 40 | 41 | filelib:ensure_dir(filename:join(Path, "dummy")), 42 | DbOpts = proplists:get_value(db_opts, Options, 43 | [{create_if_missing, true}]), 44 | 45 | KeyEncoding = proplists:get_value(key_encoding, Options, raw), 46 | ValueEncoding = proplists:get_value(value_encoding, Options, term), 47 | 48 | case erocksdb:open(Path, DbOpts, []) of 49 | {ok, Ref} -> 50 | {ok, #engine{name=Name, 51 | mod=?MODULE, 52 | ref=Ref, 53 | key_enc=KeyEncoding, 54 | val_enc=ValueEncoding, 55 | options=Options}}; 56 | Error -> 57 | Error 58 | end. 59 | 60 | close(#engine{ref=Ref}) -> 61 | erocksdb:close(Ref). 62 | 63 | destroy(#engine{name=Name, options=Options}) -> 64 | DbOpts = proplists:get_value(db_opts, Options, 65 | [{create_if_missing, true}]), 66 | 67 | erocksdb:destroy(Name, DbOpts). 68 | 69 | contains(Engine, Key) -> 70 | Fun = fun(_K, _Acc) -> true end, 71 | fold_keys(Engine, Fun, false, [{start_key, Key}, 72 | {end_key, Key}, 73 | {max, 1}]). 74 | 75 | get(#engine{ref=Ref, key_enc=KE, val_enc=VE}, Key) -> 76 | case erocksdb:get(Ref, enc(key, Key, KE), []) of 77 | {ok, Val} -> dec(value, Val, VE); 78 | not_found -> {error, not_found}; 79 | Error -> Error 80 | end. 81 | 82 | put(#engine{ref=Ref, key_enc=KE, val_enc=VE}, Key, Value) -> 83 | erocksdb:put(Ref, enc(key, Key, KE), enc(value, Value, VE), [{sync, true}]). 84 | 85 | clear(#engine{ref=Ref, key_enc=KE}, Key) -> 86 | erocksdb:delete(Ref, enc(key, Key, KE), [{sync, true}]). 87 | 88 | write_batch(#engine{ref=Ref, key_enc=KE, val_enc=VE}, Ops0) -> 89 | Ops = lists:reverse(lists:foldl(fun 90 | ({put, K, V}, Acc) -> 91 | [{put, enc(key, K, KE), enc(value, V, VE)} | Acc]; 92 | ({delete, K}, Acc) -> 93 | [{delete, enc(key, K, KE)} | Acc] 94 | end, [], Ops0)), 95 | erocksdb:write(Ref, Ops, [{sync, true}]). 96 | 97 | scan(Engine, Start, End, Max) -> 98 | AccFun = fun({K, V}, Acc) -> 99 | [{K, V} | Acc] 100 | end, 101 | AccOut = fold(Engine, AccFun, [], [{gte, Start}, 102 | {lte, End}, 103 | {max, Max}]), 104 | lists:reverse(AccOut). 105 | 106 | clear_range(Engine, Start, End, Max) -> 107 | AccFun = fun(K, Acc) -> 108 | [{delete, K} | Acc] 109 | end, 110 | Ops = fold_keys(Engine, AccFun, [], [{gte, Start}, 111 | {lte, End}, 112 | {max, Max}]), 113 | write_batch(Engine, Ops). 114 | 115 | fold_keys(#engine{ref=Ref, key_enc=KE, val_enc=VE}, Fun, Acc0, Opts) -> 116 | FillCache = proplists:get_value(fill_cache, Opts, true), 117 | {ok, Itr} = erocksdb:iterator(Ref, [{fill_cache, FillCache}], keys_only), 118 | FoldOpts0 = #fold_options{key_enc=KE, val_enc=VE}, 119 | do_fold(Itr, Fun, Acc0, rkvs_util:fold_options(Opts, FoldOpts0)). 120 | 121 | 122 | fold(#engine{ref=Ref, key_enc=KE, val_enc=VE}, Fun, Acc0, Opts) -> 123 | FillCache = proplists:get_value(fill_cache, Opts, true), 124 | {ok, Itr} = erocksdb:iterator(Ref, [{fill_cache, FillCache}]), 125 | FoldOpts0 = #fold_options{key_enc=KE, val_enc=VE}, 126 | do_fold(Itr, Fun, Acc0, rkvs_util:fold_options(Opts, FoldOpts0)). 127 | 128 | 129 | %% private 130 | 131 | do_fold(Itr, Fun, Acc0, #fold_options{gt=GT, gte=GTE, key_enc=KE}=Opts) -> 132 | {Start, Inclusive} = case {GT, GTE} of 133 | {nil, nil} -> {first, true}; 134 | {first, _} -> {first, false}; 135 | {K, _} when is_binary(K) -> {enc(key, K, KE), false}; 136 | {_, K} -> {enc(key, K, KE), true} 137 | end, 138 | try 139 | case erocksdb:iterator_move(Itr, Start) of 140 | {ok, Start} when Inclusive /= true -> 141 | fold_loop(erocksdb:iterator_move(Itr, next), Itr, Fun, 142 | Acc0, 0, Opts); 143 | {ok, Start, _V} when Inclusive /= true -> 144 | fold_loop(erocksdb:iterator_move(Itr, next), Itr, Fun, 145 | Acc0, 0, Opts); 146 | Next -> 147 | fold_loop(Next, Itr, Fun, Acc0, 0, Opts) 148 | 149 | end 150 | after 151 | erocksdb:iterator_close(Itr) 152 | end. 153 | 154 | fold_loop({error, iterator_closed}, _Itr, _Fun, Acc0, _N, _Opts) -> 155 | throw({iterator_closed, Acc0}); 156 | fold_loop({error, invalid_iterator}, _Itr, _Fun, Acc0, _N, _Opts) -> 157 | Acc0; 158 | fold_loop({ok, K}=KT, Itr, Fun, Acc0, N0, #fold_options{lt=End}=Opts) 159 | when End /= nil, K < End -> 160 | fold_loop1(KT, Itr, Fun, Acc0, N0, Opts); 161 | fold_loop({ok, K}=KT, Itr, Fun, Acc0, N0, #fold_options{lte=End}=Opts) 162 | when End =:= nil orelse K < End -> 163 | fold_loop1(KT, Itr, Fun, Acc0, N0, Opts); 164 | fold_loop({ok, K}, _Itr, Fun, Acc0, _N, #fold_options{lt=nil, lte=K}=Opts) -> 165 | Fun(dec(key, K, Opts#fold_options.key_enc), Acc0); 166 | fold_loop({ok, _K}, _Itr, _Fun, Acc0, _N, _Opts) -> 167 | Acc0; 168 | fold_loop({ok, K, _V}=KV, Itr, Fun, Acc0, N0, #fold_options{lt=End}=Opts) 169 | when End /= nil orelse K < End -> 170 | fold_loop1(KV, Itr, Fun, Acc0, N0, Opts); 171 | fold_loop({ok, K, _V}=KV, Itr, Fun, Acc0, N0, #fold_options{lte=End}=Opts) 172 | when End =:= nil orelse K < End -> 173 | fold_loop1(KV, Itr, Fun, Acc0, N0, Opts); 174 | fold_loop({ok, K, V}, _Itr, Fun, Acc0, _N, #fold_options{lt=nil, lte=K}=Opts) -> 175 | #fold_options{key_enc=KE, val_enc=VE}=Opts, 176 | Fun({dec(key, K, KE), dec(value, V, VE)}, Acc0); 177 | fold_loop({ok, _K, _V}, _Itr, _Fun, Acc0, _N, _Opts) -> 178 | Acc0. 179 | 180 | 181 | fold_loop1({ok, K}, Itr, Fun, Acc0, N0, #fold_options{max=Max}=Opts) -> 182 | Acc = Fun(dec(key, K, Opts#fold_options.key_enc), Acc0), 183 | N = N0 + 1, 184 | if ((Max =:=0) orelse (N < Max)) -> 185 | fold_loop(erocksdb:iterator_move(Itr, next), Itr, Fun, Acc, 186 | N, Opts); 187 | true -> 188 | Acc 189 | end; 190 | fold_loop1({ok, K, V}, Itr, Fun, Acc0, N0, #fold_options{max=Max}=Opts) -> 191 | #fold_options{key_enc=KE, val_enc=VE}=Opts, 192 | Acc = Fun({enc(key, K, KE), dec(value, V, VE)}, Acc0), 193 | N = N0 + 1, 194 | if 195 | ((Max =:= 0) orelse (N < Max)) -> 196 | fold_loop(erocksdb:iterator_move(Itr, next), Itr, Fun, Acc, N, 197 | Opts); 198 | true -> 199 | Acc 200 | end. 201 | 202 | %% @doc Returns true if this backend contains any values; otherwise returns false. 203 | -spec is_empty(engine()) -> boolean() | {error, term()}. 204 | is_empty(#engine{ref=Ref}) -> 205 | erocksdb:is_empty(Ref). 206 | -------------------------------------------------------------------------------- /src/rkvs_leveldb.erl: -------------------------------------------------------------------------------- 1 | %%-*- mode: erlang -*- 2 | %% 3 | %% This Source Code Form is subject to the terms of the Mozilla Public 4 | %% License, v. 2.0. If a copy of the MPL was not distributed with this 5 | %% file, You can obtain one at http://mozilla.org/MPL/2.0/. 6 | 7 | %% @doc eleveldb backendi 8 | %% 9 | %% You can pass any options from 10 | %% [eleveldb](https://github.com/basho/eleveldb/blob/develop/src/eleveldb.erl) 11 | %% when opening the database using the db_opts settings. 12 | %% 13 | %% Optionnaly you can pass a db_dir option to set the path of the database. 14 | -module(rkvs_leveldb). 15 | -behaviour(rkvs_storage_backend). 16 | 17 | -include("rkvs.hrl"). 18 | 19 | -export([open/2, 20 | close/1, 21 | destroy/1, 22 | contains/2, 23 | get/2, 24 | put/3, 25 | clear/2, 26 | write_batch/2, 27 | scan/4, 28 | clear_range/4, 29 | fold/4, 30 | fold_keys/4, 31 | is_empty/1]). 32 | 33 | -import(rkvs_util, [enc/3, dec/3]). 34 | 35 | open(Name, Options) -> 36 | Path = case proplists:get_value(db_dir, Options) of 37 | undefined -> Name; 38 | Dir -> filename:join(Dir, Name) 39 | end, 40 | 41 | filelib:ensure_dir(filename:join(Path, "dummy")), 42 | DbOpts = proplists:get_value(db_opts, Options, 43 | [{create_if_missing, true}]), 44 | 45 | KeyEncoding = proplists:get_value(key_encoding, Options, raw), 46 | ValueEncoding = proplists:get_value(value_encoding, Options, term), 47 | 48 | case eleveldb:open(Path, DbOpts) of 49 | {ok, Ref} -> 50 | {ok, #engine{name=Name, 51 | mod=?MODULE, 52 | ref=Ref, 53 | key_enc=KeyEncoding, 54 | val_enc=ValueEncoding, 55 | options=Options}}; 56 | Error -> 57 | Error 58 | end. 59 | 60 | close(#engine{ref=Ref}) -> 61 | eleveldb:close(Ref). 62 | 63 | destroy(#engine{name=Name, options=Options}) -> 64 | DbOpts = proplists:get_value(db_opts, Options, 65 | [{create_if_missing, true}]), 66 | 67 | eleveldb:destroy(Name, DbOpts). 68 | 69 | contains(Engine, Key) -> 70 | Fun = fun(_K, _Acc) -> true end, 71 | fold_keys(Engine, Fun, false, [{start_key, Key}, 72 | {end_key, Key}, 73 | {max, 1}]). 74 | 75 | get(#engine{ref=Ref, key_enc=KE, val_enc=VE}, Key) -> 76 | case eleveldb:get(Ref, enc(key, Key, KE), []) of 77 | {ok, Val} -> dec(value, Val, VE); 78 | not_found -> {error, not_found}; 79 | Error -> Error 80 | end. 81 | 82 | put(#engine{ref=Ref, key_enc=KE, val_enc=VE}, Key, Value) -> 83 | eleveldb:put(Ref, enc(key, Key, KE), enc(value, Value, VE), [{sync, true}]). 84 | 85 | clear(#engine{ref=Ref, key_enc=KE}, Key) -> 86 | eleveldb:delete(Ref, enc(key, Key, KE), [{sync, true}]). 87 | 88 | 89 | write_batch(#engine{ref=Ref, key_enc=KE, val_enc=VE}, Ops0) -> 90 | Ops = lists:reverse(lists:foldl(fun 91 | ({put, K, V}, Acc) -> 92 | [{put, enc(key, K, KE), enc(value, V, VE)} | Acc]; 93 | ({delete, K}, Acc) -> 94 | [{delete, enc(key, K, KE)} | Acc] 95 | end, [], Ops0)), 96 | 97 | eleveldb:write(Ref, Ops, [{sync, true}]). 98 | 99 | scan(Engine, Start, End, Max) -> 100 | AccFun = fun({K, V}, Acc) -> 101 | [{K, V} | Acc] 102 | end, 103 | AccOut = fold(Engine, AccFun, [], [{gte, Start}, 104 | {lte, End}, 105 | {max, Max}]), 106 | lists:reverse(AccOut). 107 | 108 | clear_range(Engine, Start, End, Max) -> 109 | AccFun = fun(K, Acc) -> 110 | [{delete, K} | Acc] 111 | end, 112 | Ops = fold_keys(Engine, AccFun, [], [{gte, Start}, 113 | {lte, End}, 114 | {max, Max}]), 115 | write_batch(Engine, Ops). 116 | 117 | fold_keys(#engine{ref=Ref, key_enc=KE, val_enc=VE}, Fun, Acc0, Opts) -> 118 | FillCache = proplists:get_value(fill_cache, Opts, true), 119 | {ok, Itr} = eleveldb:iterator(Ref, [{fill_cache, FillCache}], keys_only), 120 | FoldOpts0 = #fold_options{key_enc=KE, val_enc=VE}, 121 | do_fold(Itr, Fun, Acc0, rkvs_util:fold_options(Opts, FoldOpts0)). 122 | 123 | 124 | fold(#engine{ref=Ref, key_enc=KE, val_enc=VE}, Fun, Acc0, Opts) -> 125 | FillCache = proplists:get_value(fill_cache, Opts, true), 126 | {ok, Itr} = eleveldb:iterator(Ref, [{fill_cache, FillCache}]), 127 | FoldOpts0 = #fold_options{key_enc=KE, val_enc=VE}, 128 | do_fold(Itr, Fun, Acc0, rkvs_util:fold_options(Opts, FoldOpts0)). 129 | 130 | 131 | %% private 132 | do_fold(Itr, Fun, Acc0, #fold_options{gt=GT, gte=GTE, key_enc=KE}=Opts) -> 133 | {Start, Inclusive} = case {GT, GTE} of 134 | {nil, nil} -> {first, true}; 135 | {first, _} -> {first, false}; 136 | {K, _} when is_binary(K) -> {enc(key, K, KE), false}; 137 | {_, K} -> {enc(key, K, KE), true} 138 | end, 139 | 140 | 141 | try 142 | case eleveldb:iterator_move(Itr, Start) of 143 | {ok, Start} when Inclusive /= true -> 144 | fold_loop(eleveldb:iterator_move(Itr, prefetch), Itr, Fun, 145 | Acc0, 0, Opts); 146 | {ok, Start, _V} when Inclusive /= true -> 147 | fold_loop(eleveldb:iterator_move(Itr, prefetch), Itr, Fun, 148 | Acc0, 0, Opts); 149 | Next -> 150 | fold_loop(Next, Itr, Fun, Acc0, 0, Opts) 151 | 152 | end 153 | after 154 | eleveldb:iterator_close(Itr) 155 | end. 156 | 157 | 158 | fold_loop({error, iterator_closed}, _Itr, _Fun, Acc0, _N, _Opts) -> 159 | throw({iterator_closed, Acc0}); 160 | fold_loop({error, invalid_iterator}, _Itr, _Fun, Acc0, _N, _Opts) -> 161 | Acc0; 162 | fold_loop({ok, K}=KT, Itr, Fun, Acc0, N0, #fold_options{lt=End}=Opts) 163 | when End /= nil, K < End -> 164 | fold_loop1(KT, Itr, Fun, Acc0, N0, Opts); 165 | fold_loop({ok, K}=KT, Itr, Fun, Acc0, N0, #fold_options{lte=End}=Opts) 166 | when End =:= nil orelse K < End -> 167 | fold_loop1(KT, Itr, Fun, Acc0, N0, Opts); 168 | fold_loop({ok, K}, _Itr, Fun, Acc0, _N, #fold_options{lt=nil, lte=K}=Opts) -> 169 | Fun(dec(key, K, Opts#fold_options.key_enc), Acc0); 170 | fold_loop({ok, _K}, _Itr, _Fun, Acc0, _N, _Opts) -> 171 | Acc0; 172 | fold_loop({ok, K, _V}=KV, Itr, Fun, Acc0, N0, #fold_options{lt=End}=Opts) 173 | when End /= nil orelse K < End -> 174 | fold_loop1(KV, Itr, Fun, Acc0, N0, Opts); 175 | fold_loop({ok, K, _V}=KV, Itr, Fun, Acc0, N0, #fold_options{lte=End}=Opts) 176 | when End =:= nil orelse K < End -> 177 | fold_loop1(KV, Itr, Fun, Acc0, N0, Opts); 178 | fold_loop({ok, K, V}, _Itr, Fun, Acc0, _N, #fold_options{lt=nil, lte=K}=Opts) -> 179 | #fold_options{key_enc=KE, val_enc=VE}=Opts, 180 | Fun({dec(key, K, KE), dec(value, V, VE)}, Acc0); 181 | fold_loop({ok, _K, _V}, _Itr, _Fun, Acc0, _N, _Opts) -> 182 | Acc0. 183 | 184 | 185 | fold_loop1({ok, K}, Itr, Fun, Acc0, N0, #fold_options{max=Max}=Opts) -> 186 | Acc = Fun(dec(key, K, Opts#fold_options.key_enc), Acc0), 187 | N = N0 + 1, 188 | if ((Max =:=0) orelse (N < Max)) -> 189 | fold_loop(eleveldb:iterator_move(Itr, prefetch), Itr, Fun, Acc, 190 | N, Opts); 191 | true -> 192 | Acc 193 | end; 194 | fold_loop1({ok, K, V}, Itr, Fun, Acc0, N0, #fold_options{max=Max}=Opts) -> 195 | #fold_options{key_enc=KE, val_enc=VE}=Opts, 196 | Acc = Fun({enc(key, K, KE), dec(value, V, VE)}, Acc0), 197 | N = N0 + 1, 198 | if 199 | ((Max =:= 0) orelse (N < Max)) -> 200 | fold_loop(eleveldb:iterator_move(Itr, prefetch), Itr, Fun, Acc, N, 201 | Opts); 202 | true -> 203 | Acc 204 | end. 205 | 206 | %% @doc Returns true if this backend contains any values; otherwise returns false. 207 | -spec is_empty(engine()) -> boolean() | {error, term()}. 208 | is_empty(#engine{ref=Ref}) -> 209 | eleveldb:is_empty(Ref). 210 | -------------------------------------------------------------------------------- /doc/rkvs.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Module rkvs # 4 | * [Data Types](#types) 5 | * [Function Index](#index) 6 | * [Function Details](#functions) 7 | 8 | 9 | 10 | 11 | 12 | ## Data Types ## 13 | 14 | 15 | 16 | 17 | ### fold_options() ### 18 | 19 | 20 | 21 | 

 22 | fold_options() = [{start_key, key()} | {end_key, key()} | {gt, key()} | {gte, key()} | {lt, key()} | {lte, key()} | {max, integer()} | {fill_cache, true | false}]
 23 |
24 | 25 | 26 | 27 | 28 | 29 | ### key() ### 30 | 31 | 32 | 33 | 

 34 | key() = term()
 35 |
36 | 37 | 38 | 39 | 40 | 41 | ### keys() ### 42 | 43 | 44 | 45 | 

 46 | keys() = [key()]
 47 |
48 | 49 | 50 | 51 | 52 | 53 | ### kvs() ### 54 | 55 | 56 | 57 | 

 58 | kvs() = [{key(), value()}]
 59 |
60 | 61 | 62 | 63 | 64 | 65 | ### ops_kvs() ### 66 | 67 | 68 | 69 | 

 70 | ops_kvs() = [{put, key(), value()} | {delete, key(), value()}]
 71 |
72 | 73 | 74 | 75 | 76 | 77 | ### value() ### 78 | 79 | 80 | 81 | 

 82 | value() = term()
 83 |
84 | 85 | 86 | 87 | 88 | ## Function Index ## 89 | 90 | 91 |
clear/2delete the value associated to the key.
clear_range/4delete all K/Vs in a range.
close/1close a storage.
contains/2is the key exists in the storage.
destroy/1close a storage and remove all the data.
fold/4fold all K/Vs with a function 92 | Additionnaly you can pass the following options: 93 |
    94 |
  • 'gt', (greater than), 'gte' (greather than or equal): define the lower 95 | bound of the range to fold. Only the records where the key is greater (or 96 | equal to) will be given to the function.
    • 97 |
  • 'lt' (less than), 'lte' (less than or equal): define the higher bound 98 | of the range to fold. Only the records where the key is less than (or equal 99 | to) will be given to the function
    • 100 |
  • 'start_key', 'end_key', legacy to 'gte', 'lte'
    • 101 |
  • 'max' (default=0), the maximum of records to fold before returning the 102 | resut
    • 103 |
  • 'fill_cache' (default is true): should be the data cached in 104 | memory?
    • 105 |
fold_keys/4fold all keys with a function 106 | same parameters as in the fold function.
get/2get the value associated to the key.
is_empty/1Returns true if this backend contains any values; otherwise returns false.
open/2open a storage, amd pass options to the backend.
put/3store the value associated to the key.
scan/4retrieve a list of Key/Value in a range.
write_batch/2do multiple operations on the backend.
107 | 108 | 109 | 110 | 111 | ## Function Details ## 112 | 113 | 114 | 115 | ### clear/2 ### 116 | 117 | 118 | 

119 | clear(Engine::engine(), Key::key()) -> ok | {error, term()}
120 |
121 |
122 | 123 | delete the value associated to the key 124 | 125 | 126 | ### clear_range/4 ### 127 | 128 | 129 | 

130 | clear_range(Engine::engine(), Start::key(), End::key(), Max::integer()) -> ok | {error, term()}
131 |
132 |
133 | 134 | delete all K/Vs in a range 135 | 136 | 137 | ### close/1 ### 138 | 139 | 140 | 

141 | close(Engine::engine()) -> ok | {error, any()}
142 |
143 |
144 | 145 | close a storage 146 | 147 | 148 | ### contains/2 ### 149 | 150 | 151 | 

152 | contains(Engine::engine(), Key::key()) -> true | false
153 |
154 |
155 | 156 | is the key exists in the storage 157 | 158 | 159 | ### destroy/1 ### 160 | 161 | 162 | 

163 | destroy(Engine::engine()) -> ok | {error, any()}
164 |
165 |
166 | 167 | close a storage and remove all the data 168 | 169 | 170 | ### fold/4 ### 171 | 172 | 173 | 

174 | fold(Engine::engine(), Fun::function(), Acc0::any(), Opts::fold_options()) -> any() | {error, term()}
175 |
176 |
177 | 178 | fold all K/Vs with a function 179 | Additionnaly you can pass the following options: 180 | 181 | * 'gt', (greater than), 'gte' (greather than or equal): define the lower 182 | bound of the range to fold. Only the records where the key is greater (or 183 | equal to) will be given to the function. 184 | 185 | * 'lt' (less than), 'lte' (less than or equal): define the higher bound 186 | of the range to fold. Only the records where the key is less than (or equal 187 | to) will be given to the function 188 | 189 | * 'start_key', 'end_key', legacy to 'gte', 'lte' 190 | 191 | * 'max' (default=0), the maximum of records to fold before returning the 192 | resut 193 | 194 | * 'fill_cache' (default is true): should be the data cached in 195 | memory? 196 | 197 | 198 | 199 | 200 | ### fold_keys/4 ### 201 | 202 | 203 | 

204 | fold_keys(Engine::engine(), Fun::function(), Acc0::any(), Opts::fold_options()) -> any() | {error, term()}
205 |
206 |
207 | 208 | fold all keys with a function 209 | same parameters as in the fold function. 210 | 211 | 212 | ### get/2 ### 213 | 214 | 215 | 

216 | get(Engine::engine(), Key::key()) -> any() | {error, term()}
217 |
218 |
219 | 220 | get the value associated to the key 221 | 222 | 223 | ### is_empty/1 ### 224 | 225 | 226 | 

227 | is_empty(Engine::engine()) -> boolean() | {error, term()}
228 |
229 |
230 | 231 | Returns true if this backend contains any values; otherwise returns false. 232 | 233 | 234 | ### open/2 ### 235 | 236 | 237 | 

238 | open(Name::binary(), Options::list()) -> {ok, engine()} | {error, any()}
239 |
240 |
241 | 242 | open a storage, amd pass options to the backend. 243 | The following optinos can be used: 244 | 245 | * 'backend': default is rkvs_ets, folowin backend are provided in rkvs: 246 | rkvs_ets, rkvs_leveldb, rkvs_rocksdb, rkvs_hanoidb, rvs_bitcask. 247 | 248 | * 'db_opts': backend options, refers to the backend doc for them 249 | 250 | * 'key_encoding': to encode the key. defautl is raw (binary). can be 251 | term, {term, Opts} or sext 252 | 253 | * 'value_encoding': to encode the value. defautl is term. can be 254 | term, {term, Opts} or sext 255 | 256 | 257 | 258 | 259 | ### put/3 ### 260 | 261 | 262 | 

263 | put(Engine::engine(), Key::key(), Value::value()) -> ok | {error, term()}
264 |
265 |
266 | 267 | store the value associated to the key. 268 | 269 | 270 | ### scan/4 ### 271 | 272 | 273 | 

274 | scan(Engine::engine(), Start::key(), End::key(), Max::integer()) -> kvs() | {error, term()}
275 |
276 |
277 | 278 | retrieve a list of Key/Value in a range 279 | 280 | 281 | ### write_batch/2 ### 282 | 283 | 284 | 

285 | write_batch(Engine::engine(), Ops::ops_kvs()) -> ok | {error, term()}
286 |
287 |
288 | 289 | do multiple operations on the backend. 290 | -------------------------------------------------------------------------------- /test/rkvs_tests.erl: -------------------------------------------------------------------------------- 1 | %%-*- mode: erlang -*- 2 | %% 3 | %% This Source Code Form is subject to the terms of the Mozilla Public 4 | %% License, v. 2.0. If a copy of the MPL was not distributed with this 5 | %% file, You can obtain one at http://mozilla.org/MPL/2.0/. 6 | %% 7 | -module(rkvs_tests). 8 | 9 | -include_lib("include/rkvs.hrl"). 10 | -include("rkvs_tests.hrl"). 11 | 12 | -define(setup(F), {setup, fun setup/0, fun teardown/1, F}). 13 | -define(foreach(Fs), {foreach, fun setup/1, fun teardown/1, Fs}). 14 | 15 | -define(dbg(Backend,E), 16 | (fun() -> 17 | try (E) of 18 | __V -> 19 | ?debugFmt(<<"backend = ~p; ~s = ~P">>, [Backend,(??E), __V, 15]), 20 | __V 21 | catch 22 | error:__Err -> 23 | io:fwrite(user, 24 | "FAIL: backend = ~p; test = ~s~n" 25 | "Error = ~p~n" 26 | "Trace = ~p~n", [Backend,(??E), __Err, 27 | erlang:get_stacktrace()]), 28 | error(__Err) 29 | end 30 | end)()). 31 | 32 | backends() -> 33 | [rkvs_ets, rkvs_leveldb, rkvs_rocksdb, rkvs_hanoidb]. 34 | 35 | basic_test() -> 36 | lists:foreach(fun(Backend) -> 37 | File = ?tempdb(), 38 | {ok, Engine} = rkvs:open(File, [{backend, Backend}]), 39 | ?assert(is_record(Engine, engine)), 40 | 41 | ?assertMatch(ok, rkvs:close(Engine)), 42 | 43 | %% test db destroy 44 | ok = rkvs:destroy(Engine) 45 | end, backends()), 46 | ok. 47 | 48 | 49 | ops_test_funs() -> 50 | [ 51 | fun should_create_kv/2, 52 | fun should_get_kv/2, 53 | fun should_contains_kv/2, 54 | fun should_return_notfound/2, 55 | fun should_delete_kv/2, 56 | fun delete_undefined_return_ok/2, 57 | fun should_replace_kv/2, 58 | fun should_scan_all/2, 59 | fun should_scan_from/2, 60 | fun should_scan_end/2, 61 | fun should_scan_max/2, 62 | fun should_scan_max2/2, 63 | fun should_fold_lt/2, 64 | fun should_fold_gt/2, 65 | fun should_fold_lt_gt/2, 66 | fun should_fold_lt_gt_max/2, 67 | fun should_clear_range/2, 68 | fun should_write_batch/2, 69 | fun should_write_delete_batch/2 70 | ]. 71 | 72 | basic_ops_test_() -> 73 | RunTests = lists:foldr(fun(Backend, Acc) -> 74 | [{Backend, Fun} || Fun <- ops_test_funs()] ++ Acc 75 | end, [], backends()), 76 | 77 | {setup, 78 | fun() -> ok end, 79 | fun(_) -> ok end, 80 | [{foreachx, 81 | fun(Backend) -> 82 | {ok, Engine} = rkvs:open(?tempdb(), [{backend, Backend}]), 83 | Engine 84 | end, 85 | fun(_Backend, Engine) -> 86 | ok = rkvs:close(Engine), 87 | ok = rkvs:destroy(Engine) 88 | end, 89 | [{Backend, fun(_, Engine) -> 90 | [?_test(?dbg(Backend, Fun(Backend, Engine)))] 91 | end} || {Backend, Fun} <- RunTests] 92 | }]}. 93 | 94 | should_create_kv(_Backend, Engine) -> 95 | ?_assertMatch(ok, rkvs:put(Engine, <<"a">>, 1)). 96 | 97 | should_get_kv(_Backend, Engine) -> 98 | ok = rkvs:put(Engine, <<"a">>, 1), 99 | ?_assertMatch(1, rkvs:get(Engine, <<"a">>)). 100 | 101 | should_contains_kv(_Backend, Engine) -> 102 | ?assert(rkvs:contains(Engine, <<"a">>) =:= false), 103 | ok = rkvs:put(Engine, <<"a">>, 1), 104 | ?_assert(rkvs:contains(Engine, <<"a">>)). 105 | 106 | should_return_notfound(_Backend, Engine) -> 107 | ?_assertMatch({error, not_found}, rkvs:get(Engine, <<"a">>)). 108 | 109 | should_delete_kv(_Backend, Engine) -> 110 | ok = rkvs:put(Engine, <<"a">>, 1), 111 | ?assertMatch(ok, rkvs:clear(Engine, <<"a">>)), 112 | ?_assertMatch(false, rkvs:contains(Engine, <<"a">>)). 113 | 114 | delete_undefined_return_ok(_Backend, Engine) -> 115 | ?_assertMatch(ok, rkvs:clear(Engine, <<"a">>)). 116 | 117 | should_replace_kv(_Backend, Engine) -> 118 | ok = rkvs:put(Engine, <<"a">>, 1), 119 | ok = rkvs:put(Engine, <<"a">>, 2), 120 | ?_assertMatch(2, rkvs:get(Engine, <<"a">>)). 121 | 122 | 123 | should_scan_all(_Backend, Engine) -> 124 | ok = rkvs:put(Engine, <<"a">>, 1), 125 | ok = rkvs:put(Engine, <<"b">>, 2), 126 | ok = rkvs:put(Engine, <<"c">>, 3), 127 | 128 | ?_assertMatch([{<<"a">>, 1}, {<<"b">>, 2}, {<<"c">>, 3}], 129 | rkvs:scan(Engine, first, nil, 0)). 130 | 131 | 132 | should_scan_from(_Backend, Engine) -> 133 | ok = rkvs:put(Engine, <<"a">>, 1), 134 | ok = rkvs:put(Engine, <<"b">>, 2), 135 | ok = rkvs:put(Engine, <<"c">>, 3), 136 | 137 | ?_assertMatch([{<<"b">>, 2}, {<<"c">>, 3}], 138 | rkvs:scan(Engine, <<"b">>, nil, 0)). 139 | 140 | should_scan_end(_Backend, Engine) -> 141 | ok = rkvs:put(Engine, <<"a">>, 1), 142 | ok = rkvs:put(Engine, <<"b">>, 2), 143 | ok = rkvs:put(Engine, <<"c">>, 3), 144 | ok = rkvs:put(Engine, <<"d">>, 4), 145 | 146 | ?_assertMatch([{<<"b">>, 2}, {<<"c">>, 3}], 147 | rkvs:scan(Engine, <<"b">>, <<"c">>, 0)). 148 | 149 | 150 | should_scan_max(_Backend, Engine) -> 151 | ok = rkvs:put(Engine, <<"a">>, 1), 152 | ok = rkvs:put(Engine, <<"b">>, 2), 153 | ok = rkvs:put(Engine, <<"c">>, 3), 154 | ok = rkvs:put(Engine, <<"d">>, 4), 155 | 156 | ?_assertMatch([{<<"a">>, 1}, {<<"b">>, 2}], 157 | rkvs:scan(Engine, first, nil, 2)). 158 | 159 | 160 | should_scan_max2(_Backend, Engine) -> 161 | ok = rkvs:put(Engine, <<"a">>, 1), 162 | ok = rkvs:put(Engine, <<"b">>, 2), 163 | ok = rkvs:put(Engine, <<"c">>, 3), 164 | ok = rkvs:put(Engine, <<"d">>, 4), 165 | 166 | ?_assertMatch([{<<"b">>, 2}], 167 | rkvs:scan(Engine, <<"b">>, <<"c">>, 1)). 168 | 169 | 170 | should_fold_lt(_Backend, Engine) -> 171 | ok = rkvs:put(Engine, <<"a">>, 1), 172 | ok = rkvs:put(Engine, <<"b">>, 2), 173 | ok = rkvs:put(Engine, <<"c">>, 3), 174 | ok = rkvs:put(Engine, <<"d">>, 4), 175 | 176 | AccFun = fun({K, V}, Acc) -> 177 | [{K, V} | Acc] 178 | end, 179 | 180 | ?_assertMatch([{<<"b">>, 2}, {<<"c">>, 3}, {<<"d">>, 4}], 181 | lists:reverse(rkvs:fold(Engine, AccFun, [], 182 | [{gt, <<"a">>}]))). 183 | 184 | 185 | should_fold_gt(_Backend, Engine) -> 186 | ok = rkvs:put(Engine, <<"a">>, 1), 187 | ok = rkvs:put(Engine, <<"b">>, 2), 188 | ok = rkvs:put(Engine, <<"c">>, 3), 189 | ok = rkvs:put(Engine, <<"d">>, 4), 190 | 191 | AccFun = fun({K, V}, Acc) -> 192 | [{K, V} | Acc] 193 | end, 194 | 195 | ?_assertMatch([{<<"a">>, 1}, {<<"b">>, 2}, {<<"c">>, 3}], 196 | lists:reverse(rkvs:fold(Engine, AccFun, [], 197 | [{lt, <<"d">>}]))). 198 | 199 | should_fold_lt_gt(_Backend, Engine) -> 200 | ok = rkvs:put(Engine, <<"a">>, 1), 201 | ok = rkvs:put(Engine, <<"b">>, 2), 202 | ok = rkvs:put(Engine, <<"c">>, 3), 203 | ok = rkvs:put(Engine, <<"d">>, 4), 204 | 205 | AccFun = fun({K, V}, Acc) -> 206 | [{K, V} | Acc] 207 | end, 208 | 209 | ?_assertMatch([{<<"b">>, 2}, {<<"c">>, 3}], 210 | lists:reverse(rkvs:fold(Engine, AccFun, [], [{gt, <<"a">>}, 211 | {lt, <<"d">>}]))). 212 | 213 | should_fold_lt_gt_max(_Backend, Engine) -> 214 | ok = rkvs:put(Engine, <<"a">>, 1), 215 | ok = rkvs:put(Engine, <<"b">>, 2), 216 | ok = rkvs:put(Engine, <<"c">>, 3), 217 | ok = rkvs:put(Engine, <<"d">>, 4), 218 | 219 | AccFun = fun({K, V}, Acc) -> 220 | [{K, V} | Acc] 221 | end, 222 | 223 | ?_assertMatch([{<<"b">>, 2}], 224 | rkvs:fold(Engine, AccFun, [], [{gt, <<"a">>}, 225 | {lt, <<"d">>}, 226 | {max, 1}])). 227 | 228 | should_clear_range(_Backend, Engine) -> 229 | ok = rkvs:put(Engine, <<"a">>, 1), 230 | ok = rkvs:put(Engine, <<"b">>, 2), 231 | ok = rkvs:put(Engine, <<"c">>, 3), 232 | 233 | ok = rkvs:clear_range(Engine, first, nil, 0), 234 | ?_assertMatch([], rkvs:scan(Engine, first, nil, 0)). 235 | 236 | should_write_batch(_Backend, Engine) -> 237 | ok = rkvs:write_batch(Engine, [{put, <<"a">>, 1}, 238 | {put, <<"b">>, 2}, 239 | {put, <<"c">>, 3}]), 240 | 241 | ?_assertMatch([{<<"a">>, 1}, {<<"b">>, 2}, {<<"c">>, 3}], 242 | rkvs:scan(Engine, first, nil, 0)). 243 | 244 | 245 | should_write_delete_batch(_Backend, Engine) -> 246 | ok = rkvs:write_batch(Engine, [{put, <<"a">>, 1}, 247 | {put, <<"b">>, 2}, 248 | {put, <<"c">>, 3}]), 249 | 250 | ok = rkvs:write_batch(Engine, [{put, <<"d">>, 4}, 251 | {delete, <<"b">>}, 252 | {put, <<"e">>, 5}]), 253 | 254 | ?_assertMatch([{<<"a">>, 1}, {<<"c">>, 3}, {<<"d">>, 4}, {<<"e">>, 5}], 255 | rkvs:scan(Engine, first, nil, 0)). 256 | -------------------------------------------------------------------------------- /src/rkvs_bitcask.erl: -------------------------------------------------------------------------------- 1 | %%-*- mode: erlang -*- 2 | %% 3 | %% This Source Code Form is subject to the terms of the Mozilla Public 4 | %% License, v. 2.0. If a copy of the MPL was not distributed with this 5 | %% file, You can obtain one at http://mozilla.org/MPL/2.0/. 6 | 7 | %% @doc bitcask backend 8 | %% 9 | %% You can pass any options from 10 | %% [bitcask](https://github.com/basho/bitcask) 11 | %% when opening the database using the db_opts settings. 12 | %% 13 | %% Optionnaly you can pass a db_dir option to set the path of the database. 14 | -module(rkvs_bitcask). 15 | -behaviour(rkvs_storage_backend). 16 | 17 | -include("rkvs.hrl"). 18 | -include_lib("bitcask/include/bitcask.hrl"). 19 | 20 | -export([open/2, 21 | close/1, 22 | destroy/1, 23 | contains/2, 24 | get/2, 25 | put/3, 26 | clear/2, 27 | write_batch/2, 28 | scan/4, 29 | clear_range/4, 30 | fold/4, 31 | fold_keys/4, 32 | is_empty/1]). 33 | 34 | -import(rkvs_util, [enc/3, dec/3]). 35 | 36 | -record(fold_st, {useracc, 37 | cb, 38 | key_enc, 39 | val_enc, 40 | num = 0, 41 | max, 42 | from = <<>>, 43 | from_inclusive = false, 44 | to = nil, 45 | to_inclusive = false}). 46 | 47 | 48 | open(Name, Options) -> 49 | Path = case proplists:get_value(db_dir, Options) of 50 | undefined -> Name; 51 | Dir -> filename:join(Dir, Name) 52 | end, 53 | 54 | DbOpts = proplists:get_value(db_opts, Options, [read_write]), 55 | KeyEncoding = proplists:get_value(key_encoding, Options, raw), 56 | ValueEncoding = proplists:get_value(value_encoding, Options, term), 57 | case catch bitcask:open(Path, DbOpts) of 58 | {error, _Reason}=Error -> Error; 59 | {'EXIT', Reason} -> {error, Reason}; 60 | Ref when is_reference(Ref) -> 61 | {ok, #engine{name=Name, 62 | mod=?MODULE, 63 | ref=Ref, 64 | key_enc=KeyEncoding, 65 | val_enc=ValueEncoding, 66 | options=Options}} 67 | end. 68 | 69 | close(#engine{ref=Ref}) -> 70 | case catch bitcask:close(Ref) of 71 | ok -> ok; 72 | {error,{invalid_ref, Ref}} -> ok; %% already closed 73 | {error, Reason} -> {error, Reason}; 74 | {'EXIT', Reason} -> {error, Reason} 75 | end. 76 | 77 | destroy(#engine{name=Name, options=Options}=Engine) -> 78 | ok = close(Engine), 79 | Path = case proplists:get_value(db_dir, Options) of 80 | undefined -> Name; 81 | Dir -> filename:join(Dir, Name) 82 | end, 83 | data_directory_cleanup(Path), 84 | ok. 85 | 86 | contains(Engine, Key) -> 87 | Fun = fun(_K, _Acc) -> true end, 88 | fold_keys(Engine, Fun, false, [{start_key, Key}, 89 | {end_key, Key}, 90 | {max, 1}]). 91 | 92 | 93 | get(#engine{ref=Ref, key_enc=KE, val_enc=VE}, Key) -> 94 | case catch bitcask:get(Ref, enc(key, Key, KE)) of 95 | {ok, Val} -> dec(value, Val, VE); 96 | not_found -> {error, not_found}; 97 | {error, _Reason}=Error -> Error; 98 | {'EXIT', Reason} -> {error, Reason} 99 | end. 100 | 101 | 102 | put(#engine{ref=Ref, key_enc=KE, val_enc=VE}, Key, Value) -> 103 | case catch bitcask:put(Ref, enc(key, Key, KE), enc(value, Value, VE)) of 104 | ok -> ok; 105 | {error, _Reason}=Error -> Error; 106 | {'EXIT', Reason} -> {error, Reason} 107 | end. 108 | 109 | clear(#engine{ref=Ref, key_enc=KE}, Key) -> 110 | case catch bitcask:delete(Ref, enc(key, Key, KE)) of 111 | ok -> ok; 112 | {error, _Reason}=Error -> Error; 113 | {'EXIT', Reason} -> {error, Reason} 114 | end. 115 | 116 | write_batch(Engine, Ops) -> 117 | do_write_batch(Ops, Engine). 118 | 119 | 120 | do_write_batch([], _Engine) -> 121 | ok; 122 | do_write_batch([{put, K, V} | Rest], Engine) -> 123 | ok = rkvs_bitcask:put(Engine, K, V), 124 | do_write_batch(Rest, Engine); 125 | do_write_batch([{delete, K} | Rest], Engine) -> 126 | ok = clear(Engine, K), 127 | do_write_batch(Rest, Engine). 128 | 129 | 130 | scan(Engine, Start, End, Max) -> 131 | AccFun = fun({K, V}, Acc) -> 132 | [{K, V} | Acc] 133 | end, 134 | AccOut = fold(Engine, AccFun, [], [{gte, Start}, 135 | {lte, End}, 136 | {max, Max}]), 137 | lists:reverse(AccOut). 138 | 139 | clear_range(Engine, Start, End, Max) -> 140 | AccFun = fun(K, Acc) -> 141 | [{delete, K} | Acc] 142 | end, 143 | Ops = fold_keys(Engine, AccFun, [], [{gte, Start}, 144 | {lte, End}, 145 | {max, Max}]), 146 | write_batch(Engine, Ops). 147 | 148 | 149 | fold_keys(#engine{ref=Ref, key_enc=KE, val_enc=VE}, Fun, Acc, Opts) -> 150 | FoldAcc = fold_acc(Fun, Acc, KE, VE, Opts), 151 | case catch bitcask:fold_keys(Ref, fun fold_fun/2, FoldAcc) of 152 | #fold_st{useracc=UserAcc} -> UserAcc; 153 | {stop, #fold_st{useracc=UserAcc}} -> UserAcc; 154 | {error, Reason} -> {error, Reason}; 155 | {'EXIT', Reason} -> {error, Reason}; 156 | _ -> {error, badarg} 157 | end. 158 | 159 | fold(#engine{ref=Ref, key_enc=KE, val_enc=VE}, Fun, Acc, Opts) -> 160 | FoldAcc = fold_acc(Fun, Acc, KE, VE, Opts), 161 | case catch bitcask:fold(Ref, fun fold_fun/3, FoldAcc) of 162 | #fold_st{useracc=UserAcc} -> UserAcc; 163 | {stop, #fold_st{useracc=UserAcc}} -> UserAcc; 164 | {error, Reason} -> {error, Reason}; 165 | {'EXIT', Reason} -> {error, Reason}; 166 | _ -> {error, badarg} 167 | end. 168 | 169 | is_empty(#engine{ref=Ref}) -> 170 | bitcask:is_empty_estimate(Ref). 171 | 172 | 173 | fold_acc(Fun, Acc, KE, VE, Opts) -> 174 | fold_acc(Opts, KE, #fold_st{useracc=Acc, cb=Fun, key_enc=KE, val_enc=VE}). 175 | 176 | fold_acc([], _KE, St) -> 177 | St; 178 | fold_acc([{start_key, Start} | Rest], KE, St) -> 179 | fold_acc(Rest, KE, St#fold_st{from=start_key(Start, KE), 180 | from_inclusive=true}); 181 | fold_acc([{end_key, End} | Rest], KE, St) -> 182 | fold_acc(Rest, KE, St#fold_st{to=enc(key, End, KE), 183 | to_inclusive=true}); 184 | fold_acc([{gt, Start} | Rest], KE, St) -> 185 | fold_acc(Rest, KE, St#fold_st{from=start_key(Start, KE), 186 | from_inclusive=false}); 187 | fold_acc([{gte, Start} | Rest], KE, St) -> 188 | fold_acc(Rest, KE, St#fold_st{from=start_key(Start, KE), 189 | from_inclusive=true}); 190 | fold_acc([{lt, End} | Rest], KE, St) -> 191 | fold_acc(Rest, KE, St#fold_st{to=enc(key, End, KE), 192 | to_inclusive=false}); 193 | fold_acc([{lte, End} | Rest], KE, St) -> 194 | fold_acc(Rest, KE, St#fold_st{to=enc(key, End, KE), 195 | to_inclusive=true}); 196 | fold_acc([{max, Max} | Rest], KE, St) -> 197 | fold_acc(Rest, KE, St#fold_st{max=Max}); 198 | fold_acc([_ | Rest], KE, St) -> 199 | fold_acc(Rest, KE, St). 200 | 201 | 202 | 203 | start_key(first, _KE) -> 204 | <<>>; 205 | start_key(Key, KE) -> 206 | enc(key, Key, KE). 207 | 208 | 209 | fold_fun(#bitcask_entry{key=K}, #fold_st{from=Start, 210 | from_inclusive=Inclusive}=Acc) -> 211 | Res = case Inclusive of 212 | false when K > Start -> fold_fun1(K, Acc); 213 | true when K =:= Start orelse K > Start -> fold_fun1(K, Acc); 214 | _ -> {ok, Acc} 215 | end, 216 | 217 | case Res of 218 | {ok, Acc2} -> Acc2; 219 | {stop, Acc2} -> throw({stop, Acc2}) 220 | end. 221 | 222 | fold_fun(K, V, #fold_st{from=Start, from_inclusive=Inclusive}=Acc) -> 223 | Res = case Inclusive of 224 | false when K > Start -> fold_fun1(K, V, Acc); 225 | true when K =:= Start orelse K > Start -> fold_fun1(K, V, Acc); 226 | _ -> {ok, Acc} 227 | end, 228 | 229 | case Res of 230 | {ok, Acc2} -> Acc2; 231 | {stop, Acc2} -> throw({stop, Acc2}) 232 | end. 233 | 234 | fold_fun1(K, #fold_st{to=nil}=St) -> 235 | do_fold({ok, K}, St); 236 | fold_fun1(K, #fold_st{to=End, to_inclusive=false}=St) when K < End -> 237 | do_fold({ok, K}, St); 238 | fold_fun1(K, #fold_st{to=K, to_inclusive=true}=St) -> 239 | {stop, exec_cb(K, St)}; 240 | fold_fun1(K, #fold_st{to=End, to_inclusive=true}=St) when K < End -> 241 | do_fold({ok, K}, St); 242 | fold_fun1(_K, St) -> 243 | {stop, St}. 244 | 245 | fold_fun1(K, V, #fold_st{to=nil}=St) -> 246 | do_fold({ok, K, V}, St); 247 | fold_fun1(K, V, #fold_st{to=End, to_inclusive=false}=St) when K < End -> 248 | do_fold({ok, K, V}, St); 249 | fold_fun1(K, V, #fold_st{to=K, to_inclusive=true}=St) -> 250 | {stop, exec_cb(K, V, St)}; 251 | fold_fun1(K, V, #fold_st{to=End, to_inclusive=true}=St) when K < End -> 252 | do_fold({ok, K, V}, St); 253 | fold_fun1(_K, _V, St) -> 254 | {stop, St}. 255 | 256 | 257 | do_fold({ok, K}, #fold_st{max=0}=St) -> 258 | {ok, exec_cb(K, St)}; 259 | do_fold({ok, K}, St) -> 260 | NewSt = exec_cb(K, St), 261 | N = NewSt#fold_st.num + 1, 262 | if 263 | N < St#fold_st.max -> {ok, NewSt#fold_st{num=N}}; 264 | true -> {stop, NewSt} 265 | end; 266 | do_fold({ok, K, V}, #fold_st{max=0}=St) -> 267 | {ok, exec_cb(K, V, St)}; 268 | do_fold({ok, K, V}, St) -> 269 | NewSt = exec_cb(K, V, St), 270 | N = NewSt#fold_st.num + 1, 271 | if 272 | N < St#fold_st.max -> {ok, NewSt#fold_st{num=N}}; 273 | true -> {stop, NewSt} 274 | end. 275 | 276 | exec_cb(K, #fold_st{useracc=Acc, cb=Cb, key_enc=KE}=St) -> 277 | St#fold_st{useracc=Cb(dec(key, K, KE), Acc)}. 278 | 279 | exec_cb(K, V, #fold_st{useracc=Acc, cb=Cb, key_enc=KE, val_enc=VE}=St) -> 280 | NewAcc = Cb({dec(key, K, KE), dec(value, V, VE)}, Acc), 281 | St#fold_st{useracc=NewAcc}. 282 | 283 | data_directory_cleanup(DirPath) -> 284 | case file:list_dir(DirPath) of 285 | {ok, Files} -> 286 | [file:delete(filename:join([DirPath, File])) || 287 | File <- Files], 288 | file:del_dir(DirPath); 289 | _ -> 290 | ignore 291 | end. 292 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Mozilla Public License Version 2.0 2 | ================================== 3 | 4 | 1. Definitions 5 | -------------- 6 | 7 | 1.1. "Contributor" 8 | means each individual or legal entity that creates, contributes to 9 | the creation of, or owns Covered Software. 10 | 11 | 1.2. "Contributor Version" 12 | means the combination of the Contributions of others (if any) used 13 | by a Contributor and that particular Contributor's Contribution. 14 | 15 | 1.3. "Contribution" 16 | means Covered Software of a particular Contributor. 17 | 18 | 1.4. "Covered Software" 19 | means Source Code Form to which the initial Contributor has attached 20 | the notice in Exhibit A, the Executable Form of such Source Code 21 | Form, and Modifications of such Source Code Form, in each case 22 | including portions thereof. 23 | 24 | 1.5. "Incompatible With Secondary Licenses" 25 | means 26 | 27 | (a) that the initial Contributor has attached the notice described 28 | in Exhibit B to the Covered Software; or 29 | 30 | (b) that the Covered Software was made available under the terms of 31 | version 1.1 or earlier of the License, but not also under the 32 | terms of a Secondary License. 33 | 34 | 1.6. "Executable Form" 35 | means any form of the work other than Source Code Form. 36 | 37 | 1.7. "Larger Work" 38 | means a work that combines Covered Software with other material, in 39 | a separate file or files, that is not Covered Software. 40 | 41 | 1.8. "License" 42 | means this document. 43 | 44 | 1.9. "Licensable" 45 | means having the right to grant, to the maximum extent possible, 46 | whether at the time of the initial grant or subsequently, any and 47 | all of the rights conveyed by this License. 48 | 49 | 1.10. "Modifications" 50 | means any of the following: 51 | 52 | (a) any file in Source Code Form that results from an addition to, 53 | deletion from, or modification of the contents of Covered 54 | Software; or 55 | 56 | (b) any new file in Source Code Form that contains any Covered 57 | Software. 58 | 59 | 1.11. "Patent Claims" of a Contributor 60 | means any patent claim(s), including without limitation, method, 61 | process, and apparatus claims, in any patent Licensable by such 62 | Contributor that would be infringed, but for the grant of the 63 | License, by the making, using, selling, offering for sale, having 64 | made, import, or transfer of either its Contributions or its 65 | Contributor Version. 66 | 67 | 1.12. "Secondary License" 68 | means either the GNU General Public License, Version 2.0, the GNU 69 | Lesser General Public License, Version 2.1, the GNU Affero General 70 | Public License, Version 3.0, or any later versions of those 71 | licenses. 72 | 73 | 1.13. "Source Code Form" 74 | means the form of the work preferred for making modifications. 75 | 76 | 1.14. "You" (or "Your") 77 | means an individual or a legal entity exercising rights under this 78 | License. For legal entities, "You" includes any entity that 79 | controls, is controlled by, or is under common control with You. For 80 | purposes of this definition, "control" means (a) the power, direct 81 | or indirect, to cause the direction or management of such entity, 82 | whether by contract or otherwise, or (b) ownership of more than 83 | fifty percent (50%) of the outstanding shares or beneficial 84 | ownership of such entity. 85 | 86 | 2. License Grants and Conditions 87 | -------------------------------- 88 | 89 | 2.1. Grants 90 | 91 | Each Contributor hereby grants You a world-wide, royalty-free, 92 | non-exclusive license: 93 | 94 | (a) under intellectual property rights (other than patent or trademark) 95 | Licensable by such Contributor to use, reproduce, make available, 96 | modify, display, perform, distribute, and otherwise exploit its 97 | Contributions, either on an unmodified basis, with Modifications, or 98 | as part of a Larger Work; and 99 | 100 | (b) under Patent Claims of such Contributor to make, use, sell, offer 101 | for sale, have made, import, and otherwise transfer either its 102 | Contributions or its Contributor Version. 103 | 104 | 2.2. Effective Date 105 | 106 | The licenses granted in Section 2.1 with respect to any Contribution 107 | become effective for each Contribution on the date the Contributor first 108 | distributes such Contribution. 109 | 110 | 2.3. Limitations on Grant Scope 111 | 112 | The licenses granted in this Section 2 are the only rights granted under 113 | this License. No additional rights or licenses will be implied from the 114 | distribution or licensing of Covered Software under this License. 115 | Notwithstanding Section 2.1(b) above, no patent license is granted by a 116 | Contributor: 117 | 118 | (a) for any code that a Contributor has removed from Covered Software; 119 | or 120 | 121 | (b) for infringements caused by: (i) Your and any other third party's 122 | modifications of Covered Software, or (ii) the combination of its 123 | Contributions with other software (except as part of its Contributor 124 | Version); or 125 | 126 | (c) under Patent Claims infringed by Covered Software in the absence of 127 | its Contributions. 128 | 129 | This License does not grant any rights in the trademarks, service marks, 130 | or logos of any Contributor (except as may be necessary to comply with 131 | the notice requirements in Section 3.4). 132 | 133 | 2.4. Subsequent Licenses 134 | 135 | No Contributor makes additional grants as a result of Your choice to 136 | distribute the Covered Software under a subsequent version of this 137 | License (see Section 10.2) or under the terms of a Secondary License (if 138 | permitted under the terms of Section 3.3). 139 | 140 | 2.5. Representation 141 | 142 | Each Contributor represents that the Contributor believes its 143 | Contributions are its original creation(s) or it has sufficient rights 144 | to grant the rights to its Contributions conveyed by this License. 145 | 146 | 2.6. Fair Use 147 | 148 | This License is not intended to limit any rights You have under 149 | applicable copyright doctrines of fair use, fair dealing, or other 150 | equivalents. 151 | 152 | 2.7. Conditions 153 | 154 | Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted 155 | in Section 2.1. 156 | 157 | 3. Responsibilities 158 | ------------------- 159 | 160 | 3.1. Distribution of Source Form 161 | 162 | All distribution of Covered Software in Source Code Form, including any 163 | Modifications that You create or to which You contribute, must be under 164 | the terms of this License. You must inform recipients that the Source 165 | Code Form of the Covered Software is governed by the terms of this 166 | License, and how they can obtain a copy of this License. You may not 167 | attempt to alter or restrict the recipients' rights in the Source Code 168 | Form. 169 | 170 | 3.2. Distribution of Executable Form 171 | 172 | If You distribute Covered Software in Executable Form then: 173 | 174 | (a) such Covered Software must also be made available in Source Code 175 | Form, as described in Section 3.1, and You must inform recipients of 176 | the Executable Form how they can obtain a copy of such Source Code 177 | Form by reasonable means in a timely manner, at a charge no more 178 | than the cost of distribution to the recipient; and 179 | 180 | (b) You may distribute such Executable Form under the terms of this 181 | License, or sublicense it under different terms, provided that the 182 | license for the Executable Form does not attempt to limit or alter 183 | the recipients' rights in the Source Code Form under this License. 184 | 185 | 3.3. Distribution of a Larger Work 186 | 187 | You may create and distribute a Larger Work under terms of Your choice, 188 | provided that You also comply with the requirements of this License for 189 | the Covered Software. If the Larger Work is a combination of Covered 190 | Software with a work governed by one or more Secondary Licenses, and the 191 | Covered Software is not Incompatible With Secondary Licenses, this 192 | License permits You to additionally distribute such Covered Software 193 | under the terms of such Secondary License(s), so that the recipient of 194 | the Larger Work may, at their option, further distribute the Covered 195 | Software under the terms of either this License or such Secondary 196 | License(s). 197 | 198 | 3.4. Notices 199 | 200 | You may not remove or alter the substance of any license notices 201 | (including copyright notices, patent notices, disclaimers of warranty, 202 | or limitations of liability) contained within the Source Code Form of 203 | the Covered Software, except that You may alter any license notices to 204 | the extent required to remedy known factual inaccuracies. 205 | 206 | 3.5. Application of Additional Terms 207 | 208 | You may choose to offer, and to charge a fee for, warranty, support, 209 | indemnity or liability obligations to one or more recipients of Covered 210 | Software. However, You may do so only on Your own behalf, and not on 211 | behalf of any Contributor. You must make it absolutely clear that any 212 | such warranty, support, indemnity, or liability obligation is offered by 213 | You alone, and You hereby agree to indemnify every Contributor for any 214 | liability incurred by such Contributor as a result of warranty, support, 215 | indemnity or liability terms You offer. You may include additional 216 | disclaimers of warranty and limitations of liability specific to any 217 | jurisdiction. 218 | 219 | 4. Inability to Comply Due to Statute or Regulation 220 | --------------------------------------------------- 221 | 222 | If it is impossible for You to comply with any of the terms of this 223 | License with respect to some or all of the Covered Software due to 224 | statute, judicial order, or regulation then You must: (a) comply with 225 | the terms of this License to the maximum extent possible; and (b) 226 | describe the limitations and the code they affect. Such description must 227 | be placed in a text file included with all distributions of the Covered 228 | Software under this License. Except to the extent prohibited by statute 229 | or regulation, such description must be sufficiently detailed for a 230 | recipient of ordinary skill to be able to understand it. 231 | 232 | 5. Termination 233 | -------------- 234 | 235 | 5.1. The rights granted under this License will terminate automatically 236 | if You fail to comply with any of its terms. However, if You become 237 | compliant, then the rights granted under this License from a particular 238 | Contributor are reinstated (a) provisionally, unless and until such 239 | Contributor explicitly and finally terminates Your grants, and (b) on an 240 | ongoing basis, if such Contributor fails to notify You of the 241 | non-compliance by some reasonable means prior to 60 days after You have 242 | come back into compliance. Moreover, Your grants from a particular 243 | Contributor are reinstated on an ongoing basis if such Contributor 244 | notifies You of the non-compliance by some reasonable means, this is the 245 | first time You have received notice of non-compliance with this License 246 | from such Contributor, and You become compliant prior to 30 days after 247 | Your receipt of the notice. 248 | 249 | 5.2. If You initiate litigation against any entity by asserting a patent 250 | infringement claim (excluding declaratory judgment actions, 251 | counter-claims, and cross-claims) alleging that a Contributor Version 252 | directly or indirectly infringes any patent, then the rights granted to 253 | You by any and all Contributors for the Covered Software under Section 254 | 2.1 of this License shall terminate. 255 | 256 | 5.3. In the event of termination under Sections 5.1 or 5.2 above, all 257 | end user license agreements (excluding distributors and resellers) which 258 | have been validly granted by You or Your distributors under this License 259 | prior to termination shall survive termination. 260 | 261 | ************************************************************************ 262 | * * 263 | * 6. Disclaimer of Warranty * 264 | * ------------------------- * 265 | * * 266 | * Covered Software is provided under this License on an "as is" * 267 | * basis, without warranty of any kind, either expressed, implied, or * 268 | * statutory, including, without limitation, warranties that the * 269 | * Covered Software is free of defects, merchantable, fit for a * 270 | * particular purpose or non-infringing. The entire risk as to the * 271 | * quality and performance of the Covered Software is with You. * 272 | * Should any Covered Software prove defective in any respect, You * 273 | * (not any Contributor) assume the cost of any necessary servicing, * 274 | * repair, or correction. This disclaimer of warranty constitutes an * 275 | * essential part of this License. No use of any Covered Software is * 276 | * authorized under this License except under this disclaimer. * 277 | * * 278 | ************************************************************************ 279 | 280 | ************************************************************************ 281 | * * 282 | * 7. Limitation of Liability * 283 | * -------------------------- * 284 | * * 285 | * Under no circumstances and under no legal theory, whether tort * 286 | * (including negligence), contract, or otherwise, shall any * 287 | * Contributor, or anyone who distributes Covered Software as * 288 | * permitted above, be liable to You for any direct, indirect, * 289 | * special, incidental, or consequential damages of any character * 290 | * including, without limitation, damages for lost profits, loss of * 291 | * goodwill, work stoppage, computer failure or malfunction, or any * 292 | * and all other commercial damages or losses, even if such party * 293 | * shall have been informed of the possibility of such damages. This * 294 | * limitation of liability shall not apply to liability for death or * 295 | * personal injury resulting from such party's negligence to the * 296 | * extent applicable law prohibits such limitation. Some * 297 | * jurisdictions do not allow the exclusion or limitation of * 298 | * incidental or consequential damages, so this exclusion and * 299 | * limitation may not apply to You. * 300 | * * 301 | ************************************************************************ 302 | 303 | 8. Litigation 304 | ------------- 305 | 306 | Any litigation relating to this License may be brought only in the 307 | courts of a jurisdiction where the defendant maintains its principal 308 | place of business and such litigation shall be governed by laws of that 309 | jurisdiction, without reference to its conflict-of-law provisions. 310 | Nothing in this Section shall prevent a party's ability to bring 311 | cross-claims or counter-claims. 312 | 313 | 9. Miscellaneous 314 | ---------------- 315 | 316 | This License represents the complete agreement concerning the subject 317 | matter hereof. If any provision of this License is held to be 318 | unenforceable, such provision shall be reformed only to the extent 319 | necessary to make it enforceable. Any law or regulation which provides 320 | that the language of a contract shall be construed against the drafter 321 | shall not be used to construe this License against a Contributor. 322 | 323 | 10. Versions of the License 324 | --------------------------- 325 | 326 | 10.1. New Versions 327 | 328 | Mozilla Foundation is the license steward. Except as provided in Section 329 | 10.3, no one other than the license steward has the right to modify or 330 | publish new versions of this License. Each version will be given a 331 | distinguishing version number. 332 | 333 | 10.2. Effect of New Versions 334 | 335 | You may distribute the Covered Software under the terms of the version 336 | of the License under which You originally received the Covered Software, 337 | or under the terms of any subsequent version published by the license 338 | steward. 339 | 340 | 10.3. Modified Versions 341 | 342 | If you create software not governed by this License, and you want to 343 | create a new license for such software, you may create and use a 344 | modified version of this License if you rename the license and remove 345 | any references to the name of the license steward (except to note that 346 | such modified license differs from this License). 347 | 348 | 10.4. Distributing Source Code Form that is Incompatible With Secondary 349 | Licenses 350 | 351 | If You choose to distribute Source Code Form that is Incompatible With 352 | Secondary Licenses under the terms of this version of the License, the 353 | notice described in Exhibit B of this License must be attached. 354 | 355 | Exhibit A - Source Code Form License Notice 356 | ------------------------------------------- 357 | 358 | This Source Code Form is subject to the terms of the Mozilla Public 359 | License, v. 2.0. If a copy of the MPL was not distributed with this 360 | file, You can obtain one at http://mozilla.org/MPL/2.0/. 361 | 362 | If it is not possible or desirable to put the notice in a particular 363 | file, then You may include the notice in a location (such as a LICENSE 364 | file in a relevant directory) where a recipient would be likely to look 365 | for such a notice. 366 | 367 | You may add additional accurate notices of copyright ownership. 368 | 369 | Exhibit B - "Incompatible With Secondary Licenses" Notice 370 | --------------------------------------------------------- 371 | 372 | This Source Code Form is "Incompatible With Secondary Licenses", as 373 | defined by the Mozilla Public License, v. 2.0. 374 | --------------------------------------------------------------------------------