├── .formatter.exs
├── .gitattributes
├── .github
└── workflows
│ └── elixir.yml
├── .gitignore
├── CNAME
├── LICENSE
├── README.md
├── config
└── config.exs
├── etc
├── fs.config
├── mnesia.config
├── rocks-lists.config
└── rocks.config
├── include
├── api.hrl
├── backend.hrl
├── cursors.hrl
├── kvs.hrl
├── metainfo.hrl
└── stream.hrl
├── index.html
├── lib
└── KVS.ex
├── man
├── kvs.htm
├── kvs_fs.htm
├── kvs_mnesia.htm
├── kvs_rocks.htm
├── kvs_st.htm
└── kvs_stream.htm
├── mix.exs
├── rebar.config
├── src
├── kvs.app.src
├── kvs.erl
├── layers
│ ├── kvs_st.erl
│ └── kvs_stream.erl
└── stores
│ ├── kvs_fs.erl
│ ├── kvs_mnesia.erl
│ └── kvs_rocks.erl
├── sys.config
└── test
├── fd_test.exs
├── sc_test.exs
├── st_test.exs
└── test_helper.exs
/.formatter.exs:
--------------------------------------------------------------------------------
1 | # Used by "mix format"
2 | [
3 | inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}"],
4 | line_length: 100
5 | ]
6 |
--------------------------------------------------------------------------------
/.gitattributes:
--------------------------------------------------------------------------------
1 | *.html linguist-detectable=false
2 | *.htm linguist-detectable=false
3 | *.js linguist-detectable=false
4 | *.css linguist-detectable=false
5 | *.sh linguist-detectable=false
6 |
--------------------------------------------------------------------------------
/.github/workflows/elixir.yml:
--------------------------------------------------------------------------------
1 | name: mix
2 | on: push
3 | jobs:
4 | build:
5 | runs-on: ubuntu-latest
6 | steps:
7 | - uses: actions/checkout@v2
8 | - uses: erlef/setup-elixir@v1
9 | with:
10 | otp-version: 24.x
11 | elixir-version: 1.13.x
12 | - name: Dependencies
13 | run: |
14 | mix local.rebar --force
15 | mix local.hex --force
16 | mix deps.get
17 | - name: Compilation
18 | run: mix compile
19 |
20 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | ebin
2 | *.dets
3 | *.gz
4 | *.dump
5 | .applist
6 | rocksdb/
7 | deps/
8 | _build/
9 | *.lock
10 | doc/
11 |
--------------------------------------------------------------------------------
/CNAME:
--------------------------------------------------------------------------------
1 | kvs.n2o.dev
2 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | DHARMA License
2 |
3 | Copyright (c) 2015—2022 Synrc Research
4 |
5 | Permission to use, copy, modify, and/or distribute this software for any
6 | purpose with or without fee is hereby granted, provided that the above
7 | copyright notice and this permission notice appear in all copies.
8 |
9 | YOU CANNOT USE THIS SOFTWARE IN ANY (PROVABLE BY MONEY TRACE)
10 | PROCESS CHAIN OF EXTERMINATING UKRAINIANS BY ANY MEANS OF FASCIST
11 | ACTIONS AGAINST OUR TERRITORIAL INTEGRITY, CULTURAL DIVERSITY BY
12 | APPLYING MILITARY INVASIONS, ECONOMICAL WARS, HUMANITARIAN DISASTERS,
13 | ARTFICIAL HOLODOMORS, GENOCIDE, RAPING, LOOTING, ROBBERIES, SPREADING
14 | FAKE INFORMATION, AND OTHER CONTEMPORARY WEAPONS OF WAR AT SCALE
15 | OR IN INVIDIVUAL MANNER.
16 |
17 | YOU CANNOT USE THIS SOFTWARE BY ANY MEANS IN INTEREST OF LEGAL
18 | ENTITIES OR INDIVIDUALS WHO IS SUPPORTING NOW OR WAS SUPPORTING
19 | BACK THEN FASCISM, RUSCISM, COMMUNISM, CHAUVINISM, HUMILIATION,
20 | AND OTHER SUPPRESSIVE IDEOLOGIES IN DIFFERENT EXPRESSIONS.
21 |
22 | STOP KILLING UKRAINIANS,
23 | THE COUNTER RENDERS TENS OF MILLIONS.
24 |
25 | THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
26 | WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
27 | MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
28 | ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
29 | WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
30 | ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
31 | OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
32 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | KVS: Abstract Chain Database
2 | ============================
3 |
4 | [](https://github.com/synrc/kvs/actions)
5 | [](https://hex.pm/packages/kvs)
6 |
7 | Features
8 | --------
9 |
10 | * Polymorphic Tuples aka Extensible Records
11 | * Basic Schema for Storing Chains
12 | * Backends: MNESIA, FS, ROCKSDB
13 | * Extremely Compact: 500 LOC
14 |
15 | Usage
16 | -----
17 |
18 | ```
19 | $ git clone https://github.com/synrc/kvs && cd kvs
20 | $ open man/kvs.htm
21 | $ mad com pla rep
22 | > kvs:join().
23 | ```
24 |
25 | Release Notes
26 | -------------
27 |
28 | [1]. 2018-11-13 Новая версия KVS 5.11
29 | [2]. 2019-04-13 Новая версия KVS 6.4
30 |
31 | Credits
32 | -------
33 |
34 | * Maxim Sokhatsky
35 | * Andrii Zadorozhnii
36 | * Ivan Kulyk
37 |
--------------------------------------------------------------------------------
/config/config.exs:
--------------------------------------------------------------------------------
1 | import Config
2 |
3 | config :kvs,
4 | dba: :kvs_rocks,
5 | dba_st: :kvs_st,
6 | dba_seq: :kvs_rocks,
7 | seq_pad: [],
8 | schema: [:kvs, :kvs_stream]
9 |
--------------------------------------------------------------------------------
/etc/fs.config:
--------------------------------------------------------------------------------
1 | [ {kvx,[{dba,kvx_fs},
2 | {dba_st,kvx_stream},
3 | {schema,[kvx,kvx_stream]}]}
4 | ].
5 |
--------------------------------------------------------------------------------
/etc/mnesia.config:
--------------------------------------------------------------------------------
1 | [ {kvx,[{dba,kvx_mnesia},
2 | {dba_st,kvx_stream},
3 | {schema,[kvx,kvx_stream]}]}
4 | ].
5 |
--------------------------------------------------------------------------------
/etc/rocks-lists.config:
--------------------------------------------------------------------------------
1 | [ {kvx,[{dba,kvx_rocks},
2 | {dba_st,kvx_stream},
3 | {schema,[kvx,kvx_stream]}]}
4 | ].
5 |
--------------------------------------------------------------------------------
/etc/rocks.config:
--------------------------------------------------------------------------------
1 | [ {kvx,[{dba,kvx_rocks},
2 | {dba_st,kvx_st},
3 | {schema,[kvx,kvx_stream]}]}
4 | ].
5 |
--------------------------------------------------------------------------------
/include/api.hrl:
--------------------------------------------------------------------------------
1 | -ifndef(API_HRL).
2 | -define(API_HRL, true).
3 | -define(API,[start/0,stop/0,leave/0,leave/1,destroy/0,destroy/1,
4 | join/0,join/1,modules/0,cursors/0,get/2,get/3,put/1,put/2,index/3,index/4,
5 | match/1,match/2,index_match/2,index_match/3,key_match/2,key_match/3,
6 | delete/2,delete/3,delete_range/3,
7 | table/1,tables/0,dir/0,initialize/2,seq/2,all/1,all/2,count/1,ver/0]).
8 | -include("metainfo.hrl").
9 | -spec seq(atom() | [], integer() | []) -> term().
10 | -spec count(atom()) -> integer().
11 | -spec dir() -> list({'table',atom()}).
12 | -spec ver() -> {'version',string()}.
13 | -spec leave() -> ok.
14 | -spec destroy() -> ok.
15 | -spec join() -> ok | {error,any()}.
16 | -spec join(Node :: string()) -> [{atom(),any()}].
17 | -spec modules() -> list(atom()).
18 | -spec cursors() -> list({atom(),list(atom())}).
19 | -spec tables() -> list(#table{}).
20 | -spec table(Tab :: atom()) -> #table{} | false.
21 | -endif.
22 |
--------------------------------------------------------------------------------
/include/backend.hrl:
--------------------------------------------------------------------------------
1 | -ifndef(BACKEND_HRL).
2 | -define(BACKEND_HRL, true).
3 | -define(BACKEND, [db/0,get/3,put/1,put/2,delete/3,delete_range/3,index/3,dump/0,start/0,stop/0,destroy/0,destroy/1,keys/2,
4 | join/2,leave/0,leave/1,dir/0,create_table/2,add_table_index/2,seq/2,all/2,count/1,version/0,
5 | match/1,key_match/3,index_match/2]).
6 | -compile({no_auto_import,[get/1,put/2]}).
7 | -include("kvs.hrl").
8 | -spec put(tuple() | list(tuple())) -> ok | {error,any()}.
9 | -spec put(tuple() | list(tuple()), #kvs{}) -> ok | {error,any()}.
10 | -spec get(term() | any(), any(), #kvs{}) -> {ok,any()} | {error,not_found}.
11 | -spec delete(term(), any(), #kvs{}) -> ok | {error,not_found}.
12 | -spec delete_range(term(), any(), #kvs{}) -> ok | {error,not_found}.
13 | -spec dump() -> ok.
14 | -spec start() -> ok.
15 | -spec stop() -> ok.
16 | -spec index(term(), any(), any()) -> list(tuple()).
17 | -endif.
18 |
--------------------------------------------------------------------------------
/include/cursors.hrl:
--------------------------------------------------------------------------------
1 | -ifndef(CURSORS_HRL).
2 | -define(CURSORS_HRL, true).
3 |
4 | -record(writer, { id = [] :: term(),
5 | count = 0 :: integer(),
6 | cache = [] :: [] | {term(),term()} | {term(),term(),term()},
7 | args = [] :: [] | term(),
8 | first = [] :: [] | tuple() } ).
9 |
10 | -record(reader, { id = [] :: [] | integer(),
11 | pos = 0 :: integer() | atom(),
12 | cache = [] :: [] | {term(),term()} | {term(),term(),term()},
13 | args = [] :: [] | integer() | term(),
14 | feed = [] :: term(),
15 | seek = [] :: term(),
16 | count = 0 :: integer(),
17 | dir = 0 :: 0 | 1 } ).
18 |
19 | -endif.
20 |
--------------------------------------------------------------------------------
/include/kvs.hrl:
--------------------------------------------------------------------------------
1 | -ifndef(KVS_HRL).
2 | -define(KVS_HRL, true).
3 | -record(id_seq, { thing = []::term(), id = 0 :: integer() } ).
4 | -record(it, { id = []::[] | integer() } ).
5 | -record(ite, { id = []::[] | integer(), next = []::[] | integer() } ).
6 | -record(iter, { id = []::[] | integer(), next = []::[] | integer(), prev = []::[] | integer() } ).
7 | -record(kvs, { mod = kvs_mnesia :: kvs_mnesia | kvs_rocks | kvs_fs,
8 | st = kvs_stream :: kvs_stream | kvs_st,
9 | db = []::list(),
10 | cx = []::term() }).
11 | -endif.
12 |
--------------------------------------------------------------------------------
/include/metainfo.hrl:
--------------------------------------------------------------------------------
1 | -ifndef(METAINFO_HRL).
2 | -define(METAINFO_HRL, true).
3 | -record(schema, {name,tables=[]}).
4 | -record(table, {name,container=false,type=set,fields=[],keys=[],
5 | copy_type=application:get_env(kvx,mnesia_media,disc_copies), instance={}, mappings =[] }).
6 | -endif.
7 |
--------------------------------------------------------------------------------
/include/stream.hrl:
--------------------------------------------------------------------------------
1 | -ifndef(STREAM_HRL).
2 | -define(STREAM_HRL, true).
3 | -include("kvs.hrl").
4 | -include("cursors.hrl").
5 | -define(STREAM, [top/1, top/2, bot/1, bot/2, next/1, next/2, prev/1, prev/2, drop/1, drop/2, take/1, take/2, append/2, append/3, feed/1, feed/2,
6 | load_reader/1, load_reader/2, writer/1, writer/2, reader/1, reader/2, save/1, save/2, add/1, add/2, remove/2, remove/3, cut/1, cut/2]).
7 |
8 | -spec top(#reader{}) -> #reader{}.
9 | -spec bot(#reader{}) -> #reader{}.
10 | -spec next(#reader{}) -> #reader{}.
11 | -spec prev(#reader{}) -> #reader{}.
12 | -spec drop(#reader{}) -> #reader{}.
13 | -spec take(#reader{}) -> #reader{}.
14 | -spec feed (term()) -> list().
15 | -spec load_reader (term()) -> #reader{}.
16 | -spec writer (term()) -> #writer{}.
17 | -spec reader (term()) -> #reader{}.
18 | -spec save (#reader{} | #writer{}) -> #reader{} | #writer{}.
19 | -spec add(#writer{}) -> #writer{}.
20 | -spec append(tuple(),term()) -> any().
21 | -spec remove(tuple(),term()) -> integer().
22 | -spec cut(term(), list()) -> ok.
23 | -endif.
24 |
--------------------------------------------------------------------------------
/index.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | KVS
6 |
7 |
8 |
9 | DEV
10 | KVS
11 |
18 |
19 |
20 | KVS
21 |
22 |
23 |
24 | SYNOPSIS
25 | KVS is the light version of client interface on top of BTREE database
26 | abstractions strive to support the basic features:
27 |
28 |
Polymorphic tuples aka extensible records;
29 | Basic schema for chain storage;
30 | Backends: MNESIA, FS, ROCKSDB, SPANDB;
31 | Extremely compact and battle-tested: 500 LOC.
32 |
33 | This is an essence and fruit of KVS; the abstract term interface
34 | with naive yet productive stream implementation. Useful for simple
35 | blockchains, messaging, storage, processing systems, and banking industry.
36 | KVS is used in BPE and BANK applications.
37 |
38 | $ mad get kvs && cd kvs
39 | $ mad com pla rep
40 | > kvs:join().
41 |
42 |
43 |
44 | MODULES
45 |
51 |
52 |
53 | NOV 2021 ©
5HT ISC
54 | VER 8.10 8.4 8.3
55 |
56 |
57 |
59 |
60 |
61 |
62 |
63 | SESSION
64 |
65 |
66 | > kvs:join().
67 | ok
68 |
69 | > kvs:check().
70 | ok
71 |
72 | > kvs:all(reader).
73 | [{reader,1555175169121817000,0,[],[],
74 | {list,1555175169120161000},
75 | 0},
76 | {reader,1555175169121249000,0,[],[],
77 | {list,1555175169120161000},
78 | 0}]
79 |
80 | > rr(kvs).
81 | [emails,id_seq,iter,kvs,reader,schema,table,writer]
82 |
83 | > kvs:save(kvs:reader({list,1555175169120161000})).
84 | #reader{id = 1555175244188986000,pos = 0,
85 | cache = {emails,1555175169122304000},
86 | args = [],
87 | feed = {list,1555175169120161000},
88 | dir = 0}
89 |
90 | > kvs:take(kvs:bot((kvs:load_reader(1555175244188986000))#reader{args=-1})).
91 | #reader{id = 1555175244188986000,pos = 5,
92 | cache = {emails,1555175169127279000},
93 | args = [#emails{id = 1555175169127279000,next = [],
94 | prev = 1555175169126314000,email = []},
95 | #emails{id = 1555175169126314000,next = 1555175169127279000,
96 | prev = 1555175169125227000,email = []},
97 | #emails{id = 1555175169125227000,next = 1555175169126314000,
98 | prev = 1555175169123405000,email = []},
99 | #emails{id = 1555175169123405000,next = 1555175169125227000,
100 | prev = 1555175169122304000,email = []},
101 | #emails{id = 1555175169122304000,next = 1555175169123405000,
102 | prev = [],email = []}],
103 | feed = {list,1555175169120161000},
104 | dir = 0}
105 |
106 |
107 |
108 |
109 |
110 | CONTRIBUTORS
111 |
113 |
114 |
115 |
116 | Made with ❤ to N2O
117 |
118 |
119 |
120 |
--------------------------------------------------------------------------------
/lib/KVS.ex:
--------------------------------------------------------------------------------
1 | defmodule KVS do
2 | require Record
3 |
4 | Enum.each(Record.extract_all(from_lib: "kvs/include/cursors.hrl"), fn {name, definition} ->
5 | Record.defrecord(name, definition)
6 | end)
7 |
8 | Enum.each(Record.extract_all(from_lib: "kvs/include/metainfo.hrl"), fn {name, definition} ->
9 | Record.defrecord(name, definition)
10 | end)
11 |
12 | Enum.each(Record.extract_all(from_lib: "kvs/include/kvs.hrl"), fn {name, definition} ->
13 | Record.defrecord(name, definition)
14 | end)
15 |
16 | defmacro __using__(opts \\ []) do
17 | imports =
18 | opts
19 | |> Macro.expand(__CALLER__)
20 | |> Keyword.get(:with, [:kvs])
21 |
22 | Enum.map(imports, fn mod ->
23 | if Code.ensure_compiled(mod) do
24 | upcased = Module.concat([String.upcase(to_string(mod))])
25 |
26 | quote do
27 | import unquote(upcased)
28 | alias unquote(mod), as: unquote(upcased)
29 | end
30 | else
31 | IO.warn("🚨 Unknown module #{mod} was requested to be used by :kvs. Skipping.")
32 | end
33 | end)
34 | end
35 | end
36 |
--------------------------------------------------------------------------------
/man/kvs.htm:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | KVS
6 |
7 |
8 |
9 | DEV
10 | KVS
11 | API
12 |
19 |
20 |
21 | KVS
22 |
23 |
24 |
25 |
26 |
27 | INTRO
28 |
29 | KVS module provides user level interface for console commands.
30 | It has discovery, data manipulation and retrival features. Under the hood
31 | it handles configurable run-time backends for each supported database.
32 |
33 |
34 | put/1 — put record using id as a key.
35 | get/2 — get record by key from table.
36 | delete/1 — delete record from table.
37 | index/3 — search records by field and its value.
38 | seq/2 — generate new sequence table id.
39 | count/1 — counts records in table.
40 | dir/0 — table list.
41 |
42 |
43 | You can change backend by setting application env.
44 | This behaves well even under the heavy load.
45 |
46 |
47 |
48 | API
49 | Data operations.
50 | put(tuple()) -> ok | {error,any()}.
51 | Stores the record.
52 | get(atom(),any()) -> {ok,any()} | {error, not_found | duplicated }.
53 | Retrieves the record.
54 | delete(atom(),any()) -> ok | {error,any()}.
55 | Deletes the data record.
56 | index(atom(),any(),any()) -> list(tuple()).
57 | Searches the record by an indexed field and a given value.
58 |
59 |
60 | SEQ
61 | Sequence table id_seq stores the counter per thing.
62 | The counters are global and atomic in each supported database.
63 | Sequences are used to generate unique names for records per distributed table.
64 | If names in the table are not unique, e.g.
65 | then count function may return a different value than the current sequence.
66 | -record(id_seq, { thing = atom(),
67 | id = 0 :: integer() } ).
68 |
69 | seq(atom(), integer()) -> integer().
70 | Increments and returns id counter for the particular table.
71 | count(atom()) -> integer().
72 | Returns number of records in table.
73 |
74 |
75 | SETUP
76 | In sys.config you can specify main kvs backend module as dba parameter
77 | and list of modules containing metainfo/0 exported function. For the
78 | stream operations you can specify the stream kvs backend module dba_st parameter.
79 | [{kvs, [{dba,store_mnesia},
80 | {dba_st,store_stream},
81 | {schema,[kvs,kvs_stream]}]}].
82 | dir() -> list({'table',atom()}).
83 | Returns actual tables.
84 |
85 |
95 |
96 |
97 | 2005—2019 © Synrc Research Center
98 |
99 |
100 |
101 |
--------------------------------------------------------------------------------
/man/kvs_fs.htm:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | FS
6 |
7 |
8 |
9 | DEV
10 | KVS
11 | FS
12 |
19 |
20 |
21 | FS
22 |
23 |
24 |
25 |
26 | INTRO
27 | FS is a filesystem backend implementation for KVS.
28 | Put the {dba,kvs_fs} property for the kvs application in your sys.config.
29 |
30 |
31 | EXAMPLES
32 |
33 | All examples can be executed in Erlang REPL. Only
34 | requirement is to have kvs set in dependencies and kvs application
35 | started.
36 |
37 | Create a new kvs_fs table called test. When executed, this
38 | function will create a new directory called test under data
39 | directory in the current workspace.
40 |
41 |
42 | kvs_fs:create_table(test, []).
43 | % will return:
44 | % ok
45 |
46 |
47 | Put a new key/value in table test. When executed, a new file is
48 | created under test directory. Its name is based on the sha1 of
49 | the key encoded in base64.
50 |
51 |
52 | Table = test.
53 | Key = key.
54 | Value = <<"my_value">>.
55 | kvs_fs:put({test, Key, Value}).
56 | % will return:
57 | % ok
58 |
59 |
60 | Get a key from value test.
61 |
62 |
63 | kvs_fs:get(test, key).
64 | % will return:
65 | % {ok, {test, key, <<"my_value">>}}
66 |
67 |
68 |
69 | Delete a key
70 |
71 |
72 |
73 | This module may refer to:
74 | kvs .
75 |
76 |
77 |
78 |
79 |
80 | 2005—2019 © Synrc Research Center
81 |
82 |
83 |
84 |
--------------------------------------------------------------------------------
/man/kvs_mnesia.htm:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 | MNESIA
7 |
8 |
9 |
10 | DEV
11 | KVS
12 | MNESIA
13 |
20 |
21 |
22 | MNESIA
23 |
24 |
25 |
26 | INTRO
27 | MNESIA is a mnesia backend imlpementation for KVS.
28 | Put the {dba,kvs_mnesia} property for the kvs application in your sys.config.
29 |
30 |
31 |
32 | This module may refer to:
33 | kvs .
34 |
35 |
36 |
37 |
38 |
39 | 2005—2019 © Synrc Research Center
40 |
41 |
42 |
43 |
--------------------------------------------------------------------------------
/man/kvs_rocks.htm:
--------------------------------------------------------------------------------
1 |
2 |
3 | ROCKS
4 |
5 |
6 |
7 | DEV
8 | KVS
9 | ROCKS
10 |
17 |
18 |
19 | ROCKS
20 |
21 |
22 |
23 |
24 | INTRO
25 | ROCKS is the RocksDB backend implementation for KVS.
26 | Put the {dba,kvs_rocks} and (optionally) {dba_st,kvs_st} properties
27 | for the kvs application in your sys.config.
28 |
29 |
30 |
37 |
38 |
39 | 2005—2019 © Synrc Research Center
40 |
41 |
42 |
43 |
--------------------------------------------------------------------------------
/man/kvs_st.htm:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 | ST
10 |
11 |
12 |
13 |
14 |
15 | DEV
16 | KVS
17 | ST
18 |
25 |
26 |
27 |
28 |
29 | ST
30 |
31 |
32 |
33 |
34 | INTRO
35 | The ST module provides STREAM interface for ROCKS backend.
36 | Set the {dba,kvs_rocks} along with {dba_st,kvs_st} in
37 | order to use ST module for stream operations.
38 |
39 |
46 |
47 |
48 | 2005—2019 © Synrc Research Center
49 |
50 |
51 |
52 |
--------------------------------------------------------------------------------
/man/kvs_stream.htm:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 | STREAM
8 |
9 |
10 |
11 | DEV
12 | KVS
13 | STREAM
14 |
21 |
22 |
23 | STREAM
24 |
25 |
26 |
27 |
28 | INTRO
29 | The STREAM module provides chain persistence, accumulation and traversal.
30 |
31 | writer/1 — creates writer cursor for chain.
32 | reader/1 — creates reader cursor for chain.
33 | save/1 — stores cursor to db.
34 | load_reader/1 — loads reader cursor.
35 | top/1 — returns top of the reader.
36 | bot/1 — returns bottom of the reader.
37 | next/1 — moves reader next.
38 | prev/1 — moves reader prev.
39 | take/1 — takes N elements from reader.
40 | drop/1 — skips N elements from reader.
41 | add/1 — adds element to writer.
42 | append/2 — appends element to feed.
43 | remove/2 — removes element from feed.
44 | cut/2 — cleans-up the feed up to the key.
45 |
46 |
47 | You can grab kvs_stream
48 | and use it in your applications without importing synrc/kvs dependency,
49 | as this module is self-containing.
50 | The possible applications are: public and private feeds, FIFO queues,
51 | unread messages, chat applications, blockchain etc.
52 |
53 |
54 | WRITER
55 |
56 | Writer cursor represents unique append list chain with some cached values.
57 | E.g., elements count , first element of the chain, cache field for value of
58 | a previous written message and args field for passing arguments in fixpoint style.
59 |
60 | -record(writer, { id = [] :: term(),
61 | count = 0 :: integer(),
62 | cache = [] :: [] | tuple(),
63 | args = [] :: term(),
64 | first = [] :: [] | tuple() } ).
65 |
66 | For adding data to storage you need to create writer cursor,
67 | set the args field with the record to be added:
68 |
69 | > require KVS
70 | KVS
71 | > KVS.writer
72 | {:writer, [], 0, [], [], []}
73 |
74 | writer(list()) -> #writer{}.
75 | Creates writer cursor. After this you normally call save function for cursor persistence.
76 | As a parameter provide cursor name as a list.
77 |
78 | > w = :kvs.writer '/cursors/cur1'
79 | {:writer, '/cursors/cur1', 0, [], [], []}
80 |
81 | save(#writer{}) -> #writer{}.
82 | Flushes writer cursor to database.
83 |
84 | > :kvs.save w
85 | {:writer, '/cursors/cur1', 0, [], [], []}
86 | > :kvs.get :writer, '/cursors/cur1'
87 | {:ok, {:writer, '/cursors/cur1', 0, [], [], []}}
88 |
89 | add(#writer{}) -> #writer{}.
90 | Adds element to persistent chain declared by writer cursor.
91 | Adding elements to chain affects count field.
92 |
93 | > :lists.map fn _ ->
94 | {:ok,w} = :kvs.get(:writer,'/cursors/cur1')
95 | :kvs.save(:kvs.add(KVS.writer(w, args: {:"$msg", [], [], [], [], []})))
96 | end, :lists.seq(1, 5)
97 | [
98 | {:writer, '/cursors/cur1', 1, {:"$msg", '1313544188019000'}, [], []},
99 | {:writer, '/cursors/cur1', 2, {:"$msg", '1313544189127000'}, [], []},
100 | {:writer, '/cursors/cur1', 3, {:"$msg", '1313544189869000'}, [], []},
101 | {:writer, '/cursors/cur1', 4, {:"$msg", '1313544190519000'}, [], []},
102 | {:writer, '/cursors/cur1', 5, {:"$msg", '1313544191134000'}, [], []}
103 | ]
104 | > :kvs.get :writer, '/cursors/cur1'
105 | {:ok,
106 | {:writer, '/cursors/cur1', 5, {:"$msg", '1314009332950000'}, [], []}}
107 |
108 |
109 |
110 |
111 | READER
112 | Reader cursor represents a user reading pointer to the writer feed.
113 | Field #reader.feed is linked to #writer.id .
114 | The pos field represents current position in feed.
115 | Field dir is used for changing take and drop directions.
116 | Reader Cursor
117 | -record(reader, { id = [] :: integer(),
118 | pos = 0 :: [] | integer(),
119 | cache = [] :: [] | integer(),
120 | args = [] :: term(),
121 | feed = [] :: term(),
122 | dir = 0 :: 0 | 1 } ).
123 |
124 | reader(term()) -> #reader{}.
125 | Creates reader cursor for a given writer id.
126 |
127 | load_reader(term()) -> #reader{}.
128 | Loads reader cursor from database.
129 |
130 | save(#reader{}) -> #reader{}.
131 | Flushes reader cursor to database.
132 |
133 | bot(#reader{}) -> #reader{dir = 0}.
134 | Changes the direction of the cursor to take foraward (normal).
135 |
136 | top(#reader{}) -> #reader{dir = 1}.
137 | Changes the direction of the cursor to take backwards.
138 |
139 |
140 | ITER
141 | KVS Stream Iterator
142 | -record(iter, { id = [] :: [] | integer(),
143 | next = [] :: [] | integer(),
144 | prev = [] :: [] | integer() } ).
145 |
146 | next(#reader{}) -> #reader{}.
147 | Moves cursor to next. Consume data down from top.
148 | Return error if list is empty, otherwise next element or last.
149 |
150 | prev(#reader{}) -> #reader{}.
151 | Moves cursor to prev. Consume data up from bottom.
152 | Return error if list is empty, otherwise next element or last.
153 |
154 | drop(#reader{}) -> #reader{}.
155 | Drops N elements starting from reader.
156 |
157 | take(#reader{}) -> #reader{}.
158 | Trying to consume N records from stream using its current value and direction.
159 | Returns consumed data. Usually you seek to some position and then consume some data.
160 |
161 | append(tuple(), list()) -> term().
162 | Adds record to feed.
163 |
164 | remove(tuple(), list()) -> term().
165 | Removes record from feed.
166 |
167 | cut(list(), term()) -> #ok{} | #error{}.
168 | Cuts the feed up to a given key.
169 |
170 |
171 |
172 | This module may refer to:
173 | kvs ,
174 | kvs_st .
175 |
176 |
177 |
178 |
179 | 2005—2019 © Synrc Research Center
180 |
181 |
182 |
183 |
--------------------------------------------------------------------------------
/mix.exs:
--------------------------------------------------------------------------------
1 | defmodule KVS.Mixfile do
2 | use Mix.Project
3 |
4 | def project do
5 | [
6 | app: :kvs,
7 | version: "11.9.1",
8 | description: "KVS Abstract Chain Database",
9 | package: package(),
10 | deps: deps()
11 | ]
12 | end
13 |
14 | def application do
15 | [mod: {:kvs, []}, extra_applications: [:mnesia,:ex_doc]]
16 | end
17 |
18 | defp package do
19 | [
20 | files: ~w(include man config lib src LICENSE mix.exs README.md),
21 | licenses: ["MIT"],
22 | links: %{"GitHub" => "https://github.com/synrc/kvs"}
23 | ]
24 | end
25 |
26 | defp deps do
27 | [
28 | {:ex_doc, ">= 0.0.0", only: :dev}
29 | # {:rocksdb, "~> 1.8.0", only: :test}
30 | ]
31 | end
32 | end
33 |
--------------------------------------------------------------------------------
/rebar.config:
--------------------------------------------------------------------------------
1 | {erl_opts, [nowarn_unused_function,nowarn_duplicated_export]}.
2 | {deps, []}.
3 | {project_plugins, [rebar3_format]}.
4 | {format, [
5 | {files, ["src/*.erl", "test/*.erl"]},
6 | {formatter, otp_formatter},
7 | {options, #{ line_length => 108,
8 | paper => 250,
9 | spaces_around_fields => false,
10 | inlining => all,
11 | inline_clause_bodies => true,
12 | inline_expressions => true,
13 | inline_qualified_function_composition => true,
14 | inline_simple_funs => true,
15 | inline_items => all,
16 | inline_fields => true,
17 | inline_attributes => true
18 | }}]}.
19 |
--------------------------------------------------------------------------------
/src/kvs.app.src:
--------------------------------------------------------------------------------
1 | {application, kvs,
2 | [{description, "KVS Abstract Chain Database"},
3 | {vsn, "9.4.1"},
4 | {registered, []},
5 | {applications, [kernel,stdlib]},
6 | {mod, { kvs, []}},
7 | {env, []} ]}.
8 |
--------------------------------------------------------------------------------
/src/kvs.erl:
--------------------------------------------------------------------------------
1 | -module(kvs).
2 |
3 | -behaviour(application).
4 |
5 | -behaviour(supervisor).
6 |
7 | -include_lib("stdlib/include/assert.hrl").
8 |
9 | -include("api.hrl").
10 |
11 | -include("metainfo.hrl").
12 |
13 | -include("stream.hrl").
14 |
15 | -include("cursors.hrl").
16 |
17 | -include("kvs.hrl").
18 |
19 | -include("backend.hrl").
20 |
21 | -export([dump/0,
22 | db/0,
23 | metainfo/0,
24 | ensure/1,
25 | ensure/2,
26 | seq_gen/0,
27 | keys/1,
28 | fields/1,
29 | defined/2,
30 | field/2,
31 | setfield/3,
32 | remove/1]).
33 |
34 | -export([join/2, seq/3]).
35 |
36 | -export(?API)
37 |
38 | .
39 |
40 | -export(?STREAM)
41 |
42 | .
43 |
44 | -export([init/1, start/2, stop/1]).
45 |
46 | -record('$msg', {id, next, prev, user, msg}).
47 |
48 | init([]) -> {ok, {{one_for_one, 5, 10}, []}}.
49 |
50 | start(_, _) ->
51 | supervisor:start_link({local, kvs}, kvs, []).
52 |
53 | stop(_) -> ok.
54 |
55 | dba() -> application:get_env(kvs, dba, kvs_mnesia).
56 |
57 | seq_dba() -> application:get_env(kvs, dba_seq, kvs_mnesia).
58 |
59 | db() -> (dba()):db().
60 |
61 | kvs_stream() ->
62 | application:get_env(kvs, dba_st, kvs_stream).
63 |
64 | % kvs api
65 |
66 | all(Table) -> all(Table, #kvs{mod = dba(), db = db()}).
67 |
68 | delete(Table, Key) ->
69 | delete(Table, Key, #kvs{mod = dba(), db = db()}).
70 |
71 | get(Table, Key) ->
72 | (?MODULE):get(Table, Key, #kvs{mod = dba(), db = db()}).
73 |
74 | index(Table, K, V) ->
75 | index(Table, K, V, #kvs{mod = dba()}).
76 |
77 | keys(Feed) ->
78 | keys(Feed, #kvs{mod = dba(), db = db()}).
79 |
80 | key_match(Feed, Id) ->
81 | key_match(Feed, Id, #kvs{mod = dba(), db=db()}).
82 |
83 | match(Record) ->
84 | match(Record, #kvs{mod = dba()}).
85 |
86 | index_match(Record, Index) ->
87 | index_match(Record, Index, #kvs{mod = dba()}).
88 |
89 | join() -> join([], #kvs{mod = dba(), db = db()}).
90 |
91 | dump() -> dump(#kvs{mod = dba()}).
92 |
93 | join(Node) -> join(Node, #kvs{mod = dba(), db = db()}).
94 |
95 | leave() -> leave(#kvs{mod = dba(), db = db()}).
96 |
97 | destroy() -> destroy(#kvs{mod = dba(), db = db()}).
98 |
99 | count(Table) -> count(Table, #kvs{mod = dba()}).
100 |
101 | put(Record) -> (?MODULE):put(Record, #kvs{mod = dba(), db = db()}).
102 |
103 | stop() -> stop_kvs(#kvs{mod = dba()}).
104 |
105 | start() -> start(#kvs{mod = dba()}).
106 |
107 | ver() -> ver(#kvs{mod = dba()}).
108 |
109 | dir() -> dir(#kvs{mod = dba()}).
110 |
111 | feed(Key) ->
112 | feed(Key, #kvs{mod = dba(), st = kvs_stream(), db = db()}).
113 |
114 | seq([], DX) -> seq([], DX, #kvs{mod = kvs_rocks});
115 | seq(Table, DX) -> seq(Table, DX, #kvs{mod = seq_dba()}).
116 |
117 | remove(Rec, Feed) ->
118 | remove(Rec, Feed, #kvs{mod = dba(), st = kvs_stream(), db = db()}).
119 |
120 | put(Records, #kvs{mod = Mod, db = Db}) when is_list(Records) ->
121 | Mod:put(Records, Db);
122 | put(Record, #kvs{mod = Mod, db = Db}) -> Mod:put(Record, Db).
123 |
124 | get(RecordName, Key, #kvs{mod = Mod, db = Db}) ->
125 | Mod:get(RecordName, Key, Db).
126 |
127 | delete(Tab, Key, #kvs{mod = Mod, db = Db}) ->
128 | Mod:delete(Tab, Key, Db).
129 |
130 | delete_range(Feed, Last, #kvs{mod=DBA, db=Db}) ->
131 | DBA:delete_range(Feed,Last,Db).
132 |
133 | count(Tab, #kvs{mod = DBA}) -> DBA:count(Tab).
134 |
135 | index(Tab, Key, Value, #kvs{mod = DBA}) ->
136 | DBA:index(Tab, Key, Value).
137 |
138 | keys(Feed, #kvs{mod = DBA, db = Db}) ->
139 | DBA:keys(Feed, Db).
140 |
141 | key_match(Feed, Id, #kvs{mod = DBA, db = Db}) ->
142 | DBA:key_match(Feed, Id, Db).
143 |
144 | match(Record, #kvs{mod = DBA}) ->
145 | DBA:match(Record).
146 |
147 | index_match(Record, Index, #kvs{mod = DBA}) ->
148 | DBA:index_match(Record, Index).
149 |
150 | seq(Tab, Incr, #kvs{mod = DBA}) -> DBA:seq(Tab, Incr).
151 |
152 | dump(#kvs{mod = Mod}) -> Mod:dump().
153 |
154 | feed(Tab, #kvs{st = Mod, db = Db}) -> Mod:feed(Tab, Db).
155 |
156 | remove(Rec, Feed, #kvs{st = Mod, db = Db}) ->
157 | Mod:remove(Rec, Feed, Db).
158 |
159 | all(Tab, #kvs{mod = DBA, db = Db}) -> DBA:all(Tab, Db).
160 |
161 | start(#kvs{mod = DBA}) -> DBA:start().
162 |
163 | stop_kvs(#kvs{mod = DBA}) -> DBA:stop().
164 |
165 | join(Node, #kvs{mod = DBA, db = Db}) -> DBA:join(Node, Db).
166 |
167 | leave(#kvs{mod = DBA, db = Db}) -> DBA:leave(Db).
168 |
169 | destroy(#kvs{mod = DBA, db = Db}) -> DBA:destroy(Db).
170 |
171 | ver(#kvs{mod = DBA}) -> DBA:version().
172 |
173 | dir(#kvs{mod = DBA}) -> DBA:dir().
174 |
175 | % stream api
176 |
177 | top(X) -> (kvs_stream()):top(X).
178 |
179 | top(X,#kvs{db = Db}) -> (kvs_stream()):top(X,Db).
180 |
181 | bot(X) -> (kvs_stream()):bot(X).
182 |
183 | bot(X,#kvs{db = Db}) -> (kvs_stream()):bot(X,Db).
184 |
185 | next(X) -> (kvs_stream()):next(X).
186 |
187 | next(X,#kvs{db = Db}) -> (kvs_stream()):next(X,Db).
188 |
189 | prev(X) -> (kvs_stream()):prev(X).
190 |
191 | prev(X,#kvs{db = Db}) -> (kvs_stream()):prev(X,Db).
192 |
193 | drop(X) -> (kvs_stream()):drop(X).
194 |
195 | drop(X,#kvs{db = Db}) -> (kvs_stream()):drop(X,Db).
196 |
197 | take(X) -> (kvs_stream()):take(X).
198 |
199 | take(X,#kvs{db = Db}) -> (kvs_stream()):take(X,Db).
200 |
201 | save(X) -> (kvs_stream()):save(X).
202 |
203 | save(X,#kvs{db = Db}) -> (kvs_stream()):save(X,Db).
204 |
205 | remove(X) -> (kvs_stream()):remove(X).
206 |
207 | cut(X) -> (kvs_stream()):cut(X).
208 |
209 | cut(X,#kvs{db = Db}) -> (kvs_stream()):cut(X, Db).
210 |
211 | add(X) -> (kvs_stream()):add(X).
212 |
213 | add(X,#kvs{db = Db}) -> (kvs_stream()):add(X,Db).
214 |
215 | append(X, Y) -> (kvs_stream()):append(X, Y).
216 |
217 | append(X, Y, #kvs{db = Db}) -> (kvs_stream()):append(X, Y, Db).
218 |
219 | load_reader(X) -> (kvs_stream()):load_reader(X).
220 |
221 | load_reader(X,#kvs{db = Db}) -> (kvs_stream()):load_reader(X,Db).
222 |
223 | writer(X) -> (kvs_stream()):writer(X).
224 |
225 | writer(X,#kvs{db = Db}) -> (kvs_stream()):writer(X,Db).
226 |
227 | reader(X) -> (kvs_stream()):reader(X).
228 |
229 | reader(X,#kvs{db = Db}) -> (kvs_stream()):reader(X,Db).
230 |
231 | % unrevisited
232 |
233 | ensure(#writer{} = X) -> ensure(X,#kvs{mod = dba(), db = db()}).
234 | ensure(#writer{id = Id},#kvs{} = X) ->
235 | case kvs:get(writer, Id, X) of
236 | {error, _} ->
237 | kvs:save(kvs:writer(Id, X)),
238 | ok;
239 | {ok, _} -> ok
240 | end.
241 |
242 | cursors() ->
243 | lists:flatten([[{T#table.name, T#table.fields}
244 | || #table{name = Name} = T
245 | <- (M:metainfo())#schema.tables,
246 | Name == reader orelse Name == writer]
247 | || M <- modules()]).
248 |
249 | % metainfo api
250 |
251 | tables() ->
252 | lists:flatten([(M:metainfo())#schema.tables
253 | || M <- modules()]).
254 |
255 | modules() -> application:get_env(kvs, schema, []).
256 |
257 | metainfo() ->
258 | #schema{name = kvs, tables = core() ++ test_tabs()}.
259 |
260 | core() ->
261 | [#table{name = id_seq,
262 | fields = record_info(fields, id_seq), keys = [thing]}].
263 |
264 | test_tabs() ->
265 | [#table{name = '$msg',
266 | fields = record_info(fields, '$msg')}].
267 |
268 | table(Name) when is_atom(Name) ->
269 | lists:keyfind(Name, #table.name, tables());
270 | table(_) -> false.
271 |
272 | seq_gen() ->
273 | Init = fun (Key) ->
274 | case kvs:get(id_seq, Key) of
275 | {error, _} ->
276 | {Key, kvs:put(#id_seq{thing = Key, id = 0})};
277 | {ok, _} -> {Key, skip}
278 | end
279 | end,
280 | [Init(atom_to_list(Name))
281 | || {Name, _Fields} <- cursors()].
282 |
283 | initialize(Backend, Module) ->
284 | [begin
285 | Backend:create_table(T#table.name,
286 | [{attributes, T#table.fields},
287 | {T#table.copy_type, [node()]},
288 | {type, T#table.type}]),
289 | [Backend:add_table_index(T#table.name, Key)
290 | || Key <- T#table.keys],
291 | T
292 | end
293 | || T <- (Module:metainfo())#schema.tables].
294 |
295 | fields(Table) when is_atom(Table) ->
296 | case table(Table) of
297 | false -> [];
298 | T -> T#table.fields
299 | end.
300 |
301 | defined(TableRecord, Field) ->
302 | FieldsList = fields(element(1, TableRecord)),
303 | lists:member(Field, FieldsList).
304 |
305 | field(TableRecord, Field) ->
306 | FieldsList = fields(element(1, TableRecord)),
307 | Index = string:str(FieldsList, [Field]) + 1,
308 | element(Index, TableRecord).
309 |
310 | setfield(TableRecord, Field, Value) ->
311 | FieldsList = fields(element(1, TableRecord)),
312 | Index = string:str(FieldsList, [Field]) + 1,
313 | setelement(Index, TableRecord, Value).
314 |
--------------------------------------------------------------------------------
/src/layers/kvs_st.erl:
--------------------------------------------------------------------------------
1 | -module(kvs_st).
2 | -include("kvs.hrl").
3 | -include("stream.hrl").
4 | -include("metainfo.hrl").
5 | -export(?STREAM).
6 | -import(kvs_rocks, [fmt/1, key/2, key/1, bt/1, tb/1, ref/0, ref/1, seek_it/1, seek_it/2, move_it/3, move_it/4, take_it/4, take_it/5, delete_it/2, estimate/0, estimate/1]).
7 | -export([raw_append/2,raw_append/3, remove/1]).
8 |
9 | db() -> application:get_env(kvs,rocks_name,"rocksdb").
10 |
11 | se(X,Y,Z) -> setelement(X,Y,Z).
12 | e(X,Y) -> element(X,Y).
13 | c4(R,V) -> se(#reader.args, R, V).
14 | si(M,T) -> se(#it.id, M, T).
15 | id(T) -> e(#it.id, T).
16 |
17 | k(F,[]) -> F;
18 | k(_,{_,Id,SF}) -> iolist_to_binary([SF,<<"/">>,tb(Id)]).
19 |
20 | f2(Feed) -> X = tb(Feed),
21 | case binary:matches(X, <<"/">>,[]) of
22 | [{0,1}|_] -> binary:part(X,{1,size(X)-1});
23 | _ -> X
24 | end.
25 |
26 | read_it(C,{ok,_,[],H}) -> C#reader{cache=[], args=lists:reverse(H)};
27 | read_it(C,{ok,F,V,H}) -> C#reader{cache={e(1,V),id(V),F}, args=lists:reverse(H)};
28 | read_it(C,_) -> C#reader{args=[]}.
29 |
30 | top(#reader{}=X) -> top(X,db()).
31 | top(#reader{feed=Feed}=C,Db) -> #writer{count=Cn} = writer(f2(Feed),Db), read_it(C#reader{count=Cn},seek_it(Feed,Db)).
32 | bot(#reader{}=X) -> bot(X,db()).
33 | bot(#reader{feed=Feed}=C,Db) -> #writer{cache=Ch, count=Cn} = writer(f2(Feed),Db), C#reader{cache=Ch, count=Cn}.
34 | next(#reader{}=X) -> next(X,db()).
35 | next(#reader{feed=Feed,cache=I}=C,Db) -> read_it(C,move_it(k(Feed,I),Feed,next,Db)).
36 | prev(#reader{}=X) -> prev(X,db()).
37 | prev(#reader{cache=I,feed=Feed}=C,Db) -> read_it(C,move_it(k(Feed,I),Feed,prev,Db)).
38 | take(#reader{}=X) -> take(X,db()).
39 | take(#reader{args=N,feed=Feed,cache=I,dir=1}=C,Db) -> read_it(C,take_it(k(Feed,I),Feed,prev,N,Db));
40 | take(#reader{args=N,feed=Feed,cache=I,dir=_}=C,Db) -> read_it(C,take_it(k(Feed,I),Feed,next,N,Db)).
41 | drop(#reader{}=X) -> drop(X,db()).
42 | drop(#reader{args=N}=C,_) when N =< 0 -> C;
43 | drop(#reader{}=C,Db) -> (take(C#reader{dir=0},Db))#reader{args=[]}.
44 | remove(#reader{}=C) -> remove(C, db()).
45 | remove(#reader{feed=Feed}=C,Db) -> R = read_it(C, delete_it(Feed,Db)), kvs:delete(writer, Feed), R;
46 | remove(Rec,Feed) -> remove(Rec,Feed,db()).
47 |
48 | feed(Feed) -> feed(Feed,db()).
49 | feed(Feed,Db) ->
50 | #reader{count=Cn} = Top = top(reader(Feed,Db),Db),
51 | Halt = case {estimate(),Cn} of
52 | {E,C} when E =< 0 -> max(C,4);
53 | {E,_} -> E
54 | end,
55 | feed(fun(#reader{}=R) -> take(R#reader{args=4},Db) end,Top,[],Halt).
56 |
57 | feed(_F,#reader{},Acc,H) when H =< 0 -> Acc;
58 | feed(F,#reader{cache=C1}=R,Acc,H) ->
59 | #reader{args=A, cache=Ch, feed=Feed} = R1 = F(R),
60 | case Ch of
61 | C1 -> Acc ++ A;
62 | {_,_,K} when binary_part(K,{0,byte_size(Feed)}) == Feed
63 | andalso length(A) == 4
64 | -> feed(F, R1, Acc ++ A, H-4);
65 | _ -> Acc ++ A
66 | end.
67 |
68 | load_reader(Id) -> load_reader(Id,db()).
69 | load_reader(Id,Db) -> case kvs:get(reader,Id,#kvs{db=Db,mod=kvs_rocks}) of {ok,#reader{}=C} -> C; _ -> #reader{id=kvs:seq([],[])} end.
70 |
71 | writer(Id) -> writer(Id,db()).
72 | writer(Id,Db) -> case kvs:get(writer,Id,#kvs{db=Db,mod=kvs_rocks}) of {ok,W} -> W; {error,_} -> #writer{id=Id} end.
73 | reader(Id) -> reader(Id,db()).
74 | reader(Id,Db) -> case kvs:get(writer,Id,#kvs{db=Db,mod=kvs_rocks}) of
75 | {ok,#writer{id=Feed, count=Cn, cache=Ch}} ->
76 | read_it(#reader{id=kvs:seq([],[]),feed=key(Feed),count=Cn,cache=Ch},seek_it(key(Feed),Db));
77 | {error,_} ->
78 | read_it(#reader{id=kvs:seq([],[]),feed=key(Id),count=0,cache=[]},seek_it(key(Id),Db))
79 | end.
80 | save(C) -> save(C,db()).
81 | save(C,Db) ->
82 | N1 = case id(C) of [] -> si(C,kvs:seq([],[])); _ -> C end,
83 | NC = c4(N1,[]),
84 | kvs:put(NC,#kvs{db=Db,mod=kvs_rocks}), NC.
85 |
86 | % add
87 | raw_append(M,Feed) -> raw_append(M,Feed,db()).
88 | raw_append(M,Feed,Db) -> rocksdb:put(ref(Db), key(Feed,e(2,M)), term_to_binary(M), [{sync,true}]).
89 |
90 | add(#writer{}=X) -> add(X, db()).
91 | add(#writer{args=M}=C,Db) when element(2,M) == [] -> add(si(M,kvs:seq([],[])),C,Db);
92 | add(#writer{args=M}=C,Db) -> add(M,C,Db).
93 |
94 | add(M,#writer{id=Feed,count=S}=C,Db) ->
95 | NS=S+1,
96 | raw_append(M,Feed,Db),
97 | C#writer{cache={e(1,M),e(2,M),key(Feed)},count=NS}.
98 |
99 | cut(Feed) -> cut(Feed,db()).
100 | cut(Feed,Db) ->
101 | #writer{cache={_,Key,Fd}=Ch} = kvs:writer(Feed, #kvs{db=Db,mod=kvs_rocks}),
102 | #reader{} = kvs:prev(reader(Feed, Db)),
103 | #reader{} = kvs:next(#reader{feed=key(Feed), cache=Ch}),
104 |
105 | kvs:delete_range(Feed,{Fd,Key},#kvs{db=Db,mod=kvs_rocks}).
106 |
107 | remove(Rec,Feed,Db) ->
108 | kvs:ensure(#writer{id=Feed},#kvs{db=Db,mod=kvs_rocks}),
109 | W = #writer{count=C, cache=Ch} = kvs:writer(Feed,#kvs{db=Db,mod=kvs_rocks}),
110 | Ch1 = case {e(1,Rec),e(2,Rec),key(Feed)} of % need to keep reference for next element
111 | Ch -> R = reader(Feed,Db), e(4, prev(R#reader{cache=Ch},Db));
112 | _ -> Ch end,
113 | case kvs:delete(Feed,id(Rec),#kvs{db=Db,mod=kvs_rocks}) of
114 | ok -> Count = C - 1,
115 | save(W#writer{count = Count, cache=Ch1},Db),
116 | Count;
117 | _ -> C end.
118 |
119 | append(Rec,Feed) -> append(Rec,Feed,db()).
120 | append(Rec,Feed,Db) ->
121 | kvs:ensure(#writer{id=Feed},#kvs{db=Db,mod=kvs_rocks}),
122 | Id = e(2,Rec),
123 | W = writer(Feed,Db),
124 | case kvs:get(Feed,Id,#kvs{db=Db,mod=kvs_rocks}) of
125 | {ok,_} -> raw_append(Rec,Feed,Db), Id;
126 | {error,_} -> save(add(W#writer{args=Rec},Db),Db), Id end.
127 |
--------------------------------------------------------------------------------
/src/layers/kvs_stream.erl:
--------------------------------------------------------------------------------
1 | -module(kvs_stream).
2 | -description('KVS STREAM LAYER').
3 | -include("kvs.hrl").
4 | -include("stream.hrl").
5 | -include("metainfo.hrl").
6 | -export(?STREAM).
7 | -export([metainfo/0]).
8 |
9 | db() -> kvs:db().
10 |
11 | % boot for sample
12 |
13 | metainfo() -> #schema { name = kvs, tables = tables() }.
14 | tables() -> [ #table { name = writer, fields = record_info(fields, writer) },
15 | #table { name = reader, fields = record_info(fields, reader) } ].
16 |
17 | % section: kvs_stream prelude
18 |
19 | se(X,Y,Z) -> setelement(X,Y,Z).
20 | e(X,Y) -> element(X,Y).
21 | c4(R,V) -> se(#reader.args, R, V).
22 | sn(M,T) -> se(#iter.next, M, T).
23 | sp(M,T) -> se(#iter.prev, M, T).
24 | si(M,T) -> se(#iter.id, M, T).
25 | tab(T) -> e(1, T).
26 | id(T) -> e(#iter.id, T).
27 | en(T) -> e(#iter.next, T).
28 | ep(T) -> e(#iter.prev, T).
29 | acc(0) -> next;
30 | acc(1) -> prev.
31 |
32 | n({ok,R},C,P,Db) -> r(kvs:get(tab(R),en(R),#kvs{db=Db}),C,P);
33 | n({error,X},_,_,_) -> {error,X}.
34 |
35 | p({ok,R},C,P,Db) -> r(kvs:get(tab(R),ep(R),#kvs{db=Db}),C,P);
36 | p({error,X},_,_,_) -> {error,X}.
37 |
38 | r({ok,R},C,P) -> C#reader{cache={tab(R),id(R)},pos=P};
39 | r({error,X},_,_) -> {error,X}.
40 |
41 | w({ok,#writer{first=[]}},bot,C) -> C#reader{cache=[],pos=1,dir=0};
42 | w({ok,#writer{first=B}},bot,C) -> C#reader{cache={tab(B),id(B)},pos=1,dir=0};
43 | w({ok,#writer{cache=B,count=Size}},top,C) -> C#reader{cache={tab(B),id(B)},pos=Size,dir=1};
44 | w({error,X},_,_) -> {error,X}.
45 |
46 | % next, prev, top, bot
47 | top(#reader{}=X) -> top(X,db()).
48 | top(#reader{feed=F}=C,Db) -> w(kvs:get(writer,F,#kvs{db=Db}),top,C).
49 | bot(#reader{}=X) -> bot(X,db()).
50 | bot(#reader{feed=F}=C,Db) -> w(kvs:get(writer,F,#kvs{db=Db}),bot,C).
51 |
52 | next(#reader{}=X) -> next(X,db()).
53 | next(#reader{cache=[]},_) -> {error,empty};
54 | next(#reader{cache={T,R},pos=P}=C,Db) -> n(kvs:get(T,R,#kvs{db=Db}),C,P+1,Db).
55 |
56 | prev(#reader{}=X) -> prev(X,db()).
57 | prev(#reader{cache=[]},_) -> {error,empty};
58 | prev(#reader{cache={T,R},pos=P}=C,Db) -> p(kvs:get(T,R,#kvs{db=Db}),C,P-1,Db).
59 |
60 | % take, drop, feed
61 |
62 | drop(#reader{}=X) -> drop(X,db()).
63 | drop(#reader{cache=[]}=C,_) -> C#reader{args=[]};
64 | drop(#reader{dir=D,cache=B,args=N,pos=P}=C,Db) -> drop(acc(D),N,C,C,P,B,Db).
65 | take(#reader{}=X) -> take(X,db()).
66 | take(#reader{cache=[]}=C,_) -> C#reader{args=[]};
67 | take(#reader{dir=D,cache=_B,args=N,pos=P}=C,Db) -> take(acc(D),N,C,C,[],P,Db).
68 |
69 | take(_,_,{error,_},C2,R,P,_) -> C2#reader{args=lists:flatten(R),pos=P,cache={tab(hd(R)),en(hd(R))}};
70 | take(_,0,_,C2,R,P,_) -> C2#reader{args=lists:flatten(R),pos=P,cache={tab(hd(R)),en(hd(R))}};
71 | take(A,N,#reader{cache={T,I},pos=P}=C,C2,R,_,Db) -> take(A,N-1,?MODULE:A(C,Db),C2,[element(2,kvs:get(T,I,#kvs{db=Db}))|R],P,Db).
72 |
73 | drop(_,_,{error,_},C2,P,B,_) -> C2#reader{pos=P,cache=B};
74 | drop(_,0,_,C2,P,B,_) -> C2#reader{pos=P,cache=B};
75 | drop(A,N,#reader{cache=B,pos=P}=C,C2,_,_,Db) -> drop(A,N-1,?MODULE:A(C,Db),C2,P,B,Db).
76 |
77 | feed(Feed) -> feed(Feed,db()).
78 | feed(Feed,Db) -> #reader{args=Args} = take((reader(Feed,Db))#reader{args=-1},Db), Args.
79 |
80 | % new, save, load, writer, reader
81 |
82 | load_reader (Id) -> load_reader(Id,db()).
83 | load_reader (Id,Db) -> case kvs:get(reader,Id,#kvs{db=Db}) of {ok,C} -> C; _ -> #reader{id=[]} end.
84 |
85 | save (C) -> save(C,db()).
86 | save (C,Db) -> NC = c4(C,[]), kvs:put(NC,#kvs{db=Db}), NC.
87 | writer (Id) -> writer(Id,db()).
88 | writer (Id,Db) -> case kvs:get(writer,Id,#kvs{db=Db}) of {ok,W} -> W; {error,_} -> #writer{id=Id} end.
89 | reader (Id) -> reader(Id,db()).
90 | reader (Id,Db) -> case kvs:get(writer,Id,#kvs{db=Db}) of
91 | {ok,#writer{first=[]}} -> #reader{id=kvs:seq(reader,1),feed=Id,cache=[]};
92 | {ok,#writer{first=F}} -> #reader{id=kvs:seq(reader,1),feed=Id,cache={tab(F),id(F)}};
93 | {error,_} -> save(#writer{id=Id},Db), reader(Id,Db) end.
94 |
95 | % add, remove, append
96 |
97 | add(#writer{}=X) -> add(X,db()).
98 | add(#writer{args=M}=C,Db) when element(2,M) == [] -> add(si(M,kvs:seq(tab(M),1)),C,Db);
99 | add(#writer{args=M}=C,Db) -> add(M,C,Db).
100 |
101 | add(M,#writer{cache=[]}=C,Db) ->
102 | _Id=id(M), N=sp(sn(M,[]),[]), kvs:put(N,#kvs{db=Db}),
103 | C#writer{cache=N,count=1,first=N};
104 |
105 | add(M,#writer{cache=V1,count=S}=C,Db) ->
106 | {ok,V} = kvs:get(tab(V1),id(V1),#kvs{db=Db}),
107 | N=sp(sn(M,[]),id(V)), P=sn(V,id(M)), kvs:put([N,P],#kvs{db=Db}),
108 | C#writer{cache=N,count=S+1}.
109 |
110 | cut(Feed) -> cut(Feed, db()).
111 | cut(_,_) -> ignore.
112 | remove(Rec,Feed) -> remove(Rec,Feed,db()).
113 | remove(_Rec,Feed,Db) ->
114 | {ok,W=#writer{count=Count}} = kvs:get(writer,Feed,#kvs{db=Db}),
115 | NC = Count-1,
116 | kvs:save(W#writer{count=NC},#kvs{db=Db}),
117 | NC.
118 |
119 | append(Rec,Feed) -> append(Rec,Feed,db()).
120 | append(Rec,Feed,Db) ->
121 | kvs:ensure(#writer{id=Feed},#kvs{db=Db}),
122 | Name = element(1,Rec),
123 | Id = element(2,Rec),
124 | case kvs:get(Name,Id,#kvs{db=Db}) of
125 | {ok,_} -> Id;
126 | {error,_} -> kvs:save(kvs:add((kvs:writer(Feed,#kvs{db=Db}))#writer{args=Rec},#kvs{db=Db}),#kvs{db=Db}), Id end.
127 |
--------------------------------------------------------------------------------
/src/stores/kvs_fs.erl:
--------------------------------------------------------------------------------
1 | -module(kvs_fs).
2 | -include("backend.hrl").
3 | -include("kvs.hrl").
4 | -include("metainfo.hrl").
5 | -include_lib("stdlib/include/qlc.hrl").
6 | -export(?BACKEND).
7 |
8 | db() -> "".
9 |
10 | start() -> ok.
11 |
12 | stop() -> ok.
13 |
14 | destroy() -> ok.
15 |
16 | destroy(_) -> ok.
17 |
18 | match(_) -> [].
19 |
20 | index_match(_,_) -> [].
21 |
22 | version() -> {version,"KVS FS"}.
23 |
24 | leave() -> ok.
25 |
26 | leave(_) -> ok.
27 |
28 | dir() -> [ {table,F} || F <- filelib:wildcard("data/*"), filelib:is_dir(F) ].
29 |
30 | join(_Node,_) -> filelib:ensure_dir(dir_name()), initialize(). % should be rsync or smth
31 |
32 | initialize() ->
33 | mnesia:create_schema([node()]),
34 | [ kvs:initialize(kvs_fs,Module) || Module <- kvs:modules() ],
35 | mnesia:wait_for_tables([ T#table.name || T <- kvs:tables()],infinity).
36 |
37 | index(_Tab,_Key,_Value) -> [].
38 | get(TableName, Key, _) ->
39 | HashKey = hashkey(Key),
40 | {ok, Dir} = dir(TableName),
41 | File = filename:join([Dir,HashKey]),
42 | case file:read_file(File) of
43 | {ok,Binary} -> {ok,binary_to_term(Binary,[safe])};
44 | {error,Reason} -> {error,Reason} end.
45 |
46 | put(R) -> put(R,db()).
47 | put(Records,X) when is_list(Records) -> lists:map(fun(Record) -> put(Record,X) end, Records);
48 | put(Record,_) ->
49 | TableName = element(1,Record),
50 | HashKey = hashkey(element(2,Record)),
51 | BinaryValue = term_to_binary(Record),
52 | {ok, Dir} = dir(TableName),
53 | File = filename:join([Dir,HashKey]),
54 | filelib:ensure_dir(File),
55 | file:write_file(File,BinaryValue,[write,raw,binary,sync]).
56 |
57 | dir_name() -> "data".
58 |
59 | dir(TableName) ->
60 | {ok, Cwd} = file:get_cwd(),
61 | TablePath = filename:join([dir_name(),TableName]),
62 | case filelib:safe_relative_path(TablePath, Cwd) of
63 | unsafe -> {error, {unsafe, TablePath}};
64 | Target -> {ok, Target}
65 | end.
66 |
67 | hashkey(Key) -> encode(base64:encode(crypto:hash(sha, term_to_binary(Key)))).
68 |
69 | delete(TableName, Key, _) ->
70 | case kvs_fs:get(TableName, Key) of
71 | {ok,_} ->
72 | {ok, Dir} = dir(TableName),
73 | HashKey = hashkey(Key),
74 | File = filename:join([Dir,HashKey]),
75 | file:delete(File);
76 | {error,X} -> {error,X}
77 | end.
78 | delete_range(_,_,_) -> {error, not_found}.
79 | keys(_,_) -> [].
80 | key_match(_,_,_) -> [].
81 |
82 | count(RecordName) -> length(filelib:fold_files(filename:join([dir_name(), RecordName]), "",true, fun(A,Acc)-> [A|Acc] end, [])).
83 |
84 | all(R,_) -> lists:flatten([ begin case file:read_file(File) of
85 | {ok,Binary} -> binary_to_term(Binary,[safe]);
86 | {error,_Reason} -> [] end end || File <-
87 | filelib:fold_files(filename:join([dir_name(), R]), "",true, fun(A,Acc)-> [A|Acc] end, []) ]).
88 |
89 | seq(RecordName, Incr) -> kvs_mnesia:seq(RecordName, Incr).
90 |
91 | create_table(Name,_Options) ->
92 | {ok, Dir} = dir(Name),
93 | file:make_dir(Dir),
94 | filelib:ensure_dir(Dir).
95 |
96 | add_table_index(_Record, _Field) -> ok.
97 |
98 | % URL ENCODE
99 |
100 | encode(B) when is_binary(B) -> encode(binary_to_list(B));
101 | encode([C | Cs]) when C >= $a, C =< $z -> [C | encode(Cs)];
102 | encode([C | Cs]) when C >= $A, C =< $Z -> [C | encode(Cs)];
103 | encode([C | Cs]) when C >= $0, C =< $9 -> [C | encode(Cs)];
104 | encode([C | Cs]) when C == 16#20 -> [$+ | encode(Cs)];
105 |
106 | % unreserved
107 | encode([C = $- | Cs]) -> [C | encode(Cs)];
108 | encode([C = $_ | Cs]) -> [C | encode(Cs)];
109 | encode([C = 46 | Cs]) -> [C | encode(Cs)];
110 | encode([C = $! | Cs]) -> [C | encode(Cs)];
111 | encode([C = $~ | Cs]) -> [C | encode(Cs)];
112 | encode([C = $* | Cs]) -> [C | encode(Cs)];
113 | encode([C = 39 | Cs]) -> [C | encode(Cs)];
114 | encode([C = $( | Cs]) -> [C | encode(Cs)];
115 | encode([C = $) | Cs]) -> [C | encode(Cs)];
116 |
117 | encode([C | Cs]) when C =< 16#7f -> escape_byte(C) ++ encode(Cs);
118 | encode([C | Cs]) when (C >= 16#7f) and (C =< 16#07FF) ->
119 | escape_byte((C bsr 6) + 16#c0)
120 | ++ escape_byte(C band 16#3f + 16#80)
121 | ++ encode(Cs);
122 | encode([C | Cs]) when (C > 16#07FF) ->
123 | escape_byte((C bsr 12) + 16#e0) % (0xe0 | C >> 12)
124 | ++ escape_byte((16#3f band (C bsr 6)) + 16#80) % 0x80 | ((C >> 6) & 0x3f)
125 | ++ escape_byte(C band 16#3f + 16#80) % 0x80 | (C >> 0x3f)
126 | ++ encode(Cs);
127 | encode([C | Cs]) -> escape_byte(C) ++ encode(Cs);
128 | encode([]) -> [].
129 |
130 | hex_octet(N) when N =< 9 -> [$0 + N];
131 | hex_octet(N) when N > 15 -> hex_octet(N bsr 4) ++ hex_octet(N band 15);
132 | hex_octet(N) -> [N - 10 + $a].
133 | escape_byte(C) -> normalize(hex_octet(C)).
134 | normalize(H) when length(H) == 1 -> "%0" ++ H;
135 | normalize(H) -> "%" ++ H.
136 |
137 | dump() -> ok.
138 |
--------------------------------------------------------------------------------
/src/stores/kvs_mnesia.erl:
--------------------------------------------------------------------------------
1 | -module(kvs_mnesia).
2 | -include("backend.hrl").
3 | -include("kvs.hrl").
4 | -include("metainfo.hrl").
5 | -include_lib("mnesia/src/mnesia.hrl").
6 | -include_lib("stdlib/include/qlc.hrl").
7 | -export(?BACKEND).
8 | -export([info/1,exec/1,dump/1]).
9 |
10 | db() -> "".
11 | seq_pad() -> application:get_env(kvs, seq_pad, []).
12 | start() -> mnesia:start().
13 | stop() -> mnesia:stop().
14 | destroy() -> [mnesia:delete_table(T)||{_,T}<-kvs:dir()], mnesia:delete_schema([node()]), ok.
15 | destroy(_) -> [mnesia:delete_table(T)||{_,T}<-kvs:dir()], mnesia:delete_schema([node()]), ok.
16 | leave() -> ok.
17 | leave(_) -> ok.
18 | version() -> {version,"KVS MNESIA"}.
19 | dir() -> [{table,T}||T<-mnesia:system_info(local_tables)].
20 | join([], _) -> mnesia:start(), mnesia:change_table_copy_type(schema, node(), disc_copies), initialize();
21 |
22 | join(Node, _) ->
23 | mnesia:start(),
24 | mnesia:change_config(extra_db_nodes, [Node]),
25 | mnesia:change_table_copy_type(schema, node(), disc_copies),
26 | [{Tb, mnesia:add_table_copy(Tb, node(), Type)}
27 | || {Tb, [{N, Type}]} <- [{T, mnesia:table_info(T, where_to_commit)}
28 | || T <- mnesia:system_info(tables)], Node==N].
29 |
30 | initialize() ->
31 | mnesia:create_schema([node()]),
32 | Res = [ kvs:initialize(kvs_mnesia,Module) || Module <- kvs:modules() ],
33 | mnesia:wait_for_tables([ T#table.name || T <- kvs:tables()],infinity),
34 | Res.
35 |
36 | index(Tab,Key,Value) ->
37 | lists:flatten(many(fun() -> mnesia:index_read(Tab,Value,Key) end)).
38 |
39 | keys(Tab,_) -> mnesia:all_keys(Tab).
40 |
41 | key_match(_Tab,_Id,_) -> [].
42 |
43 | get(RecordName, Key, _) -> just_one(fun() -> mnesia:read(RecordName, Key) end).
44 | put(R) -> put(R,db()).
45 | put(Records, _) when is_list(Records) -> void(fun() -> lists:foreach(fun mnesia:write/1, Records) end);
46 | put(Record, X) -> put([Record], X).
47 | delete(Tab, Key, _) ->
48 | case mnesia:activity(context(),fun()-> mnesia:delete({Tab, Key}) end) of
49 | {aborted,Reason} -> {error,Reason};
50 | {atomic,_Result} -> ok;
51 | _ -> ok end.
52 | delete_range(_,_,_) -> {error, not_found}.
53 | match(Record) -> lists:flatten(many(fun() -> mnesia:match_object(Record) end)).
54 | index_match(Record, Index) -> lists:flatten(many(fun() -> mnesia:index_match_object(Record, Index) end)).
55 | count(RecordName) -> mnesia:table_info(RecordName, size).
56 | all(R, _) -> lists:flatten(many(fun() -> L= mnesia:all_keys(R), [ mnesia:read({R, G}) || G <- L ] end)).
57 | seq(RecordName, []) -> seq(RecordName, 1);
58 | seq(RecordName, Incr) ->
59 | Val = integer_to_list(mnesia:dirty_update_counter({id_seq, RecordName}, Incr)),
60 | case (20 - length(Val) > 0) and lists:member(RecordName, seq_pad()) of true -> string:copies("0", 20 - length(Val)); _ -> "" end ++ Val.
61 |
62 | many(Fun) -> case mnesia:activity(context(),Fun) of {atomic, [R]} -> R; {aborted, Error} -> {error, Error}; X -> X end.
63 | void(Fun) -> case mnesia:activity(context(),Fun) of {atomic, ok} -> ok; {aborted, Error} -> {error, Error}; X -> X end.
64 | info(T) -> try mnesia:table_info(T,all) catch _:_ -> [] end.
65 | create_table(Name,Options) -> mnesia:create_table(Name, Options).
66 | add_table_index(Record, Field) -> mnesia:add_table_index(Record, Field).
67 | exec(Q) -> F = fun() -> qlc:e(Q) end, {atomic, Val} = mnesia:activity(context(),F), Val.
68 | just_one(Fun) ->
69 | case mnesia:activity(context(),Fun) of
70 | {atomic, []} -> {error, not_found};
71 | {atomic, [R]} -> {ok, R};
72 | [] -> {error, not_found};
73 | [R] -> {ok,R};
74 | R when is_list(R) -> {ok,R};
75 | Error -> Error end.
76 |
77 | context() -> application:get_env(kvs,mnesia_context,async_dirty).
78 |
79 | dump() -> dump([ N || #table{name=N} <- kvs:tables() ]), ok.
80 | dump(short) ->
81 | Gen = fun(T) ->
82 | {S,M,C}=lists:unzip3([ dump_info(R) || R <- T ]),
83 | {lists:usort(S),lists:sum(M),lists:sum(C)}
84 | end,
85 | dump_format([ {T,Gen(T)} || T <- [ N || #table{name=N} <- kvs:tables() ] ]);
86 | dump(Table) when is_atom(Table) -> dump(Table);
87 | dump(Tables) ->
88 | dump_format([{T,dump_info(T)} || T <- lists:flatten(Tables) ]).
89 | dump_info(T) ->
90 | {mnesia:table_info(T,storage_type),
91 | mnesia:table_info(T,memory) * erlang:system_info(wordsize) / 1024 / 1024,
92 | mnesia:table_info(T,size)}.
93 | dump_format(List) ->
94 | io:format("~20s ~32s ~14s ~10s~n~n",["NAME","STORAGE TYPE","MEMORY (MB)","ELEMENTS"]),
95 | [ io:format("~20s ~32w ~14.2f ~10b~n",[T,S,M,C]) || {T,{S,M,C}} <- List ],
96 | io:format("~nSnapshot taken: ~p~n",[calendar:now_to_datetime(os:timestamp())]).
97 |
--------------------------------------------------------------------------------
/src/stores/kvs_rocks.erl:
--------------------------------------------------------------------------------
1 | -module(kvs_rocks).
2 | -include("backend.hrl").
3 | -include("kvs.hrl").
4 | -include("metainfo.hrl").
5 | -include_lib("stdlib/include/qlc.hrl").
6 | -export(?BACKEND).
7 | -export([ref/0,ref/1,bt/1,key/2,key/1,fd/1,tb/1,estimate/0,estimate/1]).
8 | -export([seek_it/1, seek_it/2, move_it/3, move_it/4, take_it/4, take_it/5, delete_it/1, delete_it/2]).
9 |
10 | e(X,Y) -> element(X,Y).
11 |
12 | bt([]) -> [];
13 | bt(X) -> binary_to_term(X).
14 |
15 | tb([]) -> <<>>;
16 | tb(T) when is_list(T) -> unicode:characters_to_nfkc_binary(T);
17 | tb(T) when is_atom(T) -> erlang:atom_to_binary(T, utf8);
18 | tb(T) when is_binary(T) -> T;
19 | tb(T) -> term_to_binary(T).
20 | sz([]) -> 0;
21 | sz(B) -> byte_size(B).
22 |
23 | key(R) when is_tuple(R) andalso tuple_size(R) > 1 -> key(e(1,R), e(2,R));
24 | key(R) -> key(R,[]).
25 | key(writer,R) -> % allow old writers
26 | iolist_to_binary([lists:join(<<"/">>, lists:flatten([<<>>, erlang:atom_to_binary(writer, utf8), tb(R)]))]);
27 | key(Tab,R) -> Fd = case Tab of [] -> []; _ -> tb(Tab) end,
28 | iolist_to_binary([lists:join(<<"/">>, lists:flatten([<<>>, Fd, fmt(R)]))]).
29 | keys(Tab, Db) ->
30 | Feed = key(Tab,[]),
31 | {ok, H} = rocksdb:iterator(ref(Db), []),
32 | Keys = fun KEY(K1,Acc) when binary_part(K1,{0,byte_size(Feed)}) =:= Feed ->
33 | case rocksdb:iterator_move(H, next) of
34 | {ok,K2,_} -> KEY(K2,[tb(K1)|Acc]);
35 | _ -> lists:reverse([tb(K1)|Acc])
36 | end;
37 | KEY(_,Acc) -> rocksdb:iterator_close(H), lists:reverse(Acc)
38 | end,
39 | {ok, K, _} = rocksdb:iterator_move(H, {seek, Feed}),
40 | Keys(K,[]).
41 |
42 | key_match(Tab, Id, Db) ->
43 | Feed = key(Tab,[]),
44 | {ok, H} = rocksdb:iterator(ref(Db), []),
45 | Keys = fun KEY(K1) when
46 | binary_part(K1,{0,byte_size(Feed)}) =:= Feed andalso
47 | binary_part(K1,{byte_size(K1), -byte_size(Id)}) =:= Id ->
48 | rocksdb:iterator_close(H), [K1];
49 | KEY(K1) when binary_part(K1,{0,byte_size(Feed)}) =:= Feed ->
50 | case rocksdb:iterator_move(H, next) of
51 | {ok,K2,_} -> KEY(K2);
52 | _ -> []
53 | end;
54 | KEY(_) -> rocksdb:iterator_close(H), []
55 | end,
56 | {ok, K, _} = rocksdb:iterator_move(H, {seek, Feed}),
57 | Keys(K).
58 |
59 | fmt([]) -> [];
60 | fmt(K) -> Key = tb(K),
61 | End = sz(Key),
62 | {S,E} = case binary:matches(Key, [<<"/">>], []) of
63 | [{0,1}] -> {1, End-1};
64 | [{0,1},{1,1}] -> {2, End-2};
65 | [{0,1},{1,1}|_] -> {2, End-2};
66 | [{0,1}|_] -> {1, End-1};
67 | _ -> {0, End}
68 | end,
69 | binary:part(Key,{S,E}).
70 |
71 | fd(K) -> Key = tb(K),
72 | End = sz(Key),
73 | {S,_} = case binary:matches(Key,[<<"/">>],[]) of
74 | [{0,1}] -> {End,End};
75 | [{0,1},{1,1}] -> {End,End};
76 | [{0,1},{1,1}|T] -> hd(lists:reverse(T));
77 | [{0,1}|T] -> hd(lists:reverse(T));
78 | _ -> {End,End}
79 | end,
80 | binary:part(Key,{0,S}).
81 |
82 | run(<<>>,SK,_,_,_) -> {ok,SK,[],[]};
83 | run(Key, % key
84 | SK, % sup-key
85 | Dir, % direction next/prev
86 | Compiled_Operations,
87 | Db) ->
88 | % H is iterator reference
89 |
90 | S = sz(SK),
91 | Initial_Object = {ref(Db), []},
92 |
93 | Run = fun (F,K,H,V,Acc) when binary_part(K,{0,S}) == SK -> {F(H,Dir),H,[V|Acc]}; % continue +------------+
94 | (_,K,H,V,Acc) -> stop_it(H), % fail-safe closing |
95 | throw({ok, fd(K), bt(V), [bt(A1) || A1 <- Acc]}) end, % acc unfold |
96 | % |
97 | Range_Check = fun(F,K,H,V) -> case F(H,prev) of % backward prefetch |
98 | {ok,K1,V1} when binary_part(K,{0,S}) == SK -> {{ok,K1,V1},H,[V]}; % return (range-check error) |
99 | {ok,K1,V1} -> Run(F,K1,H,V1,[]); % run prev-take chain | loop
100 | _ -> stop_it(H), % fail-safe closing |
101 | throw({ok, fd(K), bt(V), [bt(V)]}) % acc unfold |
102 | end end, % |
103 | % |
104 | State_Machine = fun % |
105 | (F,{ok,H}) -> {F(H,{seek,Key}),H}; % first move (seek) |
106 | (F,{{ok,K,V},H}) when Dir =:= prev -> Range_Check(F,K,H,V); % first chained prev-take |
107 | (F,{{ok,K,V},H}) -> Run(F,K,H,V,[]); % first chained next-take |
108 | (F,{{ok,K,V},H,A}) -> Run(F,K,H,V,A); % chained CPS-take continuator +---+
109 | (_,{{error,E},H,Acc}) -> {{error,E},H,Acc}; % error effects
110 | (F,{I,O}) -> F(I,O) % chain constructor from initial object
111 | end,
112 |
113 | catch case lists:foldl(State_Machine, Initial_Object, Compiled_Operations) of
114 | {{ok,K,Bin},H,A} -> stop_it(H), {ok, fd(K), bt(Bin), [bt(A1) || A1 <- A]};
115 | {{ok,K,Bin},H} -> stop_it(H), {ok, fd(K), bt(Bin), []};
116 | {{error,_},H,Acc} -> stop_it(H), {ok, fd(SK), bt(shd(Acc)), [bt(A1) || A1 <- Acc]};
117 | {{error,_},H} -> stop_it(H), {ok, fd(SK), [], []}
118 | end.
119 |
120 | initialize() -> [ kvs:initialize(kvs_rocks,Module) || Module <- kvs:modules() ].
121 | index(_,_,_) -> [].
122 |
123 | ref_env(Db) -> list_to_atom("rocks_ref_" ++ Db).
124 | db() -> application:get_env(kvs,rocks_name,"rocksdb").
125 | start() -> ok.
126 | stop() -> ok.
127 | destroy() -> destroy(db()).
128 | destroy(Db) -> rocksdb:destroy(Db, []).
129 | version() -> {version,"KVS ROCKSDB"}.
130 | dir() -> [].
131 | match(_) -> [].
132 | index_match(_,_) -> [].
133 | ref() -> ref(db()).
134 | ref(Db) -> application:get_env(kvs,ref_env(Db),[]).
135 | leave() -> leave(db()).
136 | leave(Db) -> case ref(Db) of [] -> skip; X -> rocksdb:close(X), application:set_env(kvs,ref_env(Db),[]), ok end.
137 | join(_,Db) ->
138 | application:start(rocksdb),
139 | leave(Db), {ok, Ref} = rocksdb:open(Db, [{create_if_missing, true}]),
140 | initialize(),
141 | application:set_env(kvs,ref_env(Db),Ref).
142 |
143 | compile(it) -> [fun rocksdb:iterator/2];
144 | compile(seek) -> [fun rocksdb:iterator/2,fun rocksdb:iterator_move/2];
145 | compile(move) -> [fun rocksdb:iterator_move/2];
146 | compile(close) -> [fun rocksdb:iterator_close/1].
147 | compile(take,N) -> lists:map(fun(_) -> fun rocksdb:iterator_move/2 end, lists:seq(1, N)).
148 | compile(delete,_, {error,E},_) -> {error,E};
149 | compile(delete,SK,{ok,_,V1,_},Db) ->
150 | F1 = key(key(fmt(SK),e(2,V1))), S = sz(SK),
151 | [fun Del(H,Dir) ->
152 | case rocksdb:delete(ref(Db), F1, []) of ok ->
153 | % {ok, K} case exist only in api, but never actually used
154 | case rocksdb:iterator_move(H,Dir) of
155 | {ok,K,_} when binary_part(K,{0,S}) == SK -> case rocksdb:delete(ref(Db), K, []) of ok -> Del(H,Dir); E -> E end;
156 | {ok,K} when binary_part(K,{0,S}) == SK -> case rocksdb:delete(ref(Db), K, []) of ok -> Del(H,Dir); E -> E end;
157 | {ok,K,V} -> {ok,K,V};
158 | {ok,K} -> {ok, K};
159 | E -> E
160 | end;
161 | E -> E
162 | end
163 | end].
164 |
165 | stop_it(H) -> try begin [F]=compile(close), F(H) end catch error:badarg -> ok end.
166 | seek_it(K) -> seek_it(K,db()).
167 | seek_it(K,Db) -> run(K,K,ok,compile(seek),Db).
168 | move_it(Key,SK,Dir) -> move_it(Key,SK,Dir,db()).
169 | move_it(Key,SK,Dir,Db) -> run(Key,SK,Dir,compile(seek) ++ compile(move),Db).
170 | take_it(Key,SK,Dir,N) -> take_it(Key,SK,Dir,N,db()).
171 | take_it(Key,SK,Dir,N,Db) when is_integer(N) andalso N >= 0 -> run(Key,SK,Dir,compile(seek) ++ compile(take,N),Db);
172 | take_it(Key,SK,Dir,_,Db) -> take_it(Key,SK,Dir,0,Db).
173 | delete_it(Fd) -> delete_it(Fd, db()).
174 | delete_it(Fd,Db) -> run(Fd,Fd,next,compile(seek) ++ compile(delete,Fd,seek_it(Fd),Db),Db).
175 |
176 | all(R,Db) -> kvs_st:feed(R,Db).
177 |
178 | get(Tab, {step,N,[208|_]=Key}, Db) -> get(Tab, {step,N,list_to_binary(Key)},Db);
179 | get(Tab, [208|_]=Key, Db) -> get(Tab, list_to_binary(Key), Db);
180 | get(Tab, Key, Db) ->
181 | case rocksdb:get(ref(Db), key(Tab,Key), []) of
182 | not_found -> {error,not_found};
183 | {ok,Bin} -> {ok,bt(Bin)} end.
184 |
185 | put(Record) -> put(Record,db()).
186 | put(Records,Db) when is_list(Records) -> lists:map(fun(Record) -> put(Record,Db) end, Records);
187 | put(Record,Db) -> rocksdb:put(ref(Db), key(Record), term_to_binary(Record), [{sync,true}]).
188 | delete(Feed, Id, Db) -> rocksdb:delete(ref(Db), key(Feed,Id), []).
189 | delete_range(Feed,{Fd,Key},Db) ->
190 | Last = key(key(fmt(Fd),Key)),
191 | ReadOps = [{'prefix_same_as_start', true}],
192 | CompactOps = [{change_level, true}],
193 | Feed1 = key(Feed),
194 | Sz = size(Feed1),
195 | Reopen = case ref(Db) of [] -> skip; _ -> leave(Db), ok end,
196 |
197 | {ok, R} = rocksdb:open(Db, [{prefix_extractor, {capped_prefix_transform, Sz}}]),
198 | {ok, H} = rocksdb:iterator(R, ReadOps),
199 | {ok, Start, _} = rocksdb:iterator_move(H, {seek, Feed1}),
200 |
201 | ok = rocksdb:delete_range(R, Start, Last, []),
202 | ok = rocksdb:delete(R, Last, []),
203 | ok = rocksdb:delete(R, key(writer,Feed), []),
204 | ok = rocksdb:compact_range(R, Start, undefined, CompactOps),
205 | ok = rocksdb:iterator_close(H),
206 | ok = rocksdb:close(R),
207 | case Reopen of skip -> ok; ok -> join([],Db) end.
208 | count(_) -> 0.
209 | estimate() -> estimate(db()).
210 | estimate(Db) -> case rocksdb:get_property(ref(Db), <<"rocksdb.estimate-num-keys">>) of
211 | {ok, Est} when is_binary(Est) -> binary_to_integer(Est);
212 | {ok, Est} when is_list(Est) -> list_to_integer(Est);
213 | {ok, Est} when is_integer(Est) -> Est;
214 | _ -> 0
215 | end.
216 |
217 | shd([]) -> [];
218 | shd(X) -> hd(X).
219 | create_table(_,_) -> [].
220 | add_table_index(_, _) -> ok.
221 | dump() -> ok.
222 |
223 | seq(_,_) ->
224 | Val =
225 | case os:type() of
226 | {win32,nt} -> {Mega,Sec,Micro} = erlang:timestamp(), integer_to_list((Mega*1000000+Sec)*1000000+Micro);
227 | _ -> integer_to_list(element(2,hd(lists:reverse(erlang:system_info(os_system_time_source)))))
228 | end,
229 | case 20 - length(Val) > 0 of true -> string:copies("0", 20 - length(Val)); _ -> "" end ++ Val.
230 |
--------------------------------------------------------------------------------
/sys.config:
--------------------------------------------------------------------------------
1 | [ {kvs,[{dba,kvs_mnesia},
2 | {dba_st,kvs_stream},
3 | {schema,[kvs,kvs_stream]}]}
4 | ].
5 |
--------------------------------------------------------------------------------
/test/fd_test.exs:
--------------------------------------------------------------------------------
1 | ExUnit.start()
2 |
3 | defmodule Fd.Test do
4 | use ExUnit.Case, async: false
5 | require KVS
6 | import Record
7 |
8 | test "fd_test", kvs do
9 | assert "//erp" = :kvs_rocks.fd "//erp/orgs"
10 | assert "/erp" = :kvs_rocks.fd "/erp/orgs"
11 | end
12 |
13 | test "key_test", kvs do
14 | assert "//erp/orgs" = :kvs_rocks.key "/erp/orgs"
15 | assert "/erp/orgs" = :kvs_rocks.key "erp/orgs"
16 | end
17 |
18 | defrecord(:msg, id: [], body: [])
19 |
20 | setup do: (on_exit(fn -> :kvs.leave();:ok = :kvs.destroy() end);:kvs.join())
21 | setup kvs, do: [
22 | id0: :lists.map(fn _ -> :kvs.append(msg(id: :kvs.seq([],[])), "/crm/duck") end, :lists.seq(1,10)),
23 | id1: :lists.map(fn _ -> :kvs.append(msg(id: :kvs.seq([],[])), "/crm/luck") end, :lists.seq(1,10)),
24 | id2: :lists.map(fn _ -> :kvs.append(msg(id: :kvs.seq([],[])), "/crm/truck") end, :lists.seq(1,10))]
25 |
26 | test "reader", kvs do
27 | ltop = Enum.at(kvs[:id1],0)
28 | dtop = Enum.at(kvs[:id0],0)
29 | ttop = Enum.at(kvs[:id2],0)
30 | assert KVS.reader(feed: "//crm/luck", count: 10, dir: 0, args: [], cache: {:msg, ^ltop, "//crm/luck"}) = :kvs.reader("/crm/luck")
31 | assert KVS.reader(feed: "//crm/duck", count: 10, dir: 0, args: [], cache: {:msg, ^dtop, "//crm/duck"}) = :kvs.reader("/crm/duck")
32 | assert KVS.reader(feed: "//crm/truck", count: 10, dir: 0, args: [], cache: {:msg, ^ttop, "//crm/truck"}) = :kvs.reader("/crm/truck")
33 | assert KVS.reader(feed: "//crm", count: 0, dir: 0, args: [], cache: {:msg, ^dtop, "//crm/duck"}) = :kvs.reader("/crm")
34 | assert KVS.reader(feed: "//noroute", count: 0, dir: 0, args: []) = :kvs.reader("/noroute")
35 | assert KVS.reader(feed: "//", count: 0, dir: 0, args: [], cache: {:msg, ^dtop, "//crm/duck"}) = :kvs.reader("/")
36 | assert KVS.reader(count: 0, dir: 0, args: []) = :kvs.reader([])
37 | end
38 |
39 | test "range", kvs do
40 | ltop = Enum.at(kvs[:id1],0)
41 | dtop = Enum.at(kvs[:id0],0)
42 | lbot = Enum.at(kvs[:id1],9)
43 |
44 | assert KVS.reader(feed: "//crm/luck", count: 10, dir: 0, args: [], cache: {:msg, ^ltop, "//crm/luck"}) = :kvs.top(:kvs.reader("/crm/luck"))
45 | assert KVS.reader(feed: "//crm", count: 0, dir: 0, args: [], cache: {:msg, ^dtop, "//crm/duck"}) = :kvs.top(:kvs.reader("/crm"))
46 | assert KVS.reader(feed: "//", count: 0, dir: 0, args: [], cache: {:msg, ^dtop, "//crm/duck"}) = :kvs.top(:kvs.reader("/"))
47 |
48 | assert KVS.reader(feed: "//crm/luck", count: 10, dir: 0, args: [], cache: {:msg, ^lbot, "//crm/luck"}) = :kvs.bot(:kvs.reader("/crm/luck"))
49 | assert KVS.reader(feed: "//crm", count: 0, dir: 0, args: [], cache: []) = :kvs.bot(:kvs.reader("/crm"))
50 | assert KVS.reader(feed: "//", count: 0, dir: 0, args: [], cache: []) = :kvs.bot(:kvs.reader("/"))
51 | end
52 |
53 | test "next", kvs do
54 | last = msg(id: Enum.at(kvs[:id1],9))
55 | r0 = KVS.reader(id: rid) = :kvs.save(:kvs.top(:kvs.reader("/crm/luck")))
56 | kvs[:id1] |> Enum.with_index
57 | |> Enum.each(fn {id,9} ->
58 | r = :kvs.load_reader(rid)
59 | r01 = :kvs.next(r)
60 | assert r1 = KVS.reader(feed: "//crm/luck", cache: c1, count: 10, dir: 0, args: [^last]) = r01
61 | assert KVS.reader(args: [], feed: "//crm/luck", cache: c1) = :kvs.save(r1)
62 | {id,i} ->
63 | v = msg(id: Enum.at(kvs[:id1],i))
64 | c = Enum.at(kvs[:id1],i+1)
65 | r = :kvs.load_reader(rid)
66 | assert r1 = KVS.reader(feed: "//crm/luck", cache: {:msg,^c,"//crm/luck"}, count: 10, dir: 0, args: [^v]) = :kvs.next(r)
67 | assert KVS.reader(args: [], feed: "//crm/luck", cache: {:msg,^c,"//crm/luck"}) = :kvs.save(r1)
68 | end)
69 | r = :kvs.load_reader(rid)
70 |
71 | assert r == :kvs.next(r)
72 | assert r == KVS.reader(:kvs.next(:kvs.bot(r)), args: [])
73 | end
74 |
75 | test "prev", kvs do
76 | out = Enum.at(kvs[:id0],9)
77 | KVS.reader(id: rid) = :kvs.save(:kvs.bot(:kvs.reader("/crm/luck")))
78 | ids = kvs[:id1] |> Enum.reverse
79 | ids |> Enum.with_index
80 | |> Enum.each(fn {id,9} ->
81 | r = :kvs.load_reader(rid)
82 | v = msg(id: Enum.at(ids, 9))
83 | assert r1 = KVS.reader(feed: "//crm/luck", cache: {:msg, ^out, "//crm/duck"}, count: 10, args: [^v]) = :kvs.prev(r)
84 | assert KVS.reader(args: [], feed: "//crm/luck", cache: c1) = :kvs.save(r1)
85 | {id,i} ->
86 | r = :kvs.load_reader(rid)
87 | v = msg(id: Enum.at(ids, i))
88 | c = Enum.at(ids, i+1)
89 | assert r1 = KVS.reader(feed: "//crm/luck", cache: {:msg, ^c, "//crm/luck"}, count: 10, args: [^v]) = :kvs.prev(r)
90 | assert KVS.reader(args: [], feed: "//crm/luck", cache: {:msg, ^c, "//crm/luck"}) = :kvs.save(r1)
91 | end)
92 | r = :kvs.load_reader(rid)
93 | assert r = :kvs.prev(r)
94 | assert r = KVS.reader(:kvs.prev(:kvs.top(r)), args: [])
95 | end
96 |
97 | test "prev to empty" do
98 | :lists.map(fn _ -> :kvs.append(msg(id: :kvs.seq([],[])), "/aco") end, :lists.seq(1,2))
99 | all = :kvs.all("/aco")
100 | head = Enum.at(all,0)
101 |
102 | r = :kvs.bot(:kvs.reader("/aco"))
103 | r1 = KVS.reader(args: args) = :kvs.take(KVS.reader(r, args: 2, dir: 1))
104 | assert all == :lists.reverse(args)
105 | assert KVS.reader(args: [^head]) = :kvs.take(KVS.reader(r1, args: 1000, dir: 1))
106 | end
107 |
108 | test "cut the *uck", kvs do
109 | :kvs.cut("/crm/luck")
110 |
111 | all = :kvs.all("/crm")
112 | assert 20 = length(all)
113 | assert all = :kvs.all("/crm/duck") ++ :kvs.all("/crm/truck")
114 |
115 | :kvs.cut("/crm/duck")
116 |
117 | all = :kvs.all("/crm")
118 | assert 10 = length(all)
119 | assert all = :kvs.all("/crm/truck")
120 |
121 | :kvs.cut("/crm/truck")
122 |
123 | all = :kvs.all("/crm")
124 | assert 0 = length(all)
125 | end
126 |
127 | test "remove the *uck with readers", kvs do
128 | :kvs.remove(:kvs.reader("/crm/luck"))
129 |
130 | all = :kvs.all("/crm")
131 | assert 20 = length(all)
132 | assert all = :kvs.all("/crm/duck") ++ :kvs.all("/crm/truck")
133 |
134 | :kvs.remove(:kvs.reader("/crm/duck"))
135 |
136 | all = :kvs.all("/crm")
137 | assert 10 = length(all)
138 | assert all = :kvs.all("/crm/truck")
139 |
140 | :kvs.remove(:kvs.reader("/crm/truck"))
141 |
142 | all = :kvs.all("/crm")
143 | assert 0 = length(all)
144 | end
145 |
146 | @tag :skip # can`t manage this within current implementation. create correct keys!
147 | test "keys with feeds separator" do
148 | :kvs.append(msg(id: "1/1"), "/one/two")
149 | :kvs.append(msg(id: "1/2"), "/one/two")
150 | assert KVS.reader(cache: {:msg, _, "//one/two"}) = :kvs.reader("/one/two")
151 | end
152 |
153 | test "corrupted writers doesn't affect all" do
154 | prev = :kvs.all("/crm/duck")
155 |
156 | KVS.writer(cache: ch) = w = :kvs.writer("/crm/duck")
157 | w1 = KVS.writer(w, cache: {:msg, "unknown", "/corrupted"})
158 |
159 | :ok = :kvs_rocks.put(w1)
160 | w2 = :kvs.writer("/crm/duck")
161 | assert {:ok, ^w2} = :kvs.get(:writer, "/crm/duck")
162 | assert w1 == w2
163 |
164 | assert prev = :kvs.all("/crm/duck")
165 |
166 | {:ok,_} = :kvs.get(:writer, "/crm/duck")
167 | :ok = :kvs.delete(:writer, "/crm/duck")
168 | {:error, :not_found} = :kvs.get(:writer, "/crm/duck")
169 | assert prev = :kvs.all("/crm/duck")
170 | end
171 |
172 | defp log(x), do: IO.puts '#{inspect(x)}'
173 | defp log(m, x), do: IO.puts '#{m} #{inspect(x)}'
174 | end
175 |
--------------------------------------------------------------------------------
/test/sc_test.exs:
--------------------------------------------------------------------------------
1 | ExUnit.start()
2 |
3 | defmodule Sc.Test do
4 | use ExUnit.Case, async: false
5 | require KVS
6 | import Record
7 | @moduledoc """
8 | refined old scenarios
9 | """
10 |
11 | defrecord(:msg, id: [], body: [])
12 |
13 | setup do: (on_exit(fn -> :ok = :kvs.leave();:ok = :kvs.destroy() end);:kvs.join())
14 | setup kvs, do: [
15 | id0: :lists.map(fn _ -> :kvs.append(msg(id: :kvs.seq([],[])), "/crm/duck") end, :lists.seq(1,10)),
16 | id1: :lists.map(fn _ -> :kvs.append(msg(id: :kvs.seq([],[])), "/crm/luck") end, :lists.seq(1,10)),
17 | id2: :lists.map(fn _ -> :kvs.append(msg(id: :kvs.seq([],[])), "/crm/truck") end, :lists.seq(1,10)),
18 | id3: :lists.map(fn _ -> :kvs.save(:kvs.add(KVS.writer(:kvs.writer(:sym),
19 | args: msg(id: :kvs.seq([],[]))))) end, :lists.seq(1,10))]
20 | test "basic", kvs do
21 | KVS.reader(id: rid1) = :kvs.save(:kvs.reader("/crm/luck"))
22 | KVS.reader(id: rid2) = :kvs.save(:kvs.reader("/crm/truck"))
23 | x1 = :kvs.take(KVS.reader(:kvs.load_reader(rid1), args: 20))
24 | x2 = :kvs.take(KVS.reader(:kvs.load_reader(rid2), args: 20))
25 | b = :kvs.feed("/crm/luck")
26 | assert 10 == length(b)
27 | assert :kvs.all("/crm/truck") == KVS.reader(x2, :args)
28 | assert KVS.reader(x1, :args) == b
29 | assert length(KVS.reader(x1, :args)) == length(KVS.reader(x2, :args))
30 | end
31 |
32 | test "sym",kvs do
33 | KVS.writer(args: last) = Enum.at(kvs[:id3],-1)
34 | {:ok, KVS.writer(id: :sym, count: 10, cache: last)} = :kvs.get(:writer, :sym)
35 | end
36 |
37 | test "take back full" do
38 | feed = "/crm/duck"
39 | KVS.reader(id: rid) = :kvs.save(:kvs.reader(feed))
40 | t = KVS.reader(args: a1) = :kvs.take(KVS.reader(:kvs.load_reader(rid), args: 10))
41 | assert a1 == :kvs.feed(feed)
42 | :kvs.save(KVS.reader(t, dir: 1))
43 | KVS.reader(args: a2) = :kvs.take(KVS.reader(:kvs.load_reader(rid), args: 10))
44 | assert :lists.reverse(a2) == :kvs.feed(feed)
45 | end
46 |
47 | test "partial take back" do
48 | KVS.reader(id: rid) = :kvs.save(:kvs.reader("/crm/luck"))
49 | r = KVS.reader(args: t) = :kvs.take(KVS.reader(:kvs.load_reader(rid), args: 2))
50 | :kvs.save(KVS.reader(r, dir: 1))
51 | KVS.reader(args: n) = :kvs.take(KVS.reader(:kvs.load_reader(rid), args: 3))
52 | assert :lists.reverse(t) == tl(n)
53 | end
54 |
55 | defp log(x), do: IO.puts '#{inspect(x)}'
56 | defp log(m, x), do: IO.puts '#{m} #{inspect(x)}'
57 | end
58 |
--------------------------------------------------------------------------------
/test/st_test.exs:
--------------------------------------------------------------------------------
1 | ExUnit.start()
2 |
3 | defmodule St.Test do
4 | use ExUnit.Case, async: false
5 | import Record
6 | require KVS
7 |
8 | defrecord(:msg, id: [], body: [])
9 |
10 | setup do: (on_exit(fn -> :ok = :kvs.leave();:ok = :kvs_rocks.destroy() end);:kvs.join())
11 | setup kvs, do: [
12 | ids: :lists.map(fn _ -> :kvs.append(msg(id: :kvs.seq([],[])), :feed) end, :lists.seq(1,10)),
13 | id0: :lists.map(fn _ -> :kvs.append(msg(id: :kvs.seq([],[])), "/crm/personal/Реєстратор А1/in/directory/duck") end, :lists.seq(1,10)),
14 | id1: :lists.map(fn _ -> :kvs.append(msg(id: :kvs.seq([],[])), "/crm/personal/Реєстратор А1/in/mail") end, :lists.seq(1,10)),
15 | id2: :lists.map(fn _ -> :kvs.append(msg(id: :kvs.seq([],[])), "/crm/personal/Реєстратор А1/in/doc") end, :lists.seq(1,10))]
16 |
17 | test "take-ø", kvs do
18 | r = KVS.reader() = :kvs.reader("/empty-feed")
19 | assert r1 = KVS.reader(feed: "//empty-feed", args: []) = :kvs.take(KVS.reader(r, args: 1))
20 | assert r1 = KVS.reader(feed: "//empty-feed", args: []) = :kvs.take(KVS.reader(r, args: 1, dir: 1))
21 | assert r2 = KVS.reader(feed: "//empty-feed", args: []) = :kvs.next(r)
22 | assert r3 = KVS.reader(feed: "//empty-feed", args: []) = :kvs.prev(r)
23 | assert r1 = KVS.reader(feed: "//empty-feed", args: []) = :kvs.take(KVS.reader(r, args: 100))
24 | assert r1 = KVS.reader(feed: "//empty-feed", args: []) = :kvs.take(KVS.reader(r, args: 100, dir: 1))
25 | KVS.reader(id: rid) = :kvs.save(r1)
26 | assert rs1 = KVS.reader(id: rid) = :kvs.load_reader(rid)
27 | assert KVS.reader(feed: "//empty-feed", args: []) = :kvs.take(KVS.reader(rs1, args: 5))
28 | assert KVS.reader(feed: "//empty-feed", args: []) = :kvs.take(KVS.reader(rs1, args: 5, dir: 1))
29 | assert KVS.reader(feed: "//empty-feed", args: []) = :kvs.next(rs1)
30 | assert KVS.reader(feed: "//empty-feed", args: []) = :kvs.prev(rs1)
31 | assert KVS.reader(feed: "//empty-feed", args: []) = :kvs.take(KVS.reader(rs1, args: 0))
32 | assert KVS.reader(feed: "//empty-feed", args: []) = :kvs.take(KVS.reader(rs1, args: 0, dir: 1))
33 | end
34 |
35 | test "take-0", kvs do
36 | feed = "//crm/personal/Реєстратор А1/in/doc"
37 | assert r = KVS.reader(id: rid, args: []) = :kvs.reader("/crm/personal/Реєстратор А1/in/doc")
38 |
39 | assert KVS.reader(id: ^rid, feed: ^feed, args: []) = :kvs.take(KVS.reader(r, args: 0, dir: 0))
40 | assert KVS.reader(id: ^rid, feed: ^feed, args: []) = :kvs.take(KVS.reader(r, args: -1, dir: 0))
41 |
42 | assert r1 = KVS.reader(id: ^rid, feed: ^feed, args: a01) = :kvs.take(KVS.reader(r, args: 10, dir: 0))
43 | assert kvs[:id2] |> Enum.map(&msg(id: &1)) == a01
44 | assert KVS.reader(id: ^rid, feed: ^feed, args: []) = :kvs.take(KVS.reader(r1, args: 10, dir: 0))
45 |
46 | assert KVS.reader(id: ^rid, feed: ^feed, args: af) = :kvs.take(KVS.reader(r, args: 100, dir: 0))
47 | assert kvs[:id2] |> Enum.map(&msg(id: &1)) == af
48 |
49 | assert r2 = KVS.reader(id: ^rid, feed: ^feed, args: a03) = :kvs.take(KVS.reader(r, args: 3, dir: 0))
50 | assert Enum.take(kvs[:id2],3) |> Enum.map(&msg(id: &1)) == a03
51 |
52 | assert KVS.reader(id: ^rid, feed: ^feed, args: a07) = :kvs.take(KVS.reader(r2, args: 7, dir: 0))
53 | assert Enum.drop(kvs[:id2],3) |> Enum.map(&msg(id: &1)) == a07
54 |
55 | assert KVS.reader(id: ^rid, feed: ^feed, args: a17) = :kvs.take(KVS.reader(r2, args: 100, dir: 0))
56 | assert Enum.drop(kvs[:id2],3) |> Enum.map(&msg(id: &1)) == a17
57 | end
58 |
59 | test "take-1", kvs do
60 | feed = "//crm/personal/Реєстратор А1/in/mail"
61 | top = Enum.at(kvs[:id1],0)
62 | bot = Enum.at(kvs[:id1],9)
63 | tpm = Enum.take(kvs[:id1],1) |> Enum.map(&msg(id: &1))
64 |
65 | assert r = KVS.reader(id: rid, args: [], cache: {:msg, ^top, ^feed}, count: 10) = :kvs.reader("/crm/personal/Реєстратор А1/in/mail")
66 | assert r = :kvs.top(r)
67 | assert KVS.reader(id: ^rid, feed: ^feed, args: ^tpm, dir: 1) = :kvs.take(KVS.reader(r, args: 1, dir: 1))
68 | assert KVS.reader(id: ^rid, feed: ^feed, args: ^tpm, dir: 1) = :kvs.take(KVS.reader(r, args: 100, dir: 1))
69 |
70 | assert r1 = KVS.reader(feed: ^feed, count: 10, args: [], cache: {:msg,^bot,^feed}) = :kvs.bot(r)
71 |
72 | assert r2 = KVS.reader(feed: ^feed, count: 10, args: a01) = :kvs.take(KVS.reader(r1, args: 5, dir: 1))
73 | x01 = Enum.drop(kvs[:id1],5) |> Enum.map(&msg(id: &1)) |> Enum.reverse
74 | assert x01 == a01
75 |
76 | assert r3 = KVS.reader(feed: ^feed, count: 10, args: a02) = :kvs.take(KVS.reader(r2, args: 10, dir: 1))
77 | x02 = Enum.take(kvs[:id1],5) |> Enum.map(&msg(id: &1)) |> Enum.reverse
78 | assert x02 == a02
79 |
80 | assert KVS.reader(feed: ^feed, count: 10, args: []) = :kvs.take(KVS.reader(r3, args: 20, dir: 1))
81 | end
82 |
83 | test "drop", kvs do
84 | feed = "/feed"
85 | assert r = KVS.reader(id: rid, args: []) = :kvs.save(:kvs.reader(:feed))
86 | assert r1 = KVS.reader(id: ^rid, feed: ^feed, args: []) = :kvs.drop(KVS.reader(r, args: 10, dir: 0))
87 |
88 | kvs[:ids] |> Enum.map(&msg(id: &1))
89 | |> Enum.each(&assert(
90 | KVS.reader(id: rid, feed: ^feed, args: [], cache: &1) =
91 | :kvs.save(:kvs.drop(KVS.reader(:kvs.load_reader(rid), args: 1, dir: 0)))))
92 |
93 | assert r2 = KVS.reader(id: ^rid, feed: ^feed, args: [], cache: c1) = :kvs.drop(KVS.reader(r, args: 1, dir: 0))
94 | assert {:msg, Enum.at(kvs[:ids], 1), feed} == c1
95 |
96 | assert KVS.reader(id: ^rid, feed: ^feed, args: [], cache: c2) = :kvs.drop(KVS.reader(r2, args: 5, dir: 0))
97 | assert {:msg, Enum.at(kvs[:ids], 6), feed} == c2
98 | assert KVS.reader(id: ^rid, feed: ^feed, args: []) = :kvs.drop(KVS.reader(r1, args: 100))
99 | end
100 |
101 | test "feed",kvs do
102 | docs = :kvs.all("/crm/personal/Реєстратор А1/in/doc")
103 | ducks = :kvs.all("/crm/personal/Реєстратор А1/in/directory/duck")
104 | mail = :kvs.all("/crm/personal/Реєстратор А1/in/mail")
105 | total = :kvs.all("/crm/personal/Реєстратор А1/in")
106 |
107 | assert ducks++docs++mail == total
108 |
109 | assert docs = :kvs.feed("/crm/personal/Реєстратор А1/in/doc")
110 | assert ducks = :kvs.feed("/crm/personal/Реєстратор А1/in/directory/duck")
111 | assert mail = :kvs.feed("/crm/personal/Реєстратор А1/in/mail")
112 | assert total = :kvs.feed("/crm/personal/Реєстратор А1/in")
113 |
114 | assert :kvs.feed("/crm/personal/Реєстратор А1/in/directory/duck")
115 | ++ :kvs.feed("/crm/personal/Реєстратор А1/in/doc")
116 | ++ :kvs.feed("/crm/personal/Реєстратор А1/in/mail") == :kvs.feed("/crm/personal/Реєстратор А1/in")
117 | end
118 |
119 | defp log(x), do: IO.puts '#{inspect(x)}'
120 | defp log(m, x), do: IO.puts '#{m} #{inspect(x)}'
121 |
122 | end
123 |
--------------------------------------------------------------------------------
/test/test_helper.exs:
--------------------------------------------------------------------------------
1 | ExUnit.start()
2 |
--------------------------------------------------------------------------------