├── .formatter.exs ├── .gitattributes ├── .github └── workflows │ └── elixir.yml ├── .gitignore ├── CNAME ├── LICENSE ├── README.md ├── config └── config.exs ├── etc ├── fs.config ├── mnesia.config ├── rocks-lists.config └── rocks.config ├── include ├── api.hrl ├── backend.hrl ├── cursors.hrl ├── kvs.hrl ├── metainfo.hrl └── stream.hrl ├── index.html ├── lib └── KVS.ex ├── man ├── kvs.htm ├── kvs_fs.htm ├── kvs_mnesia.htm ├── kvs_rocks.htm ├── kvs_st.htm └── kvs_stream.htm ├── mix.exs ├── rebar.config ├── src ├── kvs.app.src ├── kvs.erl ├── layers │ ├── kvs_st.erl │ └── kvs_stream.erl └── stores │ ├── kvs_fs.erl │ ├── kvs_mnesia.erl │ └── kvs_rocks.erl ├── sys.config └── test ├── fd_test.exs ├── sc_test.exs ├── st_test.exs └── test_helper.exs /.formatter.exs: -------------------------------------------------------------------------------- 1 | # Used by "mix format" 2 | [ 3 | inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}"], 4 | line_length: 100 5 | ] 6 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | *.html linguist-detectable=false 2 | *.htm linguist-detectable=false 3 | *.js linguist-detectable=false 4 | *.css linguist-detectable=false 5 | *.sh linguist-detectable=false 6 | -------------------------------------------------------------------------------- /.github/workflows/elixir.yml: -------------------------------------------------------------------------------- 1 | name: mix 2 | on: push 3 | jobs: 4 | build: 5 | runs-on: ubuntu-latest 6 | steps: 7 | - uses: actions/checkout@v2 8 | - uses: erlef/setup-elixir@v1 9 | with: 10 | otp-version: 24.x 11 | elixir-version: 1.13.x 12 | - name: Dependencies 13 | run: | 14 | mix local.rebar --force 15 | mix local.hex --force 16 | mix deps.get 17 | - name: Compilation 18 | run: mix compile 19 | 20 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ebin 2 | *.dets 3 | *.gz 4 | *.dump 5 | .applist 6 | rocksdb/ 7 | deps/ 8 | _build/ 9 | *.lock 10 | doc/ 11 | -------------------------------------------------------------------------------- /CNAME: -------------------------------------------------------------------------------- 1 | kvs.n2o.dev 2 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | DHARMA License 2 | 3 | Copyright (c) 2015—2022 Synrc Research 4 | 5 | Permission to use, copy, modify, and/or distribute this software for any 6 | purpose with or without fee is hereby granted, provided that the above 7 | copyright notice and this permission notice appear in all copies. 8 | 9 | YOU CANNOT USE THIS SOFTWARE IN ANY (PROVABLE BY MONEY TRACE) 10 | PROCESS CHAIN OF EXTERMINATING UKRAINIANS BY ANY MEANS OF FASCIST 11 | ACTIONS AGAINST OUR TERRITORIAL INTEGRITY, CULTURAL DIVERSITY BY 12 | APPLYING MILITARY INVASIONS, ECONOMICAL WARS, HUMANITARIAN DISASTERS, 13 | ARTFICIAL HOLODOMORS, GENOCIDE, RAPING, LOOTING, ROBBERIES, SPREADING 14 | FAKE INFORMATION, AND OTHER CONTEMPORARY WEAPONS OF WAR AT SCALE 15 | OR IN INVIDIVUAL MANNER. 16 | 17 | YOU CANNOT USE THIS SOFTWARE BY ANY MEANS IN INTEREST OF LEGAL 18 | ENTITIES OR INDIVIDUALS WHO IS SUPPORTING NOW OR WAS SUPPORTING 19 | BACK THEN FASCISM, RUSCISM, COMMUNISM, CHAUVINISM, HUMILIATION, 20 | AND OTHER SUPPRESSIVE IDEOLOGIES IN DIFFERENT EXPRESSIONS. 21 | 22 | STOP KILLING UKRAINIANS, 23 | THE COUNTER RENDERS TENS OF MILLIONS. 24 | 25 | THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 26 | WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 27 | MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 28 | ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 29 | WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 30 | ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 31 | OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 32 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | KVS: Abstract Chain Database 2 | ============================ 3 | 4 | [![Actions Status](https://github.com/synrc/kvs/workflows/mix/badge.svg)](https://github.com/synrc/kvs/actions) 5 | [![Hex pm](http://img.shields.io/hexpm/v/kvs.svg?style=flat)](https://hex.pm/packages/kvs) 6 | 7 | Features 8 | -------- 9 | 10 | * Polymorphic Tuples aka Extensible Records 11 | * Basic Schema for Storing Chains 12 | * Backends: MNESIA, FS, ROCKSDB 13 | * Extremely Compact: 500 LOC 14 | 15 | Usage 16 | ----- 17 | 18 | ``` 19 | $ git clone https://github.com/synrc/kvs && cd kvs 20 | $ open man/kvs.htm 21 | $ mad com pla rep 22 | > kvs:join(). 23 | ``` 24 | 25 | Release Notes 26 | ------------- 27 | 28 | [1]. 2018-11-13 Новая версия KVS 5.11
29 | [2]. 2019-04-13 Новая версия KVS 6.4 30 | 31 | Credits 32 | ------- 33 | 34 | * Maxim Sokhatsky 35 | * Andrii Zadorozhnii 36 | * Ivan Kulyk 37 | -------------------------------------------------------------------------------- /config/config.exs: -------------------------------------------------------------------------------- 1 | import Config 2 | 3 | config :kvs, 4 | dba: :kvs_rocks, 5 | dba_st: :kvs_st, 6 | dba_seq: :kvs_rocks, 7 | seq_pad: [], 8 | schema: [:kvs, :kvs_stream] 9 | -------------------------------------------------------------------------------- /etc/fs.config: -------------------------------------------------------------------------------- 1 | [ {kvx,[{dba,kvx_fs}, 2 | {dba_st,kvx_stream}, 3 | {schema,[kvx,kvx_stream]}]} 4 | ]. 5 | -------------------------------------------------------------------------------- /etc/mnesia.config: -------------------------------------------------------------------------------- 1 | [ {kvx,[{dba,kvx_mnesia}, 2 | {dba_st,kvx_stream}, 3 | {schema,[kvx,kvx_stream]}]} 4 | ]. 5 | -------------------------------------------------------------------------------- /etc/rocks-lists.config: -------------------------------------------------------------------------------- 1 | [ {kvx,[{dba,kvx_rocks}, 2 | {dba_st,kvx_stream}, 3 | {schema,[kvx,kvx_stream]}]} 4 | ]. 5 | -------------------------------------------------------------------------------- /etc/rocks.config: -------------------------------------------------------------------------------- 1 | [ {kvx,[{dba,kvx_rocks}, 2 | {dba_st,kvx_st}, 3 | {schema,[kvx,kvx_stream]}]} 4 | ]. 5 | -------------------------------------------------------------------------------- /include/api.hrl: -------------------------------------------------------------------------------- 1 | -ifndef(API_HRL). 2 | -define(API_HRL, true). 3 | -define(API,[start/0,stop/0,leave/0,leave/1,destroy/0,destroy/1, 4 | join/0,join/1,modules/0,cursors/0,get/2,get/3,put/1,put/2,index/3,index/4, 5 | match/1,match/2,index_match/2,index_match/3,key_match/2,key_match/3, 6 | delete/2,delete/3,delete_range/3, 7 | table/1,tables/0,dir/0,initialize/2,seq/2,all/1,all/2,count/1,ver/0]). 8 | -include("metainfo.hrl"). 9 | -spec seq(atom() | [], integer() | []) -> term(). 10 | -spec count(atom()) -> integer(). 11 | -spec dir() -> list({'table',atom()}). 12 | -spec ver() -> {'version',string()}. 13 | -spec leave() -> ok. 14 | -spec destroy() -> ok. 15 | -spec join() -> ok | {error,any()}. 16 | -spec join(Node :: string()) -> [{atom(),any()}]. 17 | -spec modules() -> list(atom()). 18 | -spec cursors() -> list({atom(),list(atom())}). 19 | -spec tables() -> list(#table{}). 20 | -spec table(Tab :: atom()) -> #table{} | false. 21 | -endif. 22 | -------------------------------------------------------------------------------- /include/backend.hrl: -------------------------------------------------------------------------------- 1 | -ifndef(BACKEND_HRL). 2 | -define(BACKEND_HRL, true). 3 | -define(BACKEND, [db/0,get/3,put/1,put/2,delete/3,delete_range/3,index/3,dump/0,start/0,stop/0,destroy/0,destroy/1,keys/2, 4 | join/2,leave/0,leave/1,dir/0,create_table/2,add_table_index/2,seq/2,all/2,count/1,version/0, 5 | match/1,key_match/3,index_match/2]). 6 | -compile({no_auto_import,[get/1,put/2]}). 7 | -include("kvs.hrl"). 8 | -spec put(tuple() | list(tuple())) -> ok | {error,any()}. 9 | -spec put(tuple() | list(tuple()), #kvs{}) -> ok | {error,any()}. 10 | -spec get(term() | any(), any(), #kvs{}) -> {ok,any()} | {error,not_found}. 11 | -spec delete(term(), any(), #kvs{}) -> ok | {error,not_found}. 12 | -spec delete_range(term(), any(), #kvs{}) -> ok | {error,not_found}. 13 | -spec dump() -> ok. 14 | -spec start() -> ok. 15 | -spec stop() -> ok. 16 | -spec index(term(), any(), any()) -> list(tuple()). 17 | -endif. 18 | -------------------------------------------------------------------------------- /include/cursors.hrl: -------------------------------------------------------------------------------- 1 | -ifndef(CURSORS_HRL). 2 | -define(CURSORS_HRL, true). 3 | 4 | -record(writer, { id = [] :: term(), 5 | count = 0 :: integer(), 6 | cache = [] :: [] | {term(),term()} | {term(),term(),term()}, 7 | args = [] :: [] | term(), 8 | first = [] :: [] | tuple() } ). 9 | 10 | -record(reader, { id = [] :: [] | integer(), 11 | pos = 0 :: integer() | atom(), 12 | cache = [] :: [] | {term(),term()} | {term(),term(),term()}, 13 | args = [] :: [] | integer() | term(), 14 | feed = [] :: term(), 15 | seek = [] :: term(), 16 | count = 0 :: integer(), 17 | dir = 0 :: 0 | 1 } ). 18 | 19 | -endif. 20 | -------------------------------------------------------------------------------- /include/kvs.hrl: -------------------------------------------------------------------------------- 1 | -ifndef(KVS_HRL). 2 | -define(KVS_HRL, true). 3 | -record(id_seq, { thing = []::term(), id = 0 :: integer() } ). 4 | -record(it, { id = []::[] | integer() } ). 5 | -record(ite, { id = []::[] | integer(), next = []::[] | integer() } ). 6 | -record(iter, { id = []::[] | integer(), next = []::[] | integer(), prev = []::[] | integer() } ). 7 | -record(kvs, { mod = kvs_mnesia :: kvs_mnesia | kvs_rocks | kvs_fs, 8 | st = kvs_stream :: kvs_stream | kvs_st, 9 | db = []::list(), 10 | cx = []::term() }). 11 | -endif. 12 | -------------------------------------------------------------------------------- /include/metainfo.hrl: -------------------------------------------------------------------------------- 1 | -ifndef(METAINFO_HRL). 2 | -define(METAINFO_HRL, true). 3 | -record(schema, {name,tables=[]}). 4 | -record(table, {name,container=false,type=set,fields=[],keys=[], 5 | copy_type=application:get_env(kvx,mnesia_media,disc_copies), instance={}, mappings =[] }). 6 | -endif. 7 | -------------------------------------------------------------------------------- /include/stream.hrl: -------------------------------------------------------------------------------- 1 | -ifndef(STREAM_HRL). 2 | -define(STREAM_HRL, true). 3 | -include("kvs.hrl"). 4 | -include("cursors.hrl"). 5 | -define(STREAM, [top/1, top/2, bot/1, bot/2, next/1, next/2, prev/1, prev/2, drop/1, drop/2, take/1, take/2, append/2, append/3, feed/1, feed/2, 6 | load_reader/1, load_reader/2, writer/1, writer/2, reader/1, reader/2, save/1, save/2, add/1, add/2, remove/2, remove/3, cut/1, cut/2]). 7 | 8 | -spec top(#reader{}) -> #reader{}. 9 | -spec bot(#reader{}) -> #reader{}. 10 | -spec next(#reader{}) -> #reader{}. 11 | -spec prev(#reader{}) -> #reader{}. 12 | -spec drop(#reader{}) -> #reader{}. 13 | -spec take(#reader{}) -> #reader{}. 14 | -spec feed (term()) -> list(). 15 | -spec load_reader (term()) -> #reader{}. 16 | -spec writer (term()) -> #writer{}. 17 | -spec reader (term()) -> #reader{}. 18 | -spec save (#reader{} | #writer{}) -> #reader{} | #writer{}. 19 | -spec add(#writer{}) -> #writer{}. 20 | -spec append(tuple(),term()) -> any(). 21 | -spec remove(tuple(),term()) -> integer(). 22 | -spec cut(term(), list()) -> ok. 23 | -endif. 24 | -------------------------------------------------------------------------------- /index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | KVS 6 | 7 | 8 |
19 | 20 |

KVS

21 |
61 |
62 | 63 |

SESSION

64 | 65 |
66 | > kvs:join(). 67 | ok 68 | 69 | > kvs:check(). 70 | ok 71 | 72 | > kvs:all(reader). 73 | [{reader,1555175169121817000,0,[],[], 74 | {list,1555175169120161000}, 75 | 0}, 76 | {reader,1555175169121249000,0,[],[], 77 | {list,1555175169120161000}, 78 | 0}] 79 | 80 | > rr(kvs). 81 | [emails,id_seq,iter,kvs,reader,schema,table,writer] 82 | 83 | > kvs:save(kvs:reader({list,1555175169120161000})). 84 | #reader{id = 1555175244188986000,pos = 0, 85 | cache = {emails,1555175169122304000}, 86 | args = [], 87 | feed = {list,1555175169120161000}, 88 | dir = 0} 89 | 90 | > kvs:take(kvs:bot((kvs:load_reader(1555175244188986000))#reader{args=-1})). 91 | #reader{id = 1555175244188986000,pos = 5, 92 | cache = {emails,1555175169127279000}, 93 | args = [#emails{id = 1555175169127279000,next = [], 94 | prev = 1555175169126314000,email = []}, 95 | #emails{id = 1555175169126314000,next = 1555175169127279000, 96 | prev = 1555175169125227000,email = []}, 97 | #emails{id = 1555175169125227000,next = 1555175169126314000, 98 | prev = 1555175169123405000,email = []}, 99 | #emails{id = 1555175169123405000,next = 1555175169125227000, 100 | prev = 1555175169122304000,email = []}, 101 | #emails{id = 1555175169122304000,next = 1555175169123405000, 102 | prev = [],email = []}], 103 | feed = {list,1555175169120161000}, 104 | dir = 0} 105 | 106 |
107 |
108 | 109 |
110 |

CONTRIBUTORS

111 |
112 |
113 |

114 |
115 |
118 | 119 | 120 | -------------------------------------------------------------------------------- /lib/KVS.ex: -------------------------------------------------------------------------------- 1 | defmodule KVS do 2 | require Record 3 | 4 | Enum.each(Record.extract_all(from_lib: "kvs/include/cursors.hrl"), fn {name, definition} -> 5 | Record.defrecord(name, definition) 6 | end) 7 | 8 | Enum.each(Record.extract_all(from_lib: "kvs/include/metainfo.hrl"), fn {name, definition} -> 9 | Record.defrecord(name, definition) 10 | end) 11 | 12 | Enum.each(Record.extract_all(from_lib: "kvs/include/kvs.hrl"), fn {name, definition} -> 13 | Record.defrecord(name, definition) 14 | end) 15 | 16 | defmacro __using__(opts \\ []) do 17 | imports = 18 | opts 19 | |> Macro.expand(__CALLER__) 20 | |> Keyword.get(:with, [:kvs]) 21 | 22 | Enum.map(imports, fn mod -> 23 | if Code.ensure_compiled(mod) do 24 | upcased = Module.concat([String.upcase(to_string(mod))]) 25 | 26 | quote do 27 | import unquote(upcased) 28 | alias unquote(mod), as: unquote(upcased) 29 | end 30 | else 31 | IO.warn("🚨 Unknown module #{mod} was requested to be used by :kvs. Skipping.") 32 | end 33 | end) 34 | end 35 | end 36 | -------------------------------------------------------------------------------- /man/kvs.htm: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | KVS 6 | 7 | 8 |
20 | 21 |

KVS

22 |
23 |
24 |
25 |
26 | 27 |

INTRO

28 | 29 |

KVS module provides user level interface for console commands. 30 | It has discovery, data manipulation and retrival features. Under the hood 31 | it handles configurable run-time backends for each supported database.

32 | 33 |
34 |
  • put/1 — put record using id as a key.
  • 35 |
  • get/2 — get record by key from table.
  • 36 |
  • delete/1 — delete record from table.
  • 37 |
  • index/3 — search records by field and its value.
  • 38 |
  • seq/2 — generate new sequence table id.
  • 39 |
  • count/1 — counts records in table.
  • 40 |
  • dir/0 — table list.
  • 41 |
42 | 43 |

You can change backend by setting application env. 44 | This behaves well even under the heavy load.

45 | 46 |
47 |
48 |

API

49 |

Data operations.

50 |

put(tuple()) -> ok | {error,any()}.

51 |

Stores the record.

52 |

get(atom(),any()) -> {ok,any()} | {error, not_found | duplicated }.

53 |

Retrieves the record.

54 |

delete(atom(),any()) -> ok | {error,any()}.

55 |

Deletes the data record.

56 |

index(atom(),any(),any()) -> list(tuple()).

57 |

Searches the record by an indexed field and a given value.

58 |
59 |
60 |

SEQ

61 |

Sequence table id_seq stores the counter per thing. 62 | The counters are global and atomic in each supported database. 63 | Sequences are used to generate unique names for records per distributed table. 64 | If names in the table are not unique, e.g. 65 | then count function may return a different value than the current sequence.

66 |
-record(id_seq, { thing = atom(), 67 | id = 0 :: integer() } ).
68 | 69 |

seq(atom(), integer()) -> integer().

70 |

Increments and returns id counter for the particular table.

71 |

count(atom()) -> integer().

72 |

Returns number of records in table.

73 |
74 |
75 |

SETUP

76 |

In sys.config you can specify main kvs backend module as dba parameter 77 | and list of modules containing metainfo/0 exported function. For the 78 | stream operations you can specify the stream kvs backend module dba_st parameter.

79 |
[{kvs, [{dba,store_mnesia}, 80 | {dba_st,store_stream}, 81 | {schema,[kvs,kvs_stream]}]}].
82 |

dir() -> list({'table',atom()}).

83 |

Returns actual tables.

84 |
85 |
86 | 87 |

This module may refer to: 88 | kvs_fs, 89 | kvs_mnesia, 90 | kvs_rocks, 91 | kvs_st, 92 | kvs_stream. 93 |

94 |
95 |
96 |
97 | 98 | 99 | 100 | 101 | -------------------------------------------------------------------------------- /man/kvs_fs.htm: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | FS 6 | 7 | 8 |
20 | 21 |

FS

22 |
23 |
24 |
25 |
26 |

INTRO

27 |

FS is a filesystem backend implementation for KVS. 28 | Put the {dba,kvs_fs} property for the kvs application in your sys.config.

29 |
30 | 31 |

EXAMPLES

32 | 33 |
All examples can be executed in Erlang REPL. Only 34 | requirement is to have kvs set in dependencies and kvs application 35 | started.
36 | 37 |

Create a new kvs_fs table called test. When executed, this 38 | function will create a new directory called test under data 39 | directory in the current workspace.

40 | 41 |
42 | kvs_fs:create_table(test, []). 43 | % will return: 44 | % ok 45 |
46 | 47 |

Put a new key/value in table test. When executed, a new file is 48 | created under test directory. Its name is based on the sha1 of 49 | the key encoded in base64.

50 | 51 |
52 | Table = test. 53 | Key = key. 54 | Value = <<"my_value">>. 55 | kvs_fs:put({test, Key, Value}). 56 | % will return: 57 | % ok 58 |
59 | 60 |

Get a key from value test.

61 | 62 |
63 | kvs_fs:get(test, key). 64 | % will return: 65 | % {ok, {test, key, <<"my_value">>}} 66 |
67 | 68 | 69 |

Delete a key

70 | 71 |
72 |
73 |

This module may refer to: 74 | kvs. 75 |

76 | 77 |
78 |
79 |
82 | 83 | 84 | -------------------------------------------------------------------------------- /man/kvs_mnesia.htm: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | MNESIA 7 | 8 | 9 |
21 | 22 |

MNESIA

23 |
24 |
25 |
26 |

INTRO

27 |

MNESIA is a mnesia backend imlpementation for KVS. 28 | Put the {dba,kvs_mnesia} property for the kvs application in your sys.config.

29 |
30 |
31 |
32 |

This module may refer to: 33 | kvs. 34 |

35 | 36 |
37 |
38 |
41 | 42 | 43 | -------------------------------------------------------------------------------- /man/kvs_rocks.htm: -------------------------------------------------------------------------------- 1 | 2 | 3 | ROCKS 4 | 5 | 6 |
18 | 19 |

ROCKS

20 |
21 |
22 |
23 |
24 |

INTRO

25 |

ROCKS is the RocksDB backend implementation for KVS. 26 | Put the {dba,kvs_rocks} and (optionally) {dba_st,kvs_st} properties 27 | for the kvs application in your sys.config.

28 |
29 |
30 |
31 |

This module may refer to: 32 | kvs_st, 33 | kvs_stream, 34 | kvs. 35 |

36 |
37 |
38 |
39 | 40 | 41 | 42 | 43 | -------------------------------------------------------------------------------- /man/kvs_st.htm: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | ST 10 | 11 | 12 | 13 | 14 | 26 |
27 | 28 | 29 |

ST

30 |
31 |
32 |
33 |
34 |

INTRO

35 |

The ST module provides STREAM interface for ROCKS backend. 36 | Set the {dba,kvs_rocks} along with {dba_st,kvs_st} in 37 | order to use ST module for stream operations.

38 |
39 |
40 |

This module may refer to: 41 | kvs_stream, 42 | kvs_rocks, 43 | kvs. 44 |

45 |
46 |
47 |
48 | 49 | 50 | 51 | 52 | -------------------------------------------------------------------------------- /man/kvs_stream.htm: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | STREAM 8 | 9 | 10 |
22 | 23 |

STREAM

24 |
25 |
26 |
27 |
28 |

INTRO

29 |

The STREAM module provides chain persistence, accumulation and traversal.

30 |
31 |
  • writer/1 — creates writer cursor for chain.
  • 32 |
  • reader/1 — creates reader cursor for chain.
  • 33 |
  • save/1 — stores cursor to db.
  • 34 |
  • load_reader/1 — loads reader cursor.
  • 35 |
  • top/1 — returns top of the reader.
  • 36 |
  • bot/1 — returns bottom of the reader.
  • 37 |
  • next/1 — moves reader next.
  • 38 |
  • prev/1 — moves reader prev.
  • 39 |
  • take/1 — takes N elements from reader.
  • 40 |
  • drop/1 — skips N elements from reader.
  • 41 |
  • add/1 — adds element to writer.
  • 42 |
  • append/2 — appends element to feed.
  • 43 |
  • remove/2 — removes element from feed.
  • 44 |
  • cut/2 — cleans-up the feed up to the key.
  • 45 |
46 | 47 |

You can grab kvs_stream 48 | and use it in your applications without importing synrc/kvs dependency, 49 | as this module is self-containing. 50 | The possible applications are: public and private feeds, FIFO queues, 51 | unread messages, chat applications, blockchain etc.

52 |
53 |
54 |

WRITER

55 | 56 |

Writer cursor represents unique append list chain with some cached values. 57 | E.g., elements count, first element of the chain, cache field for value of 58 | a previous written message and args field for passing arguments in fixpoint style.

59 | 60 |
-record(writer, { id = [] :: term(), 61 | count = 0 :: integer(), 62 | cache = [] :: [] | tuple(), 63 | args = [] :: term(), 64 | first = [] :: [] | tuple() } ).
65 | 66 |

For adding data to storage you need to create writer cursor, 67 | set the args field with the record to be added:

68 | 69 |
> require KVS 70 | KVS 71 | > KVS.writer 72 | {:writer, [], 0, [], [], []}
73 | 74 |

writer(list()) -> #writer{}.

75 |

Creates writer cursor. After this you normally call save function for cursor persistence. 76 | As a parameter provide cursor name as a list.

77 | 78 |
> w = :kvs.writer '/cursors/cur1' 79 | {:writer, '/cursors/cur1', 0, [], [], []}
80 | 81 |

save(#writer{}) -> #writer{}.

82 |

Flushes writer cursor to database.

83 | 84 |
> :kvs.save w 85 | {:writer, '/cursors/cur1', 0, [], [], []} 86 | > :kvs.get :writer, '/cursors/cur1' 87 | {:ok, {:writer, '/cursors/cur1', 0, [], [], []}}
88 | 89 |

add(#writer{}) -> #writer{}.

90 |

Adds element to persistent chain declared by writer cursor. 91 | Adding elements to chain affects count field.

92 | 93 |
> :lists.map fn _ -> 94 | {:ok,w} = :kvs.get(:writer,'/cursors/cur1') 95 | :kvs.save(:kvs.add(KVS.writer(w, args: {:"$msg", [], [], [], [], []}))) 96 | end, :lists.seq(1, 5) 97 | [ 98 | {:writer, '/cursors/cur1', 1, {:"$msg", '1313544188019000'}, [], []}, 99 | {:writer, '/cursors/cur1', 2, {:"$msg", '1313544189127000'}, [], []}, 100 | {:writer, '/cursors/cur1', 3, {:"$msg", '1313544189869000'}, [], []}, 101 | {:writer, '/cursors/cur1', 4, {:"$msg", '1313544190519000'}, [], []}, 102 | {:writer, '/cursors/cur1', 5, {:"$msg", '1313544191134000'}, [], []} 103 | ] 104 | > :kvs.get :writer, '/cursors/cur1' 105 | {:ok, 106 | {:writer, '/cursors/cur1', 5, {:"$msg", '1314009332950000'}, [], []}}
107 | 108 |
109 |
110 | 111 |

READER

112 |

Reader cursor represents a user reading pointer to the writer feed. 113 | Field #reader.feed is linked to #writer.id. 114 | The pos field represents current position in feed. 115 | Field dir is used for changing take and drop directions.

116 |
Reader Cursor
117 | -record(reader, { id = [] :: integer(), 118 | pos = 0 :: [] | integer(), 119 | cache = [] :: [] | integer(), 120 | args = [] :: term(), 121 | feed = [] :: term(), 122 | dir = 0 :: 0 | 1 } ).
123 | 124 |

reader(term()) -> #reader{}.

125 |

Creates reader cursor for a given writer id.

126 | 127 |

load_reader(term()) -> #reader{}.

128 |

Loads reader cursor from database.

129 | 130 |

save(#reader{}) -> #reader{}.

131 |

Flushes reader cursor to database.

132 | 133 |

bot(#reader{}) -> #reader{dir = 0}.

134 |

Changes the direction of the cursor to take foraward (normal).

135 | 136 |

top(#reader{}) -> #reader{dir = 1}.

137 |

Changes the direction of the cursor to take backwards.

138 |
139 |
140 |

ITER

141 |
KVS Stream Iterator
142 | -record(iter, { id = [] :: [] | integer(), 143 | next = [] :: [] | integer(), 144 | prev = [] :: [] | integer() } ).
145 | 146 |

next(#reader{}) -> #reader{}.

147 |

Moves cursor to next. Consume data down from top. 148 | Return error if list is empty, otherwise next element or last.

149 | 150 |

prev(#reader{}) -> #reader{}.

151 |

Moves cursor to prev. Consume data up from bottom. 152 | Return error if list is empty, otherwise next element or last.

153 | 154 |

drop(#reader{}) -> #reader{}.

155 |

Drops N elements starting from reader.

156 | 157 |

take(#reader{}) -> #reader{}.

158 |

Trying to consume N records from stream using its current value and direction. 159 | Returns consumed data. Usually you seek to some position and then consume some data.

160 | 161 |

append(tuple(), list()) -> term().

162 |

Adds record to feed.

163 | 164 |

remove(tuple(), list()) -> term().

165 |

Removes record from feed.

166 | 167 |

cut(list(), term()) -> #ok{} | #error{}.

168 |

Cuts the feed up to a given key.

169 | 170 |
171 |
172 |

This module may refer to: 173 | kvs, 174 | kvs_st. 175 |

176 |
177 |
178 |
179 | 180 | 181 | 182 | 183 | -------------------------------------------------------------------------------- /mix.exs: -------------------------------------------------------------------------------- 1 | defmodule KVS.Mixfile do 2 | use Mix.Project 3 | 4 | def project do 5 | [ 6 | app: :kvs, 7 | version: "11.9.1", 8 | description: "KVS Abstract Chain Database", 9 | package: package(), 10 | deps: deps() 11 | ] 12 | end 13 | 14 | def application do 15 | [mod: {:kvs, []}, extra_applications: [:mnesia,:ex_doc]] 16 | end 17 | 18 | defp package do 19 | [ 20 | files: ~w(include man config lib src LICENSE mix.exs README.md), 21 | licenses: ["MIT"], 22 | links: %{"GitHub" => "https://github.com/synrc/kvs"} 23 | ] 24 | end 25 | 26 | defp deps do 27 | [ 28 | {:ex_doc, ">= 0.0.0", only: :dev} 29 | # {:rocksdb, "~> 1.8.0", only: :test} 30 | ] 31 | end 32 | end 33 | -------------------------------------------------------------------------------- /rebar.config: -------------------------------------------------------------------------------- 1 | {erl_opts, [nowarn_unused_function,nowarn_duplicated_export]}. 2 | {deps, []}. 3 | {project_plugins, [rebar3_format]}. 4 | {format, [ 5 | {files, ["src/*.erl", "test/*.erl"]}, 6 | {formatter, otp_formatter}, 7 | {options, #{ line_length => 108, 8 | paper => 250, 9 | spaces_around_fields => false, 10 | inlining => all, 11 | inline_clause_bodies => true, 12 | inline_expressions => true, 13 | inline_qualified_function_composition => true, 14 | inline_simple_funs => true, 15 | inline_items => all, 16 | inline_fields => true, 17 | inline_attributes => true 18 | }}]}. 19 | -------------------------------------------------------------------------------- /src/kvs.app.src: -------------------------------------------------------------------------------- 1 | {application, kvs, 2 | [{description, "KVS Abstract Chain Database"}, 3 | {vsn, "9.4.1"}, 4 | {registered, []}, 5 | {applications, [kernel,stdlib]}, 6 | {mod, { kvs, []}}, 7 | {env, []} ]}. 8 | -------------------------------------------------------------------------------- /src/kvs.erl: -------------------------------------------------------------------------------- 1 | -module(kvs). 2 | 3 | -behaviour(application). 4 | 5 | -behaviour(supervisor). 6 | 7 | -include_lib("stdlib/include/assert.hrl"). 8 | 9 | -include("api.hrl"). 10 | 11 | -include("metainfo.hrl"). 12 | 13 | -include("stream.hrl"). 14 | 15 | -include("cursors.hrl"). 16 | 17 | -include("kvs.hrl"). 18 | 19 | -include("backend.hrl"). 20 | 21 | -export([dump/0, 22 | db/0, 23 | metainfo/0, 24 | ensure/1, 25 | ensure/2, 26 | seq_gen/0, 27 | keys/1, 28 | fields/1, 29 | defined/2, 30 | field/2, 31 | setfield/3, 32 | remove/1]). 33 | 34 | -export([join/2, seq/3]). 35 | 36 | -export(?API) 37 | 38 | . 39 | 40 | -export(?STREAM) 41 | 42 | . 43 | 44 | -export([init/1, start/2, stop/1]). 45 | 46 | -record('$msg', {id, next, prev, user, msg}). 47 | 48 | init([]) -> {ok, {{one_for_one, 5, 10}, []}}. 49 | 50 | start(_, _) -> 51 | supervisor:start_link({local, kvs}, kvs, []). 52 | 53 | stop(_) -> ok. 54 | 55 | dba() -> application:get_env(kvs, dba, kvs_mnesia). 56 | 57 | seq_dba() -> application:get_env(kvs, dba_seq, kvs_mnesia). 58 | 59 | db() -> (dba()):db(). 60 | 61 | kvs_stream() -> 62 | application:get_env(kvs, dba_st, kvs_stream). 63 | 64 | % kvs api 65 | 66 | all(Table) -> all(Table, #kvs{mod = dba(), db = db()}). 67 | 68 | delete(Table, Key) -> 69 | delete(Table, Key, #kvs{mod = dba(), db = db()}). 70 | 71 | get(Table, Key) -> 72 | (?MODULE):get(Table, Key, #kvs{mod = dba(), db = db()}). 73 | 74 | index(Table, K, V) -> 75 | index(Table, K, V, #kvs{mod = dba()}). 76 | 77 | keys(Feed) -> 78 | keys(Feed, #kvs{mod = dba(), db = db()}). 79 | 80 | key_match(Feed, Id) -> 81 | key_match(Feed, Id, #kvs{mod = dba(), db=db()}). 82 | 83 | match(Record) -> 84 | match(Record, #kvs{mod = dba()}). 85 | 86 | index_match(Record, Index) -> 87 | index_match(Record, Index, #kvs{mod = dba()}). 88 | 89 | join() -> join([], #kvs{mod = dba(), db = db()}). 90 | 91 | dump() -> dump(#kvs{mod = dba()}). 92 | 93 | join(Node) -> join(Node, #kvs{mod = dba(), db = db()}). 94 | 95 | leave() -> leave(#kvs{mod = dba(), db = db()}). 96 | 97 | destroy() -> destroy(#kvs{mod = dba(), db = db()}). 98 | 99 | count(Table) -> count(Table, #kvs{mod = dba()}). 100 | 101 | put(Record) -> (?MODULE):put(Record, #kvs{mod = dba(), db = db()}). 102 | 103 | stop() -> stop_kvs(#kvs{mod = dba()}). 104 | 105 | start() -> start(#kvs{mod = dba()}). 106 | 107 | ver() -> ver(#kvs{mod = dba()}). 108 | 109 | dir() -> dir(#kvs{mod = dba()}). 110 | 111 | feed(Key) -> 112 | feed(Key, #kvs{mod = dba(), st = kvs_stream(), db = db()}). 113 | 114 | seq([], DX) -> seq([], DX, #kvs{mod = kvs_rocks}); 115 | seq(Table, DX) -> seq(Table, DX, #kvs{mod = seq_dba()}). 116 | 117 | remove(Rec, Feed) -> 118 | remove(Rec, Feed, #kvs{mod = dba(), st = kvs_stream(), db = db()}). 119 | 120 | put(Records, #kvs{mod = Mod, db = Db}) when is_list(Records) -> 121 | Mod:put(Records, Db); 122 | put(Record, #kvs{mod = Mod, db = Db}) -> Mod:put(Record, Db). 123 | 124 | get(RecordName, Key, #kvs{mod = Mod, db = Db}) -> 125 | Mod:get(RecordName, Key, Db). 126 | 127 | delete(Tab, Key, #kvs{mod = Mod, db = Db}) -> 128 | Mod:delete(Tab, Key, Db). 129 | 130 | delete_range(Feed, Last, #kvs{mod=DBA, db=Db}) -> 131 | DBA:delete_range(Feed,Last,Db). 132 | 133 | count(Tab, #kvs{mod = DBA}) -> DBA:count(Tab). 134 | 135 | index(Tab, Key, Value, #kvs{mod = DBA}) -> 136 | DBA:index(Tab, Key, Value). 137 | 138 | keys(Feed, #kvs{mod = DBA, db = Db}) -> 139 | DBA:keys(Feed, Db). 140 | 141 | key_match(Feed, Id, #kvs{mod = DBA, db = Db}) -> 142 | DBA:key_match(Feed, Id, Db). 143 | 144 | match(Record, #kvs{mod = DBA}) -> 145 | DBA:match(Record). 146 | 147 | index_match(Record, Index, #kvs{mod = DBA}) -> 148 | DBA:index_match(Record, Index). 149 | 150 | seq(Tab, Incr, #kvs{mod = DBA}) -> DBA:seq(Tab, Incr). 151 | 152 | dump(#kvs{mod = Mod}) -> Mod:dump(). 153 | 154 | feed(Tab, #kvs{st = Mod, db = Db}) -> Mod:feed(Tab, Db). 155 | 156 | remove(Rec, Feed, #kvs{st = Mod, db = Db}) -> 157 | Mod:remove(Rec, Feed, Db). 158 | 159 | all(Tab, #kvs{mod = DBA, db = Db}) -> DBA:all(Tab, Db). 160 | 161 | start(#kvs{mod = DBA}) -> DBA:start(). 162 | 163 | stop_kvs(#kvs{mod = DBA}) -> DBA:stop(). 164 | 165 | join(Node, #kvs{mod = DBA, db = Db}) -> DBA:join(Node, Db). 166 | 167 | leave(#kvs{mod = DBA, db = Db}) -> DBA:leave(Db). 168 | 169 | destroy(#kvs{mod = DBA, db = Db}) -> DBA:destroy(Db). 170 | 171 | ver(#kvs{mod = DBA}) -> DBA:version(). 172 | 173 | dir(#kvs{mod = DBA}) -> DBA:dir(). 174 | 175 | % stream api 176 | 177 | top(X) -> (kvs_stream()):top(X). 178 | 179 | top(X,#kvs{db = Db}) -> (kvs_stream()):top(X,Db). 180 | 181 | bot(X) -> (kvs_stream()):bot(X). 182 | 183 | bot(X,#kvs{db = Db}) -> (kvs_stream()):bot(X,Db). 184 | 185 | next(X) -> (kvs_stream()):next(X). 186 | 187 | next(X,#kvs{db = Db}) -> (kvs_stream()):next(X,Db). 188 | 189 | prev(X) -> (kvs_stream()):prev(X). 190 | 191 | prev(X,#kvs{db = Db}) -> (kvs_stream()):prev(X,Db). 192 | 193 | drop(X) -> (kvs_stream()):drop(X). 194 | 195 | drop(X,#kvs{db = Db}) -> (kvs_stream()):drop(X,Db). 196 | 197 | take(X) -> (kvs_stream()):take(X). 198 | 199 | take(X,#kvs{db = Db}) -> (kvs_stream()):take(X,Db). 200 | 201 | save(X) -> (kvs_stream()):save(X). 202 | 203 | save(X,#kvs{db = Db}) -> (kvs_stream()):save(X,Db). 204 | 205 | remove(X) -> (kvs_stream()):remove(X). 206 | 207 | cut(X) -> (kvs_stream()):cut(X). 208 | 209 | cut(X,#kvs{db = Db}) -> (kvs_stream()):cut(X, Db). 210 | 211 | add(X) -> (kvs_stream()):add(X). 212 | 213 | add(X,#kvs{db = Db}) -> (kvs_stream()):add(X,Db). 214 | 215 | append(X, Y) -> (kvs_stream()):append(X, Y). 216 | 217 | append(X, Y, #kvs{db = Db}) -> (kvs_stream()):append(X, Y, Db). 218 | 219 | load_reader(X) -> (kvs_stream()):load_reader(X). 220 | 221 | load_reader(X,#kvs{db = Db}) -> (kvs_stream()):load_reader(X,Db). 222 | 223 | writer(X) -> (kvs_stream()):writer(X). 224 | 225 | writer(X,#kvs{db = Db}) -> (kvs_stream()):writer(X,Db). 226 | 227 | reader(X) -> (kvs_stream()):reader(X). 228 | 229 | reader(X,#kvs{db = Db}) -> (kvs_stream()):reader(X,Db). 230 | 231 | % unrevisited 232 | 233 | ensure(#writer{} = X) -> ensure(X,#kvs{mod = dba(), db = db()}). 234 | ensure(#writer{id = Id},#kvs{} = X) -> 235 | case kvs:get(writer, Id, X) of 236 | {error, _} -> 237 | kvs:save(kvs:writer(Id, X)), 238 | ok; 239 | {ok, _} -> ok 240 | end. 241 | 242 | cursors() -> 243 | lists:flatten([[{T#table.name, T#table.fields} 244 | || #table{name = Name} = T 245 | <- (M:metainfo())#schema.tables, 246 | Name == reader orelse Name == writer] 247 | || M <- modules()]). 248 | 249 | % metainfo api 250 | 251 | tables() -> 252 | lists:flatten([(M:metainfo())#schema.tables 253 | || M <- modules()]). 254 | 255 | modules() -> application:get_env(kvs, schema, []). 256 | 257 | metainfo() -> 258 | #schema{name = kvs, tables = core() ++ test_tabs()}. 259 | 260 | core() -> 261 | [#table{name = id_seq, 262 | fields = record_info(fields, id_seq), keys = [thing]}]. 263 | 264 | test_tabs() -> 265 | [#table{name = '$msg', 266 | fields = record_info(fields, '$msg')}]. 267 | 268 | table(Name) when is_atom(Name) -> 269 | lists:keyfind(Name, #table.name, tables()); 270 | table(_) -> false. 271 | 272 | seq_gen() -> 273 | Init = fun (Key) -> 274 | case kvs:get(id_seq, Key) of 275 | {error, _} -> 276 | {Key, kvs:put(#id_seq{thing = Key, id = 0})}; 277 | {ok, _} -> {Key, skip} 278 | end 279 | end, 280 | [Init(atom_to_list(Name)) 281 | || {Name, _Fields} <- cursors()]. 282 | 283 | initialize(Backend, Module) -> 284 | [begin 285 | Backend:create_table(T#table.name, 286 | [{attributes, T#table.fields}, 287 | {T#table.copy_type, [node()]}, 288 | {type, T#table.type}]), 289 | [Backend:add_table_index(T#table.name, Key) 290 | || Key <- T#table.keys], 291 | T 292 | end 293 | || T <- (Module:metainfo())#schema.tables]. 294 | 295 | fields(Table) when is_atom(Table) -> 296 | case table(Table) of 297 | false -> []; 298 | T -> T#table.fields 299 | end. 300 | 301 | defined(TableRecord, Field) -> 302 | FieldsList = fields(element(1, TableRecord)), 303 | lists:member(Field, FieldsList). 304 | 305 | field(TableRecord, Field) -> 306 | FieldsList = fields(element(1, TableRecord)), 307 | Index = string:str(FieldsList, [Field]) + 1, 308 | element(Index, TableRecord). 309 | 310 | setfield(TableRecord, Field, Value) -> 311 | FieldsList = fields(element(1, TableRecord)), 312 | Index = string:str(FieldsList, [Field]) + 1, 313 | setelement(Index, TableRecord, Value). 314 | -------------------------------------------------------------------------------- /src/layers/kvs_st.erl: -------------------------------------------------------------------------------- 1 | -module(kvs_st). 2 | -include("kvs.hrl"). 3 | -include("stream.hrl"). 4 | -include("metainfo.hrl"). 5 | -export(?STREAM). 6 | -import(kvs_rocks, [fmt/1, key/2, key/1, bt/1, tb/1, ref/0, ref/1, seek_it/1, seek_it/2, move_it/3, move_it/4, take_it/4, take_it/5, delete_it/2, estimate/0, estimate/1]). 7 | -export([raw_append/2,raw_append/3, remove/1]). 8 | 9 | db() -> application:get_env(kvs,rocks_name,"rocksdb"). 10 | 11 | se(X,Y,Z) -> setelement(X,Y,Z). 12 | e(X,Y) -> element(X,Y). 13 | c4(R,V) -> se(#reader.args, R, V). 14 | si(M,T) -> se(#it.id, M, T). 15 | id(T) -> e(#it.id, T). 16 | 17 | k(F,[]) -> F; 18 | k(_,{_,Id,SF}) -> iolist_to_binary([SF,<<"/">>,tb(Id)]). 19 | 20 | f2(Feed) -> X = tb(Feed), 21 | case binary:matches(X, <<"/">>,[]) of 22 | [{0,1}|_] -> binary:part(X,{1,size(X)-1}); 23 | _ -> X 24 | end. 25 | 26 | read_it(C,{ok,_,[],H}) -> C#reader{cache=[], args=lists:reverse(H)}; 27 | read_it(C,{ok,F,V,H}) -> C#reader{cache={e(1,V),id(V),F}, args=lists:reverse(H)}; 28 | read_it(C,_) -> C#reader{args=[]}. 29 | 30 | top(#reader{}=X) -> top(X,db()). 31 | top(#reader{feed=Feed}=C,Db) -> #writer{count=Cn} = writer(f2(Feed),Db), read_it(C#reader{count=Cn},seek_it(Feed,Db)). 32 | bot(#reader{}=X) -> bot(X,db()). 33 | bot(#reader{feed=Feed}=C,Db) -> #writer{cache=Ch, count=Cn} = writer(f2(Feed),Db), C#reader{cache=Ch, count=Cn}. 34 | next(#reader{}=X) -> next(X,db()). 35 | next(#reader{feed=Feed,cache=I}=C,Db) -> read_it(C,move_it(k(Feed,I),Feed,next,Db)). 36 | prev(#reader{}=X) -> prev(X,db()). 37 | prev(#reader{cache=I,feed=Feed}=C,Db) -> read_it(C,move_it(k(Feed,I),Feed,prev,Db)). 38 | take(#reader{}=X) -> take(X,db()). 39 | take(#reader{args=N,feed=Feed,cache=I,dir=1}=C,Db) -> read_it(C,take_it(k(Feed,I),Feed,prev,N,Db)); 40 | take(#reader{args=N,feed=Feed,cache=I,dir=_}=C,Db) -> read_it(C,take_it(k(Feed,I),Feed,next,N,Db)). 41 | drop(#reader{}=X) -> drop(X,db()). 42 | drop(#reader{args=N}=C,_) when N =< 0 -> C; 43 | drop(#reader{}=C,Db) -> (take(C#reader{dir=0},Db))#reader{args=[]}. 44 | remove(#reader{}=C) -> remove(C, db()). 45 | remove(#reader{feed=Feed}=C,Db) -> R = read_it(C, delete_it(Feed,Db)), kvs:delete(writer, Feed), R; 46 | remove(Rec,Feed) -> remove(Rec,Feed,db()). 47 | 48 | feed(Feed) -> feed(Feed,db()). 49 | feed(Feed,Db) -> 50 | #reader{count=Cn} = Top = top(reader(Feed,Db),Db), 51 | Halt = case {estimate(),Cn} of 52 | {E,C} when E =< 0 -> max(C,4); 53 | {E,_} -> E 54 | end, 55 | feed(fun(#reader{}=R) -> take(R#reader{args=4},Db) end,Top,[],Halt). 56 | 57 | feed(_F,#reader{},Acc,H) when H =< 0 -> Acc; 58 | feed(F,#reader{cache=C1}=R,Acc,H) -> 59 | #reader{args=A, cache=Ch, feed=Feed} = R1 = F(R), 60 | case Ch of 61 | C1 -> Acc ++ A; 62 | {_,_,K} when binary_part(K,{0,byte_size(Feed)}) == Feed 63 | andalso length(A) == 4 64 | -> feed(F, R1, Acc ++ A, H-4); 65 | _ -> Acc ++ A 66 | end. 67 | 68 | load_reader(Id) -> load_reader(Id,db()). 69 | load_reader(Id,Db) -> case kvs:get(reader,Id,#kvs{db=Db,mod=kvs_rocks}) of {ok,#reader{}=C} -> C; _ -> #reader{id=kvs:seq([],[])} end. 70 | 71 | writer(Id) -> writer(Id,db()). 72 | writer(Id,Db) -> case kvs:get(writer,Id,#kvs{db=Db,mod=kvs_rocks}) of {ok,W} -> W; {error,_} -> #writer{id=Id} end. 73 | reader(Id) -> reader(Id,db()). 74 | reader(Id,Db) -> case kvs:get(writer,Id,#kvs{db=Db,mod=kvs_rocks}) of 75 | {ok,#writer{id=Feed, count=Cn, cache=Ch}} -> 76 | read_it(#reader{id=kvs:seq([],[]),feed=key(Feed),count=Cn,cache=Ch},seek_it(key(Feed),Db)); 77 | {error,_} -> 78 | read_it(#reader{id=kvs:seq([],[]),feed=key(Id),count=0,cache=[]},seek_it(key(Id),Db)) 79 | end. 80 | save(C) -> save(C,db()). 81 | save(C,Db) -> 82 | N1 = case id(C) of [] -> si(C,kvs:seq([],[])); _ -> C end, 83 | NC = c4(N1,[]), 84 | kvs:put(NC,#kvs{db=Db,mod=kvs_rocks}), NC. 85 | 86 | % add 87 | raw_append(M,Feed) -> raw_append(M,Feed,db()). 88 | raw_append(M,Feed,Db) -> rocksdb:put(ref(Db), key(Feed,e(2,M)), term_to_binary(M), [{sync,true}]). 89 | 90 | add(#writer{}=X) -> add(X, db()). 91 | add(#writer{args=M}=C,Db) when element(2,M) == [] -> add(si(M,kvs:seq([],[])),C,Db); 92 | add(#writer{args=M}=C,Db) -> add(M,C,Db). 93 | 94 | add(M,#writer{id=Feed,count=S}=C,Db) -> 95 | NS=S+1, 96 | raw_append(M,Feed,Db), 97 | C#writer{cache={e(1,M),e(2,M),key(Feed)},count=NS}. 98 | 99 | cut(Feed) -> cut(Feed,db()). 100 | cut(Feed,Db) -> 101 | #writer{cache={_,Key,Fd}=Ch} = kvs:writer(Feed, #kvs{db=Db,mod=kvs_rocks}), 102 | #reader{} = kvs:prev(reader(Feed, Db)), 103 | #reader{} = kvs:next(#reader{feed=key(Feed), cache=Ch}), 104 | 105 | kvs:delete_range(Feed,{Fd,Key},#kvs{db=Db,mod=kvs_rocks}). 106 | 107 | remove(Rec,Feed,Db) -> 108 | kvs:ensure(#writer{id=Feed},#kvs{db=Db,mod=kvs_rocks}), 109 | W = #writer{count=C, cache=Ch} = kvs:writer(Feed,#kvs{db=Db,mod=kvs_rocks}), 110 | Ch1 = case {e(1,Rec),e(2,Rec),key(Feed)} of % need to keep reference for next element 111 | Ch -> R = reader(Feed,Db), e(4, prev(R#reader{cache=Ch},Db)); 112 | _ -> Ch end, 113 | case kvs:delete(Feed,id(Rec),#kvs{db=Db,mod=kvs_rocks}) of 114 | ok -> Count = C - 1, 115 | save(W#writer{count = Count, cache=Ch1},Db), 116 | Count; 117 | _ -> C end. 118 | 119 | append(Rec,Feed) -> append(Rec,Feed,db()). 120 | append(Rec,Feed,Db) -> 121 | kvs:ensure(#writer{id=Feed},#kvs{db=Db,mod=kvs_rocks}), 122 | Id = e(2,Rec), 123 | W = writer(Feed,Db), 124 | case kvs:get(Feed,Id,#kvs{db=Db,mod=kvs_rocks}) of 125 | {ok,_} -> raw_append(Rec,Feed,Db), Id; 126 | {error,_} -> save(add(W#writer{args=Rec},Db),Db), Id end. 127 | -------------------------------------------------------------------------------- /src/layers/kvs_stream.erl: -------------------------------------------------------------------------------- 1 | -module(kvs_stream). 2 | -description('KVS STREAM LAYER'). 3 | -include("kvs.hrl"). 4 | -include("stream.hrl"). 5 | -include("metainfo.hrl"). 6 | -export(?STREAM). 7 | -export([metainfo/0]). 8 | 9 | db() -> kvs:db(). 10 | 11 | % boot for sample 12 | 13 | metainfo() -> #schema { name = kvs, tables = tables() }. 14 | tables() -> [ #table { name = writer, fields = record_info(fields, writer) }, 15 | #table { name = reader, fields = record_info(fields, reader) } ]. 16 | 17 | % section: kvs_stream prelude 18 | 19 | se(X,Y,Z) -> setelement(X,Y,Z). 20 | e(X,Y) -> element(X,Y). 21 | c4(R,V) -> se(#reader.args, R, V). 22 | sn(M,T) -> se(#iter.next, M, T). 23 | sp(M,T) -> se(#iter.prev, M, T). 24 | si(M,T) -> se(#iter.id, M, T). 25 | tab(T) -> e(1, T). 26 | id(T) -> e(#iter.id, T). 27 | en(T) -> e(#iter.next, T). 28 | ep(T) -> e(#iter.prev, T). 29 | acc(0) -> next; 30 | acc(1) -> prev. 31 | 32 | n({ok,R},C,P,Db) -> r(kvs:get(tab(R),en(R),#kvs{db=Db}),C,P); 33 | n({error,X},_,_,_) -> {error,X}. 34 | 35 | p({ok,R},C,P,Db) -> r(kvs:get(tab(R),ep(R),#kvs{db=Db}),C,P); 36 | p({error,X},_,_,_) -> {error,X}. 37 | 38 | r({ok,R},C,P) -> C#reader{cache={tab(R),id(R)},pos=P}; 39 | r({error,X},_,_) -> {error,X}. 40 | 41 | w({ok,#writer{first=[]}},bot,C) -> C#reader{cache=[],pos=1,dir=0}; 42 | w({ok,#writer{first=B}},bot,C) -> C#reader{cache={tab(B),id(B)},pos=1,dir=0}; 43 | w({ok,#writer{cache=B,count=Size}},top,C) -> C#reader{cache={tab(B),id(B)},pos=Size,dir=1}; 44 | w({error,X},_,_) -> {error,X}. 45 | 46 | % next, prev, top, bot 47 | top(#reader{}=X) -> top(X,db()). 48 | top(#reader{feed=F}=C,Db) -> w(kvs:get(writer,F,#kvs{db=Db}),top,C). 49 | bot(#reader{}=X) -> bot(X,db()). 50 | bot(#reader{feed=F}=C,Db) -> w(kvs:get(writer,F,#kvs{db=Db}),bot,C). 51 | 52 | next(#reader{}=X) -> next(X,db()). 53 | next(#reader{cache=[]},_) -> {error,empty}; 54 | next(#reader{cache={T,R},pos=P}=C,Db) -> n(kvs:get(T,R,#kvs{db=Db}),C,P+1,Db). 55 | 56 | prev(#reader{}=X) -> prev(X,db()). 57 | prev(#reader{cache=[]},_) -> {error,empty}; 58 | prev(#reader{cache={T,R},pos=P}=C,Db) -> p(kvs:get(T,R,#kvs{db=Db}),C,P-1,Db). 59 | 60 | % take, drop, feed 61 | 62 | drop(#reader{}=X) -> drop(X,db()). 63 | drop(#reader{cache=[]}=C,_) -> C#reader{args=[]}; 64 | drop(#reader{dir=D,cache=B,args=N,pos=P}=C,Db) -> drop(acc(D),N,C,C,P,B,Db). 65 | take(#reader{}=X) -> take(X,db()). 66 | take(#reader{cache=[]}=C,_) -> C#reader{args=[]}; 67 | take(#reader{dir=D,cache=_B,args=N,pos=P}=C,Db) -> take(acc(D),N,C,C,[],P,Db). 68 | 69 | take(_,_,{error,_},C2,R,P,_) -> C2#reader{args=lists:flatten(R),pos=P,cache={tab(hd(R)),en(hd(R))}}; 70 | take(_,0,_,C2,R,P,_) -> C2#reader{args=lists:flatten(R),pos=P,cache={tab(hd(R)),en(hd(R))}}; 71 | take(A,N,#reader{cache={T,I},pos=P}=C,C2,R,_,Db) -> take(A,N-1,?MODULE:A(C,Db),C2,[element(2,kvs:get(T,I,#kvs{db=Db}))|R],P,Db). 72 | 73 | drop(_,_,{error,_},C2,P,B,_) -> C2#reader{pos=P,cache=B}; 74 | drop(_,0,_,C2,P,B,_) -> C2#reader{pos=P,cache=B}; 75 | drop(A,N,#reader{cache=B,pos=P}=C,C2,_,_,Db) -> drop(A,N-1,?MODULE:A(C,Db),C2,P,B,Db). 76 | 77 | feed(Feed) -> feed(Feed,db()). 78 | feed(Feed,Db) -> #reader{args=Args} = take((reader(Feed,Db))#reader{args=-1},Db), Args. 79 | 80 | % new, save, load, writer, reader 81 | 82 | load_reader (Id) -> load_reader(Id,db()). 83 | load_reader (Id,Db) -> case kvs:get(reader,Id,#kvs{db=Db}) of {ok,C} -> C; _ -> #reader{id=[]} end. 84 | 85 | save (C) -> save(C,db()). 86 | save (C,Db) -> NC = c4(C,[]), kvs:put(NC,#kvs{db=Db}), NC. 87 | writer (Id) -> writer(Id,db()). 88 | writer (Id,Db) -> case kvs:get(writer,Id,#kvs{db=Db}) of {ok,W} -> W; {error,_} -> #writer{id=Id} end. 89 | reader (Id) -> reader(Id,db()). 90 | reader (Id,Db) -> case kvs:get(writer,Id,#kvs{db=Db}) of 91 | {ok,#writer{first=[]}} -> #reader{id=kvs:seq(reader,1),feed=Id,cache=[]}; 92 | {ok,#writer{first=F}} -> #reader{id=kvs:seq(reader,1),feed=Id,cache={tab(F),id(F)}}; 93 | {error,_} -> save(#writer{id=Id},Db), reader(Id,Db) end. 94 | 95 | % add, remove, append 96 | 97 | add(#writer{}=X) -> add(X,db()). 98 | add(#writer{args=M}=C,Db) when element(2,M) == [] -> add(si(M,kvs:seq(tab(M),1)),C,Db); 99 | add(#writer{args=M}=C,Db) -> add(M,C,Db). 100 | 101 | add(M,#writer{cache=[]}=C,Db) -> 102 | _Id=id(M), N=sp(sn(M,[]),[]), kvs:put(N,#kvs{db=Db}), 103 | C#writer{cache=N,count=1,first=N}; 104 | 105 | add(M,#writer{cache=V1,count=S}=C,Db) -> 106 | {ok,V} = kvs:get(tab(V1),id(V1),#kvs{db=Db}), 107 | N=sp(sn(M,[]),id(V)), P=sn(V,id(M)), kvs:put([N,P],#kvs{db=Db}), 108 | C#writer{cache=N,count=S+1}. 109 | 110 | cut(Feed) -> cut(Feed, db()). 111 | cut(_,_) -> ignore. 112 | remove(Rec,Feed) -> remove(Rec,Feed,db()). 113 | remove(_Rec,Feed,Db) -> 114 | {ok,W=#writer{count=Count}} = kvs:get(writer,Feed,#kvs{db=Db}), 115 | NC = Count-1, 116 | kvs:save(W#writer{count=NC},#kvs{db=Db}), 117 | NC. 118 | 119 | append(Rec,Feed) -> append(Rec,Feed,db()). 120 | append(Rec,Feed,Db) -> 121 | kvs:ensure(#writer{id=Feed},#kvs{db=Db}), 122 | Name = element(1,Rec), 123 | Id = element(2,Rec), 124 | case kvs:get(Name,Id,#kvs{db=Db}) of 125 | {ok,_} -> Id; 126 | {error,_} -> kvs:save(kvs:add((kvs:writer(Feed,#kvs{db=Db}))#writer{args=Rec},#kvs{db=Db}),#kvs{db=Db}), Id end. 127 | -------------------------------------------------------------------------------- /src/stores/kvs_fs.erl: -------------------------------------------------------------------------------- 1 | -module(kvs_fs). 2 | -include("backend.hrl"). 3 | -include("kvs.hrl"). 4 | -include("metainfo.hrl"). 5 | -include_lib("stdlib/include/qlc.hrl"). 6 | -export(?BACKEND). 7 | 8 | db() -> "". 9 | 10 | start() -> ok. 11 | 12 | stop() -> ok. 13 | 14 | destroy() -> ok. 15 | 16 | destroy(_) -> ok. 17 | 18 | match(_) -> []. 19 | 20 | index_match(_,_) -> []. 21 | 22 | version() -> {version,"KVS FS"}. 23 | 24 | leave() -> ok. 25 | 26 | leave(_) -> ok. 27 | 28 | dir() -> [ {table,F} || F <- filelib:wildcard("data/*"), filelib:is_dir(F) ]. 29 | 30 | join(_Node,_) -> filelib:ensure_dir(dir_name()), initialize(). % should be rsync or smth 31 | 32 | initialize() -> 33 | mnesia:create_schema([node()]), 34 | [ kvs:initialize(kvs_fs,Module) || Module <- kvs:modules() ], 35 | mnesia:wait_for_tables([ T#table.name || T <- kvs:tables()],infinity). 36 | 37 | index(_Tab,_Key,_Value) -> []. 38 | get(TableName, Key, _) -> 39 | HashKey = hashkey(Key), 40 | {ok, Dir} = dir(TableName), 41 | File = filename:join([Dir,HashKey]), 42 | case file:read_file(File) of 43 | {ok,Binary} -> {ok,binary_to_term(Binary,[safe])}; 44 | {error,Reason} -> {error,Reason} end. 45 | 46 | put(R) -> put(R,db()). 47 | put(Records,X) when is_list(Records) -> lists:map(fun(Record) -> put(Record,X) end, Records); 48 | put(Record,_) -> 49 | TableName = element(1,Record), 50 | HashKey = hashkey(element(2,Record)), 51 | BinaryValue = term_to_binary(Record), 52 | {ok, Dir} = dir(TableName), 53 | File = filename:join([Dir,HashKey]), 54 | filelib:ensure_dir(File), 55 | file:write_file(File,BinaryValue,[write,raw,binary,sync]). 56 | 57 | dir_name() -> "data". 58 | 59 | dir(TableName) -> 60 | {ok, Cwd} = file:get_cwd(), 61 | TablePath = filename:join([dir_name(),TableName]), 62 | case filelib:safe_relative_path(TablePath, Cwd) of 63 | unsafe -> {error, {unsafe, TablePath}}; 64 | Target -> {ok, Target} 65 | end. 66 | 67 | hashkey(Key) -> encode(base64:encode(crypto:hash(sha, term_to_binary(Key)))). 68 | 69 | delete(TableName, Key, _) -> 70 | case kvs_fs:get(TableName, Key) of 71 | {ok,_} -> 72 | {ok, Dir} = dir(TableName), 73 | HashKey = hashkey(Key), 74 | File = filename:join([Dir,HashKey]), 75 | file:delete(File); 76 | {error,X} -> {error,X} 77 | end. 78 | delete_range(_,_,_) -> {error, not_found}. 79 | keys(_,_) -> []. 80 | key_match(_,_,_) -> []. 81 | 82 | count(RecordName) -> length(filelib:fold_files(filename:join([dir_name(), RecordName]), "",true, fun(A,Acc)-> [A|Acc] end, [])). 83 | 84 | all(R,_) -> lists:flatten([ begin case file:read_file(File) of 85 | {ok,Binary} -> binary_to_term(Binary,[safe]); 86 | {error,_Reason} -> [] end end || File <- 87 | filelib:fold_files(filename:join([dir_name(), R]), "",true, fun(A,Acc)-> [A|Acc] end, []) ]). 88 | 89 | seq(RecordName, Incr) -> kvs_mnesia:seq(RecordName, Incr). 90 | 91 | create_table(Name,_Options) -> 92 | {ok, Dir} = dir(Name), 93 | file:make_dir(Dir), 94 | filelib:ensure_dir(Dir). 95 | 96 | add_table_index(_Record, _Field) -> ok. 97 | 98 | % URL ENCODE 99 | 100 | encode(B) when is_binary(B) -> encode(binary_to_list(B)); 101 | encode([C | Cs]) when C >= $a, C =< $z -> [C | encode(Cs)]; 102 | encode([C | Cs]) when C >= $A, C =< $Z -> [C | encode(Cs)]; 103 | encode([C | Cs]) when C >= $0, C =< $9 -> [C | encode(Cs)]; 104 | encode([C | Cs]) when C == 16#20 -> [$+ | encode(Cs)]; 105 | 106 | % unreserved 107 | encode([C = $- | Cs]) -> [C | encode(Cs)]; 108 | encode([C = $_ | Cs]) -> [C | encode(Cs)]; 109 | encode([C = 46 | Cs]) -> [C | encode(Cs)]; 110 | encode([C = $! | Cs]) -> [C | encode(Cs)]; 111 | encode([C = $~ | Cs]) -> [C | encode(Cs)]; 112 | encode([C = $* | Cs]) -> [C | encode(Cs)]; 113 | encode([C = 39 | Cs]) -> [C | encode(Cs)]; 114 | encode([C = $( | Cs]) -> [C | encode(Cs)]; 115 | encode([C = $) | Cs]) -> [C | encode(Cs)]; 116 | 117 | encode([C | Cs]) when C =< 16#7f -> escape_byte(C) ++ encode(Cs); 118 | encode([C | Cs]) when (C >= 16#7f) and (C =< 16#07FF) -> 119 | escape_byte((C bsr 6) + 16#c0) 120 | ++ escape_byte(C band 16#3f + 16#80) 121 | ++ encode(Cs); 122 | encode([C | Cs]) when (C > 16#07FF) -> 123 | escape_byte((C bsr 12) + 16#e0) % (0xe0 | C >> 12) 124 | ++ escape_byte((16#3f band (C bsr 6)) + 16#80) % 0x80 | ((C >> 6) & 0x3f) 125 | ++ escape_byte(C band 16#3f + 16#80) % 0x80 | (C >> 0x3f) 126 | ++ encode(Cs); 127 | encode([C | Cs]) -> escape_byte(C) ++ encode(Cs); 128 | encode([]) -> []. 129 | 130 | hex_octet(N) when N =< 9 -> [$0 + N]; 131 | hex_octet(N) when N > 15 -> hex_octet(N bsr 4) ++ hex_octet(N band 15); 132 | hex_octet(N) -> [N - 10 + $a]. 133 | escape_byte(C) -> normalize(hex_octet(C)). 134 | normalize(H) when length(H) == 1 -> "%0" ++ H; 135 | normalize(H) -> "%" ++ H. 136 | 137 | dump() -> ok. 138 | -------------------------------------------------------------------------------- /src/stores/kvs_mnesia.erl: -------------------------------------------------------------------------------- 1 | -module(kvs_mnesia). 2 | -include("backend.hrl"). 3 | -include("kvs.hrl"). 4 | -include("metainfo.hrl"). 5 | -include_lib("mnesia/src/mnesia.hrl"). 6 | -include_lib("stdlib/include/qlc.hrl"). 7 | -export(?BACKEND). 8 | -export([info/1,exec/1,dump/1]). 9 | 10 | db() -> "". 11 | seq_pad() -> application:get_env(kvs, seq_pad, []). 12 | start() -> mnesia:start(). 13 | stop() -> mnesia:stop(). 14 | destroy() -> [mnesia:delete_table(T)||{_,T}<-kvs:dir()], mnesia:delete_schema([node()]), ok. 15 | destroy(_) -> [mnesia:delete_table(T)||{_,T}<-kvs:dir()], mnesia:delete_schema([node()]), ok. 16 | leave() -> ok. 17 | leave(_) -> ok. 18 | version() -> {version,"KVS MNESIA"}. 19 | dir() -> [{table,T}||T<-mnesia:system_info(local_tables)]. 20 | join([], _) -> mnesia:start(), mnesia:change_table_copy_type(schema, node(), disc_copies), initialize(); 21 | 22 | join(Node, _) -> 23 | mnesia:start(), 24 | mnesia:change_config(extra_db_nodes, [Node]), 25 | mnesia:change_table_copy_type(schema, node(), disc_copies), 26 | [{Tb, mnesia:add_table_copy(Tb, node(), Type)} 27 | || {Tb, [{N, Type}]} <- [{T, mnesia:table_info(T, where_to_commit)} 28 | || T <- mnesia:system_info(tables)], Node==N]. 29 | 30 | initialize() -> 31 | mnesia:create_schema([node()]), 32 | Res = [ kvs:initialize(kvs_mnesia,Module) || Module <- kvs:modules() ], 33 | mnesia:wait_for_tables([ T#table.name || T <- kvs:tables()],infinity), 34 | Res. 35 | 36 | index(Tab,Key,Value) -> 37 | lists:flatten(many(fun() -> mnesia:index_read(Tab,Value,Key) end)). 38 | 39 | keys(Tab,_) -> mnesia:all_keys(Tab). 40 | 41 | key_match(_Tab,_Id,_) -> []. 42 | 43 | get(RecordName, Key, _) -> just_one(fun() -> mnesia:read(RecordName, Key) end). 44 | put(R) -> put(R,db()). 45 | put(Records, _) when is_list(Records) -> void(fun() -> lists:foreach(fun mnesia:write/1, Records) end); 46 | put(Record, X) -> put([Record], X). 47 | delete(Tab, Key, _) -> 48 | case mnesia:activity(context(),fun()-> mnesia:delete({Tab, Key}) end) of 49 | {aborted,Reason} -> {error,Reason}; 50 | {atomic,_Result} -> ok; 51 | _ -> ok end. 52 | delete_range(_,_,_) -> {error, not_found}. 53 | match(Record) -> lists:flatten(many(fun() -> mnesia:match_object(Record) end)). 54 | index_match(Record, Index) -> lists:flatten(many(fun() -> mnesia:index_match_object(Record, Index) end)). 55 | count(RecordName) -> mnesia:table_info(RecordName, size). 56 | all(R, _) -> lists:flatten(many(fun() -> L= mnesia:all_keys(R), [ mnesia:read({R, G}) || G <- L ] end)). 57 | seq(RecordName, []) -> seq(RecordName, 1); 58 | seq(RecordName, Incr) -> 59 | Val = integer_to_list(mnesia:dirty_update_counter({id_seq, RecordName}, Incr)), 60 | case (20 - length(Val) > 0) and lists:member(RecordName, seq_pad()) of true -> string:copies("0", 20 - length(Val)); _ -> "" end ++ Val. 61 | 62 | many(Fun) -> case mnesia:activity(context(),Fun) of {atomic, [R]} -> R; {aborted, Error} -> {error, Error}; X -> X end. 63 | void(Fun) -> case mnesia:activity(context(),Fun) of {atomic, ok} -> ok; {aborted, Error} -> {error, Error}; X -> X end. 64 | info(T) -> try mnesia:table_info(T,all) catch _:_ -> [] end. 65 | create_table(Name,Options) -> mnesia:create_table(Name, Options). 66 | add_table_index(Record, Field) -> mnesia:add_table_index(Record, Field). 67 | exec(Q) -> F = fun() -> qlc:e(Q) end, {atomic, Val} = mnesia:activity(context(),F), Val. 68 | just_one(Fun) -> 69 | case mnesia:activity(context(),Fun) of 70 | {atomic, []} -> {error, not_found}; 71 | {atomic, [R]} -> {ok, R}; 72 | [] -> {error, not_found}; 73 | [R] -> {ok,R}; 74 | R when is_list(R) -> {ok,R}; 75 | Error -> Error end. 76 | 77 | context() -> application:get_env(kvs,mnesia_context,async_dirty). 78 | 79 | dump() -> dump([ N || #table{name=N} <- kvs:tables() ]), ok. 80 | dump(short) -> 81 | Gen = fun(T) -> 82 | {S,M,C}=lists:unzip3([ dump_info(R) || R <- T ]), 83 | {lists:usort(S),lists:sum(M),lists:sum(C)} 84 | end, 85 | dump_format([ {T,Gen(T)} || T <- [ N || #table{name=N} <- kvs:tables() ] ]); 86 | dump(Table) when is_atom(Table) -> dump(Table); 87 | dump(Tables) -> 88 | dump_format([{T,dump_info(T)} || T <- lists:flatten(Tables) ]). 89 | dump_info(T) -> 90 | {mnesia:table_info(T,storage_type), 91 | mnesia:table_info(T,memory) * erlang:system_info(wordsize) / 1024 / 1024, 92 | mnesia:table_info(T,size)}. 93 | dump_format(List) -> 94 | io:format("~20s ~32s ~14s ~10s~n~n",["NAME","STORAGE TYPE","MEMORY (MB)","ELEMENTS"]), 95 | [ io:format("~20s ~32w ~14.2f ~10b~n",[T,S,M,C]) || {T,{S,M,C}} <- List ], 96 | io:format("~nSnapshot taken: ~p~n",[calendar:now_to_datetime(os:timestamp())]). 97 | -------------------------------------------------------------------------------- /src/stores/kvs_rocks.erl: -------------------------------------------------------------------------------- 1 | -module(kvs_rocks). 2 | -include("backend.hrl"). 3 | -include("kvs.hrl"). 4 | -include("metainfo.hrl"). 5 | -include_lib("stdlib/include/qlc.hrl"). 6 | -export(?BACKEND). 7 | -export([ref/0,ref/1,bt/1,key/2,key/1,fd/1,tb/1,estimate/0,estimate/1]). 8 | -export([seek_it/1, seek_it/2, move_it/3, move_it/4, take_it/4, take_it/5, delete_it/1, delete_it/2]). 9 | 10 | e(X,Y) -> element(X,Y). 11 | 12 | bt([]) -> []; 13 | bt(X) -> binary_to_term(X). 14 | 15 | tb([]) -> <<>>; 16 | tb(T) when is_list(T) -> unicode:characters_to_nfkc_binary(T); 17 | tb(T) when is_atom(T) -> erlang:atom_to_binary(T, utf8); 18 | tb(T) when is_binary(T) -> T; 19 | tb(T) -> term_to_binary(T). 20 | sz([]) -> 0; 21 | sz(B) -> byte_size(B). 22 | 23 | key(R) when is_tuple(R) andalso tuple_size(R) > 1 -> key(e(1,R), e(2,R)); 24 | key(R) -> key(R,[]). 25 | key(writer,R) -> % allow old writers 26 | iolist_to_binary([lists:join(<<"/">>, lists:flatten([<<>>, erlang:atom_to_binary(writer, utf8), tb(R)]))]); 27 | key(Tab,R) -> Fd = case Tab of [] -> []; _ -> tb(Tab) end, 28 | iolist_to_binary([lists:join(<<"/">>, lists:flatten([<<>>, Fd, fmt(R)]))]). 29 | keys(Tab, Db) -> 30 | Feed = key(Tab,[]), 31 | {ok, H} = rocksdb:iterator(ref(Db), []), 32 | Keys = fun KEY(K1,Acc) when binary_part(K1,{0,byte_size(Feed)}) =:= Feed -> 33 | case rocksdb:iterator_move(H, next) of 34 | {ok,K2,_} -> KEY(K2,[tb(K1)|Acc]); 35 | _ -> lists:reverse([tb(K1)|Acc]) 36 | end; 37 | KEY(_,Acc) -> rocksdb:iterator_close(H), lists:reverse(Acc) 38 | end, 39 | {ok, K, _} = rocksdb:iterator_move(H, {seek, Feed}), 40 | Keys(K,[]). 41 | 42 | key_match(Tab, Id, Db) -> 43 | Feed = key(Tab,[]), 44 | {ok, H} = rocksdb:iterator(ref(Db), []), 45 | Keys = fun KEY(K1) when 46 | binary_part(K1,{0,byte_size(Feed)}) =:= Feed andalso 47 | binary_part(K1,{byte_size(K1), -byte_size(Id)}) =:= Id -> 48 | rocksdb:iterator_close(H), [K1]; 49 | KEY(K1) when binary_part(K1,{0,byte_size(Feed)}) =:= Feed -> 50 | case rocksdb:iterator_move(H, next) of 51 | {ok,K2,_} -> KEY(K2); 52 | _ -> [] 53 | end; 54 | KEY(_) -> rocksdb:iterator_close(H), [] 55 | end, 56 | {ok, K, _} = rocksdb:iterator_move(H, {seek, Feed}), 57 | Keys(K). 58 | 59 | fmt([]) -> []; 60 | fmt(K) -> Key = tb(K), 61 | End = sz(Key), 62 | {S,E} = case binary:matches(Key, [<<"/">>], []) of 63 | [{0,1}] -> {1, End-1}; 64 | [{0,1},{1,1}] -> {2, End-2}; 65 | [{0,1},{1,1}|_] -> {2, End-2}; 66 | [{0,1}|_] -> {1, End-1}; 67 | _ -> {0, End} 68 | end, 69 | binary:part(Key,{S,E}). 70 | 71 | fd(K) -> Key = tb(K), 72 | End = sz(Key), 73 | {S,_} = case binary:matches(Key,[<<"/">>],[]) of 74 | [{0,1}] -> {End,End}; 75 | [{0,1},{1,1}] -> {End,End}; 76 | [{0,1},{1,1}|T] -> hd(lists:reverse(T)); 77 | [{0,1}|T] -> hd(lists:reverse(T)); 78 | _ -> {End,End} 79 | end, 80 | binary:part(Key,{0,S}). 81 | 82 | run(<<>>,SK,_,_,_) -> {ok,SK,[],[]}; 83 | run(Key, % key 84 | SK, % sup-key 85 | Dir, % direction next/prev 86 | Compiled_Operations, 87 | Db) -> 88 | % H is iterator reference 89 | 90 | S = sz(SK), 91 | Initial_Object = {ref(Db), []}, 92 | 93 | Run = fun (F,K,H,V,Acc) when binary_part(K,{0,S}) == SK -> {F(H,Dir),H,[V|Acc]}; % continue +------------+ 94 | (_,K,H,V,Acc) -> stop_it(H), % fail-safe closing | 95 | throw({ok, fd(K), bt(V), [bt(A1) || A1 <- Acc]}) end, % acc unfold | 96 | % | 97 | Range_Check = fun(F,K,H,V) -> case F(H,prev) of % backward prefetch | 98 | {ok,K1,V1} when binary_part(K,{0,S}) == SK -> {{ok,K1,V1},H,[V]}; % return (range-check error) | 99 | {ok,K1,V1} -> Run(F,K1,H,V1,[]); % run prev-take chain | loop 100 | _ -> stop_it(H), % fail-safe closing | 101 | throw({ok, fd(K), bt(V), [bt(V)]}) % acc unfold | 102 | end end, % | 103 | % | 104 | State_Machine = fun % | 105 | (F,{ok,H}) -> {F(H,{seek,Key}),H}; % first move (seek) | 106 | (F,{{ok,K,V},H}) when Dir =:= prev -> Range_Check(F,K,H,V); % first chained prev-take | 107 | (F,{{ok,K,V},H}) -> Run(F,K,H,V,[]); % first chained next-take | 108 | (F,{{ok,K,V},H,A}) -> Run(F,K,H,V,A); % chained CPS-take continuator +---+ 109 | (_,{{error,E},H,Acc}) -> {{error,E},H,Acc}; % error effects 110 | (F,{I,O}) -> F(I,O) % chain constructor from initial object 111 | end, 112 | 113 | catch case lists:foldl(State_Machine, Initial_Object, Compiled_Operations) of 114 | {{ok,K,Bin},H,A} -> stop_it(H), {ok, fd(K), bt(Bin), [bt(A1) || A1 <- A]}; 115 | {{ok,K,Bin},H} -> stop_it(H), {ok, fd(K), bt(Bin), []}; 116 | {{error,_},H,Acc} -> stop_it(H), {ok, fd(SK), bt(shd(Acc)), [bt(A1) || A1 <- Acc]}; 117 | {{error,_},H} -> stop_it(H), {ok, fd(SK), [], []} 118 | end. 119 | 120 | initialize() -> [ kvs:initialize(kvs_rocks,Module) || Module <- kvs:modules() ]. 121 | index(_,_,_) -> []. 122 | 123 | ref_env(Db) -> list_to_atom("rocks_ref_" ++ Db). 124 | db() -> application:get_env(kvs,rocks_name,"rocksdb"). 125 | start() -> ok. 126 | stop() -> ok. 127 | destroy() -> destroy(db()). 128 | destroy(Db) -> rocksdb:destroy(Db, []). 129 | version() -> {version,"KVS ROCKSDB"}. 130 | dir() -> []. 131 | match(_) -> []. 132 | index_match(_,_) -> []. 133 | ref() -> ref(db()). 134 | ref(Db) -> application:get_env(kvs,ref_env(Db),[]). 135 | leave() -> leave(db()). 136 | leave(Db) -> case ref(Db) of [] -> skip; X -> rocksdb:close(X), application:set_env(kvs,ref_env(Db),[]), ok end. 137 | join(_,Db) -> 138 | application:start(rocksdb), 139 | leave(Db), {ok, Ref} = rocksdb:open(Db, [{create_if_missing, true}]), 140 | initialize(), 141 | application:set_env(kvs,ref_env(Db),Ref). 142 | 143 | compile(it) -> [fun rocksdb:iterator/2]; 144 | compile(seek) -> [fun rocksdb:iterator/2,fun rocksdb:iterator_move/2]; 145 | compile(move) -> [fun rocksdb:iterator_move/2]; 146 | compile(close) -> [fun rocksdb:iterator_close/1]. 147 | compile(take,N) -> lists:map(fun(_) -> fun rocksdb:iterator_move/2 end, lists:seq(1, N)). 148 | compile(delete,_, {error,E},_) -> {error,E}; 149 | compile(delete,SK,{ok,_,V1,_},Db) -> 150 | F1 = key(key(fmt(SK),e(2,V1))), S = sz(SK), 151 | [fun Del(H,Dir) -> 152 | case rocksdb:delete(ref(Db), F1, []) of ok -> 153 | % {ok, K} case exist only in api, but never actually used 154 | case rocksdb:iterator_move(H,Dir) of 155 | {ok,K,_} when binary_part(K,{0,S}) == SK -> case rocksdb:delete(ref(Db), K, []) of ok -> Del(H,Dir); E -> E end; 156 | {ok,K} when binary_part(K,{0,S}) == SK -> case rocksdb:delete(ref(Db), K, []) of ok -> Del(H,Dir); E -> E end; 157 | {ok,K,V} -> {ok,K,V}; 158 | {ok,K} -> {ok, K}; 159 | E -> E 160 | end; 161 | E -> E 162 | end 163 | end]. 164 | 165 | stop_it(H) -> try begin [F]=compile(close), F(H) end catch error:badarg -> ok end. 166 | seek_it(K) -> seek_it(K,db()). 167 | seek_it(K,Db) -> run(K,K,ok,compile(seek),Db). 168 | move_it(Key,SK,Dir) -> move_it(Key,SK,Dir,db()). 169 | move_it(Key,SK,Dir,Db) -> run(Key,SK,Dir,compile(seek) ++ compile(move),Db). 170 | take_it(Key,SK,Dir,N) -> take_it(Key,SK,Dir,N,db()). 171 | take_it(Key,SK,Dir,N,Db) when is_integer(N) andalso N >= 0 -> run(Key,SK,Dir,compile(seek) ++ compile(take,N),Db); 172 | take_it(Key,SK,Dir,_,Db) -> take_it(Key,SK,Dir,0,Db). 173 | delete_it(Fd) -> delete_it(Fd, db()). 174 | delete_it(Fd,Db) -> run(Fd,Fd,next,compile(seek) ++ compile(delete,Fd,seek_it(Fd),Db),Db). 175 | 176 | all(R,Db) -> kvs_st:feed(R,Db). 177 | 178 | get(Tab, {step,N,[208|_]=Key}, Db) -> get(Tab, {step,N,list_to_binary(Key)},Db); 179 | get(Tab, [208|_]=Key, Db) -> get(Tab, list_to_binary(Key), Db); 180 | get(Tab, Key, Db) -> 181 | case rocksdb:get(ref(Db), key(Tab,Key), []) of 182 | not_found -> {error,not_found}; 183 | {ok,Bin} -> {ok,bt(Bin)} end. 184 | 185 | put(Record) -> put(Record,db()). 186 | put(Records,Db) when is_list(Records) -> lists:map(fun(Record) -> put(Record,Db) end, Records); 187 | put(Record,Db) -> rocksdb:put(ref(Db), key(Record), term_to_binary(Record), [{sync,true}]). 188 | delete(Feed, Id, Db) -> rocksdb:delete(ref(Db), key(Feed,Id), []). 189 | delete_range(Feed,{Fd,Key},Db) -> 190 | Last = key(key(fmt(Fd),Key)), 191 | ReadOps = [{'prefix_same_as_start', true}], 192 | CompactOps = [{change_level, true}], 193 | Feed1 = key(Feed), 194 | Sz = size(Feed1), 195 | Reopen = case ref(Db) of [] -> skip; _ -> leave(Db), ok end, 196 | 197 | {ok, R} = rocksdb:open(Db, [{prefix_extractor, {capped_prefix_transform, Sz}}]), 198 | {ok, H} = rocksdb:iterator(R, ReadOps), 199 | {ok, Start, _} = rocksdb:iterator_move(H, {seek, Feed1}), 200 | 201 | ok = rocksdb:delete_range(R, Start, Last, []), 202 | ok = rocksdb:delete(R, Last, []), 203 | ok = rocksdb:delete(R, key(writer,Feed), []), 204 | ok = rocksdb:compact_range(R, Start, undefined, CompactOps), 205 | ok = rocksdb:iterator_close(H), 206 | ok = rocksdb:close(R), 207 | case Reopen of skip -> ok; ok -> join([],Db) end. 208 | count(_) -> 0. 209 | estimate() -> estimate(db()). 210 | estimate(Db) -> case rocksdb:get_property(ref(Db), <<"rocksdb.estimate-num-keys">>) of 211 | {ok, Est} when is_binary(Est) -> binary_to_integer(Est); 212 | {ok, Est} when is_list(Est) -> list_to_integer(Est); 213 | {ok, Est} when is_integer(Est) -> Est; 214 | _ -> 0 215 | end. 216 | 217 | shd([]) -> []; 218 | shd(X) -> hd(X). 219 | create_table(_,_) -> []. 220 | add_table_index(_, _) -> ok. 221 | dump() -> ok. 222 | 223 | seq(_,_) -> 224 | Val = 225 | case os:type() of 226 | {win32,nt} -> {Mega,Sec,Micro} = erlang:timestamp(), integer_to_list((Mega*1000000+Sec)*1000000+Micro); 227 | _ -> integer_to_list(element(2,hd(lists:reverse(erlang:system_info(os_system_time_source))))) 228 | end, 229 | case 20 - length(Val) > 0 of true -> string:copies("0", 20 - length(Val)); _ -> "" end ++ Val. 230 | -------------------------------------------------------------------------------- /sys.config: -------------------------------------------------------------------------------- 1 | [ {kvs,[{dba,kvs_mnesia}, 2 | {dba_st,kvs_stream}, 3 | {schema,[kvs,kvs_stream]}]} 4 | ]. 5 | -------------------------------------------------------------------------------- /test/fd_test.exs: -------------------------------------------------------------------------------- 1 | ExUnit.start() 2 | 3 | defmodule Fd.Test do 4 | use ExUnit.Case, async: false 5 | require KVS 6 | import Record 7 | 8 | test "fd_test", kvs do 9 | assert "//erp" = :kvs_rocks.fd "//erp/orgs" 10 | assert "/erp" = :kvs_rocks.fd "/erp/orgs" 11 | end 12 | 13 | test "key_test", kvs do 14 | assert "//erp/orgs" = :kvs_rocks.key "/erp/orgs" 15 | assert "/erp/orgs" = :kvs_rocks.key "erp/orgs" 16 | end 17 | 18 | defrecord(:msg, id: [], body: []) 19 | 20 | setup do: (on_exit(fn -> :kvs.leave();:ok = :kvs.destroy() end);:kvs.join()) 21 | setup kvs, do: [ 22 | id0: :lists.map(fn _ -> :kvs.append(msg(id: :kvs.seq([],[])), "/crm/duck") end, :lists.seq(1,10)), 23 | id1: :lists.map(fn _ -> :kvs.append(msg(id: :kvs.seq([],[])), "/crm/luck") end, :lists.seq(1,10)), 24 | id2: :lists.map(fn _ -> :kvs.append(msg(id: :kvs.seq([],[])), "/crm/truck") end, :lists.seq(1,10))] 25 | 26 | test "reader", kvs do 27 | ltop = Enum.at(kvs[:id1],0) 28 | dtop = Enum.at(kvs[:id0],0) 29 | ttop = Enum.at(kvs[:id2],0) 30 | assert KVS.reader(feed: "//crm/luck", count: 10, dir: 0, args: [], cache: {:msg, ^ltop, "//crm/luck"}) = :kvs.reader("/crm/luck") 31 | assert KVS.reader(feed: "//crm/duck", count: 10, dir: 0, args: [], cache: {:msg, ^dtop, "//crm/duck"}) = :kvs.reader("/crm/duck") 32 | assert KVS.reader(feed: "//crm/truck", count: 10, dir: 0, args: [], cache: {:msg, ^ttop, "//crm/truck"}) = :kvs.reader("/crm/truck") 33 | assert KVS.reader(feed: "//crm", count: 0, dir: 0, args: [], cache: {:msg, ^dtop, "//crm/duck"}) = :kvs.reader("/crm") 34 | assert KVS.reader(feed: "//noroute", count: 0, dir: 0, args: []) = :kvs.reader("/noroute") 35 | assert KVS.reader(feed: "//", count: 0, dir: 0, args: [], cache: {:msg, ^dtop, "//crm/duck"}) = :kvs.reader("/") 36 | assert KVS.reader(count: 0, dir: 0, args: []) = :kvs.reader([]) 37 | end 38 | 39 | test "range", kvs do 40 | ltop = Enum.at(kvs[:id1],0) 41 | dtop = Enum.at(kvs[:id0],0) 42 | lbot = Enum.at(kvs[:id1],9) 43 | 44 | assert KVS.reader(feed: "//crm/luck", count: 10, dir: 0, args: [], cache: {:msg, ^ltop, "//crm/luck"}) = :kvs.top(:kvs.reader("/crm/luck")) 45 | assert KVS.reader(feed: "//crm", count: 0, dir: 0, args: [], cache: {:msg, ^dtop, "//crm/duck"}) = :kvs.top(:kvs.reader("/crm")) 46 | assert KVS.reader(feed: "//", count: 0, dir: 0, args: [], cache: {:msg, ^dtop, "//crm/duck"}) = :kvs.top(:kvs.reader("/")) 47 | 48 | assert KVS.reader(feed: "//crm/luck", count: 10, dir: 0, args: [], cache: {:msg, ^lbot, "//crm/luck"}) = :kvs.bot(:kvs.reader("/crm/luck")) 49 | assert KVS.reader(feed: "//crm", count: 0, dir: 0, args: [], cache: []) = :kvs.bot(:kvs.reader("/crm")) 50 | assert KVS.reader(feed: "//", count: 0, dir: 0, args: [], cache: []) = :kvs.bot(:kvs.reader("/")) 51 | end 52 | 53 | test "next", kvs do 54 | last = msg(id: Enum.at(kvs[:id1],9)) 55 | r0 = KVS.reader(id: rid) = :kvs.save(:kvs.top(:kvs.reader("/crm/luck"))) 56 | kvs[:id1] |> Enum.with_index 57 | |> Enum.each(fn {id,9} -> 58 | r = :kvs.load_reader(rid) 59 | r01 = :kvs.next(r) 60 | assert r1 = KVS.reader(feed: "//crm/luck", cache: c1, count: 10, dir: 0, args: [^last]) = r01 61 | assert KVS.reader(args: [], feed: "//crm/luck", cache: c1) = :kvs.save(r1) 62 | {id,i} -> 63 | v = msg(id: Enum.at(kvs[:id1],i)) 64 | c = Enum.at(kvs[:id1],i+1) 65 | r = :kvs.load_reader(rid) 66 | assert r1 = KVS.reader(feed: "//crm/luck", cache: {:msg,^c,"//crm/luck"}, count: 10, dir: 0, args: [^v]) = :kvs.next(r) 67 | assert KVS.reader(args: [], feed: "//crm/luck", cache: {:msg,^c,"//crm/luck"}) = :kvs.save(r1) 68 | end) 69 | r = :kvs.load_reader(rid) 70 | 71 | assert r == :kvs.next(r) 72 | assert r == KVS.reader(:kvs.next(:kvs.bot(r)), args: []) 73 | end 74 | 75 | test "prev", kvs do 76 | out = Enum.at(kvs[:id0],9) 77 | KVS.reader(id: rid) = :kvs.save(:kvs.bot(:kvs.reader("/crm/luck"))) 78 | ids = kvs[:id1] |> Enum.reverse 79 | ids |> Enum.with_index 80 | |> Enum.each(fn {id,9} -> 81 | r = :kvs.load_reader(rid) 82 | v = msg(id: Enum.at(ids, 9)) 83 | assert r1 = KVS.reader(feed: "//crm/luck", cache: {:msg, ^out, "//crm/duck"}, count: 10, args: [^v]) = :kvs.prev(r) 84 | assert KVS.reader(args: [], feed: "//crm/luck", cache: c1) = :kvs.save(r1) 85 | {id,i} -> 86 | r = :kvs.load_reader(rid) 87 | v = msg(id: Enum.at(ids, i)) 88 | c = Enum.at(ids, i+1) 89 | assert r1 = KVS.reader(feed: "//crm/luck", cache: {:msg, ^c, "//crm/luck"}, count: 10, args: [^v]) = :kvs.prev(r) 90 | assert KVS.reader(args: [], feed: "//crm/luck", cache: {:msg, ^c, "//crm/luck"}) = :kvs.save(r1) 91 | end) 92 | r = :kvs.load_reader(rid) 93 | assert r = :kvs.prev(r) 94 | assert r = KVS.reader(:kvs.prev(:kvs.top(r)), args: []) 95 | end 96 | 97 | test "prev to empty" do 98 | :lists.map(fn _ -> :kvs.append(msg(id: :kvs.seq([],[])), "/aco") end, :lists.seq(1,2)) 99 | all = :kvs.all("/aco") 100 | head = Enum.at(all,0) 101 | 102 | r = :kvs.bot(:kvs.reader("/aco")) 103 | r1 = KVS.reader(args: args) = :kvs.take(KVS.reader(r, args: 2, dir: 1)) 104 | assert all == :lists.reverse(args) 105 | assert KVS.reader(args: [^head]) = :kvs.take(KVS.reader(r1, args: 1000, dir: 1)) 106 | end 107 | 108 | test "cut the *uck", kvs do 109 | :kvs.cut("/crm/luck") 110 | 111 | all = :kvs.all("/crm") 112 | assert 20 = length(all) 113 | assert all = :kvs.all("/crm/duck") ++ :kvs.all("/crm/truck") 114 | 115 | :kvs.cut("/crm/duck") 116 | 117 | all = :kvs.all("/crm") 118 | assert 10 = length(all) 119 | assert all = :kvs.all("/crm/truck") 120 | 121 | :kvs.cut("/crm/truck") 122 | 123 | all = :kvs.all("/crm") 124 | assert 0 = length(all) 125 | end 126 | 127 | test "remove the *uck with readers", kvs do 128 | :kvs.remove(:kvs.reader("/crm/luck")) 129 | 130 | all = :kvs.all("/crm") 131 | assert 20 = length(all) 132 | assert all = :kvs.all("/crm/duck") ++ :kvs.all("/crm/truck") 133 | 134 | :kvs.remove(:kvs.reader("/crm/duck")) 135 | 136 | all = :kvs.all("/crm") 137 | assert 10 = length(all) 138 | assert all = :kvs.all("/crm/truck") 139 | 140 | :kvs.remove(:kvs.reader("/crm/truck")) 141 | 142 | all = :kvs.all("/crm") 143 | assert 0 = length(all) 144 | end 145 | 146 | @tag :skip # can`t manage this within current implementation. create correct keys! 147 | test "keys with feeds separator" do 148 | :kvs.append(msg(id: "1/1"), "/one/two") 149 | :kvs.append(msg(id: "1/2"), "/one/two") 150 | assert KVS.reader(cache: {:msg, _, "//one/two"}) = :kvs.reader("/one/two") 151 | end 152 | 153 | test "corrupted writers doesn't affect all" do 154 | prev = :kvs.all("/crm/duck") 155 | 156 | KVS.writer(cache: ch) = w = :kvs.writer("/crm/duck") 157 | w1 = KVS.writer(w, cache: {:msg, "unknown", "/corrupted"}) 158 | 159 | :ok = :kvs_rocks.put(w1) 160 | w2 = :kvs.writer("/crm/duck") 161 | assert {:ok, ^w2} = :kvs.get(:writer, "/crm/duck") 162 | assert w1 == w2 163 | 164 | assert prev = :kvs.all("/crm/duck") 165 | 166 | {:ok,_} = :kvs.get(:writer, "/crm/duck") 167 | :ok = :kvs.delete(:writer, "/crm/duck") 168 | {:error, :not_found} = :kvs.get(:writer, "/crm/duck") 169 | assert prev = :kvs.all("/crm/duck") 170 | end 171 | 172 | defp log(x), do: IO.puts '#{inspect(x)}' 173 | defp log(m, x), do: IO.puts '#{m} #{inspect(x)}' 174 | end 175 | -------------------------------------------------------------------------------- /test/sc_test.exs: -------------------------------------------------------------------------------- 1 | ExUnit.start() 2 | 3 | defmodule Sc.Test do 4 | use ExUnit.Case, async: false 5 | require KVS 6 | import Record 7 | @moduledoc """ 8 | refined old scenarios 9 | """ 10 | 11 | defrecord(:msg, id: [], body: []) 12 | 13 | setup do: (on_exit(fn -> :ok = :kvs.leave();:ok = :kvs.destroy() end);:kvs.join()) 14 | setup kvs, do: [ 15 | id0: :lists.map(fn _ -> :kvs.append(msg(id: :kvs.seq([],[])), "/crm/duck") end, :lists.seq(1,10)), 16 | id1: :lists.map(fn _ -> :kvs.append(msg(id: :kvs.seq([],[])), "/crm/luck") end, :lists.seq(1,10)), 17 | id2: :lists.map(fn _ -> :kvs.append(msg(id: :kvs.seq([],[])), "/crm/truck") end, :lists.seq(1,10)), 18 | id3: :lists.map(fn _ -> :kvs.save(:kvs.add(KVS.writer(:kvs.writer(:sym), 19 | args: msg(id: :kvs.seq([],[]))))) end, :lists.seq(1,10))] 20 | test "basic", kvs do 21 | KVS.reader(id: rid1) = :kvs.save(:kvs.reader("/crm/luck")) 22 | KVS.reader(id: rid2) = :kvs.save(:kvs.reader("/crm/truck")) 23 | x1 = :kvs.take(KVS.reader(:kvs.load_reader(rid1), args: 20)) 24 | x2 = :kvs.take(KVS.reader(:kvs.load_reader(rid2), args: 20)) 25 | b = :kvs.feed("/crm/luck") 26 | assert 10 == length(b) 27 | assert :kvs.all("/crm/truck") == KVS.reader(x2, :args) 28 | assert KVS.reader(x1, :args) == b 29 | assert length(KVS.reader(x1, :args)) == length(KVS.reader(x2, :args)) 30 | end 31 | 32 | test "sym",kvs do 33 | KVS.writer(args: last) = Enum.at(kvs[:id3],-1) 34 | {:ok, KVS.writer(id: :sym, count: 10, cache: last)} = :kvs.get(:writer, :sym) 35 | end 36 | 37 | test "take back full" do 38 | feed = "/crm/duck" 39 | KVS.reader(id: rid) = :kvs.save(:kvs.reader(feed)) 40 | t = KVS.reader(args: a1) = :kvs.take(KVS.reader(:kvs.load_reader(rid), args: 10)) 41 | assert a1 == :kvs.feed(feed) 42 | :kvs.save(KVS.reader(t, dir: 1)) 43 | KVS.reader(args: a2) = :kvs.take(KVS.reader(:kvs.load_reader(rid), args: 10)) 44 | assert :lists.reverse(a2) == :kvs.feed(feed) 45 | end 46 | 47 | test "partial take back" do 48 | KVS.reader(id: rid) = :kvs.save(:kvs.reader("/crm/luck")) 49 | r = KVS.reader(args: t) = :kvs.take(KVS.reader(:kvs.load_reader(rid), args: 2)) 50 | :kvs.save(KVS.reader(r, dir: 1)) 51 | KVS.reader(args: n) = :kvs.take(KVS.reader(:kvs.load_reader(rid), args: 3)) 52 | assert :lists.reverse(t) == tl(n) 53 | end 54 | 55 | defp log(x), do: IO.puts '#{inspect(x)}' 56 | defp log(m, x), do: IO.puts '#{m} #{inspect(x)}' 57 | end 58 | -------------------------------------------------------------------------------- /test/st_test.exs: -------------------------------------------------------------------------------- 1 | ExUnit.start() 2 | 3 | defmodule St.Test do 4 | use ExUnit.Case, async: false 5 | import Record 6 | require KVS 7 | 8 | defrecord(:msg, id: [], body: []) 9 | 10 | setup do: (on_exit(fn -> :ok = :kvs.leave();:ok = :kvs_rocks.destroy() end);:kvs.join()) 11 | setup kvs, do: [ 12 | ids: :lists.map(fn _ -> :kvs.append(msg(id: :kvs.seq([],[])), :feed) end, :lists.seq(1,10)), 13 | id0: :lists.map(fn _ -> :kvs.append(msg(id: :kvs.seq([],[])), "/crm/personal/Реєстратор А1/in/directory/duck") end, :lists.seq(1,10)), 14 | id1: :lists.map(fn _ -> :kvs.append(msg(id: :kvs.seq([],[])), "/crm/personal/Реєстратор А1/in/mail") end, :lists.seq(1,10)), 15 | id2: :lists.map(fn _ -> :kvs.append(msg(id: :kvs.seq([],[])), "/crm/personal/Реєстратор А1/in/doc") end, :lists.seq(1,10))] 16 | 17 | test "take-ø", kvs do 18 | r = KVS.reader() = :kvs.reader("/empty-feed") 19 | assert r1 = KVS.reader(feed: "//empty-feed", args: []) = :kvs.take(KVS.reader(r, args: 1)) 20 | assert r1 = KVS.reader(feed: "//empty-feed", args: []) = :kvs.take(KVS.reader(r, args: 1, dir: 1)) 21 | assert r2 = KVS.reader(feed: "//empty-feed", args: []) = :kvs.next(r) 22 | assert r3 = KVS.reader(feed: "//empty-feed", args: []) = :kvs.prev(r) 23 | assert r1 = KVS.reader(feed: "//empty-feed", args: []) = :kvs.take(KVS.reader(r, args: 100)) 24 | assert r1 = KVS.reader(feed: "//empty-feed", args: []) = :kvs.take(KVS.reader(r, args: 100, dir: 1)) 25 | KVS.reader(id: rid) = :kvs.save(r1) 26 | assert rs1 = KVS.reader(id: rid) = :kvs.load_reader(rid) 27 | assert KVS.reader(feed: "//empty-feed", args: []) = :kvs.take(KVS.reader(rs1, args: 5)) 28 | assert KVS.reader(feed: "//empty-feed", args: []) = :kvs.take(KVS.reader(rs1, args: 5, dir: 1)) 29 | assert KVS.reader(feed: "//empty-feed", args: []) = :kvs.next(rs1) 30 | assert KVS.reader(feed: "//empty-feed", args: []) = :kvs.prev(rs1) 31 | assert KVS.reader(feed: "//empty-feed", args: []) = :kvs.take(KVS.reader(rs1, args: 0)) 32 | assert KVS.reader(feed: "//empty-feed", args: []) = :kvs.take(KVS.reader(rs1, args: 0, dir: 1)) 33 | end 34 | 35 | test "take-0", kvs do 36 | feed = "//crm/personal/Реєстратор А1/in/doc" 37 | assert r = KVS.reader(id: rid, args: []) = :kvs.reader("/crm/personal/Реєстратор А1/in/doc") 38 | 39 | assert KVS.reader(id: ^rid, feed: ^feed, args: []) = :kvs.take(KVS.reader(r, args: 0, dir: 0)) 40 | assert KVS.reader(id: ^rid, feed: ^feed, args: []) = :kvs.take(KVS.reader(r, args: -1, dir: 0)) 41 | 42 | assert r1 = KVS.reader(id: ^rid, feed: ^feed, args: a01) = :kvs.take(KVS.reader(r, args: 10, dir: 0)) 43 | assert kvs[:id2] |> Enum.map(&msg(id: &1)) == a01 44 | assert KVS.reader(id: ^rid, feed: ^feed, args: []) = :kvs.take(KVS.reader(r1, args: 10, dir: 0)) 45 | 46 | assert KVS.reader(id: ^rid, feed: ^feed, args: af) = :kvs.take(KVS.reader(r, args: 100, dir: 0)) 47 | assert kvs[:id2] |> Enum.map(&msg(id: &1)) == af 48 | 49 | assert r2 = KVS.reader(id: ^rid, feed: ^feed, args: a03) = :kvs.take(KVS.reader(r, args: 3, dir: 0)) 50 | assert Enum.take(kvs[:id2],3) |> Enum.map(&msg(id: &1)) == a03 51 | 52 | assert KVS.reader(id: ^rid, feed: ^feed, args: a07) = :kvs.take(KVS.reader(r2, args: 7, dir: 0)) 53 | assert Enum.drop(kvs[:id2],3) |> Enum.map(&msg(id: &1)) == a07 54 | 55 | assert KVS.reader(id: ^rid, feed: ^feed, args: a17) = :kvs.take(KVS.reader(r2, args: 100, dir: 0)) 56 | assert Enum.drop(kvs[:id2],3) |> Enum.map(&msg(id: &1)) == a17 57 | end 58 | 59 | test "take-1", kvs do 60 | feed = "//crm/personal/Реєстратор А1/in/mail" 61 | top = Enum.at(kvs[:id1],0) 62 | bot = Enum.at(kvs[:id1],9) 63 | tpm = Enum.take(kvs[:id1],1) |> Enum.map(&msg(id: &1)) 64 | 65 | assert r = KVS.reader(id: rid, args: [], cache: {:msg, ^top, ^feed}, count: 10) = :kvs.reader("/crm/personal/Реєстратор А1/in/mail") 66 | assert r = :kvs.top(r) 67 | assert KVS.reader(id: ^rid, feed: ^feed, args: ^tpm, dir: 1) = :kvs.take(KVS.reader(r, args: 1, dir: 1)) 68 | assert KVS.reader(id: ^rid, feed: ^feed, args: ^tpm, dir: 1) = :kvs.take(KVS.reader(r, args: 100, dir: 1)) 69 | 70 | assert r1 = KVS.reader(feed: ^feed, count: 10, args: [], cache: {:msg,^bot,^feed}) = :kvs.bot(r) 71 | 72 | assert r2 = KVS.reader(feed: ^feed, count: 10, args: a01) = :kvs.take(KVS.reader(r1, args: 5, dir: 1)) 73 | x01 = Enum.drop(kvs[:id1],5) |> Enum.map(&msg(id: &1)) |> Enum.reverse 74 | assert x01 == a01 75 | 76 | assert r3 = KVS.reader(feed: ^feed, count: 10, args: a02) = :kvs.take(KVS.reader(r2, args: 10, dir: 1)) 77 | x02 = Enum.take(kvs[:id1],5) |> Enum.map(&msg(id: &1)) |> Enum.reverse 78 | assert x02 == a02 79 | 80 | assert KVS.reader(feed: ^feed, count: 10, args: []) = :kvs.take(KVS.reader(r3, args: 20, dir: 1)) 81 | end 82 | 83 | test "drop", kvs do 84 | feed = "/feed" 85 | assert r = KVS.reader(id: rid, args: []) = :kvs.save(:kvs.reader(:feed)) 86 | assert r1 = KVS.reader(id: ^rid, feed: ^feed, args: []) = :kvs.drop(KVS.reader(r, args: 10, dir: 0)) 87 | 88 | kvs[:ids] |> Enum.map(&msg(id: &1)) 89 | |> Enum.each(&assert( 90 | KVS.reader(id: rid, feed: ^feed, args: [], cache: &1) = 91 | :kvs.save(:kvs.drop(KVS.reader(:kvs.load_reader(rid), args: 1, dir: 0))))) 92 | 93 | assert r2 = KVS.reader(id: ^rid, feed: ^feed, args: [], cache: c1) = :kvs.drop(KVS.reader(r, args: 1, dir: 0)) 94 | assert {:msg, Enum.at(kvs[:ids], 1), feed} == c1 95 | 96 | assert KVS.reader(id: ^rid, feed: ^feed, args: [], cache: c2) = :kvs.drop(KVS.reader(r2, args: 5, dir: 0)) 97 | assert {:msg, Enum.at(kvs[:ids], 6), feed} == c2 98 | assert KVS.reader(id: ^rid, feed: ^feed, args: []) = :kvs.drop(KVS.reader(r1, args: 100)) 99 | end 100 | 101 | test "feed",kvs do 102 | docs = :kvs.all("/crm/personal/Реєстратор А1/in/doc") 103 | ducks = :kvs.all("/crm/personal/Реєстратор А1/in/directory/duck") 104 | mail = :kvs.all("/crm/personal/Реєстратор А1/in/mail") 105 | total = :kvs.all("/crm/personal/Реєстратор А1/in") 106 | 107 | assert ducks++docs++mail == total 108 | 109 | assert docs = :kvs.feed("/crm/personal/Реєстратор А1/in/doc") 110 | assert ducks = :kvs.feed("/crm/personal/Реєстратор А1/in/directory/duck") 111 | assert mail = :kvs.feed("/crm/personal/Реєстратор А1/in/mail") 112 | assert total = :kvs.feed("/crm/personal/Реєстратор А1/in") 113 | 114 | assert :kvs.feed("/crm/personal/Реєстратор А1/in/directory/duck") 115 | ++ :kvs.feed("/crm/personal/Реєстратор А1/in/doc") 116 | ++ :kvs.feed("/crm/personal/Реєстратор А1/in/mail") == :kvs.feed("/crm/personal/Реєстратор А1/in") 117 | end 118 | 119 | defp log(x), do: IO.puts '#{inspect(x)}' 120 | defp log(m, x), do: IO.puts '#{m} #{inspect(x)}' 121 | 122 | end 123 | -------------------------------------------------------------------------------- /test/test_helper.exs: -------------------------------------------------------------------------------- 1 | ExUnit.start() 2 | --------------------------------------------------------------------------------